configure_sample_depot_for_sdp.sh #3

#!/bin/bash
#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE
#------------------------------------------------------------------------------
set -u

#------------------------------------------------------------------------------
# Declarations
declare Version=2.2.0
declare DownloadsDir="/opt/perforce/helix-sdp/downloads"
declare SDPUnixSetupDir="/opt/perforce/helix-sdp/p4/sdp/Server/Unix/setup"
declare SDPSetupDir="/opt/perforce/helix-sdp/p4/sdp/Server/setup"
declare TmpFile="/tmp/tmp.csd4sdp.$$.$RANDOM"
declare RunUser=perforce
declare ThisUser=
declare ThisHost=
declare CBIN=/p4/common/bin
declare ThisScript="${0##*/}"
declare CmdLine="$0 $*"
declare SDPInstance=Unset
declare CleartextPasswordFile=
declare EncryptedPasswordFile=
declare DiskSpaceAvail=
declare MinKBFor5GLimits=7340032
declare P4DInitScript=
declare P4DSystemdServiceFile=
declare P4BrokerInitScript=
declare P4BrokerSystemdServiceFile=
declare -i MaxStartDelay=120
declare -i P4DStartVerified=0
declare -i P4BrokerStartVerified=0
declare -i UseSystemd=1
declare -i i=0

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
#------------------------------------------------------------------------------
function usage {
   declare style=${1:--h}

   msg "USAGE for $ThisScript v$Version:

$ThisScript -i <sdp_instance> [-d <data_dir>] [-u <osuser>] [-no_systemd]

or

$ThisScript [-h|-man]
"
   if [[ $style == -man ]]; then
      msg "
DESCRIPTION:
	This script transforms a stock Sample Depot instance into
	one that works with the SDP.

REQUIREMENTS:
	A P4D process must be live and running with the stock
	Sample Depot data set, on a sport 

ARGUMENTS:
 -i <sdp_instance>
	Specify the SDP Instance in which the Sample Depot data set is
	running.  This argument is required.

 -d <data_dir>
	Specify the data directory where supporting files exist, such as the
	*.p4s data files used by this script.

 -u <osuser>
	Specify the Linux operating system user account under which p4d runs.
	If omitted, the default is 'perforce'.

 -no_systemd
	Specify '-no_systemd' to avoid using systemd, even if it
	appears to be available. By default, systemd is used if it
	appears to be available.

	This is helpful in operating in containerized test environments
	where systemd is not available.

 -D     Set extreme debugging verbosity.

HELP OPTIONS:
 -h	Display short help message
 -man	Display man-style help message

EXAMPLES:
	Usage to configure Instance 1:
	cd $SDPUnixSetupDir
	$ThisScript 1 2>&1 | tee log.${ThisScript%.sh}.1

	Usage to configure Instance abc:
	cd $SDPUnixSetupDir
	$ThisScript abc 2>&1 | tee log.${ThisScript%.sh}.abc
"
   fi

   exit 1
}

#------------------------------------------------------------------------------
# Function bail().
# Sample Usage:
#    bail "Missing something important. Aborting."
#    bail "Aborting with exit code 3." 3
function msg () { echo -e "$*"; }
function errmsg () { msg "\\nError: ${1:-Unknown Error}\\n"; ErrorCount+=1; }
function warnmsg () { msg "\\nWarning: ${1:-Unknown Warning}\\n"; WarningCount+=1; }
function bail () { errmsg "${1:-Unknown Error}"; exit "${2:-1}"; }

#------------------------------------------------------------------------------
# Functions.  The runCmd() function is similar to functions defined in SDP core
# libraries, but we need to duplicate them here since this script runs before
# the SDP is available on the machine (and we want no dependencies for this
# script.
function runCmd {
   declare cmd=${1:-echo Testing runCmd}
   declare desc=${2:-""}

   declare cmdToShow=$cmd

   [[ "$cmdToShow" == *"<"* ]] && cmdToShow=${cmdToShow%%<*}
   [[ "$cmdToShow" == *">"* ]] && cmdToShow=${cmdToShow%%>*}

   [[ -n "$desc" ]] && msg "$desc"
   msg "Running: $cmdToShow"
   $cmd
   return $?
}

#==============================================================================
# Command Line Processing

declare -i shiftArgs=0
declare -i ErrorCount=0
declare -i WarningCount=0
declare DataDir="$PWD"

set +u

while [[ $# -gt 0 ]]; do
   case $1 in
      (-no_systemd) UseSystemd=0;;
      (-i) SDPInstance=$2; shiftArgs=1;;
      (-d) DataDir="$2"; shiftArgs=1;;
      (-u) RunUser="$2"; shiftArgs=1;;
      (-h) usage -h;;
      (-man) usage -man;;
      (-D) set -x;; # Debug; use 'set -x' mode.
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && bail "Usage Error: Wrong numbers of args or flags to args."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#------------------------------------------------------------------------------
# Usage Validation

[[ $SDPInstance == Unset ]] && \
   bail "Bad Usage: The '<sdp_instance>' argument is required."

[[ ! -r $CBIN/p4_vars ]] && \
   bail "Missing SDP Environment File [$CBIN/p4_vars]. Aborting."

#------------------------------------------------------------------------------
# Main Program

ThisUser=$(whoami)
ThisHost=${HOSTNAME%%.*}

msg "Started $ThisScript v$Version on host $ThisHost as user $ThisUser at $(date), called as:\\n\\t$CmdLine"

if [[ "$ThisUser" != "$RunUser" ]]; then
   bail "Run as $RunUser, not $ThisUser."
else
   msg "Verified: Running as user $RunUser."
fi

# Load SDP environment and variable definitions.
# shellcheck disable=SC1090 disable=SC1091
source "$CBIN/p4_vars" "$SDPInstance" ||\
   bail "Failed to load SDP environment. Aborting."

export P4ENVIRO=/dev/null/.p4enviro
export P4CONFIG=.p4config

CleartextPasswordFile="$SDP_ADMIN_PASSWORD_FILE"
EncryptedPasswordFile="${CleartextPasswordFile}.enc"
P4DInitScript="${P4DBIN}_init"
P4BrokerInitScript="${P4BROKERBIN}_init"

if [[ "$UseSystemd" -eq 1 ]]; then
   P4DSystemdServiceFile="/etc/systemd/system/${P4DBIN##*/}.service"
   P4BrokerSystemdServiceFile="/etc/systemd/system/${P4BROKERBIN##*/}.service"
else
   P4DSystemdServiceFile=
   P4BrokerSystemdServiceFile=
fi

cd "$SDPUnixSetupDir" ||\
   bail "Could not do: cd $SDPUnixSetupDir"

msg "Operating in SDP server setup area [$PWD]."

runCmd "$P4BIN -u bruno -s info -s" "Verifying server is offline." &&\
   bail "Perforce server is unexpectedly online. Aborting."

runCmd "/p4/${SDPInstance}/bin/p4d_${SDPInstance} -jr $DownloadsDir/PerforceSample/checkpoint" \
   "Loading the Sample Depot metadata in instance ${SDPInstance}." ||\
   bail "Failed to load Sample Depot checkpoint."

runCmd "/p4/${SDPInstance}/bin/p4d_${SDPInstance} -xu" \
   "Upgrading databases (p4d -xu) for instance ${SDPInstance}." ||\
   bail "Failed to upgrade databases."

if [[ $P4PORT == "ssl:"* ]]; then
   if [[ -r /p4/ssl/certificate.txt ]]; then
      msg "Using existing OpenSSL certs in /p4/ssl"
   else
      runCmd "/p4/${SDPInstance}/bin/p4d_${SDPInstance} -Gc" \
         "Generating OpenSSL Certificates." ||\
         bail "Failed to generate OpenSSL Certs."
   fi
fi

# This next block duplicates some logic in configure_new_server.sh
# to prevent disk space failures on small demo systems.

# For filesys.*.min configurables, use 5G defaults if we have 7G+ of space
# available, otherwise assume this is a demo-scale environment and use 20M.
msg "Using 'p4d -cset' to set filesys.{P4ROOT,P4JOURNAL,P4LOG,depot,TEMP}.min settings in $P4ROOT."
DiskSpaceAvail=$(df -k "$P4ROOT/" 2>/dev/null | grep / | awk '{print $4}')
if [[ "$DiskSpaceAvail" -ge "$MinKBFor5GLimits" ]]; then
   p4d "-cset filesys.P4ROOT.min=5G" || errmsg "Error setting filesys.P4ROOT.min to 5G."
else
   p4d "-cset filesys.P4ROOT.min=20M" || errmsg "Error setting filesys.P4ROOT.min to 20M."
fi

DiskSpaceAvail=$(df -k "$LOGS/" 2>/dev/null | grep / | awk '{print $4}')
if [[ "$DiskSpaceAvail" -ge "$MinKBFor5GLimits" ]]; then
   p4d "-cset filesys.P4JOURNAL.min=5G" || errmsg "Error setting filesys.P4JOURNAL.min to 5G."
else
   p4d "-cset filesys.P4JOURNAL.min=20M" || errmsg "Error setting filesys.P4JOURNAL.min to 20M."
fi

if [[ "$DiskSpaceAvail" -ge "$MinKBFor5GLimits" ]]; then
   p4d "-cset filesys.P4LOG.min=5G" || errmsg "Error setting filesys.P4LOG.min to 5G."
else
   p4d "-cset filesys.P4LOG.min=20M" || errmsg "Error setting filesys.P4LOG.min to 20M."
fi

DiskSpaceAvail=$(df -k "$DEPOTS/" 2>/dev/null | grep / | awk '{print $4}')
if [[ "$DiskSpaceAvail" -ge "$MinKBFor5GLimits" ]]; then
   p4d "-cset filesys.depot.min=5G" || errmsg "Error setting filesys.depot.min to 5G."
else
   p4d "-cset filesys.depot.min=5G" || errmsg "Error setting filesys.depot.min to 20M."
fi

DiskSpaceAvail=$(df -k "/tmp/" 2>/dev/null | grep / | awk '{print $4}')
if [[ "$DiskSpaceAvail" -ge "$MinKBFor5GLimits" ]]; then
   p4d "-cset filesys.TEMP.min=5G" || errmsg "Error setting filesys.TEMP.min to 5G."
else
   p4d "-cset filesys.TEMP.min=5G" || errmsg "Error setting filesys.TEMP.min to 20M."
fi

if [[ -x "$P4DInitScript" ]]; then
   if [[ -f "$P4DSystemdServiceFile" ]]; then
      msg "Starting p4d as user $ThisUser with: sudo systemctl start p4d_${SDPInstance}"
      if sudo systemctl start "p4d_${SDPInstance}"; then
         msg "Service p4d_${SDPInstance} started with systemd."
      else
         warnmsg "Non-zero exit code $? starting p4d service with systemd."
         if [[ -r "$LOGS/p4d_init.log" ]]; then
            msg "Contents of $LOGS/p4d_init.log:\\n$(cat "$LOGS/p4d_init.log")"
         else
            warnmsg "No $LOGS/p4d_init.log file found."
         fi
      fi
   elif [[ -f "/etc/init.d/p4d_${SDPInstance}_init" ]]; then
      msg "Starting p4d as user $ThisUser with: sudo service p4d_${SDPInstance}_init start"
      if sudo service "p4d_${SDPInstance}_init" start; then
         msg "Service p4d_${SDPInstance}_init started with SysV init."
      else
         warnmsg "Non-zero exit code $? starting p4d service with SysV init."
         if [[ -r "$LOGS/p4d_init.log" ]]; then
            msg "Contents of $LOGS/p4d_init.log:\\n$(cat "$LOGS/p4d_init.log")"
         else
            warnmsg "No $LOGS/p4d_init.log file found."
         fi
      fi
   else
      msg "Starting p4d as user $ThisUser with: $P4DInitScript start"
      if "$P4DInitScript" start; then
         msg "Service p4d_${SDPInstance}_init started with no init mechanism."
      else
         warnmsg "Non-zero exit code $? starting p4d service with no init mechanism."
         if [[ -r "$LOGS/p4d_init.log" ]]; then
            msg "Contents of $LOGS/p4d_init.log:\\n$(cat "$LOGS/p4d_init.log")"
         else
            warnmsg "No $LOGS/p4d_init.log file found."
         fi
      fi
   fi
else
   bail "Init script is missing or not executable: $P4DInitScript"
fi

P4DStartVerified=0
i=0; while [[ "$i" -lt "$MaxStartDelay" ]]; do
   if "$P4BIN" -p "$P4PORT" info -s > "$TmpFile" 2>&1; then
      msg "Verified: p4d has started."
      P4DStartVerified=1
      rm -f "$TmpFile"
      break
   else
      # If we get an SSL trust error, then the server is indeed online.
      if grep -lq 'The authenticity of' "$TmpFile"; then
         msg "Verified: p4d has started."
         P4DStartVerified=1
         "$P4BIN" -p "$P4PORT" trust -f -y > /dev/null 2>&1
         rm -f "$TmpFile"
         break
      else
         sleep 1
      fi
   fi
   i+=1
done

[[ "$P4DStartVerified" -eq 1 ]] || \
   bail "p4d start attempted but could not be verified after $MaxStartDelay seconds.\\nTail of server log $P4LOG:\\n$(tail "$P4LOG")\\n\\n"

if [[ -x "$P4BrokerInitScript" ]]; then
   if [[ -f "$P4BrokerSystemdServiceFile" ]]; then
      msg "Starting p4broker as user $ThisUser with: sudo systemctl start p4broker_${SDPInstance}"
      if sudo systemctl start "p4broker_${SDPInstance}"; then
         msg "Service p4broker_${SDPInstance} started with systemd."
      else
         warnmsg "Non-zero exit code $? starting broker service with systemd."
      fi
   elif [[ -f "/etc/init.d/p4broker_${SDPInstance}_init" ]]; then
      msg "Starting p4broker as user $ThisUser with: sudo service p4broker_${SDPInstance}_init start"
      if sudo service "p4broker_${SDPInstance}_init" start; then
         msg "Service p4broker_${SDPInstance}_init started with SysV init."
      else
         warnmsg "Non-zero exit code $? starting broker service with SysV init."
      fi
   else
      msg "Starting p4broker as user $ThisUser with: $P4BrokerInitScript start"
      if "$P4BrokerInitScript" start; then
         msg "Service p4broker_${SDPInstance}_init started with no init mechanism."
      else
         warnmsg "Non-zero exit code $? starting broker service with no init mechanism."
      fi
   fi
else
   bail "Init script is missing or not executable: $P4BrokerInitScript"
fi

P4BrokerStartVerified=0
i=0; while [[ "$i" -lt "$MaxStartDelay" ]]; do
   if "$P4BIN" -p "$P4BROKERPORT" info -s > "$TmpFile" 2>&1; then
      msg "Verified: p4broker has started."
      P4BrokerStartVerified=1
      rm -f "$TmpFile"
      break
   else
      if grep -lq 'The authenticity of' "$TmpFile"; then
         msg "Verified: p4d has started."
         P4BrokerStartVerified=1
         "$P4BIN" -p "$P4BROKERPORT" trust -f -y > /dev/null 2>&1
         rm -f "$TmpFile"
         break
      else
         sleep 1
      fi
   fi
   i+=1
done

[[ "$P4BrokerStartVerified" -eq 1 ]] || \
   bail "p4broker start attempted but could not be verified after $MaxStartDelay seconds.\\n\\nTail of broker log $LOGS/p4broker.log:\\n$(tail "$LOGS/p4broker.log")\\n\\n"

if [[ $P4PORT == "ssl:"* ]]; then
   # Note: Automating a 'p4 trust -y' (especially with '-f') may not be appropriate
   # in a production environment, as it defeats the purpose of the Open SSL trust
   # mechanism.  But for our purposes here, where scripts spin up throw-away data
   # sets for testing or training purposes, it's just dandy.
   runCmd "/p4/${SDPInstance}/bin/p4_${SDPInstance} -p $P4PORT trust -y -f" \
      "Trusting the OpenSSL Cert of the server." ||\
      bail "Failed to trust the server."
   runCmd "/p4/${SDPInstance}/bin/p4_${SDPInstance} -p $P4BROKERPORT trust -y -f" \
      "Trusting the OpenSSL Cert of the broker." ||\
      bail "Failed to trust the broker."
fi

runCmd "$P4BIN -s info -s" "Verifying direct connection to Perforce server." ||\
   bail "Could not connect to Perforce server."

runCmd "$P4BIN -u bruno -s -p $P4BROKERPORT info -s" "Verifying via-broker connection to Perforce server." ||\
   bail "Could not connect to Perforce server via broker."

[[ "$($P4BIN -u bruno protects -m)" == super ]] ||\
   bail "Could not verify super user access for $P4USER on port $P4PORT.  Is this the Sample depot? Aborting."

msg "Super user access for bruno verified."

msg "Creating user $P4USER."
sed "s:__EDITME_ADMIN_P4USER__:$P4USER:g" "$DataDir/admin.user.p4s" > "$TmpFile"
"$P4BIN" -u bruno user -f -i < "$TmpFile"

msg "Adding user to NoTicketExpiration group."
sed "s:__EDITME_ADMIN_P4USER__:$P4USER:g" "$DataDir/NoTicketExpiration.group.p4s" > "$TmpFile"
"$P4BIN" -u bruno group -i < "$TmpFile"

msg "Promoting user $P4USER to super user."
"$P4BIN" -u bruno protect -o > "$TmpFile"
msg "\\tsuper user $P4USER * //...\\n" >> "$TmpFile"
"$P4BIN" -u bruno protect -i < "$TmpFile"

if [[ -r "$CleartextPasswordFile" ]]; then
   cat "$CleartextPasswordFile" > "$TmpFile"
   cat "$CleartextPasswordFile" >> "$TmpFile"
elif [[ -r "$EncryptedPasswordFile" ]]; then
   base64 -d - < "$EncryptedPasswordFile" > "$TmpFile"
   base64 -d - < "$EncryptedPasswordFile" >> "$TmpFile"
else
   bail "Could not find cleartext nor encrypted password files. Aborting."
fi

"$P4BIN" -u bruno passwd "$P4USER" < "$TmpFile"
rm -f "$TmpFile"

runCmd "/p4/common/bin/p4login" "Logging in $P4USER super user." ||\
   bail "Failed to login super user $P4USER. Aborting."

# Variable     Format                              Sample Values
# P4PORT       [ssl:]<P4DPortNum>                  ssl:1999, 1999
# P4BROKERPORT [ssl:]<BrokerPort>                  ssl:1666, 1666
for p in $P4PORT $P4BROKERPORT; do
   if [[ $p == "ssl:"* ]]; then
      runCmd "$P4BIN -p $p trust -y" "Trusting P4PORT=$p." ||\
         bail "Failed to trust P4PORT=$p."
   fi
   cmd="$P4BIN -u $P4USER -p $p login -a"
   if [[ -r "$CleartextPasswordFile" ]]; then
      msg "Running: $cmd < $CleartextPasswordFile"
      $cmd < "$CleartextPasswordFile" ||\
         bail "Login as perforce using P4PORT=$p failed.  Aborting."
   else
      base64 -d - < "$EncryptedPasswordFile" > "$TmpFile"
      msg "Running: $cmd < $TmpFile"
      $cmd < "$TmpFile" ||\
         bail "Login as perforce using P4PORT=$p failed.  Aborting."
      rm -f "$TmpFile"
   fi
done

runCmd "cat $P4TICKETS" "Showing P4TICKETS:"

runCmd "$SDPSetupDir/configure_new_server.sh $SDPInstance" \
   "Applying SDP configurables." ||\
   bail "Failed to set SDP configurables. Aborting."

msg "Copying Sample Depot depots to SDP location."

for depot in $(/bin/ls -d "$DownloadsDir"/PerforceSample/*); do
   [[ $depot == *"checkpoint"* ]] && continue
   [[ $depot == *"README"* ]] && continue
   [[ $depot == *"readme"* ]] && continue
   if [[ $depot == *"spec"* ]]; then
      runCmd "/usr/bin/rsync -a $depot/ /p4/$SDPInstance/depots/${depot##*/}" \
         "Copying Sample Depot archive files for spec depot [${depot##*/}]." ||\
         warnmsg "Non-zero exit code $? from rsync for depot ${depot##*/}."
   else
      runCmd "/usr/bin/rsync -a --delete $depot/ /p4/$SDPInstance/depots/${depot##*/}" \
         "Copying Sample Depot archive files for depot [${depot##*/}]." ||\
         warnmsg "Non-zero exit code $? from rsync for depot ${depot##*/}."
   fi
done

runCmd "$P4BIN admin updatespecdepot -a" \
   "Updating spec depot." || bail "Failed to udpate spec depot. Aborting."

runCmd "/usr/bin/rsync -a /p4/$SDPInstance/root/spec/ /p4/$SDPInstance/depots/spec" \
   "Copying a few spec depot files." ||\
   warnmsg "Non-zero exit code $? from rsync for spec depot."

runCmd "/bin/rm -rf /p4/$SDPInstance/root/spec" \
   "Cleanup redundant copy of spec depot files." ||:

runCmd "/p4/common/bin/live_checkpoint.sh $SDPInstance" \
   "Taking Live Checkpoint." || bail "Live checkpoint failed. Aborting."

msg "\\nSUCCESS:  SDP Instance $SDPInstance loaded with Sample Depot data, live checkpoint done, and backup created.  Good to go!\\n"

exit 0