opt_perforce_sdp_backup.sh #7

#!/bin/bash
set -u

# For documentation, run this script with the '-man' option:
# opt_perforce_sdp_backup.sh -man

#==============================================================================
# Declarations an Environment

declare ThisScript="${0##*/}"
declare Version=2.11.7
declare ThisUser=
declare ThisHost=${HOSTNAME%%.*}
declare PerforcePackageBase="/opt/perforce"
declare SDPPackageBase="$PerforcePackageBase/helix-sdp"
declare BackupMount=
declare BackupBase=
declare BackupBaseOwner=
declare BackupDir=
declare BackupDirOwner=
declare StagingBase=
declare StagingDir=
declare TarballName=
declare TarballPath=
declare TarballInBackup=
declare BackupSSLDir=
declare -a BackupCmd=
declare HomeDirBackupMode=
declare HomeDirBackupModeFile=
declare HomeDirBackupDir=
declare HomeDir=
declare CrontabBackupFile=
declare CrontabBackupFileBackup=
declare CrontabBackupFileTimestamp=
declare -i DoBackupSudoers=0
declare -i DoRecoverSudoers=0
declare SudoersFile=
declare SudoersBackupFile=
declare SystemdBackupDir=
declare -a RecoverCmd=
declare SDPRoot="${SDP_ROOT:-/p4}"
declare SDPCommon="${SDPRoot}/common"
declare SDPCommonBin="${SDPCommon}/bin"
declare SDPCommonLib="${SDPCommon}/lib"
declare SSLDir="${SDPRoot}/ssl"
declare SDPOwner=
declare SDPOwnerUID=
declare SDPGroup=
declare SDPGroupGID=
declare RecoverScript="recover_opt_perforce_sdp.sh"
declare RecoverScriptPath=
declare RecoverScriptTimestamp=
declare NewRecoverScriptPath=
declare TmpFile=
declare Symlink=
declare ServiceFile=
declare ServiceName=
declare InstanceDir=
declare InstanceBackupDir=
declare -a InstanceList
declare Instance=
declare InstanceBinDir=
declare InstanceBinBackupDir=
declare H1="=============================================================================="
declare H2="------------------------------------------------------------------------------"
declare -i Debug=${SDP_DEBUG:-0}
declare -i ErrorCount=0
declare -i WarningCount=0
declare -i InstanceCount=0
declare -i SilentMode=0
declare LogsDir="/var/log/${ThisScript%.sh}"
declare LogLink="${ThisScript%.sh}"
declare Log=

#==============================================================================
# Local Functions
# These must be defined before the library loading section below.

function msg () { echo -e "$*"; }
function dbg () { [[ "$Debug" -eq 0 ]] || msg "DEBUG: $*"; }
function errmsg () { msg "\\nError: ${1:-Unknown Error}\\n" >&2; ErrorCount+=1; }
function warnmsg () { msg "\\nWarning: ${1:-Unknown Warning}\\n" >&2; WarningCount+=1; }
function bail () { errmsg "${1:-Unknown Error}"; exit "${2:-1}"; }

#==============================================================================
# SDP Library Functions

if [[ -d "$SDPCommonLib" ]]; then
   # shellcheck disable=SC1090 disable=SC1091
   source "$SDPCommonLib/logging.lib" ||\
      bail "Failed to load bash lib [$SDPCommonLib/logging.lib]. Aborting."
   # shellcheck disable=SC1090 disable=SC1091
   source "$SDPCommonLib/utils.lib" ||\
      bail "Failed to load bash lib [$SDPCommonLib/utils.lib]. Aborting."
fi

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
# The default is -h.
#
# $2 - error message (optional).  Specify this if usage() is called due to
# user error, in which case the given message displayed first, followed by the
# standard usage message (short or long depending on $1).  If displaying an
# error, usually $1 should be -h so that the longer usage message doesn't
# obscure the error message.
#
# Sample Usage:
# usage
# usage -h
# usage -man
# usage -h "Incorrect command line usage."
#------------------------------------------------------------------------------
function usage
{
   local style=${1:--h}
   local usageErrorMessage=${2:-}

   [[ -n "$usageErrorMessage" ]] && msg "\\n\\nUsage Error:\\n\\n$usageErrorMessage\\n\\n"

   msg "USAGE for $ThisScript v$Version:

$ThisScript [-d|-D]

or

$ThisScript [-h|-man|-V]
"
   if [[ $style == -man ]]; then
      msg "
DESCRIPTION:
	This script is intended to called by a systemd timer on systems
	that support systemd.

	There is an opt_perforce_sdp_backup.service for this; it can be
	reviewed by doing:
	
	\$ systemctl cat opt_perforce_sdp_backup.service

	This service is triggered by a systemd timer, which can be viewed by:
	\$ systemctl cat opt_perforce_sdp_backup.timer

	If operating on a system without systemd, then it is OK to call this
	directly, e.g. via crontab. It must execute as root.

	This script backups key P4 Server Deployment Package (SDP) files and
	directories.  It does not back up actual P4 Server application data.
	The job of this script is to ensure any SDP files that are stored on
	the local OS root volume are backed up a data volume that is backed up.
	If no data volume is found, an extra copy of files is made on the OS
	root volume.

	To support backup to NFS volumes with root squash enabled, this script
	stages backup content in a temporary directory, creates a gzip-compressed
	tarball owned by the SDP owner user, and copies that tarball to the backup
	location. The tarball preserves all file ownership and permissions for
	recovery.

BACKUP LOCATION

	This script determines the optimal backup location based on the server
	type being backed up.  Some sample locations for backup are:

	P4 Server: /hxdepots/backup/opt_perforce_helix-sdp.<ShortHostname>
	P4 Proxy:  /hxdepots/backup/opt_perforce_helix-sdp.<ShortHostname>
	P4 Broker: /hxlogs/backup/opt_perforce_helix-sdp.<ShortHostname>

	The backup consists of a gzip-compressed tarball and a recovery script:
	* sdp_backup.<ShortHostname>.tgz - Contains all backed up files
	* $RecoverScript - Script to restore from the backup

GENERATED RECOVERY SCRIPT

	Each time a successful backup completes, this script creates a
	recovery script.  The recovery script is generated each time to
	ensure that site-specific settings such as SDP instances names
	and mount point locations are accounted for.  This ensures that
	the recovery faithfully places files in the original structure.

	The generated recovery script is in the backup directory and is
	named: $RecoverScript

	During each backup in which the generated recovery script changes,
	diffs from the prior recovery script are displayed, and the old
	recovery script is backed up in the backup location by moving it
	aside to a '.bak.<timestamp>' suffix, and the active file is
	updated.  It is expected that this backup script will change
	infrequently, e.g. when new SDP instances are added (or removed),
	and possibly in future SDP version changes. In any case, the new
	script will update and replace the old.

	This does whatever is needed to restore SDP as it was at the time
	of backup, including:
	* Creates the group of the SDP Owner if needed.
	* Creates the SDP Owner user if needed.
	* Restores the SDP Owner home directory if needed.
	* Restores the SDP Owner crontab.
	* Restores all SDP files and symlinks that exist outside the
	data volumes (e.g. /hxdepots, /hxlogs, /hxmetadata[1,2]).

	It does NOT restore operating system package installations.

RECOVERY PROCEDURE

	The recovery procedure extracts the backup tarball and runs the
	recovery script. Operating as root:

	1. Create a working directory and copy the tarball there:
	   mkdir -p /root/opt_perforce_sdp_backup
	   cd /root/opt_perforce_sdp_backup
	   cp /hxdepots/backup/opt_perforce_helix-sdp.<ShortHostname>/sdp_backup.<ShortHostname>.tgz .

	2. Extract the tarball (this restores all files with correct ownership):
	   tar -xzpf sdp_backup.<ShortHostname>.tgz

	3. Run the recovery script from the extracted content:
	   cd opt_perforce_helix-sdp.<ShortHostname>
	   ./$RecoverScript

	The gzip-compressed tarball preserves all file ownership and permissions,
	so extraction as root will restore files with their original owners
	(including root).

OTHER FILES TO RESTORE - HOME DIRECTORY

	This does NOT backup files in the '$OSUSER' user home directory, as this
	is expected to be backed up by other means and/or per local policy,
	and is outside the scope of this script.

	The home directory may include files that affect server operation, and
	thus should be recovered before using script. Some files to be recovered
	are:
	* ~/.bash_profile
	* ~/.bashrc
	* ~/.p4aliases.

	Templates for these can be found in /p4/sdp/Server/Unix/setup/bash.

	The home directory may also include an ~/.ssh directory and ~/.config
	and similar directories.

OPTIONS:
 -L <log>
	Specify the path to a log file, or the special value 'off' to disable
	logging.  By default, all output (stdout and stderr) goes to a log file
	pointed to by a symlink:

	$LogLink

	The symlink is for convenience. It refers to the log from the most recent
	run of the script.

	Each time this script is run, a new timestamped log is started, and
	the symlink updated to reference the new/latest log during startup.  Log
	files have timestamps that go to the second (or millisecond if needed)
	to differentiate logs.

	NOTE: This script is self-logging.  Output displayed on the screen is
	simultaneously captured in the log file. Using redirection operators like
	'> log' or '2>&1' are unnecessary, as is using 'tee' (though using 'tee'
	or redirects is safe and harmless).

 -d
	Display debug messages.

 -D
	Set extreme debugging verbosity using bash 'set -x' mode. Implies -d.

 -si
	Silent Mode.  No output is displayed to the terminal (except for usage errors
	on startup). Output is captured in the log.  The '-si' cannot be used with
	'-L off'.

HELP OPTIONS:
 -h	Display short help message.
 -man	Display man-style help message.
 -V	Display script name and version.

FILES:
	/etc/systemd/system/${ThisScript%.sh}.service
	/etc/systemd/system/${ThisScript%.sh}.timer

LIMITATIONS:
	If SELinux is used in enforcing mode, some 'semanage' and 'restorecon'
	commands may be needed on specific files after recovery. The generated
	recovery script does not currently handle SELinux contexts.

TO DO:
	A future version of this script may preserve the crontab of the OSUSER.
"
   fi

   exit 2
}

#==============================================================================
# Command Line Processing

declare -i shiftArgs=0

set +u
while [[ $# -gt 0 ]]; do
   case "$1" in
      (-h) usage -h;;
      (-man) usage -man;;
      (-V|-version|--version) msg "$ThisScript version $Version"; exit 0;;
      (-L) Log="$2"; shiftArgs=1;;
      (-si) SilentMode=1;;
      (-d) Debug=1;;
      (-D) Debug=1; set -x;; # Use bash 'set -x' extreme debug mode.
      (-*) usage -h "Unknown option ($1).";;
      (*) usage -h "Unknown parameter ($1).";;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usage -h "Incorrect number of arguments."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Command Line Verification

[[ "$SilentMode" -eq 1 && "$Log" == off ]] && \
   usage -h "The '-si' option cannot be used with '-Log off'."

#==============================================================================
# Main Program

trap terminate EXIT SIGINT SIGTERM

[[ -n "$Log" ]] || Log="${LogsDir}/${ThisScript%.sh}.${ThisHost}.$(date +'%Y-%m-%d-%H%M%S').log"

if [[ "$Log" != off ]]; then
   # Set LogsDir to the directory containing $Log. While LogsDir is defined above,
   # we need to reset it here in case the user used the -L option.
   LogsDir="${Log%/*}"
   if [[ ! -d "$LogsDir" ]]; then
      mkdir -p "$LogsDir" || bail "Couldn't do: mkdir -p \"$LogsDir\""
   fi

   touch "$Log" || bail "Couldn't touch log file: $Log"

   # Redirect stdout and stderr to a log file.
   if [[ "$SilentMode" -eq 0 ]]; then
      exec > >(tee "$Log")
      exec 2>&1
   else
      exec >"$Log"
      exec 2>&1
   fi

   msg "${H1}\\nLog is: $Log"
fi


ThisUser=$(id -n -u)
msg "${H1}\\nStarted $ThisScript v$Version as $ThisUser@$ThisHost on $(date)."

msg "Starting Preflight Checks."

if [[ "$ThisUser" == root ]]; then
   dbg "Verified: Running as root user."
else
   errmsg "Run this as root, not user '$ThisUser'."
fi

SDPOwner="$(stat --format=%U "$SDPCommonBin")" ||\
   errmsg "Could not owner of $SDPCommonBin with: stat --format=%U \"$SDPCommonBin\""
SDPOwnerUID="$(stat --format=%u "$SDPCommonBin")" ||\
   errmsg "Could not owner of $SDPCommonBin with: stat --format=%u \"$SDPCommonBin\""
SDPGroup="$(stat --format=%G "$SDPCommonBin")" ||\
   errmsg "Could not group of $SDPCommonBin with: stat --format=%G \"$SDPCommonBin\""
SDPGroupGID="$(stat --format=%g "$SDPCommonBin")" ||\
   errmsg "Could not group of $SDPCommonBin with: stat --format=%g \"$SDPCommonBin\""

msg "SDP owner/group is $SDPOwner:$SDPGroup, based on owner/group of $SDPCommonBin."

# Find the mount point to backup. For P4 Server and standalone P4 Proxy machines,
# this will be the mount point for /p4/N/{cache,checkpoints,depots} directory.
# For standalone Proxy and Broker machines, there may only be a local OS root
# volume, in which case we still do a local backup to a different directory on
# the current machine.
if ProbePath=$(find_p4depots_probe_path "$SDPRoot"); then
   BackupMount=$(get_mount_point "$ProbePath") ||\
      errmsg "Failed to determine mount point for: $ProbePath"
   dbg "Detected backup mount point: $BackupMount"
else
   warnmsg "Could not identify a P4Depots-related path under $SDPRoot."
   BackupMount=/
fi

# If there is no mounted volume, then we calculate the BackupBase by using
# 'pwd -P' to resolve symlinks to the ProbePath directory, which will start with
# /p4 and use the shorter symlink path, e.g. /p4/1/depots/.  The ProbePath
# reliably ends with a '/' character.  Then, we strip the ProbePath from the
# right side of the ProbePath and append "/backup" to get the BackupBase. So if
# the ProbePath is /p4/1/depots/, then a 'pwd -P' in that directory may look like
# /hxdepots/p4/1/depots (even if /hxdepots isn't a real mount point).  The
# BackupBase should then be /hxdepots/backup. Even though /hxdepots would also be
# on the OS root volume in that case, a local backup is better than no backup.
# For any production deployment for P4 Server host, it really should be a mount
# point, but that's outside the scope of this  script to control.
if [[ "$BackupMount" == / ]]; then
   warnmsg "No mount point detected for P4Depots-related dir could be determined."

   BackupBase="$(cd "$ProbePath" || return 1; pwd -P)"
   if [[ -n "$BackupBase" ]]; then
      BackupBase="${BackupBase%"${ProbePath%/}"}/backup"
   else
      BackupBase=/backup
   fi
else
   BackupBase="${BackupMount%/}/backup"
fi

# Ensure we don't have '//' in paths.
BackupBase=${BackupBase//\/\//\/}
BackupDir="$BackupBase/opt_perforce_helix-sdp.$ThisHost"

# Set up staging directory for building backup content
StagingBase="/tmp/opt_perforce_sdp_backup.$$"
StagingDir="$StagingBase/opt_perforce_helix-sdp.$ThisHost"
TarballName="sdp_backup.$ThisHost.tgz"
TarballPath="$StagingBase/$TarballName"
TarballInBackup="$BackupDir/$TarballName"

# Define subdirectories within the staging area (mirrors old BackupDir structure)
BackupSSLDir="$StagingDir/ssl"
SystemdBackupDir="${StagingDir}/systemd"
HomeDirBackupDir="${StagingDir}/HomeDir-${SDPOwner}"

msg "
Backup Base:         $BackupBase
Backup Dir:          $BackupDir
Staging Base:        $StagingBase
Staging Dir:         $StagingDir
Tarball Name:        $TarballName
Backup SSL Dir:      $BackupSSLDir
Backup Systemd Dir:  $SystemdBackupDir
Backup Home Dir:     $HomeDirBackupDir
"

if [[ -d "$SDPPackageBase" ]]; then
   dbg "Verified: $SDPPackageBase directory exists."
else
   errmsg "SDP OS Package Structure [$SDPPackageBase] does not exists. Use this script only if that structure exists."
fi

# Create BackupBase and BackupDir as SDPOwner to avoid issues with NFS root squash
if [[ ! -d "$BackupBase" ]]; then
   msg "Creating initial Backup Base directory: $BackupBase"
   runuser -u "$SDPOwner" -- mkdir -p "$BackupBase" ||\
      errmsg "Could not do: runuser -u \"$SDPOwner\" -- mkdir -p \"$BackupBase\""
   runuser -u "$SDPOwner" -- chmod 750 "$BackupBase" ||\
      errmsg "Could not do: runuser -u \"$SDPOwner\" -- chmod 750 \"$BackupBase\""
fi

if [[ -d "$BackupBase" ]]; then
   BackupBaseOwner="$(stat --format=%U "$BackupBase")"
   if [[ "$BackupBaseOwner" == "$SDPOwner" ]]; then
      dbg "Verified: $BackupBase directory exists with correct ownership ($SDPOwner)."
      if [[ -d "$BackupBase" && -w "$BackupBase" ]]; then
         dbg "Verified: $BackupBase directory exists and is writable."
      else
         errmsg "The BackupBase directory [$BackupBase] is not writable."
      fi

   elif [[ "$BackupBaseOwner" == "root" ]]; then
      msg "Adjusting ownership of $BackupBase to $SDPOwner:$SDPGroup"
      chown "$SDPOwner:$SDPGroup" "$BackupBase" ||\
         errmsg "Could not do: chown \"$SDPOwner:$SDPGroup\" \"$BackupBase\""
   else
      warnmsg "$BackupBase exists but owner is '$BackupBaseOwner', not '$SDPOwner'."
   fi
fi

if [[ ! -d "$BackupDir" ]]; then
   msg "Creating initial Backup Dir directory: $BackupDir"
   runuser -u "$SDPOwner" -- mkdir "$BackupDir" ||\
      errmsg "Could not do: runuser -u \"$SDPOwner\" -- mkdir \"$BackupDir\""
   runuser -u "$SDPOwner" -- chmod 750 "$BackupDir" ||\
      errmsg "Could not do: runuser -u \"$SDPOwner\" -- chmod 750 \"$BackupDir\""
fi

if [[ -d "$BackupDir" ]]; then
   BackupDirOwner="$(stat --format=%U "$BackupDir")"
   if [[ "$BackupDirOwner" == "$SDPOwner" ]]; then
      dbg "Verified: $BackupDir directory exists with correct ownership ($SDPOwner)."
      if [[ -d "$BackupDir" && -w "$BackupDir" ]]; then
         dbg "Verified: $BackupDir directory exists and is writable."
      else
         errmsg "The BackupDir directory [$BackupDir] is not writable."
      fi

   elif [[ "$BackupDirOwner" == "root" ]]; then
      msg "Adjusting ownership of $BackupDir to $SDPOwner:$SDPGroup"
      chown "$SDPOwner:$SDPGroup" "$BackupDir" ||\
         errmsg "Could not do: chown \"$SDPOwner:$SDPGroup\" \"$BackupDir\""
   else
      warnmsg "$BackupDir exists but owner is '$BackupDirOwner', not '$SDPOwner'."
   fi
fi

# Create staging directory structure as root (to preserve file ownership during staging)
if [[ -d "$StagingBase" ]]; then
   msg "Removing pre-existing staging directory: $StagingBase"
   rm -rf "$StagingBase" ||\
      errmsg "Could not do: rm -rf \"$StagingBase\""
fi

msg "Creating staging directory: $StagingDir"
mkdir -p "$StagingDir" ||\
   errmsg "Could not do: mkdir -p \"$StagingDir\""
chmod 750 "$StagingDir" ||\
   errmsg "Could not do: chmod 750 \"$StagingDir\""

if [[ ! -d "$SystemdBackupDir" ]]; then
   msg "Creating Systemd staging directory: $SystemdBackupDir"
   mkdir "$SystemdBackupDir" ||\
      errmsg "Could not do: mkdir \"$SystemdBackupDir\""
   chmod 750 "$SystemdBackupDir" ||\
      errmsg "Could not do: chmod 750 \"$SystemdBackupDir\""
fi

if [[ -d "$SystemdBackupDir" && -w "$SystemdBackupDir" ]]; then
   dbg "Verified: $SystemdBackupDir directory exists and is writable."
else
   errmsg "The Systemd staging directory [$SystemdBackupDir] does not exist or is not writable."
fi

if [[ ! -d "$HomeDirBackupDir" ]]; then
   msg "Creating User Home staging directory: $HomeDirBackupDir"
   mkdir "$HomeDirBackupDir" ||\
      errmsg "Could not do: mkdir \"$HomeDirBackupDir\""
   chmod 750 "$HomeDirBackupDir" ||\
      errmsg "Could not do: chmod 750 \"$HomeDirBackupDir\""
fi

if [[ -d "$HomeDirBackupDir" && -w "$HomeDirBackupDir" ]]; then
   dbg "Verified: $HomeDirBackupDir directory exists and is writable."
else
   errmsg "The User Home staging directory [$HomeDirBackupDir] does not exist or is not writable."
fi

if [[ "$ErrorCount" -eq 0 ]]; then
   dbg "Preflight checks OK."
else
   bail "Aborting due to failed preflight checks."
fi

RecoverScriptPath="$BackupDir/$RecoverScript"
NewRecoverScriptPath=$(mktemp)

msg "Initializing generated recovery script."
dbg "NewRecoverScriptPath=$NewRecoverScriptPath"

echo '#!/bin/bash' > "$NewRecoverScriptPath" ||\
   errmsg "Could not initialize tmp file with: 'echo #!/bin/bash' > $NewRecoverScriptPath"

echo "echo This recovery script was generated by $ThisScript version $Version." >> "$NewRecoverScriptPath" ||\
   errmsg "Failed to update generated recovery script."

# Add usage instructions and directory check
cat >> "$NewRecoverScriptPath" << 'EOF_USAGE' ||\
   errmsg "Failed to add usage instructions to recovery script."

# USAGE INSTRUCTIONS:
# This script must be run from inside the extracted tarball directory.
# 
# Recovery procedure:
# 1. mkdir -p /root/opt_perforce_sdp_backup
# 2. cd /root/opt_perforce_sdp_backup
# 3. cp <backup_location>/sdp_backup.<hostname>.tgz .
# 4. tar -xzpf sdp_backup.<hostname>.tgz
# 5. cd opt_perforce_helix-sdp.<hostname>
# 6. ./recover_opt_perforce_sdp.sh

# Verify we're in the correct directory by checking for expected subdirectories
if [[ ! -d "systemd" ]] || [[ ! ( -d "fgs" || -d "1" || -d "2" ) ]]; then
   echo ""
   echo "ERROR: This recovery script must be run from inside the extracted tarball directory."
   echo ""
   echo "It appears you are running this script from: $(pwd)"
   echo ""
   echo "Please follow the recovery procedure:"
   echo "  1. mkdir -p /root/opt_perforce_sdp_backup"
   echo "  2. cd /root/opt_perforce_sdp_backup"
   echo "  3. cp <backup_location>/sdp_backup.<hostname>.tgz ."
   echo "  4. tar -xzpf sdp_backup.<hostname>.tgz"
   echo "  5. cd opt_perforce_helix-sdp.<hostname>"
   echo "  6. ./recover_opt_perforce_sdp.sh"
   echo ""
   exit 1
fi

EOF_USAGE

# shellcheck disable=SC2016
echo 'EXTRACT_DIR=$(pwd)' >> "$NewRecoverScriptPath" ||\
   errmsg "Failed to capture extraction directory in recovery script."

echo 'set -x' >> "$NewRecoverScriptPath" ||\
   errmsg "Failed to update generated recovery script."

echo '[[ -d /p4 ]] || mkdir /p4' >> "$NewRecoverScriptPath" ||\
   errmsg "Failed to update generated recovery script."

msg "${H2}\\nGenerating recovery script commands to recreate SDP owner/group $SDPOwner/$SDPGroup ($SDPOwnerUID/$SDPGroupGID)"

HomeDir=$(getent passwd "$SDPOwner" | cut -d: -f6)
UserShell=$(getent passwd "$SDPOwner" | cut -d: -f7)
echo "HomeDir=$HomeDir" >> "$NewRecoverScriptPath" ||\
   errmsg "Failed to update generated recovery script with: HomeDir=$HomeDir"

# shellcheck disable=SC2028
echo "if ! getent group $SDPGroup > /dev/null; then
   echo Running: groupadd -g $SDPGroupGID \"$SDPGroup\"
   groupadd -g $SDPGroupGID \"$SDPGroup\"
fi
if ! getent passwd $SDPOwner > /dev/null; then
   echo Running: useradd -u $SDPOwnerUID -d $HomeDir -s $UserShell -G \"$SDPGroup\" $SDPOwner
   useradd -u $SDPOwnerUID -d $HomeDir -s $UserShell -G \"$SDPGroup\" $SDPOwner
fi" >> "$NewRecoverScriptPath" ||\
   errmsg "Could not append user/group creation logic to generated recovery script."

HomeDirBackupModeFile="$HomeDir/.p4-sdp.home_dir_backup"
if [[ -r "$HomeDirBackupModeFile" ]]; then
   HomeDirBackupMode=$(grep -E '^HomeDirBackupMode=' "$HomeDirBackupModeFile" 2>/dev/null | cut -d= -f2)
   HomeDir=$(grep -E '^HomeDir=' "$HomeDirBackupModeFile" 2>/dev/null | cut -d= -f2)
   if [[ "${HomeDirBackupMode^^}" == FULL ]]; then
      HomeDirBackupMode=Full
   elif [[ "${HomeDirBackupMode^^}" == BASIC ]]; then
      HomeDirBackupMode=Basic
   elif [[ "${HomeDirBackupMode^^}" == OFF ]]; then
      HomeDirBackupMode=Off
   else
      errmsg "Could not determine HomeDirBackupMode setting from file $HomeDirBackupModeFile. Not backing up '$SDPOwner' home directory."
      HomeDirBackupMode=Off
   fi
else
   errmsg "Missing file '$HomeDirBackupModeFile'. Not backing up user '$SDPOwner' home directory."
   HomeDirBackupMode=Off
fi

if [[ "$HomeDirBackupMode" == Full || "$HomeDirBackupMode" == "Basic" ]]; then
   msg "${H2}\\nDoing '$HomeDirBackupMode' backup of SDPOwner ($SDPOwner) Home Dir."

   if [[ "$HomeDirBackupMode" == Full ]]; then
      # For backup, use 'rsync -a' with '--delete' to avoid bloat. The purpose
      # here is to recovery the home directory at a s
      BackupCmd=(rsync -a --delete "$HomeDir"/ "$HomeDirBackupDir")
      # The recover script will have a captured definition of HomeDir.
      # For recovery, we avoid using the '--delete' option to rsync.
      RecoverCmd=(rsync -a "\$EXTRACT_DIR/HomeDir-${SDPOwner}"/ \$HomeDir)
      msg "Running: ${BackupCmd[*]}"
      if "${BackupCmd[@]}"; then
         echo "${RecoverCmd[*]}" >> "$NewRecoverScriptPath" ||\
            errmsg "Failed to update generated recovery script."
      else
         errmsg "Failed to backup user home dir with: ${BackupCmd[*]}"
      fi
   else # HomeDirBackupMode is 'Basic'; backup only key files.
      BackupCmd=(find "$HomeDir/" -maxdepth 1 -name ".profile*" -exec cp -pf '{}' "$HomeDirBackupDir/." ';')
      RecoverCmd=(find "\$EXTRACT_DIR/HomeDir-${SDPOwner}/" -maxdepth 1 -name ".profile*" -exec cp -pf '{}' "$HomeDir/." '\;')
      msg "Running: ${BackupCmd[*]}"
      if "${BackupCmd[@]}"; then
         echo "${RecoverCmd[*]}" >> "$NewRecoverScriptPath" ||\
            errmsg "Failed to update generated recovery script."
      else
         errmsg "Failed to Backup .profile* files in user home dir."
      fi

      BackupCmd=(find "$HomeDir/" -maxdepth 1 -name ".bash*" -exec cp -pf '{}' "$HomeDirBackupDir/." ';')
      RecoverCmd=(find "\$EXTRACT_DIR/HomeDir-${SDPOwner}/" -maxdepth 1 -name ".bash*" -exec cp -pf '{}' "$HomeDir/." '\;')
      msg "Running: ${BackupCmd[*]}"
      if "${BackupCmd[@]}"; then
         echo "${RecoverCmd[*]}" >> "$NewRecoverScriptPath" ||\
            errmsg "Failed to update generated recovery script."
      else
         errmsg "Failed to Backup .bash* files in user home dir."
      fi

      BackupCmd=(find "$HomeDir/" -maxdepth 1 -name ".p4*" -exec cp -pf '{}' "$HomeDirBackupDir/." ';')
      msg "Running: ${BackupCmd[*]}"
      RecoverCmd=(find "\$EXTRACT_DIR/HomeDir-${SDPOwner}/" -maxdepth 1 -name ".p4*" -exec cp -pf '{}' "$HomeDir/." '\;')
      if "${BackupCmd[@]}"; then
         echo "${RecoverCmd[*]}" >> "$NewRecoverScriptPath" ||\
            errmsg "Failed to update generated recovery script."
      else
         errmsg "Failed to Backup .p4* files in user home dir."
      fi

      msg "Running: ${BackupCmd[*]}"
      if [[ -d "$HomeDir/.ssh" ]]; then
         BackupCmd=(rsync -a "$HomeDir/.ssh/" "$HomeDirBackupDir/.ssh")
         RecoverCmd=(rsync -a "\$EXTRACT_DIR/HomeDir-${SDPOwner}/.ssh/" "$HomeDir/.ssh")
         msg "Running: ${BackupCmd[*]}"
         if "${BackupCmd[@]}"; then
            echo "${RecoverCmd[*]}" >> "$NewRecoverScriptPath" ||\
               errmsg "Failed to update generated recovery script."
         else
            errmsg "Failed to Backup .p4* files in user home dir."
         fi
      else
         msg "Skipping backup of ~$SDPOwner/.ssh directory as it does not exist."
      fi
   fi
else
   msg "${H2}\\nNOT backing SDPOwner ($SDPOwner) Home Dir; HomeDirBackupMode is set to 'Off'."
fi

msg "${H2}\\nBacking up SDP Package Base Dir to staging area."

BackupCmd=(rsync -av "$SDPPackageBase/" "$StagingDir")
RecoverCmd=(rsync -a "\$EXTRACT_DIR/" "$SDPPackageBase")
msg "Running: ${BackupCmd[*]}"
if "${BackupCmd[@]}"; then
   msg "SDP Package staging to $StagingDir was successful."

   echo '[[ -d /opt/perforce ]] || mkdir -p /opt/perforce' >> "$NewRecoverScriptPath" ||\
      errmsg "Failed to update generated recovery script."

   echo "${RecoverCmd[*]}" >> "$NewRecoverScriptPath" ||\
      errmsg "Could not append this command to generated recovery script: ${RecoverCmd[*]}"
else
   errmsg "SDP Package staging to $StagingDir failed."
fi

if [[ -e "$SSLDir" ]]; then
   msg "${H2}\\nBacking up SSL directory to staging area."
   BackupCmd=(rsync -av "$SSLDir/" "$BackupSSLDir")
   RecoverCmd=(rsync -a "\$EXTRACT_DIR/ssl/" "$SSLDir/")

   msg "Running: ${BackupCmd[*]}"
   if "${BackupCmd[@]}"; then
      msg "Successfully staged SDP SSL Directory to $BackupSSLDir."

      echo "${RecoverCmd[*]}" >> "$NewRecoverScriptPath" ||\
         errmsg "Could not append this to generated recovery script: ${RecoverCmd[*]}"
   else
      msg "Failed to stage SDP SSL Directory to $BackupSSLDir."
   fi
else
   msg "Skipping backup of the SSL directory [$SSLDir] because it does not exist."
fi

msg "${H2}\\nBacking up $SDPRoot/<N>/bin \"Instance Bin\" dirs to staging area."
dbg "Finding Instance Bin dirs with: find \"$SDPRoot/\" -maxdepth 2 -type d -name bin -print"
while IFS= read -r InstanceBinDir; do
   Instance="${InstanceBinDir#"$SDPRoot"/}"
   Instance="${Instance%%/*}"
   InstanceList[InstanceCount]="$Instance"
   InstanceCount+=1

   InstanceBinBackupDir="$StagingDir/${InstanceBinDir#"$SDPRoot"/}"

   msg "Backing up Instance Bin Dir: $InstanceBinDir."

   mkdir -p "$InstanceBinBackupDir" ||\
      errmsg "Could not do: mkdir -p \"$InstanceBinBackupDir\""
   chown "$SDPOwner:$SDPGroup" "$InstanceBinBackupDir" ||\
      errmsg "Could not do: chown \"$SDPOwner:$SDPGroup\" \"$InstanceBinBackupDir\""
   chmod 750 "$InstanceBinBackupDir" ||\
      errmsg "Could not do: chmod 750 \"$InstanceBinBackupDir\""

   BackupCmd=(rsync -a "$InstanceBinDir/" "$InstanceBinBackupDir")
   RecoverCmd=(rsync -a "\$EXTRACT_DIR/${InstanceBinDir#"$SDPRoot"/}/" "$InstanceBinDir/")

   dbg "Running: ${BackupCmd[*]}"
   if "${BackupCmd[@]}"; then
      msg "Successfully staged Instance Bin dir: $InstanceBinDir"

      echo "[[ -d /p4/$Instance ]] || mkdir /p4/$Instance" >> "$NewRecoverScriptPath" ||\
         errmsg "Failed to update generated recovery script."

      echo "${RecoverCmd[*]}" >> "$NewRecoverScriptPath" ||\
         errmsg "Could not append this to generated recovery script: ${RecoverCmd[*]}"
   else
      errmsg "Failed to stage Instance Bin dir: $InstanceBinDir"
   fi
done < <(find "$SDPRoot/" -maxdepth 2 -type d -name bin -print)

# For key symlinks, we don't need to copy them to the backup directory; just
# write them to the recovery script. Capture the target of the symlinks
# exactly as they were at backup time.
cd "$SDPRoot" || bail "Could not do: cd \"$SDPRoot\""
echo "cd $SDPRoot" >> "$NewRecoverScriptPath" ||\
   errmsg "Could not append this to generated recovery script: cd $SDPRoot"
for Symlink in sdp common; do
   if [[ -L "$Symlink" ]]; then
      echo "ln -s $(readlink "$Symlink") $Symlink; chown -h $SDPOwner:$SDPGroup $Symlink" >> "$NewRecoverScriptPath" ||\
         errmsg "Could not append symlink for $SDPRoot/$Symlink to generated recovery script."
   fi
done

msg "${H2}\\nBacking up .p4* and other files in $SDPRoot/<N> \"Instance\" dirs to staging area."
for Instance in "${InstanceList[@]}"; do
   InstanceDir="${SDPRoot}/$Instance"
   InstanceBackupDir="$StagingDir/${InstanceDir#"$SDPRoot"/}"
   msg "Backing up .p4* and other files in: $InstanceDir"

   BackupCmd=(find "$InstanceDir/" -maxdepth 1 -type f -exec cp -pf '{}' "$InstanceBackupDir/." ';')
   RecoverCmd=(find "\$EXTRACT_DIR/${InstanceDir#"$SDPRoot"/}/" -maxdepth 1 -type f -exec cp -pf '{}' "$InstanceDir/." '\;')
   dbg "Backing up files in $InstanceDir with this command: ${BackupCmd[*]}"

   if "${BackupCmd[@]}"; then
      msg "Successfully staged Instance Bin dir: $InstanceBinDir"

      echo "${RecoverCmd[*]}" >> "$NewRecoverScriptPath" ||\
         errmsg "Could not append this to generated recovery script: ${RecoverCmd[*]}"
   else
      errmsg "Failed to stage Instance Bin dir: $InstanceBinDir"
   fi

   # For key symlinks, we don't need to copy them to the backup directory; just
   # write them to the recovery script. Capture the target of the symlinks
   # exactly as they were at backup time.
   cd "$SDPRoot/$Instance" || bail "Could not do: cd \"$SDPRoot/$Instance\""
   echo "cd $SDPRoot/$Instance" >> "$NewRecoverScriptPath" ||\
      errmsg "Could not append this to generated recovery script: cd $SDPRoot/$Instance"

   # For all possible SDP symlinks, see if they exist, and if so, capture their targets
   # in the form of a recovery command.
   for Symlink in root offline_db cache tmp logs depots checkpoints; do
      if [[ -L "$Symlink" ]]; then
         echo "ln -s $(readlink "$Symlink") $Symlink; chown -h $SDPOwner:$SDPGroup $Symlink" >> "$NewRecoverScriptPath" ||\
            errmsg "Could not append symlink for $SDPRoot/$Instance/$Symlink to generated recovery script."
      fi
   done
done

msg "${H2}\\nBacking up any files in SDP Root ($SDPRoot) to staging area."

BackupCmd=(find "$SDPRoot/" -maxdepth 1 -type f -exec cp -pf '{}' "$StagingDir/." ';')
RecoverCmd=(find "\$EXTRACT_DIR/" -maxdepth 1 -type f -exec cp -pf '{}' "$SDPRoot/." '\;')
msg "Backing up files in $SDPRoot with this command: ${BackupCmd[*]}"

if "${BackupCmd[@]}"; then
   msg "Successfully staged files in: $SDPRoot"
   echo "${RecoverCmd[*]}" >> "$NewRecoverScriptPath" ||\
      errmsg "Could not append this to generated recovery script: ${RecoverCmd[*]}"

   # Generate a command to remove excess copies of the recover script that would clutter $SDPRoot.
   echo "rm -f $SDPRoot/${RecoverScript}*" >> "$NewRecoverScriptPath" ||\
      errmsg "Could not append command to cleanup excess copies of recover script in $SDPRoot to generated recovery script."
else
   errmsg "Failed to stage files in: $SDPRoot"
fi

msg "Staging crontab."
TmpFile=$(mktemp)
# Any stderr will be displayed and captured in the log.
CrontabBackupFile="${HomeDirBackupDir}/crontab.system_backup"
if crontab -l -u "$SDPOwner" > "$TmpFile"; then
   if [[ -r "$CrontabBackupFile" ]]; then
      if diff -q "$TmpFile" "$CrontabBackupFile" > /dev/null; then
         msg "Crontab unchanged from prior version. No need to back up."
      else
         CrontabBackupFileTimestamp=$(get_old_log_timestamp "$CrontabBackupFile")
         CrontabBackupFileBackup="${CrontabBackupFile}.bak.$CrontabBackupFileTimestamp"
         if mv -f "$CrontabBackupFile" "$CrontabBackupFileBackup"; then
            dbg "Rotated old crontab backup to: $CrontabBackupFileBackup"
         else
            errmsg "Failed to rotate old crontab backup to: $CrontabBackupFileBackup"
         fi

         if mv -f "$TmpFile" "$CrontabBackupFile"; then
            msg "Active crontab for user $SDPOwner staged to: $CrontabBackupFile"
         else
            errmsg "Failed to move active crontab in temp file [$TmpFile] to [$CrontabBackupFile]."
         fi
      fi

      RecoverCmd=(crontab -u "$SDPOwner" "\$EXTRACT_DIR/HomeDir-${SDPOwner}/crontab.system_backup")
      echo "${RecoverCmd[*]}" >> "$NewRecoverScriptPath" ||\
         errmsg "Could not append this to generated recovery script: ${RecoverCmd[*]}"
   fi
else
   if [[ -r "$CrontabBackupFile" ]]; then
     warnmsg "Crontab was NOT staged, but a prior (and possibly stale) backup exists as: $CrontabBackupFile"
   else
      warnmsg "Crontab was NOT staged. No prior backup existed, so there might not be an active crontab."
   fi
fi

rm -f "$TmpFile"

# For systemd files, remove and re-copy each backup cycle to avoid copying stale data.
msg "Removing old staged systemd service files."

# We don't need to worry about hidden (files with a '.' prefix) for systemd
# backups.
rm -rf "${SystemdBackupDir:-/tmp/DoesNotExist}"/* ||\
   errmsg "Could not do: rm -rf \"${SystemdBackupDir:-/tmp/DoesNotExist}\"/*"

msg "Staging systemd p4*.service and opt_perforce*.{service,timer} files."
for ServiceFile in /etc/systemd/system/p4*.service /etc/systemd/system/opt_perforce*.service /etc/systemd/system/opt_perforce*.timer; do
   [[ "$ServiceFile" == *'*'* ]] && continue
   BackupCmd=(cp -p "$ServiceFile" "${SystemdBackupDir}/.")
   RecoverCmd=(cp -p "\$EXTRACT_DIR/systemd/${ServiceFile##*/}" /etc/systemd/system/.)
   if "${BackupCmd[@]}"; then
      echo "
if [[ ! -e $ServiceFile ]]; then
   ${RecoverCmd[*]}
fi" >> "$NewRecoverScriptPath" ||\
         errmsg "Could not append this to generated recovery script: ${RecoverCmd[*]}"

      ServiceName="${ServiceFile##*/}"
      ServiceName="${ServiceName%.service}"
      # If the service is currently enabled at backup time, enable it at recovery time.
      if systemctl is-enabled "$ServiceName" > /dev/null; then
         RecoverCmd=(systemctl enable "$ServiceName")
         echo "${RecoverCmd[*]}" >> "$NewRecoverScriptPath" ||\
            errmsg "Could not append this to generated recovery script: ${RecoverCmd[*]}"
      fi
   else
      errmsg "Unable to stage service file: $ServiceFile"
   fi
done

RecoverCmd=(systemctl daemon-reload)
echo "${RecoverCmd[*]}" >> "$NewRecoverScriptPath" ||\
   errmsg "Could not append this to generated recovery script: ${RecoverCmd[*]}"

SudoersFile="/etc/sudoers.d/$SDPOwner"
SudoersBackupFile="$StagingDir/sudoers-$SDPOwner"

set -x
if [[ -r "$SudoersFile" ]]; then
   if [[ -r "$SudoersBackupFile" ]]; then
      if diff -q "$SudoersBackupFile" "$SudoersFile"; then
         msg "No need to stage sudoers file because the current staged file [$SudoersBackupFile] matches active file [$SudoersFile]."
         DoBackupSudoers=0
         DoRecoverSudoers=1
      else
         msg "Sudoers has been updated since last backup, so it will be staged. Diffs from prior version:"
         diff "$SudoersBackupFile" "$SudoersFile"
         DoBackupSudoers=1
         DoRecoverSudoers=1
      fi
   else
      # Simple case; if the active sudoers exist but a staged version does not yet exist, stage it.
      DoBackupSudoers=1
      DoRecoverSudoers=1
   fi
else
   if [[ -r "$SudoersBackupFile" ]]; then
      warnmsg "An active sudoers file [$SudoersFile] does not exist, but the script (if executed) would put this backup file in place: $SudoersBackupFile"
      DoBackupSudoers=0
      DoRecoverSudoers=1
   else
      msg "Neither an active sudoers file [$SudoersFile] nor a backup exist. No backup needed for sudoers."
      DoBackupSudoers=0
      DoRecoverSudoers=0
   fi
fi
set +x

if [[ "$DoBackupSudoers" -eq 1 ]]; then
   msg "Staging sudoers file."

   if [[ -r "$SudoersBackupFile" ]]; then
      BackupCmd=(mv -f "$SudoersBackupFile" "${SudoersBackupFile}.bak.$(date +'%Y-%m-%d-%H%M%S')")
      if "${BackupCmd[@]}"; then
         msg "Moved old staged sudoers file with: ${BackupCmd[*]}"
      else
         warnmsg "Failed to move old staged sudoers file with: ${BackupCmd[*]}"
      fi
   fi

   BackupCmd=(cp -pf "$SudoersFile" "$SudoersBackupFile")
   if "${BackupCmd[@]}"; then
      msg "Staged sudoers file with: ${BackupCmd[*]}"
      DoRecoverSudoers=1
   else
      errmsg "Recovery script will not restore $SudoersFile due to failure to stage sudoers file with: ${BackupCmd[*]}"
      DoRecoverSudoers=0
   fi
fi

if [[ "$DoRecoverSudoers" -eq 1 ]]; then
   msg "Generated recovery script will recover sudoers file."
   RecoverCmd=(cp -pf "\$EXTRACT_DIR/sudoers-$SDPOwner" "$SudoersFile")
   echo "${RecoverCmd[*]}" >> "$NewRecoverScriptPath" ||\
      errmsg "Could not append this to generated recovery script: ${RecoverCmd[*]}"
else
   msg "Generated recovery script will NOT recover sudoers file."
fi

echo -e "echo Recovery processing complete." >> "$NewRecoverScriptPath"

#------------------------------------------------------------------------------
# Copy recovery script to staging directory so it's included in the tarball
if [[ "$ErrorCount" -eq 0 ]]; then
   msg "Copying recovery script to staging directory for inclusion in tarball."
   RecoverScriptInStaging="$StagingDir/$RecoverScript"
   
   if cp "$NewRecoverScriptPath" "$RecoverScriptInStaging"; then
      chmod +x "$RecoverScriptInStaging" ||\
         errmsg "Could not set execute bit on recovery script in staging: $RecoverScriptInStaging"
      msg "Recovery script copied to staging directory: $RecoverScriptInStaging"
   else
      errmsg "Failed to copy recovery script to staging directory: $RecoverScriptInStaging"
   fi
else
   msg "Skipping recovery script staging due to errors."
fi

#------------------------------------------------------------------------------
# Create tarball from staging area
if [[ "$ErrorCount" -eq 0 ]]; then
   msg "${H2}\\nCreating compressed tarball from staged content."
   msg "Running: tar -czpf \"$TarballPath\" --numeric-owner -C \"$StagingBase\" \"opt_perforce_helix-sdp.$ThisHost\""
   
   if tar -czpf "$TarballPath" --numeric-owner -C "$StagingBase" "opt_perforce_helix-sdp.$ThisHost"; then
      msg "Successfully created compressed tarball: $TarballPath"
      
      # Change ownership of tarball to SDPOwner so it can be copied to NFS with root squash
      msg "Changing tarball ownership to $SDPOwner:$SDPGroup"
      if chown "$SDPOwner:$SDPGroup" "$TarballPath"; then
         msg "Successfully changed tarball ownership."
         
         # Copy tarball to backup location as SDPOwner
         msg "Copying tarball to backup location: $TarballInBackup"
         if runuser -u "$SDPOwner" -- cp -pf "$TarballPath" "$TarballInBackup"; then
            msg "Successfully copied tarball to: $TarballInBackup"
         else
            errmsg "Failed to copy tarball to backup location: $TarballInBackup"
         fi
      else
         errmsg "Failed to change tarball ownership."
      fi
   else
      errmsg "Failed to create tarball from staging area."
   fi
else
   msg "Skipping tarball creation due to errors during backup staging."
fi

#------------------------------------------------------------------------------
if [[ "$Debug" -ne 0 ]]; then
   msg "DEBUG: Generated recovery script contents:\\n$(cat "$NewRecoverScriptPath")"
fi

# If the backup has been successful, put the new recovery script in place.
if [[ "$ErrorCount" -eq 0 ]]; then
   msg "Handling update of generated recovery script: $RecoverScriptPath"
   if [[ -r "$RecoverScriptPath" ]]; then
      if diff -q "$RecoverScriptPath" "$NewRecoverScriptPath"; then
         msg "Newly generated recovery script is same as existing one. All good, nothing to do."
      else
         msg "The generated recovery script changed, so it will be updated.  Diffs from prior version:"
         diff "$RecoverScriptPath" "$NewRecoverScriptPath"
         RecoverScriptTimestamp=$(get_old_log_timestamp "$RecoverScriptPath")
         RecoverScriptBackup="${RecoverScriptPath}.bak.$RecoverScriptTimestamp"

         if mv -f "$RecoverScriptPath" "$RecoverScriptBackup"; then
            if cp "$NewRecoverScriptPath" "$RecoverScriptPath"; then
               msg "Successfully updated recovery script: $RecoverScriptPath"
               
               # Change ownership to SDPOwner for consistency
               chown "$SDPOwner:$SDPGroup" "$RecoverScriptPath" ||\
                  warnmsg "Could not change ownership of recovery script to $SDPOwner:$SDPGroup"
            else
               errmsg "Failed to update recovery script : cp \"$NewRecoverScriptPath\" \"$RecoverScriptPath\""
            fi

            dbg "Setting execute bit on recover script: $RecoverScriptPath"
            chmod +x "$RecoverScriptPath" ||\
               errmsg "Could not do: chmod +x \"$RecoverScriptPath\""

            dbg "Removing execute bit from backup file: $RecoverScriptBackup"
            chmod -x "$RecoverScriptBackup" ||\
               errmsg "Could not do: chmod -x \"$RecoverScriptBackup\""
         else
            errmsg "Not updated recovery script due to failure to move old recovery script aside with: mv -f \"$RecoverScriptPath\" \"$RecoverScriptBackup\""
         fi
      fi
   else
      if cp "$NewRecoverScriptPath" "$RecoverScriptPath"; then
         msg "Successfully generated new recovery script: $RecoverScriptPath"

         # Change ownership to SDPOwner for consistency
         chown "$SDPOwner:$SDPGroup" "$RecoverScriptPath" ||\
            warnmsg "Could not change ownership of recovery script to $SDPOwner:$SDPGroup"

         dbg "Setting execute bit on recover script: $RecoverScriptPath"
         chmod +x "$RecoverScriptPath" ||\
            errmsg "Could not do: chmod +x \"$RecoverScriptPath\""
      else
         errmsg "Failed to install new recovery script : cp \"$NewRecoverScriptPath\" \"$RecoverScriptPath\""
      fi
   fi
fi

# Clean up staging directory
if [[ -d "$StagingBase" ]]; then
   msg "Cleaning up staging directory: $StagingBase"
   rm -rf "$StagingBase" ||\
      warnmsg "Could not remove staging directory: $StagingBase"
fi

if [[ "$ErrorCount" -eq 0 && "$WarningCount" -eq 0 ]]; then
   msg "\\n${H2}\\nSummary: SUCCESS:  P4 SDP Backup completed OK. Backups are here: $BackupDir"
elif [[ "$ErrorCount" -eq 0 ]]; then
   msg "\\n${H2}\\nSummary: SUCCESS:  P4 SDP Backup completed, but there were $WarningCount warnings. Backups are here: $BackupDir"
else
   msg "\\n${H2}\\nSummary: ERRORS:  There were $ErrorCount errors encountered attempting to backup the P4 SDP."
fi

exit "$ErrorCount"