sdp_sync #2

#!/bin/bash
#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Helm Management System (HMS), and also available online:
# https://workshop.perforce.com/projects/p4-hms/main/LICENSE
#------------------------------------------------------------------------------

# This is the HMS "Tight Ship" synchronization and verification script.
# This script keeps Perforce Server Deployment Package (SDP) scripts in
# sync on a set of SDP hosts (defined in sdp_hosts.cfg), i.e. when doing
# "SDP Fleet Management."
#
# See usage() below (or run script with -man) for details.

#==============================================================================
# Declarations and Environment

# Environment isolation.  For stability and security reasons, prepend
# Allow override of P4U_HOME, which is set only when testing P4U scripts.
export P4U_HOME=${P4U_HOME:-/p4/common/bin}
export P4U_LIB=${P4U_LIB:-/p4/common/lib}
export P4U_ENV=$P4U_LIB/p4u_env.sh
export P4U_LOG=Unset

# PATH to include dirs where known-good scripts exist.
# known/tested PATH and, by implication, executables on the PATH.
export P4CBIN=${P4CBIN:-/p4/common/bin}
export P4CCFG=${P4CCFG:-/p4/common/config}
export PATH=$P4CBIN:$PATH:~/bin:.
export P4CONFIG=${P4CONFIG:-.p4config}
export P4ENVIRO=/dev/null/.p4enviro

[[ -r "$P4U_ENV" ]] || {
   echo -e "\nError: Cannot load environment from: $P4U_ENV\n\n"
   exit 1
}

declare BASH_LIBS=$P4U_ENV
BASH_LIBS+=" $P4U_LIB/libcore.sh"
BASH_LIBS+=" $P4U_LIB/libp4u.sh"
BASH_LIBS+=" $P4CBIN/backup_functions.sh"

for bash_lib in $BASH_LIBS; do
   # shellcheck disable=SC1090
   source "$bash_lib" ||\
      { echo -e "\nFATAL: Missing lib [$bash_lib]. Aborting.\n"; exit 1; }
done

declare Version=2.5.1
declare DirList=
declare HostList=
declare HostCfgFile=
declare EnvFile=$P4U_HOME/p4_vars
declare SDPInstance=${SDP_INSTANCE:-Unset}
declare HostSpecificDirListFile=
declare CronDir=/p4/common/site/cron.d
declare CronFile=
declare StatusCmd=
declare TmpFile=
declare P4Bin=/p4/common/bin/p4_bin
declare -i SilentMode=0
declare -i EmailAlways=0
declare -i EmailOnFailure=0
declare -i RemoveOldLogs=0
declare -i CheckCronFiles=0
declare -i DoStatusCheck=0
declare -i DoStatusMCheck=0
declare -i CronDefined=0
declare -i HostListSet=0
export VERBOSITY=3

#==============================================================================
# Local Functions

#------------------------------------------------------------------------------
# Function: terminate
function terminate
{
   # Disable signal trapping.
   trap - EXIT SIGINT SIGTERM

   declare -i sendEmail=0
   local mail_sender_opt=

   vvmsg "$THISSCRIPT: EXITCODE: $OverallReturnStatus"

   # Stop logging.
   if [[ "${P4U_LOG}" != off ]]; then
      stoplog

      # Email.
      [[ $EmailAlways -eq 1 ]] && sendEmail=1
      [[ $EmailOnFailure -eq 1 && $OverallReturnStatus -ne 0 ]] && sendEmail=1

      if [[ $sendEmail -eq 1 ]]; then
         emailStatus="OK"
         [[ $OverallReturnStatus -ne 0 ]] && emailStatus="Error"

         mail_sender_opt=$(get_mail_sender_opt)
         run "$SDPMAIL -s \"${HOSTNAME%%\.*} p4_$SDPInstance sdp_sync [$emailStatus]\" $mail_sender_opt $MAILTO < $P4U_LOG"
      fi
   fi

   # Don't litter.
   cleanTrash

   # With the trap removed, exit.
   exit "$OverallReturnStatus"
}

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
#------------------------------------------------------------------------------
function usage
{
   declare style=${1:--h}

   echo "USAGE for $THISSCRIPT v$Version:

$THISSCRIPT [-H <host1>[,<host2>,...]] [-i <instance>] [-a] [-s] [-m] [-c] [-r] [-e|-E] [-L <log>] [-si] [-v<n>] [-n|-N] [-D]

or

$THISSCRIPT [-h|-man|-V]
"
   if [[ $style == -man ]]; then
      echo -e "
DESCRIPTION:
	This is the \"Tight Ship\" compliance check script, a part of the
	Helm Management System (HMS).

	This script keeps the Perforce P4 Server Deployment Package (SDP)
	scripts in sync on all SDP hosts, as defined by the SDP_SYNC_HOSTS
	setting in the host config file (see FILES bellow).  It does this
	by doing a 'p4 sync' command to deploy updates, and optionally
	doing a 'p4 status' command in key directories to ensure the files
	on the machine match version control precisely.

	The 'p4 sync' is done targeting a Helm Management System server.
	Typically a dedicated 'p4hms.YourCompany.com' virtual machine is
	setup as a Linux bastion host, and a dedicated 'hms' instance of SDP
	on that machine is used to manage the SDP on all other machines. In
	some cases, the hms_ts_setup.sh script is used to initialize the
	HMS server instance automatically.

	In other cases, an //sdp depot is added to an existing SDP instance
	used used for other purposes. This can be done when there is a desire
	to maintian a single P4D instance for an entire organization.

	If used with the '-c' flag, this script also verifies that versioned
	cron files match actual crontab files on each managed host.

	This script is intended to be called by a cron job the master server.
	It does ssh calls to HMS-managed SDP hosts.

	Each target host is expect to have a P4CONFIG file named
	/p4/.p4config.SDP that defines Perforce environment settings that
	point to Perforce workspaces that enable versioning of the SDP on that
	host.

	An SDP worskpace must be configured for each host.  Typically they
	reference paths in the Perforce server that are common across
	all SDP servers, e.g. to populate the /p4/common folder.
	Other paths are host-specific, like the /p4/N/bin folders
	that indicate which instances are active on the machine as well
	as which type of servers are active for each instance (p4d,
	p4p, p4broker, etc.).

	SSH keys must be setup such that the Perforce login (as defined by
	OSUSER in $EnvFile) can ssh without a password to all SDP  hosts.
	To simplify failover, the backup servers should also be able to
	ssh to each other without a password; security implications should
	be considered here.

TIGHT SHIP COMPLIANCE
	This ship considers an environment to be \"Tight Ship\" compliant
	if:
	* A 'p4 opened' in the SDP workpsace on all hosts reports files
	checked out anywhere.
	* A 'p4 sync' in the SDP workspace on all hosts reports no errors.
	* A 'p4 status' on key directories in the SDP workspace on all hosts
	reports no errors.
	* Crontabs on all hosts match the versioned files.

	If all these conditions are met, the output will contain the message:

	Verified: Tight Ship compliant.

	This message can only appear if the '-s' and '-c' flags are used
	or implied, indicating all possible checks are done. The '-m flag
	implies '-s' and '-a' implies both '-s' and '-c'.

OPTIONS:
 -a	Specifying '-a' is a shorthand, the equivalent of specifying
	'-s -c -r -E'.

 -H <host1>[,<host2>,...]
	Specify a comma-delimited list of hosts to push to.  By default,
	the SDP_SYNC_HOSTS value defined in the config file $HostCfgFile
	determines the list of hosts to push to. If '-c' is specified to
	check crontabs, that check is also limited to the list of hosts
	specified.

 -i <instance>
	Specify the SDP instance tag.  The default is to use the
	\$SDP_INSTANCE variable if defined. This will be set if the
	standard SDP mechanism for setting shell environment variables
	is used, e.g. by souring p4_vars with the instance tag name, e.g.:

	source $EnvFile 1

	If \$SDP_INSTANCE is not defined in the shell environment, specifying
	'-i <instance>' is required.

 -s	Specify that a 'p4 status' check should be done in directories
	specified by the SDP_STATUS_DIRS variable defined in $HostCfgFile.

	Alternately, a host-specific value for SDP_STATUS_DIRS may be
	defined in a file named sdp_status_dirs.<hostname>.cfg in
	the $P4CCFG dir.

 -m	Use with '-s' to specify that the '-m' flag to 'p4 status' should
	be used for a faster check (without MD5 digest comparison).  If
	'-m' is specified without '-s', the '-s' is implied.

 -c	Specifies that the crontab for the OSUSER ($OSUSER) on each machine
	should be compared against the versioned cron file in:

	$CronDir

	In that directory, the cron files are named crontab.<user>.<hostname>

	The cron check is done for all hosts defined in the ALL_SDP_HOSTS
	setting in $HostCfgFile.

 -r
	Specify this option to remove old sdp_sync.*.log files.  If this option
	is specified, log files named /p4/<n>/logs/sdp_sync.*.log (where '<n>'
	is the SDP instance name) that are older than the number of journal
	rotations indicated by the KEEPLOGS setting in $EnvFile are removed.

	The old log removal occurs only upon successful completion.

 -e	Send email to MAILTO value defined in $EnvFile in event
	of failure only.

 -E	Send email to MAILTO value defined in $EnvFile.

 -v<n>	Set verbosity 1-5 (-v1 = quiet, -v5 = highest).

 -L <log>
	Specify the path to a log file, or the special value 'off' to disable
	logging.  By default, all output (stdout and stderr) goes to a log
	file named sdp_sync.<datestame>.log in $LOGS.

	NOTE: This script is self-logging.  That is, output displayed on the screen
	is simultaneously captured in the log file.  Do not run this script with
	redirection operators like '> log' or '2>&1', and do not use 'tee.'

-si	Operate silently.  All output (stdout and stderr) is redirected to the log
	only; no output appears on the terminal.  This cannot be used with
	'-L off'.
      
 -n	No-Op.  Prints commands instead of running them.

 -N	No-Op.  Similar to '-n', but this command does execute the 'ssh' calls to
	get to the remote host, but then does 'p4 sync -n' rather than' 'p4 sync'
	on the remote host.

 -D     Set extreme debugging verbosity.

HELP OPTIONS:
 -h	Display short help message
 -man	Display man-style help message
 -V	Dispay version info for this script and its libraries.

FILES:
	The SDP environment file $EnvFile defines various
	SDP settings, and is used by several SDP scripts.

	The host config file defines the SDP_SYNC_HOSTS and other
	values.  The default file is:

	$HostCfgFile

	There can also be an instance-specific file.  If
	$P4CCFG/sdp_hosts.<SDP_Instance>.cfg exists,
	it is used in place of sdp_hosts.cfg.  This file must
	define all values that are defined in sdp_hosts.cfg.

	If a host-specific dir list file exists, it is used
	to override the SDP_STATUS_DIRS for specific hosts.
	The host-specific dir list files are named:

	$P4CCFG/sdp_status_dirs.<host-shortname>.cfg

	Each HMS-mananged SDP host must have a P4CONFIG file,
	/p4/.p4config.SDP.  That will have settings like these
	sample values:

	P4PORT=ssl:p4hms.p4demo.com:7467
	P4CLIENT=SDP.bos-helix-01
	P4TICKETS=/p4/hms/.p4tickets
	P4TRUST=/p4/hms/.p4trust
	P4USER=perforce
	P4IGNORE=.p4ignore

EXAMPLES:
	EXAMPLE 1: Cron Usage

	Recommended crontab usage for SDP Instance 1:
	$THISSCRIPT -i 1 -si -a > /dev/null 2>&1

	The redirect to /dev/null is to avoid any output that
	would generate a duplicate email from cron.  Output is not lost;
	it is written to a timestampped log file:
	$LOGS/sdp_sync.<timestamp>.log

	EXAMPLE 2: Typical Command line usage:

	$THISSCRIPT -i 1 -a

	Note that the '-i <n>' flag is required unless

SEE ALSO:
	See this related script:
	$P4CBIN/global_replica_status
"
   fi

   exit 1
}

#==============================================================================
# Command Line Processing

declare -i shiftArgs=0

set +u
while [[ $# -gt 0 ]]; do
   case $1 in
      (-i) SDPInstance=$2; shiftArgs=1;;
      (-H) HostList=$2; HostListSet=1; shiftArgs=1;;
      (-r) RemoveOldLogs=1;;
      (-e) EmailOnFailure=1;;
      (-E) EmailAlways=1;;
      (-c) CheckCronFiles=1;;
      (-s) DoStatusCheck=1;;
      (-m) DoStatusCheck=1; DoStatusMCheck=1;;
      (-a)
         DoStatusCheck=1
         CheckCronFiles=1
         RemoveOldLogs=1
         EmailAlways=1
      ;;
      (-h) usage -h;;
      (-man) usage -man;;
      (-V) show_versions; exit 1;;
      (-v1) export VERBOSITY=1;;
      (-v2) export VERBOSITY=2;;
      (-v3) export VERBOSITY=3;;
      (-v4) export VERBOSITY=4;;
      (-v5) export VERBOSITY=5;;
      (-L) export P4U_LOG=$2; shiftArgs=1;;
      (-si) SilentMode=1;;
      (-n) export NO_OP=1;;
      (-N) export NO_OP=2;;
      (-D) set -x;; # Debug; use 'set -x' mode.
      (*) usageError "Unknown arg ($1).";;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usageError "Bad usage."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Command Line Verification

[[ $SilentMode -eq 1 && $P4U_LOG == off ]] && \
   usageError "Cannot use '-si' with '-L off'."

[[ $EmailOnFailure -eq 1 && $P4U_LOG == off ]] && \
   usageError "Cannot use '-e' or '-E' with '-L off'."

[[ $EmailAlways -eq 1 && $EmailOnFailure -eq 1 ]] && \
   usageError "The '-e' and '-E' flags are mutually exclusive."

[[ $SDPInstance == Unset ]] && \
   bail "SDP Environment is not set, and no instance parameter was provided. Aborting."

if [[ -r "${P4CCFG%config}/site/config/sdp_hosts.${SDPInstance}.cfg" ]]; then
   HostCfgFile="${P4CCFG%config}/site/config/sdp_hosts.${SDPInstance}.cfg"
elif [[ -r "${P4CCFG%config}/site/config/sdp_hosts.cfg" ]]; then
   HostCfgFile="${P4CCFG%config}/site/config/sdp_hosts.cfg"
elif [[ -r "$P4CCFG/sdp_hosts.${SDPInstance}.cfg" ]]; then
   HostCfgFile="$P4CCFG/sdp_hosts.${SDPInstance}.cfg"
else
   HostCfgFile="$P4CCFG/sdp_hosts.cfg"
fi

[[ -r "$HostCfgFile" ]] ||\
   bail "Could not find sdp_hosts.cfg or sdp_hosts.${SDPInstance}.cfg in ${P4CCFG%config}/site/config or $P4CCFG."

# shellcheck disable=SC1090
source "$HostCfgFile" ||\
   bail "Failed to load host configuration from $HostCfgFile. Aborting."

[[ -r "$EnvFile" ]] || bail "Missing SDP Environment file: $EnvFile"

# shellcheck disable=SC1090
source "$EnvFile" "$SDPInstance" ||\
   bail "Failed to load SDP Environment from $EnvFile. Aborting."

[[ $P4U_LOG == Unset ]] && P4U_LOG="${LOGS}/sdp_sync.$(date +'%Y%m%d-%H%M%S').log"

#==============================================================================
# Main Program

trap terminate EXIT SIGINT SIGTERM

declare -i OverallReturnStatus=0

if [[ "${P4U_LOG}" != off ]]; then
   touch "${P4U_LOG}" || bail "Couldn't touch log file [${P4U_LOG}]."

   # Redirect stdout and stderr to a log file.
   if [[ "$SilentMode" -eq 0 ]]; then
      exec > >(tee "${P4U_LOG}")
      exec 2>&1
   else
      exec >"${P4U_LOG}"
      exec 2>&1
   fi

   initlog
fi

export ALL_SDP_HOSTS=${ALL_SDP_HOSTS:-Unset}
export SDP_REPLICA_HOSTS=${SDP_REPLICA_HOSTS:-Unset}
export SDP_REMOTE_HOSTS=${SDP_REMOTE_HOSTS:-Unset}
export SDP_SYNC_HOSTS=${SDP_SYNC_HOSTS:-Unset}
export SDP_STATUS_DIRS=${SDP_STATUS_DIRS:-Unset}

# If '-H <host1>[,<host2>,...]' was specified as a comma-separated list
# on the command line, overide the SDP_SYNC_HOSTS value defined in
# the host config file.
[[ -n "$HostList" ]] && SDP_SYNC_HOSTS=$(echo "$HostList" | tr ',' ' ')

[[ "$SDP_SYNC_HOSTS" == Unset ]] && \
   bail "The SDP_SYNC_HOSTS variable is not defined in $HostCfgFile.  Aborting."

[[ "$ALL_SDP_HOSTS" == Unset && $CheckCronFiles -eq 1 ]] && \
   bail "The ALL_SDP_HOSTS variable is not defined in $HostCfgFile, and '-c' is specified which requires it to be defined. Aborting."

if [[ "$SDP_STATUS_DIRS" == Unset && $DoStatusCheck -eq 1 ]]; then
   HostSpecificDirListFile="$P4CCFG/sdp_status_dirs.${HOSTNAME%%.*}.cfg"

   if [[ -r "$HostSpecificDirListFile" ]]; then
      # shellcheck disable=SC1090
      source "$HostSpecificDirListFile"
      [[ "$SDP_STATUS_DIRS" == Unset ]] &&\
         bail "The SDP_STATUS_DIRS variable is not defined in $HostCfgFile or $HostSpecificDirListFile, and '-s' or '-m' is specified which requires it to be defined. Aborting."
   else
      bail "The SDP_STATUS_DIRS variable is not defined in $HostCfgFile or $HostSpecificDirListFile, and '-s' or '-m' is specified which requires it to be defined. Aborting."
   fi
fi

# Just in case commas were used instead of spaces in the config file, translate them
# to spaces here.
TmpFile=${P4TMP}/tmp.crontab.$$.$RANDOM
GARBAGE+=" $TmpFile"

for host in $(echo "$SDP_SYNC_HOSTS" | tr ',' ' '); do
   rrun "$host" "export P4CONFIG=/p4/.p4config.SDP\nexport P4ENVIRO=/dev/null/.p4enviro\n$P4Bin set\necho\n$P4Bin -s info -s\necho\n$P4Bin -s opened\n" "Checking for opened files in the SDP workspace on $host." 0 0 "not opened on this client"
   if [[ "$RCMDEXITCODE" -eq 0  ]]; then
      msg "Verified: No files are checked out in the SDP workspace on host $host."
   else
      warnmsg "Files are checked out (or login failed) in the SDP workspace on host $host."
      OverallReturnStatus=1
   fi

   if [[ "$NO_OP" -eq 0 || "$NO_OP" -eq 1 ]]; then
      rrun "$host" "export P4CONFIG=/p4/.p4config.SDP\nexport P4ENVIRO=/dev/null/.p4enviro\n$P4Bin set\necho\n$P4Bin -s info -s\necho\n$P4Bin -s sync\n" "Syncing SDP workspace on $host." 1 1 || OverallReturnStatus=1
   else
      rrun "$host" "export P4CONFIG=/p4/.p4config.SDP\nexport P4ENVIRO=/tmp/tmp.p4enviro.$$.$RANDOM\n$P4Bin set\necho\n$P4Bin -s info -s\necho\n$P4Bin -s sync -n\n" "Checking to see if files need to be sync'd on host $host." 0 1 || OverallReturnStatus=1
   fi

   if [[ "$DoStatusCheck" -eq 1 ]]; then
      if [[ $DoStatusMCheck -eq 1 ]]; then
         msg "${H}\nDoing 'p4 status -m' check in \$SDP_STATUS_DIRS: $SDP_STATUS_DIRS\n"
      else
         msg "${H}\nDoing 'p4 status' check in \$SDP_STATUS_DIRS: $SDP_STATUS_DIRS\n"
      fi

      # If a host-specifc dir list file exists, source it.  It is expected to
      # set SDP_STATUS_DIRS value.  Otherwise, the sdp_hosts.cfg file should
      # define a value for SDP_STATUS_DIRS.
      HostSpecificDirListFile="$P4CCFG/sdp_status_dirs.${host}.cfg"

      if [[ -r "$HostSpecificDirListFile" ]]; then
         msg "Loading host-specific list of dirs to status check from $HostSpecificDirListFile."
         # shellcheck disable=SC1090
         source "$HostSpecificDirListFile"
      fi

      DirList=$(echo "$SDP_STATUS_DIRS" | tr ',' ' ')
      for dir in $DirList; do
         StatusCmd="$P4Bin -s -d $dir status"
         [[ $DoStatusMCheck -eq 1 ]] && StatusCmd+=" -m"
         # shellcheck disable=SC1117
         rrun "$host" "[[ ! -d $dir ]] && exit 0; export P4CONFIG=/p4/.p4config.SDP\nexport P4ENVIRO=/tmp/tmp.p4enviro.$$.$RANDOM\n$P4Bin set\necho\n$StatusCmd" "Checking on $host with: $StatusCmd." 1 0 "No file\(s\) to reconcile"
         if [[ "$RCMDEXITCODE" -eq 0 ]]; then
            msg "Status reports clean for $host:$dir."
         else
            warnmsg "Status is not clean in $host:$dir."
            OverallReturnStatus=1
         fi
      done
   fi
done

if [[ "$CheckCronFiles" -eq 1 ]]; then
   msg "${H}\nChecking crontab files in \$ALL_SDP_HOSTS: $ALL_SDP_HOSTS\n"

   if [[ "$HostListSet" -eq 1 ]]; then
      List="$HostList"
   else
      List="$(echo "$ALL_SDP_HOSTS"|tr ',' ' ')"
   fi

   for host in $List; do
      CronFile=$CronDir/crontab.$OSUSER.$host
      msg "Checking crontab for user $OSUSER on host $host."

      if ssh -q "$host" crontab -l > "$TmpFile" 2>&1; then
         if [[ "$(cat "$TmpFile")" == *"no crontab for "* ]]; then
            CronDefined=0
         else
            CronDefined=1
         fi
      else
         errmsg "Unable to check crontab for $OSUSER@$host."
         continue
      fi

      if [[ "$CronDefined" -eq 1 ]]; then
         if [[ -e "$CronFile" ]]; then
            if diff "$CronFile" "$TmpFile" > /dev/null; then
               msg "Verified: Cron for user $OSUSER on host $host matches cron file [$CronFile]."
            else
               OverallReturnStatus=1
               errmsg "Cron for user $OSUSER on host $host does NOT match cron file [$CronFile].\nDiffs:\n$(diff "$CronFile" "$TmpFile")\n"
            fi
         else
            warnmsg "An active cron exists for $OSUSER on host $host, but the cron file is missing: $CronFile."
            OverallReturnStatus=1
         fi
      else # Cron not defined.
         if [[ -e "$CronFile" ]]; then
            warnmsg "No active cron exists for $OSUSER on host $host, but cron file exists: $CronFile."
            OverallReturnStatus=1
         else
            msg "Verified: No active cron exists for $OSUSER on host $host, and so no cron file exists."
         fi
      fi
   done
fi

if [[ $OverallReturnStatus -eq 0 ]]; then
   msg "${H}\nAll processing completed successfully.\n"

   # Display the "badge of honor" if tight ship compliance has been achieved.
   # This message is only done if all checks are done, including the 'p4 status'
   # checks and verification that crontabs match archive files.
   if [[ "$CheckCronFiles" -eq 1 && "$DoStatusCheck" -eq 1 ]]; then
      msg "Verified: Tight Ship compliant."
   fi

   if [[ $RemoveOldLogs -eq 1 ]]; then
      if [[ $NO_OP -eq 0 ]]; then
         msg "Cleanup: Removing $LOGS/sdp_sync.*.log files older than $KEEPLOGS days old (if any):"
         find "$LOGS/" -name "sdp_sync.*.log" -mtime +"$KEEPLOGS" -print -exec rm -f {} \;
      else
         msg "NO_OP: Would remove $LOGS/sdp_sync.*.log files older than $KEEPLOGS days old (if any):"
         find "$LOGS/" -name "sdp_sync.*.log" -mtime +"$KEEPLOGS" -print
      fi
   fi
else
   msg "${H}\nProcessing completed, but with warnings or errors.  Scan above output carefully.\n" 
fi

# Illustrate using $SECONDS to display runtime of a script.
msg "That took about $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\n"

# See the terminate() function, which is really where this script exits.
exit $OverallReturnStatus