prsync_depots.sh #12

#!/bin/bash
set -u

#==============================================================================
# Declarations and Environment
declare -i ErrorCount=0
declare -i WarningCount=0
declare -i SilentMode=0
declare -i Debug=0
declare -i NoOp=1

declare CmdLine="$0 $*"
declare ThisScript=${0##*/}
declare ThisHost=${HOSTNAME%%.*}
declare ThisOS=
declare Version="5.4.1"
declare ThisUser=
declare RunUser=
declare SourceHost=
declare SourceBasePath=
declare TargetBasePath=
declare WorkingDir=
declare DirName=
declare RsyncCmd=
declare RsyncScript=
declare PidList=
declare DepotListFile=
declare SSHKeyFile=
declare SSHOptions=
declare MaxRsyncProcs=
declare AbsoluteMaxRsyncProcs=
declare DefaultMaxRsyncProcs=3
declare CurrentRsyncProcs=0
declare SampleStaticCfgFile="${0%.sh}.cfg.sample"
declare StaticCfgFile=
declare StaticCfgName=default
declare MaxProcsCfgFile=
declare Log=
declare OldLog=
declare OldLogTimestamp=
declare H1="=============================================================================="
declare H2="------------------------------------------------------------------------------"

declare -i DepotDirCount=0
declare -a DepotDirList=()
declare -i SleepDelay=10
declare -i OKCount=0
declare -i DoCleanup=1
declare -i PreflightOnly=0

#==============================================================================
# Local Functions
function msg () { echo -e "$*"; }
function dbg () { [[ "$Debug" -ne 0 ]] && msg "DEBUG: $*"; }
function errmsg () { msg "\\nError: ${1:-Unknown Error}\\n"; ErrorCount+=1; }
function warnmsg () { msg "\\nWarning: ${1:-Unknown Warning}\\n"; WarningCount+=1; }
function bail () { errmsg "${1:-Unknown Error}"; exit "$ErrorCount"; }

#------------------------------------------------------------------------------
# Function: terminate
function terminate
{
   # Disable signal trapping.
   trap - EXIT SIGINT SIGTERM

   # Stop logging.
   [[ "$Log" == off ]] || msg "Log is: $Log\\n${H1}"

   # With the trap removed, exit.
   exit "$ErrorCount"
}

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
# The default is -h.
#
# $2 - error message (optional).  Specify this if usage() is called due to
# user error, in which case the given message displayed first, followed by the
# standard usage message (short or long depending on $1).  If displaying an
# error, usually $1 should be -h so that the longer usage message doesn't
# obscure the error message.
#
# Sample Usage:
# usage
# usage -h
# usage -man
# usage -h "Incorrect command line usage."
#------------------------------------------------------------------------------
function usage
{
   declare style=${1:--h}
   declare errorMessage=${2:-Unset}

   if [[ "$errorMessage" != Unset ]]; then
      msg "\\n\\nUsage Error:\\n\\n$errorMessage\\n\\n"
   fi

   echo "USAGE for $ThisScript v$Version:

$ThisScript [-c <cfg_File>] [-p] [-y] [-d|-D] [-si]

or

$ThisScript [-h|-man]
"
   if [[ $style == -man ]]; then
      msg "
DESCRIPTION:
	This script is useful for copying depot directories across two p4d
	servers, e.g. to aid it backing up or migrating data.  It is
	intended to be executed on the target server, and it uses a series
	of parallel rsync commands to pull from the source server.

	It requires that SSH keys be setup such that password-less ssh
	is enabled from the target server into the source server, so that
	'ssh' and 'rsync' commands are not prompted for a password.

	This script uses 2 configuration files, a static configuration file
	and a 'max procs' configuration file.

STATIC CONFIG FILE:
	The static configuration file defines settings that are loaded at
	startup, and cannot change once the process starts.  The following
	sample file illustrates settings:
"

if [[ -r "$SampleStaticCfgFile" ]]; then
   msg "=== BEGIN Sample Static Config File"
   cat "$SampleStaticCfgFile"
   msg "=== END Sample Static Config File"
else
   msg "The sample static config file is missing: $SampleStaticCfgFile\\n"
fi

msg "MAX PROCS CONFIG FILE:
	The 'max procs' config file is created when this script starts,
	unless it already exists.

	The file is: $MaxProcsCfgFile

	It is a one-line, file, and looks something like this:

	MAX_RSYNC_PROCS=8

	This defines the max rsync processes that can be running, and is a
	form of throttle control. It can be adjusted up or down by carefully
	editing the file even while the script is running. Simply change the
	number on the right side of the '=' in the line to change the value
	of MAX_RSYNC_PROCS.

	If increased during processing, more threads will be added quickly
	(after about $SleepDelay seconds).

	If decreased during processing, it may take a while for in-flight
	rsyncs to finish before the process count gets down to the desired
	level.

	This script will (by design) not distinguish 'rsync' commands
	launched from this script with any other rsync processing that may
	be occurring, so other (perhaps manual) rsyncs can keep this script
	from launching rsync threads, delaying its progress, but also
	avoiding overloading the machine.

	In any case, that absolute max processes that can be launched is 2
	less than the number of processors (but never less than a max of 4).

	The number of processors is determined on Linux with:
	grep -c processor /proc/cpuinfo

OPTIONS:
 -c <static_cfg_file>
 	Specify the path to the config file.  The default is:
        /p4/common/config/${ThisScript%.sh}.<SDP_INSTANCE>.cfg

 -p	Specify '-p' to indicate that only preflight checks should be
 	run, and then the script will exit. This can be useful to view
	the loaded configuration file.

 -y	Specify '-y' to indicate live processing is to occur.  By default, this
 	script operates in NoOp mode, meaning it prints data-affecting commands
	instead of running them.

 	NoOp mode is effectively a dry-run. This will verify SSH access to the
	remote server.

 -L <log>
	Specify the path to a log file, or the special value 'off' to disable
	logging.  By default, all output (stdout and stderr) goes to
	$Log

	NOTE: This script is self-logging.  That is, output displayed on the
	screen is simultaneously captured in the log file.  Do not run this
	script with redirection operators like '> log' or '2>&1', and do not
	use 'tee.'

 -si	Operate silently.  All output (stdout and stderr) is redirected to the
 	log only; no output appears on the terminal.  This cannot be used with
	'-L off'.
      
	This is useful when running from cron, as it prevents automatic
	email from being sent by cron directly, as cron does when a script
	called from cron generates output.

	Some preflight checks that occur early in processing, before logging is
	initiated, may be displayed to the output even if '-si' -is used.

 -d     Set debugging verbosity; enable debug messages.  If '-d' is
 	specified, the working directory is not removed even if all
	processing is successful.

 -D     Set extreme bash 'set -x' debugging verbosity. -D also implies -d.

HELP OPTIONS:
 -h	Display short help message
 -man	Display man-style help message

EXAMPLES:
	Example 1: Running preflight only:
	prsync_depots.sh -c /path/to/prsync_depots.cfg -p

	Example 2: Dry run specifying a config file:
	prsync_depots.sh -c /path/to/prsync_depots.cfg

	Example 3: Kicking off a live run preflight only:
	prsync_depots.sh -c /path/to/prsync_depots.cfg -y
"
   fi

   exit 1
}

#------------------------------------------------------------------------------
# Function set_absolute_max_procs()
#
# Sets global variable $AbsoluteMaxRsyncProcs to 2 less than the number of
# processors available.
#------------------------------------------------------------------------------
function set_absolute_max_procs() {

   # On Mac, there is no /proc/cpuinfo to look at.
   if [[ "$ThisOS" == "Darwin" ]]; then
      AbsoluteMaxRsyncProcs=4
      msg "Absolute max procs allowed is: $AbsoluteMaxRsyncProcs"
      return
   fi

   AbsoluteMaxRsyncProcs=$(grep -c processor /proc/cpuinfo)
   AbsoluteMaxRsyncProcs=$((AbsoluteMaxRsyncProcs-2))

   if [[ "$AbsoluteMaxRsyncProcs" =~ ^[0-9]+ ]]; then
      [[ "$AbsoluteMaxRsyncProcs" -lt 4 ]] && AbsoluteMaxRsyncProcs=4
      msg "Absolute max procs allowed is: $AbsoluteMaxRsyncProcs"
   else
      bail "Could not calculate absolute max processors."
   fi
}

#------------------------------------------------------------------------------
# Function set_max_configured_procs()
#
# Reads global variable $AbsoluteMaxRsyncProcs
# Reads global variable $DefaultMaxRsyncProcs
# Reads global variable $MaxProcsCfgFile
# Sets global variable $MaxRsyncProcs
# 
# Load max configured procs from a config file, generating the config file if
# needed. Ensure
#------------------------------------------------------------------------------
function set_max_configured_procs() {

   if [[ ! -r "$MaxProcsCfgFile" ]]; then
      if echo "MAX_RSYNC_PROCS=$DefaultMaxRsyncProcs" > "$MaxProcsCfgFile"; then
         msg "Generated config file: $MaxProcsCfgFile"
      else
         warnmsg "Failed to generate config file: $MaxProcsCfgFile; setting MaxRsyncProcs to $DefaultMaxRsyncProcs."
         MaxRsyncProcs=$DefaultMaxRsyncProcs
         return 1
      fi
   fi

   MaxRsyncProcs=$(grep MAX_RSYNC_PROCS= "$MaxProcsCfgFile"|cut -d '=' -f 2)

   if [[ ! "$MaxRsyncProcs" =~ ^[0-9]+ ]]; then
      warnmsg "Invalid MAX_RSYNC_PROCS value loaded [$MAX_RSYNC_PROCS]. It must be purely numeric. Setting MaxRsyncProcs to $DefaultMaxRsyncProcs."
      MaxRsyncProcs=$DefaultMaxRsyncProcs
   fi

   if [[ "$MaxRsyncProcs" -gt "$AbsoluteMaxRsyncProcs" ]]; then
      msg "Warning: Reducing over-limit configured max procs [$MaxRsyncProcs] to the absolute limit [$AbsoluteMaxRsyncProcs]."
      MaxRsyncProcs="$AbsoluteMaxRsyncProcs"
   fi

   msg "Throttle Status: $CurrentRsyncProcs of max $MaxRsyncProcs running."
}

#==============================================================================
# Command Line Processing

declare -i shiftArgs=0

set +u
while [[ $# -gt 0 ]]; do
   case $1 in
      (-h) usage -h;;
      (-man) usage -man;;
      (-c)
         StaticCfgFile="$2"; shiftArgs=1
         StaticCfgName=${StaticCfgFile##*/}
         StaticCfgName=${StaticCfgName%.cfg}
      ;;
      (-p) PreflightOnly=1;;
      (-y) NoOp=0;;
      (-L) Log="$2"; shiftArgs=1;;
      (-si) SilentMode=1;;
      (-d) Debug=1;;
      (-D) Debug=1; set -x;; # Use 'bash -x' extreme debugging mode.
      (*) usage -h "Unknown arg ($1).";;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usage -h "Incorrect number of arguments."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Command Line Verification

[[ "$SilentMode" -eq 1 && "$Log" == off ]] && \
   usage -h "Cannot use '-si' with '-L off'."

[[ -n "$Log" ]] || Log="${LOGS:-${HOME:-/tmp}}/${ThisScript%.sh}.${StaticCfgName}.log"

[[ -n "$StaticCfgFile" ]] || \
   StaticCfgFile="/p4/common/config/${ThisScript%.sh}.${SDP_INSTANCE:-1}.cfg"

[[ -n "$MaxProcsCfgFile" ]] || \
   MaxProcsCfgFile="${P4TMP:-/tmp}/${ThisScript%.sh}.max_procs.cfg"

#==============================================================================
# Main Program

trap terminate EXIT SIGINT SIGTERM

if [[ "$Log" != off ]]; then
   touch "$Log" || bail "Couldn't touch log file [$Log]."

   if [[ -e "$Log" ]]; then
      # shellcheck disable=SC2012
      OldLogTimestamp="$(ls -l --time-style +'%Y-%m-%d-%H%M%S' "$Log" | awk '{print $6}')"
      OldLog="${LOGS:-${HOME:-/tmp}}/${ThisScript%.sh}.${StaticCfgName}.${OldLogTimestamp}.log"
      mv -f "$Log" "$OldLog" || bail "Could not do: mv -f \"$Log\" \"$OldLog\""
      msg "Rotated log $Log to $OldLog"
   fi

   # Redirect stdout and stderr to a log file.
   if [[ "$SilentMode" -eq 0 ]]; then
      exec > >(tee "$Log")
      exec 2>&1
   else
      exec >"$Log"
      exec 2>&1
   fi

   msg "${H1}\\nLog is: $Log"
fi

msg "Started $ThisScript v$Version as $ThisUser@$ThisHost on $(date) with this command:\\n$CmdLine"

ThisUser=$(id -u -n)
ThisOS=$(uname -s)

#------------------------------------------------------------------------------
# Preflight Checks

msg "Starting Preflight Checks."

if [[ -r "$StaticCfgFile" ]]; then
   # shellcheck disable=SC1090
   source "$StaticCfgFile" || bail "Failed to load config file: $StaticCfgFile"
else
   errmsg "Missing config file [$StaticCfgFile]. Specify the path to the config file with '-c <cfg_file>'."
fi

[[ "$RunUser" == "$ThisUser" ]] || errmsg "Run as $RunUser, not $ThisUser."
[[ -n "${SourceHost:-}" ]] || errmsg "The SourceHost value is not defined."
[[ -n "${SourceBasePath:-}" ]] || errmsg "The SourceBasePath value is not defined."
[[ -n "${TargetBasePath:-}" ]] || errmsg "The TargetBasePath value is not defined."
[[ -n "${WorkingDir:-}" ]] || errmsg "The WorkingDir value is not defined."

if [[ "$ThisHost" != "$SourceHost" ]]; then
   msg "Verified: Not running on source host $SourceHost"
else
   errmsg "This must not run on the source host [$SourceHost]."
fi

if [[ "$SourceHost" = 127.0.0.1 ]]; then
   msg "Normalizing host value '127.0.0.1' to 'localhost'."
   SourceHost=localhost
fi

if [[ -n "$DepotListFile" ]]; then
   if [[ -s "$DepotListFile" ]]; then
      msg "Using user-specified depot list file: $DepotListFile"
   else
      errmsg "User-specified depot list file does not exist (or is zero-length): $DepotListFile"
   fi
fi

[[ -n "$SSHKeyFile" ]] && SSHOptions+=" -i $SSHKeyFile"

if [[ -n "$SSHOptions" ]]; then
   msg "SSH Options:\\n\\tUser-specified options to pass to ssh: \"$SSHOptions\"\\n\\tUser-specified options to pass to rsync: -e \"ssh $SSHOptions\""
fi

#------------------------------------------------------------------------------
# Start of actual processing.

if [[ "$NoOp" -eq 1 ]]; then
   msg "\\nNO_OP: Operating in Dry Run Mode; no actual rsyncs will be done. Use '-y' to do actual rsyncs.\\n"
fi

msg "Static configuration data loaded from $StaticCfgFile ($StaticCfgName):"
msg "
  SourceHost=$SourceHost
  SourceBasePath=$SourceBasePath
  TargetBasePath=$TargetBasePath
  WorkingDir=$WorkingDir
  DepotListFile=$DepotListFile
  SSHKeyFile=$SSHKeyFile
  SSHOptions=$SSHOptions"

if [[ -n "$DepotListFile" ]]; then
   msg "\\n  No depot list file specified. All directories and symlinks under $SourceBasePath on $SourceHost will be copied."
fi

msg "\\nEND Static Config File Data"

if [[ ! -d "$WorkingDir" ]]; then
   if ! mkdir -p "$WorkingDir"; then
      bail "Failed to initialize working dir $WorkingDir\\nAborting."
   fi
else
   errmsg "Working dir should not exist at startup: $WorkingDir\\nAborting."
fi

set_absolute_max_procs
set_max_configured_procs

msg "MaxProcsCfgFile is: $MaxProcsCfgFile"

if [[ "$SourceHost" != localhost ]]; then
   # shellcheck disable=SC2086
   if ssh $SSHOptions -n -q "$SourceHost" hostname > /dev/null 2>&1; then
      msg "Verified; Password-less accesss to $SourceHost is OK."
   else
      errmsg "Could not access host $SourceHost."
   fi
else
   dbg "Skipping ssh test due to SourceHost=localhost."
fi

if [[ -z "$DepotListFile" ]]; then
   DepotListFile="$(mktemp)"
   if [[ "$SourceHost" != localhost ]]; then
      dbg "Running: ssh $SSHOptions -n -q \"$SourceHost\" \"cd $SourceBasePath; ls -A\" .GT \"$DepotListFile\""
      # shellcheck disable=SC2086
      if ssh $SSHOptions -n -q "$SourceHost" "cd $SourceBasePath; ls -A" > "$DepotListFile"; then
         msg "Generated depot list file from host $SourceHost: $DepotListFile"
      else
         bail "Failed to generate depot list file from host $SourceHost: $DepotListFile\\nAborting."
      fi
   else
      dbg "Running: { cd $SourceBasePath; ls -A; } .GT \"$DepotListFile\""
      if { cd "$SourceBasePath"; ls -A; } > "$DepotListFile"; then
         msg "Generated depot list file from localhost: $DepotListFile"
      else
         bail "Failed to generate depot list file from localhost: $DepotListFile\\nAborting."
      fi
   fi
fi

msg "\\nReading list of depots."

while read -r DirName; do
   dbg "Confirming dir $SourceHost:$SourceBasePath/$DirName is a directory or symlink."
   if [[ "$SourceHost" != localhost ]]; then
      dbg "Running: ssh $SSHOptions -n -q \"$SourceHost\" \"cd $SourceBasePath; [[ -d $DirName || -L $DirName ]] || exit 1\""
      # shellcheck disable=SC2086
      if ssh $SSHOptions -n -q "$SourceHost" "cd $SourceBasePath; [[ -d $DirName || -L $DirName ]] || exit 1"; then
         DepotDirList[DepotDirCount]="$DirName"
         DepotDirCount+=1
      else
         warnmsg "Skipping bogus remote path $SourceHost:$SourceBasePath/$DirName"
      fi
   else
      dbg "Running: { cd $SourceBasePath; [[ -d $DirName || -L $DirName ]]; }"
      if { cd "$SourceBasePath"; [[ -d $DirName || -L $DirName ]]; }; then
         DepotDirList[DepotDirCount]="$DirName"
         DepotDirCount+=1
      else
         warnmsg "Skipping bogus local path $SourceBasePath/$DirName"
      fi
   fi
done < "$DepotListFile"

msg "\\nLoaded list of ${#DepotDirList[@]} depots to rsync:"
for DepotDir in "${DepotDirList[@]}"; do
   msg "  $DepotDir"
done < "$DepotListFile"

if [[ "$ErrorCount" -eq 0 ]]; then
   if [[ "$PreflightOnly" -eq 0 ]]; then
      msg "\\nPreflight checks OK."
   else
      msg "\\nExiting after preflight checks due to '-p'."
      exit 0
   fi
else
   bail "Aborting due to failed preflight checks."
fi

msg "\\nProcessing ${#DepotDirList[@]} depots."
for DepotDir in "${DepotDirList[@]}"; do
   if [[ "$NoOp" -eq 0 ]]; then
      CurrentRsyncProcs=$(ps -C rsync -o pid=|wc -l)
   else
      # In NoOp mode, check for sleep rather than rsync processes.
      CurrentRsyncProcs=$(ps -C sleep -o pid=|wc -l)
   fi
   set_max_configured_procs

   while [[ "$CurrentRsyncProcs" -ge "$MaxRsyncProcs" ]]; do
      msg "Enough rsyncs are running. Sleeping $SleepDelay before checking again."
      sleep "$SleepDelay"

      CurrentRsyncProcs=$(ps -C rsync -o pid=|wc -l)
      MaxRsyncProcs=$(grep MAX_RSYNC_PROCS= "$MaxProcsCfgFile"|cut -d '=' -f 2)
   done

   if [[ "$NoOp" -eq 0 ]]; then
      msg "Kicking off rsync for $DepotDir."
   else
      msg "NO_OP: Simulated: Kicking off rsync for $DepotDir."
   fi

   # Construct the rsync command.
   RsyncCmd="rsync"

   # Add SSHOptions if it was set in the config file.
   [[ -n "$SSHOptions" ]] && RsyncCmd+=" -e \"ssh $SSHOptions\""

   # If SourceHost is localhost, tweak the rsync command to avoid using ssh unnecessarily.
   if [[ "$SourceHost" != localhost ]]; then
      RsyncCmd+=" -a --exclude=lost+found --exclude=.snapshot $SourceHost:$SourceBasePath/$DepotDir/ $TargetBasePath/$DepotDir"
   else
      RsyncCmd+=" -a --exclude=lost+found --exclude=.snapshot $SourceBasePath/$DepotDir/ $TargetBasePath/$DepotDir"
   fi

   dbg "Constructed rsync command is: $RsyncCmd"

   RsyncScript="$WorkingDir/rsync.$DepotDir.sh"
   RsyncLog="$WorkingDir/rsync.$DepotDir.log"
   [[ -f "$TargetBasePath/$DepotDir" ]] || mkdir -p "$TargetBasePath/$DepotDir"

   # Generate each individual rsync script.
   {
      # Note intentional use use of double-quote (immediate expansion) and
      # single quote (deferred expansion) below.
      echo '#!/bin/bash'
      echo -e "\\necho Running: $RsyncCmd"

      # In No-Op mode, do sleeps rather than rsync commands.
      if [[ "$NoOp" -eq 0 ]]; then
         echo -e "\\n$RsyncCmd"
      else
         echo -e "\\necho NO_OP: Would run: $RsyncCmd"
         echo "sleep 5"
      fi
      # shellcheck disable=SC2016
      echo 'ec=$?'
      # shellcheck disable=SC2016
      echo 'echo EXIT_CODE=$ec'
      # shellcheck disable=SC2016
      echo 'echo That took $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.'
      # shellcheck disable=SC2016
      echo 'exit $ec'
   } > "$RsyncScript" || bail "Could not generate script: $RsyncScript"
   chmod +wx "$RsyncScript"

   if [[ "$Debug" -ne 0 ]]; then
      dbg "Generated rsync script [$RsyncScript]:" 
      cat "$RsyncScript"
   fi

   dbg "nohup $RsyncScript .LT. /dev/null .GT. $RsyncLog 2.GT..AM.1 .AM."
   nohup "$RsyncScript" < /dev/null > "$RsyncLog" 2>&1 &
   PidList="$PidList $!"
done

msg "\\n${H2}\\nAll rsync processes have been kicked off.  Awaiting completion of these processes:\\n$PidList"

# Redirect stderr to /dev/null, as we expect to have errors like:
# wait: pid 123459 is not a child of this shell
# That's just because this process can run for days, so long that some pids will
# be recycled.  That's not a problem, since bash 'wait' knows not to wait for those
# pids, and to only wait for completion of pids that are a child of this shell.
# shellcheck disable=SC2086
wait $PidList 2>/dev/null

msg "\\nAnalyzing parallel rsync logs."

# shellcheck disable=SC2044
for f in $(find "$WorkingDir/" -type f -name "rsync.*.log" -print); do
   if grep -q 'EXIT_CODE=0' "$f"; then
      OKCount+=1
   else
      errmsg "Error(s) found in log: $f"
      cat "$f"
   fi
done

# If there were any errors, skip the cleanup of the working directory.
if [[ "$DoCleanup" -eq 1 ]]; then
   if [[ "$DepotDirCount" -eq "$OKCount" ]]; then
      msg "\\nCopied all $DepotDirCount directories successfully."
      if [[ "$Debug" -eq 0 ]]; then
         msg "Removing working dir [$WorkingDir] due to successful rsync of all depots."
         rm -rf "$WorkingDir"
      else
         dbg "Keeping working dir: $WorkingDir"
      fi
   else
      errmsg "Tried to copy $DepotDirCount depot directories. Only $OKCount were OK."
      msg "Skipping cleanup of working dir: $WorkingDir"
   fi
else
   msg "Skipping cleanup of working dir: $WorkingDir"
fi

msg "${H2}\\nSummary:"
if [[ "$ErrorCount" -eq 0 && "$WarningCount" -eq 0 ]]; then
   msg "All processing completed successfully, with no errors or warnings."
elif [[ "$ErrorCount" -eq 0  ]]; then
   warnmsg "Processing completed with no errors, but $WarningCount warnings were encountered:"
   grep ^Warning: "$Log"
else
   msg "$ErrorCount errors and $WarningCount warnings were encountered:"
   grep ^Error: "$Log"
   grep ^Warning: "$Log"
fi

msg "\\nThat took $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n"