run_scripted_tests.sh #2

#!/bin/bash
#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE
#------------------------------------------------------------------------------
set -u

#==============================================================================
# Declarations and Environment

declare ThisScript="${0##*/}"
declare Version=3.0.0
declare ThisUser=
declare ThisHost=${HOSTNAME%%.*}
declare CmdLine="${0} $*"
declare -i AbortOnTestFailure=0
declare -i Debug=${SDP_DEBUG:-0}
declare -i ErrorCount=0
declare -i SilentMode=0
declare -i TestCount=0
declare -i TestID=0
declare -i TestPassCount=0
declare -i TestFailCount=0
declare -i TestSkipCount=0
declare -i NoOp=0
declare -i LoadSDPEnv=1
declare -i ScriptedTestDataOK=0
declare -i AllScriptedTestDataOK=1
declare -i i=0
declare H1="=============================================================================="
declare H2="------------------------------------------------------------------------------"
declare H3="||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||"
declare H4="++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
declare Log=
declare LogLink="/tmp/${ThisScript%.sh}.log"

# Values loaded from general config file.
declare RunHost=
declare TestTag=

declare -i DisplayFullLogs=0
declare UserTestGroups=
declare -i InUserTestGroup=0
declare TestGroup=
declare TestScript=
declare ExpectedExit=
declare ExpectedString=
declare TestLog=
declare TestLogHost=
declare TmpLocalLog=
declare -i RequireValidTests=1
declare -i ExpectedExitCode
declare -i ActualExitCode
declare -i ExpectedStringInOutput=0
declare -i DoBSWPreflight=1
declare Comments=
declare CfgDir="/p4/sdp/test/bsw"
declare CfgFile=
declare -a TestGroupList
declare -a TestScriptList
declare -a ExpectedExitList
declare -a TestLogList
declare -a ExpectedStringList=
declare -a TestLogHostList
declare -a CommentsList
declare ScriptedTestDataFile=
declare OutputFile="/tmp/sdp.test_output.$$.$RANDOM"
declare -i TestPassed
declare -i Line=0
declare -i GarbageCount=0
declare -a Garbage

# For Preflight checks of the Lab Environment
declare TmpFile=
declare CommitJournalCounter=

# Color support.
declare GREEN=
declare RED=
declare YELLOW=
declare RESET=

# Garbage Collection
Garbage[GarbageCount]="$OutputFile"
GarbageCount+=1

# P4 Environment Isolation
unset P4CONFIG
export P4ENVIRO=/dev/null/.p4enviro

#==============================================================================
# Local Functions

function msg () { echo -e "$*"; }
function msg_green  { msg "${GREEN}$*${RESET}"; }
function msg_yellow { msg "${YELLOW}$*${RESET}"; }
function msg_red    { msg "${RED}$*${RESET}"; }
function dbg () { [[ "$Debug" -eq 0 ]] || msg "DEBUG: $*"; }
function errmsg () { msg_red "\\nError: ${1:-Unknown Error}\\n"; ErrorCount+=1; }
function bail () { errmsg "${1:-Unknown Error}"; exit "$ErrorCount"; }

function pass () { TestCount+=1; TestPassCount+=1; msg_green "PASS Test $TestCount"; }
function fail () {
   TestCount+=1
   TestFailCount+=1
   msg_yellow "FAIL Test $TestCount"
   if [[ "$AbortOnTestFailure" -eq 1 ]]; then
      msg_yellow "Aborting after first test failure due to '-e'."
      exit 1
   fi
}

#------------------------------------------------------------------------------
# Function: terminate
# shellcheck disable=SC2317
function terminate {
   # Disable signal trapping.
   trap - EXIT SIGINT SIGTERM

   local g=

   dbg "$ThisScript: EXITCODE: $ErrorCount"

   if [[ "$Debug" -eq 0 ]]; then
      for g in "${Garbage[@]}"; do
         echo rm -rf "$g"
      done
   else
      msg "Due to '-d' (debug) mode, skipping removal of these:"
      for g in "${Garbage[@]}"; do
         msg "$g"
      done
   fi

   # Stop logging.
   [[ "$Log" == off ]] || msg "\\nLog is: $Log\\n${H1}\\n"

   # With the trap removed, exit.
   exit "$((ErrorCount+TestFailCount+TestSkipCount))"
}

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
#------------------------------------------------------------------------------
function usage {
   declare style=${1:--h}

   msg "USAGE for $ThisScript v$Version:

$ThisScript [-cd <cfg_dir>] [-sp] [-no_env] [-FL] [-g <test_group_1>[,<test_group_2>,...]] [-e] [-f] [-L <log>] [-si] [-d|-D]

or

$ThisScript [-h|-man]
"
   if [[ $style == -man ]]; then
      msg "
DESCRIPTION:
	This script runs tests for the Server Deployment Package (SDP).

	Tests are defined in the file: $ScriptedTestDataFile

OPTIONS:
 -cd	Specify '-cd <cfg_dir>' to specify an alternate location for the
	configuration directory. This is expected to include a the
	${ScriptedTestDataFile##*/} file and the test.<hostname>.cfg file.
	See FILES for more information.

	The default is: $CfgDir

	This option implies '-sp'.

 -sp	Specify '-sp' to skip preflight checks, which are run by default
 	to ensure the servers in the Battle School Workshop Lab
	Environment are online and working properly, and specifically
	that the servers in the environment are all replicating to the
	same journal counter number and have a non-zero sequence number.

 -no_env
 	Specify '-no_env' to avoid loading the SDP Shell Environment (p4_vars).

	This option is implied if /p4/common/bin/p4_vars does not exist.

 -FL	Specify '-FL' to display full logs files.  The logged output
	indicates whether the 'grep' expression detected expected output
	in logs or command ouptut.  With this option, the entire
	contents of each log scanned is also displayed.

	This will greatly increase the size of the generated log file for
	this script.  It is intended to be useful for interactive
	debugging.
	
 -g <test_group_1>[,<test_group_2>,...]
	Specify '-g <test_group>' to specify a comma-delimited list of
	test groups to run.  If not specified, the default is to execute
	all test groups.

 -e	Use '-e' to abort immediately after the first test failure.

	By default, this script attempts to execute all tests in the
	test suite (in hopes of illuminating the most issues).  The
	'-e' option changes the behavior to stop execution after
	a test failure, so as to preserve the state of the system.
	This may be useful for interactive debugging of scripted tests.

 -f	Use '-f' to bypass any invalid test entries in the scripted test
	data file and process remaining valid tets. By default, no tests
	are executed if any are invalid.

LOGGING OPTIONS:
 -L <log>
	Specify the path to a log file, or the special value 'off' to disable
	logging.  By default, all output (stdout and stderr) goes to:
	/tmp/${ThisScript%.sh}.<timestamp>.log

 -si	Operate silently.  All output (stdout and stderr) is redirected to the log
	only; no output appears on the terminal.  This cannot be used with
	'-L off'.

LOGGING:
	This script is self-logging.  That is, output displayed on the screen
	is simultaneously captured in the log file.  It is not necessary (nor harmful)
	to use redirection operators like '> log' or '2>&1' or 'tee'.

	This script uses a fixed log symlink /tmp/${ThisScript%.sh}.log
	which is updated to point to the newly created log for each run, so that
	this symlink reliably points to the most recent execution of the script
	(unless '-L off' was used).

HELP OPTIONS:
 -h	Display short help message
 -man	Display man-style help message

DEBUGGING OPTIONS:

 -d	Enable debug messages, and prevents cleanup of some interim files that
	may be useful for debugging (with details noted in the log).
      
 -D     Use bash 'set -x' extreme debugging verbosity.  Implies '-d'.

FILES:
	TEST CONFIG DIR

	Files related to testing live in a configuration directory. When used for
	testing in the Battle School Lab Environment, the directory is $CfgDir on
	the bos-helix-01 machine.

	Ths can be changed with the '-cd <cfg_dir>' option.

	HOST SPECIFIC TEST ENVIRONMENT FILE

	If a file named test_sdp.<hostname>.cfg is found for the current host, this
	file is sourced.  It is expected to look something like this:

	  declare RunHost=bos-helix-01
	  declare TestTag=\"test_sdp.\$RunHost\"
	  declare TestWS=\"bruno_jam.\$TestTag\"
	  declare TestWSRoot=\"/p4/1/tmp/\$TestWS\"
	  export SDP_TEST_HOME=/p4/sdp/test/bsw

	The file is sourced to load any shell environment settings needed by scripted
	tests to be executed, such as PATH adjustments.  It must set values for
	RunHost and TestTag; other settings are optional.

	If no such host config file exists, it is equivalent to a file exsting
	with these contents:

	  declare RunHost=\${HOSTNAME%%.*}
	  declare TestTag=\"test.\$RunHost\"
	  declare TestWS=\"bruno_jam.\$TestTag\"
	  declare TestWSRoot=\"/p4/1/tmp/\$TestWS\"
	  export SDP_TEST_HOME=/p4/sdp/test/bsw

	SCRIPTED TEST CONFIG FILE

	The ${ScriptedTestDataFile##*/} file defines the scripted tests to be
	executed, including expected exit codes and strings to grep for in
	logs/output for each.

	Open this file to get documentation of the expected format.

	ABOUT THE TEST DIRECTORY

	The /p4/sdp/test directory will not appear on a customer-deployed
	SDP in the real world. The /p4/sdp directory will exist, but the 'test'
	directory and 'bsw' subdirectory are populated only when using the
	//sdp/dev_insitu stream in a Battle School Lab Environment.

EXAMPLES:
	Example 1: Typical Usage

	For typical usage, no arguments are needed. Run this in a Battle School
	Lab Environment as perforce@bos-helix-01, after first having run 'lab qa'
	as student@bsw-lab-ui to prepare the environment.

	\$ cd /p4/sdp/test/bsw
	\$ ./$ThisScript

	Example 2: Run specific test group(s).

	\$ cd /p4/sdp/test/bsw
	\$ ./$ThisScript -g LoadCheckpoint

	See test group names as defined in scripted_tests.cfg to help select
	values for '-g'.
"
   fi

   exit 1
}

#==============================================================================
# Command Line Processing

declare -i shiftArgs=0

set +u
while [[ $# -gt 0 ]]; do
   case $1 in
      (-cd) CfgDir="$2"; DoBSWPreflight=0; shiftArgs=1;;
      (-no_env) LoadSDPEnv=0;;
      (-sp) DoBSWPreflight=0;;
      (-FL) DisplayFullLogs=1;;
      (-g) UserTestGroups="$2"; shiftArgs=1;;
      (-e) AbortOnTestFailure=1;;
      (-f) RequireValidTests=0;;
      (-h) usage -h;;
      (-man) usage -man;;
      (-L) Log="$2"; shiftArgs=1;;
      (-si) SilentMode=1;;
      (-n) NoOp=1;;
      (-d) Debug=1;;
      (-D) Debug=1; set -x;; # Debug; use 'set -x' mode.
      (-*) usage -h "Unknown option ($1).";;
      (*) usage -h "Unknown parameter ($1).";;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usageError "Bad usage."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Command Line Verification

[[ -n "$Log" ]] || Log="/tmp/${ThisScript%.sh}.$(date +'%Y-%m-%d-%H%M%S').log"

[[ "$SilentMode" -eq 1 && "$Log" == off ]] && \
   usageError "Cannot use '-si' with '-L off'."

#==============================================================================
# Main Program

trap terminate EXIT SIGINT SIGTERM

# Detect support for colors
if [[ $SilentMode -eq 0 ]] \
  && command -v tput >/dev/null 2>&1 \
  && [[ -t 1 ]] \
  && [[ "$(tput colors)" -ge 8 ]]; then
  RED="$(tput setaf 1)" 
  GREEN="$(tput setaf 2)"
  YELLOW="$(tput setaf 3)"
  RESET="$(tput sgr0)"
else
  RED=; GREEN=; YELLOW=; RESET=
fi


if [[ "$Log" != off ]]; then
   touch "$Log" || bail "Couldn't touch log file [$Log]."

   # Redirect stdout and stderr to a log file.
   if [[ "$SilentMode" -eq 0 ]]; then
      if [[ -n "$GREEN" ]]; then
         exec > >( tee \
            >(sed -r \
            -e 's/\x1B\[[0-9;]*[a-zA-Z]//g' \
            -e 's/\x1B\(B//g' >>"$Log"))
      else
         exec > >(tee "$Log")
      fi
      exec 2>&1
   else
      exec >"$Log"
      exec 2>&1
   fi

   # Setup /tmp/test_sdp.log symlink so it points to the current log.
   rm -f "$LogLink"
   ln -s "$Log" "$LogLink"

   msg "${H1}\\nLog is: $Log\\n"
fi

ThisUser=$(id -n -u)
msg "Started $ThisScript v$Version as $ThisUser@$ThisHost on $(date) as pid $$:\\nInitial Command Line:\\n$CmdLine\\n"

ScriptedTestDataFile="$CfgDir/scripted_tests.cfg"
CfgFile="$CfgDir/test_sdp.$ThisHost.cfg"

[[ -d "$CfgDir" ]] || bail "Missing confg dir [$CfgDir]."

if [[ -r "$CfgFile" ]]; then
   # shellcheck disable=SC1090
   source "$CfgFile" || bail "Failed to load config file [$CfgFile]."
else
   msg "No host config file found [$CfgFile]. Generating default host config file."
   # shellcheck disable=SC2016
   {
      echo 'declare RunHost=${HOSTNAME%%.*}'
      echo 'declare TestTag="test.$RunHost"'
      echo 'declare TestWS="bruno_jam.$TestTag"'
      echo 'declare TestWSRoot="/p4/1/tmp/$TestWS"'
      echo 'export SDP_TEST_HOME=/p4/sdp/test/bsw'
   } > "$CfgFile" || bail "Could not generate host config file [$CfgFile]."

   # shellcheck disable=SC1090
   source "$CfgFile" || bail "Failed to load config file [$CfgFile]."
fi

if [[ "$DoBSWPreflight" -eq 1 ]]; then
   msg "Pre-start check: Verify Battle School Lab Environment is running and replicating OK."

   msg "Taking a siesta for a few minutes to give replication a chance to catch up."
   sleep 180

   TmpFile=$(mktemp)
   p4master_run 1 p4 -ztag -F %ServerID%:%AppliedJournal% servers -J > "$TmpFile" ||\
      bail "Could not check 'p4 servers -J'. Aborting."

   CommitServerID=$(grep -E "(commit|master).1" "$TmpFile" | cut -d ':' -f1)
   CommitJournalCounter=$(grep -E "(commit|master).1" "$TmpFile" | cut -d ':' -f2)

   for grepString in ${CommitServerID}:${CommitJournalCounter} p4d_edge_syd:${CommitJournalCounter} p4d_fs_nyc:${CommitJournalCounter} p4d_ha_bos:${CommitJournalCounter}; do
      if ! grep -q "$grepString" "$TmpFile"; then
         errmsg "Unexpected: ServerID ${grepString%:*} is not on journal $CommitJournalCounter."
      fi
      if grep -q "${grepString}:0" "$TmpFile"; then
         errmsg "Unexpected: ServerID ${grepString%:*} is not replicating beyond journal $CommitJournalCounter yet."
      fi
   done

   if [[ "$ErrorCount" -eq 0 ]]; then
      msg "Verified: BSW Lab servers and replicating beyond sequence 0 for journal $CommitJournalCounter."
   else
      bail "Aboring before running scripted tests due to issues with Lab Environment."
   fi
else
   dbg "Skipping BSW Prelfight checks due to '-sp' or '-cd'."
fi

# Sanity check on values loaded from the config file.
[[ -z "$TestTag" ]] && \
   bail "Environment loaded from $CfgFile is missing variable definition for TestTag."

if [[ $ThisHost == "$RunHost" ]]; then
   msg "Verified: Running on $RunHost."
else
   bail "Not configured to run on host $ThisHost.  Run only on $RunHost, as configured in $CfgFile."
fi

msg "$H1\\nPre-start test preparations."

if [[ "$LoadSDPEnv" -eq 1 && -r /p4/common/bin/p4_vars ]]; then
   msg "Loading SDP Environment file."

   # shellcheck disable=SC1091
   source /p4/common/bin/p4_vars 1 ||\
      bail "Failed to load SDP environment file p4_vars."
else
   if [[ "$LoadSDPEnv" -eq 0 ]]; then
      dbg "Skipping load of SDP Environment due to '-no_env'."
   else
      dbg "Skipping load of SDP Environment due to missing /p4/common/bin/p4_vars."
   fi
fi

msg "Loading and verifying scripted test data from: $ScriptedTestDataFile"
[[ -r "$ScriptedTestDataFile" ]] || bail "Missing command line test data file [$ScriptedTestDataFile]."

# See the file scripted_tests.cfg for details on expected format of entries
# in that file we are about to parse. Short version is that we're expecting
# lines of this form:
#
# <TestGroup>|<TestScript>|<ExitCode>|[<Host>:]<TestLog>|<ExpectedStringRegex>|<Comments>

AllScriptedTestDataOK=1

i=0; Line=0; while read -r entry; do
   Line=$((Line+1))

   # Ignore blank lines and comments (but still increment line counter).
   # shellcheck disable=SC2116
   [[ -z "$(echo "$entry")" ]] && continue
   [[ $entry == "#"* ]] && continue

   ScriptedTestDataOK=1
   TestGroup=${entry%%|*}

   # Shift to next '|' char.
   entry=${entry#*|}

   # TestScript is the first field.
   if [[ "$entry" =~ ^no_script ]]; then
      TestScript="no_script"
      ExpectedExit="U"
   else
      TestScript="$CfgDir/${entry%%|*}"

      # ExpectedExit is the next field.
      ExpectedExit="${entry#*|}"
      ExpectedExit="${ExpectedExit%%|*}"
   fi

   # If user specified a comma-delimted list of test groups with '-g',
   # run only tests in specified groups.
   if [[ -n "$UserTestGroups" ]]; then
      InUserTestGroup=0
      for g in ${UserTestGroups//,/ }; do
         if [[ "${TestGroup^^}" == "${g^^}" ]]; then
            InUserTestGroup=1
         fi
      done
      if [[ "$InUserTestGroup" -eq 1 ]]; then
         dbg "Test Group [$TestGroup] is in user-specified test group list [$UserTestGroups]. Processing it."
      else
         dbg "Test Group [$TestGroup] is NOT in user-specified test group list [$UserTestGroups]. Skipping it."
         # Bypass remainder of the outer for loop if group isn't in users-specified test group list.
         continue
      fi
   fi

   # TestLog is the next field.
   TestLog="${entry#*|}"
   TestLog="${TestLog#*|}"
   TestLog="${TestLog%%|*}"

   # TestLog might look like syd-helix-04:/p4/1/logs/log, indicating to grep the log on
   # host syd-helix-04.
   if [[ "$TestLog" == *:* ]]; then
      TestLogHost="${TestLog%%:*}"
      TestLog="${TestLog##*:}"
   else
      TestLogHost=
   fi

   # ExpectedString is the next field.
   ExpectedString="${entry#*|}"
   ExpectedString="${ExpectedString#*|}"
   ExpectedString="${ExpectedString#*|}"
   ExpectedString="${ExpectedString%%|*}"

   # Comments is right-most field; strip everything to the left.
   Comments="${entry##*|}"

   if [[ -n "$TestLogHost" ]]; then
      dbg "TG:[$TestGroup]: S:[$TestScript] E:[$ExpectedExit] L:[${TestLogHost}:$TestLog] S:[$ExpectedString]\\nComments: $Comments"
   else
      dbg "TG:[$TestGroup]: S:[$TestScript] E:[$ExpectedExit] L:[$TestLog] S:[$ExpectedString]\\nComments: $Comments"
   fi

   # Do sanity tests on the loaded scripted test configuration data.
   if [[ "$ExpectedExit" == U || "$ExpectedExit" == N || "$ExpectedExit" =~ ^[0-9]+$ ]]; then
      dbg "ExpectedExit value [$ExpectedExit] is valid."
   else
      errmsg "Invalid format of expected exit code [$ExpectedExit] on line $Line of $ScriptedTestDataFile; should be numeric or 'U'."
      ScriptedTestDataOK=0
   fi

   if [[ ! "$TestLog" == /* ]]; then
      if [[ "$TestLog" != output ]]; then
         errmsg "Invalid format of TestLog [$TestLog] on line $Line of $ScriptedTestDataFile. An absolute path or the special value 'output' is expected."
         ScriptedTestDataOK=0
      fi
   fi

   if [[ "$TestScript" != no_script ]]; then
      if [[ ! -x "$TestScript" ]]; then
         errmsg "Missing or Non-Executable Test Script file [$TestScript]."
         ScriptedTestDataOK=0
      fi
   fi

   if [[ "$TestLog" == output && "$TestScript" == no_script ]]; then
      errmsg "Invalid use of both 'no_script' and 'output' on line $Line of $ScriptedTestDataFile. If no script is run, there can be no output!"
      ScriptedTestDataOK=0
   fi

   if [[ "$ScriptedTestDataOK" -eq 1 ]]; then
      TestGroupList[i]="$TestGroup"
      TestScriptList[i]="$TestScript"
      ExpectedExitList[i]="$ExpectedExit"
      TestLogList[i]="$TestLog"
      TestLogHostList[i]="$TestLogHost"
      ExpectedStringList[i]="$ExpectedString"
      CommentsList[i]="$Comments"
      i+=1
   else
      TestSkipCount+=1
      AllScriptedTestDataOK=0
   fi
done < "$ScriptedTestDataFile"

if [[ "$AllScriptedTestDataOK" -eq 1 ]]; then
   msg "Verified: All test entries are OK."
else
   if [[ "$RequireValidTests" -eq 1 ]]; then
      bail "The scripted test data file [$ScriptedTestDataFile] contained invalid entries. Use '-f' to bypass bad tests and continue using the good ones."
   else
      msg "\\n$TestSkipCount tests will be skipped due to invalid entries in $ScriptedTestDataFile. Continuing due to '-f'.\\n"
   fi
fi

msg "${H3}\\nExecuting Scripted Tests."

[[ -r "$ScriptedTestDataFile" ]] || bail "Missing scripted test config file [$ScriptedTestDataFile]."

for ((i=0; i<${#TestScriptList[@]}; i++)); do
   TestID=$((i+1))
   TestScript="${TestScriptList[i]}"
   TestGroup="${TestGroupList[i]}"
   ExpectedExit="${ExpectedExitList[i]}"
   ExpectedString="${ExpectedStringList[i]}"
   TestLog="${TestLogList[i]}"
   TestLogHost="${TestLogHostList[i]}"
   Comments="${CommentsList[i]}"
   TestPassed=1

   if [[ $ExpectedExit == U ]]; then
      ExpectedExit=Undefined
   elif [[ $ExpectedExit == N ]]; then
      ExpectedExit=NonZero
   else
      ExpectedExitCode="$ExpectedExit"
   fi

   if [[ "$TestLog" == "output" ]]; then
      ExpectedStringInOutput=1
   else
      ExpectedStringInOutput=0
   fi

   msg "${H2}\\nTest $TestID - Group $TestGroup - $Comments"
   if [[ -n "$TestLogHost" ]]; then
      msg "TG:[$TestGroup] S:[$TestScript] E:[$ExpectedExit] L:[${TestLogHost}:$TestLog] S:[$ExpectedString]"
   else
      msg "TG:[$TestGroup] S:[$TestScript] E:[$ExpectedExit] L:[$TestLog] S:[$ExpectedString]"
   fi

   if [[ "$TestScript" != no_script ]]; then
      if [[ "$NoOp" -eq 0 ]]; then
         "$TestScript" > "$OutputFile" 2>&1
         ActualExitCode=$?
      else
         msg "NO_OP: Would run: $TestScript > $OutputFile 2>&1"
         echo > "$OutputFile"
         ActualExitCode=0
      fi

      msg "\\n== Test Script Output =="
      cat "$OutputFile"

      msg "TEST_EXIT_CODE: Actual $ActualExitCode, Expected $ExpectedExit"

      if [[ "$ExpectedExit" == Undefined ]]; then
         if [[ "$ActualExitCode" -ne 2 ]]; then
            msg "Ignoring TEST_EXIT_CODE due to 'U' value."
         else
            msg "Test exit code 2 indicates test did not run correctly.  Failing test."
            TestPassed=0
         fi
      elif [[ "$ExpectedExit" == NonZero ]]; then
         [[ "$ActualExitCode" -gt 0 ]] || TestPassed=0
      else
         [[ "$ExpectedExitCode" -ne "$ActualExitCode" ]] && TestPassed=0
      fi
   fi

   if [[ -n "$ExpectedString" ]]; then
      # Check for the expected string in the command output or the script log file.
      if [[ "$ExpectedStringInOutput" -eq 1 ]]; then
         if grep -q -E "$ExpectedString" "$OutputFile"; then
            msg "\\nExpected string [$ExpectedString] found in command output."
         else
            msg "\\nExpected string [$ExpectedString] NOT found in command output."
            TestPassed=0
         fi
      else
         if [[ -n "$TestLogHost" ]]; then
            TmpLocalLog=$(mktemp)
            if rsync --copy-links "$TestLogHost:$TestLog" "$TmpLocalLog"; then
               if grep -E -q "$ExpectedString" "$TmpLocalLog"; then
                  if [[ "$DisplayFullLogs" -eq 1 ]]; then
                     msg "\\nExpected string [$ExpectedString] found in this log [${TestLogHost}:$TestLog]:\\n${H4}"
                     cat "$TmpLocalLog"
                     msg "${H4}"
                  else
                     msg "\\nExpected string [$ExpectedString] found in log [${TestLogHost}:$TestLog]."
                  fi
               else
                  if [[ "$DisplayFullLogs" -eq 1 ]]; then
                     msg "\\nExpected string [$ExpectedString] NOT found in this log [${TestLogHost}:$TestLog]:\\n${H4}"
                     cat "$TmpLocalLog"
                     msg "${H4}"
                  else
                     msg "\\nExpected string [$ExpectedString] NOT found in log [${TestLogHost}:$TestLog]."
                  fi
                  TestPassed=0
               fi
            else
               msg "String [$ExpectedString] expected in log [${TestLogHost}:$TestLog], but that log could not be copied locally to $ThisHost."
               TestPassed=0
            fi
            rm -f "$TmpLocalLog"
         else
            if [[ -r "$TestLog" ]]; then
               if grep -E -q "$ExpectedString" "$TestLog"; then
                  if [[ "$DisplayFullLogs" -eq 1 ]]; then
                     msg "\\nExpected string [$ExpectedString] found in this log [$TestLog]:\\n${H4}"
                     cat "$TestLog"
                     msg "${H4}"
                  else
                     msg "\\nExpected string [$ExpectedString] found in log [$TestLog]."
                  fi
               else
                  if [[ "$DisplayFullLogs" -eq 1 ]]; then
                     msg "\\nExpected string [$ExpectedString] NOT found in this log [$TestLog]:\\n${H4}"
                     cat "$TestLog"
                     msg "${H4}"
                  else
                     msg "\\nExpected string [$ExpectedString] NOT found in log [$TestLog]."
                  fi
                  TestPassed=0
               fi
            else
               msg "String [$ExpectedString] expected in log [$TestLog], but that log is missing."
               TestPassed=0
            fi
         fi
      fi
   else
      msg "No expected string defined, skipping check for expected string for this test."
   fi

   if [[ "$TestPassed" -eq 1 ]]; then
      pass
   else
      fail
   fi
done

if [[ "$TestPassCount" -ge 1 && "$TestFailCount" -eq 0 && "$ErrorCount" -eq 0 ]]; then
   msg "${H1}\\nScripted Test Run completed.  Summary:\\nALL $TestCount tests PASSED.\\n"
else
   msg "${H1}\\nScripted Test Run completed.  Summary:
   Total Tests:   $TestCount
   PASS Count:    $TestPassCount
   FAIL Count:    $TestFailCount"

   if [[ "$TestSkipCount" -ne 0 ]]; then
      msg "   SKIP Count:    $TestSkipCount"
   fi

   if [[ "$DisplayFullLogs" -eq 1 ]]; then
      msg "   Full Logs:     YES"
   else
      msg "   Full Logs:     NO"
   fi

   if [[ "$ErrorCount" -eq 0 ]]; then
      msg "\\nThere were no errors in test setup. Test results should be valid."
   else
      errmsg "There were $ErrorCount errors in test setup. Test results may not be valid."
   fi

   msg "\\nScan above output carefully."
fi

# Illustrate using $SECONDS to display runtime of a script.
msg "That took about $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\n"

# See the terminate() function, which is really where this script exits.
exit "$((ErrorCount+TestFailCount+TestSkipCount))"