verify_has.sh #1

#!/bin/bash
#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE
#------------------------------------------------------------------------------
 
# verify_has.sh
# Verifies HAS configuration and environment.
 
#==============================================================================
# Declarations and Environment
 
export SDP_INSTANCE="${SDP_INSTANCE:-UnsetSDPInstance}"
 
declare Version=5.24.0
declare ThisScript="${0##*/}"
declare SDPInstallRoot="/p4"
declare SDPCommon="$SDPInstallRoot/common"
declare SDPCommonBin="$SDPCommon/bin"
declare SDPCommonCfg="$SDPCommon/config"
declare SDPEnvFile="$SDPCommonBin/p4_vars"
declare SDPOwner=
declare CCheckCmd=
declare CmdArgs="$*"
declare CmdLine="$0 $CmdArgs"
declare SDPVersionA=
declare SDPVersionB=
declare SDPVersionC=
declare -i ServerOnline=0
declare -i ErrorCount=0
declare -i WarningCount=0
declare -i CheckCount=0
declare -i SilentMode=0
declare -i DoCrontabTest=1
declare -i DoCrontabTestWarn=0
declare -i DoLicenseTest=1
declare -i DoLicenseTestWarn=0
declare -i DoVersionTest=1
declare -i DoVersionTestWarn=0
declare -i DoExcessBinaryTest=1
declare -i DoExcessBinaryTestWarn=0
declare -i DoInitCompareTest=1
declare -i DoInitCompareTestWarn=0
declare -i DoMasterIDTest=1
declare -i DoMasterIDTestWarn=0
declare -i DoOfflineDBTest=1
declare -i DoOfflineDBTestWarn=0
declare -i DoP4ROOTTest=1
declare -i DoP4ROOTTestWarn=0
declare -i DoPasswordChecks=1
declare -i DoPasswordChecksWarn=0
declare -i DoP4TFilesTest=1
declare -i DoP4TFilesTestWarn=0
declare -i ExcessServerBinariesFound=0
declare -i P4DServer=0
declare -i P4BrokerServer=0
declare -i P4ProxyServer=0
declare -i DoConfigurablesCheck=0
declare SkipTestList=
declare LinkTarget=
declare ExpectedTarget=
declare WarnTestList=
declare ThisUser=
declare LicenseInfo=
declare LicenseExpiration=
declare CurrentTime=
declare ExpirationTime=
declare TimeDiff=
declare DaysDiff=
declare LicenseDaysExpirationAlert=21
declare LinkP4ROOT=
declare LinkOfflineDB=
declare H1="=============================================================================="
declare H2="------------------------------------------------------------------------------"
declare BadLog=
declare Log="Unset"
export P4TMP="Unset"
declare TmpFile=
declare TmpFile2=
declare P4DInitTemplate="/p4/common/etc/init.d/p4d_instance_init.template"
declare P4BrokerInitTemplate="/p4/common/etc/init.d/p4broker_instance_init.template"
declare P4ProxyInitTemplate="/p4/common/etc/init.d/p4p_instance_init.template"
 
# The following variables are exported and assigned in set_vars() in backup_functions.sh.
declare P4DInitScript=
declare P4DRef=
 
# shellcheck disable=SC2034
declare P4DSystemdServiceFile=
declare P4BrokerInitScript=
declare P4BrokerRef=
declare P4ProxyInitScript=
declare P4ProxyRef=
# shellcheck disable=SC2034
declare P4BrokerSystemdServiceFile=
# shellcheck disable=SC2034
declare P4ProxySystemdServiceFile=
 
#==============================================================================
# Local Functions
 
# Note: This script does not use SDP library files, as its purpose is to
# verify the integrity of an SDP installation.  Thus, it has its own
# self-contained versions of some functions for which similar versions
# would normally be sourced from files in /p4/common/lib, like libcore.sh.
 
function msg () { echo -e "$*"; }
function errmsg () { msg "\\nError: ${1:-Unknown Error}\\n"; ErrorCount+=1; }
function warnmsg () { msg "\\nWarning: ${1:-Unknown Warning}\\n"; WarningCount+=1; }
function bail () { errmsg "${1:-Unknown Error}"; exit "${2:-1}"; }
 
#------------------------------------------------------------------------------
# This function takes as input an SDP version string, and returns a version
# id of the form YYYY.N.CL, where YYYY is the year, N is an incrementing
# release ID with a given year, and CL is a changelist identifier. The
# YYYY.N together comprise the major version, often shortened to YY.N, e.g.
# r20.1 for the 2020.1 release.
#
# The full SDP Version string looks something like this:
# Rev. SDP/MultiArch/2019.3/26494 (2020/04/23).
#
# This function parses that full string and returns a value like: 2019.3.26494
function get_sdp_version_from_string () {
   local versionString="${1:-}"
   local version=
   version="20${versionString##*/20}"
   version="${version%% *}"
   version="${version/\//.}"
 
   [[ "$version" == "20" || "$version" == "200" ]] && version="Unknown"
   echo "$version"
}
 
#------------------------------------------------------------------------------
# Function: run ($cmd, $desc, $showOutput)
#
# Runs a command, with optional description, showing command line to execute
# and optionally also the output, and capturing and returning the exit code.
#
# Input:
# $1 - Command and arguments to execute. Defaults to 'echo'.
# $2 - Optional message to display describing what the command is doing.
# $3 - Numeric flag to show output; '1' indicates to show output, 0 to
#      suppress it.
#------------------------------------------------------------------------------
function run () {
   local cmd="${1:-echo}"
   local desc="${2:-}"
   local -i showOutput="${3:-1}"
   local -i exitCode=
   local log
 
   log="$(mktemp "$P4TMP/run.XXXXXXXXXXX")"
 
   [[ -n "$desc" ]] && msg "$desc"
   msg "Executing: $cmd"
   $cmd > "$log" 2>&1
   exitCode=$?
 
   if [[ "$showOutput" -eq 1 ]]; then
      echo "EXIT_CODE: $exitCode" >> "$log"
      cat "$log"
   fi
 
   /bin/rm -f "$log"
   return "$exitCode"
}
 
#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
# The default is -h.
#
# $2 - error message (optional).  Specify this if usage() is called due to
# user error, in which case the given message displayed first, followed by the
# standard usage message (short or long depending on $1).  If displaying an
# error, usually $1 should be -h so that the longer usage message doesn't
# obscure the error message.
#
# Sample Usage:
# usage 
# usage -man
# usage -h "Incorrect command line usage."
#
# This last example generates a usage error message followed by the short
# '-h' usage summary.
#------------------------------------------------------------------------------
function usage {
   declare style=${1:--h}
   declare errorMessage=${2:-Unset}
 
   if [[ $errorMessage != Unset ]]; then
      echo -e "\\n\\nUsage Error:\\n\\n$errorMessage\\n\\n" >&2
   fi
 
   echo "USAGE for verify_has.sh v$Version:
 
verify_has.sh [<instance>] [-L <log>|off
 
   or
 
verify_has.sh -h|-man
"
   if [[ $style == -man ]]; then
      echo -e "DESCRIPTION:
 
        This script verifies the current HAS (Helix Authentication Service)
        setup for the specified instance.
 
	This uses the SDP instance bin directory /p4/N/bin to determine
	what server binaries (p4d, p4broker, p4p) are expected to be configured
	on this machine.
 
	Existence of the '*_init' script indicates the given binary is
	expected. For example, for instance 1, if /p4/1/bin/p4d_1_init
	exists, a p4d server is expected to run on this machine.
 
	Checks may be executed or skipped depending on what servers are
	configured.
 
OPTIONS:
 <instance>
	Specify the SDP instances.  If not specified, the SDP_INSTANCE
	environment variable is used instead.  If the instance is not
	defined by a parameter and SDP_INSTANCE is not defined,
	exits immediately with an error message.
 
 -L <log>
	Specify the log file to use.  The default is /p4/N/logs/verify_has.log
	The special value 'off' disables logging to a file.
 
	Note that '-L off' and '-si' are mutually exclusive.
 
 -D	Set extreme debugging verbosity.
 
HELP OPTIONS:
 -h	Display short help message
 -man	Display man-style help message
 
EXAMPLES:
	Example 1: Typical usage:
 
	This script is typically called after SDP update with only the instance
	name or number as an argument, e.g.:
 
	verify_has.sh 1
 
	Example 3: Automation Usage
 
	If used from automation already doing its own logging, use -L off:
 
	verify_has.sh 1 -L off
 
LOGGING:
	This script generates a log file and also displays it to stdout at the
	end of processing.  By default, the log is:
	/p4/N/logs/verify_has.log.
 
	The exception is usage errors, which result an error being sent to
	stderr followed usage info on stdout, followed by an immediate exit.
 
	If the '-si' (silent) flag is used, the log is generated, but its
	contents are not displayed to stdout at the end of processing.
 
EXIT CODES:
	An exit code of 0 indicates no errors were encountered attempting to
	perform verifications, and that all checks verified cleanly.
"
 
   fi
 
   exit 1
}
 
#------------------------------------------------------------------------------
# Function: terminate
function terminate
{
   # Disable signal trapping.
   trap - EXIT SIGINT SIGTERM
 
   [[ "$Log" == "off" ]] || msg "\\nLog is: $Log\\n${H1}\\n"
 
   # With the trap removed, exit.
   exit "$ErrorCount"
}
 
#------------------------------------------------------------------------------
# Function: do_preflight_checks ($instance)
#
# If preflight checks fail, further tests are aborted. Failure of the very
# basic preflight checks is an indication that the SDP structure is in
# need of repair.
#
# Sample Usage:
# do_preflght_checks "$SDP_INSTANCE" ||\
#    bail "Preflight checks failed. Aborting further checks."
#------------------------------------------------------------------------------
function do_preflight_checks () {
 
   local instance="${1:-}"
   local toolsList="awk date file grep head id ls sort tail tee which"
 
   msg "$H2\\nDoing preflight sanity checks."
   msg "Preflight Check: Ensuring these utils are in PATH: $toolsList"
 
   for tool in $toolsList; do
      CheckCount+=1
      [[ -z "$(command -v "$tool")" ]] && \
         errmsg "Tool '$tool' not in PATH."
   done
 
   [[ $ErrorCount -eq 0 ]] || return 1
 
   msg "Verified: Essential tools are in the PATH."
 
   msg "Preflight Check: cd $SDPCommonBin"
 
   CheckCount+=1
   if cd "$SDPCommonBin"; then
      cd "$OLDPWD" || bail "Failed to cd to $OLDPWD. Aborting."
   else
      errmsg "Could not cd to: $SDPCommonBin"
      return 1
   fi
 
   msg "Verified: cd works to: $SDPCommonBin"
 
   msg "Preflight Check: Checking current user owns $SDPCommonBin"
 
   # shellcheck disable=SC2012
   SDPOwner="$(ls -ld "$SDPCommonBin" | awk '{print $3}')"
   ThisUser="$(id -n -u)"
 
   CheckCount+=1
   if [[ "$ThisUser" == "$SDPOwner" ]]; then
      msg "Verified: Current user [$ThisUser] owns $SDPCommonBin"
   else
      errmsg "Current user [$ThisUser] does not own $SDPCommonBin. This most likely means this script is running as the wrong user.  It could also mean the $SDPCommonBin directory is not owned by the correct owner, which should be the OS account under which the p4d process runs."
      return 1
   fi
 
   msg "Preflight Check: Checking /p4 and /p4/<instance> are local dirs."
   if ! check_local_instance_home_dir "$instance"; then
      errmsg "The SDP /p4 and /p4/<instance> dirs are NOT local."
      return 1
   fi
 
   return 0
}
 
#------------------------------------------------------------------------------
# Function: check_file ($file, $errMsg, $warningOnly)
#
# Checks for existence of a file. Returns 0 if it exists, 1 otherwise.
#
# Inputs:
# $1 - $file path to check. Required.
# $2 - Optional error message to display if file is missing.
#      Default: "Missing file [$file]."
# $3 - $warningOnly. If 0, an error is displayed if the file does not exist.
#      If 1, a warning is displayed instead of an error.  Default is 0.
#
# Allows optional custom error message describing the file, to be displayed if
# the file is missing.  Default error message is "Missing file [FILE]."
#------------------------------------------------------------------------------
function check_file () {
   local file=$1
   local errMsg=${2:-Missing file}
   local -i warningOnly=${3:-0}
   CheckCount+=1
   msg "Checking existence of file $file"
   if [[ "$warningOnly" -eq 0 ]]; then
      [[ -f "$file" ]] && return 0
      errmsg "$errMsg: [$file]."
   else
      [[ -f "$file" ]] && return 0
      warnmsg "$errMsg: [$file]."
   fi
   return 1
}
 
#------------------------------------------------------------------------------
# Function: check_file_x ($file, $errMsg, $warningOnly)
#
# Checks existence of a file with executable bit set. Returns 0 if it exists
# and is executable, 1 otherwise.
#
# Inputs:
# $1 - $file path to check. Required.
# $2 - Optional error message to display if file is missing.
#      Default: "Missing not executable [$file]."
# $3 - $warningOnly. If 0, an error is displayed if the file does not exist.
#      If 1, a warning is displayed instead of an error.  Default is 0.
#
# Allows optional custom error message describing the file, to be displayed if
# the file is missing.  Default error message is "File not executable [FILE]."
#------------------------------------------------------------------------------
function check_file_x () {
   local file=$1
   local errMsg=${2:-File not executable}
   local -i warningOnly=${3:-0}
   CheckCount+=1
 
   msg "Checking executable file $file"
   [[ -x "$file" ]] && return 0
 
   if [[ "$warningOnly" -eq 0 ]]; then
      errmsg "$errMsg: [$file]."
   else
      warnmsg "$errMsg: [$file]."
   fi
 
   return 1
}
 
#------------------------------------------------------------------------------
# Function: check_file_dne ($file, $errMsg, $warningOnly)
#
# Confirm that a specific file does not exist, e.g. a semaphore file.  If the
# specified files does not exist, return 0, or 1 if it exists. This is the
# opposite of check_file().
#
# Inputs:
# $1 - $file path to check. Required.
# $2 - Optional error message to display if file exists.
#      Default: "This file should not exist: [$file]."
# $3 - $warningOnly. If 0, an error is displayed if the file does not exist.
#      If 1, a warning is displayed instead of an error.  Default is 0.
#
#
# Allows optional custom error message describing the file, to be displayed if
# the file is found.  Default error message is "This file should not exist: [FILE]."
#------------------------------------------------------------------------------
function check_file_dne () {
   local file=$1
   local errMsg=${2:-This file should not exist}
   local -i warningOnly=${3:-0}
   CheckCount+=1
 
   msg "Confirming this file does not exist: $file"
   [[ ! -f "$file" ]] && return 0
 
   if [[ "$warningOnly" -eq 0 ]]; then
      errmsg "$errMsg: [$file]."
   else
      warnmsg "$errMsg: [$file]."
   fi
 
   return 1
}
 
#------------------------------------------------------------------------------
# Function: check_file_is_shell_script ($file)
#
# Confirm that a specific file exists, and is a regular file (not a symlink),
# and is a shell script rather than a binary.
#
# Inputs:
# $1 - $file path to check. Required.
# $2 - Optional error message to display if file exists.
#      Default: "This file must be a shell script: [$file]."
# $3 - $warningOnly. If 0, an error is displayed if the file does not exist,
#      is a symlink, or is not a shell script.  If 1, a warning is displayed
#      instead of an error.  Default is 0.
#
# Allows optional custom error message describing the file, to be displayed if
# the file is not a shell script.  Default error message is:
# "This file should be a script: [FILE]."
#------------------------------------------------------------------------------
function check_file_is_shell_script () {
   local file=$1
   local errMsg=${2:-This file must be a shell script}
   local -i warningOnly=${3:-0}
   local fileType=
 
   msg "Confirming this file is a script: $file"
   CheckCount+=1
   if [[ -e "$file" ]]; then
      msg "Verified: File [$file] exists."
   else
      if [[ "$warningOnly" -eq 0 ]]; then
         errmsg "$errMsg: [$file]. It does not exist."
      else
         warnmsg "$errMsg: [$file]. It does not exist."
      fi
      return 1
   fi
 
   CheckCount+=1
   if [[ ! -L "$file" ]]; then
      msg "Verified: File [$file] is not a symlink."
   else
      if [[ "$warningOnly" -eq 0 ]]; then
         errmsg "$errMsg: [$file]. It is a symlink."
      else
         warnmsg "$errMsg: [$file]. It is a symlink."
      fi
      return 1
   fi
 
   CheckCount+=1
   fileType="$(file "$file" 2>&1)"
   if [[ "${fileType,,}" == *"shell script"* ]]; then
      msg "Verified: File [$file] is a shell script."
   else
      if [[ "$warningOnly" -eq 0 ]]; then
         errmsg "$errMsg: [$file]. Type is: $fileType"
      else
         warnmsg "$errMsg: [$file]. Type is: $fileType"
      fi
      return 1
   fi
 
   return 0
}
 
#------------------------------------------------------------------------------
# Function: check_configurable ($instance, $configurable, $scope, $expectedVal, $errMsg1, $errMsg2, $warningOnly)
#
# Check that a configurable is set, and optionally check that it is set to
# an expected value.
#
# Inputs:
# $1 - SDP Instance. Required.
# $2 - Configurable name. Required.
# $3 - Configurable scope/ServerID, as per 'p4 help configure'.  The default
#      is "any", meaning what it means with 'p4 configure set', i.e. that the
#      configurable is a global default.  The special value 'ALL' can
#      also be supplied parameter, which is has the special meaning of checking
#      if the configurable is defined for any ServerID, including the 'any'
#      value.  The value returned is that of the first setting encountered.
# $4 - Expected value of configurable. Optional. If defined, an additional check is
#      done, checking the current value against the expected value.  Optionally,
#      the special value UNDEF can be used, which reverses the exit code, such
#      that a happy zero is returned only if the value is not set.
# $5 - Optional error message to display if no value is defined.  See code
#      below for the default message.
# $6 - Optional error message to display if a value is defined but does not
#      match the expected value.  See code below for the default message.
# $7 - $warningOnly. If 0, an error is displayed if the configurable is not
#      defined or does not have the expected value.
#      If 1, a warning is displayed instead of an error.  Default is 0.
#
# Return Codes:
# 1 - Verifications failed.
# 0 - Verifications passed.
# 
# Sample Usage: 
# check_configurable "$SDP_INSTANCE" journalPrefix
#
# check_configurable "$SDP_INSTANCE" journalPrefix any "$CHECKPOINTS/$P4SERVER"
#
# check_configurable "$SDP_INSTANCE" journalPrefix any "$CHECKPOINTS/$P4SERVER" ||\
#   bail "Yikes, journalPrefix is not set, all bets are off. Aborting."
#------------------------------------------------------------------------------
function check_configurable () {
   local instance="$1"
   local configurable="$2"
   local scope="${3:-any}"
   local expectedValue="${4:-NoExpectedValue}"
   local errMsgMissing="${5:-No value defined}"
   local errMsgBadValue="${6:-Value does not match what is expected}"
   local -i warningOnly=${7:-0}
   local detectedScope=
   local value=
 
   # If skipping P4ROOT tests, don't bother with configurable checks, as
   # they require P4ROOT.
   [[ "$DoP4ROOTTest" -eq 1 ]] || return 0
 
   CheckCount+=1
 
   if [[ ! -r "$P4ROOT"/db.config ]]; then
      warnmsg "Skipping check for configurable $configurable; no db.config."
      return 1
   fi
 
   if [[ "$scope" != "ALL" ]]; then
      value=$($P4DBIN -r "$P4ROOT" -cshow | grep "^${scope}: ${configurable} = ")
   else
      value=$($P4DBIN -r "$P4ROOT" -cshow | grep ": ${configurable} = " | head -1)
      detectedScope="$value"
      value=${value##* = }
      detectedScope="${detectedScope%%:*}"
   fi
 
   if [[ "$expectedValue" != "UNDEF" ]]; then
      if [[ -n "$value" ]]; then
         value=${value##* = }
         if [[ "$scope" != "ALL" ]]; then
            msg "Verified: Configurable ${scope}:${configurable} is defined."
         else
            msg "Verified: Configurable ${configurable} is defined at least once."
         fi
      else
         if [[ "$warningOnly" -eq 0 ]]; then
            errmsg "$errMsgMissing for configurable [${scope}:${configurable}]."
         else
            warnmsg "$errMsgMissing for configurable [${scope}:${configurable}]."
         fi
         return 1
      fi
   else
      if [[ -n "$value" ]]; then
         if [[ "$scope" != "ALL" ]]; then
            if [[ "$warningOnly" -eq 0 ]]; then
               errmsg "Configurable ${configurable} should not be set with 'p4 configure set' but has a value for ServerID ${scope} of: ${value}"
            else
               warnmsg "Configurable ${configurable} should not be set with 'p4 configure set' but has a value for ServerID ${scope} of: ${value}"
            fi
            return 1
         else
            if [[ "$warningOnly" -eq 0 ]]; then
               errmsg "Configurable ${configurable} should not be set with 'p4 configure set' but has a value for ServerID ${detectedScope} of: ${value} (and possibly for other ServerIDs)."
            else
               warnmsg "Configurable ${configurable} should not be set with 'p4 configure set' but has a value for ServerID ${detectedScope} of: ${value} (and possibly for other ServerIDs)."
            fi
            return 1
         fi
      else
         if [[ "$scope" != "ALL" ]]; then
            msg "Verified: Configurable ${scope}:${configurable} is undefined (as expected)."
         else
            msg "Verified: Configurable ${configurable} is undefined (as expected) for all ServerID values."
         fi
      fi
   fi
 
   [[ "$expectedValue" == "NoExpectedValue" ]] && return 0
 
   CheckCount+=1
 
   if [[ "$expectedValue" != "UNDEF" ]]; then
      if [[ "$value" == "$expectedValue" ]]; then
         msg "Verified: Configurable ${scope}:${configurable} has expected value [$value]."
      else
         if [[ "$warningOnly" -eq 0 ]]; then
            errmsg "$errMsgBadValue for variable [${scope}:${configurable}]\\n\\tExpected value: [$expectedValue]\\n\\tActual value:   [$value]"
         else
            warnmsg "$errMsgBadValue for variable [${scope}:${configurable}]\\n\\tExpected value: [$expectedValue]\\n\\tActual value:   [$value]"
         fi
         return 1
      fi
   fi
 
   return 0
}
 
#------------------------------------------------------------------------------
# Function: check_env_var ($instance, $var, $expectedval, $msg1, $msg2, $warningOnly)
#
# Check that a shell environment variable is set when sourcing the SDP
# environment. Optionally checks that variables are set to expected values.
#
# Inputs:
# $1 - SDP Instance. Required.
# $2 - Variable name. Required.
# $3 - Expected value of variable. Optional. If defined, an additional check is
#      done, checking the current value against the expected value.
# $4 - Optional error message to display if no value is defined.  See code
#      below for the default message.
# $5 - Optional error message to display if a value is defined but does not match
#      the expected value.  See code below for the default message.
# $6 - $warningOnly. If 0, an error is displayed if the environment variable
#      is not set or does not match the expected value.
#      If 1, a warning is displayed instead of an error.  Default is 0.
# 
# Return Codes:
# 1 - Verifications failed.
# 0 - Verifications passed.
# Sample Usage: 
# check_env_var $SDP_INSTANCE P4JOURNAL "/p4/$SDP_INSTANCE/logs/journal"
#
# check_env_var $SDP_INSTANCE P4JOURNAL "/p4/$SDP_INSTANCE/logs/journal" ||\
#   bail "Yikes, P4JOURNAL is not set, all bets are off. Aborting."
#------------------------------------------------------------------------------
function check_env_var () {
   local instance="$1"
   local var="$2"
   local expectedValue="${3:-NoExpectedValue}"
   local errMsgMissing="${4:-No value defined}"
   local errMsgBadValue="${5:-Value does not match what is expected}"
   local -i warningOnly=${6:-0}
   local value=
   CheckCount+=1
 
   eval unset "${var}"
   # shellcheck disable=SC1090
   source "$SDPEnvFile" "$instance"
 
   set +u
   if [[ -n "$(eval echo \$"${var}")" ]]; then
      msg "Verified: Variable ${var} is defined."
      set -u
   else
      if [[ "$warningOnly" -eq 0 ]]; then
         errmsg "$errMsgMissing for variable [$var]."
      else
         warnmsg "$errMsgMissing for variable [$var]."
      fi
      set -u
      return 1
   fi
 
   [[ "$expectedValue" == "NoExpectedValue" ]] && return 0
 
   CheckCount+=1
   value="$(eval echo \$"${var}")"
 
   if [[ "$value" == "$expectedValue" ]]; then
      msg "Verified: Variable ${var} has expected value [$value]."
   else
      if [[ "$warningOnly" -eq 0 ]]; then
         errmsg "$errMsgBadValue for variable [$var]\\n\\tExpected value: [$expectedValue]\\n\\tActual value:   [$value]"
      else
         warnmsg "$errMsgBadValue for variable [$var]\\n\\tExpected value: [$expectedValue]\\n\\tActual value:   [$value]"
      fi
      return 1
   fi
 
   return 0
}
 
#------------------------------------------------------------------------------
# Function: check_local_instance_home_dir ($instance)
#
# Check that the '/p4' directory and the instance home directory '/p4/N' are
# local directories on the root volume, per SDP structural intent.
#
# Inputs:
# $1 - SDP Instance. Required.
#
# Return Codes:
# 0 - Verifications were able to at least run; ErrorCount is incremented
#     if tests fail.
# 1 - Verifications could not even complete. This is a pre-flight failure.
#
# This increments globals CheckCount and possibly ErrorCount.
#
# Sample Usage: 
# check_local_instance_home_dir "$SDP_INSTANCE" ||\
#    bail "Error checking p4dir and/or local instance home dir."
#------------------------------------------------------------------------------
function check_local_instance_home_dir () {
   local instance="$1"
   local p4Dir="/p4"
   local p4HomeDir="$p4Dir/$instance"
 
   CheckCount+=1
   if [[ "$P4HOME" == "$p4HomeDir" ]]; then
      msg "Verified: P4HOME has expected value: $p4HomeDir"
   else
      errmsg "P4HOME has unexpected value: $p4HomeDir"
   fi
 
   CheckCount+=1
   if [[ -L "$p4HomeDir" ]]; then
      errmsg "This is a symlink; it should be a local directory: $p4HomeDir"
   else
      msg "Verified: This P4HOME path is not a symlink: $p4HomeDir"
   fi
 
   CheckCount+=1
   if cd "$p4Dir"; then
      msg "Verified: cd to $p4Dir OK."
      CheckCount+=1
      if [[ "$(pwd -P)" == "$p4Dir" ]]; then
         msg "Verified: Dir $p4Dir is a local dir."
      else
         errmsg "Dir $p4Dir is NOT a local dir."
      fi
      cd - > /dev/null || bail "Failed to cd to $OLDPWD. Aborting."
 
      CheckCount+=1
      if cd "$p4HomeDir"; then
         msg "Verified: cd to $p4HomeDir OK."
         CheckCount+=1
         if [[ "$(pwd -P)" == "$p4HomeDir" ]]; then
            msg "Verified: P4HOME dir $p4HomeDir is a local dir."
         else
            errmsg "P4HOME dir $p4HomeDir is NOT a local dir."
         fi
         cd - > /dev/null || bail "Failed to cd to $OLDPWD. Aborting."
      else
         errmsg "Could not cd to $p4HomeDir."
         return 1
      fi
   else
      errmsg "Could not cd to $p4Dir."
      return 1
   fi
 
   return 0
}
 
#==============================================================================
# Command Line Processing
 
declare -i shiftArgs=0
 
set +u
while [[ $# -gt 0 ]]; do
   case $1 in
      (-h) usage -h;;
      (-man) usage -man;;
      (-L) Log="$2"; shiftArgs=1;;
      (-D) set -x;; # Debug; use 'set -x' mode.
      (-*) usage -h "Unknown command line option ($1).";;
      (*) export SDP_INSTANCE=$1;;
   esac
 
   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usage -h "Incorrect number of arguments."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u
 
#==============================================================================
# Command Line Verification
 
[[ "$SDP_INSTANCE" == "UnsetSDPInstance" ]] && \
   usage -h "Missing <instance> parameter. The <instance> must be given as a parameter to this script, or else the \$SDP_INSTANCE environment variable defined.  It can be set by doing:\\n\\n\\tsource $SDPEnvFile <instance>\\n\\nor by passing in the instance name as a parameter to this script.\\n"
 
 
#==============================================================================
# Main Program

# Mark
# check if service is installed
# check if service is running
# check if extension is present
# check for 3 extension configs
# check if the groups listed exist
# check if the triggers have logging enabled
# check if there are auth triggers installed

# shellcheck disable=SC1090
source "$SDPEnvFile" "$SDP_INSTANCE" ||\
   bail "Failed to load SDP environment for instance $SDP_INSTANCE."
 
# shellcheck disable=SC1090 disable=SC1091
source "$P4CBIN/backup_functions.sh" ||\
   bail "Failed to load backup_functions.sh."
 
[[ "${OSUSER:-Unset}" == "Unset" ]] &&\
   bail "The critical OSUSER setting is not defined in $SDPEnvFile. Aborting."
 
# If this verify_has.sh script is called by root, change user to OSUSER.
if [[ $(id -u) -eq 0 ]]; then
   exec su - "$OSUSER" -c "$SDPCommonBin/${0##*/} $CmdArgs"
elif [[ $(id -u -n) != "${OSUSER:-UnknownOSUSER}" ]]; then
   bail "${0##*/} can only be run by root or $OSUSER"
fi
 
trap terminate EXIT SIGINT SIGTERM

 
 
# Logs should be defined to /p4/N/logs after sourcing the environment
# file above; default to /tmp for cases of incomplete environment where
# LOGS is not defined.
export LOGS="${LOGS:-/tmp}"
 
[[ "$Log" == "Unset" ]] && Log="${LOGS}/verify_has.log"
 
if [[ "$Log" != "off" ]]; then
   if [[ -f "$Log" ]]; then
      if [[ ! -w "$Log" ]]; then
         BadLog="$Log"
         Log="off"
         bail "Existing log file [$BadLog] is not writable. Aborting."
      fi
      rotate_log_file "$Log" ".gz"
   else
      if [[ ! -d "${LOGS}" ]]; then
         Log="off"
         bail "Logs directory [$LOGS] is not writable. Aborting."
      fi
   fi
 
   if ! touch "$Log"; then
      BadLog="$Log"
      Log="off"
      bail "Couldn't touch log file [$BadLog]. Aborting."
   fi
 
   # Redirect stdout and stderr to a log file.
   if [[ "$SilentMode" -eq 0 ]]; then
      exec > >(tee "$Log")
      exec 2>&1
   else
      exec >"$Log"
      exec 2>&1
   fi
 
   msg "${H1}\\nLog is: $Log"
fi
 
 
msg "${H2}\\nRunning standard checks typically called within SDP scripts."
CheckCount+=1
check_vars
set_vars
 
 if [[ "$P4DServer" -eq 1 ]]; then
    CheckCount+=1
    check_dirs
 fi
 
msg "${H2}\\nChecking *_init scripts in instance bin dir [/p4/$SDP_INSTANCE/bin] to see what servers are configured on this machine."
echo P4DInitScript is $P4DInitScript

 if [[ -e "$P4DInitScript" ]]; then
    msg "A p4d server is here."
    P4DServer=1
    check_file_x "$P4DInitScript"
 
    CheckCount+=1
    if [[ -r "$P4DInitTemplate" ]]; then
      msg "Verified: P4D init script template exists: $P4DInitTemplate"
      if [[ "$DoInitCompareTest" -eq 1 ]]; then
         CheckCount+=1
         TmpFile=$(mktemp)
         TmpFile2=$(mktemp)
         grep -v -e '^#' -e '^[[:space:]]*$' "$P4DInitScript" > "$TmpFile"
         sed -e "s/REPL_SDP_INSTANCE/${SDP_INSTANCE}/g" "$P4DInitTemplate" | grep -v -e '^#' -e '^[[:space:]]*$' > "$TmpFile2"
 
         if diff -q "$TmpFile" "$TmpFile2" > /dev/null; then
            msg "Verified: P4D init script contents are OK."
         else
            if [[ "$DoInitCompareTestWarn" -eq 1 ]]; then
               warnmsg "P4D init script contents are not as expected:\\n== Expected Contents (Trimmed) ==\\n$(cat "$TmpFile2")\\n== Actual Contents (Trimmed) ==\\n$(cat "$TmpFile")\\n"
            else
               errmsg "P4D init script contents are not as expected:\\n== Expected Contents (Trimmed) ==\\n$(cat "$TmpFile2")\\n== Actual Contents (Trimmed) ==\\n$(cat "$TmpFile")\\n"
            fi
         fi
         rm -f "$TmpFile" "$TmpFile2"
      else
         msg "Skipping P4D init script compare per '-skip'."
      fi
   else
      errmsg "P4D init script template does not exist: $P4DInitTemplate"
   fi
 
   CheckCount+=1
   if [[ -e "$P4DRef" ]]; then
      ExpectedTarget="/p4/common/bin/p4d_${SDP_INSTANCE}_bin"
      if [[ -L "$P4DRef" ]]; then
         CheckCount+=1
         LinkTarget=$(readlink "$P4DRef")
 
         if [[ "$LinkTarget" == "$ExpectedTarget" ]]; then
            msg "Verified: Symlink target for $P4DRef is correct ($LinkTarget)."
         else
            errmsg "P4D Instance Symkink target value is wrong:\\Expected: $ExpectedTarget\\nActual: $LinkTarget\\n\\n"
         fi
      elif [[ -f "$P4DRef" ]]; then
         CheckCount+=1
         # For case-insensitive instances, /p4/N/bin/p4d_N is a script rather
         # than a symlink, but still references a target in /p4/common/bin.
         LinkTarget=$(grep ^P4D= "$P4DRef" | cut -d '=' -f 2)
 
         if [[ "$LinkTarget" == "$ExpectedTarget" ]]; then
            msg "Verified: Target for P4D= in $P4DRef is correct ($LinkTarget)."
         else
           errmsg "P4D Instance P4D= target value in $P4DRef is wrong:\\nExpected: $ExpectedTarget\\nActual: $LinkTarget\\n\\nTo fix this error, run these commands:\\n\\tmv -f $P4DRef ${P4DRef}.junk\\n\\techo '#!/bin/bash' > $P4DRef\\n\\techo 'P4D=/p4/common/bin/p4d_${SDP_INSTANCE}_bin' >> $P4DRef\\n\\techo 'exec \$P4D -C1 \"\$@\"' >> $P4DRef\\n\\tchmod +x $P4DRef\\n\\n"
          fi
      else
         errmsg "Element $P4DRef exists but is neither a file or symlink."
      fi
   else
      errmsg "A p4d server is here, but $P4DRef does not exist."
   fi
fi
 

if [[ "$P4DServer" -eq 1 ]]; then 
   msg "${H2}\\nChecking configurables values."
   # check if the auth.sso.allow.passwd configurable is set
   CheckCount+=1
   msg "This next test checks that auth.sso.allow.passwd is set to 1 to tell p4d that users can auth with either SSO or database password, and as such that p4d can handle password changes as well.\\n"
   check_configurable "$SDP_INSTANCE" auth.sso.allow.passwd any 1
fi

if [[ "$P4DServer" -eq 1 && "$DoP4ROOTTest" -eq 1 ]]; then
   msg "${H2}\\nChecking helix-auth service."
   ServiceInfo=$( systemctl list-units --type=service -all | grep helix-auth ) #systemctl --type=service | grep helix-auth)
   CheckCount+=1
   if [[ -n "$ServiceInfo" ]]; then
      if [[ "$ServiceInfo" == *"helix-auth"* ]]; then
          if [[ "$ServiceInfo" == *"running"* ]]; then
              msg "The helix-auth service is installed and running."
          else
              errmsg "The Helix-auth service is installed but it is NOT running."
          fi
      else
      	 errmsg "The Helix-auth service is not installed."
      fi
   fi
fi

if [[ "$P4DServer" -eq 1 && "$DoP4ROOTTest" -eq 1 ]]; then
   msg "${H2}\\nChecking existance of loginhook extension.\\n"
   loginhook=$( p4 -ztag -F "%extension%" extension --list --type extensions )
   CheckCount+=1
   if [[ -n $loginhook ]]; then
      msg "It appears that the Auth::loginhook extension is installed."
      loginhookEnabled=$( p4 -ztag -F "%enabled%" extension --list --type extensions )
      CheckCount+=1
      if [[ "$loginhookEnabled" == *"true" ]]; then
	 msg "It appears that the Auth::loginhook extension is enabled."
	 else
             errmsg "However, the Auth::loginhook extension is DISABLED."
      fi
      CheckCount+=1
      UsersAndGroups=$( p4 -ztag -F "%ExtConfig%" extension --configure Auth::loginhook --name loginhook-a1 -o | grep Those | wc -l )
      if [[ $UsersAndGroups < 5 ]]; then
	 errmsg "It appears that there are no users or groups configured for the extension."
      fi  
    else
        errmsg "The Auth::loginhook extension does not appear to be installed."
    fi
fi

if [[ "$P4DServer" -eq 1 && "$DoP4ROOTTest" -eq 1 ]]; then
   msg "${H2}\\nChecking for authentication triggers.\\n"
   CheckCount+=1
   AuthTriggers=$( p4 triggers -o | grep -v \# | grep auth | wc -l )
   if [[ $AuthTriggers != 0 ]]; then
      warnmsg "There appear to be authentication triggers in place. This may cause undesired behavior in combination with HAS."
   else
      msg "No potentially conflicting authentication triggers found."
   fi
fi



# TODO:
# - Check if logging is enabled to make sure disk doesn't fill up unexpectedly .
 
if [[ "$ErrorCount" -eq 0 && "$WarningCount" -eq 0 ]]; then
   msg "\\n${H1}\\n\\nALL CLEAN: $CheckCount verifications completed OK."
elif [[ "$ErrorCount" -eq 0 ]]; then
   msg "\\n${H1}\\n\\nNO ERRORS: $CheckCount verifications completed, with $WarningCount warnings detected."
else
   msg "\\n${H1}\\n\\nVerifications completed, with $ErrorCount errors and $WarningCount warnings detected in $CheckCount checks."
fi
 
# See the terminate() function, which is really where this script exits.
exit 0