global_replica_status.sh #2

#!/bin/bash
#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Helix Management System (HMS), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce_software-hms/view/main/LICENSE
#------------------------------------------------------------------------------

#==============================================================================
# Declarations and Environment

# Allow override of P4U_HOME, which is set only when testing P4U scripts.
export P4U_HOME=${P4U_HOME:-/p4/common/bin}
export P4U_LIB=${P4U_LIB:-/p4/common/lib}
export P4U_ENV=$P4U_LIB/p4u_env.sh
export P4U_LOG=Unset
export VERBOSITY=${VERBOSITY:-3}

# Environment isolation.  For stability and security reasons, prepend
# PATH to include dirs where known-good scripts exist.
# known/tested PATH and, by implication, executables on the PATH.
export PATH=$P4U_HOME:$PATH:~/bin:.
export P4CONFIG=${P4CONFIG:-.p4config}

[[ -r "$P4U_ENV" ]] || {
   echo -e "\\nError: Cannot load environment from: $P4U_ENV\\n\\n"
   exit 1
}

declare BASH_LIBS=$P4U_ENV
BASH_LIBS+=" $P4U_LIB/libcore.sh"
BASH_LIBS+=" $P4U_LIB/libp4u.sh"

for bash_lib in $BASH_LIBS; do
   # shellcheck disable=SC1090
   source "$bash_lib"
done

declare Version=1.2.1
declare HostList=
declare CfgDir="/p4/common/site/config"
declare SDPHostsCfg=$CfgDir/sdp_hosts.cfg
declare EnvFile=$P4U_HOME/p4_vars
declare SDPInstance=${SDP_INSTANCE:-Unset}
declare -i CheckFirewall=0
declare -i SilentMode=0
declare -i EmailAlways=0
declare -i EmailOnFailure=0
declare -i RemoveOldLogs=0

#==============================================================================
# Local Functions

#------------------------------------------------------------------------------
# Function: terminate
function terminate
{
   # Disable signal trapping.
   trap - EXIT SIGINT SIGTERM

   declare -i sendEmail=0
   local mail_sender_opt=

   vvmsg "$THISSCRIPT: EXITCODE: $OverallReturnStatus"

   # Stop logging.
   if [[ "${P4U_LOG}" != off ]]; then
      stoplog

      # Email.
      [[ $EmailAlways -eq 1 ]] && sendEmail=1
      [[ $EmailOnFailure -eq 1 && $OverallReturnStatus -ne 0 ]] && sendEmail=1

      if [[ $sendEmail -eq 1 ]]; then
         emailStatus="OK"
         [[ $OverallReturnStatus -ne 0 ]] && emailStatus="Error"

         mail_sender_opt=$(get_mail_sender_opt)
         run "$SDPMAIL -s \"${HOSTNAME%%\.*} p4_$SDPInstance global_replica_status [$emailStatus]\" $mail_sender_opt $MAILTO < $P4U_LOG"
      fi
   fi

   # Don't litter.
   cleanTrash

   # With the trap removed, exit.
   exit "$OverallReturnStatus"
}

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
#------------------------------------------------------------------------------
function usage
{
   declare style=${1:--h}

   echo "USAGE for $THISSCRIPT v$Version:

$THISSCRIPT [-H <host1>[,<host2>,...]] [-i <tag>] [-f] [-r] [-e|-E] [-L <log>] [-si] [-v<n>] [-n|-N] [-D]

or

$THISSCRIPT [-h|-man|-V]
"
   if [[ $style == -man ]]; then
      echo -e "
DESCRIPTION:
	This script checks the statis of Helix replicas on all
	replica hosts, as defined in the SDP_REPLICA_HOSTS variable set in
	configuration file, $SDPHostsCfg.  The
	underlying Perforce status command is a 'p4 pull -lj' command.

	This script is intended to be called by a cron job the master server.
	It then does ssh calls to the remaining SDP hosts.

	This depends on ssh keys being setup such that the Perforce login
	(as defined by OSUSER in $EnvFile) can ssh without a
	password to all SDP  hosts.  (To simplify failover, the
	backup servers should also be able to ssh to each other
	without a password; security implications should be
	considered here.)

OPTIONS:
 -H <host1>[,<host2>,...]
	Specify a comma-delimited list of hosts to push to.  By default,
	the SDP_REPLICA_HOSTS value defined in the config file
	$SDPHostsCfg determines the list of hosts to
	push to.

 -i <tag>
	Specify the SDP instance tag (e.g '-i 1' for /p4/1 or '-i fgs' for
	/p4/fgs) for the SDP instance to check replcas for.  The default is
	to use the \$SDP_INSTANCE variable if defined, or else '1'.

	The default host config file $SDPHostsCfg is
	ignored in favor of an instance-specific file if one is found named
	$CfgDir/${SDPHostsCfg%.cfg}.<tag>.cfg.  This can
	be used for cases where instances share the master/commit server,
	but have differing hosts on which replicas for each instance are run.

 -f	Do a 'sudo service iptables status' to check firewalls.  Currently
	only supports firewalls that work with that command (including
	RHEL 6/CentOS 6).

	This verifies that sudo access works, and displays information about
	the firewwall.  This option is only suitable for sites where the
	Perforce admin account is allowed sudo access.

 -r
	Specify this option to remove old global_replica_status.*.log files.
	If this option is specified, log files named
	/p4/<n>/logs/global_replica_status.*.log (where '<n>' is the SDP
	instance name) that are older than the number of days
	indicated by the KEEPLOGS setting in $EnvFile are removed.

	The old log removal occurs only upon successful completion.

 -e	Send email to MAILTO value defined in $EnvFile in event
	of failure only.

 -E	Send email to MAILTO value defined in $EnvFile.

 -v<n>	Set verbosity 1-5 (-v1 = quiet, -v5 = highest).

 -L <log>
	Specify the path to a log file, or the special value 'off' to disable
	logging.  By default, all output (stdout and stderr) goes to a log
	file named global_replica_status.<datestame>.log in $LOGS.

	NOTE: This script is self-logging.  That is, output displayed on the screen
	is simultaneously captured in the log file.  Do not run this script with
	redirection operators like '> log' or '2>&1', and do not use 'tee.'

-si	Operate silently.  All output (stdout and stderr) is redirected to the log
	only; no output appears on the terminal.  This cannot be used with
	'-L off'.
      
 -n	No-Op.  Prints commands instead of running them.

 -N	No-Op.  Similar to '-n', but this command does execute the 'ssh' calls to
	get to the remote host, but then does 'p4 sync -n' rather than' 'p4 sync'
	on the remote host.

 -D     Set extreme debugging verbosity.


HELP OPTIONS:
 -h	Display short help message
 -man	Display man-style help message
 -V	Dispay version info for this script and its libraries.

FILES:
	The SDP environment file $EnvFile defines various
	SDP settings, and is used by several SDP scripts.

	The host config file $SDPHostsCfg defines the SDP_HOSTS and other values.

EXAMPLES:
	Recommended crontab usage for SDP Instance 1:
	$P4CBIN/$THISSCRIPT -i 1 -si -r -E < /dev/null > /dev/null 2>&1

	The redirect to /dev/null is to avoid any output that
	would generate a duplicate email from cron.  Output is not lost;
	it is written to a timestampped log file:
	$LOGS/global_replica_status.<timestamp>.log

SEE ALSO:
	See this related script:
	$P4CBIN/sdp_sync.sh
"
   fi

   exit 1
}

#==============================================================================
# Command Line Processing

declare -i shiftArgs=0

set +u
while [[ $# -gt 0 ]]; do
   case $1 in
      (-i) SDPInstance=$2; shiftArgs=1;;
      (-H) HostList=$2; shiftArgs=1;;
      (-f) CheckFirewall=1;;
      (-r) RemoveOldLogs=1;;
      (-e) EmailOnFailure=1;;
      (-E) EmailAlways=1;;
      (-h) usage -h;;
      (-man) usage -man;;
      (-V) show_versions; exit 1;;
      (-v1) export VERBOSITY=1;;
      (-v2) export VERBOSITY=2;;
      (-v3) export VERBOSITY=3;;
      (-v4) export VERBOSITY=4;;
      (-v5) export VERBOSITY=5;;
      (-L) export P4U_LOG=$2; shiftArgs=1;;
      (-si) SilentMode=1;;
      (-n) export NO_OP=1;;
      (-D) set -x;; # Debug; use 'set -x' mode.
      (*) usageError "Unknown arg ($1).";;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usageError "Bad usage."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Command Line Verification

[[ $SilentMode -eq 1 && $P4U_LOG == off ]] && \
   usageError "Cannot use '-si' with '-L off'."

[[ $EmailOnFailure -eq 1 && $P4U_LOG == off ]] && \
   usageError "Cannot use '-e' or '-E' with '-L off'."

[[ $EmailAlways -eq 1 && $EmailOnFailure -eq 1 ]] && \
   usageError "The '-e' and '-E' flags are mutually exclusive."

[[ $SDPInstance == Unset ]] && \
   bail "SDP Environment is not set, and no instance parameter was provided. Aborting."

# shellcheck disable=SC1090
source "$EnvFile" "$SDPInstance" ||\
   bail "Failed to load environment from $EnvFile. Aborting."

# shellcheck disable=SC1090
source "$P4CBIN/backup_functions.sh" ||\
   warnmsg "Failed to load backup_functions.sh; email will not send."

[[ $P4U_LOG == Unset ]] && P4U_LOG="$LOGS/global_replica_status.$(date +'%Y%m%d-%H%M%S').log"

#==============================================================================
# Main Program

declare -i OverallReturnStatus=0

trap terminate EXIT SIGINT SIGTERM

if [[ "${P4U_LOG}" != off ]]; then
   touch "${P4U_LOG}" || bail "Couldn't touch log file [${P4U_LOG}]."

   # Redirect stdout and stderr to a log file.
   if [[ $SilentMode -eq 0 ]]; then
      exec > >(tee "${P4U_LOG}")
      exec 2>&1
   else
      exec >"${P4U_LOG}"
      exec 2>&1
   fi

   initlog
fi

if [[ -r $CfgDir/sdp_hosts.${SDPInstance}.cfg ]]; then
   SDPHostsCfg=$CfgDir/sdp_hosts.${SDPInstance}.cfg
   msg "Using instance-specific host config file $SDPHostsCfg."
fi

# shellcheck disable=SC1090
source "$SDPHostsCfg" ||\
   bail "Failed to load host configuration from $SDPHostsCfg. Aborting."

# Export vars defined in the host config file.
export P4MS_HOST ALL_SDP_HOSTS SDP_REPLICA_HOSTS SDP_REMOTE_HOSTS SDP_SYNC_HOSTS SDP_STATUS_DIRS

msg "$THISSCRIPT v$Version reporting host status at $(date) for these hosts: $SDP_REPLICA_HOSTS."

# If '-H <host1>[,<host2>,...]' was specified as a comma-separated list
# on the command line, overide the SDP_REPLICA_HOSTS value defined in
# the host config file.
[[ -n "$HostList" ]] && SDP_REPLICA_HOSTS="$(echo "$HostList"|tr ',' ' ')"

for host in $SDP_REPLICA_HOSTS; do
   msg "${H}\\nChecking replication status on $host."

   if [[ $NO_OP -eq 0 ]]; then
      ssh -q "$host" "source $EnvFile $SDPInstance; $P4BIN serverid; $P4BIN pull -lj" ||\
         OverallReturnStatus=1
   else
      msg "NO_OP: Would run: ssh -q $host \"source $EnvFile $SDPInstance; $P4BIN serverid; $P4BIN pull -lj\""
   fi
done

if [[ $CheckFirewall -eq 1 ]]; then
   for host in $SDP_REPLICA_HOSTS; do
      msg "${H}\\nChecking sudo access and firewall on $host."

      if [[ $NO_OP -eq 0 ]]; then
         ssh -q -t "$host" "source $EnvFile $SDPInstance; $P4BIN serverid; sudo service iptables status" ||\
            OverallReturnStatus=1
      else
         msg "NO_OP: Would run: ssh -q -t $host \"source $EnvFile $SDPInstance; $P4BIN serverid; $P4BIN pull -lj\""
      fi
   done
fi

if [[ $OverallReturnStatus -eq 0 ]]; then
   msg "${H}\\nAll replicas reported back.\\n";

   if [[ $RemoveOldLogs -eq 1 ]]; then
      if [[ $NO_OP -eq 0 ]]; then
         msg "Cleanup: Removing $LOGS/global_replica_status.*.log files older than $KEEPLOGS days old (if any):"
         find "$LOGS/" -name "global_replica_status.*.log" -mtime +"$KEEPLOGS" -print -exec /bin/rm -f {} \;
      else
         msg "NO_OP: Would remove $LOGS/global_replica_status.*.log files older than $KEEPLOGS days old (if any):"
         find "$LOGS/" -name "global_replica_status.*.log" -mtime +"$KEEPLOGS" -print
      fi
   fi
else
   msg "${H}\\nSome replicas did not report back.  Review the output above carefully."
fi

# Illustrate using $SECONDS to display runtime of a script.
msg "That took $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n"

# See the terminate() function, which is really where this script exits.
exit $OverallReturnStatus