#!/bin/bash
set -u
#==============================================================================
# Declarations and Environment
declare THISSCRIPT=${0##*/}
declare CMDLINE="$0 $*"
declare Version=1.0.3
declare ThisUser=
declare ThisHost=
declare HostList=
declare HostList2=
declare ExcludeHostList=
declare -i Excluded=0
declare -i HostCount=0
declare -i HostProcessedCount=0
declare -i OKCount=0
declare -i ErrorCount=0
declare -A HostStatus
declare CmdAndArgs=
declare SDPEnvFile="/p4/common/bin/p4_vars"
declare SDPInstance="${SDP_INSTANCE:-sn}"
export NO_OP=0
export P4U_LOG=
if ! source "$SDPEnvFile" "$SDPInstance"; then
echo -e "\\nError: Failed to load SDP shell environment for instancce $SDPInstance."
exit 1
fi
if ! source "$P4CLIB/p4u_env.sh"; then
echo -e "\\nError: Failed to load p4u_env.sh."
exit 1
fi
if ! source "$P4CLIB/libcore.sh"; then
echo -e "\\nError: Failed to load libcore.sh."
exit 1
fi
if ! source "$P4CCFG/sdp_hosts.cfg"; then
echo -e "\\nError: Failed to load sdp_hosts.cfg file."
exit 1
fi
export VERBOSITY=4
P4U_LOG="${LOGS}/hrun.$(date +'%Y%m%d-%H%M%S').log"
#==============================================================================
# Local Functions
#------------------------------------------------------------------------------
# Function: terminate
function terminate
{
# Disable signal trapping.
trap - EXIT SIGINT SIGTERM
# Stop logging.
[[ "$P4U_LOG" != off && -f "$P4U_LOG" ]] && stoplog
# With the trap removed, exit.
exit "$ErrorCount"
}
#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
# The default is -h.
#
# $2 - error message (optional). Specify this if usage() is called due to
# user error, in which case the given message displayed first, followed by the
# standard usage message (short or long depending on $1). If displaying an
# error, usually $1 should be -h so that the longer usage message doesn't
# obscure the error message.
#
# Sample Usage:
# usage
# usage -h
# usage -man
# usage -h "Incorrect command line usage."
#------------------------------------------------------------------------------
function usage
{
declare style=${1:--h}
declare errorMessage=${2:-Unset}
if [[ "$errorMessage" != Unset ]]; then
msg "\\n\\nUsage Error:\\n\\n$errorMessage\\n\\n"
fi
msg "USAGE for $THISSCRIPT v$Version:
$THISSCRIPT {-a [-e <host1>[,<host2>...]] | -r [-e <host1>[,<host2>...]] | -H <host1>[,<host2>...]} [-n] [-L <log>] [-si] [-v<n>]
[-n] [-c cmd and args ...]
or
$THISSCRIPT [-h|-man|-V]
"
if [[ $style == -man ]]; then
echo -e "
DESCRIPTION:
This script executes a command on multiple SDP hosts.
OPTIONS:
-c Specify the command with options/arguments/flags to run on remote hosts.
After the '-c' flag occurs on the command line, the remainder of the
command line is interpreted as part of the command, not as options to
this script. Therefore, '-c' must always be specified after all other
options to this script.
If '-c' is not specified, the 'hostname' command is executed. This can be
useful for exercising and verifying SSH access to all machines.
Quoting what comes after '-c' is necessary if the command to run on
others hosts uses '&&' or ';' to separate multi-part commands.
-H <host1>[,<host2>...]
Specify an explict, comma-delimted list of hosts to execute
the command on.
-a Specify the command is to be run on all SDP hosts, per
the ALL_SDP_HOSTS setting in $P4CCFG/sdp_hosts.cfg.
-r Specify the command is to be run on all replica hosts. per
the SDP_REPLICA_HOSTS setting in $P4CCFG/sdp_hosts.cfg.
-e <host1>[,<host2>...]
Specify an explict, comma-delimted list of hosts to exclude from
the list of hosts to command on; use this with '-a' or '-r'.
-v<N> Specify the verbosity level, from most quiet (1) to loudest (6):
-v1 quiet (errors, warnings, vital info)
-v3 normal
-v6 debugging verbosity
-L <log>
Specify the path to a log file, or the special value 'off' to disable
logging. By default, all output (stdout and stderr) goes to
${LOGS}/log
NOTE: This script is self-logging. That is, output displayed on the screen
is simultaneously captured in the log file. Do not run this script with
redirection operators like '> log' or '2>&1', and do not use 'tee.'
-si Operate silently. All output (stdout and stderr) is redirected to the log
only; no output appears on the terminal. This cannot be used with
'-L off'.
-n No-Op. Prints commands instead of running them.
-D Set extreme debugging verbosity.
HELP OPTIONS:
-h Display short help message
-man Display man-style help message
-V Display version info for this script and its libraries.
FILES:
This script uses $P4CCFG/sdp_hosts.cfg to provide a list of all hosts.
EXAMPLES:
EXAMPLE 1: Check the p4d version running on all hosts:
hrun -a -c p4 -ztag -F %serverVersion% info -s
EXAMPLE 2: Check the p4d version running on a list of hosts:
hrun -H perforce1,fmt-p4edge,perforceindev1,perforce1-tx,perforce2-tx,perforce3-tx,perforce1-ot,perforce2-ot -c p4 -ztag -F %serverVersion% info
EXAMPLE 3: Check the p4d version staged on all hosts. Note usage of quoting:
hrun -a -c \"/p4/sdp/helix_binaries/p4d -V | grep ^Rev\"
EXAMPLE 4: Check replica health (replicas only). Note usage of quoting:
hrun -r -c \"p4 pull -lj; p4 pull -ls\"
EXAMPLE 5: Check replica health (replicas only), skipping some. Note usage of
quoting:
hrun -r -e perforce2-ot,perforce2 -c \"p4 pull -lj; p4 pull -ls\"
EXAMPLE 6: Check server.id on all hosts:
hrun -a -c cat /p4/sn/root/server.id
"
fi
exit 1
}
#------------------------------------------------------------------------------
function host_cmd_failed () {
msg "FAILED on ${1:-UnknownHost}"
ErrorCount+=1
}
#==============================================================================
# Command Line Processing
declare -i shiftArgs=0
declare -i SilentMode=0
set +u
while [[ $# -gt 0 ]]; do
case $1 in
(-h) usage -h;;
(-man) usage -man;;
(-V) show_versions; exit 1;;
(-H) HostList="$2"; shiftArgs=1;;
(-e) ExcludeHostList="$2"; shiftArgs=1;;
(-a) HostList="${ALL_SDP_HOSTS:-}";;
(-r) HostList="${SDP_REPLICA_HOSTS:-}";;
(-v1) export VERBOSITIY=1;;
(-v2) export VERBOSITIY=2;;
(-v3) export VERBOSITIY=3;;
(-v4) export VERBOSITIY=4;;
(-v5) export VERBOSITIY=5;;
(-v6) export VERBOSITIY=6;;
(-L) P4U_LOG="$2"; shiftArgs=1;;
(-si) SilentMode=1;;
(-n) export NO_OP=1;;
(-D) set -x;; # Debug; use 'set -x' mode.
(-c) shift; CmdAndArgs="$*"; shiftArgs=$#; shiftArgs=$((shiftArgs-1));;
(*) usageError "Unknown argument/option [$1].";;
esac
# Shift (modify $#) the appropriate number of times.
shift; while [[ $shiftArgs -gt 0 ]]; do
[[ $# -eq 0 ]] && usage -h "Incorrect number of arguments."
shiftArgs=$shiftArgs-1
shift
done
done
set -u
#==============================================================================
# Main Program
trap terminate EXIT SIGINT SIGTERM
ThisUser=$(whoami)
ThisHost=${HOSTNAME%%.*}
[[ -z "$CmdAndArgs" ]] && CmdAndArgs="hostname"
if [[ -n "$HostList" ]]; then
HostList=$(echo "$HostList"| tr ',' ' ')
else
bail "No target hosts specified. Specify '-a', '-r', or '-H <host1>[,<host2>...]'."
fi
if [[ -n "$ExcludeHostList" ]]; then
ExcludeHostList=$(echo "$ExcludeHostList" | tr ',' ' ')
HostList2=
for host in $HostList; do
Excluded=0
for ehost in $ExcludeHostList; do
[[ "$host" == "$ehost" ]] && Excluded=1
done
[[ "$Excluded" -eq 0 ]] && HostList2+=" $host"
done
HostList=$(echo "$HostList2")
fi
if [[ "$P4U_LOG" != off ]]; then
touch "$P4U_LOG" || bail "Couldn't touch log file: $P4U_LOG"
# Redirect stdout and stderr to a log file.
if [[ "$SilentMode" -eq 0 ]]; then
exec > >(tee "$P4U_LOG")
exec 2>&1
else
exec >"$P4U_LOG"
exec 2>&1
fi
initlog
fi
msg "User: $ThisUser\\nLauch Host: $ThisHost\\nCommand and Args:\\n\\t$CmdAndArgs\\n"
msg "Host List: $HostList"
if [[ "$NO_OP" -eq 1 ]]; then
msg "\\nNO_OP (No Operation/preview) mode enabled."
fi
for host in $HostList; do
HostCount+=1
HostStatus[$host]="Not Attempted."
done
for host in $HostList; do
HostProcessedCount+=1
if rrun "$host" "$CmdAndArgs"; then
msg "Command OK on host $host."
OKCount+=1
HostStatus[$host]="OK"
else
host_cmd_failed "$host"
HostStatus[$host]="FAIL"
fi
done
msg "\\nSummary:"
if [[ "$ErrorCount" -eq 0 ]]; then
msg "Command executed successfully on all $HostProcessedCount hosts."
else
msg "Command executed on $HostProcessedCount of $HostCount hosts, OK on $OKCount, $ErrorCount with errors."
fi
msg "\\nStatus on each host:"
for host in $HostList; do
printf " %-16s %-s\n" "$host" "${HostStatus[$host]}"
done
msg "That took $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n"