#!/bin/bash
# Declarations and Environment
export SDP_ENV=/p4/common/bin/p4_vars
export SDP_INSTANCE=${SDP_INSTANCE:-Unset}
export SDP_INSTANCE=${1:-$SDP_INSTANCE}
if [[ $SDP_INSTANCE == Undefined ]]; then
echo "Instance parameter not supplied."
echo "You must supply the Perforce instance as a parameter to this script."
exit 1
fi
declare StatusMessage="OK: All scanned depots verified OK."
declare -i VerifyOnlyRecentChanges=0
declare -i VerifyFailed=
declare -i ExitCode=0
declare VerifyCmd=
declare Log=Unset
declare Version=6.0.0
declare Threads=1
#==============================================================================
# Local Functions
# Micro-functions, one-liners used to avoid external dependencies.
function msg () { if [[ $Log != Unset ]]; then echo -e "$*" >> $Log; else echo -e "$*"; fi; }
function bail () { msg "\nError: ${1:-Unknown Error}"; exit ${2:-1}; }
function cmd ()
{
# echo === Running $* on $(date). >> "$cmd_log"
$* >> "$cmd_log" 2>&1
status=$?
if [ $status -ne 0 ]; then
VerifyFailed=1
fi
echo === $* completed on $(date). >> "$cmd_log"
}
#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
# The default is -h.
#
# $2 - error message (optional). Specify this if usage() is called due to
# user error, in which case the given message displayed first, followed by the
# standard usage message (short or long depending on $1). If displaying an
# errror, usually $1 should be -h so that the longer usage message doesn't
# obsure the error message.
#
# Sample Usage:
# usage
# usage -man
# usage -h "Incorrect command line usage."
#
# This last example generates a usage error message followed by the short
# '-h' usage summary.
#------------------------------------------------------------------------------
function usage
{
declare style=${1:--h}
declare errorMessage=${2:-Unset}
if [[ $errorMessage != Unset ]]; then
echo -e "\n\nUsage Error:\n\n$errorMessage\n\n" >&2
fi
echo "USAGE for p4verify.sh v$Version:
p4verify.sh [<instance>] [-P num_threads]
or
p4verify.sh -h|-man
"
if [[ $style == -man ]]; then
echo -e "DESCRIPTION:
This script performs a 'p4 verify' of all submitted and shelved versioned
files in depots of all types except 'remote' and 'archive' type depots.
If run on a replica, it schedules archive failures for transfer to the
replica.
OPTIONS:
<instance>
Specify the SDP instances. If not specified, the SDP_INSTANCE
environment variable is used instead. If the instance is not
defined by a parameter and SDP_INSTANCE is not defined, p4verify.sh
exists immediately with an error message.
-D Set extreme debugging verbosity.
-P #
Specify the number of depots to verify in parallel.
HELP OPTIONS:
-h Display short help message
-man Display man-style help message
EXAMPLES:
This script is typically called via cron with only the instance
paramter as an argument, e.g.:
p4verify.sh N
LOGGING:
This script generates no output by default. All (stdout and stderr) is
logged to /p4/N/logs/p4verify.log.
The exception is usage errors, which result an error being sent to
stderr followed usage info on stdout, followed by an immediate exit.
If the '-v' flag is used, the contents of the log are displayed to
stdout at the end of processing.
EXIT CODES:
An exit code of 0 indicates no errors were encounted attempting to
perform verifications, AND that all verifications attempted
reported no problems.
A exit status of 1 indicates that verifications could not be
attempted for some reason.
A exit status of 2 indicates that verifications were successfully
performed, but that problems such as BAD or MISSING files
were detected, or else system limits prevented verification.
"
fi
exit 1
}
#==============================================================================
# Command Line Processing
declare -i shiftArgs=0
set +u
while [[ $# -gt 0 ]]; do
case $1 in
(-h) usage -h;;
(-man) usage -man;;
(-D) set -x;; # Debug; use 'set -x' mode.
(-P) Threads=$2; shiftArgs=1;;
(-*) usage -h "Unknown command line option ($1).";;
(*) export SDP_INSTANCE=$1;;
esac
# Shift (modify $#) the appropriate number of times.
shift; while [[ $shiftArgs -gt 0 ]]; do
[[ $# -eq 0 ]] && usage -h "Incorrect number of arguments."
shiftArgs=$shiftArgs-1
shift
done
done
set -u
#==============================================================================
# Command Line Verification
[[ $SDP_INSTANCE == Unset ]] && \
bail "The \$SDP_INSTANCE setting is not defined. It must be defined by doing:\n\n\tsource /p4/common/bin/p4_vars <instance>\n\nor by passing in the instance name as a parameter to this script.\n"
#==============================================================================
# Main Program
source $SDP_ENV $SDP_INSTANCE ||\
bail "Failed to load SDP environment for instance $SDP_INSTANCE."
source $P4CBIN/backup_functions.sh ||\
bail "Failed to load backup_functions.sh."
[[ $Log == Unset ]] && Log=$LOGS/verify_output.log
# This is needed because everything in backup_functions.sh uses LOGFILE including the mail_log_file function.
LOGFILE=$LOGS/p4verify.log
LOG=$LOGS/p4verify
check_vars
set_vars
get_journalnum
# Clean up old logs
rm -f ${LOG}-*
rm -f $Log
rotate_log_file $LOGFILE ".gz"
msg "${0##*/} v$Version Starting verify at $(date +'%a %Y-%m-%d %H:%M:%S %Z')."
P4="$P4BIN -p $P4PORT -u $P4USER"
$P4CBIN/p4login
if [[ $SERVER_TYPE == "p4d_edge" || $SERVER_TYPE == "p4d_edgerep" ]]; then
msg "Exiting because server type is $SERVER_TYPE, and we do not verify those since they should be running in cache mode."
exit 0
fi
msg "If there are errors in this log, contact your local support team."
VerifyFailed=0
depots=() # Clear array
depotdirs=() # Clear array
VerifyCmd="verify --only MISSING -q"
msg "Starting verify of unload depot."
$P4 -s verify -qUt --only MISSING //$($P4 -ztag -F %name%,%type% depots | grep unload | cut -d "," -f 1)/... >> $Log 2>&1 &
thread=$($P4 monitor show | grep -c verify)
echo "Starting verify of shelves." > ${LOG}-shelves.txt
date >> ${LOG}-shelves.txt
for change in $(p4 -ztag -F %change% changes -s shelved); do
echo "Verifying shelf $change" >> ${LOG}-shelves.txt
($P4 ${VerifyCmd}S @=$change >> ${LOG}-shelves.txt 2>&1 >> ${LOG}-shelves.txt ) &
while [ $thread -ge $Threads ]; do
sleep 5
thread=$($P4 monitor show | grep -c verify)
done
thread=$((thread+1)) # add one to the thread count and start a new verify
done
echo "End verify of shelves." >> ${LOG}-shelves.txt
date >> ${LOG}-shelves.txt
IFS=$'\n'
P4=p4
# Build array depots of all of the depot names
for d in $($P4 -ztag -F %name% depots); do
depot_type=$($P4 -ztag -F %Type% depot -o $d)
[[ "$depot_type" == remote || "$depot_type" == achive || "$depot_type" == unload ]] && continue
depots+=( "$d" ) # Append depot to the array
for dir in $($P4 -ztag -F %dir% dirs //$d/*);do
depotdirs+=( "$dir" )
done
done
thread=$($P4 monitor show | grep -c verify)
for dir in "${depotdirs[@]}"; do
depotname=$(echo "$dir" | cut -d "/" -f 3)
dirname=$(echo "$dir" | cut -d "/" -f 4)
cmd_log=${LOG}-${depotname}.log
cd $P4TMP
unset IFS
# echo "Running: $P4 $VerifyCmd $dir/*" >> "$cmd_log"
$P4 $VerifyCmd "$dir/*" >> "$cmd_log" 2>&1
tfile=$(mktemp $depotname.XXXXXXXXX)
$P4 -ztag -F %depotFile% files "$dir/..." > ${tfile}
linecount=$(wc -l ${tfile} | cut -d " " -f 1)
if [ "$linecount" -ge 0 ] && [ "$linecount" -le 1000000 ]; then
split -l500 ${tfile} ${tfile}_
elif [ "$linecount" -ge 1000000 ] && [ "$linecount" -le 10000000 ]; then
split -l1000 ${tfile} ${tfile}_
else
split -l3000 ${tfile} ${tfile}_
fi
rm ${tfile}
for f in $(ls $P4TMP/${tfile}_*); do
# Loop to see if we are over our thread count. If so wait until we drop below it again
while [ $thread -ge $Threads ]; do
sleep 5
thread=$($P4 monitor show -a | grep -v qS | grep -c verify)
done
echo "Verifying files in $f from $dir" >> "$cmd_log"
(cmd $P4 -s -x "$f" $VerifyCmd;rm $f) &
thread=$((thread+1)) # add one to the thread count and start a new verify
done
IFS=$'\n'
done
# now that we have started all of them wait until all of our processes have finished before continuing.
while [ $thread -gt 0 ]; do
sleep 60
thread=$($P4 monitor show -a | grep -v qS | grep -c verify)
done
# now that the processes have finished combine all of the log file together
for d in "${depots[@]}"; do
if [ -f ${LOG}-$d.log ]; then
cat ${LOG}-$d.log >> "$Log"
rm -f ${LOG}-$d.log
fi
done
cat ${LOG}-shelves.txt >> "$Log"
if [[ $VerifyFailed -ne 0 ]]; then
StatusMessage="Error: Verify attempt failed. Review the log [$LOGFILE]."
ExitCode=1
fi
if [[ $ExitCode -eq 0 ]]; then
egrep '(BAD!|MISSING!|p4 help max)' $Log >> $LOGFILE 2>&1
if [[ $? -eq 0 ]]; then
StatusMessage="Warning: Verify errors detected. Review the log [$LOGFILE]."
ExitCode=2
fi
fi
msg "Completed verifications at $(date)."
if [[ $ExitCode -ne 0 ]]; then
mail_log_file "$HOSTNAME $P4SERVER P4Verify Log ($StatusMessage)"
fi
exit $ExitCode