#!/bin/bash
#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE
#------------------------------------------------------------------------------
#==============================================================================
# Declarations and Environment
export SDP_ENV=/p4/common/bin/p4_vars
export SDP_INSTANCE=${SDP_INSTANCE:-Unset}
declare StatusMessage="OK: All scanned depots verified OK."
declare ThisScript=${0##*}
declare ThisUser=
declare Version=5.6.2
declare CmdLine="$0 $*"
declare -i VerifyOnlyRecentChanges=0
declare -i VerifyFailed=
declare -i ShowLog=0
declare -i ExitCode=0
declare -i VerifyArchived=0
declare -i VerifyChunks=0
declare -i VerifyShelved=1
declare -i VerifySpecDepot=1
declare -i VerifySubmitted=1
declare -i VerifyUnload=1
declare -i ScheduleTransfers=1
declare -i ShelvedCLCount=0
declare -i ShelvedCLErrorCount=0
declare -i DepotCount=0
declare -a DepotList
declare DepotListFile=
declare RevRange=
declare RecentChangesToVerify=200
declare VerifyCmd=
declare ChangesCmd=
declare ShelvedChangesToVerify=
declare ShelvedCL=
declare VerifyOnlyOption=
declare VerifyOptions=
declare ChunkFile=
declare ChunkPath=
declare ChunkSize=
declare ChunkScript="/p4/common/bin/depot_verify_chunks.py"
declare Log=Unset
declare TmpFile=
declare TmpLog=
#==============================================================================
# Local Functions
# Micro-functions, one-liners used to avoid external dependencies.
function msg () { if [[ $Log != Unset ]]; then echo -e "$*" >> $Log; else echo -e "$*"; fi; }
function bail () { msg "\\nError: ${1:-Unknown Error}"; exit "${2:-1}"; }
#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
# The default is -h.
#
# $2 - error message (optional). Specify this if usage() is called due to
# user error, in which case the given message displayed first, followed by the
# standard usage message (short or long depending on $1). If displaying an
# error, usually $1 should be -h so that the longer usage message doesn't
# obscure the error message.
#
# Sample Usage:
# usage
# usage -man
# usage -h "Incorrect command line usage."
#
# This last example generates a usage error message followed by the short
# '-h' usage summary.
#------------------------------------------------------------------------------
function usage
{
declare style=${1:--h}
declare errorMessage=${2:-Unset}
if [[ $errorMessage != Unset ]]; then
echo -e "\\n\\nUsage Error:\\n\\n$errorMessage\\n\\n" >&2
fi
echo "USAGE for $ThisScript v$Version:
p4verify.sh [<instance>] [-nu] [-nr] [-ns] [-nS] [-a] [-nt] [-o BAD|MISSING] [-chunks <ChunkSize>] [-recent] [-dlf <depot_list_file>] [-L <log>] [-v] [-D]
or
p4verify.sh -h|-man
"
if [[ $style == -man ]]; then
echo -e "DESCRIPTION:
This script performs a 'p4 verify' of all submitted and shelved versioned
files in depots of all types except 'remote' and 'archive' type depots.
If run on a replica, it schedules archive failures for transfer to the
replica.
OPTIONS:
<instance>
Specify the SDP instance. If not specified, the SDP_INSTANCE
environment variable is used instead. If the instance is not
defined by a parameter and SDP_INSTANCE is not defined, p4verify.sh
exists immediately with an error message.
-nu Specify '-nu' (No Unload) to skip verification of the singleton depot
of type 'unload' (if created). The 'unload' depot is verified
by default.
-nr Specify '-nr' (No Regular) to skip verification of regular submitted
archive files. The '-nr' option is not compatible with '-recent'.
Regular submitted archive files are verified by default.
-ns Specify '-ns' (No Spec Depot) to skip verification of singleton depot
of type 'spec' (if created). The 'spec' depot is verified by default.
-nS Specify '-nS' (No Shelves) to skip verification of shelved archive
files, i.e. to skip the 'p4 verify -qS'.
-a Specify '-a' (Archive Depots) to do verification of depots of type
'archive'. Depots of type 'archive' are not verified by default, as
archive depots are often physically removed from the server's
storage subsystem for long-term cold storage.
-nt Specify the '-nt' option to avoid passing the '-t' flag to 'p4 verify'
on a replica. By default, p4verify.sh detects if it is running on a
replica, and if so automatically applies the '-t' flag to 'p4 verify'.
That causes the replica to attempt to self-heal, as files that fail
verification are scheduled for transfer from the P4TARGET server. This
default behavior results in 'Transfer scheduled' messages in the log,
and MISSING/BAD files are listed as 'info:' rather than 'error:'. There
is no clear indication of whether or which of the scheduled transfers
complete successfully, and so there may be a mix of transient/correctable
and \"real\"/persistent transfer errors for files that are also BAD/MISSING
on the master server. Specify '-nt' to ensure the log contains a list
of files that currently fail a 'p4 verify' without attempting to transfer
them from the master.
-o BAD|MISSING
Specify '-o MISSING' to check only whether expected archive files exist
or not, skipping the checksum calculation of existing files. This results
in dramatically faster, if less comprehensive, verification. This
is particularly well suited when verification is being used to schedule
archive file transfers of missing files on replicas. This translates into
passing the '--only MISSING' option to 'p4 verify'.
Specify '-o BAD' to check only for BAD revisions. This translates into
passing the '--only BAD' option to 'p4 verify'.
This option requires p4d to be 2021.1 or newer. For older p4d versions,
this option is silently ignored.
-chunks <ChunkSize>
Specify the maximum amount of content by size to verify at once. If
this is specified, the depot_verify_chunks.py script is used to
break up depots into chunks of a given size, e.g. 100M or 4G.
The <ChunkSize> parameter must be a size value valid to pass to the
depot_verify_chunks.py script with the '-m' option.
This requires the perforce-p4python3 module to be installed and the
python3 in the PATH must be the correct one that uses the P4 module.
The -recent and -chunks options are mutually exclusive.
-recent
Specify that only recent changelists should be verified.
The \$SDP_RECENT_CHANGES_TO_VERIFY variable defines how many
changelists are considered recent; the default is $RecentChangesToVerify.
If the default is not appropriate for your site, add
\"export SDP_RECENT_CHANGES_TO_VERIFY\" to /p4/common/config/p4_N.vars to
change the default for an instance, or to /p4/common/bin/p4_vars to
change it globally. If \$SDP_RECENT_CHANGES_TO_VERIFY is unset, the
default is $RecentChangesToVerify.
When -recent is used, neither shelves nor files in the unload depot
are verified.
-dlf <depot_list_file>
Specify a file containing a list of depots to process in the desired
order. By default, all depots reported by 'p4 depots', which
effectively results in depots processed in alphabetical order.
This can be useful in time-sensitive situations where the order
of processing can be prioritized, and/or to prevent processing
certain depots.
The format fo the depot list file is straighforward, one line per
depot, without the '//' nor trailling /..., so a list might look
like this sample:
ProjA
ProjB
spec
.swarm
unload
archive
ProjC
Blank lines and lines starting with a '#' are treated as comments and
ignored.
WARNING: This is not intended to be the primary method of verification,
because it would be easy to forget to add new depots to the list file.
If the depot list file is not readable, processing aborts.
-v Verbose. Show output of verify attempts, which is suppressed by default.
Setting SDP_SHOW_LOG=1 in the shell environment has the same effect as -v.
The default behavior of this script is to generate no terminal output,
but instead to write output into a log file -- see LOGGING below. If
'-v' is specified, the generated log is sent to stdout at the end of
processing. This flag is not recommended for routine cron operation or
for large data sets.
The -chunks and -recent options are mutually exclusive.
-L <log>
Specify the log file to use. The default is /p4/N/logs/p4verify.log
Log rotation and old log cleanup logic does not apply to log files
specified with -L. Thus, using -L is not recommended for routine scheduled
operation, e.g. via crontab.
-D Set extreme debugging verbosity.
HELP OPTIONS:
-h Display short help message
-man Display man-style help message
EXAMPLES:
This script is typically called via cron with only the instance
parameter as an argument, e.g.:
p4verify.sh N
LOGGING:
This script generates no output by default. All (stdout and stderr) is
logged to /p4/N/logs/p4verify.log.
The exception is usage errors, which result an error being sent to
stderr followed usage info on stdout, followed by an immediate exit.
If the '-v' flag is used, the contents of the log are displayed to
stdout at the end of processing.
EXIT CODES:
An exit code of 0 indicates no errors were encountered attempting to
perform verifications, AND that all verifications attempted
reported no problems.
A exit status of 1 indicates that verifications could not be
attempted for some reason.
A exit status of 2 indicates that verifications were successfully
performed, but that problems such as BAD or MISSING files
were detected, or else system limits prevented verification.
"
fi
exit 1
}
#------------------------------------------------------------------------------
# Function: get_verify_rev_range ($depot, $recentChanges)
#------------------------------------------------------------------------------
function get_verify_rev_range () {
declare depot=${1:-}
declare recentChangesToVerify=${2:-}
declare nowChange=
[[ -z "$depot" || -z "$recentChangesToVerify" ]] && return 0
nowChange=$("$P4BIN" -ztag -F %change% changes -m 1 "//$depot/...")
thenChange=$("$P4BIN" -ztag -F %change% changes -m "$recentChangesToVerify" "//$depot/..." | tail -1)
[[ -z "$nowChange" ]] && return 1
[[ -z "$thenChange" ]] && return 1
echo "@$thenChange,@$nowChange"
return 0
}
#==============================================================================
# Command Line Processing
declare -i shiftArgs=0
set +u
while [[ $# -gt 0 ]]; do
case "$1" in
(-h) usage -h;;
(-man) usage -man;;
(-ns) VerifySpecDepot=0;;
(-nS) VerifyShelved=0;;
(-nr) VerifySubmitted=0;;
(-nu) VerifyUnload=0;;
(-a) VerifyArchived=1;;
(-nt) ScheduleTransfers=0;;
(-o) VerifyOnlyOption="--only ${2^^}"; shiftArgs=1;;
(-chunks) VerifyChunks=1; ChunkSize="$2"; shiftArgs=1;;
(-recent) VerifyOnlyRecentChanges=1;;
(-dlf) DepotListFile="${2:-}"; shiftArgs=1;;
(-v) ShowLog=1;;
(-L) Log=$2; shiftArgs=1;;
(-D) set -x;; # Debug; use 'set -x' mode.
(-*) usage -h "Unknown command line option ($1).";;
(*) export SDP_INSTANCE=$1;;
esac
# Shift (modify $#) the appropriate number of times.
shift; while [[ $shiftArgs -gt 0 ]]; do
[[ $# -eq 0 ]] && usage -h "Incorrect number of arguments."
shiftArgs=$shiftArgs-1
shift
done
done
set -u
#==============================================================================
# Command Line Verification
[[ $SDP_INSTANCE == Unset ]] && \
usage -h "The \\$SDP_INSTANCE setting is not defined. It must be defined by doing:\\n\\n\\tsource /p4/common/bin/p4_vars <instance>\\n\\nor by passing in the instance name as a parameter to this script.\\n"
[[ "$VerifySubmitted" -eq 0 && "$VerifyOnlyRecentChanges" -eq 1 ]] && \
usage -h "The '-recent' and '-nr' options are mutually exclusive."
if [[ -n "$VerifyOnlyOption" ]]; then
# shellcheck disable=SC2072
if [[ "$P4D_VERSION" > "2021.1" ]]; then
if [[ ! "$VerifyOnlyOption" =~ ^--only\ (MISSING|BAD)$ ]]; then
usage -h "The option specified with '-o' must be BAD or MISSING. Specified value was: $VerifyOnlyOption"
fi
else
VerifyOnlyOption=
fi
fi
[[ "$VerifyOnlyRecentChanges" -eq 1 && "$VerifyChunks" -eq 1 ]] && \
usage -h "The -recent and -chunks options are mutually exclusive."
if [[ -n "$ChunkSize" && ! -x "$ChunkScript" ]]; then
msg "Warning: -chunks ignored because chunking script is not executable by user $ThisUser: $ChunkScript"
VerifyChunks=0
fi
if [[ -n "$DepotListFile" && ! -r "$DepotListFile" ]]; then
usage -h "The file specified with '-dlf <depot_list-file>' does not exist: $DepotListFile"
fi
#==============================================================================
# Main Program
# shellcheck disable=SC1090
source "$SDP_ENV" "$SDP_INSTANCE" ||\
bail "Failed to load SDP environment for instance $SDP_INSTANCE."
# shellcheck disable=SC1090 disable=SC1091
source "$P4CBIN/backup_functions.sh" ||\
bail "Failed to load backup_functions.sh."
#------------------------------------------------------------------------------
# Function: cmd()
# We need to define cmd() after we have sourced backup_functions.sh.
function cmd ()
{
tmpf="${P4TMP}/t_verify.$$.$RANDOM"
msg "$*" >> "$Log"
# Disable SC2048 for $* because we want any spaces to be expanded here.
# shellcheck disable=SC2048
eval "$*" > "$tmpf" 2>&1
status=$?
# Suppress 'no such file' errors from older p4d versions. In newer p4d
# versions, this is a mere warning, which we don't suppress.
grep -v "^error: //$d/... - no such file(s)." "$tmpf" >> "$Log"
rm -f "$tmpf"
return $status
}
# If $SDP_RECENT_CHANGES_TO_VERIFY, use that, else default to the value set for
# $RecentChangesToVerify in this script.
[[ -n "${SDP_RECENT_CHANGES_TO_VERIFY:-}" ]] && \
RecentChangesToVerify="${SDP_RECENT_CHANGES_TO_VERIFY}"
[[ "$Log" == Unset ]] && Log=$LOGS/p4verify.log
# This is needed because everything in backup_functions.sh uses LOGFILE including the mail_log_file function.
LOGFILE=$Log
rotate_log_file "$LOGFILE" ".gz"
ThisUser=$(id -n -u)
msg "Started $ThisScript v$Version as $ThisUser@${HOSTNAME%%.*} on $(date +'%a %Y-%m-%d %H:%M:%S %Z') as:\\n$CmdLine"
P4="$P4BIN -p $P4PORT -u $P4USER"
"$P4CBIN/p4login"
msg "If there are errors in this log, contact support-helix-core@perforce.com"
VerifyFailed=0
if [[ "$VerifyOnlyRecentChanges" -eq 1 ]]; then
msg "Info: Verifying only $RecentChangesToVerify changelists per depot due to -recent.\\n"
fi
RevRange=
# Determine the list of depots to process, either from 'p4 depots' or from a
# specified depot list file.
if [[ -n "$DepotListFile" ]]; then
TmpFile=$(mktemp)
grep -Ev '^\s*$|^\s*\#' "$DepotListFile" > "$TmpFile"
if [[ -s "$TmpFile" ]]; then
while read -r d; do
DepotList[$DepotCount]="$d"
DepotCount+=1
done < "$TmpFile"
else
bail "No depots to process from the depot list file $DepotListFile (trimmed in $TmpFile)."
fi
else
for d in $($P4 -ztag -F %name% depots); do
DepotList[$DepotCount]="$d"
DepotCount+=1
done
fi
msg "Processing ${#DepotList[@]} depots."
for d in "${DepotList[@]}"; do
depotType=$($P4 -ztag -F %Type% depot -o "$d")
case "$depotType" in
(archive)
[[ "$VerifyArchived" -eq 1 ]] || continue
# For archive depots, use -A and -q but not -z.
if [[ -n "$VerifyOnlyOption" ]]; then
VerifyOptions="-q $VerifyOnlyOption -A"
else
VerifyOptions="-q -A"
fi
[[ "${P4REPLICA}" == "FALSE" || "${SHAREDDATA}" == "TRUE" || "$ScheduleTransfers" -eq 0 ]] || VerifyOptions+=" -t"
msg "=== Started verify of archive depot //$d/... at $(date)."
VerifyCmd="$P4 -s verify $VerifyOptions //$d/..."
cmd "$VerifyCmd" || VerifyFailed=1
;;
(remote)
msg "\\nInfo: Skipping verify of remote depot $d.\\n"
continue
;;
(spec)
[[ "$VerifySpecDepot" -eq 1 ]] || continue
# For the spec depot, there are no lazy copies, so no need for -z.
if [[ -n "$VerifyOnlyOption" ]]; then
VerifyOptions="-q $VerifyOnlyOption"
else
VerifyOptions="-q"
fi
[[ "${P4REPLICA}" == "FALSE" || "${SHAREDDATA}" == "TRUE" || "$ScheduleTransfers" -eq 0 ]] || VerifyOptions+=" -t"
msg "=== Started verify of spec depot //$d/... at $(date)."
VerifyCmd="$P4 -s verify $VerifyOptions //$d/..."
cmd "$VerifyCmd" || VerifyFailed=1
;;
(unload)
[[ "$VerifyUnload" -eq 1 ]] || continue
# For the unload depot, there are no lazy copies, so no need for -z.
if [[ -n "$VerifyOnlyOption" ]]; then
VerifyOptions="-q $VerifyOnlyOption -U"
else
VerifyOptions="-q -U"
fi
[[ "${P4REPLICA}" == "FALSE" || "${SHAREDDATA}" == "TRUE" || "$ScheduleTransfers" -eq 0 ]] || VerifyOptions+=" -t"
msg "=== Started verify of unload depot at $(date)."
VerifyCmd="$P4 -s verify $VerifyOptions //$d/..."
cmd "$VerifyCmd" || VerifyFailed=1
;;
(*)
[[ "$depotType" != "local" && "$depotType" != "stream" ]] && \
msg "Warning: Unknown depot type [$depotType]. Attempting normal verify."
if [[ "$VerifyOnlyRecentChanges" -eq 1 ]]; then
RevRange=$(get_verify_rev_range "$d" "$RecentChangesToVerify")
fi
ChunkFile="$P4TMP/depot_chunks.$d.txt"
rm -f "$ChunkFile"
if [[ "$VerifyChunks" -eq 1 ]]; then
msg "Running: $ChunkScript -v ERROR -m $ChunkSize //$d/..."
if ! "$ChunkScript" -v ERROR -m "$ChunkSize" "//$d/..." > "$ChunkFile"; then
msg "Warning: Not chunking //$d/... due to script error."
rm -f "$ChunkFile"
fi
if [[ ! -s "$ChunkFile" ]]; then
msg "Warning: Not chunking //$d/... due to zero-size chunk file."
rm -f "$ChunkFile"
fi
else
rm -f "$ChunkFile"
fi
# For stream/local depots, use -z unless --only is used.
if [[ -n "$VerifyOnlyOption" ]]; then
VerifyOptions="-q $VerifyOnlyOption"
else
VerifyOptions="-qz"
fi
[[ "${P4REPLICA}" == "FALSE" || "${SHAREDDATA}" == "TRUE" || "$ScheduleTransfers" -eq 0 ]] || VerifyOptions+=" -t"
if [[ "$VerifySubmitted" -eq 1 ]]; then
if [[ "$VerifyChunks" -eq 1 && -s "$ChunkFile" ]]; then
msg "=== Started verify of submitted files in //$d/... in $ChunkSize chunks at $(date)."
while read -r ChunkPath; do
VerifyCmd="$P4 -s verify $VerifyOptions \"$ChunkPath\""
cmd "$VerifyCmd" || VerifyFailed=1
done < "$ChunkFile"
else
msg "=== Started verify of submitted files in //$d/... at $(date)."
VerifyCmd="$P4 -s verify $VerifyOptions //$d/...$RevRange"
cmd "$VerifyCmd" || VerifyFailed=1
fi
fi
;;
esac
done
if [[ "$VerifyShelved" -eq 1 ]]; then
msg "=== Started verify of shelved changelists at $(date)."
TmpFile=$(mktemp)
TmpLog=$(mktemp)
ChangesCmd="$P4BIN -ztag -F %change% changes -s shelved"
[[ "$VerifyOnlyRecentChanges" -eq 1 ]] && \
ChangesCmd+=" -m $RecentChangesToVerify"
ShelvedCLCount=0
ShelvedCLErrorCount=0
# For shelves, don't use -z since we verify change-by-change.
if [[ -n "$VerifyOnlyOption" ]]; then
VerifyOptions="-q $VerifyOnlyOption -S"
else
VerifyOptions="-q -S"
fi
[[ "${P4REPLICA}" == "FALSE" || "${SHAREDDATA}" == "TRUE" || "$ScheduleTransfers" -eq 0 ]] || VerifyOptions+=" -t"
if $ChangesCmd > "$TmpFile"; then
ShelvedChangesToVerify=$(wc -l "$TmpFile" | cut -d ' ' -f 1)
msg "Verifying $ShelvedChangesToVerify shelved changelists."
while read -r ShelvedCL; do
ShelvedCLCount+=1
VerifyCmd="$P4 -s verify $VerifyOptions @=$ShelvedCL"
# Capture temp log for each CL verify command. Display logs with error; discard others.
if eval "$VerifyCmd" > "$TmpLog" 2>&1; then
rm -f "$TmpLog"
else
ShelvedCLErrorCount+=1
cat "$TmpLog" >> "$Log"
rm -f "$TmpLog"
fi
done < "$TmpFile"
else
msg "Error: Could not get or write list of shelved changes."
VerifyFailed=1
fi
if [[ "$ShelvedCLCount" -ne 0 ]]; then
if [[ "$ShelvedCLErrorCount" -eq 0 ]]; then
msg "\\nAll $ShelvedCLCount shelved changes verified OK."
else
msg "\\nOf $ShelvedCLCount shelved changes verified, $ShelvedCLErrorCount had errors."
VerifyFailed=1
fi
else
msg "\\nThere were no shelved changelists to verify."
fi
else
msg "Skipping verify of shelved changelists due to '-nS'."
fi
if [[ "$VerifyFailed" -ne 0 ]]; then
StatusMessage="Error: Verify attempt failed. Review the log [$Log]."
ExitCode=1
fi
if [[ $ExitCode -eq 0 ]]; then
# Note: Ignore SC2196 due to 'grep -e' and 'egrep' not having consistent
# results on all platforms.
# shellcheck disable=SC2196
if egrep -m1 '(BAD!|MISSING!|p4 help max)' "$Log" > /dev/null 2>&1; then
StatusMessage="Error: Verify errors detected. Review the log [$Log]."
ExitCode=2
fi
fi
msg "Completed verifications at $(date), taking $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n"
mail_log_file "$HOSTNAME $P4SERVER P4Verify Log ($StatusMessage)"
[[ "$ShowLog" -eq 1 && -s "$Log" ]] && cat "$Log"
exit $ExitCode