fix_ktext_only.sh #3

#!/bin/bash
set -u

#==============================================================================
# Declarations and Environment

declare ThisScript=${0##*/}
declare Version=1.2.0
declare ThisUser=
declare ThisHost=${HOSTNAME%%.*}
declare P4VerifyLog=
declare Log=
declare FileRev=
declare File=
declare Rev=
declare VerifyCmd=
declare VerifyFixCmd=
declare Instance="${SDP_INSTANCE:-1}"
declare TmpFile=
declare TmpLog=
declare -i ErrorCount=0
declare -i Debug=0
declare -i NoOp=0
declare AllVerifyErrorsCount=
declare -i SkippedOKCount=0
declare -i SkippedNotKTextCount=0
declare -i FixAttemptedCount=0
declare -i FixVerifiedCount=0
declare -i FixFailedCount=0
declare H1="=============================================================================="
declare H2="------------------------------------------------------------------------------"

#==============================================================================
# Local Functions

function msg () { echo -e "$*"; }
function dbg () { [[ "$Debug" -eq 0 ]] || msg "DEBUG: $*"; }
function errmsg () { msg "\\nError: ${1:-Unknown Error}\\n"; ErrorCount+=1; }
function bail () { errmsg "${1:-Unknown Error}"; exit "${2:-1}"; }

#------------------------------------------------------------------------------
# Function: terminate
function terminate
{
   # Disable signal trapping.
   trap - EXIT SIGINT SIGTERM

   dbg "$ThisScript: EXITCODE: $ErrorCount"

   # Stop logging.
   [[ "$Log" == off ]] || msg "Log is: $Log\\n${H1}"

   # With the trap removed, exit.
   exit "$ErrorCount"
}

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
# The default is -h.
#
# $2 - error message (optional).  Specify this if usage() is called due to
# user error, in which case the given message displayed first, followed by the
# standard usage message (short or long depending on $1).  If displaying an
# error, usually $1 should be -h so that the longer usage message doesn't
# obscure the error message.
#
# Sample Usage:
# usage
# usage -h
# usage -man
# usage -h "Incorrect command line usage."
#------------------------------------------------------------------------------
function usage
{
   declare style=${1:--h}
   declare errorMessage=${2:-Unset}

   if [[ "$errorMessage" != Unset ]]; then
      msg"\\n\\nUsage Error:\\n\\n$errorMessage\\n\\n"
   fi

   msg "USAGE for $ThisScript v$Version:

$ThisScript <P4VerifyLog> [-L <log>] [-n] [-d|-D]

or

$ThisScript [-h|-man]
"
   if [[ $style == -man ]]; then
      msg "
DESCRIPTION:
	This script process a log from the p4verify.sh script
	containing errors suspected to be mere kxtext errors.

	The call to p4verify.sh script looks something like this:
	nohup p4verify.sh -paths /p4/common/site/cvsfix/PathsToTarget.txt -dlf /p4/common/site/cvsfix/DepotListFile.txt -nS -L /p4/1/logs/p4verify.cvsfix.fresh_sandbox.\$(date +'%Y-%m-%d-%H%M%S').log -d < /dev/null > /dev/null 2>&1 &

	The p4verify.cvsfix.fresh_sandbox.<timestamp>.log file is then fed to
	this script, which is executed like so:
	nohup ./$ThisScript /p4/1/logs/p4verify.cvsfix.fresh_sandbox.<timestamp>.log < /dev/null > /p4/1/logs/cvsfix/fko.log 2>&1 &

	This scripts then fixes such errors just by recalculating
	the expected checksum with 'p4 verify -v' commands.

	The pattern deemed to be mere ktext errors have this profile:
	* Files fail to verify cleanly on the sandbox commit server,
	even though all files in target paths are known to verify
	cleanly on the production commit server.
	* Files types include the '+k' filetype modifier.
	* The verify error type is 'BAD' (not 'MISSING').

	This script is intended to be called twice. The first run will
	fix errors, and the second run should detect that all files are
	already OK.

OPTIONS:
 <P4VerifyLog>
 	Specify the path to a log file from p4verify.sh that was
	executed on the edge server, and which contains suspected
	CVS import failures.

 -L <log>
	Specify the path to a log file, or the special value 'off' to disable
	logging.  By default, all output (stdout and stderr) goes to:

	${LOGS:-/tmp}/cvsfix/${ThisScript%.sh}.<DateStamp>.log

	The [.NN.RandomNumber] are used only if needed, if this script is
	called more than once per second.

	NOTE: This script is self-logging.  That is, output displayed on the screen
	is simultaneously captured in the log file.  Using redirection operators like
	'>' or '2>&1' are optional, as is using tee.

 -n	Enable No-Op (No Operation).  Commands that that significantly affect data
	are displayed rather than executed.  This is useful for performing dry runs.

 -d     Debug mode, enables additional output.

 -D     Set extreme debugging verbosity.

HELP OPTIONS:
 -h	Display short help message with synopsis of command line usage.
 -man	Display full manual-style help message (this page).

SETUP:
	In an SDP environmet, create the /p4/common/site/cvsfix directory,
	and deploy this script fix_ktext_only.sh in that folder, making
	sure the +x execution bit is set on the file.

EXAMPLES:
	EXAMPLE 1: Targeted

	In this example, our goal is to fix ktext files only in paths
	//abc/dev/... and \"//xyz/import/code/my files/...\".

	This examples targets ktext errors in certain folders, using a
	targeted call to p4verify.sh.

	First, create a file DepotListFile.txt to contain depots to be
	processed, as the OS user that owns /p4/common, like so:

	cd /p4/common/site/cvsfix

	echo abc > /p4/common/site/cvsfix/DepotListFile.txt
	echo xyz >> /p4/common/site/cvsfix/DepotListFile.txt

	Next, create a file PathsToTarget.txt to contain the specific paths
	to be processed within that depot (noting handling of a path with
	spaces), like so: 
	echo //abc/dev/... > /p4/common/site/cvsfix/PathsToTarget.txt
	echo \"//xyz/import/code/my files/...\" >> /p4/common/site/cvsfix/PathsToTarget.txt

	Next, call the targeted p4verify.sh script like so:

	nohup p4verify.sh -paths /p4/common/site/cvsfix/PathsToTarget.txt -dlf /p4/common/site/cvsfix/DepotListFile.txt -nS -L /p4/1/logs/p4verify.cvsfix.fix_ktext.log -d < /dev/null > /dev/null 2>&1 &

	Monitor the log: /p4/1/logs/p4verify.cvsfix.fix_ktext.log

	When that finishes (it may take a long while), call this script as a dry run:
	nohup ./fix_ktext_only.sh -n -d /p4/1/logs/p4verify.cvsfix.fix_ktext.log < /dev/null > \$LOGS/fix_ktext_only.dry_run.log 2>&1 &

	Wait for that to complete, and then review the log: \$LOGS/fix_ktext_only.dry_run.log

	If that looks good, proceed:
	nohup ./fix_ktext_only.sh /p4/1/logs/p4verify.cvsfix.fix_ktext.log < /dev/null > \$LOGS/fix_ktext_only.live_run.log 2>&1 &

	Wait for that to complete, and then review the log: \$LOGS/fix_ktext_only.live_run.log

	EXAMPLE 2: Fix all ktext file issues that we can.

	In this example, we use a call to p4verify.sh that skips shelves but is otherwise
	complete:

	nohup p4verify.sh -nS -L /p4/1/logs/p4verify.cvsfix.fix_ktext.log -d < /dev/null > /dev/null 2>&1 &

	Monitor the log: /p4/1/logs/p4verify.cvsfix.fix_ktext.log

	When that finishes (it may take a long while), call this script as a dry run:
	nohup ./fix_ktext_only.sh -n -d /p4/1/logs/p4verify.cvsfix.fix_ktext.log < /dev/null > \$LOGS/fix_ktext_only.dry_run.log 2>&1 &

	Wait for that to complete, and then review the log: \$LOGS/fix_ktext_only.dry_run.log

	If that looks good, proceed:
	nohup ./fix_ktext_only.sh /p4/1/logs/p4verify.cvsfix.fix_ktext.log < /dev/null > \$LOGS/fix_ktext_only.live_run.log 2>&1 &

	TIP:  Even if your intent is to fix all ktext issues, if you know from prior runs of
	p4verify.sh where the errors are, you can save time by targeting the p4verify.sh script
	to the known locations of verify errors, using the targeting method illustrated in
	EXAMPL1.

	TIP: If you have a prior p4verify.sh generated from the normal weekly SDP crontab
	call to p4verify.sh, that may contain shelved file errors that will confuse this
	script. If you can confirm from the summary that there are no errors, then you can
	use this script and feed it that previously generaeted log that is free of shelved
	file errors. This will save the step of doing a separate run of p4verify.sh.  If
	there are any shelved errors in the weekly log, then do the procedure as noted
	above, calling 'p4verify.sh' with the '-nS' (no shelves) option.
"
   fi

   exit 1
}

#==============================================================================
# Command Line Processing

declare -i shiftArgs=0

set +u
while [[ $# -gt 0 ]]; do
   case $1 in
      (-h) usage -h;;
      (-man) usage -man;;
      (-L) Log="$2"; shiftArgs=1;;
      (-n) NoOp=1;;
      (-d) Debug=1;; # Enable dbg() function.
      (-D) Debug=1; set -x;; # Debug; use bash 'set -x' extreme debug mode.
      (-*) usage -h "Unknown arg ($1).";;
      (*)
         if [[ -n "$P4VerifyLog" ]]; then
            usage -h "Unknown positional parameter ($1)."
         else
            P4VerifyLog="$1"
	 fi
      ;;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usage -h "Incorrect number of arguments."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Command Line Verification

[[ -n "$P4VerifyLog" ]] || usage -h "The <P4VerifyLog> parameter is required."

[[ -r "$P4VerifyLog" ]] ||\
   usage -h "The <P4VerifyLog> specified does not exist: $P4VerifyLog"

dbg "Loading SDP Environment for Instance $Instance."
# shellcheck disable=SC1091
source /p4/common/bin/p4_vars "$Instance" ||\
   bail "Could not do: source /p4/common/bin/p4_vars \"$Instance\""

[[ -n "$Log" ]] ||\
   Log="${LOGS:-/tmp}/cvsfix/${ThisScript%.sh}.$(date +'%Y-%m-%d-%H%M%S').log"

if [[ ! -d "${Log%/*}" ]]; then
   dbg "Making log dir: ${Log%/*}"
   mkdir -p "${Log%/*}" ||\
      bail "Could not do: mkdir -p \"${Log%/*}\""
fi

#==============================================================================
# Main Program

trap terminate EXIT SIGINT SIGTERM

if [[ "$Log" != off ]]; then
   touch "$Log" || bail "Couldn't touch log file [$Log]."

   # Redirect stdout and stderr to a log file.
   exec > >(tee "$Log")
   exec 2>&1

   msg "${H1}\\nLog is: $Log"
fi

ThisUser=$(id -n -u)
msg "Starting $ThisScript v$Version as $ThisUser@$ThisHost on $(date)."

[[ "$NoOp" -eq 0 ]] || msg "\\nRunning in DRY RUN/NO_OP (No Operation) mode."

msg "\\nPhase 0: Preflight Checks."

if grep -qi -E '(master|commit)' "$P4ROOT/server.id"; then
   IsMaster=1
else
   IsMaster=0
fi

if [[ "$IsMaster" -eq 1 ]]; then
   msg "Verified: Running on master server with ServerID $SERVERID."
else
   bail "Run this only on the master host, not host with serverid $SERVERID."
fi

msg "${H2}\\nPreflight checks OK. Proceeding."

msg "Getting list of file revisions to examine."
TmpFile="$(mktemp $P4TMP/tmp.ktext_file_revs.XXXXXX.txt)"
TmpLog="$(mktemp)"
grep '^error: //' "$P4VerifyLog" | sed -E -e 's@^error: @@g' -e 's@ - (add|branch|edit|integrate|move/add) .*$@@g' > "$TmpFile" ||\
   bail "Could not generate temp file: $TmpFile"

AllVerifyErrorsCount="$(wc -l "$TmpFile" | awk '{print $1}')"
msg "Checking $AllVerifyErrorsCount from $TmpFile."

while read -r FileRev; do
   File="${FileRev%#*}"
   Rev="${FileRev##*#}"
   [[ -n "$File" ]] || errmsg "Could not extract File from FileRev: $FileRev"
   [[ -n "$Rev" ]] || errmsg "Could not extract Rev from FileRev: $FileRev"
   [[ -n "$File"  && -n "$Rev" ]] || continue
   dbg "F:[$File] R:[$Rev]"

   VerifyCmd="p4 -s verify -q \"$File#$Rev,#$Rev\""
   msg "Checking, Running: $VerifyCmd"
   if eval $VerifyCmd > "$TmpLog" 2>&1; then
      msg "Skipping file rev (already OK): $FileRev"
      SkippedOKCount+=1
      continue
   else
      cat "$TmpLog"
      if grep -q ' BAD\!' "$TmpLog" && grep -q -- +k "$TmpLog"; then
         VerifyFixCmd=
         [[ "$NoOp" -eq 0 ]] || VerifyFixCmd="echo "
         VerifyFixCmd+="p4 -s verify -v \"$File#$Rev,#$Rev\""
         msg "Fixing, Running: $VerifyFixCmd"
         # Always discard output of 'p4 verify -v'; it doesn't clearly indicate
         # success or failure; it always fails. The next check below is needed
         # to confirm success.
         eval $VerifyFixCmd > /dev/null 2>&1
         FixAttemptedCount+=1

         msg "Re-Checking, Running: $VerifyCmd"
         if eval $VerifyCmd > "$TmpLog" 2>&1; then
            cat "$TmpLog"
            FixVerifiedCount+=1
	 else
            cat "$TmpLog"
            errmsg "Failed to fix FileRev: $FileRev"
            FixFailedCount+=1
	 fi
      else
         msg "Skipping file rev (does not fit profile): $FileRev"
         SkippedNotKTextCount+=1
         continue
      fi
   fi
done < "$TmpFile"

rm -f "$TmpFile" "$TmpLog"

msg "${H2}\\nSummary:
   All Verify Errors Count:        $AllVerifyErrorsCount

   Skipped - Already OK:           $SkippedOKCount
   Skipped - Did not fit profile:  $SkippedNotKTextCount

   Fixes Attempted:                $FixAttemptedCount
   Fixes Verified:                 $FixVerifiedCount
   Fixes Failed:                   $FixFailedCount

   Errors:                         $ErrorCount
"

if [[ "$ErrorCount" -eq 0 ]]; then
   msg "\\nAll Processing Completed Successfully."
else
   errmsg "Processing Completed, but with $ErrorCount errors."
fi

msg "Total time was $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n"

exit "$ErrorCount"