move_p4_depot.sh #1

#!/bin/bash
#------------------------------------------------------------------------------
set -u

#==============================================================================
# Declarations and Environment

declare ThisScript=${0##*/}
declare Version="2.2.0"
declare ThisUser=
declare ThisHost=${HOSTNAME%%.*}
declare SrcP4Depots=
declare TgtP4Depots=
declare SrcDir=
declare TgtDir=
declare OpMode=Prep
declare Depot=
declare -a Cmd=
declare Log=
declare LogTimestamp=
declare -i NoOp=1
declare -i SilentMode=0
declare -i ErrorCount=0
declare -i Debug=${DEBUG:-0}
declare -i i=0
declare H1="=============================================================================="
declare SDPRoot="${SDP_ROOT:-/p4}"
declare SDPCommon="$SDPRoot/common"
declare SDPCommonBin="$SDPCommon/bin"
declare SDPCommonLib="$SDPCommon/lib"
declare SDPInstance=

#==============================================================================
# SDP Library Functions

if [[ -d "$SDPCommonLib" ]]; then
   # shellcheck disable=SC1090 disable=SC1091
   source "$SDPCommonLib/logging.lib" ||\
      bail "Failed to load bash lib [$SDPCommonLib/logging.lib]. Aborting."
fi

#==============================================================================
# Local Functions
function msg () { echo -e "$*"; }
function dbg () { [[ "$Debug" -eq 0 ]] || msg "DEBUG: $*"; }
function errmsg () { msg "\\nError: ${1:-Unknown Error}\\n"; ErrorCount+=1; }
function bail () { errmsg "${1:-Unknown Error}"; exit "${2:-1}"; }

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
# The default is -h.
#
# $2 - error message (optional).  Specify this if usage() is called due to
# user error, in which case the given message displayed first, followed by the
# standard usage message (short or long depending on $1).  If displaying an
# error, usually $1 should be -h so that the longer usage message doesn't
# obscure the error message.
#
# Sample Usage:
# usage
# usage -h
# usage -man
# usage -h "Incorrect command line usage."
#------------------------------------------------------------------------------
function usage
{
   local style=${1:--h}
   local usageErrorMessage=${2:-}

   [[ -n "$usageErrorMessage" ]] && msg "\\n\\nUsage Error:\\n\\n$usageErrorMessage\\n\\n"

   msg "USAGE for $ThisScript version $Version:

$ThisScript <Depot> <SrcP4Depots> <TgtP4Depots> [-mode {Prep,Cutover}] [-i <instance>] [-L <Log>|off] [-y] [-d|-D]

or

$ThisScript [-h|-man|-V]
"
   if [[ $style == -man ]]; then
      msg "
DESCRIPTION:
	This script is a utility intended to minimize the impact of
	moving a depot on a P4 Server from one depots volume to
	another.

OPTIONS:
 -m <OpMode>
	Specify the operational mode, which can be either 'Prep' or 'Cutover'.
	If this option is ommitted, 'Prep' mode is assumed.

	This script is intended to be run in Prep mode before running in
	Cutover mode.  In Prep mode, it is non-disruptive to the live
	running server. It copies data from the source volume to the
	target volume, but does not disturb data in the source.  It is OK
	if the source is actively being written in Prep mode because Prep
	mode is just getting a head start on the eventual copy in Cutover
	mode. In Prep mode, a having a perfect and complete copy is not
	required.

 -L <log>
	Specify the path to a log file, or the special value 'off' to disable
	logging.  By default, all output (stdout and stderr) goes to a log file
	pointed to by a symlink:

	\$LOGS/${ThisScript%.sh}.<OpMode>.<Depot>[.DryRun].log

	The symlink is for convenience. It refers to the log from the most recent
	run where '-L' was not used.

	Each time this script is run, a new timestamped log is started, and
	the symlink updated to reference the new/latest log during startup.  Log
	files have timestamps that go to the second (or millisecond if needed)
	to differentiate logs.

	NOTE: This script is self-logging.  That is, output displayed on the screen
	is simultaneously captured in the log file. Using redirection operators like
	'> log' or '2>&1' are unnecessary, as is using 'tee' (though using 'tee'
	or redirects is safe and harmless).

 -y	Live mode.  By default, any commands that affect data, such as rsync commands,
	are displayed but not executed.  With the '-y' option, commands that affect
	data are executed.

 -d	Display debug messages.

 -D     Set extreme debugging verbosity using bash 'set -x' mode. Implies -d.

 -si	Silent Mode.  No output is displayed to the terminal (except for usage errors
	on startup). Output is captured in the log.  The '-si' cannot be used with
	'-L off'. This option is intended to be used if calling from crontab to
	prevent crontab from sending emails.

HELP OPTIONS:
 -h	Display short help message.
 -man	Display man-style help message.
 -V	Display script name and version.

EXAMPLES:
	Example 1: Move depot xyz from /hxdepots to /hxdepots-2

	$ThisScript -m Prep xyz /hxdepots/p4/1/depots /hxdepots-2/p4/1/depots
	$ThisScript -m Prep xyz /hxdepots/p4/1/depots /hxdepots-2/p4/1/depots -y
	$ThisScript -m Cutover xyz /hxdepots/p4/1/depots /hxdepots-2/p4/1/depots
	$ThisScript -m Cutover xyz /hxdepots/p4/1/depots /hxdepots-2/p4/1/depots -y
"
   fi

   exit 2
}

#------------------------------------------------------------------------------
# Function: terminate
# shellcheck disable=SC2317
function terminate ()
{
   # Disable signal trapping.
   trap - EXIT SIGINT SIGTERM

   dbg "ExitCode: $ErrorCount"

   [[ "$Log" == "off" ]] || msg "\\nLog is: $Log\\n${H1}"

   # With the trap removed, exit.
   exit "$ErrorCount"
}

#==============================================================================
# Command Line Processing

declare -i shiftArgs=0

set +u
while [[ $# -gt 0 ]]; do
   case $1 in
      (-h) usage -h;;
      (-man) usage -man;;
      (-V|-version|--version) msg "$ThisScript version $Version"; exit 0;;
      (-m)
         if [[ "${1^^}" == PREP ]]; then
            OpMode=Prep
         elif [[ "${1^^}" == CUTOVER ]]; then
            OpMode=Cutover
         else
            usage -h "Invalid OpMode specifed with '-m $1'; valid values are 'Prep' and 'Cutover'."
         fi
      ;;
      (-i) SDPInstance="$2"; shiftArgs=1;;
      (-L) Log="$2"; shiftArgs=1;;
      (-y|--yes) NoOp=0;;
      (-si) SilentMode=1;;
      (-d) Debug=1;;
      (-D) Debug=2; set -x;; # Use bash 'set -x' extreme debug mode.
      (-*) usage -h "Unknown option ($1).";;
      (*)
         if [[ -z "$Depot" ]]; then
            Depot="$1"
         elif [[ -z "$SrcP4Depots" ]]; then
            SrcP4Depots="$1"
         elif [[ -z "$TgtP4Depots" ]]; then
            TgtP4Depots="$1"
         else
            usage -h "Extra parameter [$1] is unknown; all 3 positional parameters already provided: Depot=[$Depot], SrcP4Depots=[$SrcP4Depots], TgtP4Depots=[$TgtP4Depots]."
         fi
      ;;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usage -h "Incorrect number of arguments."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Command Line Verification

[[ "$SilentMode" -eq 1 && "$Log" == off ]] && \
   usage -h "The '-si' option cannot be used with '-Log off'."

[[ -n "$Depot" ]] || usage -h "Missing required parameter '<Depot>'."
[[ -n "$SrcP4Depots" ]] || usage -h "Missing required parameter '<SrcP4Depots>'."
[[ -n "$TgtP4Depots" ]] || usage -h "Missing required parameter '<TgtP4Depots>'."

[[ -n "$SDPInstance" ]] || SDPInstance="${SDP_INSTANCE:-}"
[[ -n "$SDPInstance" ]] ||\
   usage -h "The SDP_INSTANCE must be defined or '-i <instance>' provided."

# shellcheck disable=SC1090 disable=SC1091
source "$SDPCommonBin/p4_vars" "$SDPInstance" ||\
   bail "Could not do: source \"$SDPCommonBin/p4_vars\" \"$SDPInstance\""A

#==============================================================================
# Main Program

trap terminate EXIT SIGINT SIGTERM

# If the user specifies a log file file with '-L', write to the specified file.
# If no log was specified, create a default log file using a timestamp in the
# LOGS dir, and immediately update the symlink for the default log to point to
# it.
if [[ "$Log" != off ]]; then
   # If $Log is not yet defined, set it to a reasonable default.
   if [[ -z "$Log" ]]; then
      LogTimestamp=$(date +'%Y-%m-%d-%H%M%S')
      if [[ "$NoOp" -eq 0 ]]; then
         Log="$LOGS/${ThisScript%.sh}.$OpMode.$Depot.$LogTimestamp.log"
      else
         Log="$LOGS/${ThisScript%.sh}.$OpMode.$Depot.DryRun.$LogTimestamp.log"
      fi
      # Make sure we have a unique log file. Prefer a human-readable timestamp
      # using hours/minutes/seconds. Append milliseconds if needed to ensure
      # a unique filename.
      while [[ -e "$Log" ]]; do
         LogTimestamp=$(date +'%Y-%m-%d-%H%M%S.%3N')
         Log="$LOGS/${ThisScript%.sh}.${LogTimestamp}.$i.log"
         i+=1
      done
   fi
   # The LogLink symlink has no timestamp. It points to the most recent log file.
   if [[ "$NoOp" -eq 0 ]]; then
      LogLink="$LOGS/${ThisScript%.sh}.$OpMode.$Depot.log"
   else
      LogLink="$LOGS/${ThisScript%.sh}.$OpMode.$Depot.DryRun.log"
   fi

   if [[ -e "$LogLink" ]]; then
      if [[ -L "$LogLink" ]]; then
         rm -f "$LogLink"
      else
         # If the name that should be a symlink is not a symlink, move it aside before
         # creating the symlink.
         OldLogTimestamp=$(get_old_log_timestamp "$LogLink")
         mv -f "$LogLink" "${LogLink%.log}.${OldLogTimestamp}.log" ||\
            bail "Could not move old log file aside; tried: mv -f \"$LogLink\" \"${LogLink%.log}.${OldLogTimestamp}.log\""
      fi
   fi

   touch "$Log" || bail "Couldn't touch new log file [$Log]."

   # Use a subshell so the 'cd' doesn't persist.
   ( cd "$LOGS"; ln -s "${Log##*/}" "${LogLink##*/}"; ) ||\
       bail "Couldn't initialize log symlink; tried: ln -s \"$Log\" \"$LogLink\""

   # Redirect stdout and stderr to a log file.
   if [[ "$SilentMode" -eq 0 ]]; then
      exec > >(tee "$Log")
      exec 2>&1
   else
      exec >"$Log"
      exec 2>&1
   fi

   msg "${H1}\\nLog is: $Log\\n"
fi

ThisUser=$(id -n -u)
msg "Started ${0##*/} version $Version as $ThisUser@$ThisHost on $(date)."

msg "Preflight checks."

SrcDir="$SrcP4Depots/$Depot"
TgtDir="$TgtP4Depots/$Depot"

[[ -d "$SrcP4Depots" ]] || errmsg "Missing SrcP4Depots: $SrcP4Depots"

[[ -d "$TgtP4Depots" ]] || errmsg "Missing TgtP4Depots: $TgtP4Depots"

[[ -d "$SrcDir" ]] || errmsg "Missing SrcDir: $SrcDir"

if [[ ! -d "$TgtDir" ]]; then
   msg "Running: mkdir -p $TgtDir"

   if mkdir -p "$TgtDir"; then
      msg "Mkdir OK."
   else
      errmsg "Mkdir failed."
   fi
fi

if [[ "$ErrorCount" -eq 0 ]]; then
   msg "\\nPreflight checks passed. Moving on."
else
   bail "Aborting early due to failed preflight checks."
fi

if [[ "$OpMode" == Cutover ]]; then
   msg "Disk space checks BEFORE:\\ndf -h $SrcP4Depots $TgtP4Depots\\n$(df -h "$SrcP4Depots" "$TgtP4Depots")\\n"
fi

#------------------------------------------------------------------------------
# Do the rsync from source to target.
Cmd=(); [[ "$NoOp" -eq 0 ]] || Cmd+=("echo")
Cmd+=('rsync' '-a' "$SrcDir/" "$TgtDir")

msg "Running: ${Cmd[*]}"
if [[ "$OpMode" == Prep ]]; then
   if "${Cmd[@]}"; then
      msg "\\nRsync was OK. Prep is complete."
   else
      errmsg "Rsync reported erros. Preparation failed."
   fi
else
   if "${Cmd[@]}"; then
      msg "\\nRsync was OK. Moving original source."
   else
      errmsg "Rsync reported erros. Cutover aborted."
   fi

   Cmd=(); [[ "$NoOp" -eq 0 ]] || Cmd+=("echo")
   Cmd+=('mv' "$SrcDir" "${SrcDir}.JUNK")
   msg "Running: ${Cmd[*]}"

   if "${Cmd[@]}"; then
      msg "\\nMove OK, doing symlink."

      Cmd=(); [[ "$NoOp" -eq 0 ]] || Cmd+=("echo")
      Cmd+=('ln' '-s' "$TgtDir" "$SrcDir")
      msg "Running: ${Cmd[*]}"
      if "${Cmd[@]}"; then
         msg "Symlink OK. Doing remove."

         Cmd=(); [[ "$NoOp" -eq 0 ]] || Cmd+=("echo")
         Cmd+=('rm' '-rf' "${SrcDir}.JUNK")
         msg "Running: ${Cmd[*]}"
         if "${Cmd[@]}"; then
            msg "Removal OK."
         else
            errmsg "Removal did not go well. Cutover is already done, but addtional cleanup may be needded."
         fi
      else
         errmsg "Symlink did not go well. Cutover is incompete. Manual intervention is required."
      fi
   else
      errmsg "The local move did not go well. Cutover is incomplete. Manual intervention is required."
   fi
fi

msg "\\nThat took $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n"

if [[ "$ErrorCount" -eq 0 ]]; then
   msg "\\nAll processing completed successfully."
else
   errmsg "Done, but with with errors.  See above."
fi

if [[ "$OpMode" == Cutover ]]; then
   msg "Disk space checks AFTER:\\ndf -h $SrcP4Depots $TgtP4Depots\\n$(df -h "$SrcP4Depots" "$TgtP4Depots")\\n"
fi

exit "$ErrorCount"