#!/bin/bash #============================================================================== # Copyright and license info is available in the LICENSE file included with # the Server Deployment Package (SDP), and also available online: # https://workshop.perforce.com/view/p4-sdp/main/LICENSE #------------------------------------------------------------------------------ set -u #------------------------------------------------------------------------------ # Version ID Block. Relies on +k filetype modifier. # VersionID='$Id: //p4-sdp/r26.1.0.BETA/Server/Unix/p4/common/bin/backup_functions.sh#1 $ $Change: 33444 $' #============================================================================== # Global Variables. export P4DInitScript= export P4DRef= export P4DSystemdServiceFile= export P4BrokerInitScript= export P4BrokerRef= export P4BrokerSystemdServiceFile= export P4ProxyInitScript= export P4ProxyRef= export P4ProxySystemdServiceFile= export EDGESERVER= export STANDBYSERVER= export CHECKPOINT= declare -i IsP4DCaseSensitive= declare -i CreateParallelCheckpoint= declare -i CreateMultifileParallelCheckpoint= declare -i LoadParallelCheckpoint= declare -i CheckpointDumpSkipped=0 declare -i UseGrepM= #============================================================================== # Common functions used in various SDP scripts. #------------------------------------------------------------------------------ # Verify key variables in the shell environment exist, or else abort. # # If checks in this function fail, this function an 'echo' and 'exit 1' # rather than calling 'log' or 'die', as this function is generally called # early in processing, before the log is initialized. #------------------------------------------------------------------------------ function check_vars () { local CheckVarsPreflightOK=1 CommonVars="SDP_INSTANCE P4HOME P4PORT P4ROOT OFFLINE_DB P4JOURNAL P4BIN P4DBIN P4TICKETS P4TRUST KEEPCKPS KEEPJNLS KEEPLOGS CHECKPOINTS LOGS OSUSER" InstanceVars="P4MASTER_ID P4MASTERPORT" # First, check vars that should be set in /p4/common/bin/p4_vars. for var in $CommonVars; do # Detect unset variables, using ':-' to avoid 'unbound variable' errors. # shellcheck disable=SC1083 if [[ -z "$(eval echo \${"$var":-})" ]]; then echo "Error: Required variable \$$var is NOT set. It should be set in /p4/common/bin/p4_vars." CheckVarsPreflightOK=0 fi done # Next, check vars that should be set in /p4/common/config/p4_N.instance. # For some variables, provide additional details that help help users correct # the problem. for var in $InstanceVars; do # shellcheck disable=SC1083 if [[ -z "$(eval echo \${"$var":-})" ]]; then echo "Error: Required variable \$$var is NOT set. It should be set in /p4/common/config/p4_N.vars, where N is the SDP instance name." if [[ "$var" == "P4MASTER_ID" ]]; then echo "The value for P4MASTER_ID should be the name of the ServerID of the master server." fi CheckVarsPreflightOK=0 fi done if [[ "$CheckVarsPreflightOK" -eq 0 ]]; then echo "Use p4master_run or source p4_vars before calling this script." echo "Aborting due to errors in shell environment preflight checks." exit 1 fi } #------------------------------------------------------------------------------ # Function: copy_jd_table ($TableName, $RootDir) # # Copies the specified table to a temp dir for dumping to avoid locks taken by # p4d -jd if done against P4ROOT # # The caller must ensure the specified table exists in the specified root dir. # # Input: # $1 - TableName (required) # $2 - RootDir (required, root or offline_db) # # Exports JDTmpDir #------------------------------------------------------------------------------ function copy_jd_table () { local TableName=${1:-Unset} local RootDir=${2:-Unset} JDTmpDir=$(mktemp -d 2>/dev/null) if [[ ! -d "$JDTmpDir" ]]; then JDTmpDir=$(mktemp -d -p "$P4TMP" -t 'tmp_jdtmpdir.XXXXXXXX') fi if [[ ! -d "$JDTmpDir" ]]; then echo -e "\\nError: Could not initialize JDTmpDir [$JDTmpDir]\\n" exit 1 fi export JDTmpDir cp "$RootDir/$TableName" "$JDTmpDir/." || die "Failed to copy $RootDir/$TableName to $JDTmpDir/." } #------------------------------------------------------------------------------ # Function: remove_jd_tables () # # Cleanup the JDTmpDir, with extra precautions to avoid doing an 'rm -rf' on the # wrong path. #------------------------------------------------------------------------------ function remove_jd_tables () { if [[ -n "${JDTmpDir:-}" && -d "$JDTmpDir" ]]; then if [[ "$JDTmpDir" =~ ^/tmp/tmp. || "$JDTmpDir" == "$P4TMP/tmp_jdtmpdir."* ]]; then rm -rf "${JDTmpDir:-/tmp/does_not_exist}/" fi fi } #------------------------------------------------------------------------------ # is_edge ($ServerID, $RootDir) # # Determine if a given ServerID is an edge server or not, checking a given # database root dir (e.g. $P4ROOT or $OFFLINE_DB). # # Input: # $1 - ServerID (required) # $2 - RootDir (optional, defaults to $P4ROOT) # # Output YES if an edge server, NO otherwise. # #------------------------------------------------------------------------------ function is_edge () { local ServerID=${1:-Unset} local RootDir=${2:-$P4ROOT} local ServicesData= local EdgeCheck= local grepCmd= if [[ -r "$RootDir/db.server" ]]; then copy_jd_table "db.server" "$RootDir" else echo NO return fi # Prepare grep to be case sensitive or insensitive based on SDP instance # configuration. IsP4DCaseSensitive is set in set_vars(). # Sensitive data can have case-varied server specs, but exactly one will # match grep without the -i. Insensitive data can have only one server # spec. grepCmd="$GREP" [[ "${IsP4DCaseSensitive:-0}" -eq 0 ]] && grepCmd+=" -i" [[ "${UseGrepM:-0}" -eq 1 ]] && grepCmd+=" -m 1" # Extract a slice of db.server referencing the given ServerID, and then grab # the field containing Services data. if [[ "${UseGrepM:-0}" -eq 1 ]]; then ServicesData=$("$P4DBIN" -r "$JDTmpDir" -v track=-1 -J off -L /dev/null -k db.server -jd - 2>&1 |\ $grepCmd "@db.server@ @${ServerID}@" | "$CUT" -d '@' -f 13) else ServicesData=$("$P4DBIN" -r "$JDTmpDir" -v track=-1 -J off -L /dev/null -k db.server -jd - 2>&1 |\ $grepCmd "@db.server@ @${ServerID}@" | head -1 | "$CUT" -d '@' -f 13) fi remove_jd_tables # Do a bitwise operation to determine if the ServicesData value indicates # this is an edge server. if [[ -n "$ServicesData" ]]; then EdgeCheck=$((ServicesData & 4096)) if [[ "$EdgeCheck" -gt 0 ]]; then echo YES else echo NO fi else echo NO fi } #------------------------------------------------------------------------------ # is_replica ($ServerID, $RootDir) # # Determine if a given ServerID is a replica server or not, checking a given # database root dir (e.g. $P4ROOT or $OFFLINE_DB). # # Input: # $1 - ServerID (required) # $2 - RootDir (optional, defaults to $P4ROOT) # # Output YES if an standby server, NO otherwise. # #------------------------------------------------------------------------------ is_replica () { local ServerID="${1:-Unset}" local RootDir="${2:-$P4ROOT}" local ServicesData= local grepCmd= if [[ -r "$RootDir/db.server" ]]; then copy_jd_table "db.server" "$RootDir" else echo NO return fi # Prepare grep to be case sensitive or insensitive based on SDP instance # configuration. IsP4DCaseSensitive is set in set_vars(). # Sensitive data can have case-varied server specs, but exactly one will # match grep without the -i. Insensitive data can have only one server # spec. grepCmd="$GREP" [[ "${IsP4DCaseSensitive:-0}" -eq 0 ]] && grepCmd+=" -i" [[ "${UseGrepM:-0}" -eq 1 ]] && grepCmd+=" -m 1" # Extract a slice of db.server referencing the given ServerID, and then grab # the field containing Services data. if [[ "${UseGrepM:-0}" -eq 1 ]]; then ServicesData=$("$P4DBIN" -r "$JDTmpDir" -v track=-1 -J off -L /dev/null -k db.server -jd - 2>&1 |\ $grepCmd "@db.server@ @${ServerID}@" | "$CUT" -d '@' -f 13 | tr -d ' ') else ServicesData=$("$P4DBIN" -r "$JDTmpDir" -v track=-1 -J off -L /dev/null -k db.server -jd - 2>&1 |\ $grepCmd "@db.server@ @${ServerID}@" | head -1 | "$CUT" -d '@' -f 13 | tr -d ' ') fi remove_jd_tables # Do a check to see if the ServicesData value indicates # this is an standby server. if [[ -n "$ServicesData" ]]; then if [[ "$ServicesData" -eq '2533' ]]; then echo YES else echo NO fi else echo NO fi } #------------------------------------------------------------------------------ # is_standby ($ServerID, $RootDir) # # Determine if a given ServerID is an standby server or not, checking a given # database root dir (e.g. $P4ROOT or $OFFLINE_DB). # # Input: # $1 - ServerID (required) # $2 - RootDir (optional, defaults to $P4ROOT) # # Output YES if an standby server, NO otherwise. # #------------------------------------------------------------------------------ function is_standby () { local ServerID="${1:-Unset}" local RootDir="${2:-$P4ROOT}" local ServicesData= local grepCmd= if [[ -r "$RootDir/db.server" ]]; then copy_jd_table "db.server" "$RootDir" else echo NO return fi # Prepare grep to be case sensitive or insensitive based on SDP instance # configuration. IsP4DCaseSensitive is set in set_vars(). # Sensitive data can have case-varied server specs, but exactly one will # match grep without the -i. Insensitive data can have only one server # spec. grepCmd="$GREP" [[ "${IsP4DCaseSensitive:-0}" -eq 0 ]] && grepCmd+=" -i" [[ "${UseGrepM:-0}" -eq 1 ]] && grepCmd+=" -m 1" # Extract a slice of db.server referencing the given ServerID, and then grab # the field containing Services data. if [[ "${UseGrepM:-0}" -eq 1 ]]; then ServicesData=$("$P4DBIN" -r "$JDTmpDir" -v track=-1 -J off -L /dev/null -k db.server -jd - 2>&1 |\ $grepCmd "@db.server@ @${ServerID}@" | "$CUT" -d '@' -f 13 | tr -d ' ') else ServicesData=$("$P4DBIN" -r "$JDTmpDir" -v track=-1 -J off -L /dev/null -k db.server -jd - 2>&1 |\ $grepCmd "@db.server@ @${ServerID}@" | head -1 | "$CUT" -d '@' -f 13 | tr -d ' ') fi remove_jd_tables # Do a check to see if the ServicesData value indicates # this is an standby server. if [[ -n "$ServicesData" ]]; then if [[ "$ServicesData" -eq '35141' || "$ServicesData" -eq '35301' ]]; then echo YES else echo NO fi else echo NO fi } #------------------------------------------------------------------------------ # Set variables for use in various scripts: # EDGESERVER=1 if this is an edge server, 0 otherwise. # STANDBYSERVER=1 if this is a standby server, 0 otherwise. # # This must be called after loading the standard shell environment by # doing: # source /p4/common/bin/p4_vars N # # This sets P4HOME, SERVERID, etc. needed by this function. #------------------------------------------------------------------------------ function set_vars () { P4DInitScript="$P4HOME/bin/p4d_${SDP_INSTANCE}_init" P4DRef="${P4DInitScript%_init}" P4DSystemdServiceFile="/etc/systemd/system/p4d_${SDP_INSTANCE}.service" P4BrokerInitScript="$P4HOME/bin/p4broker_${SDP_INSTANCE}_init" P4BrokerRef="${P4BrokerInitScript%_init}" P4BrokerSystemdServiceFile="/etc/systemd/system/p4broker_${SDP_INSTANCE}.service" P4ProxyInitScript="$P4HOME/bin/p4p_${SDP_INSTANCE}_init" P4ProxyRef="${P4ProxyInitScript%_init}" P4ProxySystemdServiceFile="/etc/systemd/system/p4p_${SDP_INSTANCE}.service" # If /p4/N/bin/p4d_N is a symlink, the current instance is configured to be # case-sensitive, otherwise it is case-insensitive. (On a non-p4d server # machine, the IsP4DCaseSensitive value may be wrong, but this is of no # consequence, as it is only used on p4d server machines). if [[ -L "${P4DInitScript%_init}" ]]; then IsP4DCaseSensitive=1 else IsP4DCaseSensitive=0 fi # Determine if 'grep' supports the '-m' option. Use it when we can as it is # fast. If it's not supported, use other methods, like "|head -1", which are # more portable but less efficient. if echo "test" | "$GREP" -m 1 "test" >/dev/null 2>&1; then UseGrepM=1 else UseGrepM=0 fi # shellcheck disable=SC2072 if [[ -n "${DO_PARALLEL_CHECKPOINTS:-}" && "$DO_PARALLEL_CHECKPOINTS" != "0" && "$P4D_VERSION" > "2022.2" ]]; then CreateParallelCheckpoint=1 # shellcheck disable=SC2072 [[ "$P4D_VERSION" > "2023.1" ]] && CreateMultifileParallelCheckpoint=1 if [[ "$DO_PARALLEL_CHECKPOINTS" =~ ^[1-9]{1}[0-9]*$ ]]; then if [[ "$DO_PARALLEL_CHECKPOINTS" == 1 ]]; then Threads=4 else # Strip leading zeros from the DO_PARALLEL_CHECKPOINTS value # (if the 'bc' utility is available). if [[ -n "$(command -v bc)" ]]; then Threads=$(echo "$DO_PARALLEL_CHECKPOINTS"|bc) else Threads="$DO_PARALLEL_CHECKPOINTS" fi fi else Threads=4 fi else CreateParallelCheckpoint=0 fi # shellcheck disable=SC2153 if [[ -n "$SERVERID" ]]; then if [[ "$(is_edge "$SERVERID")" == YES ]]; then export EDGESERVER=1 else export EDGESERVER=0 fi else export EDGESERVER=0 fi if [[ -n "$SERVERID" ]]; then if [[ "$(is_replica "$SERVERID")" == YES ]]; then export REPLICASERVER=1 # Get commit server from P4TARGET setting in database else export REPLICASERVER=0 fi else export REPLICASERVER=0 fi if [[ -n "$SERVERID" ]]; then if [[ "$(is_standby "$SERVERID")" == YES ]]; then export STANDBYSERVER=1 # Get commit server from P4TARGET setting in database else export STANDBYSERVER=0 fi else export STANDBYSERVER=0 fi # Ensure that SDP_ADMIN_PASSWORD_FILE is set, using existing value if set (e.g. # in p4_vars), otherwise set it to the SDP standard value. export SDP_ADMIN_PASSWORD_FILE="${SDP_ADMIN_PASSWORD_FILE:-Unset}" if [[ "$SDP_ADMIN_PASSWORD_FILE" == Unset ]]; then export SDP_ADMIN_PASSWORD_FILE="$P4CCFG/.p4passwd.${P4SERVER}.admin" fi } #------------------------------------------------------------------------------ # Check if user is running as required OS user. #------------------------------------------------------------------------------ function check_uid () { user=$(id -un) if [[ "${user}" != "${OSUSER}" ]]; then die "Must be run by user: ${OSUSER}. Abort!" fi } #------------------------------------------------------------------------------ # Function log() - echo message to logfile or stdout. # # If $LOGFILE is defined, write message to the log file only; nothing goes to # stdout. Prepend a datestamp. # If $LOGFILE isn't defined, just echo to stdout, w/o timestamp or. # In all cases, support '-e' formatting. # Input: # $1 - message to log (must be quoted). #------------------------------------------------------------------------------ function log () { if [[ "${LOGFILE:-Unset}" != Unset ]]; then echo -n "$(date +'%Y-%m-%d %H:%M:%S')" >> "$LOGFILE" 2>&1 echo -e " $0: $*" >> "$LOGFILE" 2>&1 else echo -e "$@" fi } #------------------------------------------------------------------------------ # Decide depending on our mail utility, how to specify sender (if we need to). # Mail on some platforms sets sender by default. # If the mail utility returns what looks like a version identifier # when given the '-V' flag, use a '-S' flag. If it does not return a # version identifier, don't set a mail sender option. # Allow GNU Mailutils alternative flag instead. Also check for s-nail/nail. #------------------------------------------------------------------------------ # Decide depending on our mail utility, how to specify sender (if we need to). # Mail on some platforms sets sender by default. # Check the mail utility version/type and set appropriate sender options. #------------------------------------------------------------------------------ function get_mail_sender_opt () { local mail_sender_opt= local mail_ver= if [[ -n "$MAILFROM" ]]; then # Capture both stdout and stderr for reliable version detection; some # mail clients write version info to stderr rather than stdout. # shellcheck disable=SC2076 mail_ver=$($SDPMAIL -V 2>&1) # Check for specific mail clients; s-nail/nail before numeric-version # variants so the more-specific pattern wins. if [[ "$mail_ver" =~ "GNU Mailutils" ]]; then # GNU Mailutils: combined -a flag form, no spaces (e.g. -aFrom:email) mail_sender_opt="-aFrom:$MAILFROM" elif [[ "$mail_ver" =~ "s-nail" ]]; then # s-nail requires -r for sender; comma-separated recipients are handled # by splitting MAILTO into separate args at the call site. mail_sender_opt="-r $MAILFROM" elif [[ "$mail_ver" =~ "nail" ]]; then # Regular nail/mailx variant mail_sender_opt="-r $MAILFROM" elif [[ "$mail_ver" =~ ^[0-9]+\.[0-9] ]]; then # Numeric version string — likely a mailx variant supporting -r mail_sender_opt="-r $MAILFROM" elif [[ "$SDPMAIL" == "s-nail" ]]; then # Fallback: version check failed but binary name identifies it as s-nail mail_sender_opt="-r $MAILFROM" elif [[ "$SDPMAIL" == "mailx" ]]; then # Fallback: version check failed but binary name identifies it as mailx mail_sender_opt="-r $MAILFROM" else # Unknown mail client or basic 'mail'; omit sender option as it may # not be supported. mail_sender_opt= fi fi echo "$mail_sender_opt" } #------------------------------------------------------------------------------ # Email the log file by $LOGFILE. #------------------------------------------------------------------------------ function mail_log_file () { local subject=$1 local mail_sender_opt mail_sender_opt=$(get_mail_sender_opt) # If the configured mail program is not in the PATH, just do an echo rather # than emailing. [[ -n "$(command -v "$SDPMAIL")" ]] || SDPMAIL="echo Simulated $SDPMAIL" # Split comma-delimited MAILTO into separate recipient arguments; s-nail and # some other clients reject a comma-separated list passed as a single arg. IFS=',' read -ra mailto_arr <<< "$MAILTO" # Send email # shellcheck disable=SC2086 $SDPMAIL -s "$subject" $mail_sender_opt "${mailto_arr[@]}" < "$LOGFILE" # Check for SNS_ALL_ALERT_TOPIC_ARN to send SNS notifications on both success and failure if [[ -n "${SNS_ALL_ALERT_TOPIC_ARN:-}" ]]; then sns_log_file_all "$subject" fi } #------------------------------------------------------------------------------ # Deliver the $LOGFILE via AWS SNS. #------------------------------------------------------------------------------ function sns_log_file () { # AWS SNS has a 100 character limit for subject field local subject="$1" local short_subject= # shellcheck disable=SC2116 disable=SC2155 short_subject="$(echo "${subject:0:100}")" aws --region "$AWS_DEFAULT_REGION" sns publish --topic-arn "$SNS_ALERT_TOPIC_ARN" --subject "$short_subject" --message "$(cat "$LOGFILE")" } function sns_log_file_all () { local subject="$1" # shellcheck disable=SC2116 disable=SC2155 local short_subject="$(echo "${subject:0:100}")" aws --region "$AWS_DEFAULT_REGION" sns publish --topic-arn "$SNS_ALL_ALERT_TOPIC_ARN" --subject "$short_subject" --message "$(cat "$LOGFILE")" } #------------------------------------------------------------------------------ # Deliver the $LOGFILE via PagerDuty. # See SDP documentation on how to configure: # https://workshop.perforce.com/view/p4-sdp/main/doc/SDP_Guide.Unix.html#_configuring_pagerduty_for_notifications # # Required environment variables # PAGERDUTY_ROUTING_KEY #------------------------------------------------------------------------------ function pd_log_file () { # shellcheck disable=SC2116 echo -e "Sending alert and log file contents to administrator via PagerDuty." >&2 # set these defaults to empty strings in case the user not defined them in their local p4 vars file PAGERDUTY_CUSTOM_FIELD="${PAGERDUTY_CUSTOM_FIELD:-''}" /usr/local/bin/pd event alert \ --routing_key="$PAGERDUTY_ROUTING_KEY" \ --source="$(hostname)" \ --summary="$(basename "$0") has failed in the SDP!" \ --keys="hostname" --keys="P4INSTANCE" --keys="component" --keys="log_file" \ --keys="script_name" --keys="log_contents" --keys="custom_field" \ --values="$(hostname)" --values="$P4INSTANCE" --values="SDP" --values="$LOGFILE" \ --values="$(basename "$0")" --values="$(cat "$LOGFILE")" --values="$PAGERDUTY_CUSTOM_FIELD" } #------------------------------------------------------------------------------ # Function die() - log message, send email/SNS, and exit. # If $LOGFILE is defined, write message to the log file, email/SNS log, # and exit. # If $LOGFILE is not defined, write message to the stdout, and skip # email/SNS. # If in terminal session, display message to stderr as well. #------------------------------------------------------------------------------ function die () { # mail the error (with more helpful subject line than cron) log "ERROR!!! - $HOSTNAME $P4SERVER $0: $*" if [[ "${LOGFILE:-Unset}" != Unset ]]; then if [[ "${SNS_ALERT_TOPIC_ARN:-Unset}" != Unset ]]; then log "Using SNS for log file delivery..." sns_log_file "ERROR!!! - $HOSTNAME $P4SERVER $0: $*" elif [[ "${PAGERDUTY_ROUTING_KEY:-Unset}" != Unset ]] && [[ -f "/usr/local/bin/pd" ]]; then log "Using PagerDuty for log file delivery..." pd_log_file "ERROR!!! - $HOSTNAME $P4SERVER $0: $*" else log "Using email for log file delivery..." mail_log_file "ERROR!!! - $HOSTNAME $P4SERVER $0: $*" fi fi # if running from terminal, also send to stderr if tty >/dev/null; then echo -e "$@" >&2 fi rm -f "${LOGS}/ckp_running.txt" exit 1 } #------------------------------------------------------------------------------ # Convert various byte values (K,M,G,T,P,E,%) to bytes. If passing in # Pass in values such as 1024K, 512M, 1G, 2T, 10%, or raw bytes (no suffix) # Returns: # - Echoes the byte count on success # - Returns exit code 0 on success, 1 on failure # # Usage Examples: # convert_to_bytes 2T && echo "OK" || echo "Conversion failed." # 2199023255552 # OK # # convert_to_bytes 10% || echo "Conversion failed." # Conversion failed. (bad input example) # # See more examples in the SizeConversion test group in /p4/sdp/test/bsw/cli_tests.cfg. #------------------------------------------------------------------------------ function convert_to_bytes() { local value=$1 local totalsize=${2:-0} local size= local unit= # Normalize input to uppercase value=$(echo "$value" | tr '[:lower:]' '[:upper:]') # Extract numeric and unit parts size=$("$GREP" -Eo '^-?[0-9]+' <<< "$value") unit=$("$GREP" -Eo '[KMGTPE%]$' <<< "$value") case "$unit" in K) echo $((size * 1024)) ;; M) echo $((size * 1024**2)) ;; G) echo $((size * 1024**3)) ;; T) echo $((size * 1024**4)) ;; P) echo $((size * 1024**5)) ;; E) echo $((size * 1024**6)) ;; %) if (( totalsize > 0 )); then echo $((totalsize * size / 100)) else return 1 fi ;; ''|[0-9]) echo "$size" ;; # Default: treat as bytes *) return 1 ;; esac return 0 } #------------------------------------------------------------------------------ # Convert bytes into a human-readable value (K,M,G,T,P,E), # choosing the largest unit where the value is >= 1. # Includes 1 decimal place of precision where appropriate. # Adds 'B' only for values < 1K. # For good input, echo converted value and return 0. # If invalid input is given, echo nothing, and return 1. # # Usage Examples: # convert_to_human_readable 1536 # 1.5K # convert_to_human_readable 999 # 999B # convert_to_human_readable 1099511627776 # 1.0T # # See more examples in the SizeConversion test group in /p4/sdp/test/bsw/cli_tests.cfg. #------------------------------------------------------------------------------ function convert_to_human_readable() { local bytes=$1 local result if (( bytes < 0 )); then return 1 fi if (( bytes >= 1024**6 )); then result=$(awk -v b="$bytes" 'BEGIN{printf "%.1fE", b/(1024^6)}') elif (( bytes >= 1024**5 )); then result=$(awk -v b="$bytes" 'BEGIN{printf "%.1fP", b/(1024^5)}') elif (( bytes >= 1024**4 )); then result=$(awk -v b="$bytes" 'BEGIN{printf "%.1fT", b/(1024^4)}') elif (( bytes >= 1024**3 )); then result=$(awk -v b="$bytes" 'BEGIN{printf "%.1fG", b/(1024^3)}') elif (( bytes >= 1024**2 )); then result=$(awk -v b="$bytes" 'BEGIN{printf "%.1fM", b/(1024^2)}') elif (( bytes >= 1024 )); then result=$(awk -v b="$bytes" 'BEGIN{printf "%.1fK", b/(1024)}') else result="${bytes}B" echo "$result" return 0 fi echo "$result" return 0 } #------------------------------------------------------------------------------ # Write a semaphore file, $LOGS/ckp_running.txt. This file is written at # the start of processing, and removed upon successful completion. It # prevents multiple concurrent operations from being launched accidentally # e.g. by multiple human admins, or a human inadvertently competing with a # cron job. # # It is also intended to get human admins to determine the root cause of # checkpoint failures. #------------------------------------------------------------------------------ function ckp_running() { if [[ -f "${LOGS}/ckp_running.txt" ]]; then die "Last checkpoint not complete. Check the backup process or contact support." fi echo "Checkpoint running on $(date)." > "${LOGS}/ckp_running.txt" } #------------------------------------------------------------------------------ # Remove the ckp_running.txt semaphore file when checkpoint processing is # complete. #------------------------------------------------------------------------------ function ckp_complete() { rm -f "${LOGS}/ckp_running.txt" } #------------------------------------------------------------------------------ # Ensure key directories are writable. Abort if they are not. # Input: # $1 - serverType, specify 1 (default) to check dirs for p4d, 2 to check # dirs for a standalone proxy or broker. #------------------------------------------------------------------------------ function check_dirs () { local -i serverType=${1:-1} # Check that key dirs are writable local -i dirsOK=1 local dirList # shellcheck disable=SC2153 case "$serverType" in (1) dirList="$OFFLINE_DB $CHECKPOINTS $LOGS $P4TMP";; (2) dirList="$LOGS $P4TMP";; (*) dirList="$LOGS $P4TMP";; esac [[ "$EDGESERVER" -eq 1 ]] && dirList+=" ${CHECKPOINTS}.${SERVERID#p4d_}" for dir in $dirList; do if [[ ! -d "$dir" || ! -w "$dir" ]]; then log "Error: Dir $dir does not exist or is not writable." dirsOK=0 fi done [[ "$dirsOK" -eq 1 ]] || die "Some expected dirs are missing or not writable. Aborting." } #------------------------------------------------------------------------------ # Add the results of df -h or df -m to the log file. #------------------------------------------------------------------------------ function check_disk_space () { log "Checking disk space..." $P4BIN diskspace >> "$LOGFILE" 2>&1 } #------------------------------------------------------------------------------ # Check value of journal; ensure it is an integer. #------------------------------------------------------------------------------ function check_journalnum () { local JNLNUM=${1:-Unset} re='^[0-9]+$' if ! [[ $JNLNUM =~ $re ]] ; then die "The journal counter value [$JNLNUM] is invalid. It must be numeric." fi } #------------------------------------------------------------------------------ # Check the checkpoints directory for the oldest checkpoint #------------------------------------------------------------------------------ function get_ckpnum () { if [[ "$EDGESERVER" -eq 0 ]]; then # shellcheck disable=SC2034 disable=SC2012 disable=SC2016 OLDESTCHECKPOINT=$(ls -1tr "${CHECKPOINTS}/" | "$GREP" \\.ckp | "$GREP" -v \\.md5$ | head -n 1 | "$AWK" -F '.ckp.' '{ print $(2) }' | tr -d '.gz') else # shellcheck disable=SC2034 disable=SC2012 disable=SC2016 OLDESTCHECKPOINT=$(ls -1tr "${CHECKPOINTS}.${SERVERID#p4d_}/" | "$GREP" \\.ckp | "$GREP" -v \\.md5$ | head -n 1 | "$AWK" -F '.ckp.' '{ print $(2) }' | tr -d '.gz') fi } #------------------------------------------------------------------------------ # Determine journal counter by checking counter in db.counters. #------------------------------------------------------------------------------ function get_journalnum () { # get the current journal and checkpoint serial numbers. local nextCheckpointNum if [[ -r "$P4ROOT/db.counters" ]]; then copy_jd_table "db.counters" "$P4ROOT" nextCheckpointNum=$("$P4DBIN" -r "$JDTmpDir" -v track=-1 -k db.counters -jd - 2>&1 | grep @journal@ | cut -d '@' -f 8) remove_jd_tables if [[ -n "$nextCheckpointNum" ]]; then check_journalnum "$nextCheckpointNum" JOURNALNUM="$nextCheckpointNum" else # Special case: If db.counters is empty, then we have a new/empty data # set, so just set the value to 0. JOURNALNUM=0 fi else # Special case: If db.counters doesn't exist, then we have a new/empty # data set, so just set the value to 0. JOURNALNUM=0 fi # If we are on an edge server, the journal has already rotated, so we have to decrement the value # so that we replay the correct journal file and create the correct checkpoint number on the # edge server. # # In the case of a standby server, the journal rotation occurs on the master server, # so we don't need to increment the journal number again, so we decrement by 1. # Also, when replaying the journals to the offline db, we don't want to play to the live journal # because it is still being replicated. if [[ "$EDGESERVER" -eq 1 || "$REPLICASERVER" -eq 1 || "$STANDBYSERVER" -eq 1 ]]; then JOURNALNUM=$((JOURNALNUM - 1)) fi CHECKPOINTNUM=$((JOURNALNUM + 1)) } #------------------------------------------------------------------------------ # Determine journal space usage and minimum disk space requirement #------------------------------------------------------------------------------ function get_journal_stats () { # Get minimum disk space required on server journal filesystem before server rejects commands # This will return the configured and default value, but grab the configured value which shows first # If a configured value is not present, it will use the default value # shellcheck disable=SC2034 disable=SC2016 P4JOURNALMIN=$("$P4BIN" configure show filesys.P4JOURNAL.min | "$AWK" '{ print $1 }' | $CUT -d'=' -f2 | head -1) # Get current journal free disk space # shellcheck disable=SC2034 P4JOURNALFREE=$("$P4BIN" -ztag -F "%freeBytes%" diskspace P4JOURNAL) # Get total available disk space for journal # shellcheck disable=SC2034 P4JOURNALTOTAL=$("$P4BIN" -ztag -F "%totalBytes%" diskspace P4JOURNAL) } #------------------------------------------------------------------------------ # Verify that the offline databases are usable by checking the existence # of a 'offline_db_usable.txt' file that is written only when databases # are in a known-good state, following successful recovery from a checkpoint. #------------------------------------------------------------------------------ function check_offline_db_usable () { # Check it is OK if [[ ! -f "$OFFLINE_DB/offline_db_usable.txt" ]]; then die "Offline database not in a usable state. Check the backup process." fi if [[ ! -f "$OFFLINE_DB/db.counters" ]]; then die "Offline database not found. Consider creating it with live_checkpoint.sh. Be aware that live_checkpoint.sh locks the live system and may take a long time. Aborting." fi } #------------------------------------------------------------------------------ # Determine journal counter in offline databases. #------------------------------------------------------------------------------ function get_offline_journal_num () { check_offline_db_usable if [[ -r "$OFFLINE_DB/db.counters" ]]; then copy_jd_table "db.counters" "$OFFLINE_DB" else die "Cannot check offline journal counter; offline_db is missing db.counters." fi # Get the journal number of the offline database export OFFLINEJNLNUM OFFLINEJNLNUM=$("$P4DBIN" -r "$JDTmpDir" -v track=-1 -jd - db.counters 2>&1 | grep '@journal@' | cut -d "@" -f 8 2>> "$LOGFILE") || die "Cannot get the offline journal number. Abort!" remove_jd_tables check_journalnum "$OFFLINEJNLNUM" log "Offline journal number is: $OFFLINEJNLNUM" } #------------------------------------------------------------------------------ # Request replica checkpoints. function request_replica_checkpoint () { local checkpointCmd= if [[ "$CreateParallelCheckpoint" -eq 1 ]]; then if [[ "$CreateMultifileParallelCheckpoint" -eq 1 ]]; then checkpointCmd="$P4BIN admin checkpoint -Z -p -m -N ${Threads:-4}" else checkpointCmd="$P4BIN admin checkpoint -Z -p -N ${Threads:-4}" fi else checkpointCmd="$P4BIN admin checkpoint -Z" fi log "Running: $checkpointCmd" $checkpointCmd >> "$LOGFILE" 2>&1 & } #------------------------------------------------------------------------------ # Cleanup old checkpoint and numbered journal files. #------------------------------------------------------------------------------ function remove_old_checkpoints_and_journals () { local checkpointsDir= local filePrefix= local ckpDir= local ckpFile= local jnlFile= local md5File= local journalPrefix= local standbyReplicaJournalsDir= local tmpFile= local lastCheckpointJournalCounter= local journalCounter= local keepCheckpointCounter= local keepJournalCounter= if [[ "$KEEPCKPS" -eq 0 ]]; then log "Skipping cleanup of old checkpoints because KEEPCKPS is set to 0." return 0 fi if [[ "$CheckpointDumpSkipped" -eq 1 ]]; then log "Skipping cleanup of old checkpoints and journals because the checkpoint dump was skipped." return 0 fi if get_latest_checkpoint_with_md5; then lastCheckpointJournalCounter="${CHECKPOINT##*.ckp.}" lastCheckpointJournalCounter="${lastCheckpointJournalCounter%%.*}" if [[ "$lastCheckpointJournalCounter" =~ ^[0-9]+$ ]]; then if [[ "$KEEPCKPS" -lt "$lastCheckpointJournalCounter" ]]; then keepCheckpointCounter=$((lastCheckpointJournalCounter-KEEPCKPS+1)) else log "Not removing any checkpoints or journals because KEEPCKPS ($KEEPCKPS) is greater than or equal to the highest checkpoint counter ($lastCheckpointJournalCounter)." return fi if [[ "$KEEPJNLS" -lt "$lastCheckpointJournalCounter" ]]; then keepJournalCounter=$((lastCheckpointJournalCounter-KEEPJNLS)) else log "Not removing any numbered journals because KEEPJNLS ($KEEPJNLS) is greater than or equal to the highest checkpoint counter ($lastCheckpointJournalCounter)." keepJournalCounter=ALL fi log "Deleting obsolete checkpoints and journals since last checkpoint counter ($lastCheckpointJournalCounter). Removing checkpoints with counter less than $keepCheckpointCounter (KEEPCKPS=$KEEPCKPS) and journals with counter less than $keepJournalCounter (KEEPJNLS=$KEEPJNLS)." else log "Error: Invalid journal counter [$lastCheckpointJournalCounter] extracted from file name: $CHECKPOINT - Skipping removal of obsolete checkpoints and journals." return 1 fi else log "Skipping removal of checkpoints and journal as no checkpoints are available." return 0 fi # For the master server, we can safely rely on the SDP standard that the journalPrefix # is of the form '/p4/N/checkpoints/p4_N'. For replicas and edge servers, determine # the values dynamically based on the current journalPrefix value for the given ServerID. if [[ "$SERVERID" == "$P4MASTER_ID" ]]; then # Refer to ckp/jnl files starting like: /p4/N/checkpoints/p4_N checkpointsDir="${CHECKPOINTS}" filePrefix="${P4SERVER}" else if [[ -r "$P4ROOT/db.config" ]]; then copy_jd_table "db.config" "$P4ROOT" journalPrefix="$("$P4DBIN" -r "$JDTmpDir" -v track=-1 -k db.config -jd - | grep "@${SERVERID}@ @journalPrefix@" | cut -d '@' -f 10)" remove_jd_tables fi if [[ -n "$journalPrefix" ]]; then checkpointsDir="${journalPrefix%/*}" filePrefix="${journalPrefix##*/}" else log "Warning: Could not determine journalPrefix for ServerID $SERVERID." checkpointsDir="${CHECKPOINTS}" filePrefix="${P4SERVER}" fi fi # Find old checkpoints and journals to remove. if [[ -d "$checkpointsDir" ]]; then # First, build a list of all checkpoints based on *.md5 and *.OK files. This list will include checkpoint files and # parallel checkpoint directories, but will ignore uncompressed checkpoints. Ignoring uncompressed checkpoints is # for safety; SDP deals only in compressed checkpoints, and any uncompressed checkpoint is likely due to some kind # of recovery or other non-routine operation. Leave uncompressed files alone, but list them as warnings. For # optimal safety, ignore files that don't precisely match patterns of checkpoint and journal files that could be # created by p4d with the given journalPrefix using tight regular expressions. # # The *.OK files are essentially stand-ins for for *.md5 files for a small range of p4d versions that that did not generate top-level # *.md5 files for parallel checkpoint directories. For P4D 2023.2+, top-level *.md5 files are generated for parallel # checkpoint directories, so they'll have both *.OK and *.md5 files. tmpFile=$(mktemp "$P4TMP/checkpoint_MD5_files.XXXXXXXX") log "Finding old *.md5 and *.OK files and corresponding checkpoints to remove." log "Running: find \"$checkpointsDir/\" -mindepth 1 -maxdepth 1 | grep -E \"^${checkpointsDir}/${filePrefix}\.ckp\.[1-9]{1}[0-9]*(\.gz|)\.(md5|OK)\$\" \"$tmpFile\"" find "$checkpointsDir/" -mindepth 1 -maxdepth 1 | grep -E "^${checkpointsDir}/${filePrefix}\.ckp\.[1-9]{1}[0-9]*(\.gz|)\.(md5|OK)$" > "$tmpFile" if [[ -n "${SDP_DEBUG:-}" ]]; then log "DEBUG: Contents of $tmpFile:" cat "$tmpFile" >> "$LOGFILE" 2>&1 fi # Given an MD5 file, there are a few possibilities for the corresponding checkpoint: # - A compressed checkpoint file created by 'p4 admin checkpoint -Z'. # - An uncompressed checkpoint file created by 'p4 admin checkpoint'. # - A checkpoint directory created if parallel checkpoints were enabled. # # Symlinks for checkpoints and journal files are never created by SDP; such are created only by # external processing, e.g. by human admins. We avoid deleting those. while read -r md5File; do journalCounter="${md5File##*.ckp.}" journalCounter="${journalCounter%%.*}" if [[ "$journalCounter" =~ ^[0-9]+$ ]]; then if [[ "$journalCounter" -ge "$keepCheckpointCounter" ]]; then [[ -n "${SDP_DEBUG:-}" ]] && log "DEBUG: $md5File not removed; journal counter $journalCounter is greater than or equal to $keepCheckpointCounter." continue fi else log "Error: Invalid journal counter [$journalCounter] extracted from file name: $md5File - Not removing this file." continue fi ckpFile= ckpDir= if [[ "$md5File" == *.gz.md5 ]]; then if [[ -r "${md5File%.md5}" ]]; then if [[ -f "${md5File%.md5}" ]]; then ckpFile="${md5File%.md5}" elif [[ -L "${md5File%.md5}" ]]; then log "Warning: For MD5 file [$md5File], expected checkpoint .gz file was a symlink instead. Not removing: ${md5File%.md5}" fi else log "Warning: For MD5 file [$md5File], did not find expected checkpoint .gz file to cleanup: ${md5File%.md5}" fi elif [[ "$md5File" == *.md5 ]]; then if [[ -r "${md5File%.md5}" ]]; then if [[ -d "${md5File%.md5}" ]]; then ckpDir="${md5File%.md5}" elif [[ -f "${md5File%.md5}" ]]; then log "Warning: For MD5 file [$md5File], expected checkpoint directory was a file, possibly an uncompressed checkpoint. Not removing: ${md5File%.md5}" elif [[ -L "${md5File%.md5}" ]]; then log "Warning: For MD5 file [$md5File], expected checkpoint directory was a symlink. Not removing: ${md5File%.md5}" fi elif [[ -r "${md5File%.md5}.gz" ]]; then if [[ -f "${md5File%.md5}.gz" ]]; then ckpFile="${md5File%.md5}.gz" elif [[ -L "${md5File%.md5}.gz" ]]; then log "Warning: For MD5 file [$md5File], expected checkpoint file was a symlink. Not removing: ${md5File%.md5}.gz" fi else log "Warning: For MD5 file [$md5File], did not find expected checkpoint file or directory to cleanup: ${md5File%.md5} or ${md5File%.md5}.gz" fi elif [[ "$md5File" == *.OK ]]; then if [[ -r "${md5File%.OK}" ]]; then if [[ -d "${md5File%.OK}" ]]; then ckpDir="${md5File%.OK}" elif [[ -f "${md5File%.OK}" ]]; then log "Warning: For MD5 file [$md5File], expected checkpoint directory was a file, possibly an uncompressed checkpoint. Not removing: ${md5File%.OK}" elif [[ -L "${md5File%.OK}" ]]; then log "Warning: For MD5 file [$md5File], expected checkpoint directory was a symlink. Not removing: ${md5File%.OK}" fi fi fi log "Removing checkpoint MD5 file with: rm -f \"$md5File\"" rm -f "$md5File" >> "$LOGFILE" 2>&1 ||\ log "Warning: Error doing: rm -f \"$md5File\"" if [[ -n "$ckpFile" ]]; then log "Removing checkpoint file: rm -f \"$ckpFile\"" rm -f "$ckpFile" >> "$LOGFILE" 2>&1 ||\ log "Warning: Error doing: rm -f \"$ckpFile\"" fi if [[ -n "$ckpDir" ]]; then log "Removing checkpoint dir: rm -rf \"$ckpDir\"" rm -rf "$ckpDir" >> "$LOGFILE" 2>&1 ||\ log "Warning: Error doing: rm -rf \"$ckpDir\"" fi done < "$tmpFile" [[ -n "${SDP_DEBUG:-}" ]] || rm -f "$tmpFile" if [[ "$keepJournalCounter" != "ALL" ]]; then # Use KEEPJNLS to allow for separate journal rotation at a different frequency than KEEPCKPS. tmpFile=$(mktemp "$P4TMP/journal_files.XXXXXXXX") log "Finding old numbered journal files to remove." log "Running: find \"$checkpointsDir/\" -mindepth 1 -maxdepth 1 | grep -E \"^${checkpointsDir}/${filePrefix}\.jnl\.[0-9]+\$\" \"$tmpFile\"" find "$checkpointsDir/" -mindepth 1 -maxdepth 1 | grep -E "^${checkpointsDir}/${filePrefix}\.jnl\.[0-9]+$" > "$tmpFile" if [[ -n "${SDP_DEBUG:-}" ]]; then log "DEBUG: Contents of $tmpFile:" cat "$tmpFile" >> "$LOGFILE" 2>&1 fi while read -r jnlFile; do journalCounter="${jnlFile##*.jnl.}" if [[ "$journalCounter" =~ ^[0-9]+$ ]]; then if [[ "$journalCounter" -ge "$keepJournalCounter" ]]; then [[ -n "${SDP_DEBUG:-}" ]] && log "DEBUG: $jnlFile not removed; journal counter $journalCounter is greater than or equal to $keepJournalCounter." continue fi else log "Error: Invalid journal counter [$journalCounter] extracted from file name: $jnlFile - Not removing this file." continue fi log "Removing journal file with: rm -f \"$jnlFile\"" rm -f "$jnlFile" >> "$LOGFILE" 2>&1 ||\ log "Warning: Error doing: rm -f \"$jnlFile\"" done < "$tmpFile" [[ -n "${SDP_DEBUG:-}" ]] || rm -f "$tmpFile" fi else log "Warning: Expected checkpoints directory [$checkpointsDir] does not exist." fi # The 'journals.rep' structure has been obsolete for some time, but we still advise cleanup in # that structure in case modern SDP is installed in an environment where older SDP recently # had been operating. standbyReplicaJournalsDir="${P4HOME}/journals.rep" if [[ -d "$standbyReplicaJournalsDir" ]]; then log "Obsolete folder detected: $standbyReplicaJournalsDir - this folder can be removed after sufficient checkpoints and numbered journals appear in ${CHECKPOINTS%/*}/" fi # This is a workaround to cleanup $LOGS/journal.NNN files on standby replicas. # These files are normally removed by p4d during journal rotation on the standby # replica. Use only if standby journals are not removed due to a standby replica # sharing /p4depots with its P4TARGET server. To use this workround, add this # line to the end of the /p4/common/config/p4_N.vars file: # # export SDP_REMOVE_STANDBY_JOURNALS=1 # if [[ "${SDP_REMOVE_STANDBY_JOURNALS:-0}" == 1 && "$(is_standby "$SERVERID")" == YES ]]; then tmpFile=$(mktemp "$P4TMP/standby_journal_N_files.XXXXXXXX") log "Finding old numbered journal files to remove on standby." log "Running: find \"$LOGS/\" -mindepth 1 -maxdepth 1 | grep -E \"^${LOGS}\.journal\.[1-9]{1}[0-9]*\$\" \"$tmpFile\"" find "$LOGS/" -mindepth 1 -maxdepth 1 | grep -E "^${LOGS}\.journal\.[1-9]{1}[0-9]*$" > "$tmpFile" if [[ -n "${SDP_DEBUG:-}" ]]; then log "DEBUG: Contents of $tmpFile:" cat "$tmpFile" >> "$LOGFILE" 2>&1 fi log "Removing excess journal.NNN files due to SDP_REMOVE_STANDBY_JOURNALS=1." while read -r standbyJnlFile; do journalCounter="${standbyJnlFile##*journal.}" if [[ "$journalCounter" =~ ^[0-9]+$ ]]; then if [[ "$journalCounter" -ge "$keepJournalCounter" ]]; then [[ -n "${SDP_DEBUG:-}" ]] && log "DEBUG: $standbyJnlFile not removed; journal counter $journalCounter is greater than or equal to $keepJournalCounter." continue fi else log "Error: Invalid journal counter [$journalCounter] extracted from file name: $standbyJnlFile - Not removing this file." continue fi log "Removing standby journal file with: rm -f \"$standbyJnlFile\"" rm -f "$standbyJnlFile" >> "$LOGFILE" 2>&1 ||\ log "Warning: Error doing: rm -f \"$standbyJnlFile\"" done < "$tmpFile" [[ -n "${SDP_DEBUG:-}" ]] || rm -f "$tmpFile" fi } #------------------------------------------------------------------------------ # Function: is_server_up ($server) # # Input: # $1 - server, one of 'p4d', 'p4p', or 'p4broker' # # Output: None # # Return Codes: # 0: Server is up. # 1: Server is down. # 2: Bad usage. # # Server up/down status is checked using the appropriate init script. #------------------------------------------------------------------------------ function is_server_up () { local server="${1:-Unset}" case "$server" in (p4d) "$P4DInitScript" status > /dev/null 2>&1 return $? ;; (p4broker) "$P4BrokerInitScript" status > /dev/null 2>&1 return $? ;; (p4p) "$P4ProxyInitScript" status > /dev/null 2>&1 return $? ;; (Unset) log "Internal Error: is_server_up(): No server type specified." return 2 ;; (*) log "Internal Error: is_server_up(): Unknown server specified: $server" return 2 ;; esac } #------------------------------------------------------------------------------ # Shutdown p4d using systemd if configured for systemd. Otherwise call the # underlying init script directly. # # Log the shutdown activity. # #------------------------------------------------------------------------------ function stop_p4d () { log "Shutting down the ${P4DBIN##*/} server." local -i maxStopDelay=${SDP_MAX_STOP_DELAY_P4D:-43200} local -i stopVerified=0 local -i i=0 local -i useSystemd=0 local serviceName= if [[ -n "$(command -v systemctl)" && ! -r /.dockerenv ]]; then serviceName="${P4DBIN##*/}" if [[ -n "$(systemctl is-enabled "$serviceName" 2>/dev/null)" ]]; then useSystemd=1 fi fi if [[ "$useSystemd" -eq 1 ]]; then { sudo systemctl stop "${P4DBIN##*/}"; } >> "$LOGFILE" 2>&1 ||\ die "Failed to execute: sudo systemctl stop ${P4DBIN##*/}" # With systemd, we must independently confirm service stop, # waiting if needed. stopVerified=0 i=0; while [[ "$i" -lt "$maxStopDelay" ]]; do if is_server_up p4d; then sleep 1 else stopVerified=1 break fi i+=1 done else "$P4DInitScript" stop >> "$LOGFILE" 2>&1 stopVerified=1 fi if [[ "$stopVerified" -eq 1 ]]; then log "Stopped ${P4DBIN##*/} server." return 0 else log "Error: Server ${P4DBIN##*/} did not stop after $maxStopDelay seconds. Tailing $P4LOG:" tail "$P4LOG" >> "$LOGFILE" 2>&1 die "Aborting due to failed p4d stop." fi } #------------------------------------------------------------------------------ # Shutdown p4broker using systemd if configured for systemd. Otherwise call the # underlying init script directly. # # Log the shutdown activity. # #------------------------------------------------------------------------------ function stop_p4broker () { log "Shutting down the ${P4BROKERBIN##*/} server." local -i maxStopDelay=${SDP_MAX_STOP_DELAY_P4BROKER:-600} local -i stopVerified=0 local -i i=0 local -i useSystemd=0 local serviceName= if [[ -n "$(command -v systemctl)" ]]; then serviceName="${P4BROKERBIN##*/}" if [[ -n "$(systemctl is-enabled "$serviceName" 2>/dev/null)" ]]; then useSystemd=1 fi fi if [[ "$useSystemd" -eq 1 ]]; then { sudo systemctl stop "${P4BROKERBIN##*/}"; } >> "$LOGFILE" 2>&1 ||\ die "Failed to execute: sudo systemctl stop ${P4BROKERBIN##*/}" # With systemd, we must independently confirm service stop, # waiting if needed. stopVerified=0 i=0; while [[ "$i" -lt "$maxStopDelay" ]]; do if is_server_up p4broker; then sleep 1 else stopVerified=1 break fi i+=1 done else "$P4BrokerInitScript" stop >> "$LOGFILE" 2>&1 stopVerified=1 fi if [[ "$stopVerified" -eq 1 ]]; then log "Stopped ${P4BROKERBIN##*/} server." return 0 else die "Server ${P4BROKERBIN##*/} did not stop after $maxStopDelay seconds." fi } #------------------------------------------------------------------------------ # Shutdown p4p using systemd if configured for systemd. Otherwise call the # underlying init script directly. # # Log the shutdown activity. # #------------------------------------------------------------------------------ function stop_p4p () { log "Shutting down the ${P4PBIN##*/} server." local -i maxStopDelay=${SDP_MAX_STOP_DELAY_P4P:-600} local -i stopVerified=0 local -i i=0 local -i useSystemd=0 local serviceName= if [[ -n "$(command -v systemctl)" ]]; then serviceName="${P4PBIN##*/}" if [[ -n "$(systemctl is-enabled "$serviceName" 2>/dev/null)" ]]; then useSystemd=1 fi fi if [[ "$useSystemd" -eq 1 ]]; then { sudo systemctl stop "${P4PBIN##*/}"; } >> "$LOGFILE" 2>&1 ||\ die "Failed to execute: sudo systemctl stop ${P4PBIN##*/}" # With systemd, we must independently confirm service stop, # waiting if needed. stopVerified=0 i=0; while [[ "$i" -lt "$maxStopDelay" ]]; do if is_server_up p4p; then sleep 1 else stopVerified=1 break fi i+=1 done else "$P4ProxyInitScript" stop >> "$LOGFILE" 2>&1 stopVerified=1 fi if [[ "$stopVerified" -eq 1 ]]; then log "Stopped ${P4PBIN##*/} server." return 0 else die "Server ${P4PBIN##*/} did not stop after $maxStopDelay seconds." fi } #------------------------------------------------------------------------------ # Start p4d using systemd if configured for systemd. Otherwise call the # underlying init script directly. # # Log the startup activity. # # This is a do-or-die function. It returns success upon successful server # startup, or else dies. #------------------------------------------------------------------------------ function start_p4d () { log "Starting the ${P4DBIN##*/} server." local -i maxStartDelay=${SDP_MAX_START_DELAY_P4D:-120} local -i startVerified=0 local -i i=0 local -i useSystemd=0 local serviceName= if [[ -n "$(command -v systemctl)" && ! -r /.dockerenv ]]; then serviceName="${P4DBIN##*/}" if [[ -n "$(systemctl is-enabled "$serviceName" 2>/dev/null)" ]]; then useSystemd=1 fi fi if [[ "$useSystemd" -eq 1 ]]; then { sudo systemctl start "${P4DBIN##*/}"; } ||\ die "Failed to execute: sudo systemctl start ${P4DBIN##*/}" else "$P4DInitScript" start >> "$LOGFILE" 2>&1 fi # Confirm that p4d started, waiting if needed. startVerified=0 i=0; while [[ "$i" -lt "$maxStartDelay" ]]; do if is_server_up p4d; then startVerified=1 break else sleep 1 fi i+=1 done if [[ "$startVerified" -eq 1 ]]; then log "Server ${P4DBIN##*/} started successfully." return 0 else log "Error: Server ${P4DBIN##*/} did not start after $maxStartDelay seconds. Tailing $P4LOG:" tail "$P4LOG" >> "$LOGFILE" 2>&1 die "Aborting due to failed p4d start." fi } #------------------------------------------------------------------------------ # Start p4broker using systemd if configured for systemd. Otherwise call the # underlying init script directly. # # Log the startup activity. # # This is a do-or-die function. It returns success upon successful server # startup, or else dies. #------------------------------------------------------------------------------ function start_p4broker () { log "Starting the ${P4BROKERBIN##*/} server." local -i maxStartDelay=${SDP_MAX_START_DELAY_P4BROKER:-60} local -i startVerified=0 local -i i=0 local -i useSystemd=0 local serviceName= if [[ -n "$(command -v systemctl)" ]]; then serviceName="${P4BROKERBIN##*/}" if [[ -n "$(systemctl is-enabled "$serviceName" 2>/dev/null)" ]]; then useSystemd=1 fi fi if [[ "$useSystemd" -eq 1 ]]; then { sudo systemctl start "${P4BROKERBIN##*/}"; } ||\ die "Failed to execute: sudo systemctl start ${P4BROKERBIN##*/}" else "$P4BrokerInitScript" start >> "$LOGFILE" 2>&1 fi # Confirm that p4broker started, waiting if needed. startVerified=0 i=0; while [[ "$i" -lt "$maxStartDelay" ]]; do if is_server_up p4broker; then startVerified=1 break else sleep 1 fi i+=1 done if [[ "$startVerified" -eq 1 ]]; then log "Server ${P4BROKERBIN##*/} started successfully." return 0 else die "Server ${P4BROKERBIN##*/} did not start after $maxStartDelay seconds." fi } #------------------------------------------------------------------------------ # Start p4p using systemd if configured for systemd. Otherwise call the # underlying init script directly. # # Log the startup activity. # # This is a do-or-die function. It returns success upon successful server # startup, or else dies. #------------------------------------------------------------------------------ function start_p4p () { log "Starting the ${P4PBIN##*/} server." local -i maxStartDelay=${SDP_MAX_START_DELAY_P4P:-60} local -i startVerified=0 local -i i=0 local -i useSystemd=0 local serviceName= if [[ -n "$(command -v systemctl)" ]]; then serviceName="${P4PBIN##*/}" if [[ -n "$(systemctl is-enabled "$serviceName" 2>/dev/null)" ]]; then useSystemd=1 fi fi if [[ "$useSystemd" -eq 1 ]]; then { sudo systemctl start "${P4PBIN##*/}"; } ||\ die "Failed to execute: sudo systemctl start ${P4PBIN##*/}" else "$P4ProxyInitScript" start >> "$LOGFILE" 2>&1 fi # Confirm that p4p started, waiting if needed. startVerified=0 i=0; while [[ "$i" -lt "$maxStartDelay" ]]; do if is_server_up p4p; then startVerified=1 break else sleep 1 fi i+=1 done if [[ "$startVerified" -eq 1 ]]; then log "Server ${P4PBIN##*/} started successfully." return 0 else die "Server ${P4PBIN##*/} did not start after $maxStartDelay seconds." fi } #------------------------------------------------------------------------------ # Do a front-door 'p4d admin journal' command to rotate the current/active # journal file on the master server, starting a fresh new P4JOURNAL file. # # In a distributed topology with replicas/edge servers, this function must # be called only on the master/commit server. #------------------------------------------------------------------------------ function truncate_journal () { local CheckpointFile="${CHECKPOINTS}/${P4SERVER}.ckp.${CHECKPOINTNUM}.gz" local JournalFile="${CHECKPOINTS}/${P4SERVER}.jnl.${JOURNALNUM}" local JournalCmd= if [[ "$SERVERID" == "$P4MASTER_ID" ]]; then [[ -f "$CheckpointFile" ]] && \ die "Checkpoint $CheckpointFile already exists, check the backup process." [[ -f "$JournalFile" ]] && \ die "Journal $JournalFile already exists, check the backup process." log "Truncating journal..." # During journal rotation, either by a front-door 'p4 admin journal' or a # back-door 'p4d -jj', p4d does a copy-then-delete rather than an mv at # the OS level. During rotation, the perforce server will pause # responses to clients (just as with a checkpoint), but this should be # for a short period of time even for large data sets, as the journal # typically represents a single day of metadata. # Curly braces capture output of 'time'. "$P4CBIN"/p4login -p "$P4MASTERPORT" JournalCmd="$P4BIN -p $P4MASTERPORT admin journal" log "Running: $JournalCmd" { time $JournalCmd; } >> "$LOGFILE" 2>&1 || { die "Journal rotation failed. Abort!"; } # The test below waits until the journal file exists in the checkpoints directory before proceeding. test=1 while [[ $test != 0 ]]; do sleep 5 if [[ -f "$JournalFile" ]]; then test=0 fi done "$P4CBIN"/p4login else log "Warning: The truncate_journal () function should only be called on the master server. It is ignored on edge and replica servers." fi } #------------------------------------------------------------------------------ # Call 'p4d -jj' to rotate the current/active journal file on the master # server from an edge server, starting a fresh new P4JOURNAL file. # # In a distributed topology with edge and standby servers, this function can be # used to trigger a journal rotation on master/commit server. It's not meant to # be used from the master server itself. #------------------------------------------------------------------------------ function truncate_journal_on_master () { # Increment Edge journal number since the journal will increment on the master after calling journal rotation local EdgeJournalNum=$((JOURNALNUM + 1)) local StandbyJournalNum=$((JOURNALNUM + 2)) # If using journalcopy, have to add 2 since live journal is in checkpoints folder local JournalCmd= local JournalFile= if [[ "$EDGESERVER" -eq 1 ]]; then # Refer to ckp/jnl files starting like (example ServerID=p4d_edge_nyc): # /p4/N/checkpoints.edge_nyc/p4_N.edge_nyc JournalFile="${CHECKPOINTS}.${SERVERID#p4d_}/${P4SERVER}.${SERVERID#p4d_}.jnl.${EdgeJournalNum}" elif [[ "$STANDBYSERVER" -eq 1 ]]; then JournalFile="${CHECKPOINTS}/${P4SERVER}.jnl.${StandbyJournalNum}" fi if [[ "$SERVERID" != "$P4MASTER_ID" ]]; then [[ -f "$JournalFile" ]] && \ die "Journal $JournalFile already exists, check the backup process." log "Truncating journal on ${P4MASTERPORT}." # 'p4d -jj' does a copy-then-delete, instead of a simple mv. # During 'p4d -jj' the perforce server will hang the responses to clients, # this should be for a very short period of time even for large data # sets, as the journal represents a single day of metadata. # Curly braces capture output of 'time'. "$P4CBIN"/p4login -p "$P4MASTERPORT" JournalCmd="$P4BIN -p $P4MASTERPORT admin journal" log "Running: $JournalCmd" { time $JournalCmd; } >> "$LOGFILE" 2>&1 || { die "Journal rotation failed. Abort!"; } # The test below waits until the journal file exists in the checkpoints directory before proceeding. test=1 while [[ $test != 0 ]]; do sleep 5 if [[ -f "$JournalFile" ]]; then test=0 fi done "$P4CBIN"/p4login -service else log "Warning: truncate_journal_on_master () function should not be called on a master server. Ignoring." fi } #------------------------------------------------------------------------------ # Similar to truncate_journal() above, p4d_truncate_journal() is intended to be # usable form the p4d_base init script, to allow journal rotation just before # p4d is started. As it may be called from the init script, it may be called # on the master, a replica, or an edge. However, it will only do the journal # rotation if called on the master. #------------------------------------------------------------------------------ function p4d_truncate_journal () { local JournalFile="${CHECKPOINTS}/${P4SERVER}.jnl.${JOURNALNUM}" if [[ "$SERVERID" == "$P4MASTER_ID" ]]; then [[ -f "$JournalFile" ]] && \ die "Journal $JournalFile already exists, check the backup process." log "Rotating journal prior to starting p4d." "$P4DBIN" -r "$P4ROOT" -J "$P4JOURNAL" -jj >> "$LOGFILE" 2>&1 ||\ die "Failed to rotate journal. Aborting p4d server start." else log "Warning: The p4d_truncate_journal() function has no effect if called on a server other than the master. Ignoring." fi } #------------------------------------------------------------------------------ # Replay any and all numbered journal files into the offline databases. #------------------------------------------------------------------------------ function replay_journals_to_offline_db () { local checkpointCmd= local checkpointsDir= local filePrefix= local numberedJournal= local journalPrefix= local useTargetJournalPrefix=${1:-0} log "Replay any unreplayed journals to the offline database." check_offline_db_usable # For the master server, we can safely rely on the SDP standard that the # journalPrefix is of the form '/p4/N/checkpoints/p4_N'. For replicas and # edge servers, determine the values dynamically based on the current journal # Prefix value for the given ServerID. if [[ "$SERVERID" == "$P4MASTER_ID" ]]; then # Refer to ckp/jnl files starting like: /p4/N/checkpoints/p4_N checkpointsDir="${CHECKPOINTS}" filePrefix="${P4SERVER}" else if [[ "$useTargetJournalPrefix" -eq 1 ]]; then journalPrefix=$(get_target_config_value journalPrefix) else if [[ -r "$P4ROOT/db.config" ]]; then copy_jd_table "db.config" "$P4ROOT" journalPrefix="$("$P4DBIN" -r "$JDTmpDir" -v track=-1 -k db.config -jd - | grep "@${SERVERID}@ @journalPrefix@" | cut -d '@' -f 10)" remove_jd_tables fi fi if [[ -n "$journalPrefix" ]]; then checkpointsDir="${journalPrefix%/*}" filePrefix="${journalPrefix##*/}" else log "Warning: Could not determine journalPrefix for ServerID $SERVERID." checkpointsDir="${CHECKPOINTS}" filePrefix="${P4SERVER}" fi fi for (( j=OFFLINEJNLNUM; j <= JOURNALNUM; j++ )); do numberedJournal="${checkpointsDir}/${filePrefix}.jnl.${j}" log "Replay journal $numberedJournal to offline db." rm -f "${OFFLINE_DB}/offline_db_usable.txt" >> "$LOGFILE" 2>&1 # Curly braces capture output of 'time'. checkpointCmd="$P4DBIN -r $OFFLINE_DB -v track=-1 -jr -f $numberedJournal" log "Running: $checkpointCmd" { time $checkpointCmd; } >> "$LOGFILE" 2>&1 || { die "Offline journal replay failed. Abort!"; } echo "Offline db file restored successfully." > "${OFFLINE_DB}/offline_db_usable.txt" done } #------------------------------------------------------------------------------ # Replay the live, active P4JOURNAL file into the offline database. #------------------------------------------------------------------------------ function replay_active_journal_to_offline_db () { log "Replay active journal to offline db." local ActiveJournal= local CheckpointCmd= # On a standby server, the current/active journal is named /p4/N/logs/journal.. # On the master and other server types, the active journal is $P4JOURNAL. if [[ "$STANDBYSERVER" -eq 1 ]]; then local _JNLNUM if [[ -r "$P4ROOT/db.counters" ]]; then copy_jd_table "db.counters" "$P4ROOT" _JNLNUM=$("$P4DBIN" -r "$JDTmpDir" -v track=-1 -jd - db.counters 2>&1 | grep '@journal@' | cut -d "@" -f 8 2>> "$LOGFILE") ||\ die "Cannot get $P4ROOT journal number. Abort!" remove_jd_tables ActiveJournal="$LOGS/journal.$_JNLNUM" else die "Cannot get $P4ROOT journal number. No db.counters found in $P4ROOT. Abort!" fi else ActiveJournal="$P4JOURNAL" fi # Curly braces capture output of 'time'. CheckpointCmd="$P4DBIN -r $OFFLINE_DB -v track=-1 -jr -f $ActiveJournal" log "Running: $CheckpointCmd" { time $CheckpointCmd; } >> "$LOGFILE" 2>&1 || { die "Active Journal replay failed. Abort!"; } } #------------------------------------------------------------------------------ # Function: get_target_config_value($configName) # # Get a 'p4 configure' configuration value from the P4TARGET server. # # Example: # # targetJournalPrefix=$(get_target_config_value journalPrefix) #------------------------------------------------------------------------------ function get_target_config_value() { local configName=${1:-} local configValue= local targetPort= local grepCmd= if [[ -r "$P4ROOT/db.config" ]]; then copy_jd_table "db.config" "$P4ROOT" # Prepare grep to be case sensitive or insensitive based on SDP instance # configuration. IsP4DCaseSensitive is set in set_vars(). # Sensitive data can have case-varied server specs, but exactly one will # match grep without the -i. Insensitive data can have only one server # spec. grepCmd="$GREP" [[ "${IsP4DCaseSensitive:-0}" -eq 0 ]] && grepCmd+=" -i" [[ "${UseGrepM:-0}" -eq 1 ]] && grepCmd+=" -m 1" if [[ "${UseGrepM:-0}" -eq 1 ]]; then targetPort=$("$P4DBIN" -r "$JDTmpDir" -cshow | $grepCmd "${SERVERID}: P4TARGET" | cut -d ' ' -f 4) else targetPort=$("$P4DBIN" -r "$JDTmpDir" -cshow | $grepCmd "${SERVERID}: P4TARGET" | head -1 | cut -d ' ' -f 4) fi remove_jd_tables fi if [[ -n "$targetPort" ]]; then configValue=$("$P4BIN" -p "$targetPort" configure show "$configName" 2>/dev/null | head -1 | sed -E -e 's/^.*=//g' -e 's/ .*$//g') fi echo "$configValue" } #------------------------------------------------------------------------------ # Function: get_latest_checkpoint_with_md5 () # # This function returns the most recent available checkpoint for which a *.md5 # file exists, based on timestamps on the .md5 files. The md5 files are # generated by p4d upon successful completion of the checkpoint. # # The checkpoint can be a file or a directory (for parallel checkpoints), and # may be compressed or uncompressed. # # This function reads several global variables. It sets the global # LoadParallelCheckpoint=1 if the checkpoint is a directory (requiring # parallel replay options), or 0 if a file (for serial replay) # It sets the global CHECKPOINT to the the path to the checkpoint file. # # The function returns a 0 (happy) if a checkpoint is found (a checkpoint file # or directory), or 1 (unhappy) otherwise. #------------------------------------------------------------------------------ function get_latest_checkpoint_with_md5 () { local useTargetJournalPrefix=${1:-0} local checkpointsDir= local filePrefix= local journalPrefix= local lastCheckpointMD5= local lastCheckpoint= local tmpFile= local journalCounter=0 local highJournalCounter=0 # For the master server, we can safely rely on the SDP standard that the journalPrefix # is of the form '/p4/N/checkpoints/p4_N'. For replicas and edge servers, determine # the values dynamically based on the current journalPrefix value for the given ServerID. if [[ "$SERVERID" == "$P4MASTER_ID" ]]; then # Refer to ckp/jnl files starting like: /p4/N/checkpoints/p4_N checkpointsDir="${CHECKPOINTS}" filePrefix="${P4SERVER}" else if [[ "$useTargetJournalPrefix" -eq 1 ]]; then journalPrefix=$(get_target_config_value journalPrefix) else if [[ -r "$P4ROOT/db.config" ]]; then copy_jd_table "db.config" "$P4ROOT" journalPrefix="$("$P4DBIN" -r "$JDTmpDir" -v track=-1 -k db.config -jd - | grep "@${SERVERID}@ @journalPrefix@" | cut -d '@' -f 10)" remove_jd_tables fi fi if [[ -n "$journalPrefix" ]]; then checkpointsDir="${journalPrefix%/*}" filePrefix="${journalPrefix##*/}" else log "Warning: Could not determine journalPrefix for ServerID $SERVERID." checkpointsDir="${CHECKPOINTS}" filePrefix="${P4SERVER}" fi fi # Make sure we have at least one checkpoint available, which may be a file or # directory. Search for the most recent among the *.md5 and *.OK files. # # Set lastCheckpoint to the path of a the most recent valid checkpoint, or # set it to empty to indicate no checkpoint is available. tmpFile=$(mktemp "$P4TMP/md5_files.XXXXXXXX") log "Seeking highest journal counter among checkpoint *.md5 files with prefix $checkpointsDir/$filePrefix" log "Running: find \"$checkpointsDir/\" -mindepth 1 -maxdepth 1 | grep -E \"^${checkpointsDir}/${filePrefix}\.ckp\.[1-9]{1}[0-9]*(\.gz|)\.(md5|OK)\$\" \"$tmpFile\"" find "$checkpointsDir/" -mindepth 1 -maxdepth 1 | grep -E "^${checkpointsDir}/${filePrefix}\.ckp\.[1-9]{1}[0-9]*(\.gz|)\.(md5|OK)$" > "$tmpFile" [[ -n "${SDP_DEBUG:-}" ]] && log "Contents of tmpFile[$tmpFile]:\\n$(cat "$tmpFile")" while read -r md5File; do journalCounter="${md5File##*.ckp.}" journalCounter="${journalCounter%%.*}" if [[ "$journalCounter" =~ ^[0-9]+$ ]]; then [[ "$journalCounter" -gt "$highJournalCounter" ]] && highJournalCounter="$journalCounter" [[ "$journalCounter" -eq "$highJournalCounter" ]] && lastCheckpointMD5="$md5File" else log "Error: Invalid journal counter [$journalCounter] extracted from file name: $md5File" fi done < "$tmpFile" [[ -n "${SDP_DEBUG:-}" ]] && log "DEBUG: lastCheckpointMD5=$lastCheckpointMD5" [[ -n "${SDP_DEBUG:-}" ]] || rm -f "$tmpFile" if [[ -n "$lastCheckpointMD5" ]]; then # Account for the idiosyncrasy that MD5 files for checkpoints may look # like p4_N.ckp.gz.md5, p4_N.ckp.md5, or p4_N.ckp.OK. if [[ "$lastCheckpointMD5" == *.gz.md5 ]]; then lastCheckpoint="${lastCheckpointMD5%.md5}" LoadParallelCheckpoint=0 if [[ ! -r "$lastCheckpoint" ]]; then if [[ -f "${lastCheckpointMD5%.gz.md5}" ]]; then lastCheckpoint="${lastCheckpointMD5%.gz.md5}" log "Warning: Missing last checkpoint [${lastCheckpointMD5%.md5}], which is expected to exist because this exists: [$lastCheckpointMD5]. However, an uncompressed checkpoint [$lastCheckpoint] was found, so using that instead." else log "Error: Missing last checkpoint [$lastCheckpoint], which is expected to exist because this exists: $lastCheckpointMD5" lastCheckpoint= fi fi elif [[ "$lastCheckpointMD5" == *.md5 ]]; then if [[ -r "${lastCheckpointMD5%.md5}" ]]; then lastCheckpoint="${lastCheckpointMD5%.md5}" if [[ -d "$lastCheckpoint" ]]; then LoadParallelCheckpoint=1 else LoadParallelCheckpoint=0 fi elif [[ -r "${lastCheckpointMD5%.md5}.gz" ]]; then lastCheckpoint="${lastCheckpointMD5%.md5}.gz" LoadParallelCheckpoint=0 else log "Error: Missing last checkpoint [${lastCheckpointMD5%.md5}.gz] or [${lastCheckpoint%.md5}], one of which is expected to exist because this exists: $lastCheckpointMD5" lastCheckpoint= fi elif [[ "$lastCheckpointMD5" == *.OK ]]; then lastCheckpoint="${lastCheckpointMD5%.OK}" LoadParallelCheckpoint=1 if [[ ! -r "$lastCheckpoint" ]]; then log "Error: Missing last checkpoint [$lastCheckpoint], which is expected to exist because this exists: $lastCheckpointMD5" lastCheckpoint= fi else log "Error: Could not determine checkpoint file from last checkpoint MD5 file [$lastCheckpointMD5]." fi else if [[ "$SERVERID" == "$P4MASTER_ID" ]]; then log "Error: No checkpoints found while scanning for checkpoint files or parallel checkpoint directories, compressed or uncompressed, with a journalPrefix of '$checkpointsDir/$filePrefix'. Consider creating a live checkpoint here on the commit server, which will cause a period of server unresponsiveness, like so:\\n\\tnohup live_checkpoint.sh $SDP_INSTANCE < /dev/null > /dev/null 2>&1 &\\n\\ttail -f $LOGS/checkpoint.log\\n" else log "Error: No checkpoints found while scanning for checkpoint files or parallel checkpoint directories, compressed or uncompressed, with a journalPrefix of '$checkpointsDir/$filePrefix'.\\n" fi LoadParallelCheckpoint="$CreateParallelCheckpoint" fi # The primary job of this function is to set a value for CHECKPOINTS, and # return happy (0) if a valid value was set, or unhappy (1) otherwise. export CHECKPOINT="$lastCheckpoint" [[ -z "$CHECKPOINT" ]] && return 1 return 0 } #------------------------------------------------------------------------------ # Recreate offline databases from the latest checkpoint. #------------------------------------------------------------------------------ # shellcheck disable=SC2120 function recreate_offline_db_files () { local checkpointCmd= local checkpointsDir= local useTargetJournalPrefix=${1:-0} if [[ "$CheckpointDumpSkipped" -eq 1 ]]; then log "Skipping recreation of offline_db because the checkpoint dump was skipped." return 0 fi get_latest_checkpoint_with_md5 "$useTargetJournalPrefix" ||\ die "Error: Aborting due to unavailable checkpoint." # At this point we know we have a valid checkpoint to load, # referenced by $LastCheckpoint, and we know that # $LoadParallelCheckpoint is correctly set based on the # latest checkpoint found above. { rm -f "${OFFLINE_DB}"/offline_db_usable.txt rm -f "${OFFLINE_DB}"/db.* rm -f "${OFFLINE_DB}"/save/db.* } >> "$LOGFILE" 2>&1 if [[ "$LoadParallelCheckpoint" -eq 1 ]]; then log "Recovering from parallel checkpoint directory: $CHECKPOINT" checkpointCmd="$P4DBIN -r $OFFLINE_DB -v track=-1 -z -N ${Threads:-4} -jrp $CHECKPOINT" else log "Recovering from serial checkpoint file: $CHECKPOINT" checkpointCmd="$P4DBIN -r $OFFLINE_DB -v track=-1 -jr $CHECKPOINT" fi log "Running: $checkpointCmd" # Curly braces capture output of 'time'. { time $checkpointCmd; } >> "$LOGFILE" 2>&1 || { die "Restore of checkpoint to $OFFLINE_DB failed!"; } echo "Offline db file restored successfully." > "${OFFLINE_DB}/offline_db_usable.txt" } #------------------------------------------------------------------------------ # Take a live checkpoint from db.* files in P4ROOT. #------------------------------------------------------------------------------ function checkpoint () { local checkpointCmd= local checkpointsDir= local filePrefix= local parallelCheckpointDir= local parallelCheckpointOKFile= local journalPrefix= local checkpointsDir= local filePrefix= local expectedMD5File= local -i isCommitServer=0 log "Create a new checkpoint from live db files in $P4ROOT." if [[ "$SERVERID" == "$P4MASTER_ID" ]]; then # Refer to ckp/jnl files starting like: /p4/N/checkpoints/p4_N checkpointsDir="${CHECKPOINTS}" filePrefix="${P4SERVER}" isCommitServer=1 else isCommitServer=0 fi if [[ "$isCommitServer" -eq 1 ]]; then if [[ "$CreateParallelCheckpoint" -eq 1 ]]; then parallelCheckpointDir="${checkpointsDir}/${filePrefix}.ckp.${CHECKPOINTNUM}" parallelCheckpointOKFile="${parallelCheckpointDir}.OK" if [[ "$CreateMultifileParallelCheckpoint" -eq 1 ]]; then checkpointCmd="$P4DBIN -r $P4ROOT -v track=-1 -Z -N ${Threads:-4} -jcpm ${checkpointsDir}/${filePrefix}" else checkpointCmd="$P4DBIN -r $P4ROOT -v track=-1 -Z -N ${Threads:-4} -jcp ${checkpointsDir}/${filePrefix}" fi else checkpointCmd="$P4DBIN -r $P4ROOT -v track=-1 -jc -Z ${checkpointsDir}/${filePrefix}" fi log "Running: $checkpointCmd" # Curly braces capture output of 'time'. { time $checkpointCmd; } >> "$LOGFILE" 2>&1 || { die "ERROR - New live checkpoint failed!"; } if [[ "$CreateParallelCheckpoint" -eq 1 ]]; then touch "$parallelCheckpointOKFile" log "Live parallel checkpoint completed OK. Writing: $parallelCheckpointOKFile" else log "Live checkpoint completed OK." fi else if [[ -r "$P4ROOT/db.config" ]]; then copy_jd_table "db.config" "$P4ROOT" journalPrefix="$("$P4DBIN" -r "$JDTmpDir" -k db.config -v track=-1 -jd - | grep "@${SERVERID}@ @journalPrefix@" | cut -d '@' -f 10)" remove_jd_tables fi if [[ -n "$journalPrefix" ]]; then checkpointsDir="${journalPrefix%/*}" filePrefix="${journalPrefix##*/}" else die "Could not determine journalPrefix for ServerID $SERVERID." fi log "Starting live checkpoint process for ServerID $SERVERID with journal $((JOURNALNUM+2))." log "Calling: $P4CBIN/request_replica_checkpoint.sh $SDP_INSTANCE -now" "$P4CBIN"/request_replica_checkpoint.sh "$SDP_INSTANCE" -now expectedMD5File="$checkpointsDir/${filePrefix}.ckp.$((JOURNALNUM+2)).md5" log "Waiting for MD5 to appear for server $SERVERID: $expectedMD5File" # Wait until the *.md5 file appears before proceeding. while true; do sleep 5 [[ -f "$expectedMD5File" ]] && break done "$P4CBIN"/p4login # shellcheck disable=SC2119 recreate_offline_db_files fi } #------------------------------------------------------------------------------A # Take a checkpoint from the ROOTDIR, typically either /p4/N/root or # /p4/N/offline_db. #------------------------------------------------------------------------------ function dump_checkpoint () { local CheckpointCmd= local CheckpointsDir= local NewCheckpointFile= local NewCheckpointDir= local NewCheckpointMD5= local FilePrefix= local JournalPrefix= local -i DoSnapshot=0 local -i SnapshotOK=1 local -i CheckpointOK=1 local ParallelCheckpointDir= local ParallelCheckpointOKFile= # shellcheck disable=SC2153 log "Dump out new checkpoint from db files in $ROOTDIR." # For the master server, we can safely rely on the SDP standard that the journalPrefix # is of the form '/p4/N/checkpoints/p4_N'. For replicas and edge servers, determine # the values dynamically based on the current journalPrefix value for the given ServerID. if [[ "$SERVERID" == "$P4MASTER_ID" ]]; then # Refer to ckp/jnl files starting like: /p4/N/checkpoints/p4_N CheckpointsDir="${CHECKPOINTS}" FilePrefix="${P4SERVER}" else if [[ -r "$P4ROOT/db.config" ]]; then copy_jd_table "db.config" "$P4ROOT" JournalPrefix="$("$P4DBIN" -r "$JDTmpDir" -v track=-1 -k db.config -jd - | grep "@${SERVERID}@ @journalPrefix@" | cut -d '@' -f 10)" remove_jd_tables fi if [[ -n "$JournalPrefix" ]]; then CheckpointsDir="${JournalPrefix%/*}" FilePrefix="${JournalPrefix##*/}" else log "Warning: Could not determine journalPrefix for ServerID $SERVERID." CheckpointsDir="${CHECKPOINTS}" FilePrefix="${P4SERVER}" fi fi if [[ "$CreateParallelCheckpoint" -eq 1 ]]; then NewCheckpointDir="${CheckpointsDir}/${FilePrefix}.ckp.${CHECKPOINTNUM}" NewCheckpointMD5="${NewCheckpointDir}.md5" else NewCheckpointFile="${CheckpointsDir}/${FilePrefix}.ckp.${CHECKPOINTNUM}.gz" NewCheckpointMD5="${NewCheckpointFile}.md5" fi log "Ensuring MD5 file $NewCheckpointMD5 does not already exist." if [[ -r "$NewCheckpointMD5" ]]; then if [[ -r "$NewCheckpointFile" ]] ; then log "Warning: Skipping this checkpoint dump due to existing checkpoint file $NewCheckpointFile. Verified MD5 file exists: $NewCheckpointMD5. Checkpoints should continue after next journal rotation." CheckpointDumpSkipped=1 return elif [[ -d "$NewCheckpointDir" ]]; then log "Warning: Skipping this checkpoint dump due to existing checkpoint dir $NewCheckpointDir. Verified MD5 file exists: $NewCheckpointMD5. Checkpoints should continue after next journal rotation." CheckpointDumpSkipped=1 return else if [[ "$CreateParallelCheckpoint" -eq 1 ]]; then die "Found file $NewCheckpointMD5, but did not see expected checkpoint file $NewCheckpointFile. Aborting checkpoint dump." else die "Found file $NewCheckpointMD5, but did not see expected checkpoint dir $NewCheckpointDir. Aborting checkpoint dump." fi fi # If the MD5 file exists but checkpoint does not in file or directory form, abort. elif [[ -r "$NewCheckpointFile" ]]; then die "Aborting because checkpoint file $NewCheckpointFile exists but $NewCheckpointMD5 does not." elif [[ -d "$NewCheckpointDir" ]]; then die "Aborting because checkpoint dir $NewCheckpointDir exists but $NewCheckpointMD5 does not." fi if [[ "$CreateParallelCheckpoint" -eq 1 ]]; then ParallelCheckpointDir="${CheckpointsDir}/${FilePrefix}.ckp.${CHECKPOINTNUM}" ParallelCheckpointOKFile="${ParallelCheckpointDir}.OK" if [[ "$CreateMultifileParallelCheckpoint" -eq 1 ]]; then CheckpointCmd="$P4DBIN -r $ROOTDIR -v track=-1 -z -N ${Threads:-4} -jdpm $ParallelCheckpointDir" else CheckpointCmd="$P4DBIN -r $ROOTDIR -v track=-1 -z -N ${Threads:-4} -jdp $ParallelCheckpointDir" fi else CheckpointCmd="$P4DBIN -r $ROOTDIR -v track=-1 -jd -z ${CheckpointsDir}/${FilePrefix}.ckp.${CHECKPOINTNUM}.gz" fi log "Running: $CheckpointCmd" # Curly braces capture output of 'time'. if { time $CheckpointCmd; } >> "$LOGFILE" 2>&1; then CheckpointOK=1 if [[ "$CreateParallelCheckpoint" -eq 1 ]]; then log "Parallel checkpoint dump completed OK. Writing: $ParallelCheckpointOKFile" touch "$ParallelCheckpointOKFile" else log "Checkpoint dump completed OK." fi else CheckpointOK=0 fi if [[ -n "${SNAPSHOT_SCRIPT:-}" ]]; then DoSnapshot=1 log "Calling site-specific snapshot script: $SNAPSHOT_SCRIPT" if "$SNAPSHOT_SCRIPT" >> "$LOGFILE" 2>&1; then SnapshotOK=1 else SnapshotOK=0 fi fi if [[ "$DoSnapshot" -eq 0 ]]; then if [[ "$CheckpointOK" -eq 1 ]]; then log "New checkpoint dump succeeded." else die "New checkpoint dump FAILED." fi else if [[ "$CheckpointOK" -eq 0 && "$SnapshotOK" -eq 0 ]]; then die "Both checkpoint dump and snapshot FAILED." elif [[ "$CheckpointOK" -eq 1 && "$SnapshotOK" -eq 0 ]]; then die "New checkpoint dump succeeded, but snapshot FAILED." elif [[ "$CheckpointOK" -eq 0 && "$SnapshotOK" -eq 1 ]]; then die "New checkpoint dump FAILED, but snapshot succeeded." else log "New checkpoint dump and snapshot succeeded." fi fi } #------------------------------------------------------------------------------ # Compare journal numbers between live and offline databases, to ensure # they can be safely swapped out. #------------------------------------------------------------------------------ function compare_journal_numbers () { local _OFFLINEJNLNUM if [[ -r "$OFFLINE_DB/db.counters" ]]; then copy_jd_table "db.counters" "$OFFLINE_DB" else die "Cannot get $OFFLINE_DB journal number. No db.counters found in $OFFLINE_DB. Abort!" fi _OFFLINEJNLNUM=$("$P4DBIN" -r "$JDTmpDir" -v track=-1 -jd - db.counters 2>&1 | grep '@journal@' | cut -d "@" -f 8 2>> "$LOGFILE") ||\ die "Cannot get $OFFLINE_DB journal number. Abort!" remove_jd_tables check_journalnum "$_OFFLINEJNLNUM" # Get the journal number of the root database if [[ ! -f "$P4ROOT/db.counters" ]]; then die "$P4ROOT database not found. Something is seriously wrong since the server was just running a minute ago! Contact support-p4@perforce.com" fi local _JNLNUM if [[ -r "$P4ROOT/db.counters" ]]; then copy_jd_table "db.counters" "$P4ROOT" _JNLNUM=$("$P4DBIN" -r "$JDTmpDir" -v track=-1 -jd - db.counters 2>&1 | grep '@journal@' | cut -d "@" -f 8 2>> "$LOGFILE") ||\ die "Cannot get $P4ROOT journal number. Abort!" remove_jd_tables else die "Cannot get $P4ROOT journal number. No db.counters found in $P4ROOT. Abort!" fi check_journalnum "$_JNLNUM" if [[ "$_JNLNUM" -gt "$_OFFLINEJNLNUM" ]]; then log "$P4ROOT journal number is: $_JNLNUM" log "$OFFLINE_DB journal number is: $_OFFLINEJNLNUM" die "$OFFLINE_DB journal number is less than $P4ROOT, cannot switch." fi } #------------------------------------------------------------------------------ # Function: rsync_with_preflight ($src, $path, $rsyncOpts, $safetyThresholdKB) function rsync_with_preflight () { local srcPath=${1:-} local tgtPath=${2:-} local rsyncOptions=${3:--a} local safetyThresholdKB=${4:-1048576} local rsyncCmd= local rsyncDryRunCmd= local rsyncTransferSize= local tgtAvail= local tgtFs= local remainingSpace= [[ -n "$srcPath" && -n "$tgtPath" ]] || return 1 rsyncCmd="rsync $rsyncOptions \"$srcPath/\" \"$tgtPath\"" rsyncDryRunCmd="rsync $rsyncOptions --dry-run --stats \"$srcPath/\" \"$tgtPath\"" log "Planned rsync: $rsyncCmd" # Get source directory size in KB rsyncTransferSize=$(eval "$rsyncDryRunCmd" | awk '/Total transferred file size:/ {print $5}') log "Pre-rsync safety check: Disk space required for rsync is $rsyncTransferSize KB." # Get available space on target filesystem in KB. tgtFs=$(df -k --output=avail "${tgtPath%/*}" | tail -1) tgtAvail=$(echo "$tgtFs" | tr -d '[:space:]') log "Pre-rsync safety check: Available space in ${tgtPath%/*} is $tgtAvail KB." # Calculate remaining space after rsync remainingSpace=$((tgtAvail - rsyncTransferSize)) # Check against safety threshold log "Pre-rsync safety check results: Required to transfer: $rsyncTransferSize KB, Available: $tgtAvail KB, Safety Threshold: $safetyThresholdKB KB." if (( remainingSpace < safetyThresholdKB )); then log "Error: Not enough disk space to safely rsync to $tgtPath. Aborting rsync operation." return 1 else log "Disk space sufficient. Proceeding with rsync." fi log "Proceeding with this command: $rsyncCmd" if eval "$rsyncCmd" >> "$LOGFILE" 2>&1; then log "The rsync completed OK." else log "The rsync failed to copy." return 1 fi return 0 } #------------------------------------------------------------------------------ # Function: copy_readonly_clients_dir() # # Duplicate the data for partitioned clients. This is part of the procedure to # swap P4ROOT and offline_db. It must be called when the p4d service is stopped, # and before the call to switch_db_files(), so the P4ROOT still points at the # live/current data. #------------------------------------------------------------------------------ function copy_readonly_clients_dir() { local tmpFile= local srcPath= local tgtPath= #local safetyThresholdKB=10485760 # 10 GB in KB (adjust as needed) #local safetyThresholdKB=5242880 # 5G in KB local safetyThresholdKB=1048576 # 1G in KB log "Checking to see if client.readonly.dir is set." tmpFile=$(mktemp) "$P4DBIN" -r "$P4ROOT" -cshow > "$tmpFile" 2>&1 if grep -q "$SERVERID: client.readonly.dir = " "$tmpFile"; then srcPath=$(grep "$SERVERID: client.readonly.dir = " "$tmpFile" | cut -d= -f2) srcPath=${srcPath# } elif grep -q "any: client.readonly.dir = " "$tmpFile"; then srcPath=$(grep "any: client.readonly.dir = " "$tmpFile" | cut -d= -f2) srcPath=${srcPath# } fi rm -f "$tmpFile" >> "$LOGFILE" 2>&1 if [[ -n "$srcPath" ]]; then log "The client.readonly.dir value is set to: $srcPath" # If the client.readonly.dir value looks like "/p4/N/root/", # the copy target path is determined by changing /p4/N/root to /p4/N/offline_db. if [[ "$srcPath" == "$P4ROOT/"* ]]; then tgtPath="$OFFLINE_DB/$(echo "$srcPath" | sed -E -e "s@$P4ROOT/@@g")" # If the clientReadonly.Dir path is an absolute path but not under # /p4/N/root, we don't need to handle it, as its location is outside # SDP control. elif [[ "$srcPath" == /* ]]; then log "The client.readonly.dir [$srcPath] is an absolute path outside P4ROOT, so copying is not needed. Skipping." tgtPath= elif [[ -L "$srcPath" ]]; then log "The client.readonly.dir [$srcPath] targets a symlink, so copying is not needed. Skipping." tgtPath= # If the client.readonly.dir value is a relative path to P4ROOT, # check to see if that directory exists under P4ROOT. If so, # set srcPath to that directory, and set tgtPath to the # the same directory under /p4/N/offline_db. else if [[ -d "$P4ROOT/$srcPath" ]]; then tgtPath="$OFFLINE_DB/$srcPath" srcPath="$P4ROOT/$srcPath" else log "The client.readonly.dir [$srcPath] is a relative path under P4ROOT, but $P4ROOT/$srcPath does not exist, so copying is not needed. Skipping." tgtPath= fi fi if [[ -n "$tgtPath" ]]; then if rsync_with_preflight "$srcPath" "$tgtPath" "-a --delete --exclude='db.sendq*' --exclude='*.lck' --exclude='*.tmp'" "$safetyThresholdKB"; then log "The client.readonly.dir copied OK." else log "Error: Failed to copy the client.readonly.dir." return 1 fi fi else log "The client.readonly.dir is not set, so no need to copy it." fi return 0 } #------------------------------------------------------------------------------ # Swap out live db.* database files in P4ROOT with those in offline_db. #------------------------------------------------------------------------------ function switch_db_files () { local verifyCmd="$P4CBIN/verify_sdp.sh -skip crontab,excess,init,license,masterid,version -L off" log "Calling 'verify_sdp.sh' before swapping db.* files:\\n$verifyCmd" $verifyCmd >> "$LOGFILE" 2>&1 ||\ die "Error: Cannot confirm all is well with $P4CBIN/verify_sdp.sh. Aborting" # Compare the Offline and Master journal numbers before switching to make # sure they match. compare_journal_numbers log "Preparing for P4ROOT/offline_db symlink swap." [[ -d "${P4ROOT}"/save ]] || mkdir -p "${P4ROOT}"/save >> "$LOGFILE" 2>&1 # shellcheck disable=SC2129 echo "P4ROOT is not available during switch_db_files() processing." > "$P4ROOT/P4ROOT_not_usable.txt" 2>> "$LOGFILE" echo "P4ROOT is not available during switch_db_files() processing." > "$OFFLINE_DB/P4ROOT_not_usable.txt" 2>> "$LOGFILE" # shellcheck disable=SC2129 rm -f "${P4ROOT}"/save/db.* >> "$LOGFILE" 2>&1 rm -rf "${P4ROOT}"/server.locks >> "$LOGFILE" 2>&1 rm -rf "${OFFLINE_DB}"/server.locks >> "$LOGFILE" 2>&1 mv -f "${P4ROOT}"/db.* "${P4ROOT}"/save/. >> "$LOGFILE" 2>&1 if [[ -r "$P4ROOT"/license ]]; then mv -f "${P4ROOT}"/license "${OFFLINE_DB}/." >> "$LOGFILE" 2>&1 fi if [[ -n "$(ls "$P4ROOT"/license* 2>/dev/null)" ]]; then mv -f "${P4ROOT}"/license* "${OFFLINE_DB}/." >> "$LOGFILE" 2>&1 fi if [[ -r "${P4ROOT}"/rdb.lbr ]]; then mv -f "${P4ROOT}"/rdb.lbr "${OFFLINE_DB}/." >> "$LOGFILE" 2>&1 fi if [[ -n "$(ls "$P4ROOT"/state* 2>/dev/null)" ]]; then mv -f "${P4ROOT}"/state* "${OFFLINE_DB}/." >> "$LOGFILE" 2>&1 fi if [[ -r "${P4ROOT}"/server.id ]]; then mv -f "${P4ROOT}"/server.id "${OFFLINE_DB}/." >> "$LOGFILE" 2>&1 fi if [[ -n "$(ls "$P4ROOT"/server.id* 2>/dev/null)" ]]; then mv -f "${P4ROOT}"/server.id* "${OFFLINE_DB}/." >> "$LOGFILE" 2>&1 fi rm -f "${OFFLINE_DB}/offline_db_usable.txt" >> "$LOGFILE" 2>&1 log "Switching root and offline_db symlinks." LinkOfflineDB="$(readlink "$OFFLINE_DB")" LinkP4ROOT="$(readlink "$P4ROOT")" unlink "$OFFLINE_DB" unlink "$P4ROOT" ln -s "$LinkOfflineDB" "$P4ROOT" >> "$LOGFILE" 2>&1 ||\ die "Link of $LinkOfflineDB to $P4ROOT failed." ln -s "$LinkP4ROOT" "$OFFLINE_DB" >> "$LOGFILE" 2>&1 ||\ die "Link of $LinkP4ROOT to $OFFLINE_DB failed." rm -f "$P4ROOT/P4ROOT_not_usable.txt" >> "$LOGFILE" 2>&1 rm -f "$OFFLINE_DB/P4ROOT_not_usable.txt" >> "$LOGFILE" 2>&1 } #------------------------------------------------------------------------------ # Function: trim_log_file ($LogToTrim, $MaxLines) # # For log files expected to be short, keep them at a max size. # # When they get too big, trim them from the top first, so the most recent # output is retained. # # Thus function treats its processing as non-essential; most errors are # silently ignored and output discarded. Only error output related to replacing # the original log is retained in LOGFILE or displayed. #------------------------------------------------------------------------------ function trim_log_file () { local LogToTrim="${1:-}" local MaxLines="${2:-5000}" local TmpFile= local Lines= [[ -w "$LogToTrim" ]] || return # Abort if MaxLines isn't numeric. [[ "$MaxLines" =~ ^[0-9]+$ ]] || return TmpFile="${LogToTrim}.trimming.$$.$RANDOM" Lines=$(wc -l "$LogToTrim") Lines=${Lines%% *} # Confirm Lines is a number, else just abort. [[ "$Lines" =~ ^[0-9]+$ ]] || return # If the file isn't big enough to need trimming, abort. [[ "$Lines" -gt "$MaxLines" ]] || return log "Trimming log $LogToTrim from $Lines to $MaxLines lines." # If the trimming fails, discard output and just return. if tail -"$MaxLines" "$LogToTrim" > "$TmpFile" 2>/dev/null; then if [[ -n "${LOGFILE:-}" ]]; then mv -f "$TmpFile" "$LogToTrim" >> "$LOGFILE" 2>&1 else mv -f "$TmpFile" "$LogToTrim" fi else return fi } #------------------------------------------------------------------------------ # Rotate specified log files, and compress with gzip. #------------------------------------------------------------------------------ function rotate_log_file () { local LogToRotate="${1:-}" local GzExt="${2:-}" local -i i=1 local Datestamp= local RotatedLog= local RotatedZippedLog= [[ -n "$LogToRotate" ]] || return if [[ -n "${LOGFILE:-}" ]]; then pushd "$LOGS" > /dev/null 2>> "$LOGFILE" || die "Could not cd to: $LOGS" else pushd "$LOGS" > /dev/null || die "Could not cd to: $LOGS" fi Datestamp=$(date +'%Y-%m-%d_%H-%M-%S') RotatedLog="${LogToRotate}.${Datestamp}" if [[ -f "${LogToRotate}" ]]; then if [[ -n "${LOGFILE:-}" ]]; then mv -f "${LogToRotate}" "${RotatedLog}" >> "$LOGFILE" 2>&1 if [[ -n "$GzExt" ]]; then RotatedZippedLog="${RotatedLog}${GzExt}" # If needed, move existing zipped log aside. if [[ -e "$RotatedZippedLog" ]]; then while [[ -e "${LogToRotate}.${Datestamp}.${i}${GzExt}" ]]; do i+=1 done log "Moving pre-existing $RotatedZippedLog aside to ${LogToRotate}.${Datestamp}.${i}${GzExt}" >> "$LOGFILE" 2>&1 mv -f "$RotatedZippedLog" "${LogToRotate}.${Datestamp}.${i}${GzExt}" >> "$LOGFILE" 2>&1 fi gzip "$RotatedLog" >> "$LOGFILE" 2>&1 fi else mv -f "${LogToRotate}" "${RotatedLog}" if [[ -n "$GzExt" ]]; then RotatedZippedLog="${RotatedLog}${GzExt}" # If needed, move existing zipped log aside. if [[ -e "$RotatedZippedLog" ]]; then while [[ -e "${LogToRotate}.${Datestamp}.${i}${GzExt}" ]]; do i+=1 done log "Moving pre-existing $RotatedZippedLog aside to ${LogToRotate}.${Datestamp}.${i}${GzExt}" mv -f "$RotatedZippedLog" "${LogToRotate}.${Datestamp}.${i}${GzExt}" fi gzip "$RotatedLog" fi fi fi if [[ -n "${LOGFILE:-}" ]]; then popd > /dev/null 2>> "$LOGFILE" || die "Could not cd to: $OLDPWD" else popd > /dev/null || die "Could not cd to: $OLDPWD" fi } #------------------------------------------------------------------------------ # At the start of each run for live_checkpoint.sh, daily_checkpoint.sh, and # recreate_db_checkpoint.sh, before *any* logging activity occurs, rotate the # logs from the most recent prior run, always named "checkpoint.log" or "log". #------------------------------------------------------------------------------ function rotate_last_run_logs () { # Rotate prior log file for the current script. rotate_log_file "$LOGFILE" # Rotate prior server log. rotate_log_file "log" ".gz" # Rotate prior broker log. rotate_log_file "p4broker.log" ".gz" # Rotate prior audit log. rotate_log_file "audit.log" ".gz" # Rotate prior monitor metrics log. rotate_log_file "monitor_metrics.log" ".gz" # Rotate prior p4triggers log. rotate_log_file "p4triggers.log" ".gz" # Rotate prior p4login log. rotate_log_file "p4login.log" ".gz" # Additions for p4p.log, rotate_proxy.log, and p4pcm.log rotate_log_file "p4p.log" ".gz" rotate_log_file "rotate_proxy.log" ".gz" rotate_log_file "p4pcm.log" ".gz" } #------------------------------------------------------------------------------ # Remove log files matching a specified name prefix, preserving a specified # number of the recent logs. #------------------------------------------------------------------------------ function remove_log_files () { local removeLog=${1:-} local keepNum=${2:-} local log= [[ -n "${removeLog:-}" && -n "${keepNum:-}" ]] || return 1 # shellcheck disable=SC2012 for log in $(ls -t "${removeLog}"* 2>/dev/null | $AWK "NR > $keepNum"); do log "rm -f $log" rm -f "$log" done return 0 } #------------------------------------------------------------------------------ # Remove old SDP logs. #------------------------------------------------------------------------------ function remove_old_logs () { if [[ "$CheckpointDumpSkipped" -eq 1 ]]; then log "Skipping cleanup of old logs because the checkpoint dump was skipped." return 0 fi pushd "$LOGS" > /dev/null 2>> "$LOGFILE" || die "Could not cd to: $LOGS" # First remove checkpoint logs. For checkpoint logs specifically, use # KEEPJNLS rather than KEEPLOGS to determine how many to keep, so we keep # the same number of checkpoint logs as we keep checkpoints. if [[ "$KEEPJNLS" -eq 0 ]]; then log "Skipping cleanup of old checkpoint logs because KEEPJNLS is set to 0." else log "Deleting old checkpoint logs. Keeping latest $KEEPJNLS, per KEEPJNLS setting in p4_vars." remove_log_files "checkpoint.log" "$KEEPJNLS" fi # Next, use KEEPLOGS to determine how many other logs to cleanup. if [[ "$KEEPLOGS" -eq 0 ]]; then log "Skipping cleanup of old server logs because KEEPLOGS is set to 0." else log "Deleting old server logs. Keeping latest $KEEPLOGS, per KEEPLOGS setting in p4_vars." remove_log_files "log" "$KEEPLOGS" remove_log_files "p4broker.log" "$KEEPLOGS" remove_log_files "broker_rotate.log" "$KEEPLOGS" remove_log_files "audit.log" "$KEEPLOGS" remove_log_files "sync_replica.log" "$KEEPLOGS" remove_log_files "replica_status.log" "$KEEPLOGS" remove_log_files "replica_cleanup.log" "$KEEPLOGS" remove_log_files "request_checkpoint.log" "$KEEPLOGS" remove_log_files "recreate_offline_db.log" "$KEEPLOGS" remove_log_files "edge_shelf_replicate.log" "$KEEPLOGS" remove_log_files "upgrade.log" "$KEEPLOGS" remove_log_files "p4login" "$KEEPLOGS" remove_log_files "p4verify.log" "$KEEPLOGS" remove_log_files "journal_watch.log" "$KEEPLOGS" remove_log_files "refresh_P4ROOT_from_offline_db.log" "$KEEPLOGS" remove_log_files "purge_revisions.log" "$KEEPLOGS" remove_log_files "monitor_metrics.log" "$KEEPLOGS" remove_log_files "p4p.log" "$KEEPLOGS" remove_log_files "rotate_proxy.log" "$KEEPLOGS" remove_log_files "p4pcm.log" "$KEEPLOGS" fi popd > /dev/null 2>>"$LOGFILE" || die "Could not cd to: $OLDPWD" # Cleanup old/excess JDTmpDir find /tmp/ -mindepth 1 -maxdepth 1 -type d -name "tmp.*" -user "$OSUSER" -mtime +1 -exec rm -rf {} \; find "$P4TMP/" -mindepth 1 -maxdepth 1 -type d -name "tmp_jdtmpdir.*" -user "$OSUSER" -mtime +1 -exec rm -rf {} \; } #------------------------------------------------------------------------------ # Set the SDP Checkpoint counter to indicate last successful SDP checkpoint # operation for a given ServerID. #------------------------------------------------------------------------------ function set_counter() { local counterName="LastSDPCheckpoint.${SERVERID:-UnsetServerID}" local counterValue= "$P4CBIN"/p4login -p "$P4MASTERPORT" counterValue="$(date +'%s (%Y/%m/%d %H:%M:%S %z %Z)')" log "Setting checkpoint counter $counterName to value $counterValue." echo "$P4BIN -s -u $P4USER -p $P4MASTERPORT counter $counterName \"$counterValue\"" >> "$LOGFILE" "$P4BIN" -s -u "$P4USER" -p "$P4MASTERPORT" counter "$counterName" "$counterValue" >> "$LOGFILE" 2>&1 ||\ log "Warning: Unable to set checkpoint counter $counterName to value [$counterValue]." }