journal_watch.sh #5

#!/bin/bash
#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE
#------------------------------------------------------------------------------

# This script will check diskspace available to P4JOURNAL and trigger a journal
# rotation based on specified thresholds. This is useful in case you are in danger of
# running out of disk space and your rotated journal files are stored on a separate
# partition than the active journal.
#
# This script is using the following external variables:
#
# SDP_INSTANCE - The instance of Perforce that is being backed up. If not
# set in environment, pass in as argument to script.
#
# P4HOME - Server's home directory.
# P4BIN - Command line client name for the instance where disk space is monitored.
# P4DBIN - Server executable name for the instance where disk space is monitored.
# P4ROOT - Server's root directory. p4/root, p4_N/root
# P4PORT - TCP/IP port for the server instance where disk space is monitored.
# P4JOURNAL - Location of the Journal for the server where disk space is monitored.
# P4JOURNALWARN - Amount of space left (K,M,G,%) before min journal space where an email alert is sent
# P4JOURNALWARNALERT - Send an alert if warn threshold is reached (true/false, default: false)
# P4JOURNALROTATE - Amount of space left (K,M,G,%) before min journal space to trigger a journal rotation
# P4OVERRIDEKEEPJNL - Allow script to temporarily override KEEPJNL to retain enough journals to replay against oldest checkpoint (true/false, default: false)
#
# USAGE: ./journal_watch.sh <P4JOURNALWARN> <P4JOURNALWARNALERT> <P4JOURNALROTATE> <P4OVERRIDEKEEPJNL (Optional)>
#
# EXAMPLES:
#   Run from CLI that will warn via email if less than 20% is available and rotate journal when less than 10% is available
#      ./journal_watch.sh 20% TRUE 10% TRUE
#
#   Cron job that will warn via email if less than 20% is available and rotate journal when less than 10% is available
#      30 * * * * [ -e /p4/common/bin ] && /p4/common/bin/run_if_master.sh ${INSTANCE} /p4/common/bin/journal_watch.sh ${INSTANCE} 20\% TRUE 10\% TRUE
#
export SDP_INSTANCE=${SDP_INSTANCE:-Undefined}
export SDP_INSTANCE=${1:-$SDP_INSTANCE}
if [[ "$SDP_INSTANCE" == Undefined ]]; then
   echo "Instance parameter not supplied."
   echo "You must supply the Perforce instance as a parameter to this script."
   exit 1
fi

# If environment variable is defined, then use that as default value if cli arguments not supplied
export P4JOURNALWARN="${P4JOURNALWARN:-Undefined}"
export P4JOURNALWARN="${2:-$P4JOURNALWARN}"
export P4JOURNALWARNALERT="${P4JOURNALWARNALERT:-false}"
export P4JOURNALWARNALERT="${3:-$P4JOURNALWARNALERT}"
export P4JOURNALROTATE="${P4JOURNALROTATE:-Undefined}"
export P4JOURNALROTATE="${4:-$P4JOURNALROTATE}"
export P4OVERRIDEKEEPJNL="${P4OVERRIDEKEEPJNL:-false}"
export P4OVERRIDEKEEPJNL="${5:-$P4OVERRIDEKEEPJNL}"

# shellcheck disable=SC1091
source /p4/common/bin/p4_vars "$SDP_INSTANCE"
# shellcheck disable=SC1091
source /p4/common/bin/backup_functions.sh
LOGFILE="$LOGS/journal_watch.log"

######### Start of Script ##########

check_vars
set_vars
rotate_log_file "$LOGFILE"
log "Start $P4SERVER Journal Watch"
check_uid
check_dirs
"$P4CBIN/p4login"
get_ckpnum
get_journalnum
get_journal_stats

# Convert values to a common unit (bytes)
if [[ "$P4JOURNALMIN" =~ .*%$ ]]; then
   # If the min value contains a %, pass in total disk size to calculate bytes
   P4JOURNALMINBYTES=$(convert_to_bytes "$P4JOURNALMIN", "$P4JOURNALTOTAL")
else
   P4JOURNALMINBYTES=$(convert_to_bytes "$P4JOURNALMIN")
fi

if [[ "$P4JOURNALWARN" =~ .*%$ ]]; then
   # If the min value contains a %, pass in total disk size to calculate bytes
   P4JOURNALWARNBYTES=$(convert_to_bytes "$P4JOURNALWARN", "$P4JOURNALTOTAL")
else
   P4JOURNALWARNBYTES=$(convert_to_bytes "$P4JOURNALWARN")
fi

if [[ "$P4JOURNALROTATE" =~ .*%$ ]]; then
   # If the min value contains a %, pass in total disk size to calculate bytes
   P4JOURNALROTATEBYTES=$(convert_to_bytes "$P4JOURNALROTATE", "$P4JOURNALTOTAL")
else
   P4JOURNALROTATEBYTES=$(convert_to_bytes "$P4JOURNALROTATE")
fi

# Calculate available space for journal file (free space minus minimum required)
P4JOURNALAVAILABLE=$((P4JOURNALFREE-P4JOURNALMINBYTES))

# If available space is less than warning threshold, log warning and optionally email alert
if [[ "$P4JOURNALAVAILABLE" -lt "$P4JOURNALWARNBYTES" ]]; then
   log "Warning disk space threshold of ${P4JOURNALWARNBYTES} bytes reached, only ${P4JOURNALAVAILABLE} bytes available"
   # Convert to all lower case before comparison
   if [[ "${P4JOURNALWARNALERT,,}" == "true" ]]; then
      mail_log_file "${HOSTNAME} ${P4SERVER} Journal watch space warning"
   fi
fi

# If available space is less than rotate threshold, trigger a journal rotation on the master server
if [[ "$P4JOURNALAVAILABLE" -lt "$P4JOURNALROTATEBYTES" ]]; then
   log "Rotate disk space threshold of ${P4JOURNALROTATEBYTES} bytes reached, only ${P4JOURNALAVAILABLE} bytes available"
   if [[ "$EDGESERVER" -eq 0 && "$STANDBYSERVER" -eq 0 ]]; then
      ckp_running
      truncate_journal
      ckp_complete
   else
      truncate_journal_on_master
   fi
   mail_log_file "${HOSTNAME} ${P4SERVER} Journal watch space warning"
fi

# If checkpoints exist, calculate the number of journals to keep based on oldest checkpoint on disk to ensure enough
# journals are available for recovery (if P4OVERRIDEKEEPJNL is set to true)
if [[ -n "$OLDESTCHECKPOINT" && "${P4OVERRIDEKEEPJNL,,}" == "true" ]]; then
   if [[ "$EDGESERVER" -eq 0 && "$STANDBYSERVER" -eq 0 ]]; then
      KEEPJNLS=$((JOURNALNUM + 1 - OLDESTCHECKPOINT))
   else
      KEEPJNLS=$((JOURNALNUM + 2 - OLDESTCHECKPOINT))
   fi
   log "Based on oldest checkpoint of ${OLDESTCHECKPOINT}, temporarily setting KEEPJNLS to ${KEEPJNLS} to ensure enough journals are available for recovery."
fi

remove_old_checkpoints_and_journals
check_disk_space
remove_old_logs
log "End $P4SERVER Journal Watch"