maintenance #14

#!/bin/bash
#
# weekly_maintenance.sh – perform routine Perforce cleanup and notifications.
#
# This script performs a series of maintenance tasks against a Helix Core
# installation using the SDP environment.  It logs all output to a file
# and emails the results to the configured MAILTO address.  If a large
# number of clients are queued for unloading, it alerts the administrator
# instead of attempting the unload.

# Fail fast on errors, unset variables and pipeline failures.
set -uo pipefail

# Determine the SDP instance either from the first positional parameter or
# the environment.  Require it to be set to avoid accidental misuse.
P4INSTANCE="${1:-${P4INSTANCE:-}}"
if [[ -z "$P4INSTANCE" ]]; then
  echo "Instance parameter not supplied."
  echo "You must supply the Perforce instance as a parameter to this script or set \$P4INSTANCE in the environment."
  exit 1
fi

# Initialize pyenv so scripts use the correct Python version.
export PYENV_ROOT="$HOME/.pyenv"
export PATH="$PYENV_ROOT/bin:$PATH"
eval "$(pyenv init -)"

# Source SDP environment variables and ensure we can talk to Perforce.
if ! source "/p4/common/bin/p4_vars" "$P4INSTANCE"; then
  echo "Unable to source p4_vars for instance $P4INSTANCE" >&2
  exit 1
fi

# sdp_structured_log.sh is standalone (no backup_functions.sh dependency) and
# only needs $LOGS, already exported by p4_vars above.
# shellcheck disable=SC1091
[[ -r /p4/common/bin/sdp_structured_log.sh ]] && source /p4/common/bin/sdp_structured_log.sh
declare -F sdp_log_init >/dev/null && sdp_log_init maintenance

# Authenticate using the SDP helper.  Suppress output but do not fail the script if login fails.
/p4/common/bin/p4login >/dev/null 2>&1 || true

# Prepare our log file and client names.  Use the hostname and server name for uniqueness.
LOG="/p4/sdp/Maintenance/${HOSTNAME}_${P4SERVER}_log.txt"
P4CLIENT="${HOSTNAME}_unload"

# Ensure the maintenance working directory exists and change into it for relative paths.
MAINT_DIR="/p4/sdp/Maintenance"
if [[ ! -d "$MAINT_DIR" ]]; then
  echo "Maintenance directory $MAINT_DIR does not exist." >&2
  exit 1
fi
cd "$MAINT_DIR"

# Helper function to timestamp and append a message to the log.
log_msg() {
  local msg="${1:-}"
  printf '%s %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$msg" >>"$LOG"
}

# Tracks whether any step below actually failed, so the end-of-run email
# only fires on a real problem instead of on every single run (it was
# unconditional before - "Weekly maintenance log" landed in MAILTO's inbox
# every night regardless of outcome). Also mirrors into the structured
# jsonl record via sdp_set_status/sdp_set_error, since none of the `|| true`
# guards below were ever reporting failure to it either.
HAD_ERROR=0
run_step() {
  local desc="$1"
  shift
  log_msg "$desc"
  "$@" >>"$LOG" 2>&1
  local rc=$?
  if (( rc != 0 )); then
    log_msg "ERROR: $desc failed (exit $rc)"
    HAD_ERROR=1
    declare -F sdp_set_status >/dev/null && sdp_set_status failure
    declare -F sdp_set_error >/dev/null && sdp_set_error "$desc failed (exit $rc)"
  fi
  return 0
}

# Start logging.
log_msg "--- Weekly maintenance started for instance $P4INSTANCE ---"

# Remove any leftover server lock files.  This can prevent DB replays from
# succeeding if the previous run terminated uncleanly.
run_step "Removing server lock files from /p4/${P4INSTANCE}/root/server.locks" \
  rm -rf "/p4/${P4INSTANCE}/root/server.locks"

# Generate lists of inactive clients.  accessdates.py produces clients.txt
# based on the configured number of weeks.
run_step "Generating client inactivity report via accessdates.py" accessdates.py "$P4INSTANCE"

# Count the number of clients to be unloaded and decide what to do.  Use
# wc -l with a redirection to avoid counting the filename itself.  Fall back
# to zero if the file is missing.
client_count=0
if [[ -f clients.txt ]]; then
  client_count=$(wc -l < clients.txt || echo 0)
fi
log_msg "Inactive client count: $client_count"
declare -F sdp_add_field >/dev/null && sdp_add_field inactive_client_count "$client_count"

# If there are fewer than 200 clients queued, unload them.  Otherwise alert
# the administrator by emailing the list of clients rather than unloading
# them, as processing a very large set may impact server performance.
if (( client_count > 0 )); then
  if (( client_count < 200 )); then
    run_step "Unloading $client_count clients via unload_clients.py" \
      unload_clients.py
  else
    log_msg "Too many clients ($client_count) to unload automatically; sending alert email instead."
    declare -F sdp_set_status >/dev/null && sdp_set_status warning
    declare -F sdp_add_field >/dev/null && sdp_add_field unload_threshold_exceeded true
    if [[ -n "${MAILTO:-}" ]]; then
      mail -s "${HOSTNAME} ${P4SERVER} client unload threshold exceeded" "$MAILTO" < clients.txt || true
    fi
  fi
fi

# Notify users about pending client deletions.  This script emails owners of
# workspaces that will be removed soon.  Only run once per maintenance window.
# run_step "Running email_pending_client_deletes.py" email_pending_client_deletes.py "$P4INSTANCE"

# Remove empty pending changelists.  These accumulate over time and can slow
# down server operations.
# run_step "Removing empty pending changelists via remove_empty_pending_changes.py" remove_empty_pending_changes.py "$P4INSTANCE"

# Finish logging. Only email MAILTO when something actually failed - the
# structured jsonl record (emitted automatically on exit by sdp_log_init's
# trap, status/error set by run_step above) is the routine, non-noisy
# record of every run; this email is reserved for the cases that need a
# human to look.
log_msg "--- Weekly maintenance completed ---"
if (( HAD_ERROR )) && [[ -n "${MAILTO:-}" ]]; then
  mail -s "${HOSTNAME} ${P4SERVER} Weekly maintenance FAILED" "$MAILTO" <"$LOG" || true
fi