edge_maintenance #19

#!/bin/bash
#
# weekly_maintenance.sh – perform routine Perforce cleanup and notifications.
#
# This script performs a series of maintenance tasks against a Helix Core
# installation using the SDP environment.  It logs all output to a file
# and emails failures to the configured MAILTO address. Client unloading
# is capped at 200 workspaces per run without a threshold alert.

# Fail fast on errors, unset variables and pipeline failures.
set -uo pipefail

# Determine the SDP instance either from the first positional parameter or
# the environment.  Require it to be set to avoid accidental misuse.
P4INSTANCE="${1:-${P4INSTANCE:-}}"
if [[ -z "$P4INSTANCE" ]]; then
  echo "P4PORT=${P4PORT:-unknown}: Instance parameter not supplied." >&2
  echo "You must supply the Perforce instance as a parameter to this script or set \$P4INSTANCE in the environment."
  exit 1
fi

# Initialize pyenv so scripts use the correct Python version.
export PYENV_ROOT="$HOME/.pyenv"
export PATH="$PYENV_ROOT/bin:$PATH"
eval "$(pyenv init -)"

# Source SDP environment variables and ensure we can talk to Perforce.
if ! source "/p4/common/bin/p4_vars" "$P4INSTANCE"; then
  echo "P4PORT=${P4PORT:-unknown}: Unable to source p4_vars for instance $P4INSTANCE" >&2
  exit 1
fi

# sdp_structured_log.sh is standalone (no backup_functions.sh dependency) and
# only needs $LOGS, already exported by p4_vars above.
# shellcheck disable=SC1091
[[ -r /p4/common/bin/sdp_structured_log.sh ]] && source /p4/common/bin/sdp_structured_log.sh
declare -F sdp_log_init >/dev/null && sdp_log_init edge_maintenance

# Prepare our log file and client names.  Use the hostname and server name for uniqueness.
LOG="/p4/sdp/Maintenance/${HOSTNAME}_${P4SERVER}_log.txt"
P4CLIENT="${HOSTNAME}_unload"

# Ensure the maintenance working directory exists and change into it for relative paths.
MAINT_DIR="/p4/sdp/Maintenance"
if [[ ! -d "$MAINT_DIR" ]]; then
  echo "P4PORT=${P4PORT:-unknown}: Maintenance directory $MAINT_DIR does not exist." >&2
  exit 1
fi

# Both weekly entry points share one instance lock; each run gets its own
# reports and scratch files. The server.locks cleanup below is intentional.
source "$MAINT_DIR/maintenance_workspace.sh" || exit 1
maintenance_workspace_init "$P4INSTANCE" || {
  workspace_status=$?
  if (( workspace_status == 75 )); then exit 0; fi
  exit "$workspace_status"
}

# Acquire the instance lock before replacing the shared run log. A competing
# invocation must not rotate the active owner's diagnostics.
if declare -F sdp_prepare_run_log >/dev/null; then
  sdp_prepare_run_log "$LOG" || exit 1
  exec >"$LOG" 2>&1 || exit 1
else
  exec >>"$LOG" 2>&1 || exit 1
fi
printf 'Maintenance run directory: %s run_id=%s\n' "$SDP_MAINTENANCE_WORKDIR" "${SDP_RUN_ID:-unavailable}"

# Helper function to timestamp and append a message to the log.
log_msg() {
  local msg="${1:-}"
  printf '%s %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$msg" >>"$LOG"
}

# Tracks whether any step below actually failed, so the end-of-run email
# only fires on a real problem instead of on every single run (it was
# unconditional before - "Weekly maintenance log" landed in MAILTO's inbox
# every night regardless of outcome). Also mirrors into the structured
# jsonl record via sdp_set_status/sdp_set_error, since none of the `|| true`
# guards below were ever reporting failure to it either.
HAD_ERROR=0
run_step() {
  local desc="$1"
  shift
  log_msg "$desc"
  "$@" >>"$LOG" 2>&1
  local rc=$?
  if (( rc != 0 )); then
    log_msg "ERROR: $desc failed (exit $rc)"
    HAD_ERROR=1
    declare -F sdp_set_status >/dev/null && sdp_set_status failure
    declare -F sdp_set_error >/dev/null && sdp_set_error "$desc failed (exit $rc)"
  fi
  return "$rc"
}

# Start logging.
log_msg "--- Weekly maintenance started for instance $P4INSTANCE (P4PORT=${P4PORT:-unknown}, run_id=${SDP_RUN_ID:-unavailable}) ---"

# Authentication failures must be visible and stop all cleanup work.
run_step "Authenticating with p4login" /p4/common/bin/p4login || exit 1

# Remove any leftover server lock files.  This can prevent DB replays from
# succeeding if the previous run terminated uncleanly.
run_step "Removing server lock files from /p4/${P4INSTANCE}/root/server.locks" \
  rm -rf "/p4/${P4INSTANCE}/root/server.locks"

# Generate lists of inactive clients.  accessdates.py produces clients.txt
# based on the configured number of weeks.
REPORT_OK=0
if run_step "Generating client inactivity report via accessdates.py" \
  accessdates.py "$P4INSTANCE" --client-snapshot client_snapshot.json; then
  REPORT_OK=1
fi

# Count the number of clients to be unloaded and decide what to do.  Use
# wc -l with a redirection to avoid counting the filename itself.  Fall back
# to zero if the file is missing.
client_count=0
if (( REPORT_OK )) && [[ -f clients.txt ]]; then
  client_count=$(wc -l < clients.txt || echo 0)
fi
log_msg "Inactive client count: $client_count"
declare -F sdp_add_field >/dev/null && sdp_add_field inactive_client_count "$client_count"

# Unload at most 200 clients per run; leave any remainder for later runs.
if (( client_count > 0 )); then
  unload_count=$(( client_count < 200 ? client_count : 200 ))
  run_step "Unloading up to $unload_count clients via unload_clients.py" \
    unload_clients.py "$P4INSTANCE" --limit 200 --clients-file client_snapshot.json
fi

# Notify users about pending client deletions.  This script emails owners of
# workspaces that will be removed soon.  Only run once per maintenance window.
# run_step "Running email_pending_client_deletes.py" email_pending_client_deletes.py "$P4INSTANCE"

# Remove empty pending changelists.  These accumulate over time and can slow
# down server operations.
run_step "Removing empty pending changelists via remove_empty_pending_changes.py" \
  remove_empty_pending_changes.py "$P4INSTANCE"

# Finish logging. Only email MAILTO when something actually failed - the
# structured jsonl record (emitted automatically on exit by sdp_log_init's
# trap, status/error set by run_step above) is the routine, non-noisy
# record of every run; this email is reserved for the cases that need a
# human to look.
log_msg "--- Weekly maintenance completed run_id=${SDP_RUN_ID:-unavailable} ---"
if (( HAD_ERROR )) && [[ -n "${MAILTO:-}" ]]; then
  if ! mail -s "${HOSTNAME} ${P4SERVER} P4PORT=${P4PORT:-unknown} Weekly maintenance FAILED" "$MAILTO" <"$LOG"; then
    log_msg "ERROR: Unable to send maintenance failure notification"
  fi
fi

exit "$HAD_ERROR"