sdp_checkpoint_lock.sh #2

  • //
  • guest/
  • russell_jackson/
  • sdp/
  • Server/
  • Unix/
  • p4/
  • common/
  • bin/
  • sdp_checkpoint_lock.sh
  • View
  • Commits
  • Open Download .zip Download (3 KB)
#!/bin/bash
# Shared checkpoint ownership for backup/recovery scripts. Sourcing this helper
# changes no shell options and installs no traps. Shared die() releases its
# owned marker; abrupt termination and other handlers may leave it in place.
function ckp_lock_fail () {
   if declare -F die >/dev/null; then die "$*";
   elif declare -F bail >/dev/null; then bail "$*";
   else printf '%s\n' "$*" >&2; exit 1; fi
}

#------------------------------------------------------------------------------
# Acquire the checkpoint semaphore before initializing/rotating shared logs.
# noclobber uses exclusive file creation: exactly one concurrent caller wins.
# A legacy marker, stale marker, or interrupted write is deliberately blocking.
# Operators must investigate and remove it manually; PID reuse/shared mounts
# make automatic stale-lock removal unsafe. Completion and shared die() release
# only the acquiring shell's marker with its matching owner token.
# Optional "quiet" keeps rotate_journal's contention path silent.
#------------------------------------------------------------------------------
function ckp_running() {
   local LockFile="${LOGS}/ckp_running.txt"
   local Token="${HOSTNAME:-unknown}:${BASHPID:-$$}:$RANDOM:$RANDOM"
   local Owner=
   if [[ -n "${CkpLockToken:-}" && "${CkpLockPid:-}" == "${BASHPID:-$$}" &&
         "${CkpLockSubshell:-0}" == "${BASH_SUBSHELL:-0}" ]]; then
      IFS= read -r Owner < "$LockFile" 2>/dev/null || Owner=
      [[ "$Owner" == "$CkpLockToken" ]] && return 0
      echo "Checkpoint lock ownership changed: $LockFile. Aborting." >&2
      exit 1
   fi
   if (set -o noclobber; printf '%s\n' "$Token" > "$LockFile") 2>/dev/null; then
      CkpLockToken=$Token
      CkpLockPid=${BASHPID:-$$}
      CkpLockSubshell=${BASH_SUBSHELL:-0}
      return 0
   fi
   if [[ -e "$LockFile" || -L "$LockFile" ]]; then
      # Do not append to or rotate the active owner's checkpoint log. The
      # separate skip log is append-only; quiet callers preserve cron silence.
      if [[ "${1:-}" != quiet ]]; then
         (
            LOGFILE="$LOGS/checkpoint_skipped.log"
            if declare -F sdp_log_init >/dev/null; then
               sdp_log_init "${0##*/}"
               sdp_set_status skipped
               sdp_add_field skip_reason checkpoint_semaphore_exists
            fi
            printf '%s\n' "Skipping: checkpoint semaphore $LockFile already exists." >> "$LOGFILE"
         )
      fi
      exit 0
   fi
   echo "Cannot create checkpoint semaphore $LockFile. Aborting." >&2
   exit 1
}

#------------------------------------------------------------------------------
# Remove only this shell's successful-run semaphore. Fail closed if an operator
# or another process replaced it while this run was active.
#------------------------------------------------------------------------------
function ckp_complete() {
   local Owner=
   if [[ -z "${CkpLockToken:-}" || "${CkpLockPid:-}" != "${BASHPID:-$$}" ||
         "${CkpLockSubshell:-0}" != "${BASH_SUBSHELL:-0}" ]]; then
      ckp_lock_fail "Cannot release a checkpoint semaphore this shell does not own."
   fi
   IFS= read -r Owner < "${LOGS}/ckp_running.txt" 2>/dev/null || Owner=
   [[ "$Owner" == "$CkpLockToken" ]] || ckp_lock_fail "Checkpoint semaphore ownership changed; leaving it untouched."
   rm -- "${LOGS}/ckp_running.txt" 2>> "$LOGFILE" || ckp_lock_fail "Cannot release the checkpoint semaphore."
   CkpLockToken=
   CkpLockPid=
   CkpLockSubshell=
}

# Change User Description Committed
#2 34016 Russell C. Jackson (Rusty) Restore owned checkpoint-marker cleanup on handled failures and use MD5 completion markers for recovery.

Release only the acquiring shell’s ckp_running.txt marker in die(), including Bash 3.2 subshell ownership checks. Remove the separate full checkpoint verification pass from load_checkpoint and recreate_db_checkpoint, retain readable sidecar/source checks, and fix gzip sidecar path selection. Update regression tests and operational documentation.

Validation: combined acceptance gate passed; 44 platform stages, 30 topology scenarios, and 112 historical release scenarios across 14 RHEL 8/9 cells passed. All 8 Linux offline suites passed 526 tests each. Focused recovery tests passed on Bash 3.2 and 5.3.
#1 34004 Russell C. Jackson (Rusty) Improve SDP backup/recovery, Maintenance, logging and cloud operation.

Add owned checkpoint locks, recovery-aware retention, preserved database switches,
bounded cleanup and coordinated AWS/Azure snapshots. Fix service-user password
initialization, edge staging, standby promotion state parsing, proxy cache setup,
typed depot verification and historical-server configuration/error handling.
Remove unused OTP programs; keep Windows removed, server.locks cleanup and the
journal-count condition unchanged. Add no global shell strict modes.

Update operational documentation and add reproducible mkdirs.sh/mkdirs.cfg
container assurance. Validate Ubuntu 26.04 and UBI 8.10/9.8/10.2 across both Python
runtimes: 44 stages; 428 offline tests per runtime/OS, zero offline skips;
19 deployed-archive comparisons per combination. Three six-node topology runs
pass all 30 recovery/failure scenarios. P4D 2023.1 through 2026.1 on UBI 8/9
pass all 14 cells and 98 scenarios. Static, documentation and provenance checks
pass. Containers do not certify every fleet kernel, patch build or integration.

Tested source manifest SHA-256:
e1a23e17b8ced8c1148d9a250589977c6f6e98606bf2a81dc3ff5d7c909e3695
Deployed comparison archive SHA-256:
1bd23217356ee3808e040d39aea5be9519bda7c8f2b4924656a70aff8ad7293a