sdp_test_harness.sh #4

#!/bin/bash
#==============================================================================
# sdp_test_harness.sh - Functional test harness for the Perforce Helix Core SDP.
#
# Exercises the SDP scripts on a running instance: environment, auth, service
# lifecycle (start/stop/status/restart), checkpoint/journal maintenance, archive
# verification, housekeeping, role-gating wrappers, and diagnostics. Scripts that
# only apply to replica/edge/standby/broker/proxy topologies are smoke-tested
# (argument parsing / usage) since this harness targets a single commit-server.
#
# Usage:  sdp_test_harness.sh [instance]   (default instance: 1)
#         Run as the SDP OS user (e.g. 'perforce').
#
# Exit:   0 if all non-skipped tests pass, 1 otherwise.
#==============================================================================

set -u
INSTANCE="${1:-1}"

#------------------------------------------------------------------------------
# Load the SDP environment.
#------------------------------------------------------------------------------
if [[ ! -r "/p4/common/bin/p4_vars" ]]; then
   echo "FATAL: /p4/common/bin/p4_vars not found. Is the SDP installed?" >&2
   exit 2
fi
# shellcheck disable=SC1090
source /p4/common/bin/p4_vars "$INSTANCE" || { echo "FATAL: could not source p4_vars $INSTANCE" >&2; exit 2; }

# SDP scripts pushd/popd and must run from a directory the SDP OS user can read.
# When invoked via 'sudo -u perforce' the cwd is the caller's home (often
# unreadable), which breaks checkpoint/journal scripts; move to the SDP home.
cd "${P4HOME:-/p4}" 2>/dev/null || cd /tmp || true

CBIN="/p4/common/bin"
INITSCRIPT="/p4/$INSTANCE/bin/p4d_${INSTANCE}_init"
WORKDIR="$(mktemp -d "${TMPDIR:-/tmp}/sdp_test.XXXXXX")"
REPORT="$WORKDIR/report.log"
RUNLOG="$WORKDIR/commands.log"
TS_START="$(date +%s)"

declare -i T_TOTAL=0 T_PASS=0 T_FAIL=0 T_SKIP=0
declare -a FAILED_TESTS=()

#------------------------------------------------------------------------------
# Output helpers.
#------------------------------------------------------------------------------
if [[ -t 1 ]]; then C_G=$'\e[32m'; C_R=$'\e[31m'; C_Y=$'\e[33m'; C_B=$'\e[1m'; C_0=$'\e[0m'
else C_G=""; C_R=""; C_Y=""; C_B=""; C_0=""; fi

_hdr()  { printf '\n%s== %s ==%s\n' "$C_B" "$1" "$C_0"; printf '\n== %s ==\n' "$1" >>"$REPORT"; }
_pass() { T_TOTAL+=1; T_PASS+=1; printf '  %s[PASS]%s %s\n' "$C_G" "$C_0" "$1"; printf '  [PASS] %s\n' "$1" >>"$REPORT"; }
_skip() { T_TOTAL+=1; T_SKIP+=1; printf '  %s[SKIP]%s %s (%s)\n' "$C_Y" "$C_0" "$1" "$2"; printf '  [SKIP] %s (%s)\n' "$1" "$2" >>"$REPORT"; }
_fail() {
   T_TOTAL+=1; T_FAIL+=1; FAILED_TESTS+=("$1")
   printf '  %s[FAIL]%s %s -- %s\n' "$C_R" "$C_0" "$1" "$2"
   printf '  [FAIL] %s -- %s\n' "$1" "$2" >>"$REPORT"
}

# Run a command, capturing combined output and exit code into globals. stdin is
# redirected from /dev/null so a tested script that reads stdin (e.g. clean_log.sh
# is a sed filter) gets EOF and cannot block the harness -- important when run
# non-interactively (via Ansible/cron) where stdin is an open pipe, not a tty.
_OUT=""; _RC=0
_exec() {
   printf '\n$ %s\n' "$*" >>"$RUNLOG"
   _OUT="$("$@" 2>&1 </dev/null)"; _RC=$?
   printf '%s\n(exit %d)\n' "$_OUT" "$_RC" >>"$RUNLOG"
}

#------------------------------------------------------------------------------
# Assertions. Each is one test.
#------------------------------------------------------------------------------
# Pass if the command exits 0.
t_ok() { local name="$1"; shift; _exec "$@"; if [[ $_RC -eq 0 ]]; then _pass "$name"; else _fail "$name" "exit=$_RC: $(tail -n1 <<<"$_OUT")"; fi; }

# Pass if the command exits 0 AND output contains the needle.
t_ok_has() {
   local name="$1" needle="$2"; shift 2; _exec "$@"
   if [[ $_RC -eq 0 && "$_OUT" == *"$needle"* ]]; then _pass "$name"
   elif [[ $_RC -ne 0 ]]; then _fail "$name" "exit=$_RC: $(tail -n1 <<<"$_OUT")"
   else _fail "$name" "missing '$needle' in output"; fi
}

# Pass if output contains the needle (exit code ignored).
t_has() {
   local name="$1" needle="$2"; shift 2; _exec "$@"
   if [[ "$_OUT" == *"$needle"* ]]; then _pass "$name"; else _fail "$name" "missing '$needle' (exit=$_RC)"; fi
}

# Pass if output does NOT contain the needle.
t_hasnot() {
   local name="$1" needle="$2"; shift 2; _exec "$@"
   if [[ "$_OUT" != *"$needle"* ]]; then _pass "$name"; else _fail "$name" "unexpected '$needle' in output"; fi
}

# Pass if the path exists.
t_file() { local name="$1" path="$2"; if [[ -e "$path" ]]; then _pass "$name"; else _fail "$name" "missing path: $path"; fi; }

# Audit tool: a structural/health report whose exit code reflects findings, not
# whether the tool works. Pass if it ran (no crash); report error/warning counts
# so genuine findings are visible without failing the suite on environment gaps.
t_audit() {
   local name="$1"; shift; _exec "$@"
   local errs warns
   errs=$(grep -c -iE "^Error:" <<<"$_OUT"); warns=$(grep -c -iE "^Warning:" <<<"$_OUT")
   if [[ $_RC -eq 127 || $_RC -ge 128 ]]; then _fail "$name" "did not run (exit=$_RC)"
   else _pass "$name [ran; ${errs} errors, ${warns} warnings to review]"; fi
}

# Smoke test: the script exists, is executable, and runs its arg-validation
# (prints usage / an instance complaint) without a fatal interpreter/crash.
t_smoke() {
   local name="$1"; shift
   local script="$1"
   if [[ ! -e "$script" ]]; then _skip "$name" "not present"; return; fi
   if [[ ! -x "$script" ]]; then _fail "$name" "not executable: $script"; return; fi
   _exec "$@"
   # 127 = the script ran but a command it needs is absent (an uninstalled
   # component, e.g. broker/proxy on a commit-only server) -> skip.
   # >=128 = killed by a signal -> real problem.
   if [[ $_RC -eq 127 ]]; then _skip "$name" "dependency not installed (exit 127)"
   elif [[ $_RC -ge 128 ]]; then _fail "$name" "did not run cleanly (exit=$_RC)"
   else _pass "$name"; fi
}

# Like t_ok, but skip (do not fail) when the script is absent -- for optional
# components such as monitoring probe scripts installed by a separate role.
t_opt() {
   local name="$1" script="$2"
   if [[ ! -e "$script" ]]; then _skip "$name" "not present (optional)"; return; fi
   shift  # remaining args: script [args...]
   t_ok "$name" "$@"
}

#------------------------------------------------------------------------------
# Ensure the server is left running no matter how we exit.
#------------------------------------------------------------------------------
_cleanup() {
   "$INITSCRIPT" start >/dev/null 2>&1
   "$CBIN/p4login" >/dev/null 2>&1
}
trap _cleanup EXIT

p4up()   { p4 -p "$P4PORT" info >/dev/null 2>&1; }
relogin(){ "$CBIN/p4login" >/dev/null 2>&1; }

echo "SDP Test Harness  instance=$INSTANCE  P4PORT=$P4PORT  $(date)"
echo "Work dir: $WORKDIR"
echo "SDP Test Harness instance=$INSTANCE P4PORT=$P4PORT $(date)" >"$REPORT"

#==============================================================================
# 1. Environment & sanity
#==============================================================================
_hdr "Environment & sanity"
[[ -n "${P4PORT:-}" ]] && _pass "p4_vars sets P4PORT ($P4PORT)" || _fail "p4_vars sets P4PORT" "P4PORT empty"
[[ -n "${P4ROOT:-}" && -d "$P4ROOT" ]] && _pass "p4_vars sets P4ROOT ($P4ROOT)" || _fail "p4_vars sets P4ROOT" "bad P4ROOT"
t_ok_has  "p4 set shows P4PORT"            "P4PORT"             bash -c "p4 set"
t_ok_has  "p4 info reachable (ServerID)"   "ServerID: master"  p4 info
# p4sanity_check is run after the data fixture below: it requires at least one
# submitted changelist to exist, which a brand-new server does not have yet.
# Accept any PASS (Grade A or B): Grade B = all required configurables pass, with
# only recommended (e.g. rejectList, filesys.depot.min) left to admin policy.
# ccheck exits non-zero for Grade B, so check the output rather than the rc.
t_has     "ccheck.sh PASS (required configs)" "Result: PASS"   "$CBIN/ccheck.sh" "$INSTANCE" -y

#==============================================================================
# 2. Authentication
#==============================================================================
_hdr "Authentication"
t_ok      "p4login"                        "$CBIN/p4login" "$INSTANCE"
t_ok_has  "p4 login -s (valid ticket)"     "expires"           p4 login -s
t_ok      "p4login_master"                 "$CBIN/p4login_master" "$INSTANCE"

#==============================================================================
# 3. Role-gating wrappers
#==============================================================================
_hdr "Role-gating wrappers"
t_has     "run_if_master runs on master"   "MASTER_OK"  "$CBIN/run_if_master.sh"  "$INSTANCE" echo MASTER_OK
t_hasnot  "run_if_replica skips on master" "REPLICA_RUN" "$CBIN/run_if_replica.sh" "$INSTANCE" echo REPLICA_RUN
t_hasnot  "run_if_edge skips on master"    "EDGE_RUN"    "$CBIN/run_if_edge.sh"    "$INSTANCE" echo EDGE_RUN

#==============================================================================
# 4. Service lifecycle (status / stop / start / restart)
#==============================================================================
_hdr "Service lifecycle"
t_ok      "p4d init: status (running)"     "$INITSCRIPT" status
"$INITSCRIPT" stop >/dev/null 2>&1; sleep 2
if ! p4up; then _pass "p4d init: stop brings server down"; else _fail "p4d init: stop brings server down" "server still answering"; fi
"$INITSCRIPT" start >/dev/null 2>&1; sleep 3; relogin
if p4up; then _pass "p4d init: start brings server up"; else _fail "p4d init: start brings server up" "server not answering"; fi
"$INITSCRIPT" restart >/dev/null 2>&1; sleep 3; relogin
if p4up; then _pass "p4d init: restart brings server up"; else _fail "p4d init: restart brings server up" "server not answering"; fi

#==============================================================================
# 5. Test data fixture (needed by checkpoint/verify)
#==============================================================================
_hdr "Test data fixture"
relogin
FIX_RC=0
{
   p4 depot -o test 2>/dev/null | p4 depot -i 2>/dev/null
   CLIENT="sdp_test_client_$$"
   CROOT="$WORKDIR/ws"; mkdir -p "$CROOT"
   p4 client -o "$CLIENT" | sed -e "s#^Root:.*#Root: $CROOT#" \
       -e "/^View:/,\$d" > "$WORKDIR/cspec"
   { cat "$WORKDIR/cspec"; printf 'View:\n\t//test/... //%s/...\n' "$CLIENT"; } | p4 client -i
   echo "sdp harness test content $(date)" > "$CROOT/harness_file.txt"
   P4CLIENT="$CLIENT" p4 add "$CROOT/harness_file.txt"
   P4CLIENT="$CLIENT" p4 submit -d "sdp harness test submit"
} >>"$RUNLOG" 2>&1 || FIX_RC=1
if P4CLIENT="" p4 files //test/... >/dev/null 2>&1; then _pass "create depot + submit test file"; else _fail "create depot + submit test file" "no files under //test/"; fi
# Now that at least one changelist exists, p4sanity_check can pass.
t_ok      "p4sanity_check.sh"              "$CBIN/p4sanity_check.sh" "$INSTANCE"

#==============================================================================
# 6. Checkpoint & journal maintenance
#==============================================================================
_hdr "Checkpoint & journal maintenance"
JNL_BEFORE="$(p4 counter journal 2>/dev/null)"
# live_checkpoint seeds the offline db and takes the first checkpoint.
t_ok      "live_checkpoint.sh"             "$CBIN/live_checkpoint.sh" "$INSTANCE"
relogin
t_ok      "rotate_journal.sh"              "$CBIN/rotate_journal.sh" "$INSTANCE"
relogin
JNL_AFTER="$(p4 counter journal 2>/dev/null)"
if [[ "$JNL_AFTER" =~ ^[0-9]+$ && "$JNL_BEFORE" =~ ^[0-9]+$ && "$JNL_AFTER" -gt "$JNL_BEFORE" ]]; then
   _pass "journal counter advanced ($JNL_BEFORE -> $JNL_AFTER)"
else _fail "journal counter advanced" "before=$JNL_BEFORE after=$JNL_AFTER"; fi
t_ok      "daily_checkpoint.sh"            "$CBIN/daily_checkpoint.sh" "$INSTANCE"
relogin
t_ok      "recreate_offline_db.sh"         "$CBIN/recreate_offline_db.sh" "$INSTANCE"
relogin
# A checkpoint file should now exist.
if compgen -G "/p4/$INSTANCE/checkpoints/*.ckp.*" >/dev/null; then _pass "checkpoint file(s) created"; else _fail "checkpoint file(s) created" "none in /p4/$INSTANCE/checkpoints"; fi

#==============================================================================
# 7. Archive verification
#==============================================================================
_hdr "Archive verification"
t_ok      "p4verify.py"                    "$CBIN/p4verify.py" "$INSTANCE" -P 60
t_smoke   "verify_shelves.sh (smoke)"      "$CBIN/verify_shelves.sh" "$INSTANCE"

#==============================================================================
# 8. Housekeeping & monitoring
#==============================================================================
_hdr "Housekeeping & monitoring"
t_ok      "p4_vars / p4master_run wrapper" "$CBIN/p4master_run" "$INSTANCE" p4 info
# sizes.sh ends in 'grep G'; exits non-zero when no GB-scale files exist. Smoke.
t_smoke   "sizes.sh (smoke)"               "$CBIN/sizes.sh" "$INSTANCE"
t_opt     "p4_disk_space.sh"               "$CBIN/p4_disk_space.sh" "$INSTANCE"
t_ok      "clean_log.sh"                   "$CBIN/clean_log.sh" "$INSTANCE"
t_ok      "journal_watch.sh"               "$CBIN/journal_watch.sh" "$INSTANCE" "20%" TRUE "10%" TRUE
t_ok      "update_limits.py"               "$CBIN/update_limits.py" "$INSTANCE"
# kill_idle pipes to 'xargs kill'; exit code is unreliable (123 when a target
# already exited), so smoke-test that it runs rather than asserting exit 0.
t_smoke   "kill_idle.sh (smoke)"           "$CBIN/kill_idle.sh" "$INSTANCE"
t_opt     "p4_healthcheck.sh"              "$CBIN/p4_healthcheck.sh" "$INSTANCE"
t_smoke   "recommend_limits.py (smoke)"    "$CBIN/recommend_limits.py" "$INSTANCE"

#==============================================================================
# 8b. Audit & reporting tools (informational: exit reflects findings, not health)
#==============================================================================
_hdr "Audit & reporting tools (informational)"
t_audit   "verify_sdp.sh"                  "$CBIN/verify_sdp.sh" "$INSTANCE"
# sdp_health_check auto-discovers instances and takes NO arguments.
t_audit   "sdp_health_check.sh"            "$CBIN/sdp_health_check.sh"

#==============================================================================
# 9. Diagnostics
#==============================================================================
_hdr "Diagnostics"
t_ok      "p4dstate.sh"                     "$CBIN/p4dstate.sh" "$INSTANCE"
t_smoke   "p4_network_latency.sh (smoke)"  "$CBIN/p4_network_latency.sh" "$INSTANCE"

#==============================================================================
# 10. Replica / edge / standby / broker tooling (smoke only on single master)
#==============================================================================
_hdr "Replica/edge/standby tooling (smoke)"
for s in mkrep.sh mkedge.sh mkstandby.sh mkstandby_shared.sh sync_replica.sh \
         replica_status.sh recreate_db_sync_replica.sh request_replica_checkpoint.sh \
         edge_dump.sh broker_rotate.sh proxy_rotate.sh cacheclean.sh; do
   if [[ -f "$CBIN/$s" ]]; then t_smoke "$s (smoke)" "$CBIN/$s"; else _skip "$s (smoke)" "not present"; fi
done

#==============================================================================
# Summary
#==============================================================================
TS_END="$(date +%s)"
DUR=$(( TS_END - TS_START ))
printf '\n%s==================== SUMMARY ====================%s\n' "$C_B" "$C_0"
printf '  total=%d  %spass=%d%s  %sfail=%d%s  %sskip=%d%s   (%ds)\n' \
   "$T_TOTAL" "$C_G" "$T_PASS" "$C_0" "$C_R" "$T_FAIL" "$C_0" "$C_Y" "$T_SKIP" "$C_0" "$DUR"
{
   echo; echo "==================== SUMMARY ===================="
   echo "total=$T_TOTAL pass=$T_PASS fail=$T_FAIL skip=$T_SKIP duration=${DUR}s"
} >>"$REPORT"
if (( T_FAIL > 0 )); then
   printf '  Failed: %s\n' "${FAILED_TESTS[*]}"
   echo "Failed: ${FAILED_TESTS[*]}" >>"$REPORT"
fi
printf '  Report: %s\n  Command log: %s\n' "$REPORT" "$RUNLOG"

(( T_FAIL == 0 ))