#!/bin/bash
# Perforce SDP Health Check
CURRENT_USER=$(id -un)
RFI_POINTS=0
RFI_ITEMS=()
add_rfi() {
RFI_POINTS=$((RFI_POINTS + 1))
RFI_ITEMS+=("$1")
}
# Detect the user(s) owning p4d processes
mapfile -t P4D_OWNERS < <(ps -eo user,comm | awk '$2 ~ /^p4d/ {print $1}' | sort -u)
if [[ ${#P4D_OWNERS[@]} -eq 0 ]]; then
echo "ERROR: No p4d processes found on this host. Is the server running?" >&2
exit 1
fi
if [[ ${#P4D_OWNERS[@]} -gt 1 ]]; then
echo "ERROR: p4d processes are owned by multiple users: ${P4D_OWNERS[*]}" >&2
echo " Cannot determine which account to run as." >&2
exit 1
fi
P4D_USER="${P4D_OWNERS[0]}"
if [[ "$CURRENT_USER" != "$P4D_USER" ]]; then
echo "ERROR: This script must run as '$P4D_USER' (owner of p4d processes)." >&2
echo " Currently running as '$CURRENT_USER'." >&2
echo " Re-run as: sudo -u $P4D_USER $0 $*" >&2
exit 1
fi
export HOME=$(getent passwd "$P4D_USER" | cut -d: -f6)
cd "$HOME" || exit 1
# Discover all SDP instances: subdirs of /p4 that have a matching vars file
NON_INSTANCE_DIRS="sdp ssl common"
mapfile -t INSTANCES < <(
for dir in /p4/*/; do
name=$(basename "$dir")
[[ " $NON_INSTANCE_DIRS " =~ " $name " ]] && continue
[[ -f "/p4/common/config/p4_${name}.vars" ]] && echo "$name"
done | sort
)
if [[ ${#INSTANCES[@]} -eq 0 ]]; then
echo "ERROR: No SDP instances found under /p4." >&2
exit 1
fi
LOG_DIR="$HOME/p4health/logs"
HOSTNAME_SHORT=$(hostname -s)
LOG_FILE="$LOG_DIR/p4health_${HOSTNAME_SHORT}_$(date +%Y%m%d_%H%M%S).log"
mkdir -p "$LOG_DIR"
echo "=== Perforce SDP Health Check ===" | tee "$LOG_FILE"
echo "Date/Time : $(date -u '+%Y-%m-%d %H:%M:%S UTC')" | tee -a "$LOG_FILE"
echo "Host : $(hostname -f)" | tee -a "$LOG_FILE"
echo "Run as : $CURRENT_USER" | tee -a "$LOG_FILE"
echo "" | tee -a "$LOG_FILE"
echo "--- Instances discovered on this host ---" | tee -a "$LOG_FILE"
for inst in "${INSTANCES[@]}"; do
echo " $inst" | tee -a "$LOG_FILE"
done
echo "" | tee -a "$LOG_FILE"
OVERALL_EXIT=0
TOTAL_DB_SIZE=0
TOTAL_DB_REV_SIZE=0
# --- Global: SDP version check ---
SDP_VERSION_FILE="/p4/sdp/Version"
SDP_VERSION_URL="https://swarm.workshop.perforce.com/view/guest/perforce_software/sdp/main/Version"
echo "--- SDP Version Check ---" | tee -a "$LOG_FILE"
if [[ -f "$SDP_VERSION_FILE" ]]; then
LOCAL_SDP_VERSION=$(tr -d '[:space:]' < "$SDP_VERSION_FILE")
REMOTE_SDP_VERSION=$(curl -sf --max-time 10 "$SDP_VERSION_URL" | tr -d '[:space:]')
if [[ -z "$REMOTE_SDP_VERSION" ]]; then
echo "WARNING: Could not fetch remote SDP version (network issue?)" | tee -a "$LOG_FILE"
elif [[ "$LOCAL_SDP_VERSION" == "$REMOTE_SDP_VERSION" ]]; then
echo "OK: SDP is current ($LOCAL_SDP_VERSION)" | tee -a "$LOG_FILE"
else
echo "WARNING: SDP version mismatch." | tee -a "$LOG_FILE"
echo " Installed : $LOCAL_SDP_VERSION" | tee -a "$LOG_FILE"
echo " Available : $REMOTE_SDP_VERSION" | tee -a "$LOG_FILE"
add_rfi "SDP version out of date (installed: $LOCAL_SDP_VERSION, available: $REMOTE_SDP_VERSION)"
fi
else
echo "WARNING: SDP version file not found: $SDP_VERSION_FILE" | tee -a "$LOG_FILE"
fi
echo "" | tee -a "$LOG_FILE"
# --- Global: mail command check ---
echo "--- Mail Command Check ---" | tee -a "$LOG_FILE"
MAIL_PATH=$(bash -c "source /p4/common/bin/p4_vars ${INSTANCES[0]} > /dev/null 2>&1 && which mail 2>/dev/null")
if [[ "$MAIL_PATH" == "/p4/common/site/bin/mail" ]]; then
echo "ERROR: 'mail' resolves to the SDP stub at $MAIL_PATH — real email is suppressed." | tee -a "$LOG_FILE"
echo " SDP alerts and reports will not be delivered via email." | tee -a "$LOG_FILE"
add_rfi "mail command is the SDP stub ($MAIL_PATH) — real email is suppressed on this server"
elif [[ -z "$MAIL_PATH" ]]; then
echo "WARNING: 'mail' command not found in PATH." | tee -a "$LOG_FILE"
else
echo "OK: 'mail' resolves to $MAIL_PATH" | tee -a "$LOG_FILE"
fi
echo "" | tee -a "$LOG_FILE"
for inst in "${INSTANCES[@]}"; do
vars_file="/p4/common/config/p4_${inst}.vars"
# Source vars file in a subshell to extract instance-specific settings
eval "$(bash -c "source '$vars_file' 2>/dev/null && printf 'INST_P4USER=%s\nINST_P4PORT=%s\nINST_SSL_PREFIX=%s\nINST_P4PORTNUM=%s\n' \"\$P4USER\" \"\$P4PORT\" \"\$SSL_PREFIX\" \"\$P4PORTNUM\"")"
SSL_MODE=$([[ -n "$INST_SSL_PREFIX" ]] && echo "yes" || echo "no")
# Capture p4 info once — used for display and version parsing
P4INFO_OUT=$(/p4/common/bin/p4 -p "$INST_P4PORT" -u "$INST_P4USER" info 2>&1)
INST_VERSION=$(echo "$P4INFO_OUT" | grep "Server version" | grep -oE '[0-9]{4}\.[0-9]+' | head -1)
if /p4/common/bin/run_if_master.sh "$inst" echo yes 2>/dev/null | grep -q yes; then
SERVER_TYPE="commit"
elif /p4/common/bin/run_if_replica.sh "$inst" echo yes 2>/dev/null | grep -q yes; then
SERVER_TYPE="replica"
elif /p4/common/bin/run_if_edge.sh "$inst" echo yes 2>/dev/null | grep -q yes; then
SERVER_TYPE="edge"
else
SERVER_TYPE="unknown"
fi
# Version comparison — 2 releases per year; flag if more than 2 behind MOST_RECENT
MOST_RECENT="2026.1"
MR_YEAR=${MOST_RECENT%%.*}
MR_REL=${MOST_RECENT##*.}
MR_LINEAR=$(( MR_YEAR * 2 + MR_REL - 1 ))
if [[ -n "$INST_VERSION" ]]; then
IV_YEAR=${INST_VERSION%%.*}
IV_REL=${INST_VERSION##*.}
IV_LINEAR=$(( IV_YEAR * 2 + IV_REL - 1 ))
RELEASES_BEHIND=$(( MR_LINEAR - IV_LINEAR ))
else
RELEASES_BEHIND=-1
fi
echo "========================================" | tee -a "$LOG_FILE"
echo "Instance : $inst" | tee -a "$LOG_FILE"
echo "Server type : $SERVER_TYPE" | tee -a "$LOG_FILE"
echo "P4D version : ${INST_VERSION:-unknown} ($RELEASES_BEHIND release(s) behind $MOST_RECENT)" | tee -a "$LOG_FILE"
echo "P4USER : $INST_P4USER" | tee -a "$LOG_FILE"
echo "P4PORT : $INST_P4PORT" | tee -a "$LOG_FILE"
echo "SSL : $SSL_MODE" | tee -a "$LOG_FILE"
echo "" | tee -a "$LOG_FILE"
[[ "$SSL_MODE" == "no" ]] && add_rfi "Instance '$inst': is not running in SSL mode"
if [[ $RELEASES_BEHIND -gt 2 ]]; then
add_rfi "Instance '$inst': P4D version $INST_VERSION is $RELEASES_BEHIND release(s) behind current ($MOST_RECENT)"
fi
# Tally db.* and db.rev sizes (perforce owns these files, no sudo needed)
P4ROOT="/p4/${inst}/root"
INST_DB_SIZE=$(du -cb "${P4ROOT}"/db.* 2>/dev/null | tail -1 | awk '{print $1}')
INST_DB_REV_SIZE=$(du -sb "${P4ROOT}/db.rev" 2>/dev/null | awk '{print $1}')
INST_DB_SIZE=${INST_DB_SIZE:-0}
INST_DB_REV_SIZE=${INST_DB_REV_SIZE:-0}
TOTAL_DB_SIZE=$((TOTAL_DB_SIZE + INST_DB_SIZE))
TOTAL_DB_REV_SIZE=$((TOTAL_DB_REV_SIZE + INST_DB_REV_SIZE))
echo "DB files size : $(numfmt --to=iec --suffix=B $INST_DB_SIZE)" | tee -a "$LOG_FILE"
echo "db.rev size : $(numfmt --to=iec --suffix=B $INST_DB_REV_SIZE)" | tee -a "$LOG_FILE"
echo "" | tee -a "$LOG_FILE"
echo "--- p4 info (instance: $inst) ---" | tee -a "$LOG_FILE"
echo "$P4INFO_OUT" | tee -a "$LOG_FILE"
echo "" | tee -a "$LOG_FILE"
echo "--- Disk Space Check (instance: $inst) ---" | tee -a "$LOG_FILE"
for dir_label in "root" "logs" "depots"; do
LINK_PATH="/p4/${inst}/${dir_label}"
if [[ -L "$LINK_PATH" ]]; then
REAL_PATH=$(readlink -f "$LINK_PATH")
DF_LINE=$(df -P "$REAL_PATH" 2>/dev/null | tail -1)
DISK_PCT=$(echo "$DF_LINE" | awk '{gsub(/%/,"",$5); print $5}')
DISK_AVAIL=$(echo "$DF_LINE" | awk '{print $4}')
DISK_AVAIL_HR=$(numfmt --to=iec --suffix=B $((DISK_AVAIL * 1024)) 2>/dev/null || echo "${DISK_AVAIL}K")
MOUNT=$(echo "$DF_LINE" | awk '{print $6}')
printf " %-8s -> %-35s mount: %-15s free: %-8s used: %s%%\n" \
"$dir_label" "$REAL_PATH" "$MOUNT" "$DISK_AVAIL_HR" "$DISK_PCT" | tee -a "$LOG_FILE"
if [[ "${DISK_PCT:-0}" -ge 90 ]]; then
add_rfi "Instance '$inst': ${dir_label} partition (${MOUNT}) is ${DISK_PCT}% full (-> ${REAL_PATH})"
fi
else
echo " WARNING: $LINK_PATH is not a symlink" | tee -a "$LOG_FILE"
fi
done
echo "" | tee -a "$LOG_FILE"
echo "--- License Check (instance: $inst) ---" | tee -a "$LOG_FILE"
if [[ "$SERVER_TYPE" == "commit" || "$SERVER_TYPE" == "replica" ]]; then
LICENSE_FILE="/p4/${inst}/root/license"
if [[ ! -f "$LICENSE_FILE" ]]; then
echo "ERROR: No license file found at $LICENSE_FILE" | tee -a "$LOG_FILE"
add_rfi "Instance '$inst': No license file found at $LICENSE_FILE"
else
echo "License file : $LICENSE_FILE" | tee -a "$LOG_FILE"
EXPIRY_DATE=$(grep -iE 'expires' "$LICENSE_FILE" | grep -oE '[0-9]{4}/[0-9]{2}/[0-9]{2}' | head -1)
if [[ -n "$EXPIRY_DATE" ]]; then
EXPIRY_EPOCH=$(date -d "${EXPIRY_DATE//\// -}" +%s 2>/dev/null)
DAYS_REMAINING=$(( (EXPIRY_EPOCH - $(date +%s)) / 86400 ))
echo "Expiry date : $EXPIRY_DATE ($DAYS_REMAINING day(s) remaining)" | tee -a "$LOG_FILE"
if [[ $DAYS_REMAINING -le 30 ]]; then
echo "WARNING: License expires within 30 days." | tee -a "$LOG_FILE"
add_rfi "Instance '$inst': License expires in $DAYS_REMAINING day(s) on $EXPIRY_DATE"
else
echo "OK: License is valid." | tee -a "$LOG_FILE"
fi
else
echo "WARNING: Could not parse expiry date from $LICENSE_FILE" | tee -a "$LOG_FILE"
fi
fi
else
echo "Skipped (not applicable for $SERVER_TYPE server)" | tee -a "$LOG_FILE"
fi
echo "" | tee -a "$LOG_FILE"
echo "--- SSL Certificate Check (instance: $inst) ---" | tee -a "$LOG_FILE"
if [[ "$SSL_MODE" == "yes" ]]; then
SSL_CERT="/p4/ssl/certificate.txt"
if [[ ! -f "$SSL_CERT" ]]; then
echo "ERROR: SSL is enabled but $SSL_CERT not found." | tee -a "$LOG_FILE"
add_rfi "Instance '$inst': SSL enabled but no certificate found at $SSL_CERT"
else
CERT_ENDDATE=$(openssl x509 -in "$SSL_CERT" -noout -enddate 2>/dev/null | cut -d= -f2)
CERT_EPOCH=$(date -d "$CERT_ENDDATE" +%s 2>/dev/null)
DAYS_LEFT=$(( (CERT_EPOCH - $(date +%s)) / 86400 ))
echo "Certificate : $SSL_CERT" | tee -a "$LOG_FILE"
echo "Expires : $CERT_ENDDATE ($DAYS_LEFT day(s) remaining)" | tee -a "$LOG_FILE"
if [[ $DAYS_LEFT -le 30 ]]; then
echo "WARNING: Certificate expires within 30 days." | tee -a "$LOG_FILE"
add_rfi "Instance '$inst': SSL certificate expires in $DAYS_LEFT day(s) on $CERT_ENDDATE"
else
echo "OK: Certificate expiry is valid." | tee -a "$LOG_FILE"
fi
CERT_ISSUER=$(openssl x509 -in "$SSL_CERT" -noout -issuer 2>/dev/null)
if echo "$CERT_ISSUER" | grep -q "Perforce Autogen Cert"; then
echo "WARNING: Certificate is a Perforce auto-generated self-signed certificate." | tee -a "$LOG_FILE"
add_rfi "Instance '$inst': SSL certificate is a Perforce auto-generated self-signed cert — a CA-signed certificate is recommended"
else
echo "OK: Certificate is not a Perforce auto-generated self-signed certificate." | tee -a "$LOG_FILE"
fi
fi
else
echo "Skipped (SSL not enabled for this instance)" | tee -a "$LOG_FILE"
fi
echo "" | tee -a "$LOG_FILE"
echo "--- Checkpoint Check (instance: $inst) ---" | tee -a "$LOG_FILE"
# Resolve journalPrefix: try live configurable first, fall back to SDP convention
JOURNAL_PREFIX=$(bash -c "source /p4/common/bin/p4_vars '$inst' > /dev/null 2>&1 && \
p4 configure show journalPrefix 2>/dev/null" | awk -F= '/journalPrefix/{print $2}' | awk '{print $1}')
if [[ -z "$JOURNAL_PREFIX" ]]; then
JOURNAL_PREFIX="/p4/${inst}/checkpoints/p4_${inst}"
echo "Note: journalPrefix not configured; using SDP default." | tee -a "$LOG_FILE"
fi
echo "Checkpoint prefix : $JOURNAL_PREFIX" | tee -a "$LOG_FILE"
mapfile -t CKPT_FILES < <(ls -t "${JOURNAL_PREFIX}".ckp.* 2>/dev/null)
if [[ ${#CKPT_FILES[@]} -eq 0 ]]; then
echo "WARNING: No checkpoint files found." | tee -a "$LOG_FILE"
add_rfi "Instance '$inst': No checkpoint files found at ${JOURNAL_PREFIX}.ckp.*"
else
local_count=${#CKPT_FILES[@]}
echo "Last $([[ $local_count -lt 10 ]] && echo $local_count || echo 10) checkpoint file(s):" | tee -a "$LOG_FILE"
for f in "${CKPT_FILES[@]:0:10}"; do
echo " $(ls -lh "$f" | awk '{print $5, $6, $7, $8, $9}')" | tee -a "$LOG_FILE"
done
NEWEST_CKPT="${CKPT_FILES[0]}"
CKPT_AGE_HOURS=$(( ( $(date +%s) - $(stat -c %Y "$NEWEST_CKPT") ) / 3600 ))
if [[ $CKPT_AGE_HOURS -gt 24 ]]; then
echo "WARNING: Most recent checkpoint is ${CKPT_AGE_HOURS} hours old (> 24 hours)." | tee -a "$LOG_FILE"
add_rfi "Instance '$inst': Most recent checkpoint is ${CKPT_AGE_HOURS}h old — no checkpoint in the last 24 hours"
else
echo "OK: Most recent checkpoint is ${CKPT_AGE_HOURS} hour(s) old." | tee -a "$LOG_FILE"
fi
fi
echo "" | tee -a "$LOG_FILE"
echo "--- p4verify.log Check (instance: $inst) ---" | tee -a "$LOG_FILE"
VERIFY_LOG="/p4/${inst}/logs/p4verify.log"
if [[ ! -f "$VERIFY_LOG" ]]; then
echo "WARNING: $VERIFY_LOG does not exist — p4verify.sh may never have run for this instance." | tee -a "$LOG_FILE"
add_rfi "Instance '$inst': $VERIFY_LOG not found — verify script may never have run"
else
VERIFY_LOG_AGE_HOURS=$(( ( $(date +%s) - $(stat -c %Y "$VERIFY_LOG") ) / 3600 ))
echo "Log file : $VERIFY_LOG (${VERIFY_LOG_AGE_HOURS}h old)" | tee -a "$LOG_FILE"
BAD_COUNT=$(grep -c '\bBAD\b' "$VERIFY_LOG" 2>/dev/null || true)
MISSING_COUNT=$(grep -c 'MISSING' "$VERIFY_LOG" 2>/dev/null || true)
if [[ "${BAD_COUNT:-0}" -gt 0 || "${MISSING_COUNT:-0}" -gt 0 ]]; then
echo "WARNING: $BAD_COUNT BAD and $MISSING_COUNT MISSING error(s) found." | tee -a "$LOG_FILE"
echo " First 5 error lines:" | tee -a "$LOG_FILE"
grep -E '\bBAD\b|MISSING' "$VERIFY_LOG" | head -5 | while IFS= read -r line; do
echo " $line" | tee -a "$LOG_FILE"
done
add_rfi "Instance '$inst': p4verify.log contains $BAD_COUNT BAD and $MISSING_COUNT MISSING error(s) — depot archive integrity issue"
else
echo "OK: No BAD or MISSING errors in p4verify.log." | tee -a "$LOG_FILE"
fi
fi
echo "" | tee -a "$LOG_FILE"
echo "--- Workspace Staleness Check (instance: $inst) ---" | tee -a "$LOG_FILE"
if [[ "$SERVER_TYPE" == "commit" || "$SERVER_TYPE" == "edge" ]]; then
ONE_YEAR_AGO=$(date -d "1 year ago" +%s)
WS_COUNTS=$(bash -c "source /p4/common/bin/p4_vars '$inst' > /dev/null 2>&1 && \
p4 -ztag clients 2>/dev/null" | \
awk -v threshold="$ONE_YEAR_AGO" '
/^\.\.\. client / { total++ }
/^\.\.\. Access / { if ($NF+0 < threshold) stale++ }
END { print total+0, stale+0 }
')
WS_TOTAL=$(echo "$WS_COUNTS" | awk '{print $1}')
WS_STALE=$(echo "$WS_COUNTS" | awk '{print $2}')
if [[ "$WS_TOTAL" -eq 0 ]]; then
echo "No workspaces found." | tee -a "$LOG_FILE"
else
WS_PCT=$(awk "BEGIN {printf \"%.1f\", ($WS_STALE / $WS_TOTAL) * 100}")
echo "Total workspaces : $WS_TOTAL" | tee -a "$LOG_FILE"
echo "Not accessed in 1 year : $WS_STALE ($WS_PCT%)" | tee -a "$LOG_FILE"
if awk "BEGIN {exit !(($WS_STALE / $WS_TOTAL) * 100 > 10)}"; then
echo "WARNING: Over 10% of workspaces are stale." | tee -a "$LOG_FILE"
add_rfi "Instance '$inst': ${WS_PCT}% of workspaces (${WS_STALE}/${WS_TOTAL}) not accessed in over 1 year"
else
echo "OK: Stale workspace percentage is within threshold." | tee -a "$LOG_FILE"
fi
fi
else
echo "Skipped (not applicable for $SERVER_TYPE server)" | tee -a "$LOG_FILE"
fi
echo "" | tee -a "$LOG_FILE"
echo "--- Swarm Check (instance: $inst) ---" | tee -a "$LOG_FILE"
if [[ "$SERVER_TYPE" == "commit" ]]; then
SWARM_TRIGGERS=$(bash -c "source /p4/common/bin/p4_vars '$inst' > /dev/null 2>&1 && \
p4 triggers -o 2>/dev/null | grep -i swarm")
SWARM_EXTENSIONS=$(bash -c "source /p4/common/bin/p4_vars '$inst' > /dev/null 2>&1 && \
p4 extension --list --type extensions 2>/dev/null | grep -i swarm")
SWARM_PROPERTY=$(bash -c "source /p4/common/bin/p4_vars '$inst' > /dev/null 2>&1 && \
p4 property -l 2>/dev/null | grep -i swarm")
if [[ -z "$SWARM_TRIGGERS" && -z "$SWARM_EXTENSIONS" && -z "$SWARM_PROPERTY" ]]; then
echo "WARNING: No Swarm indicators found (triggers, extensions, or properties)." | tee -a "$LOG_FILE"
add_rfi "Instance '$inst': Swarm does not appear to be installed — no Swarm triggers, extensions, or properties found"
else
echo "OK: Swarm detected." | tee -a "$LOG_FILE"
[[ -n "$SWARM_TRIGGERS" ]] && echo " Via triggers : $(echo "$SWARM_TRIGGERS" | head -1)" | tee -a "$LOG_FILE"
[[ -n "$SWARM_EXTENSIONS" ]] && echo " Via extensions : $(echo "$SWARM_EXTENSIONS" | head -1)" | tee -a "$LOG_FILE"
[[ -n "$SWARM_PROPERTY" ]] && echo " Via property : $(echo "$SWARM_PROPERTY" | head -1)" | tee -a "$LOG_FILE"
fi
else
echo "Skipped (not a commit server)" | tee -a "$LOG_FILE"
fi
echo "" | tee -a "$LOG_FILE"
echo "--- External Authentication Check (instance: $inst) ---" | tee -a "$LOG_FILE"
if [[ "$SERVER_TYPE" == "commit" ]]; then
LDAP_OUT=$(bash -c "source /p4/common/bin/p4_vars '$inst' > /dev/null 2>&1 && \
p4 ldaps 2>/dev/null")
SSO_OUT=$(bash -c "source /p4/common/bin/p4_vars '$inst' > /dev/null 2>&1 && \
p4 extension --list --type extensions 2>/dev/null | grep -o login")
if [[ -z "$LDAP_OUT" && -z "$SSO_OUT" ]]; then
echo "WARNING: No external authentication configured (no LDAP specs, no SSO extension)." | tee -a "$LOG_FILE"
add_rfi "Instance '$inst': No external authentication configured — no LDAP specs and no SSO extension found"
else
echo "OK: External authentication is configured." | tee -a "$LOG_FILE"
[[ -n "$LDAP_OUT" ]] && echo "$LDAP_OUT" | while IFS= read -r line; do
echo " LDAP : $line" | tee -a "$LOG_FILE"
done
[[ -n "$SSO_OUT" ]] && echo " SSO : login extension present" | tee -a "$LOG_FILE"
fi
else
echo "Skipped (not a commit server)" | tee -a "$LOG_FILE"
fi
echo "" | tee -a "$LOG_FILE"
echo "--- Server Topology Check (instance: $inst) ---" | tee -a "$LOG_FILE"
if [[ "$SERVER_TYPE" == "commit" ]]; then
SERVERS_RAW=$(bash -c "source /p4/common/bin/p4_vars '$inst' > /dev/null 2>&1 && \
p4 -ztag servers -J 2>/dev/null")
# Collapse tagged output into one line per server: id type jnl seq alive
SERVERS_SUMMARY=$(echo "$SERVERS_RAW" | awk '
/^\.\.\. ServerID / { id=$3 }
/^\.\.\. ServerType / { type=$3 }
/^\.\.\. AppliedJournal / { jnl=$3 }
/^\.\.\. AppliedSequence / { seq=$3 }
/^\.\.\. IsAlive / { print id, type, jnl, seq, $3 }
')
# 'standard' is the legacy type for standalone servers; treat it the same as 'commit-server'
COMMIT_JNL=$(echo "$SERVERS_SUMMARY" | awk '$2=="commit-server"||$2=="standard"{print $3; exit}')
COMMIT_SEQ=$(echo "$SERVERS_SUMMARY" | awk '$2=="commit-server"||$2=="standard"{print $4; exit}')
echo "Commit journal position : $COMMIT_JNL/$COMMIT_SEQ" | tee -a "$LOG_FILE"
echo "" | tee -a "$LOG_FILE"
DR_COUNT=0
while IFS=' ' read -r srv_id srv_type srv_jnl srv_seq srv_alive; do
[[ "$srv_type" == "commit-server" || "$srv_type" == "standard" ]] && continue
[[ -z "$srv_id" ]] && continue
ALIVE_STR=$([[ "$srv_alive" == "1" ]] && echo "alive" || echo "not alive")
printf " %-30s %-22s jnl: %s/%s [%s]\n" \
"$srv_id" "$srv_type" "$srv_jnl" "$srv_seq" "$ALIVE_STR" | tee -a "$LOG_FILE"
# Count DR-capable servers (anything that is not an edge)
[[ "$srv_type" != "edge-server" ]] && DR_COUNT=$((DR_COUNT + 1))
# Journal lag check
if [[ "$srv_jnl" != "$COMMIT_JNL" || "$srv_seq" != "$COMMIT_SEQ" ]]; then
echo " WARNING: $srv_id is behind commit ($srv_jnl/$srv_seq vs $COMMIT_JNL/$COMMIT_SEQ)" | tee -a "$LOG_FILE"
add_rfi "Instance '$inst': $srv_id ($srv_type) is not at commit journal level ($srv_jnl/$srv_seq vs $COMMIT_JNL/$COMMIT_SEQ)"
fi
done <<< "$SERVERS_SUMMARY"
echo "" | tee -a "$LOG_FILE"
# DR coverage check
if [[ $DR_COUNT -eq 0 ]]; then
echo "WARNING: No DR-capable servers (replica/standby) found — this server has no disaster recovery coverage." | tee -a "$LOG_FILE"
add_rfi "Instance '$inst': No DR-capable replica or standby servers defined — no disaster recovery coverage"
else
echo "OK: $DR_COUNT DR-capable server(s) defined." | tee -a "$LOG_FILE"
fi
else
echo "Skipped (not a commit server)" | tee -a "$LOG_FILE"
fi
echo "" | tee -a "$LOG_FILE"
echo "--- db.have Fragmentation Check (instance: $inst) ---" | tee -a "$LOG_FILE"
DBHAVE_ROOT="/p4/${inst}/root/db.have"
DBHAVE_OFFDB="/p4/${inst}/offline_db/db.have"
ROOT_SIZE=$(du -sb "$DBHAVE_ROOT" 2>/dev/null | awk '{print $1}')
OFFDB_SIZE=$(du -sb "$DBHAVE_OFFDB" 2>/dev/null | awk '{print $1}')
if [[ -z "$ROOT_SIZE" ]]; then
echo "WARNING: $DBHAVE_ROOT not found — cannot compare." | tee -a "$LOG_FILE"
elif [[ -z "$OFFDB_SIZE" ]]; then
echo "WARNING: $DBHAVE_OFFDB not found — offline_db may not be initialised." | tee -a "$LOG_FILE"
else
OFFDB_PCT=$(awk "BEGIN {printf \"%.1f\", ($OFFDB_SIZE / $ROOT_SIZE) * 100}")
echo "root/db.have : $(numfmt --to=iec --suffix=B $ROOT_SIZE)" | tee -a "$LOG_FILE"
echo "offline_db/db.have : $(numfmt --to=iec --suffix=B $OFFDB_SIZE) (${OFFDB_PCT}% of root)" | tee -a "$LOG_FILE"
if awk "BEGIN {exit !($OFFDB_PCT < 75)}"; then
echo "WARNING: offline_db/db.have is less than 75% of root/db.have — database may be fragmented." | tee -a "$LOG_FILE"
add_rfi "Instance '$inst': offline_db/db.have is ${OFFDB_PCT}% of root/db.have — database defragmentation is recommended"
else
echo "OK: db.have sizes are within acceptable range." | tee -a "$LOG_FILE"
fi
fi
echo "" | tee -a "$LOG_FILE"
echo "--- verify_sdp.sh $inst -online ---" | tee -a "$LOG_FILE"
VERIFY_TMPFILE=$(mktemp)
/p4/common/bin/verify_sdp.sh "$inst" -online -L off 2>&1 | tee "$VERIFY_TMPFILE" | tee -a "$LOG_FILE"
INST_EXIT=${PIPESTATUS[0]}
VERIFY_ERRORS=$(grep -oE '[0-9]+ error' "$VERIFY_TMPFILE" | tail -1 | grep -oE '[0-9]+')
VERIFY_WARNINGS=$(grep -oE '[0-9]+ warning' "$VERIFY_TMPFILE" | tail -1 | grep -oE '[0-9]+')
rm -f "$VERIFY_TMPFILE"
if [[ "${VERIFY_ERRORS:-0}" -gt 0 ]] || [[ "${VERIFY_WARNINGS:-0}" -gt 0 ]]; then
add_rfi "Instance '$inst': verify_sdp.sh reported ${VERIFY_ERRORS:-0} error(s) and ${VERIFY_WARNINGS:-0} warning(s)"
fi
echo "--- p4sanity_check.sh $inst ---" | tee -a "$LOG_FILE"
SANITY_TMPFILE=$(mktemp)
bash -c "source /p4/common/bin/p4_vars '$inst' > /dev/null 2>&1 && /p4/common/bin/p4sanity_check.sh '$inst'" 2>&1 | tee "$SANITY_TMPFILE" | tee -a "$LOG_FILE"
SANITY_EXIT=${PIPESTATUS[0]}
if [[ $SANITY_EXIT -ne 0 ]]; then
SANITY_RESULT=$(grep -E '^(PASS|FAIL):' "$SANITY_TMPFILE" | tail -1)
add_rfi "Instance '$inst': p4sanity_check.sh reported non-clean output (${SANITY_RESULT:-exit code $SANITY_EXIT})"
fi
rm -f "$SANITY_TMPFILE"
echo "" | tee -a "$LOG_FILE"
echo "--- ccheck.sh $inst ---" | tee -a "$LOG_FILE"
CCHECK_TMPFILE=$(mktemp)
/p4/common/bin/ccheck.sh "$inst" -L off 2>&1 | tee "$CCHECK_TMPFILE" | tee -a "$LOG_FILE"
CCHECK_EXIT=${PIPESTATUS[0]}
CCHECK_BAD=$(grep -oE 'Configs BAD \(Required\):[[:space:]]+[0-9]+' "$CCHECK_TMPFILE" | grep -oE '[0-9]+$')
CCHECK_WARN=$(grep -oE 'Configs WARN \(Recommended\):[[:space:]]+[0-9]+' "$CCHECK_TMPFILE" | grep -oE '[0-9]+$')
rm -f "$CCHECK_TMPFILE"
if [[ "${CCHECK_BAD:-0}" -gt 0 ]] || [[ "${CCHECK_WARN:-0}" -gt 0 ]]; then
add_rfi "Instance '$inst': ccheck.sh reported ${CCHECK_BAD:-0} BAD (required) and ${CCHECK_WARN:-0} WARN (recommended) configurable(s)"
fi
echo "" | tee -a "$LOG_FILE"
[[ $CCHECK_EXIT -ne 0 ]] && OVERALL_EXIT=$CCHECK_EXIT
[[ $INST_EXIT -ne 0 ]] && OVERALL_EXIT=$INST_EXIT
done
# --- Memory and database size check ---
RAM_KB=$(grep MemTotal /proc/meminfo | awk '{print $2}')
RAM_BYTES=$((RAM_KB * 1024))
echo "========================================" | tee -a "$LOG_FILE"
echo "--- Memory and Database Size Check ---" | tee -a "$LOG_FILE"
echo "Server RAM : $(numfmt --to=iec --suffix=B $RAM_BYTES)" | tee -a "$LOG_FILE"
echo "Total db.* size : $(numfmt --to=iec --suffix=B $TOTAL_DB_SIZE) (across all instances)" | tee -a "$LOG_FILE"
echo "Total db.rev size : $(numfmt --to=iec --suffix=B $TOTAL_DB_REV_SIZE) (across all instances)" | tee -a "$LOG_FILE"
echo "" | tee -a "$LOG_FILE"
if [[ $TOTAL_DB_SIZE -gt $RAM_BYTES ]]; then
echo "WARNING: Total db.* size exceeds available RAM." | tee -a "$LOG_FILE"
add_rfi "Total db.* size ($(numfmt --to=iec --suffix=B $TOTAL_DB_SIZE)) exceeds server RAM ($(numfmt --to=iec --suffix=B $RAM_BYTES))"
else
echo "OK: Total db.* size is within RAM." | tee -a "$LOG_FILE"
fi
if [[ $TOTAL_DB_REV_SIZE -gt $RAM_BYTES ]]; then
echo "WARNING: Total db.rev size exceeds available RAM." | tee -a "$LOG_FILE"
add_rfi "Total db.rev size ($(numfmt --to=iec --suffix=B $TOTAL_DB_REV_SIZE)) exceeds server RAM ($(numfmt --to=iec --suffix=B $RAM_BYTES))"
else
echo "OK: Total db.rev size is within RAM." | tee -a "$LOG_FILE"
fi
echo "" | tee -a "$LOG_FILE"
# --- Room for Improvement summary ---
echo "========================================" | tee -a "$LOG_FILE"
echo "--- Room for Improvement ---" | tee -a "$LOG_FILE"
if [[ $RFI_POINTS -eq 0 ]]; then
echo "No issues found. Server looks healthy." | tee -a "$LOG_FILE"
else
echo "Total RFI points: $RFI_POINTS" | tee -a "$LOG_FILE"
for item in "${RFI_ITEMS[@]}"; do
echo " - $item" | tee -a "$LOG_FILE"
done
fi
echo "" | tee -a "$LOG_FILE"
echo "=== Health check complete. Exit code: $OVERALL_EXIT ===" | tee -a "$LOG_FILE"
echo "Log written to: $LOG_FILE"
exit $OVERALL_EXIT
| # | Change | User | Description | Committed | |
|---|---|---|---|---|---|
| #1 | 32832 | Mark Zinthefer |
Adding vibe-coded p4health.sh. Meant to be run at a customer site to generate a report of areas to improve upon. |