setup_lab.sh #4

#!/bin/bash
# setup_lab.sh
#
# PURPOSE:
#   Set up the trim_excess_metadata.sh test environment on a Battle School
#   Workshop lab that has been reset to "Lab 0" baseline.
#
# RUNS ON: p4c-bos-01 (as the perforce OS user)
#
# STARTING STATE: Standard Battle School Lab 0 reset.  All 5 servers up,
#   replication working, no gf-site customisations present.
#
# RESULT:
#   p4c-nyc-03 is a standalone commit server (ServerID: p4d_commit_gf) that
#   was once a filtered forwarding replica keeping only //jam/... and //pb/...
#   Ready for trim_excess_metadata.sh to run.
#
# USAGE:
#   cd /home/perforce/tem/test
#   bash setup_lab.sh [--svc-password <password>] [--dry-run]
#
# OPTIONS:
#   --svc-password  Password to set for the svc_p4d_ffr_gf service user.
#                   Default: ChangeMe99!  (fine for a lab)
#   --dry-run       Print commands without executing them.

set -euo pipefail

# ---------------------------------------------------------------------------
# Configuration
# ---------------------------------------------------------------------------

# Host names
BOS01="p4c-bos-01"
NYC03="p4c-nyc-03"

# SDP instance number
SDP_INSTANCE=1

# Site tag for the divested unit
SITE_TAG="gf"
SITE_TAG_DESC="GF - Filtered Forwarding Replica test site"

# New ServerID after promotion to standalone
STANDALONE_ID="p4d_commit_gf"

# Service user created by mkrep.sh
SVC_USER="svc_p4d_ffr_gf"

# Default lab password for the service user (safe for non-production lab only)
SVC_PASSWORD="ChangeMe99!"

# Depot paths kept by the RevisionDataFilter / ArchiveDataFilter
KEPT_DEPOT_PATHS="//jam/...
//pb/..."

# Directory where trim_excess_metadata.sh and keep files live
TEM_DIR="/home/perforce/tem"

# Checkpoint directory for the filtered seed checkpoint
FILTERED_CKP_DIR="/p4/${SDP_INSTANCE}/checkpoints.ffr_${SITE_TAG}"

DRY_RUN=0

# Unset P4CONFIG so that shell SDP environment variables (P4USER, P4PORT, etc.)
# take effect cleanly without being overridden by any P4CONFIG file.
unset P4CONFIG

# ---------------------------------------------------------------------------
# Argument parsing
# ---------------------------------------------------------------------------

while [[ $# -gt 0 ]]; do
   case "$1" in
      --svc-password) SVC_PASSWORD="$2"; shift 2 ;;
      --dry-run)      DRY_RUN=1; shift ;;
      *) echo "Unknown option: $1" >&2; exit 1 ;;
   esac
done

# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------

H1="============================================================"
H2="------------------------------------------------------------"

msg()  { echo -e "$*"; }
hdr()  { msg "\n${H1}\n$*\n${H2}"; }
step() { msg "  --> $*"; }

run() {
   # run CMD [ARGS...]  — execute or print depending on DRY_RUN
   if [[ "$DRY_RUN" -eq 1 ]]; then
      echo "  [DRY RUN] $*"
   else
      "$@"
   fi
}

run_ssh() {
   # run_ssh HOST CMD  — run CMD on HOST via ssh
   local host="$1"; shift
   if [[ "$DRY_RUN" -eq 1 ]]; then
      echo "  [DRY RUN] ssh perforce@${host}: $*"
   else
      ssh -n "perforce@${host}" "$@"
   fi
}

wait_for_p4d() {
   # wait_for_p4d HOST PORT  — poll until p4d responds or timeout
   local host="$1" port="${2:-1999}" tries=0 max=20
   step "Waiting for p4d on ${host}:${port} ..."
   while ! ssh -n "perforce@${host}" "p4 -p localhost:${port} -ztag info -s" \
         > /dev/null 2>&1; do
      tries=$((tries + 1))
      if [[ "$tries" -ge "$max" ]]; then
         echo "ERROR: p4d on ${host}:${port} did not come up after ${max} attempts." >&2
         return 1
      fi
      sleep 2
   done
   step "p4d is up on ${host}:${port}."
}

die() { echo "FATAL: $*" >&2; exit 1; }

# ---------------------------------------------------------------------------
# Phase 0: Pre-flight
# ---------------------------------------------------------------------------

hdr "Phase 0: Pre-flight checks"

# Must run on bos-01
THIS_HOST=$(hostname -s)
if [[ "$THIS_HOST" != "${BOS01%%.*}" ]]; then
   die "This script must run on ${BOS01} (currently on ${THIS_HOST})."
fi

step "Checking that all required commands are available ..."
for cmd in p4 p4d mkrep.sh rotate_journal.sh load_checkpoint.sh p4login; do
   command -v "$cmd" > /dev/null || die "Required command not found: $cmd"
done

step "Checking SSH access to ${NYC03} ..."
ssh -n "perforce@${NYC03}" "hostname" > /dev/null || die "Cannot SSH to ${NYC03}."

step "Verifying Lab 0 starting state (commit server at bos-01 is up) ..."
p4 -p "localhost:1999" -ztag info -s > /dev/null || die "p4d on bos-01 not responding."

msg "Pre-flight OK."

# ---------------------------------------------------------------------------
# Phase 1: Add gf site tag
# ---------------------------------------------------------------------------

hdr "Phase 1: Add '${SITE_TAG}' site tag to SiteTags.cfg"

SITE_TAGS_CFG="/p4/common/config/SiteTags.cfg"
if grep -q "^${SITE_TAG}:" "$SITE_TAGS_CFG" 2>/dev/null; then
   step "Site tag '${SITE_TAG}' already present — skipping."
else
   step "Appending site tag '${SITE_TAG}' to ${SITE_TAGS_CFG} ..."
   run bash -c "echo '${SITE_TAG}: ${SITE_TAG_DESC}' >> '${SITE_TAGS_CFG}'"
   step "Done."
fi

# ---------------------------------------------------------------------------
# Phase 2: Run mkrep.sh to create FFR server spec and service user
# ---------------------------------------------------------------------------

hdr "Phase 2: Run mkrep.sh -t ffr -s ${SITE_TAG} -r ${NYC03} -i ${SDP_INSTANCE}"

# Check whether the server spec already exists.
# `p4 server --exists -o <name>` errors if the spec doesn't exist (unlike plain
# `p4 server -o` which always returns a default template).
if p4 -p "localhost:1999" server --exists -o "p4d_ffr_${SITE_TAG}" \
      > /dev/null 2>&1; then
   step "Server spec 'p4d_ffr_${SITE_TAG}' already exists — skipping mkrep."
else
   step "Running mkrep.sh ..."
   run mkrep.sh -t ffr -s "$SITE_TAG" -r "$NYC03" -i "$SDP_INSTANCE"
   step "mkrep.sh complete."
fi

# ---------------------------------------------------------------------------
# Phase 3: Add RevisionDataFilter and ArchiveDataFilter to server spec
# ---------------------------------------------------------------------------

hdr "Phase 3: Add RevisionDataFilter / ArchiveDataFilter to p4d_ffr_${SITE_TAG}"

SPEC_TMP=$(mktemp)
p4 -p "localhost:1999" server -o "p4d_ffr_${SITE_TAG}" > "$SPEC_TMP"

# Check for a non-empty RevisionDataFilter (empty field always present in template)
if grep -qE "^[[:space:]]+//" "$SPEC_TMP" && grep -A5 "^RevisionDataFilter:" "$SPEC_TMP" | grep -q "^[[:space:]]//"; then
   step "RevisionDataFilter already set — skipping filter update."
else
   step "Setting RevisionDataFilter and ArchiveDataFilter via in-place sed ..."
   # p4 spec has empty "RevisionDataFilter:" and "ArchiveDataFilter:" lines;
   # replace each with the depot paths (in-place sed, newline via $'\n').
   sed -i "s|^RevisionDataFilter:\$|RevisionDataFilter:\n\t//jam/...\n\t//pb/...|" "$SPEC_TMP"
   sed -i "s|^ArchiveDataFilter:\$|ArchiveDataFilter:\n\t//jam/...\n\t//pb/...|" "$SPEC_TMP"

   if [[ "$DRY_RUN" -eq 1 ]]; then
      echo "  [DRY RUN] p4 server -i < (modified spec)"
   else
      p4 -p "localhost:1999" server -i < "$SPEC_TMP"
   fi
   step "Filter fields set."
fi
rm -f "$SPEC_TMP"

# ---------------------------------------------------------------------------
# Phase 4: Fix service user password
# ---------------------------------------------------------------------------

hdr "Phase 4: Set password for ${SVC_USER} (dm.user.resetpassword=1 workaround)"

# mkrep.sh creates the service user but dm.user.resetpassword=1 marks it as
# needing a reset.  Set a known lab password so the replica can log in.
step "Setting password for ${SVC_USER} ..."
if [[ "$DRY_RUN" -eq 1 ]]; then
   echo "  [DRY RUN] echo <password> | p4 passwd -P <password> ${SVC_USER}"
else
   printf '%s\n%s\n' "$SVC_PASSWORD" "$SVC_PASSWORD" \
      | p4 -p "localhost:1999" passwd "$SVC_USER" 2>&1 || true
fi
step "Password set."

# ---------------------------------------------------------------------------
# Phase 5: Rotate journal + create filtered seed checkpoint
# ---------------------------------------------------------------------------

hdr "Phase 5: Rotate journal and create filtered seed checkpoint"

FILTERED_CKP="${FILTERED_CKP_DIR}/p4_${SDP_INSTANCE}.ckp.ffr_${SITE_TAG}.1.gz"

if [[ -f "$FILTERED_CKP" ]]; then
   step "Filtered checkpoint already exists at ${FILTERED_CKP} — skipping."
else
   step "Rotating journal on bos-01 ..."
   run rotate_journal.sh "$SDP_INSTANCE"

   step "Creating filtered checkpoint directory on nyc-03 ..."
   run_ssh "$NYC03" "mkdir -p '${FILTERED_CKP_DIR}'"

   # Also need the dir locally for scp after remote creation
   run mkdir -p "$FILTERED_CKP_DIR"

   step "Generating filtered seed checkpoint from offline_db ..."
   # p4d_1 = SDP wrapper for instance 1; -P = server spec filter; -Z = gzip
   run p4d -r "/p4/${SDP_INSTANCE}/offline_db" \
      -P "p4d_ffr_${SITE_TAG}" -J off -Z \
      -jd "${FILTERED_CKP}" < /dev/null

   step "Checkpoint created: ${FILTERED_CKP}"
fi

# ---------------------------------------------------------------------------
# Phase 6: Initialise nyc-03 as the filtered replica
# ---------------------------------------------------------------------------

hdr "Phase 6: Stop nyc-03 p4d, set ServerID, load checkpoint, start"

TARGET_FFR_ID="p4d_ffr_${SITE_TAG}"

# Idempotency: check if p4d is actually running on nyc-03 with the right ServerID.
# The server.id file alone is not enough — the load may have failed.
NYC03_RUNNING_ID=$(ssh -n "perforce@${NYC03}" \
   "p4 -p localhost:1999 -ztag -F %ServerID% info -s 2>/dev/null || echo '(down)'" 2>/dev/null \
   || echo "(down)")

if [[ "$NYC03_RUNNING_ID" == "$TARGET_FFR_ID" ]]; then
   step "nyc-03 p4d is up with ServerID=${TARGET_FFR_ID} — skipping checkpoint load."
else
   step "Stopping p4d on nyc-03 ..."
   run_ssh "$NYC03" "sudo systemctl stop p4d_${SDP_INSTANCE}"
   sleep 3

   step "Copying filtered checkpoint (and .md5) to nyc-03 ..."
   if [[ "$DRY_RUN" -eq 0 ]]; then
      # Remove any pre-existing (read-only) checkpoint files before copying
      run_ssh "$NYC03" "rm -f '${FILTERED_CKP}' '${FILTERED_CKP}.md5'"
      scp "${FILTERED_CKP}" "${FILTERED_CKP}.md5" "perforce@${NYC03}:${FILTERED_CKP_DIR}/"
   else
      echo "  [DRY RUN] scp ${FILTERED_CKP} ${FILTERED_CKP}.md5 perforce@${NYC03}:${FILTERED_CKP_DIR}/"
   fi

   step "Setting ServerID to ${TARGET_FFR_ID} on nyc-03 ..."
   run_ssh "$NYC03" "echo '${TARGET_FFR_ID}' > /p4/${SDP_INSTANCE}/root/server.id"

   step "Loading checkpoint on nyc-03 ..."
   run_ssh "$NYC03" "load_checkpoint.sh '${FILTERED_CKP}' -i ${SDP_INSTANCE} -y"

   step "Starting p4d on nyc-03 ..."
   run_ssh "$NYC03" "sudo systemctl start p4d_${SDP_INSTANCE}"
   sleep 4

   wait_for_p4d "$NYC03"
fi

# ---------------------------------------------------------------------------
# Phase 7: Log in the service user on nyc-03
# ---------------------------------------------------------------------------

hdr "Phase 7: Log in service user and perforce user on nyc-03"

step "Logging in ${SVC_USER} on nyc-03 (required for replication pull threads) ..."
if [[ "$DRY_RUN" -eq 1 ]]; then
   echo "  [DRY RUN] echo <password> | p4 -p localhost:1999 -u ${SVC_USER} login"
else
   run_ssh "$NYC03" \
      "echo '${SVC_PASSWORD}' | p4 -p localhost:1999 -u '${SVC_USER}' login" || true
fi

step "Logging in perforce user on nyc-03 (required for p4verify in Phase 9) ..."
if [[ "$DRY_RUN" -eq 1 ]]; then
   echo "  [DRY RUN] ssh perforce@${NYC03}: p4login -v 1"
else
   ssh -n "perforce@${NYC03}" "p4login -v 1" 2>&1 || true
fi
step "User logins done."

# ---------------------------------------------------------------------------
# Phase 8: Blast existing archive files on nyc-03
# ---------------------------------------------------------------------------

hdr "Phase 8: Remove pre-existing depot archives on nyc-03"

step "Removing all files under /p4/${SDP_INSTANCE}/depots/ on nyc-03 ..."
run_ssh "$NYC03" "rm -rf /p4/${SDP_INSTANCE}/depots/*"
step "Archives cleared."

# ---------------------------------------------------------------------------
# Phase 9: Pull filtered archives via p4verify on nyc-03
# ---------------------------------------------------------------------------

hdr "Phase 9: Pull filtered archives via p4verify on nyc-03 (may take a few minutes)"

step "Running p4verify.sh ${SDP_INSTANCE} on nyc-03 ..."
step "(Only //jam/... and //pb/... archives should transfer,"
step " but lazy-copy backing files from //depot/... may also be pulled.)"
if [[ "$DRY_RUN" -eq 1 ]]; then
   echo "  [DRY RUN] ssh perforce@${NYC03}: p4verify.sh ${SDP_INSTANCE}"
else
   # p4verify can take several minutes; capture output for reference
   ssh -n "perforce@${NYC03}" "p4verify.sh ${SDP_INSTANCE}" 2>&1 \
      | grep -v "^$" | tail -20 || true
fi
step "p4verify done."

step "Pulling lazy-copy backing archives from //depot/... on nyc-03 ..."
step "(jam and pb streams have lazy copies backed by //depot/... archives;"
step " these must be present before standalone promotion so 'p4 snap' can work.)"
if [[ "$DRY_RUN" -eq 1 ]]; then
   echo "  [DRY RUN] ssh perforce@${NYC03}: p4 -p localhost:1999 verify -q //depot/..."
else
   ssh -n "perforce@${NYC03}" \
      "p4 -p localhost:1999 -u perforce verify -q //depot/..." 2>&1 \
      | grep -v "^$" || true
fi
step "Backing archive pull done."

# ---------------------------------------------------------------------------
# Phase 10: Promote nyc-03 to standalone commit server
# ---------------------------------------------------------------------------

hdr "Phase 10: Promote nyc-03 to standalone (ServerID: ${STANDALONE_ID})"

# Idempotency: check if p4d is actually running as the standalone ServerID.
NYC03_STANDALONE_CHECK=$(ssh -n "perforce@${NYC03}" \
   "p4 -p localhost:1999 -ztag info -s 2>/dev/null | awk '/^[.][.][.] ServerID /{id=\$3} /^[.][.][.] services /{svc=\$3} END{print id\"/\"svc}'" \
   2>/dev/null || echo "(down)")

if [[ "$NYC03_STANDALONE_CHECK" == "${STANDALONE_ID}/standard" ]]; then
   step "nyc-03 p4d is up as standalone (${STANDALONE_ID}) — skipping promotion."
else
   step "Stopping p4d on nyc-03 ..."
   run_ssh "$NYC03" "sudo systemctl stop p4d_${SDP_INSTANCE}"
   sleep 3

   step "Setting ServerID to ${STANDALONE_ID} (new ID avoids inheriting db.replication=readonly) ..."
   run_ssh "$NYC03" "echo '${STANDALONE_ID}' > /p4/${SDP_INSTANCE}/root/server.id"

   step "Starting p4d on nyc-03 ..."
   run_ssh "$NYC03" "sudo systemctl start p4d_${SDP_INSTANCE}"
   sleep 4

   wait_for_p4d "$NYC03"

   step "Logging in as perforce on nyc-03 (must precede configure commands) ..."
   if [[ "$DRY_RUN" -eq 1 ]]; then
      echo "  [DRY RUN] p4login on nyc-03"
   else
      ssh -n "perforce@${NYC03}" "p4login -v 1" 2>&1 || true
   fi

   step "Unsetting auth.id (standalone handles its own auth) ..."
   run_ssh "$NYC03" \
      "P4PORT=localhost:1999 p4 configure unset auth.id" || true

   step "Promotion complete."
fi

# ---------------------------------------------------------------------------
# Phase 11: Create test config files in TEM_DIR
# ---------------------------------------------------------------------------

hdr "Phase 11: Create test config files in ${TEM_DIR}"

P4CONFIG_GF="${TEM_DIR}/.p4config.gf"
KEEP_USERS="${TEM_DIR}/keep_users.gf.txt"
KEEP_GROUPS="${TEM_DIR}/keep_groups.gf.txt"

if [[ ! -f "$P4CONFIG_GF" ]]; then
   step "Creating ${P4CONFIG_GF} ..."
   run bash -c "cat > '${P4CONFIG_GF}' <<'EOCFG'
P4PORT=${NYC03}:1999
P4USER=perforce
P4TICKETS=${TEM_DIR}/.p4tickets
EOCFG"
else
   step "${P4CONFIG_GF} already exists — leaving as-is."
fi

if [[ ! -f "$KEEP_USERS" ]]; then
   step "Creating ${KEEP_USERS} ..."
   run bash -c "echo 'perforce' > '${KEEP_USERS}'"
else
   step "${KEEP_USERS} already exists — leaving as-is."
fi

if [[ ! -f "$KEEP_GROUPS" ]]; then
   step "Creating ${KEEP_GROUPS} ..."
   # 'testers' group has Randall_Scott as its only member — tests the
   # last-group-member retry logic in Phase 7 of trim_excess_metadata.sh.
   run bash -c "echo 'testers' > '${KEEP_GROUPS}'"
else
   step "${KEEP_GROUPS} already exists — leaving as-is."
fi

step "Config files ready."

# Get a fresh auth ticket for nyc-03 standalone into the tem-specific tickets
# file. p4login (SDP) targets bos-01 and won't help here; use raw 'p4 login'.
# The admin password file is the standard SDP location for instance 1.
P4ADMIN_PASSWD="/p4/common/config/.p4passwd.p4_1.admin"
step "Getting fresh auth ticket for nyc-03 into ${TEM_DIR}/.p4tickets ..."
if [[ "$DRY_RUN" -eq 1 ]]; then
   echo "  [DRY RUN] P4CONFIG=${P4CONFIG_GF} p4 login -a < ${P4ADMIN_PASSWD}"
elif [[ -f "$P4ADMIN_PASSWD" ]]; then
   P4CONFIG="$P4CONFIG_GF" p4 login -a < "$P4ADMIN_PASSWD"
   step "Fresh ticket obtained."
else
   step "Warning: ${P4ADMIN_PASSWD} not found — enter password manually:"
   P4CONFIG="$P4CONFIG_GF" p4 login -a
fi

# ---------------------------------------------------------------------------
# Phase 12: Verify final state
# ---------------------------------------------------------------------------

hdr "Phase 12: Verify final state on nyc-03"

step "Checking ServerID and server type ..."
if [[ "$DRY_RUN" -eq 0 ]]; then
   P4_INFO=$(ssh -n "perforce@${NYC03}" \
      "p4 -p localhost:1999 -ztag info -s" 2>/dev/null)
   SERVER_ID=$(echo "$P4_INFO"   | awk '/^[.][.][.] ServerID /     {print $3}')
   SERVICES=$(echo "$P4_INFO"    | awk '/^[.][.][.] serverServices /{print $3}')
   SERVER_ROOT=$(echo "$P4_INFO" | awk '/^[.][.][.] serverRoot /    {print $3}')

   if [[ "$SERVER_ID" == "$STANDALONE_ID" && "$SERVICES" == "standard" ]]; then
      step "✅  ServerID=${SERVER_ID}, Services=${SERVICES} — promotion confirmed."
   else
      step "⚠️   ServerID=${SERVER_ID}, Services=${SERVICES} — check promotion."
   fi
   step "P4ROOT: ${SERVER_ROOT}"
fi

step "Checking depot archive counts on nyc-03 ..."
if [[ "$DRY_RUN" -eq 0 ]]; then
   ARCHIVE_SUMMARY=$(ssh -n "perforce@${NYC03}" \
      "for d in /p4/${SDP_INSTANCE}/depots/*/; do \
         echo \"\$(find \$d -type f 2>/dev/null | wc -l) files: \$d\"; \
       done" 2>/dev/null)
   echo "$ARCHIVE_SUMMARY" | sed 's/^/    /'
fi

msg ""
msg "${H1}"
msg "Setup complete!"
msg ""
msg "Next step: cd ${TEM_DIR} && bash test/run_trim_test.sh"
msg "${H1}"