#!/bin/bash
################################################################################
# Script: check_replica_readiness.sh
# Purpose: Perform read-only checks on a Perforce replica server to verify
# readiness for p4 failover operation
# Usage: check_replica_readiness.sh [-text] <instance>
################################################################################
set -u # Exit on undefined variables
# Color codes for terminal output
RED='\033[0;31m'
YELLOW='\033[1;33m'
GREEN='\033[0;32m'
NC='\033[0m' # No Color
# Global variables
SCRIPT_NAME=$(basename "$0")
INSTANCE=""
TEXT_MODE=0
ERROR_COUNT=0
WARNING_COUNT=0
PASS_COUNT=0
REPORT=""
LOG_FILE=""
################################################################################
# Functions
################################################################################
usage() {
cat <<EOF
Usage: $SCRIPT_NAME [-text] <instance>
Options:
-text Output report to command line instead of email
Arguments:
instance The Perforce instance identifier (e.g., 1, 2, p4prod, etc.)
Description:
Performs comprehensive readiness checks for p4 failover on a replica server.
Checks include: replica status, license validity, trigger table, LDAP
connectivity, replication status, offline_db, sync logs, SSH connectivity,
and verify logs.
Output is always logged to /p4/<instance>/logs/check_replica_readiness.log
EOF
exit 1
}
log_pass() {
local message="$1"
PASS_COUNT=$((PASS_COUNT + 1))
if [ $TEXT_MODE -eq 1 ]; then
echo -e "${GREEN}[PASS]${NC} $message"
fi
REPORT+="[PASS] $message\n"
# Also write to log file if available
if [ -n "$LOG_FILE" ]; then
echo "[PASS] $message" >> "$LOG_FILE"
fi
}
log_warning() {
local message="$1"
WARNING_COUNT=$((WARNING_COUNT + 1))
if [ $TEXT_MODE -eq 1 ]; then
echo -e "${YELLOW}[WARN]${NC} $message"
fi
REPORT+="[WARN] $message\n"
# Also write to log file if available
if [ -n "$LOG_FILE" ]; then
echo "[WARN] $message" >> "$LOG_FILE"
fi
}
log_error() {
local message="$1"
ERROR_COUNT=$((ERROR_COUNT + 1))
if [ $TEXT_MODE -eq 1 ]; then
echo -e "${RED}[ERROR]${NC} $message"
fi
REPORT+="[ERROR] $message\n"
# Also write to log file if available
if [ -n "$LOG_FILE" ]; then
echo "[ERROR] $message" >> "$LOG_FILE"
fi
}
log_section() {
local section="$1"
if [ $TEXT_MODE -eq 1 ]; then
echo ""
echo "=========================================="
echo "$section"
echo "=========================================="
fi
REPORT+="\n==========================================\n"
REPORT+="$section\n"
REPORT+="==========================================\n"
# Also write to log file if available
if [ -n "$LOG_FILE" ]; then
echo "" >> "$LOG_FILE"
echo "==========================================" >> "$LOG_FILE"
echo "$section" >> "$LOG_FILE"
echo "==========================================" >> "$LOG_FILE"
fi
}
################################################################################
# Setup log file with rotation
################################################################################
setup_log_file() {
local log_dir="/p4/${INSTANCE}/logs"
LOG_FILE="${log_dir}/check_replica_readiness.log"
# Create logs directory if it doesn't exist
if [ ! -d "$log_dir" ]; then
mkdir -p "$log_dir" 2>/dev/null
if [ $? -ne 0 ]; then
echo "Warning: Could not create log directory: $log_dir"
LOG_FILE=""
return 1
fi
fi
# Rotate existing log file if it exists
if [ -f "$LOG_FILE" ]; then
local timestamp=$(date +%Y%m%d-%H%M%S)
local rotated_log="${log_dir}/check_replica_readiness.log.${timestamp}.log"
mv "$LOG_FILE" "$rotated_log" 2>/dev/null
if [ $? -eq 0 ]; then
echo "Rotated existing log to: $rotated_log" | tee -a "$LOG_FILE"
fi
fi
# Create new log file with header
{
echo "=========================================="
echo "Perforce Replica Readiness Check"
echo "Instance: ${INSTANCE}"
echo "Started: $(date '+%Y-%m-%d %H:%M:%S')"
echo "=========================================="
} > "$LOG_FILE"
return 0
}
################################################################################
# Get P4PORT from instance config
################################################################################
get_p4port() {
local config_file="/p4/common/config/p4_${INSTANCE}.vars"
if [ -f "$config_file" ]; then
source "$config_file"
echo "${P4PORT:-localhost:1666}"
else
echo "localhost:1666"
fi
}
################################################################################
# Check 1: Verify this is a replica server
################################################################################
check_replica_status() {
log_section "Replica Status Check"
if [ ! -x "/p4/common/bin/run_if_replica.sh" ]; then
log_error "run_if_replica.sh script not found or not executable"
return 1
fi
if /p4/common/bin/run_if_replica.sh "${INSTANCE}" echo is_replica > /dev/null 2>&1; then
log_pass "Server is confirmed as a replica server"
return 0
else
log_error "Server is NOT a replica server - cannot proceed with failover readiness checks"
return 1
fi
}
################################################################################
# Check 2: License validation
################################################################################
check_license() {
log_section "License Validation"
local license_file=""
# Check for license file existence
if [ -f "/p4/${INSTANCE}/root/license" ]; then
license_file="/p4/${INSTANCE}/root/license"
elif [ -f "/p4/${INSTANCE}/root/license.txt" ]; then
license_file="/p4/${INSTANCE}/root/license.txt"
else
log_error "License file not found (checked /p4/${INSTANCE}/root/license and /p4/${INSTANCE}/root/license.txt)"
return 1
fi
log_pass "License file found at: $license_file"
# Extract p4port from license (may not always be present)
local license_p4port=$(grep -i "^P4PORT:" "$license_file" | awk '{print $2}' | tr -d '\r')
# Get server hostname and IP
local server_hostname=$(hostname -f)
local server_ip=$(hostname -I | awk '{print $1}')
# Extract IP address from license file (could be IPv4 or IPv6)
local license_ip=$(grep -i "^IPaddress:" "$license_file" | awk '{print $2}' | tr -d '\r')
# Check if license p4port or IP matches hostname or IP
local license_match=0
if [ -n "$license_p4port" ]; then
if [[ "$license_p4port" == *"$server_hostname"* ]] || [[ "$license_p4port" == *"$server_ip"* ]]; then
license_match=1
fi
fi
# Also check if license IP matches server IP (for IPv4 and IPv6)
if [ -n "$license_ip" ]; then
# Extract just the address part from formats like [00:50:56:bc:74:42]:1664
local clean_license_ip=$(echo "$license_ip" | sed 's/\[//g' | sed 's/\].*//g')
# Get all network interfaces and MAC addresses
local server_macs=$(ip link show 2>/dev/null | grep -i "link/ether" | awk '{print $2}' | tr '[:lower:]' '[:upper:]')
if [ -z "$server_macs" ]; then
# Fallback for systems without ip command
server_macs=$(ifconfig 2>/dev/null | grep -i "ether" | awk '{print $2}' | tr '[:lower:]' '[:upper:]')
fi
# Convert license address to uppercase for comparison
local upper_license_ip=$(echo "$clean_license_ip" | tr '[:lower:]' '[:upper:]')
# Check if it's an IPv4 address match
if [[ "$clean_license_ip" == "$server_ip"* ]] || [[ "$server_ip" == "$clean_license_ip"* ]]; then
license_match=1
fi
# Check all IPs on the server
local all_ips=$(hostname -I)
for ip in $all_ips; do
if [[ "$clean_license_ip" == "$ip" ]] || [[ "$license_ip" == *"$ip"* ]]; then
license_match=1
break
fi
done
# Check if it matches a MAC address (format like 00:50:56:bc:74:42)
if [[ "$upper_license_ip" =~ ^[0-9A-F]{2}:[0-9A-F]{2}:[0-9A-F]{2}:[0-9A-F]{2}:[0-9A-F]{2}:[0-9A-F]{2}$ ]]; then
for mac in $server_macs; do
if [[ "$upper_license_ip" == "$mac" ]]; then
license_match=1
break
fi
done
fi
fi
if [ $license_match -eq 1 ]; then
if [ -n "$license_p4port" ]; then
log_pass "License P4PORT/IPaddress matches server hostname/IP (P4PORT: $license_p4port, IPaddress: $license_ip)"
else
log_pass "License IPaddress matches server hostname/IP/MAC (IPaddress: $license_ip)"
fi
else
if [ -n "$license_p4port" ] || [ -n "$license_ip" ]; then
log_error "License P4PORT ($license_p4port) and IPaddress ($license_ip) do NOT match server hostname ($server_hostname) or IPs ($server_ip)"
else
log_error "Could not extract P4PORT or IPaddress from license file"
fi
fi
# Check expiration date
local expiration_date=$(grep -i "^Expiration:" "$license_file" | awk '{print $2}' | tr -d '\r')
# If Expiration field not found, try Support-Expires (epoch format)
if [ -z "$expiration_date" ]; then
local support_expires=$(grep -i "^Support-Expires:" "$license_file" | awk '{print $2}' | tr -d '\r')
if [ -n "$support_expires" ]; then
# Convert epoch to human-readable date
expiration_date=$(date -d "@$support_expires" "+%Y/%m/%d" 2>/dev/null || date -r "$support_expires" "+%Y/%m/%d" 2>/dev/null)
local exp_epoch="$support_expires"
else
log_warning "Could not extract expiration date from license file"
return 0
fi
else
# Convert expiration date to epoch (assuming YYYY/MM/DD format)
local exp_epoch=$(date -d "${expiration_date/\//-}" +%s 2>/dev/null)
if [ $? -ne 0 ]; then
log_warning "Could not parse expiration date: $expiration_date"
return 0
fi
fi
local current_epoch=$(date +%s)
local days_until_expiration=$(( (exp_epoch - current_epoch) / 86400 ))
if [ $days_until_expiration -lt 0 ]; then
log_error "License has EXPIRED (expiration date: $expiration_date)"
elif [ $days_until_expiration -le 30 ]; then
log_warning "License expires in $days_until_expiration days (expiration date: $expiration_date)"
else
log_pass "License is valid, expires in $days_until_expiration days (expiration date: $expiration_date)"
fi
}
################################################################################
# Check 3: Trigger table validation
################################################################################
check_triggers() {
log_section "Trigger Table Validation"
log_warning "Trigger validation is experimental. Please validate manually regardless of output."
# Get P4PORT from instance config
local p4port=$(get_p4port)
# Get trigger table
local triggers_output=$(p4 -p "${p4port}" triggers -o 2>&1)
if [ $? -ne 0 ]; then
log_error "Failed to retrieve trigger table: $triggers_output"
return 1
fi
# Parse trigger lines (skip comments and empty lines)
local trigger_count=0
local missing_files=0
local temp_file=$(mktemp)
local missing_triggers_file=$(mktemp)
# Save triggers to temp file for processing
echo "$triggers_output" | grep -v "^#" | grep -v "^$" | grep "^[[:space:]]*[^T]" > "$temp_file"
while IFS= read -r line; do
# Skip the "Triggers:" header line
if [[ "$line" =~ ^Triggers: ]]; then
continue
fi
# Extract trigger definition (format: name type path command)
# Trigger line format: trigger-name trigger-type trigger-path "command %quote%"
# Use cut instead of awk to avoid printf issues with % characters
local trigger_name=$(echo "$line" | awk '{print $1}')
if [ -z "$trigger_name" ]; then
continue
fi
trigger_count=$((trigger_count + 1))
# Extract the command part (everything after the third field)
# Remove quotes and get file paths
local trigger_command=$(echo "$line" | sed 's/^[[:space:]]*[^[:space:]]*[[:space:]]*[^[:space:]]*[[:space:]]*[^[:space:]]*[[:space:]]*//' | sed 's/"//g')
if [ -z "$trigger_command" ]; then
continue
fi
# Extract local file paths (absolute paths starting with /)
# Look for interpreter paths and script paths
local files_to_check=$(echo "$trigger_command" | grep -oE '(/[^[:space:]"]+)' | grep -v '%')
for file_path in $files_to_check; do
# Skip if path contains perforce placeholders
if [[ "$file_path" =~ % ]]; then
continue
fi
# Remove any trailing % characters
file_path=$(echo "$file_path" | sed 's/%.*$//')
# Skip empty paths
if [ -z "$file_path" ]; then
continue
fi
# Check if file exists
if [ ! -f "$file_path" ] && [ ! -x "$file_path" ]; then
echo "Trigger file missing: $file_path (from trigger: $trigger_name)" >> "$missing_triggers_file"
missing_files=$((missing_files + 1))
fi
done
done < "$temp_file"
# Report missing files with digest if there are many
if [ $missing_files -gt 0 ]; then
local missing_count=$(wc -l < "$missing_triggers_file")
if [ $missing_count -le 10 ]; then
# Show all missing files
while IFS= read -r missing_line; do
log_error "$missing_line"
done < "$missing_triggers_file"
else
# Show first 10 and indicate there are more
local shown=0
while IFS= read -r missing_line && [ $shown -lt 10 ]; do
log_error "$missing_line"
shown=$((shown + 1))
done < "$missing_triggers_file"
local remaining=$((missing_count - 10))
log_error "... and $remaining more missing trigger files (total: $missing_count missing files)"
fi
fi
rm -f "$temp_file" "$missing_triggers_file"
if [ $trigger_count -eq 0 ]; then
log_pass "No triggers defined in trigger table"
elif [ $missing_files -eq 0 ]; then
log_pass "All trigger files are present on replica server ($trigger_count triggers checked)"
fi
}
################################################################################
# Check 4: LDAP connectivity
################################################################################
check_ldap() {
log_section "LDAP Connectivity Check"
# Get P4PORT from instance config
local p4port=$(get_p4port)
# Get LDAP configurations
local ldap_output=$(p4 -p "${p4port}" ldaps -o 2>&1)
if [ $? -ne 0 ]; then
log_warning "Failed to retrieve LDAP configurations or none configured"
return 0
fi
# Parse LDAP specs
local ldap_count=0
local current_name=""
local current_host=""
local current_port=""
local current_enabled=""
while IFS= read -r line; do
if [[ "$line" =~ ^Name:[[:space:]]*(.+) ]]; then
# Process previous LDAP spec if exists
if [ -n "$current_name" ]; then
ldap_count=$((ldap_count + 1))
test_ldap_connection "$current_name" "$current_host" "$current_port" "$current_enabled"
fi
current_name="${BASH_REMATCH[1]}"
current_host=""
current_port=""
current_enabled=""
elif [[ "$line" =~ ^Host:[[:space:]]*(.+) ]]; then
current_host="${BASH_REMATCH[1]}"
elif [[ "$line" =~ ^Port:[[:space:]]*(.+) ]]; then
current_port="${BASH_REMATCH[1]}"
elif [[ "$line" =~ ^Enabled:[[:space:]]*(.+) ]]; then
current_enabled="${BASH_REMATCH[1]}"
fi
done <<< "$ldap_output"
# Process last LDAP spec
if [ -n "$current_name" ]; then
ldap_count=$((ldap_count + 1))
test_ldap_connection "$current_name" "$current_host" "$current_port" "$current_enabled"
fi
if [ $ldap_count -eq 0 ]; then
log_pass "No LDAP specifications configured"
fi
}
test_ldap_connection() {
local name="$1"
local host="$2"
local port="$3"
local enabled="$4"
if [ -z "$host" ] || [ -z "$port" ]; then
log_warning "LDAP spec '$name': Missing host or port information"
return
fi
# Test connectivity using timeout and nc (netcat) or telnet
if command -v nc > /dev/null 2>&1; then
timeout 5 nc -zv "$host" "$port" > /dev/null 2>&1
local result=$?
elif command -v telnet > /dev/null 2>&1; then
timeout 5 bash -c "echo quit | telnet $host $port" > /dev/null 2>&1
local result=$?
else
log_warning "LDAP spec '$name': Cannot test connectivity (nc or telnet not available)"
return
fi
if [ $result -eq 0 ]; then
log_pass "LDAP spec '$name': Connection successful to $host:$port (enabled: ${enabled:-unknown})"
else
if [[ "$enabled" == "enabled" ]]; then
log_error "LDAP spec '$name': Cannot connect to $host:$port (ENABLED spec)"
else
log_warning "LDAP spec '$name': Cannot connect to $host:$port (disabled spec)"
fi
fi
}
################################################################################
# Check 5: Replication journal status
################################################################################
check_replication_journal() {
log_section "Replication Journal Status (p4 pull -lj)"
# Get P4PORT from instance config
local p4port=$(get_p4port)
# Get replication status
local pull_output=$(p4 -p "${p4port}" pull -lj 2>&1)
if [ $? -ne 0 ]; then
log_error "Failed to execute 'p4 pull -lj': $pull_output"
return 1
fi
# Parse output for journal numbers and byte counts
# Expected format variations:
# "Current replica journal state is: Journal 13556, Sequence 382606269."
# OR older format: "... journal 123 pos 456 ..."
# Can have multiple lines if replica is behind - use tail -1 to get current state
local master_journal=""
local replica_journal=""
local master_bytes=""
local replica_bytes=""
# Try new format first (with "Journal" and "Sequence")
# Use tail -1 to get the most recent/current state if there are multiple lines
master_journal=$(echo "$pull_output" | grep -i "master.*journal" | tail -1 | grep -oE 'Journal [0-9]+' | awk '{print $2}')
replica_journal=$(echo "$pull_output" | grep -i "replica.*journal" | tail -1 | grep -oE 'Journal [0-9]+' | awk '{print $2}')
master_bytes=$(echo "$pull_output" | grep -i "master.*sequence" | tail -1 | grep -oE 'Sequence [0-9]+' | awk '{print $2}')
replica_bytes=$(echo "$pull_output" | grep -i "replica.*sequence" | tail -1 | grep -oE 'Sequence [0-9]+' | awk '{print $2}')
# If new format didn't work, try old format
if [ -z "$master_journal" ]; then
master_journal=$(echo "$pull_output" | grep -i "master" | tail -1 | grep -oE 'journal [0-9]+' | awk '{print $2}')
fi
if [ -z "$replica_journal" ]; then
replica_journal=$(echo "$pull_output" | grep -i "replica" | tail -1 | grep -oE 'journal [0-9]+' | awk '{print $2}')
fi
if [ -z "$master_bytes" ]; then
master_bytes=$(echo "$pull_output" | grep -i "master" | tail -1 | grep -oE 'pos [0-9]+' | awk '{print $2}')
fi
if [ -z "$replica_bytes" ]; then
replica_bytes=$(echo "$pull_output" | grep -i "replica" | tail -1 | grep -oE 'pos [0-9]+' | awk '{print $2}')
fi
# Check journal numbers
if [ -z "$master_journal" ] || [ -z "$replica_journal" ]; then
log_warning "Could not parse journal numbers from 'p4 pull -lj' output"
log_warning "Raw output: $pull_output"
return 0
fi
if [ "$master_journal" -ne "$replica_journal" ]; then
log_error "Replica is NOT syncing on current journal (Master: $master_journal, Replica: $replica_journal)"
else
log_pass "Replica is syncing on current journal number ($replica_journal)"
# Check byte count difference (only if on same journal)
if [ -n "$master_bytes" ] && [ -n "$replica_bytes" ]; then
local byte_diff=$((master_bytes - replica_bytes))
local byte_diff_abs=${byte_diff#-} # Absolute value
local percent_diff=0
if [ $master_bytes -gt 0 ]; then
percent_diff=$(( (byte_diff_abs * 100) / master_bytes ))
fi
if [ $percent_diff -gt 10 ]; then
log_warning "Replica byte position differs by more than 10% (Master: $master_bytes, Replica: $replica_bytes, Diff: ${percent_diff}%)"
else
log_pass "Replica byte position is within 10% of master (Master: $master_bytes, Replica: $replica_bytes, Diff: ${percent_diff}%)"
fi
fi
fi
}
################################################################################
# Check 6: Replication state status
################################################################################
check_replication_state() {
log_section "Replication State Status (p4 pull -ls)"
# Get P4PORT from instance config
local p4port=$(get_p4port)
# Get replication state
local pull_state=$(p4 -p "${p4port}" pull -ls 2>&1)
if [ $? -ne 0 ]; then
log_error "Failed to execute 'p4 pull -ls': $pull_state"
return 1
fi
# Check if output contains all zeros (indicating no pending changes)
# The output format typically shows pending integrates, submits, etc.
local non_zero=$(echo "$pull_state" | grep -oE '[1-9][0-9]*')
if [ -z "$non_zero" ]; then
log_pass "Replication state shows all zeros (no pending operations)"
else
log_error "Replication state shows non-zero values (pending operations detected)"
if [ $TEXT_MODE -eq 1 ]; then
echo "$pull_state"
fi
REPORT+="$pull_state\n"
fi
}
################################################################################
# Check 7: Offline database directory
################################################################################
check_offline_db() {
log_section "Offline Database Check"
local offline_db_dir="/p4/${INSTANCE}/offline_db"
local current_time=$(date +%s)
# Check if directory exists
if [ ! -d "$offline_db_dir" ]; then
log_error "Offline database directory does not exist: $offline_db_dir"
return 1
fi
log_pass "Offline database directory exists: $offline_db_dir"
# Check for offline_db_usable.txt
local usable_file="${offline_db_dir}/offline_db_usable.txt"
if [ ! -f "$usable_file" ]; then
log_error "File offline_db_usable.txt not found in $offline_db_dir"
else
log_pass "File offline_db_usable.txt is present"
# Check timestamp on usable file (within past 7 days)
local usable_mtime=$(stat -c %Y "$usable_file" 2>/dev/null || stat -f %m "$usable_file" 2>/dev/null)
local usable_age_days=$(( (current_time - usable_mtime) / 86400 ))
if [ $usable_age_days -gt 7 ]; then
log_error "File offline_db_usable.txt is older than 7 days ($usable_age_days days old)"
else
log_pass "File offline_db_usable.txt is recent ($usable_age_days days old)"
fi
fi
# Check for db.* files
local db_files_count=$(ls -1 "$offline_db_dir"/db.* 2>/dev/null | wc -l)
if [ "$db_files_count" -eq 0 ]; then
log_error "No db.* files found in $offline_db_dir"
else
log_pass "Found $db_files_count db.* files in offline_db directory"
# Find the most recent db.* file using ls with sorting by modification time
local most_recent_db=$(ls -t "$offline_db_dir"/db.* 2>/dev/null | head -1)
if [ -n "$most_recent_db" ]; then
local db_mtime=$(stat -c %Y "$most_recent_db" 2>/dev/null || stat -f %m "$most_recent_db" 2>/dev/null)
local db_age_days=$(( (current_time - db_mtime) / 86400 ))
if [ $db_age_days -gt 7 ]; then
log_warning "Most recent db.* file is older than 7 days ($db_age_days days old): $(basename $most_recent_db)"
else
log_pass "Most recent db.* file is recent ($db_age_days days old): $(basename $most_recent_db)"
fi
fi
fi
}
################################################################################
# Check 8: Sync replica log
################################################################################
check_sync_replica_log() {
log_section "Sync Replica Log Check"
local sync_log="/p4/${INSTANCE}/logs/sync_replica.log"
if [ ! -f "$sync_log" ]; then
log_warning "Sync replica log file not found: $sync_log"
return 0
fi
log_pass "Sync replica log file exists: $sync_log"
# Check log file age
local log_mtime=$(stat -c %Y "$sync_log" 2>/dev/null || stat -f %m "$sync_log" 2>/dev/null)
local current_time=$(date +%s)
local log_age_days=$(( (current_time - log_mtime) / 86400 ))
if [ $log_age_days -gt 7 ]; then
log_warning "Sync replica log file is older than 7 days ($log_age_days days old)"
else
log_pass "Sync replica log file is recent ($log_age_days days old)"
fi
# Check for error strings in the log
local error_count=$(grep -c -i "error" "$sync_log" 2>/dev/null)
# Ensure error_count is a valid integer
if ! [[ "$error_count" =~ ^[0-9]+$ ]]; then
error_count=0
fi
if [ "$error_count" -gt 0 ]; then
log_warning "Sync replica log contains $error_count instances of 'error'"
# Include sample of errors
if [ $TEXT_MODE -eq 1 ]; then
echo "Sample of errors:"
grep -i "error" "$sync_log" | tail -5
fi
REPORT+="Sample of errors:\n"
REPORT+="$(grep -i 'error' "$sync_log" | tail -5)\n"
else
log_pass "Sync replica log contains no error strings"
fi
}
################################################################################
# Check 9: SSH connectivity to P4TARGET
################################################################################
check_ssh_to_target() {
log_section "SSH Connectivity to P4TARGET"
# Get P4PORT from instance config
local p4port=$(get_p4port)
# Get server ID from replica
local server_id_file="/p4/${INSTANCE}/root/server.id"
if [ ! -f "$server_id_file" ]; then
log_warning "Server ID file not found: $server_id_file"
return 0
fi
local server_id=$(cat "$server_id_file" | tr -d '\r\n')
if [ -z "$server_id" ]; then
log_warning "Could not read server ID from $server_id_file"
return 0
fi
log_pass "Replica server ID: $server_id"
# Get P4TARGET for this server ID
local p4target=$(p4 -p "${p4port}" configure show allservers 2>/dev/null | grep "P4TARGET" | grep "$server_id" | awk -F'=' '{print $2}' | awk '{print $1}' | tr -d '\r\n')
if [ -z "$p4target" ]; then
log_warning "Could not find P4TARGET configuration for server ID: $server_id"
return 0
fi
log_pass "P4TARGET found: $p4target"
# Extract hostname from P4TARGET (format might be hostname:port or just hostname)
local target_host=$(echo "$p4target" | cut -d':' -f1)
# Test SSH connectivity
if timeout 10 ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$target_host" date > /dev/null 2>&1; then
log_pass "SSH connectivity to P4TARGET ($target_host) successful"
# Check timezone match between replica and target
local replica_tz=$(date +%Z)
local target_tz=$(timeout 10 ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$target_host" "date +%Z" 2>/dev/null | tr -d '\r\n')
if [ -z "$target_tz" ]; then
log_warning "Could not retrieve timezone from P4TARGET server"
elif [ "$replica_tz" != "$target_tz" ]; then
log_warning "Timezone mismatch: Replica is $replica_tz, P4TARGET is $target_tz - replica timezone needs adjustment"
else
log_pass "Timezone match confirmed: Both servers are in $replica_tz"
fi
else
log_warning "SSH connectivity to P4TARGET ($target_host) failed - SSH keys might be broken"
fi
}
################################################################################
# Check 10: Verify log analysis
################################################################################
check_verify_log() {
log_section "P4 Verify Log Analysis"
local verify_log="/p4/${INSTANCE}/logs/p4verify.log"
if [ ! -f "$verify_log" ]; then
log_warning "Verify log file not found: $verify_log"
return 0
fi
log_pass "Verify log file exists: $verify_log"
# Check log file age
local log_mtime=$(stat -c %Y "$verify_log" 2>/dev/null || stat -f %m "$verify_log" 2>/dev/null)
local current_time=$(date +%s)
local log_age_days=$(( (current_time - log_mtime) / 86400 ))
if [ $log_age_days -gt 7 ]; then
log_warning "Verify log file is older than 7 days ($log_age_days days old)"
else
log_pass "Verify log file is recent ($log_age_days days old)"
fi
# Check for MISSING or BAD files
local missing_count=$(grep -c "MISSING!" "$verify_log" 2>/dev/null || echo "0")
local bad_count=$(grep -c "BAD!" "$verify_log" 2>/dev/null || echo "0")
if [ $missing_count -gt 0 ] || [ $bad_count -gt 0 ]; then
log_warning "Verify log shows issues: $missing_count MISSING! files, $bad_count BAD! files"
# Include sample of issues
if [ $TEXT_MODE -eq 1 ]; then
echo "Sample of issues:"
grep "MISSING!\|BAD!" "$verify_log" | head -10
fi
REPORT+="Sample of issues:\n"
REPORT+="$(grep 'MISSING!\|BAD!' "$verify_log" | head -10)\n"
else
log_pass "Verify log shows no MISSING! or BAD! files"
fi
}
################################################################################
# Generate and send report
################################################################################
send_report() {
local summary="Perforce Replica Readiness Report - Instance ${INSTANCE}\n"
summary+="Generated: $(date '+%Y-%m-%d %H:%M:%S')\n"
summary+="Hostname: $(hostname -f)\n\n"
summary+="Summary: ${PASS_COUNT} passed, ${WARNING_COUNT} warnings, ${ERROR_COUNT} errors\n"
local full_report="${summary}\n${REPORT}"
# Write summary to log file
if [ -n "$LOG_FILE" ]; then
{
echo ""
echo "=========================================="
echo "SUMMARY"
echo "=========================================="
echo -e "${summary}"
echo ""
if [ $ERROR_COUNT -gt 0 ]; then
echo "Total Errors: ${ERROR_COUNT}"
fi
if [ $WARNING_COUNT -gt 0 ]; then
echo "Total Warnings: ${WARNING_COUNT}"
fi
if [ $ERROR_COUNT -eq 0 ] && [ $WARNING_COUNT -eq 0 ]; then
echo "All checks passed!"
fi
echo ""
echo "Log file: $LOG_FILE"
} >> "$LOG_FILE"
fi
if [ $TEXT_MODE -eq 1 ]; then
echo ""
echo "=========================================="
echo "SUMMARY"
echo "=========================================="
echo -e "${summary}"
echo ""
if [ $ERROR_COUNT -gt 0 ]; then
echo -e "${RED}Total Errors: ${ERROR_COUNT}${NC}"
fi
if [ $WARNING_COUNT -gt 0 ]; then
echo -e "${YELLOW}Total Warnings: ${WARNING_COUNT}${NC}"
fi
if [ $ERROR_COUNT -eq 0 ] && [ $WARNING_COUNT -eq 0 ]; then
echo -e "${GREEN}All checks passed!${NC}"
fi
if [ -n "$LOG_FILE" ]; then
echo ""
echo "Log file: $LOG_FILE"
fi
else
# Send email report
local email_address=""
if [ -f "/p4/common/config/p4_${INSTANCE}.vars" ]; then
source "/p4/common/config/p4_${INSTANCE}.vars"
email_address="${MAILTO:-}"
fi
if [ -z "$email_address" ]; then
log_error "No email address found in /p4/common/config/p4_${INSTANCE}.vars (MAILTO variable)"
echo -e "$full_report"
return 1
fi
local subject="Perforce Replica Readiness Report - Instance ${INSTANCE}"
if [ $ERROR_COUNT -gt 0 ]; then
subject="${subject} - ERRORS DETECTED"
elif [ $WARNING_COUNT -gt 0 ]; then
subject="${subject} - Warnings"
else
subject="${subject} - All Clear"
fi
echo -e "$full_report" | mail -s "$subject" "$email_address"
if [ $? -eq 0 ]; then
echo "Report sent successfully to $email_address"
else
echo "Failed to send email report"
echo -e "$full_report"
fi
fi
}
################################################################################
# Main execution
################################################################################
main() {
# Parse arguments
while [ $# -gt 0 ]; do
case "$1" in
-text)
TEXT_MODE=1
shift
;;
-h|--help)
usage
;;
*)
if [ -z "$INSTANCE" ]; then
INSTANCE="$1"
else
echo "Error: Unknown argument '$1'"
usage
fi
shift
;;
esac
done
# Validate instance argument
if [ -z "$INSTANCE" ]; then
echo "Error: Instance identifier is required"
usage
fi
# Validate instance format (alphanumeric)
if ! [[ "$INSTANCE" =~ ^[0-9a-zA-Z]+$ ]]; then
echo "Error: Instance must be alphanumeric (letters and numbers only)"
usage
fi
# Start checks
if [ $TEXT_MODE -eq 1 ]; then
echo "=========================================="
echo "Perforce Replica Readiness Check"
echo "Instance: ${INSTANCE}"
echo "Started: $(date '+%Y-%m-%d %H:%M:%S')"
echo "=========================================="
fi
# Setup log file with rotation
setup_log_file
# Run all checks
check_replica_status
if [ $? -ne 0 ]; then
# If not a replica, stop here
send_report
exit 1
fi
check_license
check_triggers
check_ldap
check_replication_journal
check_replication_state
check_offline_db
check_sync_replica_log
check_ssh_to_target
check_verify_log
# Generate and send report
send_report
# Exit with appropriate code
if [ $ERROR_COUNT -gt 0 ]; then
exit 1
elif [ $WARNING_COUNT -gt 0 ]; then
exit 2
else
exit 0
fi
}
# Run main function
main "$@"