SDP-functions.ps1 #1

# ============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE
# ----------------------------------------------------------------------------

<#
    .Synopsis
        Various SDP related shared functions
        
    .Description
        Functions intended to be called by other scripts.
        This script is not intended to be run directly.

        $Id: //p4-sdp/dev/Server/Windows/p4/common/bin/SDP-functions.ps1#1 $
#>

Set-StrictMode -Version 2.0

# Try and avoid 2 byte output in log files!
$PSDefaultParameterValues['*:Encoding'] = 'utf8'

Function Parse-SDPConfigFile([string]$scriptpath) {
    $SDPConfigFile = Split-Path -parent $scriptpath | Join-Path -childpath "..\..\config\sdp_config.ini"
    $SDPConfigFile = Resolve-Path $SDPConfigFile
    $ini = Parse-Inifile($SDPConfigFile)

    $section = "${SDPInstance}:${env:computername}".tolower()
    if ($ini[$section] -eq $null) {
        throw @"
Section '$section' not found in ${SDPConfigFile} - did you specify a valid SDPInstance parameter
(before the ':') or are you on the wrong machine (hostname - is the value after the ':')?
"@
    }
    
    try {
        $global:sdp_global_root = $ini[$section]["sdp_global_root"]
        $global:sdp_serverid = $ini[$section]["sdp_serverid"]
        $global:sdp_p4serviceuser = $ini[$section]["sdp_p4serviceuser"]
        $global:sdp_p4superuser = $ini[$section]["sdp_p4superuser"]
        $global:admin_pass_filename = $ini[$section]["admin_pass_filename"]
        $global:email_pass_filename = $ini[$section]["email_pass_filename"]
        $global:mailfrom = $ini[$section]["mailfrom"]
        $global:maillist = $ini[$section]["maillist"]
        $global:mailhost = $ini[$section]["mailhost"]
        $global:mailhostport = $ini[$section]["mailhostport"]
        $global:python = $ini[$section]["python"]
        $global:remote_depotdata_root = $ini[$section]["remote_depotdata_root"]
        $global:keepckps = 0
        $global:keeplogs = 0
        $result = [int32]::TryParse($ini[$section]["keepckps"], [ref]$global:keepckps)
        $result = [int32]::TryParse($ini[$section]["keeplogs"], [ref]$global:keeplogs)
        $global:limit_one_daily_checkpoint = $false
        $value = $ini[$section]["limit_one_daily_checkpoint"]
        if ($value -match "true|1|yes|y") {
            $global:limit_one_daily_checkpoint = $true
        }
        $global:remote_sdp_instance = $ini[$section]["remote_sdp_instance"]
        $global:p4target = $ini[$section]["p4target"]
    }
    catch {
        $message = $_.Exception.GetBaseException().Message 
        Write-Error $message 
        throw "Error parsing ini file: section ${section}"
    }

    $global:SDP_INSTANCE_HOME = -join($global:SDP_GLOBAL_ROOT, "\p4\", $SDPInstance)
    $global:SDP_INSTANCE_BIN_DIR = -join($global:SDP_INSTANCE_HOME, "\bin")
    $global:P4exe =  -join($global:SDP_INSTANCE_BIN_DIR, "\p4.exe")
    $global:P4Dexe =  -join($global:SDP_INSTANCE_BIN_DIR, "\p4d.exe")
    $global:P4DVersion = "1970.0" # See set-vars below
    Ensure-PathExists $global:p4exe
    Ensure-PathExists $global:p4dexe
    $global:SDP_INSTANCE_SSL_DIR = -join($global:SDP_INSTANCE_HOME, "\ssl")
    $global:P4CONFIG = "p4config.txt"
    $global:P4ROOT = -join($global:SDP_INSTANCE_HOME, "\root")
    $global:P4JOURNAL = -join($global:SDP_INSTANCE_HOME, "\logs\journal")
    $global:P4USER = $global:SDP_P4SUPERUSER
    $env:P4TICKETS = -join($global:SDP_INSTANCE_HOME, "\P4Tickets.txt")
    $env:P4TRUST = -join($global:SDP_INSTANCE_HOME, "\P4Trust.txt")
    $env:P4ENVIRO = -join($global:SDP_INSTANCE_HOME, "\P4Enviro.txt")
    $global:SDP_INSTANCE_P4SERVICE_NAME = -join("p4_", $SdpInstance)
    $global:P4PORT = Get-P4PORT
    $global:SCRIPTS_DIR = -join($global:SDP_GLOBAL_ROOT, "\p4\common\bin")
    $global:SVCINST_EXE = -join($global:SCRIPTS_DIR, "\svcinst.exe")
    $global:CONFIG_DIR = -join($global:SDP_GLOBAL_ROOT, "\p4\config")
    $global:LOGS_DIR = -join($global:SDP_INSTANCE_HOME, "\logs")
    $global:OFFLINE_DB_DIR = -join($global:SDP_INSTANCE_HOME, "\offline_db")
    $global:CHECKPOINTS_DIR = -join($global:SDP_INSTANCE_HOME, "\checkpoints")
    $global:REMOTE_CHECKPOINTS_DIR = -join($global:REMOTE_DEPOTDATA_ROOT, "\p4\", $global:REMOTE_SDP_INSTANCE, "\checkpoints")
    $global:ADMIN_PASS_FILE = -join($global:CONFIG_DIR, "\", $global:ADMIN_PASS_FILENAME)
    $global:EMAIL_PASS_FILE = -join($global:CONFIG_DIR, "\", $global:EMAIL_PASS_FILENAME)
    $Global:IS_EDGESERVER = $false
    $global:IS_STANDBYSERVER = $false
    $global:IS_REPLICASERVER = $false
    # The various Edge-specific tables for checkpoint/restore - used with -K/k parameter
    # Note that this is p4d version specific - see set-vars
    $global:EXCLUDED_EDGE_TABLES = "db.have,db.working,db.resolve,db.locks,db.revsh,db.workingx,db.resolvex"
    $global:EDGE_CHECKPOINT_TABLES = "$global:EXCLUDED_EDGE_TABLES,db.view,db.label,db.revsx,db.revux"
    $serverid_file = -join($global:P4ROOT, "\server.id")
    $global:SERVERID = get-content $serverid_file
    $global:P4LOG = -join($global:SDP_INSTANCE_HOME, "\logs\", $global:SERVERID, ".log")
    # PATH=%SDP_INSTANCE_BIN_DIR%;%SCRIPTS_DIR%;%PATH%
    Ensure-PathExists $global:SDP_INSTANCE_BIN_DIR
    Ensure-PathExists $global:P4ROOT
    Ensure-PathExists $global:LOGS_DIR
    Ensure-PathExists $global:OFFLINE_DB_DIR
    Ensure-PathExists $global:CHECKPOINTS_DIR
    Ensure-PathExists $global:ADMIN_PASS_FILE
    Set-vars
}

Function Set-vars {
    $dbfile = "$global:P4ROOT\db.server"
    if (!(Test-Path -path $dbfile)) {
        return
    }
    # SERVERVAL from the server record bitwise anded with 4096 indicates an edge server.
    # See https://www.perforce.com/perforce/doc.current/schema/#ServerServiceType
    $serverval = & $p4dexe -r $global:P4ROOT -jd - db.server | select-string "@$global:SERVERID@" | %{$_.line.split()[8]}
    if ($serverval -is [array]) {
        $val = $serverval[0]
    } else {
        $val = $serverval
    }
    [int]$serverint = [convert]::toint32($val, 10)
    if ($serverint -eq 0x19ED) {
        $global:IS_EDGESERVER = $true
    }
    if (($serverint -eq 0x8945) -or ($serverint -eq 0x89E5)) {
        $global:IS_STANDBYSERVER = $true
    }
    if (($serverint -eq 0x0945) -or ($serverint -eq 0x09E5) -or ($serverint -eq 0x094D)) {
        $global:IS_REPLICASERVER = $true
    }
    
    $global:p4dversion = & $global:P4dexe -V | select-string "^Rev" | %{$_.line.split("/")[2]}
    if ($global:p4dversion -gt "19.0") {
        $global:EXCLUDED_EDGE_TABLES = "$global:EXCLUDED_EDGE_TABLES,db.stash,db.haveg,db.workingg,db.locksg,db.resolveg"
        $global:EDGE_CHECKPOINT_TABLES = "$global:EXCLUDED_EDGE_TABLES,db.view,db.label,db.revsx,db.revux"
    }
}

Function Write-Env-Var-File([string]$path) {
    # Writes a file containing CMD env var statements
    @"
set SDP_INSTANCE_HOME=$global:SDP_INSTANCE_HOME
set SDP_INSTANCE_BIN_DIR=$global:SDP_INSTANCE_BIN_DIR
set SDP_INSTANCE_SSL_DIR=$global:SDP_INSTANCE_SSL_DIR
set P4CONFIG=$global:P4CONFIG
set P4ROOT=$global:P4ROOT
set P4LOG=$global:P4LOG
set P4JOURNAL=$global:P4JOURNAL
set P4USER=$global:P4USER
set P4TICKETS=$env:P4TICKETS
set P4TRUST=$env:P4TRUST
set P4ENVIRO=$env:P4ENVIRO
set SDP_INSTANCE_P4SERVICE_NAME=$global:SDP_INSTANCE_P4SERVICE_NAME
set P4PORT=$global:P4PORT
set SCRIPTS_DIR=$global:SCRIPTS_DIR
set SVCINST_EXE=$global:SVCINST_EXE
set CONFIG_DIR=$global:CONFIG_DIR
set LOGS_DIR=$global:LOGS_DIR
set OFFLINE_DB_DIR=$global:OFFLINE_DB_DIR
set CHECKPOINTS_DIR=$global:CHECKPOINTS_DIR
set REMOTE_CHECKPOINTS_DIR=$global:REMOTE_CHECKPOINTS_DIR
set SERVERID=$global:SERVERID
"@ | set-content -path $path -encoding oem
}

Function Get-P4PORT {
    $regkey = "HKLM:\SYSTEM\CurrentControlSet\services\${global:SDP_INSTANCE_P4SERVICE_NAME}\parameters"
    (get-itemproperty $regkey).P4PORT
}

Function Get-Date-Time () {
    Get-Date -format "yyyy\/MM\/dd HH\:mm\:ss"
}

Function Ensure-PathExists([string]$path) {
    if (!(Test-Path -path $path)) {
        throw "Error - required path $path doesn't exist!"
    }
}

Function Create-LogFile () {
    if (!$global:LogFileName) {
        throw "ERROR - Global:LogFileName not set"
    }
    $global:LogFile = Join-Path $global:LOGS_DIR $global:LogFileName
    Rotate-CurrentLogFile
    write-debug "Using logfile: $global:logfile"
    write-output "Starting ${global:ScriptTask}" | set-content -path $global:Logfile
}

Function Append-To-File([string]$message, [string]$filename) {
    "$message" | add-content -path $filename -encoding UTF8
}

Function Log([string]$message) {
    # Logs output to file and to console
    $datetime = get-date-time
    write-host "$datetime $message"
    Append-To-File "$datetime $message" $global:Logfile
}

Function LogException([exception]$exception) {
    $datetime = get-date-time
    $message = $exception.message
    write-error -message "$datetime $message" -Exception $exception
    write-output "$datetime $message" | add-content -path $global:Logfile -encoding UTF8
}

Function Parse-IniFile ($file) {
    $ini = @{}

    # Create a default section if none exist in the file. Like a java prop file.
    $section = "NO_SECTION"
    $ini[$section] = @{}

    switch -regex -file $file {
        "^\[(.+)\]$" {
            $section = $matches[1].Trim().ToLower()
            $ini[$section] = @{}
        }
        "^\s*([^#].+?)\s*=\s*(.*)" {
            $name,$value = $matches[1..2]
            # skip comments that start with semicolon:
            if (!($name.StartsWith(";"))) {
                $ini[$section][$name] = $value.Trim()
            }
        }
    }
    $ini
}

Function Invoke-P4Login () {
    Log "Logging in to P4 Instance"
    # Due to the vagaries of powershell launching processes with redirection
    # it is easier to spawn a cmd prompt.
    $cmd = "$global:P4exe -p $global:P4PORT -u $global:SDP_P4SUPERUSER login -a < $global:ADMIN_PASS_FILE >> $global:LogFile 2>&1"
    Log $cmd
    cmd /c $cmd
    if ($lastexitcode -ne 0) {
        throw "ERROR - failed to login!"
    }
}

Function Run-ReplicaStatus ([string]$statuslog) {
    Log "Getting replica status"
    $result = & $global:P4exe -p $global:P4PORT -u $global:SDP_P4SUPERUSER pull -lj 2>&1
    Log $result
    if ((Test-Path $statuslog -pathtype leaf)) {
        remove-item $statuslog
    }
    Append-To-File "$result" $statuslog
    if ($lastexitcode -ne 0) {
        Log "WARNING - pull command exited with error code"
    }
}

Function Check-OfflineDBExists () {
    Log "Checking offline DB exists"
    $path = Join-Path $global:OFFLINE_DB_DIR "db.counters"
    Log "checking existence of $path"
    if (!(Test-Path $path -pathtype leaf)) {
        throw "ERROR - offline_db doesn't exist!"
    }
}

Function Ensure-CheckpointNotRunning () {
    Log "Ensuring there is no other checkpoint script running"
    $checkFile = Join-Path $global:CHECKPOINTS_DIR "ckp_running.txt"
    Log "checking existence of $checkFile"
    if (Test-Path $checkFile -pathtype leaf) {
        throw "Error - a checkpoint operation is already running! If in doubt please check with [email protected] for advice!! If you know that this is from a previous failure you may delete the file manually and re-run the checkpoint operation you were trying..."
    }
    write-output "Checkpoint running" | set-content -path $checkFile
}

Function Signal-CheckpointComplete () {
    $checkFile = Join-Path $global:CHECKPOINTS_DIR "ckp_running.txt"
    remove-item $checkFile
}

Function Get-CurrentJournalCounter () {
    Log "Getting current live journal counter"
    $cmd = "$global:P4exe -p $global:P4PORT -u $global:SDP_P4SUPERUSER counter journal"
    $result = run-cmd-get-output $cmd "Error - failed to get live journal counter!"
    [int]$global:JOURNAL_NUM = $result
    [int]$global:CHECKPOINT_NUM = $global:JOURNAL_NUM + 1        
    Log "Live journal/checkpoint counters ${global:JOURNAL_NUM}/${global:CHECKPOINT_NUM}"
}

Function Get-JournalCounter ([string]$rootdir) {
    $cmd = "$global:P4Dexe -r $rootdir -jd - db.counters"
    Log $cmd
    $rawjournal = Invoke-Expression $cmd | select-string "@journal@" | %{$_.line.split()[4]}
    Log $rawjournal
    $journal = [regex]::match($rawjournal, '^@([0-9]+)@$').Groups[1].Value
    if ($lastexitcode -ne 0) {
        throw "Error - failed to get $rootdir journal counter!"
    }
    return [int]$journal
}

Function Get-OfflineJournalCounter () {
    Log "Getting offline journal counter"
    $journal = Get-JournalCounter $global:OFFLINE_DB_DIR
    Log "Offline journal counter $journal"
    [int]$global:Offline_journal_num = $journal
}

Function Rotate-File ([string]$Rotate_file) {
    if (Test-Path $Rotate_file -pathtype leaf) {
        $suffix = (Get-ChildItem $Rotate_file).lastwritetime | Get-Date -format "yyyyMMdd\-HHmmss"
        $newname = "$Rotate_file.$suffix"
        Log "Rotating $Rotate_file to $newname"
        rename-file "$Rotate_file" "$newname"
    }
}

Function Rotate-CurrentLogFile () {
    $Rotate_logname = $global:LOGFILE
    if (Test-Path $Rotate_logname -pathtype leaf) {
        $suffix = (Get-ChildItem $Rotate_logname).lastwritetime | Get-Date -format "yyyyMMdd\-HHmmss"
        $newname = "$Rotate_logname.$suffix"
        Log "Rotating $Rotate_logname to $newname"
        rename-file "$Rotate_logname" "$newname"
    }
}

Function Rename-File ([string]$old_name, [string]$new_name) {
    # Retries frequently to cope with busy servers which may lock log files
    $max_retries = 60 * 10  # 60 seconds at 10 times per second
    $retries = $max_retries
    $sleep_ms = 100
    if (!(Test-Path $old_name -pathtype leaf)) {
        return
    }
    While ($true) {
        try {
            # The -ea stop turns error into terminating error so it can be caught as exception
            rename-item -path $old_name -newname $new_name -ea stop
            return
        }
        catch {
            if ($retries -eq 0) {
                Log "Failed to rename file after $max_retries retries."
                return
            }
            Start-Sleep -MilliSeconds $sleep_ms
            $retries = $retries - 1
        }
    }
}

Function Rotate-LogFile () {
    param([string]$Rotate_logname, [bool]$zip)
    if (Test-Path $Rotate_logname -pathtype leaf) {
        $newname = "$Rotate_logname.$global:logid"
        Log "Rotating $Rotate_logname to $newname"
        rename-file "$Rotate_logname" "$newname"
        if ($zip) {
            $gzip = -join($global:SCRIPTS_DIR, "\gzip.exe")
            $cmd = "$gzip $newname"
            run-cmd $cmd "failed to gzip file"
        }
    }
}

Function Rotate-Logfiles () {
    Log "Rotating various logfiles"
    $datetime = Get-Date -format "yyyyMMdd\-HHmmss"
    $global:LOGID = "${global:JOURNAL_NUM}.$datetime"

    Rotate-LogFile "${global:sdp_serverid}.log" $true
    Rotate-LogFile "log" $true
    Rotate-LogFile "p4broker.log" $true
    Rotate-LogFile "audit.log" $true
    Rotate-LogFile "sync_replica.log" $false
    Rotate-LogFile "rotate-log-files.log" $false
    Rotate-LogFile "create-filtered-edge-checkpoint.log" $false
    Rotate-LogFile "recreate-offline-db-from-checkpoint.log" $false
    Rotate-LogFile "recover-edge.log" $false
    Rotate-LogFile "p4verify.log" $false
    Rotate-LogFile "replica_status.log" $false
    Rotate-LogFile "Weekly-sync-replica.log" $false
}

Function Move-Files ([string]$source, [string]$target) {
    Get-ChildItem -Path $source |
        foreach-object {
            $f = $_.FullName
            Log "Moving $f to $target"
            move-item $f $target
        }
}

Function Copy-Files ([string]$source, [string]$target) {
    Get-ChildItem -Path $source |
        foreach-object {
            $f = $_.FullName
            Log "Copying $f to $target"
            copy-item $f $target
        }
}

Function Remove-Files {
    # Remove older files while keeping a specified minimum number
    param([string]$remove_filename, [int]$keepnum)

    $file_path = -join($remove_filename, "*")
    $files = @(Get-ChildItem -Path $file_path | Sort-Object -Property LastWriteTime -Descending)
    if ($files) {
        for ($j = $keepnum; $j -lt $files.count; $j++) {
            $f = $files[$j].FullName
            Log("Removing: $f")
            remove-item $f -Force
        }
    }
}

Function Remove-OldLogs () {
    # Remove old Checkpoint Logs
    # Use KEEPCKPS rather than KEEPLOGS, so we keep the same number
    # of checkpoint logs as we keep checkpoints.
    if ($global:keepckps -eq 0) {
        Log "Skipping cleanup of old checkpoint logs because KEEPCKPS is set to 0."
    } else {
        log "Deleting old checkpoint logs.  Keeping latest $global:KEEPCKPS, per KEEPCKPS setting in sdp_config.ini."
        remove-files "checkpoint.log" $global:KEEPCKPS
        remove-files $global:P4LOG $global:KEEPLOGS
        remove-files "log" $global:KEEPLOGS
        remove-files "p4broker.log" $global:KEEPLOGS
        remove-files "audit.log" $global:KEEPLOGS
        remove-files "sync_replica.log" $global:KEEPLOGS
        remove-files "replica_status.log" $global:KEEPLOGS
        remove-files "p4verify.log" $global:KEEPLOGS
        remove-files "recreate_offline_db.log" $global:KEEPLOGS
        remove-files "upgrade.log" $global:KEEPLOGS
        remove-files "rotate-log-files.log" $global:KEEPLOGS
        remove-files "create-filtered-edge-checkpoint.log" $global:KEEPLOGS
        remove-files "recreate-offline-db-from-checkpoint.log" $global:KEEPLOGS
        remove-files "recover-edge.log" $global:KEEPLOGS
        remove-files "p4verify.log" $global:KEEPLOGS
        remove-files "replica_status.log" $global:KEEPLOGS
        remove-files "Weekly-sync-replica.log" $global:KEEPLOGS
    }
}

Function Is-Replica () {
    # If a replica then add -t for transfer to args
    $target = & $global:P4exe -p $global:P4PORT -u $global:SDP_P4SUPERUSER configure show 2>&1 | select-string "P4TARGET="
    Log $target
    if ($lastexitcode -ne 0) {
        return $false
    }
    if ($target -ne $null -and $target -ne "") {
        return $true
    }
    return $false
}

Function Truncate-Journal () {
	if ($global:IS_EDGESERVER) {
		Log "[EDGE] Truncating live journal"
		$checkpoint_path = -join($global:CHECKPOINTS_DIR, ".", $global:sdp_serverid, "\", $global:SDP_INSTANCE_P4SERVICE_NAME, ".", $global:sdp_serverid, ".ckp.", $global:Checkpoint_num, ".gz")
	}
	else {
    Log "Truncating live journal"
		$checkpoint_path = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME, ".ckp.", $global:Checkpoint_num, ".gz")
	}
    if (Test-Path $checkpoint_path -pathtype leaf) {
		#### HERE is edge
		if ($global:IS_EDGESERVER) {
		    $checkFile = Join-Path $global:CHECKPOINTS_DIR "ckp_running.txt"
			remove-item $checkFile
			Log "[EDGE] Removed ckp_running.txt awaiting likely a journal rotation"
			throw "Likely awaiting journal rotation"
		}
		else {
			throw "ERROR - $checkpoint_path BUT ITS ELSE already exists, check the backup process"
		}
		
		
    }
    if ($global:IS_EDGESERVER -or $global:IS_STANDBYSERVER -or $global:IS_REPLICASERVER) {
        log "Skipping truncation of journal as this is not standard/commit server..."
    } else {
        # Specific path customization for edge servers
        #if ($global:IS_EDGESERVER) {
		#	log "[EDGE] Truncating journal..."
        #    $journalprefix = -join($global:CHECKPOINTS_DIR, ".", $global:sdp_serverid, "\", $global:SDP_INSTANCE_P4SERVICE_NAME, ".", $global:sdp_serverid)
        #} else {
            $journalprefix = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME)
        #}
        $journalpath = -join($journalprefix, ".jnl.", $global:JOURNAL_NUM)
        if (Test-Path $journalpath -pathtype leaf) {
            throw "ERROR - $journalpath already exists, 2check the backup process"
        }
        log "Truncating journal..."
        # 'p4d -jj' does a copy-then-delete, instead of a simple mv.
        # during 'p4d -jj' the perforce server will hang the responses to clients.
        $cmd = "$global:P4Dexe -r $global:P4ROOT -J $global:P4JOURNAL -jj"
        run-cmd $cmd "ERROR - attempting to truncate journal"
    }
}


Function Run-Checkpoint () {
	if ($global:IS_EDGESERVER) {
		$checkpoint_prefix = -join($global:CHECKPOINTS_DIR, ".", $global:sdp_serverid, "\", $global:SDP_INSTANCE_P4SERVICE_NAME, ".", $global:sdp_serverid)
	}
	else {
		$checkpoint_prefix = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME)
	}
    $cmd = "$global:P4DEXE -r $global:P4ROOT -J $global:P4JOURNAL -jc -Z $checkpoint_prefix"
    run-cmd $cmd "ERROR - attempting to checkpoint live database"
}

Function Replay-JournalsToOfflineDB () {
    Log "Applying all outstanding journal files to offline_db"
    # On Master/Commit we replay up to live journal, on edge/standby one less
    $last_journal = $global:JOURNAL_NUM
    # Adjust journal processing for edge/standby/replica
    if ($global:IS_EDGESERVER -or $global:IS_STANDBYSERVER -or $global:IS_REPLICASERVER) {
        $last_journal = $last_journal - 1
    }
    for ($j = $global:Offline_journal_num; $j -le $last_journal; $j++) {
        # Specific path customization only for edge servers
        if ($global:IS_EDGESERVER) {
			Log "[EDGE] Replaying Journals To Offlinedb"
            $journalprefix = -join($global:CHECKPOINTS_DIR, ".", $global:sdp_serverid, "\", $global:SDP_INSTANCE_P4SERVICE_NAME, ".", $global:sdp_serverid)

        } else {
            $journalprefix = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME)
        }
        $journalpath = -join($journalprefix, ".jnl.", $j)
        $cmd = "$global:P4DEXE -r $global:OFFLINE_DB_DIR -f -jr $journalpath"
        run-cmd $cmd "ERROR - attempting to replay journal"
    }
}

function Create-OfflineCheckpoint {
	if ($global:IS_EDGESERVER) {
		Log "[EDGE] Creating offline checkpoint"
		$checkpoint_prefix = -join($global:CHECKPOINTS_DIR, ".", $global:sdp_serverid, "\", $global:SDP_INSTANCE_P4SERVICE_NAME, ".", $global:sdp_serverid, ".ckp.")
	}
	else {
		Log "Creating offline checkpoint"
		$checkpoint_prefix = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME, ".ckp.")
	}
    if ($global:limit_one_daily_checkpoint -ne 0) {
        $curr_date = Get-Date -format "yyyyMMdd"
        $checkpoint_path = -join($checkpoint_prefix, "*.gz")
        $files = @(Get-ChildItem -Path $checkpoint_path | Sort-Object -Property LastWriteTime -Descending)
        if (!$files -or $files.count -eq 0) {
           Log "LIMIT_ONE_DAILY_CHECKPOINT is set to true, but no checkpoint exists."
		   
        } else {
            $ckp_date = $files[0].LastWriteTime | Get-Date -format "yyyyMMdd"
            if ($curr_date -eq $ckp_date) {
                Log "A checkpoint was already created today, and LIMIT_ONE_DAILY_CHECKPOINT is set to true."
                Log "Skipping offline checkpoint dump."
                return
            }
        }
    }
    $checkpoint_path = -join($checkpoint_prefix, $global:checkpoint_num, ".gz")
    $cmd = "$global:P4DEXE -r $global:OFFLINE_DB_DIR -jd -z $checkpoint_path"
    run-cmd $cmd "ERROR - attempting to create offline checkpoint"
}

Function Check-OfflineDBUsable () {
    $offline_db_usable = -join($global:OFFLINE_DB_DIR, "\offline_db_usable.txt")
    if (!(test-path $offline_db_usable -pathtype leaf)) {
        $msg = "Offline database not in a usable state. Check the backup process."
        Log $msg
        throw $msg
    }
}

Function Recreate-OfflineDBFiles () {
    Log "Recreating offline db files"
    $offline_db_usable = -join($global:OFFLINE_DB_DIR, "\offline_db_usable.txt")
	if ($global:IS_EDGESERVER) {
		$checkpoint_prefix = -join($global:CHECKPOINTS_DIR, ".", $global:sdp_serverid, "\", $global:SDP_INSTANCE_P4SERVICE_NAME, ".", $global:sdp_serverid, ".ckp.")
	}
	else {
		$checkpoint_prefix = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME, ".ckp.")
	}
    $checkpoint_path = -join($checkpoint_prefix, "*.gz")
    $files = @(Get-ChildItem -Path $checkpoint_path | Sort-Object -Property LastWriteTime -Descending)
    if (!$files -or $files.count -eq 0) {
        $msg = "No checkpoints found - run live-checkpoint.ps1"
        Log $msg
        throw $msg
    }
    if (test-path $offline_db_usable -pathtype leaf) {
        remove-item $offline_db_usable
    }
    remove-files "${global:OFFLINE_DB_DIR}\db." 0
    
    Log "Recovering from latest checkpoint found"
    $checkpoint_path = $files[0]
    $cmd = "$global:P4DEXE -r $global:OFFLINE_DB_DIR -f -jr -z $checkpoint_path"
    run-cmd $cmd "ERROR - attempting to recover offline db files"
    "Offline db file restored successfully." | set-content -path $offline_db_usable
}

Function Remove-OldCheckpointsAndJournals () {
    if ($global:keepckps -eq 0) {
        Log "Skipping cleanup of old checkpoints because KEEPCKPS is set to 0."
    } else {
        log "Deleting obsolete checkpoints and journals.  Keeping latest $global:KEEPCKPS, per KEEPCKPS setting in sdp_config.ini."
		if ($global:IS_EDGESERVER) {
			Log "[EDGE] Deleting"
			$checkpoint_prefix = -join($global:CHECKPOINTS_DIR, ".", $global:sdp_serverid, "\", $global:SDP_INSTANCE_P4SERVICE_NAME, ".", $global:sdp_serverid, ".ckp.")
			$journal_prefix = -join($global:CHECKPOINTS_DIR, ".", $global:sdp_serverid, "\", $global:SDP_INSTANCE_P4SERVICE_NAME, ".", $global:sdp_serverid, ".jnl.")
	}
		else {
			Log "Deleting"
			$checkpoint_prefix = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME, ".ckp.")
			$journal_prefix = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME, ".jnl.")
		}
        # We multiply KEEPCKP by 2 for the ckp files because of the md5 files.
        remove-files $checkpoint_prefix ($global:KEEPCKPS * 2)
        remove-files $journal_prefix ($global:KEEPCKPS)
    }
}

Function run-cmd-no-check ([string]$cmd) {
    Log $cmd
    $result = Invoke-Expression "$cmd >> $global:Logfile 2>&1"
    Log $result
}

# This version returns the result
Function run-cmd-get-output ([string]$cmd, [string]$errmsg) {
    Log $cmd
    $result = Invoke-Expression $cmd
    Log $result
    if ($lastexitcode -ne 0) {
        throw "$errmsg"
    }
    return $result
}

Function run-cmd ([string]$cmd, [string]$errmsg) {
    Log $cmd
    # Avoid NativeCommandError by preventing PS wrapping stderr output in error objects
    Invoke-Expression $cmd >> $global:Logfile 2>&1 | %{ "$_" }
    # Old version for posterity
    # Invoke-Expression "$cmd >> $global:Logfile 2>&1"
    if ($lastexitcode -ne 0) {
        throw "$errmsg"
    }
}

Function Log-DiskSpace () {
    log "Checking disk space..."
    $cmd = "$global:P4exe -p $global:P4PORT -u $global:SDP_P4SUPERUSER diskspace"
    run-cmd "$cmd"
}

Function set-counter () {
    Invoke-P4Login
    $datetime = get-date-time
    $cmd = "$global:P4exe -p $global:P4PORT -u $global:SDP_P4SUPERUSER counter lastSDPCheckpoint ""$datetime"""
    run-cmd $cmd
}

Function Compare-JournalNumbers () {
    # Ensures offline and live db counters are the same - avoids using out-of-date offline_db
    $offlineJournal = Get-JournalCounter $global:OFFLINE_DB_DIR
    $liveJournal = Get-JournalCounter $global:P4ROOT
    if ($liveJournal -ne $offlineJournal) {
      log "$global:P4ROOT journal number is: $liveJournal"
      log "$global:OFFLINE_DB_DIR journal number is: $offlineJournal"
      throw "$global:P4ROOT and $global:OFFLINE_DB_DIR journal numbers do not match." 
    }
}

Function Move-OfflineDBToLive () {
    # Compare the Offline and Master journal numbers before switching to make sure they match.
    Compare-JournalNumbers
    log "Switching out db files..."
    $savedir = "${global:P4ROOT}\save"
    if (!(test-path -path $savedir -type container)) {
        New-Item $savedir -ItemType directory
    }
    remove-files "$savedir\db." 0
    move-files "${global:P4ROOT}\db.*" $savedir
    move-files "${global:offline_db_dir}\db.*" $global:P4ROOT
    remove-item "${global:offline_db_dir}\offline_db_usable.txt"
}

Function Test-IsAdmin {
    ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator")
}

Function Check-AdminPrivileges {
    if (!(Test-IsAdmin)) {
        throw "Please run this script with admin privileges"
    }
}

Function Stop-LiveService () {
    $cmd = "$global:SVCINST_EXE stop -n $global:SDP_INSTANCE_P4SERVICE_NAME"
    run-cmd $cmd "Failed to stop Live-Service $global:SDP_INSTANCE_P4SERVICE_NAME"
}

Function Start-LiveService () {
    $cmd = "$global:SVCINST_EXE start -n $global:SDP_INSTANCE_P4SERVICE_NAME"
    run-cmd $cmd "Failed to start Live-Service $global:SDP_INSTANCE_P4SERVICE_NAME"
}

Function send-email ([string]$subject, [string]$logfilename) {
    Log "Sending email with subject: $subject"
    if ($logfilename) {
        $contents = get-content $logfilename
    } else {
        $contents = get-content $global:logfile
    }
    $SMTPServer = $global:mailhost
    $SMTPPort = "25"
    if ($global:mailhostport -and $global:mailhostport -ne "") {
        $SMTPPort = $global:mailhostport
    }
    try {
        $email_password = ""
        if ((test-path $global:email_pass_file -pathtype leaf)) {
            Log "Found  $global:email_pass_file"
            $email_password = get-content $global:email_pass_file
        }
        $message = New-Object System.Net.Mail.MailMessage
        $message.subject = $subject
        $message.body = $contents -join "<br>`n"
        $message.isbodyhtml = $true
        $message.to.add($global:maillist)
        $message.from = $global:mailfrom
        $smtp = New-Object System.Net.Mail.SmtpClient($SMTPServer, $SMTPPort)
        if ($SMTPPort -ne "25") {
            $smtp.EnableSSL = $true
        }
        if ($email_password -ne "") {
            $smtp.Credentials = New-Object System.Net.NetworkCredential($global:mailfrom, $email_password)
        }
        $smtp.send($message)
    }
    Catch
    {
        $ErrorMessage = $_.Exception.Message
        Log "Failed to send email to ${global:mailhost}: ${ErrorMessage}"
    }
}