#!/bin/bash
set -u
#------------------------------------------------------------------------------
# Usage: To get usage info, run: ./aws_sync.sh - man
#==============================================================================
# Declarations and Environment
declare ThisScript=${0##*/}
declare ThisUser=
declare DepotsDir=
declare DefaultCfgFile="/p4/common/config/aws_s3_sync.cfg"
declare SDPInstance="${SDP_INSTANCE:-}"
declare SDPEnvFile="/p4/common/bin/p4_vars"
declare SDPInstanceVarsFile=
declare HxDepots=
declare CfgFile=
declare Version=1.1.5
declare -i NoOp=0
declare -i ErrorCount=0
declare -i SilentMode=0
declare -i CmdExitCode=0
declare Cmd=
declare Log=
declare H="=============================================================================="
# Values for these variables are defined in the confgiration values. See the
# CONFIGURATION FILES section of the manual pages.
declare S3Bucket=
declare StorageClass=
#==============================================================================
# Local Functions
function msg () { echo -e "$*"; }
function errmsg () { msg "\\nError: ${1:-Unknown Error}\\n"; ErrorCount+=1; }
function bail () { errmsg "${1:-Unknown Error}"; exit "${2:-1}"; }
#------------------------------------------------------------------------------
# Function: terminate
function terminate
{
# Disable signal trapping.
trap - EXIT SIGINT SIGTERM
[[ "$Log" != off ]] && msg "Log is: $Log\\n${H}"
# With the trap removed, exit.
exit "$ErrorCount"
}
#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
# The default is -h.
#
# $2 - error message (optional). Specify this if usage() is called due to
# user error, in which case the given message displayed first, followed by the
# standard usage message (short or long depending on $1). If displaying an
# error, usually $1 should be -h so that the longer usage message doesn't
# obscure the error message.
#
# Sample Usage:
# usage
# usage -h
# usage -man
# usage -h "Incorrect command line usage."
#------------------------------------------------------------------------------
function usage
{
declare style=${1:--h}
declare errorMessage=${2:-Unset}
if [[ "$errorMessage" != Unset ]]; then
msg "\\n\\nUsage Error:\\n\\n$errorMessage\\n\\n"
fi
msg "USAGE for $ThisScript v$Version:
$ThisScript [-c <cfg_file>] [-L <log>] [-si] [-n] [-D]
or
$ThisScript [-h|-man|-V]
"
if [[ $style == -man ]]; then
msg "
DESCRIPTION:
This script wraps the AWS command line, specifically the 'aws s3 sync'
command, to sync the depots directory to AWS.
CONFIGURATION FILE:
This script requires a configuration file to define the following
settings:
* S3Bucket - Define the AWS S3 Bucket to copy to.
* StorageClass Define the storage class to use. This impacts storage
costs, accessibility, etc. For more information, see:
https://aws.amazon.com/s3/storage-classes
The default configuration file location is: $DefaultCfgFile"
if [[ -r "$DefaultCfgFile" ]]; then
msg "\\n${H}\\nBEGIN CONTENTS $DefaultCfgFile\\n$(cat "$DefaultCfgFile")\\n${H}\\nEND FILE CONTENTS"
fi
msg "
EXAMPLES:
This script can be called directly, though it is intended to be called
via a crontab entry like this sample:
INSTANCE=1
0 1 * * * [[ -e /hxdepots ]] && /p4/common/bin/run_if_master.sh \${INSTANCE} -si
OPTIONS:
-c <cfg_file>
Specify the path to the configuration file. The default is:
$DefaultCfgFile
-L <log>
Specify the path to a log file, or the special value 'off' to disable
logging. By default, all output (stdout and stderr) goes to
${LOGS:-\$LOGS}/aws_s3_sync.<datestamp>.log
NOTE: This script is self-logging. That is, output displayed on the screen
is simultaneously captured in the log file. Do not run this script with
redirection operators like '> log' or '2>&1', and do not use 'tee.'
If an existing log file is specified, this script will append the the
specified log file.
-si Operate silently. All output (stdout and stderr) is redirected to the log
only; no output appears on the terminal. This cannot be used with
'-L off'.
This is useful when running from cron, as it prevents automatic
email from being sent by cron directly, as cron does when a script called
from cron generates any output. This script is then responsible for
email handling, if any is to be done.
-n No-Op. Prints commands instead of running them.
-D Set extreme debugging verbosity.
HELP OPTIONS:
-h Display short help message
-man Display man-style help message
"
fi
exit 1
}
#==============================================================================
# Command Line Processing
declare -i shiftArgs=0
set +u
while [[ $# -gt 0 ]]; do
case $1 in
(-i) SDPInstance="$2"; shiftArgs=1;;
(-h) usage -h;;
(-man) usage -man;;
(-c) CfgFile="$2"; shiftArgs=1;;
(-L) Log="$2"; shiftArgs=1;;
(-si) SilentMode=1;;
(-n) NoOp=1;;
(-D) set -x;; # Debug; use 'set -x' mode.
(*) usage -h "Unknown arg ($1).";;
esac
# Shift (modify $#) the appropriate number of times.
shift; while [[ $shiftArgs -gt 0 ]]; do
[[ $# -eq 0 ]] && usage -h "Incorrect number of arguments."
shiftArgs=$shiftArgs-1
shift
done
done
set -u
#==============================================================================
# Command Line Verification
[[ "$SilentMode" -eq 1 && "$Log" == off ]] && \
usage -h "Cannot use '-si' with '-L off'."
[[ -n "$SDPInstance" ]] ||\
bail "The SDP '-i <SDP_Instance>' parameter is required unless \$SDP_INSTANCE is defined."
[[ -z "$CfgFile" ]] && CfgFile="$DefaultCfgFile"
[[ -r "$SDPEnvFile" ]] ||\
bail "The SDP Environment file is missing: $SDPEnvFile"
SDPInstanceVarsFile="/p4/common/config/p4_${SDPInstance}.vars"
[[ -r "$SDPInstanceVarsFile" ]] ||\
bail "The SDP Instances Vars file is missing: $SDPInstanceVarsFile"
# shellcheck disable=SC1090
source "$SDPEnvFile" "$SDPInstance" ||\
bail "Failed to load SDP Environment."
[[ -r "$CfgFile" ]] ||\
bail "The SDP Environment file is missing: $SDPEnvFile"
# shellcheck disable=SC1090
source "$CfgFile" ||\
bail "Failed to load the configuration file: $CfgFile"
[[ -z "$Log" ]] && Log="$LOGS/${ThisScript%.sh}.$(date +'%Y%m%d-%H%M%S').log"
#==============================================================================
# Main Program
trap terminate EXIT SIGINT SIGTERM
if [[ "$Log" != off ]]; then
touch "$Log" || bail "Couldn't touch log file [$Log]."
# Redirect stdout and stderr to a log file.
if [[ "$SilentMode" -eq 0 ]]; then
exec > >(tee -a "$Log")
exec 2>&1
else
exec >"$Log"
exec 2>&1
fi
msg "${H}\\nLog is: $Log"
fi
ThisUser=$(id -n -u)
msg "Started $ThisScript v$Version as $ThisUser@${HOSTNAME%%.*} on $(date)."
# shellcheck disable=SC2164
HxDepots=$(cd /p4/common; d=$(pwd -P); echo "${d%/p4/common}")
[[ -d "$HxDepots" ]] || bail "Couldn't determine HxDepots with 'cd /p4/common; pwd -P'."
DepotsDir="${HxDepots}${DEPOTS}"
[[ -d "$DepotsDir" ]] || bail "The expected depots directory does not exist or is not accessible: $DepotsDir"
Cmd="/usr/bin/aws s3 sync --storage-class $StorageClass $DepotsDir $S3Bucket"
msg "Running:\\n$Cmd\\nLogging to: $Log"
if [[ "$NoOp" -eq 0 ]]; then
$Cmd
CmdExitCode=$?
msg "Exit code from 'aws s3 sync' command: $CmdExitCode"
if [[ "$CmdExitCode" -eq 0 || "$CmdExitCode" -eq 2 ]]; then
msg "Verified: The AWS Command returned with a happy zero exit code."
else
errmsg "AWS Command returned with a non-zero exit code."
fi
else
msg "NO_OP: Would run: $Cmd"
fi
if [[ "$ErrorCount" -eq 0 ]]; then
msg "\\nProcessing competed successfully."
else
errmsg "\\nErrors were encountered during processing."
fi
msg "That took $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n"
exit "$ErrorCount"