p4pstate.sh #1

#!/bin/bash
 
# Copyright (c) 2017, Perforce Software, Inc.  All rights reserved.
# 
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
# 
# 1.  Redistributions of source code must retain the above copyright
#     notice, this list of conditions and the following disclaimer.
# 
# 2.  Redistributions in binary form must reproduce the above copyright
#     notice, this list of conditions and the following disclaimer in the
#     documentation and/or other materials provided with the distribution.
# 
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL PERFORCE SOFTWARE, INC. BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
RCSCHANGE="$Change: 30154 $"
RCSDATE="$Date: 2024/03/05 $"
 
# === CONFIGURATION BEGIN ===
#
# This script supports two modes:
#
# 1) SDP mode (default):
#    - sources SDP environment
#    - expects SDP_INSTANCE as positional arg or from environment
#
#    Example:
#      nohup sudo /p4/common/bin/p4pstate.sh 1 > state.out &
#
# 2) Non-SDP mode:
#    - skip SDP environment
#    - provide values via parameters (see usage)
#
#    Example:
#      sudo ./p4pstate.sh --non-sdp --p4p-bin /usr/local/bin/p4p \
#         --p4-bin /usr/local/bin/p4 --p4port ssl:1667 --p4user perforce \
#         --p4plog /var/log/p4p.log --p4pcache /p4pcache
#
# === CONFIGURATION END ===

# Initialise these for bash
USAGE=0
NOP4=0
CONFIGONLY=0
SKIP_CONFIG_CHECK=0
SDP_MODE=1
SDP_INSTANCE_CLI=

p4p=
p4=
COLLECT_P4PORT=
outfile=

while [ $# -gt 0 ]; do
	key="$1"

	case $key in
	    --non-sdp|-N)
	    SDP_MODE=0
	    shift
	    ;;
	    --instance|-i)
	    SDP_INSTANCE_CLI="$2"
	    shift
	    shift
	    ;;
	    --p4p-bin|-x)
	    p4p="$2"
	    shift
	    shift
	    ;;
	    --p4-bin|-b)
	    p4="$2"
	    shift
	    shift
	    ;;
	    --p4port|-P)
	    COLLECT_P4PORT="$2"
	    shift
	    shift
	    ;;
	    --p4user|-u)
	    P4USER="$2"
	    shift
	    shift
	    ;;
	    --p4plog|-l)
	    P4PLOG="$2"
	    shift
	    shift
	    ;;
	    --p4pcache|-C)
	    P4PCACHE="$2"
	    shift
	    shift
	    ;;
	    --outfile|-o)
	    outfile="$2"
	    shift
	    shift
	    ;;
	    # In case the proxy is nonresponse, use '--skipp4'
	    --skipp4|-s)
	    NOP4=1
	    shift
	    ;;
	    # Config check only
	    --check-config|-c)
	    CONFIGONLY=1
	    shift
	    ;;
	    -h|--help)
	    # Print usage
	    USAGE=1
	    shift
	    ;;
	    --bypass-check-config|-B)
	    SKIP_CONFIG_CHECK=1
	    shift
	    ;;
	    -*)
	    echo "Unknown option: $1"
	    USAGE=1
	    shift
	    ;;
	    *)
	    # Support legacy positional SDP instance argument.
	    if [ -z "$SDP_INSTANCE_CLI" ]; then
	       SDP_INSTANCE_CLI="$1"
	    else
	       echo "Unexpected argument: $1"
	       USAGE=1
	    fi
	    shift
	    ;;
	esac
done

if [ $SDP_MODE -eq 1 ] && [ $USAGE -eq 0 ]; then
	export SDP_INSTANCE=${SDP_INSTANCE_CLI:-${SDP_INSTANCE:-}}
	if [[ $SDP_INSTANCE == Undefined || -z "$SDP_INSTANCE" ]]; then
	   echo "Instance parameter not supplied."
	   echo "You must supply the Perforce instance as a parameter to this script."
	   exit 1
	fi

	# Setup SDP environment
	. /p4/common/bin/p4_vars $SDP_INSTANCE
	. /p4/common/bin/backup_functions.sh

	outfile=${outfile:-$LOGS/p4pstate-`date +%Y%m%d%H%M%S`.out}
	p4p=${p4p:-$P4PBIN}
	p4=${p4:-$P4BIN}
	COLLECT_P4PORT=${COLLECT_P4PORT:-${PROXY_PORT:-$P4PORT}}
elif [ $SDP_MODE -eq 1 ]; then
	# Allow --help without requiring SDP instance or environment.
	outfile=${outfile:-./p4pstate-`date +%Y%m%d%H%M%S`.out}
	p4p=${p4p:-p4p}
	p4=${p4:-p4}
else
	outfile=${outfile:-./p4pstate-`date +%Y%m%d%H%M%S`.out}
	p4p=${p4p:-p4p}
	p4=${p4:-p4}
	P4USER=${P4USER:-perforce}
	P4PORT=${P4PORT:-$COLLECT_P4PORT}
fi
 
RED=$(tput setaf 1)
GREEN=$(tput setaf 2)
YELLOW=$(tput setaf 3)
normal=$(tput sgr0)

NPARALLELPIDS=32	# number of concurrent per-PID data collectors
 
TIMEOUT=120
 
export P4PORT
export P4USER
 
SIGKILL=9
SIGTERM=15
 
plural()
{
	if [ $1 != "1" ]
	then
	    printf "%s %ss" $1 $2
	else
	    if [ $# -lt 3 ]
	    then
		printf "%s %s" $1 $2
	    else
		printf "%s" $3
	    fi
	fi
}
 
header()
{
	printf "Output from "
	if [ $# -gt 1 ]
	then
	    plural $2 second
	    printf " of "
	fi
	printf "\"%s\"" "$1"
	if [ $# -gt 2 ]
	then
	    printf " repeated every "
	    plural $3 second second
	fi
	printf ", starting at %s\n" "`date "+%Y/%m/%d %H:%M:%S"`"
}
 
initmask()
{
	printf "%.${1}d" 0
}
 
setbit()
{
	masklen=`expr length $1`
 
	if [ $2 -gt 1 ]
	then
	    printf "%s" `expr substr $1 1 \`expr $2 - 1\``
	fi
 
	printf "1"
 
	if [ $2 -lt $masklen ]
	then
	    printf "%s" `expr substr $1 \`expr $2 + 1\` \`expr $masklen - $2\``
	fi
}
 
isset()
{
	test `expr substr $1 $2 1` = "1"
}
 
sleepkill()
{
	sleep $1 > /dev/null	# redirection required so as to not stall pipe
	kill -$2 $3 2> /dev/null
}
 
run()
{
	rcount=${2:-1}
	scount=${3:-1}
	header "$1" $rcount $scount
 
	if [ $# -lt 3 ]
	then
	    $1 2>&1 &
	else
	    {
		while [ 1 ]
		do
		    date +%H:%M:%S
		    $1 2>&1
		    printf "=======\n"
		    sleep $3
		done
	    } &
	fi
	runpid=$!
 
	reportkills=`initmask 64`
	if [ $# -gt 1 ]
	then
	    reportkills=`setbit $reportkills $SIGKILL`
 
	    sleepkill $2 $SIGTERM $runpid
 
	    {
		sleepkill $TIMEOUT $SIGKILL $runpid
	    } &
	else
	    reportkills=`setbit $reportkills $SIGKILL`
	    reportkills=`setbit $reportkills $SIGTERM`
 
	    {
		sleepkill $TIMEOUT $SIGTERM $runpid
		sleepkill $TIMEOUT $SIGKILL $runpid
	    } &
	fi
	waitpid=$!
	wait $runpid 2> /dev/null
	signal=`expr $? - 128`
	test $signal -gt 0 && isset $reportkills $signal &&
	{
	    printf "Command timed out; terminated using \"kill %s\".\n" -$signal
	}
	kill $waitpid 2> /dev/null
 
	wait 2> /dev/null
 
	printf "\n"
}
 
collectfile()
{
	echo $1.`printf "%.5d" $2`
}
 
getrcsvalue()
{
	#
	# Return an RCS value from a string. The RCS value is the
	# substring between the first and last space characters.
	#
	echo $1 | sed 's/^[^ ]* \(.*\) [^ ]*$/\1/'
}
 
reportscript()
{
	printf "%s/%s (%s)\n" "`basename "$0" | tr [:lower:] [:upper:]`" \
	    "`getrcsvalue "$RCSCHANGE"`" "`getrcsvalue "$RCSDATE"`"
}
 
lsofrun()
{
	targets=
	if [ -n "$P4PCACHE" ] && [ -d "$P4PCACHE" ]; then
	    [ -e "$P4PCACHE/pdb.monitor" ] && targets="$targets $P4PCACHE/pdb.monitor"
	    [ -e "$P4PCACHE/pdb.lbr" ] && targets="$targets $P4PCACHE/pdb.lbr"
	fi
	if [ -n "$P4PLOG" ] && [ -f "$P4PLOG" ]; then
	    targets="$targets $P4PLOG"
	fi

	if [ -n "$targets" ]; then
	    run "lsof $targets"
	else
	    echo "Skipping lsof file scan; no existing --p4pcache/--p4plog targets found."
	fi
}
 
getpids()
{
	# Handle SDP process names which may be exact (p4p_1) or with _bin suffix (p4p_1_bin)
	pname=`echo $p4p | sed 's/.*\/\([^\/]*\)$/\1/'`
	ps -eo pid=,comm= | awk -v p="$pname" '$2 == p || $2 == p "_bin" {print $1}'
}
 
pidruns()
{
	collecting=`collectfile $outfile $ncollects`
 
	iparallelpid=0
 
	for pid in `getpids`
	do
	    iparallelpid=`expr $iparallelpid + 1`
 
	    {
		run "lsof -p $pid"
		run "strace -qT -e trace=network,select,poll,epoll_wait,futex -p $pid" 0.50
	    } \
		> `collectfile $collecting $iparallelpid` &
 
	    if [ $iparallelpid -eq $NPARALLELPIDS ]
	    then
		wait
 
		package $collecting $NPARALLELPIDS
 
		iparallelpid=0
	    fi
	done
 
	wait
 
	package $collecting $iparallelpid
}
 
collect()
{
	ncollects=`expr $ncollects + 1`
 
	"$@" > `collectfile $outfile $ncollects` &
}
 
package()
{
	icollect=0
	while [ $icollect -lt $2 ]
	do
	    icollect=`expr $icollect + 1`
 
	    collected=`collectfile $1 $icollect`
	    cat $collected
	    rm -f $collected
	done
}

collect_tail_if_exists()
{
	if [ -n "$1" ] && [ -f "$1" ]; then
	    collect run "tail -f -n10000 $1" 15
	else
	    collect run "echo Skipping log tail; file not found: $1"
	fi
}

collect_connections()
{
	if type ss > /dev/null 2>&1; then
	    collect run "ss -tanp" 15 1
	else
	    collect run "netstat -antp" 15 1
	fi
}
 
check_binary()
{
	printf '    Checking if %s installed and in PATH ' "$1"
	if ! type "$1" > /dev/null; then
	    return 0;
	else
	    return 1;
	fi
}

check_p4()
{
	case $1 in
	    p4p)
	    printf '    Checking configured p4p binary (%s) ' "$p4p"
	    eval $p4p -V > /dev/null 2>&1
	    ;;
	    p4)
	    printf '    Checking configured p4 binary (%s) ' "$p4"
	    eval $p4 -V > /dev/null 2>&1
	    ;;
	    P4PLOG)
	    printf '    Checking configured P4PLOG (%s) ' "$P4PLOG"
	    if [ -z "$P4PLOG" ]; then
	        return 0;
	    fi
	    eval tail -n1 "$P4PLOG" > /dev/null 2>&1
	    ;;
	    P4PCACHE)
	    printf '    Checking configured P4PCACHE (%s) ' "$P4PCACHE"
	    if [ -z "$P4PCACHE" ]; then
	        return 0;
	    fi
	    eval ls -ld "$P4PCACHE" > /dev/null 2>&1
	    ;;
	esac

	if [ $? -gt 0 ]
	then
	    return 0;
	else
	    return 1;
	fi
}

check_config()
{
	fatal_error=0;
	warnings=0

	printf "Configuration Check\n"
	if [ $NOP4 -eq 0 ]; then
	    if [ -z "$COLLECT_P4PORT" ]; then
	        printf "    Checking p4 info precondition "
	        printf '[%s]\n' "${RED}fail${normal}"
	        printf "        -> Set --p4port in --non-sdp mode.\n"
	        fatal_error=1
	    else
	        printf "    Checking 'p4 info' against configured proxy port (%s) " "$COLLECT_P4PORT"
	        eval $p4 -p $COLLECT_P4PORT -ztag info  > /dev/null 2>&1
	        if [ $? -gt 0 ]; then
	            printf '[%s]\n' "${YELLOW}warning${normal}"
	            printf "        -> Check connectivity to configured proxy port or use the "
	            printf "'--skipp4' option to p4pstate.sh.\n"
	            warnings=1
	        else
	            printf '[%s]\n' "${GREEN}pass${normal}"
	        fi
	    fi
	fi

	p4_binaries="p4p p4"
	for binary in ${p4_binaries}
	do
	    if check_p4 "$binary"; then
	        case $binary in
	            p4p)
	            printf '[%s]\n' "${RED}fail${normal}"
	            fatal_error=1
	            ;;
	            p4)
	            printf '[%s]\n' "${YELLOW}warning${normal}"
	            warnings=1
	            ;;
	        esac
	        printf '        -> No %s binary found, check p4pstate configuration.\n' "$binary"
	    else
	        printf '[%s]\n' "${GREEN}pass${normal}"
	    fi
	done

	p4_components="P4PLOG P4PCACHE"
	for component in ${p4_components}
	do
	    if check_p4 "$component"; then
	        printf '[%s]\n' "${YELLOW}warning${normal}"
	        printf '        -> No %s found, check p4pstate configuration.\n' "$component"
	        warnings=1
	    else
	        printf '[%s]\n' "${GREEN}pass${normal}"
	    fi
	done;

	utilities="lsof strace netstat lslocks top"
	for utility in ${utilities}
	do
	    if check_binary "$utility"; then
	        if [ "$utility" != "lslocks" ] && [ "$utility" != "netstat" ]\
	             && [ "$utility" != "top" ];then
	             printf '[%s]\n' "${RED}fail${normal}"
	             fatal_error=1
	        else
	             printf '[%s]\n' "${YELLOW}warning${normal}"
	             warnings=1
	        fi
	    else
	        printf '[%s]\n' "${GREEN}pass${normal}"
	    fi
	done;

	if [ $fatal_error -gt 0 ]; then
	    if [ $warnings -gt 0 ]; then
	        printf "Required components missing, fix failed and warning.\n"
	    else
	        printf "Required components missing, fix failed.\n"
	    fi
	    exit 1
	fi

	if [ $warnings -gt 0 ]; then
	    if [ $fatal_error -gt 0 ]; then
	        printf "Required components missing, fix failed and warning.\n"
	        exit 1
	    else
	        printf "For a more complete capture of data, fix warnings.\n"
	    fi
	fi

	if [ $CONFIGONLY -eq 1 ];then
	    exit 1
	fi

}

usage()
{
	printf "Usage:\n"
	printf "\n\tp4pstate.sh [ options ]\n"
	printf "\n\toptions:"
	printf "\n\t\t${RED}-h, --help ${normal}Display usage"
	printf "\n\t\t${RED}-s, --skipp4 ${normal}Skip running p4 commands"
	printf "\n\t\t${RED}-c, --check-config ${normal}Check configuration status only"
	printf "\n\t\t${RED}-B, --bypass-check-config ${normal}Skip prerequisite checks"
	printf "\n\t\t${RED}-N, --non-sdp ${normal}Run without sourcing SDP environment"
	printf "\n\t\t${RED}-i, --instance N ${normal}SDP instance (alternative to positional N)"
	printf "\n\t\t${RED}-x, --p4p-bin PATH ${normal}Path to p4p binary (non-SDP or override)"
	printf "\n\t\t${RED}-b, --p4-bin PATH ${normal}Path to p4 binary (non-SDP or override)"
	printf "\n\t\t${RED}-P, --p4port PORT ${normal}Proxy listener port for p4 commands"
	printf "\n\t\t${RED}-u, --p4user USER ${normal}User for p4 commands"
	printf "\n\t\t${RED}-l, --p4plog PATH ${normal}Proxy log path to tail and inspect"
	printf "\n\t\t${RED}-C, --p4pcache PATH ${normal}Proxy cache root to inspect with lsof"
	printf "\n\t\t${RED}-o, --outfile PATH ${normal}Output file path"
	printf "\n\n\tp4pstate.sh is used when a Linux P4 Proxy is exhibiting "
	printf "problematic behaviors like:\n"
	printf "\n\t   * Slow or unresponsive P4 Proxy\n"
	printf "\t   * Slow or stalled client connections through proxy\n"
	printf "\t   * Unexpected proxy resource usage\n"
	printf "\n\tp4pstate.sh collects data from multiple sources, during the time of an "
	printf "event, writing to its output \n\tfile a comprehensive set of data when "
	printf "paired with proxy and server logs covering the time of \n\tthe event "
	printf "provides a more complete picture of events to aid in investigation of "
	printf "issues. The information \n\tgathered helps facilitate issue resolution and "
	printf "root cause investigations.\n\n"
	printf "\tPrerequisites for use:"
	printf "\n\n\t1. Ensure SDP environment is configured for the target instance and proxy.\n"
	printf "\t2. Ensure the lsof, strace, lslocks, and netstat Linux utilities are installed and "
	printf "available in the\n\t   \$PATH of the root Linux user running p4pstate.sh."
	printf "\n\t3. Run p4pstate.sh as a Linux root user or under sudo."
	printf "\n\n\tUse 'p4pstate.sh --check-config' to check if prerequisites are met."
	printf "\n\n\tSDP examples:"
	printf "\n\t   sudo p4pstate.sh 1"
	printf "\n\t   sudo p4pstate.sh alpha"
	printf "\n\n\tNon-SDP example:"
	printf "\n\t   sudo p4pstate.sh --non-sdp --p4p-bin /usr/local/bin/p4p --p4-bin /usr/local/bin/p4 \\\n\n\t      --p4port ssl:1667 --p4user perforce --p4plog /var/log/p4p.log --p4pcache /p4pcache"
	printf "\n\n\tSee ${RED}https://portal.perforce.com/s/article/15261 ${normal}for further details.\n\n"
}
 
if [ $USAGE -gt 0 ]
then
	usage
	exit 1
fi

if [ $SKIP_CONFIG_CHECK -eq 0 ]; then
	check_config
fi

outfiletemplate="$outfile"
 
eval outfile=\"$outfiletemplate\"
 
ncollects=0
 
printf "Collecting p4p state from proxy at P4PORT=%s into \"%s\"..." "$COLLECT_P4PORT" "$outfile"
 
collect run reportscript
 
collect run "$p4p -V"
collect run "$p4 -V"
 
collect run "uname -a"
 
if [ -n "$P4PLOG" ]; then
	collect_tail_if_exists "$P4PLOG"
else
	collect run "echo Skipping proxy log tail; no --p4plog configured"
fi
 
collect lsofrun
 
collect pidruns
 
collect run "ps -elfjH" 15 1
collect_connections
 
collect run "sysctl -a"
 
collect_tail_if_exists "/var/log/messages"
collect_tail_if_exists "/var/log/syslog"
collect_tail_if_exists "/var/log/dmesg"
 
collect run "lslocks -J -o +BLOCKER"
collect run "lslocks -o +BLOCKER"

collect run "top -b -n 4"
 
#
# Depending upon the state of the proxy, the following commands
# might not behave as expected.
#
if [ $NOP4 -eq 0 ]
then
	if [ -n "$COLLECT_P4PORT" ]; then
		collect run "$p4 -u $P4USER -p $COLLECT_P4PORT -ztag info"
	else
		collect run "echo Skipping p4 info; no --p4port configured"
	fi
fi
 
#
# All commands have been started; wait for them to finish.
#
wait
 
package $outfile $ncollects > $outfile
 
printf " done.\n"
 
exit 0