p4dstate.sh #14

#!/bin/bash
 
# Copyright (c) 2017, Perforce Software, Inc.  All rights reserved.
# 
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
# 
# 1.  Redistributions of source code must retain the above copyright
#     notice, this list of conditions and the following disclaimer.
# 
# 2.  Redistributions in binary form must reproduce the above copyright
#     notice, this list of conditions and the following disclaimer in the
#     documentation and/or other materials provided with the distribution.
# 
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL PERFORCE SOFTWARE, INC. BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
RCSCHANGE="$Change: 30154 $"
RCSDATE="$Date: 2024/03/05 $"
 
# === CONFIGURATION BEGIN ===
#
# Change the "<absolute-path-to->" in the following "outfile" definition, but
# leave the " and \` characters as they are.
#
# SDP version of the original p4dstate.sh - differences:
# - works with bash
# - sources SDP environment - expects SDP_INSTANCE to be passed as parameter
#
# Recommend way to run this:
#   nohup sudo /p4/common/bin/p4dstate.sh 1 > state.out &
#
# Then check the state.out file for output, and also the resulting /p4/1/logs/p4dstate*.log
#

export SDP_INSTANCE=${1:-$SDP_INSTANCE}
if [[ $SDP_INSTANCE == Undefined ]]; then
   echo "Instance parameter not supplied."
   echo "You must supply the Perforce instance as a parameter to this script."
   exit 1
fi
[[ $SDP_INSTANCE == $1 ]] && shift	# Skip supplied param to avoid issues with later arg parsing


# Setup SDP environment
. /p4/common/bin/p4_vars $SDP_INSTANCE
. /p4/common/bin/backup_functions.sh

outfile=$LOGS/p4dstate-`date +%Y%m%d%H%M%S`.out
 
#
# Change the values of the following definitions as appropriate
# for the installation.
#
p4d=$P4DBIN
p4=$P4BIN
# The following already set by p4_vars - so ignored for SDP version of this script.
# P4ROOT=/<absolute-path-to-P4ROOT>
# P4LOG=/<absolute-path-to->/log
# P4JOURNAL=/<absolute-path-to->/journal
# P4JOURNALPREFIX=/<absolute-path-to->/jnlPrefix    # use if interested in rotated journals
P4JOURNALPREFIX=""
# P4PORT=<p4d-listener>
# P4USER=<perforce-superuser>
# === CONFIGURATION END ===

RED=$(tput setaf 1)
GREEN=$(tput setaf 2)
YELLOW=$(tput setaf 3)
normal=$(tput sgr0)

NPARALLELPIDS=128	# number of concurrent per-PID data collectors
 
TIMEOUT=120
 
export P4PORT
export P4USER
 
SIGKILL=9
SIGTERM=15
# Initialise these for bash
USAGE=0
NOP4=0
CONFIGONLY=0
SKIP_CONFIG_CHECK=0

while [ $# -gt 0 ]; do
	key="$1"

	case $key in
	    # In case the server is nonresponse, use '--skipp4'
	    --skipp4)
	    NOP4=1
	    shift
	    ;;
	    # Config check only
	    --check-config)
	    CONFIGONLY=1
	    shift
	    ;;
	    -h)
	    # Print usage
	    USAGE=1
	    shift
	    ;;
	    --bypass-check-config)
	    SKIP_CONFIG_CHECK=1
	    shift
	    ;;
	esac
done
 
plural()
{
	if [ $1 != "1" ]
	then
	    printf "%s %ss" $1 $2
	else
	    if [ $# -lt 3 ]
	    then
		printf "%s %s" $1 $2
	    else
		printf "%s" $3
	    fi
	fi
}
 
header()
{
	printf "Output from "
	if [ $# -gt 1 ]
	then
	    plural $2 second
	    printf " of "
	fi
	printf "\"%s\"" "$1"
	if [ $# -gt 2 ]
	then
	    printf " repeated every "
	    plural $3 second second
	fi
	printf ", starting at %s\n" "`date "+%Y/%m/%d %H:%M:%S"`"
}
 
initmask()
{
	printf "%.$1d" 0
}
 
setbit()
{
	masklen=`expr length $1`
 
	if [ $2 -gt 1 ]
	then
	    printf "%s" `expr substr $1 1 \`expr $2 - 1\``
	fi
 
	printf "1"
 
	if [ $2 -lt $masklen ]
	then
	    printf "%s" `expr substr $1 \`expr $2 + 1\` \`expr $masklen - $2\``
	fi
}
 
isset()
{
	test `expr substr $1 $2 1` = "1"
}
 
sleepkill()
{
	sleep $1 > /dev/null	# redirection required so as to not stall pipe
	kill -$2 $3 2> /dev/null
}
 
run()
{
	rcount=${2:-1} 
	scount=${3:-1} 
	header "$1" $rcount $scount 
 
	if [ $# -lt 3 ]
	then
	    $1 2>&1 &
	else
	    {
		while [ 1 ]
		do
		    date +%H:%M:%S
		    $1 2>&1
		    printf "=======\n"
		    sleep $3
		done
	    } &
	fi
	runpid=$!
 
	reportkills=`initmask 64`
	if [ $# -gt 1 ]
	then
	    reportkills=`setbit $reportkills $SIGKILL`
 
	    sleepkill $2 $SIGTERM $runpid
 
	    {
		sleepkill $TIMEOUT $SIGKILL $runpid
	    } &
	else
	    reportkills=`setbit $reportkills $SIGKILL`
	    reportkills=`setbit $reportkills $SIGTERM`
 
	    {
		sleepkill $TIMEOUT $SIGTERM $runpid
		sleepkill $TIMEOUT $SIGKILL $runpid
	    } &
	fi
	waitpid=$!
	wait $runpid 2> /dev/null
	signal=`expr $? - 128`
	test $signal -gt 0 && isset $reportkills $signal &&
	{
	    printf "Command timed out; terminated using \"kill %s\".\n" -$signal
	}
	kill $waitpid 2> /dev/null
 
	wait 2> /dev/null
 
	printf "\n"
}
 
collectfile()
{
	echo $1.`printf "%.5d" $2`
}
 
getrcsvalue()
{
	#
	# Return an RCS value from a string. The RCS value is the
	# substring between the first and last space characters.
	#
	echo $1 | sed 's/^[^ ]* \(.*\) [^ ]*$/\1/'
}
 
reportscript()
{
	printf "%s/%s (%s)\n" "`basename "$0" | tr [:lower:] [:upper:]`" \
	    "`getrcsvalue "$RCSCHANGE"`" "`getrcsvalue "$RCSDATE"`"
}
 
lsofrun()
{
	run "lsof -r1 $P4ROOT/* $P4ROOT/server.locks/*/* $P4JOURNAL $P4JOURNALPREFIX* $P4LOG" 15 \
	    | grep -v '^tail'
}
 
getpids()
{
	# Handle SDP processes which are case sensitive (p4d_1) or insensitive (p4d_1_bin)
	pname=`echo $p4d | sed 's/.*\/\([^\/]*\)$/\1/'`
	printf "ps -e | grep -E '%s$|%s_bin$'" $pname $pname \
	    | sh \
	    | sed 's/^[^0-9]*\([0-9]*\)[^0-9]*.*/\1/'
}
 
pidruns()
{
	collecting=`collectfile $outfile $ncollects`
 
	iparallelpid=0
 
	for pid in `getpids`
	do
	    iparallelpid=`expr $iparallelpid + 1`
 
	    {
		run "lsof -p $pid"
		run "strace -qT -p $pid" 0.50
	    } \
		> `collectfile $collecting $iparallelpid` &
 
	    if [ $iparallelpid -eq $NPARALLELPIDS ]
	    then
		wait
 
		package $collecting $NPARALLELPIDS
 
		iparallelpid=0
	    fi
	done
 
	wait
 
	package $collecting $iparallelpid
}
 
collect()
{
	ncollects=`expr $ncollects + 1`
 
	"$@" > `collectfile $outfile $ncollects` &
}
 
package()
{
	icollect=0
	while [ $icollect -lt $2 ]
	do
	    icollect=`expr $icollect + 1`
 
	    collected=`collectfile $1 $icollect`
	    cat $collected
	    rm -f $collected
	done
}
 
check_binary()
{
	printf '    Checking if %s installed and in PATH ' "$1"
	if ! type "$1" > /dev/null; then
	    return 0;
	else
	    return 1;
	fi
}

check_p4()
{
	case $1 in
	    p4d)
	    printf '    Checking configured p4d binary (%s) ' "$p4d"
	    eval $p4d -V > /dev/null 2>&1
	    ;;
	    p4)
	    printf '    Checking configured p4 binary (%s) ' "$p4"
	    eval $p4 -V > /dev/null 2>&1
	    ;;
	    P4LOG)
	    printf '    Checking configured P4LOG (%s) ' "$P4LOG"
	    eval tail -n1 "$P4LOG" > /dev/null 2>&1
	    ;;
	    P4JOURNAL)
	    printf '    Checking configured P4JOURNAL (%s) ' "$P4JOURNAL"
	    eval tail -n1 "$P4JOURNAL" > /dev/null 2>&1
	    ;;
	esac

	if [ $? -gt 0 ]
	then
	    return 0;
	else
	    return 1;
	fi
}

check_config()
{
	fatal_error=0;
	warnings=0

	printf "Configuration Check\n"
	if [ $NOP4 -eq 0 ]; then
	    printf "    Checking 'p4 info' against configured P4PORT (%s) " "$P4PORT"
	    eval $p4 -p $P4PORT -ztag info  > /dev/null 2>&1
	    if [ $? -gt 0 ]; then
	        printf '[%s]\n' "${YELLOW}warning${normal}"
	        printf "        -> Check connectivity to configured P4PORT or use the "
	        printf "'--skipp4' option to p4dstate.sh.\n"
	        warnings=1
	    else
	        printf '[%s]\n' "${GREEN}pass${normal}"
	        printf "    Checking 'p4 configure show' against configured P4PORT "
	        printf "as configured P4USER (%s,%s) " $P4PORT $P4USER
	        eval $p4 -p $P4PORT -u $P4USER configure show  > /dev/null 2>&1
	        if [ $? -gt 0 ]; then
	            printf '[%s]\n' "${YELLOW}warning${normal}"
	            printf "       -> Check for valid login ticket for configured P4USER "
	            printf "and P4PORT (%s,%s)\n" "$P4USER" "$P4PORT"
	            warnings=1
	         else
	            printf '[%s]\n' "${GREEN}pass${normal}"
	        fi
	    fi
	fi

	p4_binaries="p4d p4"
	for binary in ${p4_binaries}
	do
	    if check_p4 "$binary"; then
	        case $binary in
	            p4d)
	            printf '[%s]\n' "${RED}fail${normal}"
	            fatal_error=1
	            ;;
	            p4)
	            printf '[%s]\n' "${YELLOW}warning${normal}"
	            warnings=1
	            ;;
	        esac
	        printf '        -> No %s binary found, check p4dstate configuration.\n' "$binary"
	    else  
	        printf '[%s]\n' "${GREEN}pass${normal}"
	    fi
	done

	p4_components="P4LOG P4JOURNAL"
	for component in ${p4_components}
	do
	    if check_p4 "$component"; then
	        printf '[%s]\n' "${YELLOW}warning${normal}"
	        printf '        -> No %s found, check p4dstate configuration.\n' "$component"
	        warnings=1
	    else  
	        printf '[%s]\n' "${GREEN}pass${normal}"
	    fi
	done;

	utilities="lsof strace netstat lslocks top"
	for utility in ${utilities}
	do
	    if check_binary "$utility"; then
	        if [ "$utility" != "lslocks" ] && [ "$utility" != "netstat" ]\
	             && [ "$utility" != "top" ];then
	             printf '[%s]\n' "${RED}fail${normal}"
	             fatal_error=1
	        else
	             printf '[%s]\n' "${YELLOW}warning${normal}"
	             warnings=1
	        fi
	    else  
	        printf '[%s]\n' "${GREEN}pass${normal}"
	    fi
	done;

	if [ $fatal_error -gt 0 ]; then
	    if [ $warnings -gt 0 ]; then
	        printf "Required components missing, fix failed and warning.\n"
	    else
	        printf "Required components missing, fix failed.\n"
	    fi
	    exit 1
	fi

	if [ $warnings -gt 0 ]; then
	    if [ $fatal_error -gt 0 ]; then
	        printf "Required components missing, fix failed and warning.\n"
	        exit 1
	    else
	        printf "For a more complete capture of data, fix warnings.\n"
	    fi
	fi

	if [ $CONFIGONLY -eq 1 ];then
	    exit 1
	fi

}

usage()
{
	printf "Usage:\n"
	printf "\n\tp4dstate.sh [ options ]\n"
	printf "\n\toptions:"
	printf "\n\t\t${RED}-h ${normal}Display usage"
	printf "\n\t\t${RED}--skipp4 ${normal}Skip running p4 commands"
	printf "\n\t\t${RED}--check-config ${normal}Check configuration status only"
	printf "\n\n\tp4dstate.sh is used when a Linux Helix Core Server is exhibiting "
	printf "problematic behaviors like:\n"
	printf "\n\t   * Slow or unresponsive Helix Core Server\n"
	printf "\t   * Slow or unresponsive Helix Core Server command\n"
	printf "\t   * Replication stalled or moving slowly on a Helix Core Replica Server\n"
	printf "\n\tp4dstate.sh collects data from multiple sources, during the time of an "
	printf "event, writing to its output \n\tfile a comprehensive set of data when "
	printf "paired with a Helix Core Server log file covering the time of \n\tthe event "
	printf "provides a more complete picture of events to aid in investigation of "
	printf "issues. The information \n\tgathered helps facilitate issue resolution and "
	printf "root cause investigations.\n\n"
	printf "\tPrerequisites for use:"
	printf "\n\n\t1. Configure the section near the top of the script to set the location of "
	printf "the output file and the \n\t   relevant p4 settings for the Helix Core Server "
	printf "p4dstate.sh will be run against.\n"
	printf "\t2. Ensure the lsof, strace, lslocks, and netstat Linux utilities are installed and "
	printf "available in the\n\t   \$PATH of the root Linux user running p4dstate.sh." 
	printf "\n\t3. Ensure a valid Helix Core Server login ticket for the ${RED}<perforce-superuser> "
	printf "${normal}defined in the \n\t   p4dstate.sh configuration."
	printf "\n\t4. Run p4dstate.sh as a Linux root user or under sudo."
	printf "\n\n\tUse 'p4dstate.sh --check-config' to check if prerequesites are met."
	printf "\n\n\tSee ${RED}https://portal.perforce.com/s/article/15261 ${normal}for further details.\n\n"
}
 
if [ $USAGE -gt 0 ]
then
	usage
	exit 1
fi

if [ $SKIP_CONFIG_CHECK -eq 0 ]; then
	check_config
fi

outfiletemplate="$outfile"
 
eval outfile=\"$outfiletemplate\"
 
ncollects=0
 
printf "Collecting p4d state from server at P4PORT=%s into \"%s\"..." "$P4PORT" "$outfile"
 
collect run reportscript
 
collect run "$p4d -r $P4ROOT -V"
collect run "$p4d -r $P4ROOT -c show"
collect run "$p4 -V"
 
collect run "uname -a"
 
collect run "tail -f -n10000 $P4LOG" 15
collect run "tail -f -n10000 $P4JOURNAL" 15
 
collect lsofrun
 
collect pidruns
 
collect run "ps -elfjH" 15 1
collect run "netstat -antp" 15 1
 
collect run "sysctl -a"
 
collect run "tail -f -n10000 /var/log/messages" 15
collect run "tail -f -n10000 /var/log/syslog" 15
collect run "tail -f -n10000 /var/log/dmesg" 15
 
collect run "lslocks -J -o +BLOCKER"
collect run "lslocks -o +BLOCKER"

collect run "top -b -n 4"
 
#
# Depending upon the state of the server, the following commands
# might not behave as expected.
#
if [ $NOP4 -eq 0 ]
then
	collect run "$p4 -u $P4USER -p $P4PORT -ztag info"
	collect run "$p4 -u $P4USER -p $P4PORT configure show"
	collect run "$p4 -u $P4USER -p $P4PORT configure show allservers"
	collect run "$p4 -u $P4USER -p $P4PORT configure history"
	collect run "$p4 -u $P4USER -p $P4PORT -ztag servers -J"
	collect run "$p4 -u $P4USER -p $P4PORT journals"
	collect run "$p4 -u $P4USER -p $P4PORT monitor show -el" 15 1
fi
 
#
# All commands have been started; wait for them to finish.
#
wait
 
package $outfile $ncollects > $outfile
 
printf " done.\n"
 
exit 0