install_sdp_python.sh #2

#!/bin/bash
set -u

#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE
#------------------------------------------------------------------------------

#==============================================================================
# Declarations and Environment

# Allow override of P4U_HOME, which is set only when testing P4U scripts.
# Version ID Block. Relies on +k filetype modifier.
#------------------------------------------------------------------------------
# shellcheck disable=SC2016
declare VersionID='$Id: //p4-sdp/dev_c2s/Unsupported/setup/install_sdp_python.sh#2 $ $Change: 31472 $'
declare VersionStream=${VersionID#*//}; VersionStream=${VersionStream#*/}; VersionStream=${VersionStream%%/*};
declare VersionCL=${VersionID##*: }; VersionCL=${VersionCL%% *}
declare Version=${VersionStream}.${VersionCL}
[[ "$VersionStream" == r* ]] || Version="${Version^^}"

export P4CBIN=${P4CBIN:-/p4/common/bin}
export P4U_HOME=${P4U_HOME:-/p4/common/bin}
export P4U_LIB=${P4U_LIB:-/p4/common/lib}
export P4U_ENV=$P4U_LIB/p4u_env.sh
export P4U_LOG=
export VERBOSITY=${VERBOSITY:-3}

# Environment isolation.  For stability and security reasons, prepend
# PATH to include dirs where known-good scripts exist.
# known/tested PATH and, by implication, executables on the PATH.
export PATH=$P4U_HOME:$PATH:~/bin:/usr/local/bin:$PATH
export P4CONFIG=${P4CONFIG:-.p4config}

[[ -r "$P4U_ENV" ]] || {
   echo -e "\nError: Cannot load environment from: $P4U_ENV\n\n"
   exit 1
}

declare BASH_LIBS="$P4U_ENV $P4U_LIB/libcore.sh $P4U_LIB/libp4u.sh"

for bash_lib in $BASH_LIBS; do
   # shellcheck disable=SC1090
   source "$bash_lib"
done

declare BuildCmd=
declare InstallCmd=
declare PipUpgradeCmd=
declare OpenSSLDir=
declare -i UseSSL=1
declare -i BuildP4PythonFromSource=0
declare -i RequireSSL=1
declare -i SilentMode=0

#==============================================================================
# Local Functions

#------------------------------------------------------------------------------
# Function: terminate
function terminate
{
   # Disable signal trapping.
   trap - EXIT SIGINT SIGTERM

   # Don't litter.
   cleanTrash

   vvmsg "$THISSCRIPT: EXITCODE: $OverallReturnStatus"

   # Stop logging.
   [[ "${P4U_LOG}" == off ]] || stoplog

   # With the trap removed, exit.
   exit "$OverallReturnStatus"
}

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
#------------------------------------------------------------------------------
function usage
{
   declare style=${1:--h}

   echo "USAGE for $THISSCRIPT version $Version:

$THISSCRIPT [-R rel] [-P python_rel] [-r python_root] [-src] [-f] [-no_ssl] [-w work_dir] [-L <log>] [-si] [-v<n>] [-n] [-D]

or

$THISSCRIPT [-h|-man|-V]
"
   if [[ $style == -man ]]; then
      echo -e "
DESCRIPTION:
	This installs a self-contained Python 3 with P4Python, the Perforce
	API for Python, in /p4/common/python.

	On operating systems the support it, the recommended way to install
	is using the OS package perforce-p4python3 rather than this script.
	For inormation on the Perforce Package repository, see

	https://package.perforce.com

      	This script is useful on platforms for which the package is not
	available, such as Amazon Linux 2 (as of April 2023).

	This script will first download and build Python 3 from source,
	getting a source tarball from python.org.

	For installing P4Python, there are 2 options:
	1) Use pip to install the P4Python package. When this works, it
	is the preferred method.

	2) If '-src' is specified, then this script will download the
	pre-built C++ P4API from ftp.perforce.com, and then download and
	build P4Python from source, getting the tarball from ftp.perforce.com.

PLATFORM SUPPORT:
	This has worked at one point or other on Red Hat Enterprise Linux
	(RHEL), CentOS 6 and 7, Rocky Linux 8, and Amazon Linux 2. It may
	work on Ubuntu 20.04 and other Linux derivatives with no modification.

	It supports the bin.linux26_64 (Linux) and bin.darwin90x86_64
	(Mac OSX/Darwin) architectures.

REQUIREMENTS:
	The Perforce Server Deployment Package (SDP 2020.1+)
	must be installed and configured.  In particular the SDP
	Environment file [$SDPEnvFile] must define a value for OSUSER.

	Development utilities such as 'make' and the 'gcc' compiler
	must be installed and available in the PATH.

	The 'wget' utility must be installed and available in the
	PATH.  (Note that 'wget' is not installed with OSX by
	default, at least not as of OSX 10.11/El Capitan.  However,
	it can be acquired and compiled using XCode).

OPTIONS:
 -R rel
	Specify the Perforce release, e.g. r21.1.  The default is $PerforceRel.
	This is used for both the Perforce API and P4Python.

	This option is ignored if installing P4Python with pip.

 -P python_rel
	Specify the Python release, e.g. 3.9.6.  The default is $PythonRel.

 -r python_root
	Specify the python root.  The default is
	$PythonRoot

	This can be used doing a dry run of an an installation.
	It should not be used for a production install, since the SDP
	environment file, /p4/common/bin/p4_vars, defines the default
	Python root in the PATH.

 -src	Specify that P4Python should be build from source, built locally,
 	and installed. This is an alternative to the default behaviour
	of installing with pip.

 -f 	Specify -f (force) to re-install the SDP Python if it is
	already installed.  By default, in will be installed only
	if the Python root dir (see -r) does not exist.

	This will also cause the downloads directory to be deleted to
	ensure a clean build.

 -no_ssl
	Specify -no_ssl if SSL is neither required nor available.  SSL
	will *always* be used to build P4Python if 'openssl' is
	available, regardless of whether this flag is supplied.  If
	openssl is not available, this script will abort by default.

	Specify '-no_ssl' to consider lack of openssl a warning rather
	than an error condition, and thus to continue and build P4Python
	without SSL.  If openssl is not available and '-no_ssl' is
	specified, the resulting P4Python build will not be able to
	connect to SSL-enabled Perforce Helix servers, i.e. those with
	a P4PORT prefixed with 'ssl:'.

	This flag is ignored if the 'pip' install method is used for
	P4Python.

 -w work_dir
	Specify the working dir.  By default, a temporary working directory
	is created under $Tmp, with a random name.  This temporary working
	directory can be removed upon successful completion.

 -v<n>	Set verbosity 1-5 (-v1 = quiet, -v5 = highest).

 -L <log>
	Specify the path to a log file, or the special value 'off' to disable
	logging.  By default, all output (stdout and stderr) goes to:
	$(dirname "${P4U_LOG}").

	NOTE: This script is self-logging.  That is, output displayed on the screen
	is simultaneously captured in the log file.  Do not run this script with
	redirection operators like '> log' or '2>&1', and do not use 'tee.'

 -si	Operate silently.  All output (stdout and stderr) is redirected to the log
	only; no output appears on the terminal.  This cannot be used with
	'-L off'.
      
 -n	No-Op.  Prints commands instead of running them.

 -D     Set extreme debugging verbosity.

HELP OPTIONS:
 -h	Display short help message
 -man	Display man-style help message
 -V	Dispay version info for this script and its libraries.

DIRECTORIES:
	The Downloads Directory is used to use to store or find existing downloads:
	$DownloadsDir

	The downloads directory is checked for these needed tarfiles:
	$PythonTarFile
	$P4APITarFile (if building P4Python from source)
	$P4PythonTarFile (if building P4Python from source)

	The downloads directory is removed if '-f' is specified.

EXAMPLES:
"
   fi

   exit 1
}

#==============================================================================
# Command Line Processing

declare -i shiftArgs=0
declare -i Force=0
declare -i KeepWorkingDir=1
declare PythonRoot=/p4/common/python
declare Tmp=${P4TMP:-/tmp}
declare WorkingDir=$Tmp/python.$$.$RANDOM
declare DownloadsDir=$Tmp/downloads/p4python
# Set the PerforceRel to the latest API version for which there is also
# a P4Python build available.
declare PerforceRel=r21.1
declare PythonRel=3.10.2
declare PythonTarFile=
declare P4APITarFile=p4api.tgz
declare P4PythonTarFile=p4python.tgz
declare PythonBuildDir=
declare P4PythonBuildDir=
declare APIDir=
declare ApiArch=
declare RunUser=
declare RunArch=x86_64
declare SDPEnvFile=/p4/common/bin/p4_vars
declare ThisArch=
declare ThisOS=

set +u
while [[ $# -gt 0 ]]; do
   case $1 in
      (-R) PerforceRel=$2; shiftArgs=1;;
      (-P) PythonRel=$2; shiftArgs=1;;
      (-r) PythonRoot=$2; shiftArgs=1;;
      (-src) BuildP4PythonFromSource=1;;
      (-f) Force=1;;
      (-no_ssl) RequireSSL=0;;
      (-w)
         if [[ ${2^^} == KEEP ]]; then
            KeepWorkingDir=1
         else
            WorkingDir=$2
            KeepWorkingDir=1
         fi
      ;;
      (-h) usage -h;;
      (-man) usage -man;;
      (-V) show_versions; exit 1;;
      (-v1) export VERBOSITY=1;;
      (-v2) export VERBOSITY=2;;
      (-v3) export VERBOSITY=3;;
      (-v4) export VERBOSITY=4;;
      (-v5) export VERBOSITY=5;;
      (-L) export P4U_LOG="$2"; shiftArgs=1;;
      (-si) SilentMode=1;;
      (-n) export NO_OP=1;;
      (-D) set -x;; # Debug; use 'set -x' mode.
      (*) usageError "Unknown arg ($1).";;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usageError "Bad usage."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

[[ $KeepWorkingDir -eq 0 ]] && GARBAGE+="$WorkingDir"

PythonTarFile=Python-$PythonRel.tgz

#==============================================================================
# Command Line Verification

[[ -n "$P4U_LOG" ]] || P4U_LOG="/tmp/install_sdp_python.sh.$(date +'%Y%m%d-%H%M%S').log"
[[ $SilentMode -eq 1 && $P4U_LOG == off ]] && \
   usageError "Cannot use '-si' with '-L off'."

#==============================================================================
# Main Program

trap terminate EXIT SIGINT SIGTERM

declare -i OverallReturnStatus=0

[[ ! -d $Tmp ]] && bail "Missing SDP tmp dir [$Tmp]. Aborting."

if [[ "${P4U_LOG}" != off ]]; then
   touch "${P4U_LOG}" || bail "Couldn't touch log file [${P4U_LOG}]."

   # Redirect stdout and stderr to a log file.
   if [[ $SilentMode -eq 0 ]]; then
      exec > >(tee "${P4U_LOG}")
      exec 2>&1
   else
      exec >"${P4U_LOG}"
      exec 2>&1
   fi

   initlog
fi

if [[ ! -r "$SDPEnvFile" ]]; then
   bail "Missing or unreadable SDP Environment File [$SDPEnvFile].  Aborting."
fi

RunUser=$(grep '^export OSUSER=' /p4/common/bin/p4_vars |\
   tail -1 | cut -d '=' -f 2)

# shellcheck disable=SC2116
RunUser=$(echo "$RunUser")

if [[ -n "$RunUser" ]]; then
   msg "The OSUSER defined in the SDP environment file is $RunUser."
else
   bail "Could not detect OSUSER in SDP environment file [$SDPEnvFile]. Aborting."
fi

if [[ $USER == "$RunUser" ]]; then
   msg "Verified:  Running as $USER."
else
   bail "Running as $USER.  Run this only as the OSUSER [$RunUser] defined in the SDP Environment File [$SDPEnvFile]. Aborting."
fi

msg "Starting $THISSCRIPT version $Version at $(date) with command line:\n\t$CMDLINE\n\n"

msg "Verifying dependencies."

ThisArch=$(uname -m)

if [[ $ThisArch == "$RunArch" ]]; then
   msg "Verified:  Running on a supported architecture [$ThisArch]."
   ThisOS=$(uname -s)
   ApiArch=UNDEFINED_API_ARCH
   case $ThisOS in
      (Darwin) ApiArch="darwin90x86_64";;
      (Linux) ApiArch="linux26x86_64";;
      (*) bail "Unsupported value returned by 'uname -m': $ThisOS. Aborting.";;
   esac
else
   bail "Running on architecture $ThisArch.  Run this only on hosts with '$RunArch' architecture. Aborting."
fi

   
#------------------------------------------------------------------------------
# Check that we have a compiler.  Give recommendations for package installation
# on 'yum' and 'apt' systems. To Do: For SuSE builds, add package checks for
# zypper, e.g. for zlib, openssl, and openssl-devel.
[[ -z "$(which gcc 2>/dev/null)" || -z "$(which g++ 2>/dev/null)" ]] && \
   bail "No gcc found in the path.  You may need to install it.  Please\n check that the gcc.x86_64 and gcc-c++.x86_64 packages are\n installed, e.g. with:\n\tyum install -y gcc.x86_64 gcc-c++.x86_64\n\n"

[[ -z "$(which wget 2>/dev/null)" ]] && \
   bail "No wget found in the path.  You may need to install it.  Please check that the wget.x86_64 packages is installed, e.g. with:\n\tyum install -y wget.x86_64\n\n"

if [[ -n "$(which openssl 2>/dev/null)" ]]; then
   OpenSSLDir=$(openssl version -a|grep "^OPENSSLDIR:"|cut -d '"' -f 2)

   [[ -z "$OpenSSLDir" ]] && \
      bail "Could not determine OPENSSLDIR with: openssl version -a.\n"

   [[ $RequireSSL -eq 0 ]] && \
      warnmsg "Ignoring '-no_ssl' as openssl is available."

   msg "Verfied: openssl found in PATH: $(which openssl).  Using OPENSSLDIR=$OpenSSLDir."
else
   if [[ $RequireSSL -eq 1 ]]; then
      bail "No openssl found in PATH.  Aborting.  To continue to build without SSL support, specify '-no_ssl'.  The resulting P4Python build will not be able to connect to SSL-enabled Perforce Helix servers.\n"
   else
      warnmsg "No openssl found in PATH.  Continuing to build without SSL support due to '-no_ssl'.  The resulting P4Python build will not be able to connect to SSL-enabled Perforce Helix servers."
      UseSSL=0
   fi
fi

if [[ -d $PythonRoot ]]; then
   if [[ $Force -eq 0 ]]; then
      bail "The SDP Python root directory exists: [$PythonRoot]. Aborting."
   else
      run "/bin/rm -rf $PythonRoot" || bail "Could not remove SDP Python root dir [$PythonRoot]. Aborting."
      run "/bin/mkdir -p $PythonRoot" || bail "Could not recreate SDP Python Root [$PythonRoot]. Aborting."
   fi
fi

if [[ ! -d $WorkingDir ]]; then
   run "/bin/mkdir -p $WorkingDir" || bail "Could not create working dir [$WorkingDir]."
fi

if [[ -d "$DownloadsDir" ]]; then
   if [[ $Force -eq 1 ]]; then
      run "/bin/rm -rf $DownloadsDir" "Removing downloads directory due to -f."
   fi
   run "/bin/mkdir -p $DownloadsDir" || bail "Could not create downloads dir [$DownloadsDir]."
else
   run "/bin/mkdir -p $DownloadsDir" || bail "Could not create downloads dir [$DownloadsDir]."
fi

cd "$DownloadsDir" || bail "Could not cd to [$DownloadsDir]."

msg "Downloading dependencies to $DownloadsDir."

if [[ ! -r $PythonTarFile ]]; then
   run "wget -q --no-check-certificate https://python.org/ftp/python/$PythonRel/$PythonTarFile" ||\
      bail "Could not get $PythonTarFile."
else
   msg "Skipping download of existing $PythonTarFile file."
fi

if [[ "$BuildP4PythonFromSource" -eq 1 ]]; then
   if [[ ! -r $P4APITarFile ]]; then
      run "wget -q https://ftp.perforce.com/perforce/$PerforceRel/bin.$ApiArch/$P4APITarFile" ||\
         bail "Could not get file: $P4APITarFile"
   else
      msg "Skipping download of existing $P4APITarFile file."
   fi

   if [[ ! -r $P4PythonTarFile ]]; then
      run "wget -q http://ftp.perforce.com/perforce/$PerforceRel/bin.tools/$P4PythonTarFile" ||\
         bail "Could not get file '$P4PythonTarFile'"
   else
      msg "Skipping download of existing p4python.tgz file."
   fi
fi

cd "$WorkingDir" || bail "Could not cd to working dir [$WorkingDir]."

run "tar -xzpf $DownloadsDir/$PythonTarFile"

PythonBuildDir="$WorkingDir/$(tar -tzf "$DownloadsDir/$PythonTarFile"|head -1|cut -d '/' -f1)"

cd "$PythonBuildDir" || bail "Could not cd to Python build dir [$PythonBuildDir]."

configureScript="$PWD/tmp.configure.python.sh"
echo -e "#!/bin/bash\n\n$PWD/configure --prefix=$PythonRoot < /dev/null > $PWD/configure_with_prefix.log 2>&1\n\nexit $?\n" > "$configureScript"
chmod +x "$configureScript"
msg "Running this configure script:\n$(cat "$configureScript")\n"
$configureScript || bail "Failed to configure Python."

msg "Building and installing Python to $PythonRoot."
msg "make install in $PWD, logging to make_install.log."
if make install > make_install.log 2>&1; then
   msg "Python install was successful. The install log is: $PWD/make_install.log"
else
   errmsg "Failed to build Python.  Checking the log for known errors ..."

   if grep "zlib not available" "$PWD/make_install.log" 2>/dev/null; then
      errmsg "Known Error Detected - zlib lib missing.  See: https://stackoverflow.com/questions/3905615/zlib-module-missing"
      msg "Analysis of the failed Python build log indicates the zlib library is missing.\nIf on a 'yum' system like RHEL/CentOS, try: sudo yum install -y zlib zlib-devel\nIf on an 'apt' system like Ubuntu, try: sudo apt install -y make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev"
   else
      errmsg "No familiar errors found in the Python build log."
   fi

   bail "Failed to build Python.  Review the log: $PWD/make_install.log"
fi

if [[ "$BuildP4PythonFromSource" -eq 1 ]]; then
   cd "$WorkingDir" || bail "Could not cd to working dir [$WorkingDir]."

   APIDir=$PWD/$(tar -tf "$DownloadsDir/$P4APITarFile"|head -1|cut -d '/' -f1)
   run "tar -xzpf $DownloadsDir/$P4APITarFile"

   P4PythonBuildDir=$(tar -tzf "$DownloadsDir/$P4PythonTarFile"|head -1|cut -d '/' -f1)

   run "tar -xzpf $DownloadsDir/$P4PythonTarFile"

   cd "$P4PythonBuildDir" || bail "Could not cd to P4Python build dir [$P4PythonBuildDir]."

   if [[ -r setup.cfg ]]; then
      mv setup.cfg setup.cfg.orig
      grep -v '^p4_ssl=' setup.cfg.orig > setup.cfg
      echo "p4_api=$APIDir" >> setup.cfg
      if [[ "$UseSSL" -eq 1 ]]; then
         echo -e "p4_ssl=$OpenSSLDir" >> setup.cfg
      fi

      msg "Contents of setup.cfg $PWD:"
      msg "\\n===== START setup.cfg ======"
      cat setup.cfg
      msg "\\n====== END setup.cfg ======"
   else
      errmsg "No setup.cfg found in P4Python dir. This may cause a build failure."
   fi

   msg "Building P4Python in $PWD, logging to build.log"
   BuildCmd="$PythonRoot/bin/python3 setup.py build"

   msg "Executing: $BuildCmd"

   if $BuildCmd > build.log 2>&1; then
      msg "\\nBuild of P4Python was successful."

   else
      errmsg "Failed to build P4Python.  Checking the log for known errors ..."

      msg "Contents of failed build of P4Python in $PWD:"
      msg "\\n===== START build.log ======"
      cat build.log
      msg "\\n====== END build.log ======"

      if grep -q 'cannot find -lcrypto' "$PWD/build.log" 2>/dev/null; then
         errmsg "Known Error Detected - missing crypto libs."
         msg "Analysis of the failed Python build log indicates the crpypto library is missing.\\nIf on a 'yum' system like RHEL/CentOS, try: sudo yum install -y openssl-devel\\nIf on an 'apt' system like Ubuntu, try: sudo apt install libssl-dev"
      else
         errmsg "No familiar errors found in the P4Python build log."
      fi

      bail "Failed to build P4Python.  Review the log: $PWD/build.log."
   fi

   msg "Contents of successful build of P4Python in $PWD:"
   msg "\\n===== START build.log ======"
   cat build.log
   msg "\\n====== END build.log ======"

   msg "Installing P4Python in $PWD, logging to install.log"
   InstallCmd="$PythonRoot/bin/python3 setup.py install"

   msg "Executing: $InstallCmd"

   if $InstallCmd > install.log 2>&1; then
      msg "\\nInstall of P4Python was successful."
   else
      msg "Contents of failed install of P4Python in $PWD:"
      msg "\\n===== START install.log ======"
      cat build.log
      msg "\\n====== END install.log ======"
      bail "Failed to install P4Python.  Review the log: $PWD/install.log."
   fi

   msg "Contents of successful install of P4Python in $PWD:"
   msg "\\n===== START install.log ======"
   cat build.log
   msg "\\n====== END install.log ======"
else
   PipUpgradeCmd="$PythonRoot/bin/python3 -m pip install --upgrade pip"
   run "$PipUpgradeCmd" "Upgrading pip." ||\
      warnmsg "Python pip upgrade failed. Ignoring this failure."

   InstallCmd="$PythonRoot/bin/python3 -m pip install p4python"
   run "$InstallCmd" "Installing P4Python with Pip." ||\
      bail "Failed to install P4Python with pip.  Perhaps try running again with '-src' to build from source?"
fi

msg "\\nEnsure that this directory is in your PATH: $PythonRoot/bin"

if [[ "$OverallReturnStatus" -eq 0 ]]; then
   msg "${H}\nSuccess.  P4Python is ready.\n"
else
   msg "${H}\nProcessing completed, but with errors.  Scan above output carefully.\n" 
fi

# Illustrate using $SECONDS to display runtime of a script.
msg "That took about $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\n"

# See the terminate() function, which is really where this script exits.
exit $OverallReturnStatus