build.sh #2

#!/usr/bin/env bash
#
# build.sh
# --------
#
# Script to drive the build and test cycle of p4ruby across several different 
# OSes managed via Salt.
#
# Environment Setup
# -----------------
#
# We assume this script is running on the salt master as a user with nopasswd
# sudo access.
#
# ## Salt Master Setup
#
# The salt master should include:
#
# 1. The 'p4' command line app available on the path
#
# 2. An RVM multi-user installation with the current user referencing a 
# 	 'build' gemset for Ruby 2.0. (This is the lowest version that's supported
# 	 via RVM.)
#
# 3. A Salt file root at /srv/salt that's writeable by the executing user
#
# ## The build instance: node_type: build
#
# There should be a minion with the grain configuration 'node_type: build'. 
# We assume there's only one of these, and it's a Windows 7 machine with 
# the RubyDevKit and Ruby 2.0, 2.1, 2.2 distributions installed for both
# 32 and 64 bit deployments. The setup work of the instance is not yet 
# automated.
#


#-----------------------------------------------------------------------------
# Configuration
#-----------------------------------------------------------------------------

# P4Ruby is branched to follow it's MAJOR.MINOR versions. Except for main,
# which basically should never be released, just tested.
p4ruby_branch=${P4RUBY_BRANCH:main}

# This is overridden by 'rake test' after we've setup the ruby dependencies
p4ruby_version=invalid

# Our internal codeline specifier. This should align with p4ruby, but it's
# an internal specifier that people generally don't see.
p4_branch=r15.1

# The workspace root directory. This should appear like a 'wide open' client
# that maps in our internal Perforce depot repository. This basically includes:
#
#  1. The workshop branch of p4ruby
#  2. p4api.tgz, p4, and p4.exe of our required platforms
#  3. OpenSSL of MinGW-w64 builds for Windows
#
# Our internal CI system doesn't easily allow for me to remap directories, 
# so the directory layout of the working area follows our internal conventions,
# which can appear quite nested.
ws_root=${BUILD_P4CLIENTROOT}

# This is generally set via the EC build environment, and helps us identify
# where the dependent components come from.
codeline=${CODELINE}

p4_bin=${ws_root}/depot/${CODELINE}/p4-bin

p4ruby_dir=${ws_root}/public/guest/perforce_software/p4ruby/$p4ruby_branch

build_gemset=build
build_ruby_version=2.0.0-p643
build_user=perforce
build_wrappers=/home/perforce/.rvm/wrappers/ruby-2.0.0-p643

wrapper_dir=${HOME}/.rvm/wrappers/ruby-${build_ruby_version}@${build_gemset}

# This is the root of the local salt file directory that we copy temporary 
# from this server that get transferred to minions via cp.get_file
#
# Currently for this build this is really just a temp directory which should
# get cleaned out regularly.
salt_file_root=/srv/salt

openssl_root=${ws_root}/3rd_party/cpp_libraries/openssl/1.0.1m

# The salt parameter used to target the 'build' machines
# Right now, since I'm only running this on one machine, I target it via
# the hostname directly. It's much faster than -G 'grain' targeting.
salt_build='p4ruby-crossbuild.das.perforce.com'

salt_test_windows='p4ruby-test-windows7.das.perforce.com'

salt_test_ubuntu12='p4ruby-test-ubuntu12-pkg.das.perforce.com'

salt_test_ubuntu14='p4ruby-test-ubuntu14-rvm.das.perforce.com'

salt_test_centos6='p4ruby-test-centos6-pkg.das.perforce.com'

#-----------------------------------------------------------------------------
# create_p4ruby_tarball
#-----------------------------------------------------------------------------
# One of the first tasks is to generate the p4ruby.zip and p4ruby.tgz archives.
# They are just placed in the salt root here just to make sure 
create_p4ruby_tarball()
{
	cd $p4ruby_dir
	$wrapper_dir/bundle install || false
	$wrapper_dir/rake package || false
	ls pkg

	cp pkg/p4ruby.tgz $salt_file_root
	cp pkg/p4ruby.zip $salt_file_root
}

#-----------------------------------------------------------------------------
# set_p4ruby_version
#-----------------------------------------------------------------------------
# Update the p4ruby version string based on what's set in ruby. Depends upon
# create_p4ruby_tarball having been run, and already setup our Ruby dependencies
set_p4ruby_version()
{
    cd $p4ruby_dir
    p4ruby_version=`$wrapper_dir/rake version`
}

#-----------------------------------------------------------------------------
# prepare_build
#-----------------------------------------------------------------------------
# Transfer dependencies and the source tree to the crosscompile machine
prepare_build()
{
	# salt-cp doesn't copy binary files on windows
	# https://github.com/saltstack/salt/issues/23566
	# sudo salt-cp $salt_build $salt_file_root/p4ruby.zip C:\\Builds\\p4ruby.zip
	sudo salt $salt_build cp.get_file salt://p4ruby.tgz /var/tmp/p4ruby.tgz
	
	# I get a strange 'NoneType is not an object' error
	#sudo salt $salt_build archive.tar xzvf /var/tmp/p4ruby.tgz
	sudo salt $salt_build cmd.run 'tar xzv -C /var/tmp -f /var/tmp/p4ruby.tgz' runas=$build_user

	mkdir -p $salt_file_root/openssl/mingw64x64
	cp -r $openssl_root/artifacts/inc.mingw64x64 $salt_file_root/openssl/mingw64x64
	cp -r $openssl_root/artifacts/lib.mingw64x64 $salt_file_root/openssl/mingw64x64

	mkdir -p $salt_file_root/openssl/mingw64x86
	cp -r $openssl_root/artifacts/inc.mingw64x86 $salt_file_root/openssl/mingw64x86
	cp -r $openssl_root/artifacts/lib.mingw64x86 $salt_file_root/openssl/mingw64x86

	# salt-cp doesn't copy binary files on windows
	# https://github.com/saltstack/salt/issues/23566
	#sudo salt-cp $salt_build $salt_file_root/openssl C:\\Builds\\openssl
	sudo salt $salt_build cp.get_dir salt://openssl /var/tmp

	mkdir -p $salt_file_root/depot/$codeline/p4-bin
	cp -r $p4_bin/bin.mingw64x64 $salt_file_root/depot/$codeline/p4-bin
	cp -r $p4_bin/bin.mingw64x86 $salt_file_root/depot/$codeline/p4-bin
	cp -r $p4_bin/bin.ntx64 $salt_file_root/depot/$codeline/p4-bin

	sudo salt $salt_build cp.get_dir salt://depot /var/tmp

	# Unpack p4api archives
	sudo salt $salt_build archive.unzip /var/tmp/depot/$codeline/p4-bin/bin.mingw64x64/p4api.zip /var/tmp/depot/$codeline/p4-bin/bin.mingw64x64
	sudo salt $salt_build archive.unzip /var/tmp/depot/$codeline/p4-bin/bin.mingw64x86/p4api.zip /var/tmp/depot/$codeline/p4-bin/bin.mingw64x86
}

#-----------------------------------------------------------------------------
# run_build
#-----------------------------------------------------------------------------
run_build()
{
	sudo salt $salt_build cmd.run "/bin/bash -c 'P4RUBY_CROSS_PLATFORM=x64-mingw32 p4api_dir=`echo /var/tmp/depot/$codeline/p4-bin/bin.mingw64x64/p4api-${p4ruby_branch}.*` ssl_include_dir=/var/tmp/openssl/mingw64x64/inc.mingw64x64 ssl_lib_dir=/var/tmp/openssl/mingw64x64/lib.mingw64x64 $build_wrappers/rake cross compile RUBY_CC_VERSION=2.0.0:2.1.5:2.2.2'" runas=$build_user cwd=/var/tmp/p4ruby

    sudo salt $salt_build cmd.run "/bin/bash -c 'P4RUBY_CROSS_PLATFORM=x86-mingw32 p4api_dir=`echo /var/tmp/depot/$codeline/p4-bin/bin.mingw64x86/p4api-${p4ruby_branch}.*` ssl_include_dir=/var/tmp/openssl/mingw64x86/inc.mingw64x86 ssl_lib_dir=/var/tmp/openssl/mingw64x86/lib.mingw64x86 $build_wrappers/rake cross compile RUBY_CC_VERSION=2.0.0:2.1.5:2.2.2'" runas=$build_user cwd=/var/tmp/p4ruby

	sudo salt $salt_build cmd.run "/bin/bash -c 'P4RUBY_CROSS_PLATFORM=x64-mingw32 p4api_dir=`echo /var/tmp/depot/$codeline/p4-bin/bin.mingw64x64/p4api-${p4ruby_branch}.*` ssl_include_dir=/var/tmp/openssl/mingw64x64/inc.mingw64x64 ssl_lib_dir=/var/tmp/openssl/mingw64x64/lib.mingw64x64 $build_wrappers/rake cross native gem RUBY_CC_VERSION=2.0.0:2.1.5:2.2.2'" runas=$build_user cwd=/var/tmp/p4ruby

    sudo salt $salt_build cmd.run "/bin/bash -c 'P4RUBY_CROSS_PLATFORM=x86-mingw32 p4api_dir=`echo /var/tmp/depot/$codeline/p4-bin/bin.mingw64x86/p4api-${p4ruby_branch}.*` ssl_include_dir=/var/tmp/openssl/mingw64x86/inc.mingw64x86 ssl_lib_dir=/var/tmp/openssl/mingw64x86/lib.mingw64x86 $build_wrappers/rake cross native gem RUBY_CC_VERSION=2.0.0:2.1.5:2.2.2'" runas=$build_user cwd=/var/tmp/p4ruby

	# Push gems back to the host
	sudo salt $salt_build cp.push /var/tmp/p4ruby/pkg/p4ruby-${p4ruby_version}.gem
	sudo salt $salt_build cp.push /var/tmp/p4ruby/pkg/p4ruby-${p4ruby_version}-x64-mingw32.gem
	sudo salt $salt_build cp.push /var/tmp/p4ruby/pkg/p4ruby-${p4ruby_version}-x86-mingw32.gem

	# Copy the gems into our salt root
	cp /var/cache/salt/master/minions/p4ruby-crossbuild.das.perforce.com/files/var/tmp/p4ruby/pkg/*.gem $salt_file_root
}

#-----------------------------------------------------------------------------
# test_windows
#-----------------------------------------------------------------------------
# Test both 32- and 64- bit gems out on p4ruby-test-windows7.das.perforce.com
# This requires the p4d.exe NT build, and SSL, which should be prepared
# in Salt by prepare_build()
test_windows()
{
    sudo salt $salt_test_windows file.mkdir C:\\test || false

    # Note that windows *does not* use the source gem.
    sudo salt $salt_test_windows cp.get_file salt://p4ruby-${p4ruby_version}-x64-mingw32.gem C:\\test\\p4ruby-${p4ruby_version}-x64-mingw32.gem
    sudo salt $salt_test_windows cp.get_file salt://p4ruby-${p4ruby_version}-x86-mingw32.gem C:\\test\\p4ruby-${p4ruby_version}-x86-mingw32.gem

    sudo salt $salt_test_windows cp.get_file salt://depot/$codeline/p4-bin/bin.ntx64/p4d.exe C:\\test\\p4d.exe

    cp $p4ruby_dir/config/install_windows.bat $salt_file_root
    cp $p4ruby_dir/config/test_windows.bat $salt_file_root
    cp $p4ruby_dir/config/uninstall_windows.bat $salt_file_root

    # The source distribution should only be needed for executing tests.
    sudo salt $salt_test_windows cp.get_file salt://p4ruby.zip C:\\test\\p4ruby.zip || false
    sudo salt $salt_test_windows archive.unzip C:\\test\\p4ruby.zip C:\\test || false

    sudo salt $salt_test_windows cmd.script salt://install_windows.bat template=jinja context="{\"ruby_version\":\"Ruby200\", \"gem_file\":\"p4ruby-${p4ruby_version}-x86-mingw32.gem\"}"
    sudo salt $salt_test_windows cmd.script salt://test_windows.bat template=jinja context="{\"ruby_version\":\"Ruby200\"}"
    sudo salt $salt_test_windows cmd.script salt://uninstall_windows.bat template=jinja context="{\"ruby_version\":\"Ruby200\"}"

    sudo salt $salt_test_windows cmd.script salt://install_windows.bat template=jinja context="{\"ruby_version\":\"Ruby21\", \"gem_file\":\"p4ruby-${p4ruby_version}-x86-mingw32.gem\"}"
    sudo salt $salt_test_windows cmd.script salt://test_windows.bat template=jinja context="{\"ruby_version\":\"Ruby21\"}"
    sudo salt $salt_test_windows cmd.script salt://uninstall_windows.bat template=jinja context="{\"ruby_version\":\"Ruby21\"}"

    sudo salt $salt_test_windows cmd.script salt://install_windows.bat template=jinja context="{\"ruby_version\":\"Ruby22\", \"gem_file\":\"p4ruby-${p4ruby_version}-x86-mingw32.gem\"}"
    sudo salt $salt_test_windows cmd.script salt://test_windows.bat template=jinja context="{\"ruby_version\":\"Ruby22\"}"
    sudo salt $salt_test_windows cmd.script salt://uninstall_windows.bat template=jinja context="{\"ruby_version\":\"Ruby22\"}"

    sudo salt $salt_test_windows cmd.script salt://install_windows.bat template=jinja context="{\"ruby_version\":\"Ruby200-x64\", \"gem_file\":\"p4ruby-${p4ruby_version}-x64-mingw32.gem\"}"
    sudo salt $salt_test_windows cmd.script salt://test_windows.bat template=jinja context="{\"ruby_version\":\"Ruby200-x64\"}"
    sudo salt $salt_test_windows cmd.script salt://uninstall_windows.bat template=jinja context="{\"ruby_version\":\"Ruby200-x64\"}"

    sudo salt $salt_test_windows cmd.script salt://install_windows.bat template=jinja context="{\"ruby_version\":\"Ruby21-x64\", \"gem_file\":\"p4ruby-${p4ruby_version}-x64-mingw32.gem\"}"
    sudo salt $salt_test_windows cmd.script salt://test_windows.bat template=jinja context="{\"ruby_version\":\"Ruby21-x64\"}"
    sudo salt $salt_test_windows cmd.script salt://uninstall_windows.bat template=jinja context="{\"ruby_version\":\"Ruby21-x64\"}"

    sudo salt $salt_test_windows cmd.script salt://install_windows.bat template=jinja context="{\"ruby_version\":\"Ruby22-x64\", \"gem_file\":\"p4ruby-${p4ruby_version}-x64-mingw32.gem\"}"
    sudo salt $salt_test_windows cmd.script salt://test_windows.bat template=jinja context="{\"ruby_version\":\"Ruby22-x64\"}"
    sudo salt $salt_test_windows cmd.script salt://uninstall_windows.bat template=jinja context="{\"ruby_version\":\"Ruby22-x64\"}"

    sudo salt $salt_test_windows cmd.run 'RMDIR C:\\test /S /Q'
}

#-----------------------------------------------------------------------------
# test_ubuntu12_system19
#-----------------------------------------------------------------------------
# This test will run against the Ubuntu 1.9 system installer, but downloading
# the p4api from ftp.perforce.com
#
# Note 1: bundler and rake should be installed.
#
# Note 2: The rubygems package supported by Debian is very old, so we have to
#         work around the fact that the rake-compiler package can't be installed
test_ubuntu12_system19()
{
    sudo salt $salt_test_ubuntu12 cp.get_file salt://p4ruby-${p4ruby_version}.gem /var/tmp/p4ruby-${p4ruby_version}.gem

    cp $ws_root/depot/$codeline/p4-bin/bin.linux26x86_64/p4d $salt_file_root
    sudo salt $salt_test_ubuntu12 cp.get_file salt://p4d /var/tmp/p4d
    sudo salt $salt_test_ubuntu12 cmd.run 'chmod +x /var/tmp/p4d'

    sudo salt $salt_test_ubuntu12 cmd.run "gem install /var/tmp/p4ruby-${p4ruby_version}.gem"

    sudo salt $salt_test_ubuntu12 cp.get_file salt://p4ruby.tgz /var/tmp/p4ruby.tgz
    sudo salt $salt_test_ubuntu12 cmd.run "tar xzv -C /var/tmp -f /var/tmp/p4ruby.tgz"

    sudo salt $salt_test_ubuntu12 cmd.run "bundle install --without=development" cwd=/var/tmp/p4ruby
    sudo salt $salt_test_ubuntu12 cmd.run "bash -c 'P4D_BIN=/var/tmp/p4d rake test'" cwd=/var/tmp/p4ruby

    sudo salt $salt_test_ubuntu12 cmd.run "gem uninstall p4ruby"

    sudo salt $salt_test_ubuntu12 cmd.run "rm -rf /var/tmp/*"
}

#-----------------------------------------------------------------------------
# test_centos6_system19
#-----------------------------------------------------------------------------
# This comes from the "Software Collections Library", hence, we have to
# prefix our commands with 'scl enable ruby193'
test_centos6_system19()
{
    sudo salt $salt_test_centos6 cp.get_file salt://p4ruby-${p4ruby_version}.gem /var/tmp/p4ruby-${p4ruby_version}.gem

    cp $ws_root/depot/$codeline/p4-bin/bin.linux26x86_64/p4d $salt_file_root
    sudo salt $salt_test_centos6 cp.get_file salt://p4d /var/tmp/p4d
    sudo salt $salt_test_centos6 cmd.run 'chmod +x /var/tmp/p4d'

    sudo salt $salt_test_centos6 cmd.run "scl enable ruby193 \"gem install /var/tmp/p4ruby-${p4ruby_version}.gem\""

    sudo salt $salt_test_centos6 cp.get_file salt://p4ruby.tgz /var/tmp/p4ruby.tgz
    sudo salt $salt_test_centos6 cmd.run "tar xzv -C /var/tmp -f /var/tmp/p4ruby.tgz"

    sudo salt $salt_test_centos6 cmd.run "scl enable ruby193 \"bundle install --without=development\"" cwd=/var/tmp/p4ruby
    sudo salt $salt_test_centos6 cmd.run "bash -c 'export P4D_BIN=/var/tmp/p4d; scl enable ruby193 \"rake test\"'" cwd=/var/tmp/p4ruby

    sudo salt $salt_test_centos6 cmd.run "scl enable ruby193 \"gem uninstall p4ruby\""

    sudo salt $salt_test_centos6 cmd.run "rm -rf /var/tmp/*"
}

#-----------------------------------------------------------------------------
# prepare_distribution
#-----------------------------------------------------------------------------
# Make sure that output files are made available at the root of the workspace
# which will not be deleted
prepare_distribution()
{
    cd $ws_root

    mkdir p4-bin/bin.tools

    cp $salt_file_root/*.gem p4-bin/bin.tools
    cp $salt_file_root/p4ruby.tgz p4-bin/bin.tools
    cp $salt_file_root/p4ruby.zip p4-bin/bin.tools
}

#-----------------------------------------------------------------------------
# clean_build
#-----------------------------------------------------------------------------
clean_build()
{	
	sudo salt $salt_build cmd.run 'rm -rf /var/tmp/*'
}

#-----------------------------------------------------------------------------
# clean_salt_file_root
#-----------------------------------------------------------------------------
clean_salt_file_root()
{
	sudo rm -rf $salt_file_root/*
}


#-----------------------------------------------------------------------------
# Main Script
#-----------------------------------------------------------------------------

set -ex

create_p4ruby_tarball
set_p4ruby_version

prepare_build

run_build

test_windows
test_ubuntu12_system19
test_centos6_system19

prepare_distribution

clean_build
clean_salt_file_root