startanalytics.sh #4

#!/bin/bash
# 
# Simple tool for starting HDFS, HBase, and Spark in sequence. Displaying jps for current machine
# and all registered slaves.
#
# $Id: //guest/cmclouth/projects/htd-deployment/htd-source/tools/startanalytics.sh#4 $
#

function waitforrunning() {
	# echo "$@"
	expected=0
	for parm in $@; do
		if [ $expected -gt 0 ]; then
			if [ -z $regex ]; then
				regex="\<$parm\>"
			else
				regex="$regex\|\<$parm\>"
			fi
		else
			remoteserver=$parm
			unset regex
		fi
		((expected += 1))
	done
	((expected -= 1))
	if [ "$remoteserver" = "$HOSTNAME" ]; then
		found=$(jps | grep -v "^.* Jps$" | grep "$regex" | wc -l)
	else
		found=$(ssh "$remoteserver" jps | grep -v "^.* Jps$" | grep "$regex" | wc -l)
	fi
	if [ $found -eq $expected ]; then
		echo "all services running on $remoteserver: $@."
	else
		# determine which ones are missing
		expected=0
		for parm in $@; do
			if [ $expected -gt 0 ]; then
				regex="\<$parm\>"
				if [ "$remoteserver" = "$HOSTNAME" ]; then
					found=$(jps | grep -v "^.* Jps$" | grep "$regex" | wc -l)
				else
					found=$(ssh "$remoteserver" jps | grep -v "^.* Jps$" | grep "$regex" | wc -l)
				fi
				if [ $found -lt 1 ]; then
					if [ -z notfound ]; then
						notfound="$parm"
					else
						notfound="$notfound $parm"
					fi
				fi
			fi
			((expected += 1))
		done
		echo "expected services not running on $remoteserver: $notfound."
		exit 1
	fi
}
function waitforslaves() {
	for htdanalyticsdataslave in $(grep -v "^$" "/opt/interset/hadoop/etc/hadoop/slaves" | grep -v "^#"); do
		waitforrunning "$htdanalyticsdataslave" $@
	done
}
# Exit immediately on non-zero status.
set -e
HOSTNAME=$(hostname)
echo "Start hdfs"
/opt/interset/hadoop/sbin/start-dfs.sh
sleep 3
waitforrunning "$HOSTNAME" "NameNode" "SecondaryNameNode"
# check slaves
waitforslaves "DataNode"
echo "Start hbase"
# sudo service hbase start
/opt/interset/hbase/bin/start-hbase.sh
sleep 3
waitforrunning "$HOSTNAME" "NameNode" "SecondaryNameNode" "HQuorumPeer" "HMaster"
# check slaves
waitforslaves "DataNode" "HRegionServer"
echo "Start spark"
/opt/interset/spark/sbin/start-all.sh
# sudo service spark start
sleep 3
# check slaves
waitforslaves "DataNode" "HRegionServer" "Worker"
waitforrunning "$HOSTNAME" "NameNode" "SecondaryNameNode" "HQuorumPeer" "HMaster" "Master"