netssltransport.cc #1

/**
 * @file netssltransport.cc
 *
 * @brief SSL driver for NetTransport
 *	NetSslTransport - a TCP with SSL subclass of NetTcpTransport
 *
 * Threading: underlying SSL library contains threading
 *
 * @invariants:
 *
 * Copyright (c) 2011 Perforce Software
 * Confidential.  All Rights Reserved.
 * @author Wendy Heffner
 *
 * Creation Date: August 19, 2011
 */

/**
 * NOTE:
 * The following file only defined if USE_SSL true.
 * The setting of this definition is controlled by
 * the Jamrules file.  If the jam build is specified
 * with -sSSL=yes this class will be defined.
 */
# ifdef USE_SSL

# define NEED_ERRNO
# define NEED_SIGNAL
# ifdef OS_NT
# define NEED_FILE
# endif
# define NEED_FCNTL
# define NEED_IOCTL
# define NEED_TYPES
# define NEED_SOCKET_IO
# define NEED_SLEEP

# ifdef OS_MPEIX
# define _SOCKET_SOURCE /* for sys/types.h */
# endif

# define GET_MJR_SSL_VERSION(x) \
    (((x) >> 12) & 0xfffffL)

# include <stdhdrs.h>

# ifndef OS_NT
# include <pthread.h>
# endif // not OS_NT

# include <error.h>
# include <strbuf.h>
# include <md5.h>
# include "netaddrinfo.h"
# include <errorlog.h>
# include <debug.h>
# include <bitarray.h>
# include <tunable.h>
# include <error.h>

# include <enviro.h>
# include <msgrpc.h>
# include <timer.h>
# include "filesys.h"
# include "pathsys.h"

# include <keepalive.h>
# include "netsupport.h"
# include "netport.h"
# include "netportparser.h"
# include "netconnect.h"
# include "nettcpendpoint.h"
# include "nettcptransport.h"
# include "netselect.h"
# include "netdebug.h"


extern "C"
{
    // OpenSSL
# include "openssl/bio.h"
# include "openssl/ssl.h"
# include "openssl/err.h"
# if !( defined( OPENSSL_VERSION_TEXT ) && defined( OPENSSL_VERSION_NUMBER ))
# include "openssl/opensslv.h"
# endif

   // For strerror
# include <stdio.h>
}
# include "netsslcredentials.h"
# include "netsslendpoint.h"
# include "netssltransport.h"
# include "netsslmacros.h"
# include "datetime.h"


////////////////////////////////////////////////////////////////////////////
//  Defines and Globals                                                   //
////////////////////////////////////////////////////////////////////////////

/*
 * Current Cypher Suite Configuration:
 * Primary:   AES256-SHA        SSLv3 Kx=RSA      Au=RSA  Enc=AES(256)      Mac=SHA1
 * Secondary: CAMELLIA256-SHA   SSLv3 Kx=RSA      Au=RSA  Enc=Camellia(256) Mac=SHA1
 */
# define SSL_PRIMARY_CIPHER_SUITE "AES256-SHA"
# define SSL_SECONDARY_CIPHER_SUITE "CAMELLIA256-SHA"

const char *P4SslVersionString = OPENSSL_VERSION_TEXT;
unsigned long sCompileVersion =  OPENSSL_VERSION_NUMBER;

# ifdef OS_NT
static HANDLE *mutexArray = NULL;
# else
static pthread_mutex_t *mutexArray = NULL;
# endif

////////////////////////////////////////////////////////////////////////////
//  MutexLocker                                                           //
////////////////////////////////////////////////////////////////////////////
# ifdef OS_NT
# define ONE_SECOND 1000
class MutexLocker
{
    public:
	MutexLocker( HANDLE &mutex );
	~MutexLocker();

    private:
	DWORD   retval;
	HANDLE &mutex;
};

MutexLocker::MutexLocker( HANDLE &theLock )
    : mutex(theLock)
{
	retval = WaitForSingleObject( mutex, ONE_SECOND );
	if( retval != WAIT_OBJECT_0 )
	    DEBUGPRINT( SSLDEBUG_ERROR,
		    "Unable to acquire lock on windows." );
}

MutexLocker::~MutexLocker()
{
	if( retval == WAIT_OBJECT_0 )
	    ReleaseMutex( mutex );
}

# endif // OS_NT


////////////////////////////////////////////////////////////////////////////
//  Global                                                                //
////////////////////////////////////////////////////////////////////////////
# ifdef OS_NT
HANDLE                    sClientMutex = CreateMutex (NULL, FALSE, NULL);
HANDLE                    sServerMutex = CreateMutex (NULL, FALSE, NULL);
# endif // OS_NT

/**
 * MillisecondDifference
 *
 * @brief Utility to find the difference between two DateTimeHighPrecision
 * Note: Assumes that the difference is bounded by at most one second
 * since this is used to measure passage of time in a select with a 1/2 sec
 * timeout.
 *
 * @param lop left operator in subtraction
 * @param rop right operator in subtraction
 * @return count of milliseconds difference
 */
int
MillisecondDifference(const DateTimeHighPrecision &lop,
		      const DateTimeHighPrecision &rop)
{
    int sec = lop.Seconds() - rop.Seconds();
    int leftMs = (sec * 1000) + (lop.Nanos() / 1000000);
    int rightMs = (rop.Nanos() / 1000000);
    return (leftMs - rightMs);
}

////////////////////////////////////////////////////////////////////////////
//  NetSslTransport - STATIC MEMBERS                                      //
////////////////////////////////////////////////////////////////////////////
SSL_CTX *NetSslTransport::sServerCtx = NULL;
SSL_CTX *NetSslTransport::sClientCtx = NULL;


/**
 * NetSslTransport::constructor
 *
 * @brief constructor
 * @invariant assures that bio and ssl are NULL prior to use
 *
 * @param int socket
 * @param bool flag indicating if server or client side
 */
NetSslTransport::NetSslTransport( int t, bool fromClient )
    : NetTcpTransport( t, fromClient )
{
	this->bio = NULL;
	this->ssl = NULL;
	this->clientNotSsl = false;
	cipherSuite.Set("encrypted");
}

NetSslTransport::NetSslTransport( int t, bool fromClient,
				NetSslCredentials &cred )
    : NetTcpTransport( t, fromClient ), credentials(cred)
{
	this->bio = NULL;
	this->ssl = NULL;
	this->clientNotSsl = false;
	cipherSuite.Set("encrypted");
}
NetSslTransport::~NetSslTransport()
{
	Close();
}

/**
 * NetSslTransport::SslClientInit
 *
 * @brief static method to initialize the SSL client-side CTX structure
 *
 * @param error structure
 * @return none
 */
void
NetSslTransport::SslClientInit(Error *e)
{
	if( sClientCtx )
	    return;

# ifdef OS_NT
	/*
	 * Windows multi-threaded so must synchronize
	 * CTX creation, since we only want one.
	 */
	MutexLocker locker( sClientMutex );
	/*
	 * Now that we have the lock see if any
	 * thread created the CTX while we were
	 * waiting.
	 */
# endif // OS_NT

	if( !sClientCtx )
	{
#ifdef OS_NT
	    InitLockCallbacks( e );
	    if( e->Test())
		return;
#endif // OS_NT

	    TRANSPORT_PRINT( SSLDEBUG_FUNCTION,
		"NetSslTransport::SslClientInit - Initializing client CTX structure." );


	    ValidateRuntimeVsCompiletimeSSLVersion( e );
	    if( e->Test() )
	    {
		TRANSPORT_PRINT( SSLDEBUG_ERROR,
			"Version mismatch between compile OpenSSL version and runtime OpenSSL version." );
		return;
	    }

	    SSL_load_error_strings();
	    SSLCHECKERROR( e,
                       "NetSslTransport::SslClientInit SSL_load_error_strings",
                       MsgRpc::SslInit,
                       fail );
	    ERR_load_BIO_strings();
	    SSLCHECKERROR( e, "NetSslTransport::SslClientInit ERR_load_BIO_strings",
                       MsgRpc::SslInit,
                       fail );
	    if ( !SSL_library_init() )
	    {
		// executable not compiled supporting SSL
		// need to link with open SSL libraries
		e->Set(MsgRpc::SslNoSsl);
		return;
	    }
	    SSLCHECKERROR( e, "NetSslTransport::SslClientInit SSL_library_init",
                       MsgRpc::SslInit,
                       fail );

	    // WSAstartup code in NetTcpEndPoint constructor

	    sClientCtx = SSL_CTX_new( TLSv1_method() );
	    SSLNULLHANDLER( sClientCtx, e,
                        "NetSslTransport::SslClientInit SSL_CTX_new",
                        fail );

	    SSL_CTX_set_mode(
		sClientCtx,
		SSL_MODE_ENABLE_PARTIAL_WRITE | SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER );
	    SSLLOGFUNCTION( "NetSslTransport::SslClientInit SSL_CTX_set_mode" );
	}
	return;

fail:
	e->Set( MsgRpc::SslCtx );
	return;
}

void
NetSslTransport::GetPeerFingerprint(StrBuf &value)
{
	if( !isAccepted && credentials.GetFingerprint() &&
		credentials.GetFingerprint()->Length() )
	    value.Set( credentials.GetFingerprint()->Text() );
	else
	    value.Clear();
}


/**
 * NetSslTransport::SslServerInit
 *
 * @brief One time initialization for server side of connection for SSL.
 * @invariant Assures that the server-side SSL CTX structure has been initialized.
 *
 * @param error structure
 * @return nothing
 */
void
NetSslTransport::SslServerInit(StrPtr *hostname, Error *e)
{
	if( sServerCtx )
	    return;

# ifdef OS_NT
	/*
	 * Windows multi-threaded so must synchronize
	 * CTX creation, since we only want one.
	 */
	MutexLocker locker(sServerMutex);
	/*
	 * Now that we have the lock see if any
	 * thread created the CTX while we were
	 * waiting.
	 */
# endif // OS_NT

	if( !sServerCtx )
	{
#ifdef OS_NT
	    InitLockCallbacks( e );
	    if( e->Test())
		return;
#endif // OS_NT

	    TRANSPORT_PRINT( SSLDEBUG_FUNCTION,
		"NetSslTransport::SslServerInit - Initializing server CTX structure." );
	    SSL_load_error_strings();
	    SSLCHECKERROR( e,
                       "NetSslTransport::SslServerInit SSL_load_error_strings",
                       MsgRpc::SslInit,
                       fail );
	    ERR_load_BIO_strings();
	    SSLCHECKERROR( e, "NetSslTransport::SslServerInit ERR_load_BIO_strings",
                       MsgRpc::SslInit,
                       fail );
	    if ( !SSL_library_init() )
	    {
		// executable not compiled supporting SSL
		// need to link with open SSL libraries
		e->Set(MsgRpc::SslNoSsl);
		return;
	    }
	    SSLCHECKERROR( e, "NetSslTransport::SslServerInit SSL_library_init",
                       MsgRpc::SslInit,
                       fail );

	    /* Set up cert and key:
	     * Note that we have already verified in RpcService::Listen
	     * that sslKey and sslCert exist in P4SSLDIR and are valid.
	     * Since we are doing lazy init of the SslCtx the
	     * files could have been deleted before hitting this code.  We will do
	     * another verify check here when we finally load the credentials
	     * for the first read/write.
	     */
	    credentials.ReadCredentials(e);
	    P4CHECKERROR( e, "NetSslTransport::SslServerInit ReadCredentials", fail );


	    sServerCtx = SSL_CTX_new( TLSv1_method() );
	    SSLNULLHANDLER( sServerCtx, e,
                        "NetSslTransport::SslServerInit SSL_CTX_new",
                        fail );

	    SSL_CTX_set_mode(
		sServerCtx,
		SSL_MODE_ENABLE_PARTIAL_WRITE | SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER );
	    SSLLOGFUNCTION( "NetSslTransport::SslServerInit SSL_CTX_set_mode" );


	    SSL_CTX_use_PrivateKey( sServerCtx, credentials.GetPrivateKey() );
	    SSLLOGFUNCTION(
		"NetSslTransport::SslServerInit SSL_CTX_use_PrivateKey" );
	    credentials.SetOwnKey(false);
	    /*
	     * Note: if want key passphrase protected then need to implement
	     * a callback function to supply the passphrase:
	     *     int passwd_cb(char *buf, int size, int flag, void *userdata);
	     *
	     * Alternatively strip the passphrase protection off the key via
	     *     cp server.key server.key.org
	     *     openssl [rsa|dsa] -in server.key.org -out server.key
	     */
	    SSL_CTX_use_certificate( sServerCtx, credentials.GetCertificate() );
	    SSLLOGFUNCTION(
		"NetSslTransport::SslServerInit SSL_CTX_use_certificate" );
	    credentials.SetOwnCert(false);

	    /*
	     * Set context to not verify certificate authentication with CA.
	     */
	    SSL_CTX_set_verify( sServerCtx, SSL_VERIFY_NONE, NULL /* no callback */);
	    SSLLOGFUNCTION(
		"NetSslTransport::SslServerInit SSL_CTX_set_verify server ctx" );

	    /*
	     * NOTE: The way this code is written there is no CA check on the cert.
	     * If want to do this then SSL_CTX_load_verify_locations(sServerCtx, NULL, sslCACert);
	     * The certificate authority certificate directory must be hashed:
	     *     c_rehash /path/to/certfolder
	     */
	}

	return;

fail:
	e->Set( MsgRpc::SslCtx );
	return;
}

/**
 * NetSslTransport::DoHandshake
 *
 * @brief transport level method to invoke SSL handshake and provide debugging output
 * @invariant if SSL and CTX structures are non-null handshake has completed successfully
 *
 * @param error structure
 * @return none
 */
void
NetSslTransport::DoHandshake( Error *e )
{
	int sslRetval;

	if(ssl)
	    return;

	/*
	 * Lazy initialization: If no SSL context has been created for
	 * this type of connection (server side or client side) then do it
	 * now, then use it to create an SSL structure for this connection.
	 */
	if( this->isAccepted )
	{
	    ssl = SSL_new( sServerCtx );
	    SSLNULLHANDLER( ssl, e, "NetSslTransport::DoHandshake SSL_new", fail );
	    if ( p4tunable.Get( P4TUNE_SSL_SECONDARY_SUITE ) )
	    {
		SSL_set_cipher_list( ssl, SSL_SECONDARY_CIPHER_SUITE );
		SSLLOGFUNCTION( "NetSslTransport::DoHandshake SSL_set_cipher_list secondary" );
	    } else
	    {
		SSL_set_cipher_list( ssl, SSL_PRIMARY_CIPHER_SUITE );
		SSLLOGFUNCTION( "NetSslTransport::DoHandshake SSL_set_cipher_list primary" );
	    }
	}
	else
	{
	    ssl = SSL_new( sClientCtx );
	    SSLNULLHANDLER( ssl, e, "NetSslTransport::DoHandshake SSL_new", fail );
	}

	/*
	 * If debugging output configured, dump prioritized list of
	 * supported cipher-suites.
	 */
	if( SSLDEBUG_TRANS )
	{
	    int priority = 0;
	    bool shouldContinue = true;
	    p4debug.printf( "List of Cipher Suites supported:\n" );
	    while( shouldContinue )
	    {
		const char *cipherStr = SSL_get_cipher_list( ssl, priority );
		priority++;
		shouldContinue = (cipherStr) ? true : false;
		if( shouldContinue )
		{
		    p4debug.printf( "  Priority %d: %s\n", priority, cipherStr );
		}
	    }
	}

	/* Create a bio for this new socket */
	bio = BIO_new_socket( t, BIO_NOCLOSE );
	SSLNULLHANDLER( bio, e, "NetSslTransport::DoHandshake BIO_new_socket",
                    fail );

	/**
	 * Hooks the BIO up with the SSL *. Note once this is called then the
	 * SSL * owns the BIOs and will cleanup their allocation in an SSL_free
	 * if the reference count becomes zero.
	 */
	SSL_set_bio( ssl, bio, bio );
	SSLLOGFUNCTION( "NetSslTransport::DoHandshake SSL_set_bio" );

	if( !SslHandshake(e) )
	    goto fail;

	if( !isAccepted )
	{

	    X509 *serverCert = SSL_get_peer_certificate( ssl );
	    credentials.SetCertificate( serverCert, e );

	    if ( e->Test() )
	    {
		X509_free( serverCert );
		goto failNoRead;
	    }
	    else
	    {
		SSLLOGFUNCTION( credentials.GetFingerprint()->Text() );
	    }


	    if( SSLDEBUG_TRANS )
	    {
		char *str = NULL;
		// print out CERT info from server

		p4debug.printf( "Server certificate:" );
		str = X509_NAME_oneline( X509_get_subject_name( serverCert ), 0,
				     0 );
		SSLNULLHANDLER( str, e, "connect X509_get_subject_name", fail );
		p4debug.printf( "\t subject: %s\n", str );
		free( str );
		str = X509_NAME_oneline( X509_get_issuer_name( serverCert ), 0,
				     0 );
		SSLNULLHANDLER( str, e, "connect X509_get_issuer_name", fail );
		p4debug.printf( "\t issuer: %s\n", str );
		free( str );

	    }
	    X509_free( serverCert );
	    SSLLOGFUNCTION( "X509_free" );
	}

	return;

	/* Error Handling */
fail:
	lastRead = 1;

failNoRead:
	TRANSPORT_PRINT( SSLDEBUG_ERROR,
		"NetSslTransport::DoHandshake In fail error code." );
	if(ssl)
	{
	    SSL_free(ssl);
	    SSLLOGFUNCTION( "NetSslTransport::DoHandshake SSL_free" );
	    bio = NULL;
	    ssl = NULL;
	}
	// NET_CLOSE_SOCKET(t);
	if( isAccepted )
	{
	    TRANSPORT_PRINT( SSLDEBUG_ERROR,
		    "NetSslTransport::DoHandshake failed on server side.");
	    if( !e->Test() )
		e->Set( MsgRpc::SslAccept ) << "";
	}
	else
	{
	    TRANSPORT_PRINT( SSLDEBUG_ERROR,
		    "NetSslTransport::DoHandshake failed on client side.");
	    if( !e->Test() )
		e->Set( MsgRpc::SslConnect ) << GetPortParser().String() << "";
	}

}

/**
 * NetSslTransport::SslHandshake
 *
 * @brief low level method to invoke ssl handshake and handle asynchronous behavior of
 * the BIO structure.
 *
 * @param error structure
 * @return bool true = success, false = fail
 */
bool
NetSslTransport::SslHandshake( Error *e )
{
	bool      done = false;
	int       readable = isAccepted? 1 : 0;
	int       writable = isAccepted? 0 : 1;
	int       counter = 0;
	/* select timeout */
	const int tv = HALF_SECOND;
	int	  sslClientTimeoutMs
	    = p4tunable.GetLevel("ssl.client.timeout") * 1000;
	DateTimeHighPrecision dtBeforeSelect, dtAfterSelect;
	int maxwait = p4tunable.Get( P4TUNE_NET_MAXWAIT ) * 1000;
	if( maxwait && 
	    ( sslClientTimeoutMs == 0 || maxwait < sslClientTimeoutMs ) )
	    sslClientTimeoutMs = maxwait;

	while ( !done )
	{
	    // Perform an SSL handshake.
	    int ret = 0;
	    int errorRet = 0;
	    if( isAccepted )
		ret = SSL_accept(ssl);
	    else
		ret = SSL_connect( ssl );

	    switch( errorRet = SSL_get_error( ssl, ret ) )
	    {
	    case SSL_ERROR_NONE:
		done = true;
		break;

	    case SSL_ERROR_WANT_READ:
		{
		    readable = 1;
		    writable = 0;
# ifdef WSAEWOULDBLOCK
	            int error = WSAGetLastError();

# else
	            int error = errno;
# endif
	            dtBeforeSelect.Now();
		    int selectRet =
			    selector->Select( readable, writable, tv );
		    dtAfterSelect.Now();
		    counter += MillisecondDifference(dtAfterSelect, dtBeforeSelect);

		    if( selectRet < 0 )
		    {
			e->Sys( "select", "socket" );
			return false;
		    }
		    /*
		     * Changed to prevent a bad guy forming a DOS attack on
		     * Linux by running nc against a p4d.  Previous code
		     * executed a tight loop calling SSL_accept and erroring
		     * out with a WANT_WRITE and a EAGAIN error. We stop this
		     * by sleeping for a millisecond before trying again.
		     * Some OSs return EWOULDBLOCK under this same condition.
		     *
		     * Note POSIX 2008 spec prevents the select call in this
		     * case statement from returning prematurely with EAGAIN,
		     * but Ubuntu seems to follow an earlier version of the
		     * spec. It appears that Mac OS X is complient since this
		     * problem does not exist on that platform.
		     */
# ifdef WSAEWOULDBLOCK
		    if ( WSAEWOULDBLOCK == error)
# else
		    if(( EAGAIN == error ) || ( EWOULDBLOCK == error) )
# endif
		    {

			if( counter > 10 )
			{
			    // Timeout code for new client using SSL going against old server.
			    if(!isAccepted && (counter > sslClientTimeoutMs)) {
				TRANSPORT_PRINTF( SSLDEBUG_ERROR,"NetSslTransport::SslHandshake failed on client side: %d", errorRet);
				e->Set( MsgRpc::SslConnect) << GetPortParser().String();
				Close();
				return false;
			    }
			    msleep(1);
			    counter++;
			}
			else
			{
			    TRANSPORT_PRINT( SSLDEBUG_FUNCTION,
				"NetSslTransport::SslHandshake WANT_READ with EAGAIN or EWOULDBLOCK");
			}
		    }
		}
		break;

	    case SSL_ERROR_WANT_WRITE:
		readable = 0;
		writable = 1;
		if( selector->Select( readable, writable, tv ) < 0 )
		{
		    e->Sys( "select", "socket" );
		    return false;
		}
		TRANSPORT_PRINTF( SSLDEBUG_FUNCTION,"NetSslTransport::SslHandshake WANT_WRITE ret=%d", ret);
		break;
	    case SSL_ERROR_SSL:
		/* underlying protocol error dump error to 
		 * debug output
		 */
		char sslError[256];
		ERR_error_string( ERR_get_error(), sslError );
		TRANSPORT_PRINTF( SSLDEBUG_ERROR, "Handshake Failed: %s", sslError );
		e->Net( "ssl handshake", sslError);
		return false;
		break;
	    default:
		StrBuf errBuf;
		if( Error::IsNetError() )
		{
		    StrBuf tmp;
		    Error::StrNetError( tmp );
		    errBuf.Set( " (" );
		    errBuf.Append( &tmp );
		    errBuf.Append( ")" );
		}
		if( isAccepted )
		{
		    TRANSPORT_PRINTF( SSLDEBUG_ERROR,
			    "NetSslTransport::SslHandshake failed on server side: %d",
			    errorRet );
		    e->Set( MsgRpc::SslAccept) << errBuf;
		}
		else
		{
		    TRANSPORT_PRINTF( SSLDEBUG_ERROR,
		 	"NetSslTransport::SslHandshake failed on client side: %d",
		 	errorRet);
		    e->Set( MsgRpc::SslConnect) << GetPortParser().String() << errBuf;
		}
		return false;
	    } // end switch - last SSL error
	} // end while - keep going until the handshake is done.
	return true;
}


/*
 * NetSslTransport::SendOrReceive() - send or receive data as ready
 *
 * If data to write and no room to read, block writing.
 * If room to read and no data to write, block reading.
 * If both data to write and room to read, block until one is ready.
 *
 * If neither data to write nor room to read, don't call this!
 *
 * Brain-dead version of NetSslSelector::Select() indicates both
 * read/write are ready.  To avoid blocking on read, we only do so
 * if not instructed to write.
 *
 * Returns 1 if either data read or written.
 * Returns 0 if neither, meaning EOF or error.
 */

int
NetSslTransport::SendOrReceive( NetIoPtrs &io, Error *se, Error *re )
{
	// Hacky solution to allow us to send out a cleartext error
	// message to the client if they are attempting a cleartext
	// connection to our SSL server.
	if( clientNotSsl )
	{
	    int retval = NetTcpTransport::SendOrReceive( io, se, re );
	    Close();
	    return retval;
	}
	if( t < 0 )
	{
	    TRANSPORT_PRINT( SSLDEBUG_ERROR,"NetSslTransport::SendOrReceive connection "
		    "closed, returning w/o doing anything." );
	    // Socket has been closed don't try to use
	    return 0;
	}
	StrBuf buf;
	int  doRead = 0;
	int  doWrite = 0;

	int dataReady;
	int maxwait = p4tunable.Get( P4TUNE_NET_MAXWAIT );
	Timer waitTime;
	if( maxwait )
	{
	    maxwait *= 1000;
	    waitTime.Start();
	}

	int  readable = 0;
	int  writable = 0;

	/* flags set by check_availability(  ) that poll for I/O status */
	bool can_read = false;
	bool can_write = false;

	/* flags to mark all the combinations of why we're blocking */
	bool read_waiton_write = false;
	bool read_waiton_read = false;
	bool write_waiton_write = false;
	bool write_waiton_read = false;

	/* select timeout */
	int  tv;

	/* ssl status */
	int  sslError;
	int  sslPending;
	long errErrorNum;
	char errErrorStr[256];

	// Lazy call of handshake code.
	if( !ssl )
	{
	    DoHandshake( se );
	    if( se->Test() )
	    {
		re = se;
		goto end;
	    }
	}

	for ( ;; )
	{
	    doRead = io.recvPtr != io.recvEnd && !re->Test();
	    doWrite = io.sendPtr != io.sendEnd && !se->Test();
	    sslPending = 0;

	    if( !doRead && !doWrite )
	    {
		// cannot read or write
		return 0;
	    }
	    /*
	     * If we enter this loop and are supposed to read AND
	     * there is something already in the SSL read buffers
	     * then don't wait in the select call for something
	     * to appear in the kernel buffers.
	     */
	    sslPending = SSL_pending( ssl );


	    /*
	     * Complex logic to check OS level buffers for
	     * read or write:
	     * For Write:
	     * - if entered this function to write then always check
	     * - if entered to read but handshake is blocking operation
	     *   then write before continue writing
	     * DO NOT check write otherwise since will cause tight loop
	     * burning CPU on interactive commands.
	     *
	     * For Read:
	     * - if entered this function to read then always check
	     * - if entered to write but handshake is blocking operation
	     *   then read before continue writing
	     */
	    readable =
		( doRead || write_waiton_read || read_waiton_read ) ? 1 : 0;
	    writable =
		( doWrite || write_waiton_write || read_waiton_write ) ? 1 : 0;

	    if (readable && sslPending)
		tv = 0;
	    else if( (readable && breakCallback) || maxwait )
		tv = HALF_SECOND;
	    else
		tv = -1;


	    if( ( dataReady = selector->Select( readable, writable, tv ) < 0 ) )
	    {
		re->Sys( "select", "socket" );
		return 0;
	    }

	    if( !dataReady && maxwait && waitTime.Time() >= maxwait )
	    {
	        lastRead = 0;
	        TRANSPORT_PRINT( SSLDEBUG_ERROR, "SSL SendOrReceive maxwait expired." );
	        if( doRead )
	            re->Set( MsgRpc::MaxWait ) << "receive" << ( maxwait / 1000 );
	        else
	            se->Set( MsgRpc::MaxWait ) << "send" << ( maxwait / 1000 );
		return 0;
	    }

	    // Before checking for data do the callback isalive test.
	    // If the user defined IsAlive() call returns a zero
	    // value then the user wants this request to terminate.

	    if( doRead && breakCallback && !breakCallback->IsAlive() )
	    {
		lastRead = 0;
		re->Set( MsgRpc::Break );
		return 0;
	    }

	    if( SSLDEBUG_BUFFER )
	    {
		DateTimeHighPrecision dt;
		char buftime[ DTHighPrecisionBufSize ];

		dt.Now();
		dt.Fmt( buftime );

		p4debug.printf(
		    "State status:	time: %s\n"
		    "\tsslPending         %d - is something in the SSL read buffer?\n"
		    "\treadable           %d - is something in the OS read buffer?\n"
		    "\twritable           %d - is there available room OS write buffer?\n"
		    "\tdoRead             %d - we have room in P4rpc read buffer\n"
		    "\tdoWrite            %d - we have stuff to write in P4rpc write buffer\n"
		    "\twrite_waiton_write %d - ssl write buffer not available, try again when net net write buffer ready\n"
		    "\twrite_waiton_read  %d - ssl write buffer not available due to handshake, try again when net read buffer ready\n"
		    "\tread_waiton_write  %d - ssl read buffer not available due to handshake, try again when net write buffer ready\n"
		    "\tread_waiton_read   %d - ssl read buffer not available, try again when net read buffer ready\n",
		    buftime, sslPending, readable, writable, doRead, doWrite,
		    write_waiton_write, write_waiton_read, read_waiton_write,
		    read_waiton_read );
	    }

	    /* This "if" statement reads data. It will only be entered if
	     * the following conditions are all true:
	     * 1. we're not in the middle of a write
	     * 2. there's space left in the recv buffer
	     * 3. the kernel write buffer is available signaling that a handshake
	     *    write has completed (this write had been blocking our read, and
	     *    now we are free to continue it), or either SSL or OS says we can
	     *    read regardless of whether we're blocking for availability to read.
	     */
	    if( !( write_waiton_read || write_waiton_write ) && doRead
            && ( sslPending || readable || ( writable && read_waiton_write )) )
	    {
		/* clear the flags since we'll set them based on the I/O call's
		 * return
		 */
		read_waiton_read = false;
		read_waiton_write = false;

		/* read into the buffer after the current position */

		int l = SSL_read( ssl, io.recvPtr, io.recvEnd - io.recvPtr );
		SSLLOGFUNCTION( "NetSslTransport::SendOrReceive SSL_read" );

		switch ( sslError = SSL_get_error( ssl, l ) )
		{
		case SSL_ERROR_NONE:
		    /*
		     * no errors occurred.  update the new buffer size and signal
		     * that "have data" by returning 1.
		     */
		    if( l > 0 )
			TRANSPORT_PRINTF( SSLDEBUG_TRANS,
			    "NetSslTransport::SendOrReceive recv %d bytes",
			    l );

		    lastRead = 1;
		    io.recvPtr += l;
		    return 1;

		    break;
		case SSL_ERROR_ZERO_RETURN:
		    /* connection closed */
		    TRANSPORT_PRINT( SSLDEBUG_ERROR, "SSL_read returned SSL_ERROR_ZERO_RETURN" );
		    goto end;
		    break;
		case SSL_ERROR_WANT_READ:
		    /*
		     * we need to retry the read after kernel buffer available for
		     * reading
		     */
		    TRANSPORT_PRINT( SSLDEBUG_ERROR, "SSL_read returned SSL_ERROR_WANT_READ" );
		    read_waiton_read = true;
		    break;
		case SSL_ERROR_WANT_WRITE:
		    /*
		     * we need to retry the read after kernel buffer is available for
		     * writing
		     */
		    TRANSPORT_PRINT( SSLDEBUG_ERROR, "SSL_read returned SSL_ERROR_WANT_WRITE" );
		    read_waiton_write = true;
		    break;
		case SSL_ERROR_SYSCALL:
		    /*
		     * an I/O error occurred check underlying SSL ERR for info
		     * if none then report errno
		     */
		    errErrorNum = ERR_get_error();
	            if ( errErrorNum == 0 )
	            {
	        	if( l == 0)
	        	{
	        	    TRANSPORT_PRINT( SSLDEBUG_ERROR, "SSL_read encountered an EOF." );
	        	    re->Net( "read", "SSL_read encountered an EOF." );
	        	}
	        	else if ( l < 0 )
	        	{
	        	    Error::StrError(buf, errno);
	        	    TRANSPORT_PRINTF( SSLDEBUG_ERROR, "SSL_read encountered a system error: %s", buf.Text() );
	        	    re->Net( "read", buf.Text() );
	        	}
	        	else
	        	{
	        	    TRANSPORT_PRINT( SSLDEBUG_FUNCTION,
	        		    "SSL_read claims SSL_ERROR_SYSCALL but returns data." );
	        	    if( l > 0 )
	        		TRANSPORT_PRINTF( SSLDEBUG_TRANS,
	        			"NetSslTransport::SendOrReceive recv %d bytes\n",
	        			l );
	        	    lastRead = 1;
	        	    io.recvPtr += l;
	        	    return 1;
	        	}
	            }
	            else
	            {
	        	ERR_error_string( errErrorNum, errErrorStr );
	        	TRANSPORT_PRINTF( SSLDEBUG_ERROR,
	        		"SSL_read encountered a syscall ERR: %s", errErrorStr );
	        	re->Net( "read", errErrorStr );
	            }
	            re->Set( MsgRpc::SslRecv );
	            goto end;
	            break;
		default:
		    if( l == 0 )
		    {
			TRANSPORT_PRINT( SSLDEBUG_FUNCTION,
			    "SSL_read attempted on closed connection." );
			if( doWrite )
			{
			    // Error: connection was closed but we had stuff to write
			    re->Net( "read", "socket" );
			    re->Set( MsgRpc::SslRecv );
			}
			/*
			 * Connection closed but did not shutdown SSL cleanly.
			 * The p4 proxy will do this if the command issued by
			 * its client does not require forwarding to the p4d server.
			 */
			goto end;
		    }

		    /* ERROR */
		    TRANSPORT_PRINTF( SSLDEBUG_ERROR,
			    "SSL_read returned unknown error: %d",
                                  sslError );
		    re->Net( "read", "socket" );
		    re->Set( MsgRpc::SslRecv );
		    goto end;
		}
	    }

	    /* This "if" statement writes data. It will only be entered if
	     * the following conditions are all true:
	     * 1. we're not in the middle of a read
	     * 2. there's room in the buffer
	     * 3. the kernel read buffer is available signaling that a handshake
	     *    read has completed (this rad had been blocking our write, and
	     *    now we are free to continue it), or OS says we can write regardless
	     *    of whether we're blocking for availability to write.
	     */
	    if( !( read_waiton_write || read_waiton_read ) && doWrite
            && ( writable || ( readable && write_waiton_read )) )
	    {
		/* clear the flags */
		write_waiton_read = false;
		write_waiton_write = false;

		/* perform the write from the start of the buffer */
		int l = SSL_write( ssl, io.sendPtr, io.sendEnd - io.sendPtr );
		SSLLOGFUNCTION( "NetSslTransport::SendOrReceive SSL_write" );

		switch ( sslError = SSL_get_error( ssl, l ) )
		{
		case SSL_ERROR_NONE:
		    /*
		     * no errors occurred.  update the new buffer size and signal
		     * that "have data" by returning 1.
		     */
		    if( l > 0 )
			TRANSPORT_PRINTF( SSLDEBUG_TRANS,
				"NetSslTransport send %d bytes\n", l );
		    lastRead = 0;
		    io.sendPtr += l;
		    return 1;

		    break;
		case SSL_ERROR_ZERO_RETURN:
		    /* connection closed */
		    TRANSPORT_PRINT( SSLDEBUG_ERROR,
			    "SSL_write returned SSL_ERROR_ZERO_RETURN" );
		    goto end;
		case SSL_ERROR_WANT_READ:
		    /*
		     * we need to retry the write after A is available for
		     * reading
		     */
		    TRANSPORT_PRINT( SSLDEBUG_ERROR,
			    "SSL_write returned SSL_ERROR_WANT_READ" );
		    write_waiton_read = true;
		    break;
		case SSL_ERROR_WANT_WRITE:
		    /*
		     * we need to retry the write after A is available for
		     * writing
		     */
		    TRANSPORT_PRINT( SSLDEBUG_ERROR,
			    "SSL_write returned SSL_ERROR_WANT_WRITE" );
		    write_waiton_write = true;
		    break;
		case SSL_ERROR_SYSCALL:
		    /*
		     * an I/O error occurred check underlying SSL ERR for info
		     * if none then report errno
		     */
		    errErrorNum = ERR_get_error();
	            if ( errErrorNum == 0 )
	            {
	        	if( l == 0)
	        	{
	        	    TRANSPORT_PRINT( SSLDEBUG_ERROR, "SSL_write encountered an EOF." );
	        	    se->Net( "write", "SSL_write encountered an EOF." );
	        	}
	        	else if ( l < 0 )
	        	{
	        	    Error::StrError(buf, errno);
	        	    TRANSPORT_PRINTF( SSLDEBUG_ERROR,
	        		    "SSL_write encountered a system error: %s", buf.Text() );
	        	    se->Net( "write", buf.Text() );
	        	}
	        	else
	        	{
	        	    TRANSPORT_PRINT( SSLDEBUG_ERROR,
	        		    "SSL_write claims SSL_ERROR_SYSCALL but returns data." );
	        	    if( l > 0 )
	        		TRANSPORT_PRINTF( SSLDEBUG_TRANS, "NetSslTransport send %d bytes", l );
	        	    lastRead = 0;
	        	    io.sendPtr += l;
	        	    return 1;
	        	}
	            }
	            else
	            {
	        	ERR_error_string( errErrorNum, errErrorStr );
	        	TRANSPORT_PRINTF( SSLDEBUG_ERROR,
	        		"SSL_write encountered a syscall ERR: %s", errErrorStr );
	        	se->Net( "write", errErrorStr );
	            }
	            se->Set( MsgRpc::SslSend );
	            goto end;
		    break;
		default:
		    if( l == 0 )
		    {
			/*
			 * Connection closed but did not shutdown SSL cleanly.
			 * The p4 proxy will do this if the command issued by
			 * its client does not require forwarding to the p4d server.
			 */
			TRANSPORT_PRINT( SSLDEBUG_FUNCTION,
			    "SSL_write attempted on closed connection." );
			goto end;
		    }

		    /* ERROR */
		    TRANSPORT_PRINTF( SSLDEBUG_ERROR,
			    "SSL_write returned unknown error: %d",
                                  sslError );
		    se->Net( "write", "socket" );
		    se->Set( MsgRpc::SslSend );
		    goto end;
		}
	    }
	}

end:
	Close();
	return 0;
}


/**
 * NetSslTransport::Close
 *
 * @brief handles teardown of the SSL connection and deletion of the
 * of the SSL and BIO structures, closes socket
 * @invariant SSL and BIO are NULL if the connection is closed
 *
 * @return none
 */
void
NetSslTransport::Close( void )
{
	int ret;
	if( t < 0 )
	    return;

	TRANSPORT_PRINTF( SSLDEBUG_CONNECT, "NetSslTransport %s closing %s",
                           GetAddress( RAF_PORT )->Text(),
                           GetPeerAddress( RAF_PORT )->Text() );

	// Avoid TIME_WAIT on the server by reading the EOF after
	// the last message sent by the client.  Getting the EOF
	// means we've received the TH_FIN packet, which means we
	// don't have to send our own on close().  He who sends
	// a TH_FIN goes into the 2 minute TIME_WAIT imposed by TCP.

	TRANSPORT_PRINTF( SSLDEBUG_TRANS, "NetSslTransport lastRead=%d", lastRead );

	if( lastRead )
	{
	    int  r = 1;
	    int  w = 0;
	    char buf[1];

	    if( selector->Select( r, w, -1 ) >= 0 && r )
		read( t, buf, 1 );
	}

	if (ssl)
	{
	    if( SSL_get_shutdown( ssl ) & SSL_RECEIVED_SHUTDOWN )
	    {
		// clean shutdown
		SSL_shutdown( ssl );
		SSLLOGFUNCTION( "NetSslTransport::Close SSL_shutdown" );
	    }
	    else
	    {
		/*
		 * An error is causing this shutdown, so we remove session
		 * from cache.
		 */
		SSL_clear( ssl );
		SSLLOGFUNCTION( "NetSslTransport::Close SSL_clear" );
	    }
	    BIO_pop( bio );
	    SSLLOGFUNCTION( "NetSslTransport::Close BIO_pop" );
	    SSL_free( ssl );
	    SSLLOGFUNCTION( "NetSslTransport::Close SSL_free" );
	}

	bio = NULL;
	ssl = NULL;

	if( lastRead )
	{
	    int  r = 1;
	    int  w = 0;
	    char buf[1];

	    if( selector->Select( r, w, -1 ) >= 0 && r )
		read( t, buf, 1 );
	}

	NET_CLOSE_SOCKET(t);
}


void
NetSslTransport::ClientMismatch( Error *e )
{
	if ( CheckForHandshake(t) == PeekCleartext )
	{
	    TRANSPORT_PRINT( SSLDEBUG_ERROR,
		    "Handshake peek appears not to be for SSL.");
	    // this is a non-ssl connection
	    e->Set( MsgRpc::SslCleartext );
	    clientNotSsl = true;
	}
}

void
NetSslTransport::ValidateRuntimeVsCompiletimeSSLVersion( Error *e )
{
	StrBuf sb;
	GetVersionString( sb, SSLeay() );
	TRANSPORT_PRINTF( SSLDEBUG_ERROR,
		"OpenSSL runtime version %s", sb.Text() );
	sb.Clear();
	GetVersionString( sb, OPENSSL_VERSION_NUMBER );
	TRANSPORT_PRINTF( SSLDEBUG_ERROR,
		"OpenSSL compile version %s", sb.Text() );

	if ( GET_MJR_SSL_VERSION(SSLeay())
		< GET_MJR_SSL_VERSION( OPENSSL_VERSION_NUMBER ))
	{
	    e->Set(MsgRpc::SslLibMismatch) << sb;
	}
}

void
NetSslTransport::GetVersionString( StrBuf &sb, unsigned long version )
{
	/* Version numbering explanation from OpenSSL openssl/opensslv.h file:
	 *
	 * Numeric release version identifier:
	 * MNNFFPPS: major minor fix patch status
	 * The status nibble has one of the values 0 for development, 1 to e for betas
	 * 1 to 14, and f for release.  The patch level is exactly that.
	 * For example:
	 * 0.9.3-dev          0x00903000
	 * 0.9.3-beta1        0x00903001
	 * 0.9.3-beta2-dev    0x00903002
	 * 0.9.3-beta2        0x00903002 (same as ...beta2-dev)
	 * 0.9.3              0x0090300f
	 * 0.9.3a             0x0090301f
	 * 0.9.4              0x0090400f
	 * 1.2.3z             0x102031af
	 */
	unsigned long M = (version >> 28) & ~0xfffffff0L;
	unsigned long N = (version >> 20) & ~0xffffff00L;
	unsigned long P = (version >> 12) & ~0xffffff00L;
	sb << M << "." << N << "." << P;
}

////////////////////////////////////////////////////////////////////////////
//  OpenSSL Dynamic Locking Callback Functions                            //
////////////////////////////////////////////////////////////////////////////
// Dynamic locking code only being used on NT in 2012.1, will be used on
// other platforms in 2012.2 (by that time I will include pthreads to the
// HPUX build). In 2012.1 HPUX has many compile errors for pthreads.
#ifndef OS_HPUX

static void LockingFunction( int mode, int n, const char *file, int line )
{
# ifdef OS_NT
    if (mode & CRYPTO_LOCK)
    {
	WaitForSingleObject( mutexArray[n], INFINITE );
    }
    else
    {
	ReleaseMutex( mutexArray[n] );
    }
# else
    if( mode & CRYPTO_LOCK )
    {
	pthread_mutex_lock( &mutexArray[n] );
    }
    else
    {
	pthread_mutex_unlock( &mutexArray[n] );
    }
# endif // OS_NT
}

/**
 * OpenSSL uniq id function.
 *
 * @return    thread id
 */
static unsigned long IdFunction( void )
{
# ifdef OS_NT
    return ((unsigned long) GetCurrentThreadId());
# else
    return ((unsigned long) pthread_self());
# endif
}

/**
 * OpenSSL allocate and initialize dynamic crypto lock.
 *
 * @param    file    source file name
 * @param    line    source file line number
 */
static struct CRYPTO_dynlock_value *
DynCreateFunction( const char *file, int line )
{
    struct CRYPTO_dynlock_value *value;

    value = (struct CRYPTO_dynlock_value *) malloc(
	    sizeof(struct CRYPTO_dynlock_value) );
    if( !value )
    {
	goto err;
    }
# ifdef OS_NT
    value->mutex = CreateMutex( NULL, FALSE, NULL );
# else
    pthread_mutex_init( &value->mutex, NULL );
# endif // OS_NT
    return value;

    err: return (NULL);
}

/**
 * OpenSSL dynamic locking function.
 *
 * @param    mode    lock mode
 * @param    l        lock structure pointer
 * @param    file    source file name
 * @param    line    source file line number
 * @return    none
 */
static void DynLockFunction(
        int mode,
        struct CRYPTO_dynlock_value *l,
        const char *file,
        int line )
{
# ifdef OS_NT
    if (mode & CRYPTO_LOCK)
    {
	WaitForSingleObject( l->mutex, INFINITE );
    }
    else
    {
	ReleaseMutex( l->mutex );
    }
# else
    if( mode & CRYPTO_LOCK )
    {
	pthread_mutex_lock( &l->mutex );
    }
    else
    {
	pthread_mutex_unlock( &l->mutex );
    }
# endif // OS_NT
}

/**
 * OpenSSL destroy dynamic crypto lock.
 *
 * @param    l        lock structure pointer
 * @param    file    source file name
 * @param    line    source file line number
 * @return    none
 */

static void DynDestroyFunction(
        struct CRYPTO_dynlock_value *l,
        const char *file,
        int line )
{
# ifdef OS_NT
    CloseHandle(l->mutex);
# else
    pthread_mutex_destroy( &l->mutex );
# endif // OS_NT
    free( l );
}

static int InitLockCallbacks( Error *e )
{
	int i;
	int numlocks = CRYPTO_num_locks();

	/* static locks area */
# ifdef OS_NT
	mutexArray = (HANDLE *) malloc( CRYPTO_num_locks() * sizeof(HANDLE) );
# else
	mutexArray = (pthread_mutex_t *) malloc( CRYPTO_num_locks() * sizeof(pthread_mutex_t) );
# endif // OS_NT

	if( mutexArray == NULL )
	{
	    e->Set(MsgRpc::Operat) << "malloc";
	    return -1;
	}

	for ( i = 0; i < numlocks; i++ )
	{
# ifdef OS_NT
	    mutexArray[i] = CreateMutex( NULL, FALSE, NULL );
# else
	    pthread_mutex_init( &mutexArray[i], NULL );
# endif // OS_NT
	}
	/* static locks callbacks */
	CRYPTO_set_locking_callback( LockingFunction );
	CRYPTO_set_id_callback( IdFunction );
	/* dynamic locks callbacks */
	CRYPTO_set_dynlock_create_callback( DynCreateFunction );
	CRYPTO_set_dynlock_lock_callback( DynLockFunction );
	CRYPTO_set_dynlock_destroy_callback( DynDestroyFunction );

	return (0);
}

/**
 * Cleanup TLS library.
 *
 * @return    0
 */
static int ShudownLockCallbacks( void )
{
    int i;
    int numlocks = CRYPTO_num_locks();

    if( mutexArray == NULL )
    {
	return (0);
    }

    CRYPTO_set_dynlock_create_callback( NULL );
    CRYPTO_set_dynlock_lock_callback( NULL );
    CRYPTO_set_dynlock_destroy_callback( NULL );

    CRYPTO_set_locking_callback( NULL );
    CRYPTO_set_id_callback( NULL );

    for ( i = 0; i < numlocks; i++ )
    {
# ifdef OS_NT
	CloseHandle( mutexArray[i] );
# else
	pthread_mutex_destroy( &mutexArray[i] );
# endif // OS_NT
    }

    return (0);
}

# endif // !OS_HPUX

# endif //USE_SSL