# # Python module for utilities for remote computer communication. # import os, sys, Setup from cutil import log, DEBUG, VERBOSE, INFO, WARN, ERR, FATAL,\ normalPopen, read_write def scp(srcFile, destFile, scpCmd = Setup.SERVER.SCP, remoteUser = Setup.BACKUP.user, remoteHost = Setup.BACKUP.host): ''' Run SCP on the given files, with the scrFile on the local machine and destFile on the remote machine. ''' log(INFO, ("Running [", scpCmd, "] [", srcFile, "] [", remoteUser, "@", remoteHost, ":", destFile, "]")) conn = normalPopen( (scpCmd, srcFile, "%s@%s:%s" % (remoteUser, remoteHost, destFile) )) out = [] while conn.poll() == -1: lines = read_write(conn.fromchild, None, 0, out) for x in lines: log(VERBOSE, ("scp: ", x)) # cmd returned ret = conn.poll() # read in the remainder of the input lines = read_write(conn.fromchild, None, 0, out) for x in lines: log(VERBOSE, ("scp: ", x)) log(VERBOSE, ("Stopped scp: return code = ", ret)) # what to do with the output? return ret def rsync(srcBaseDir, destBaseDir, relFiles, rsyncCmd = Setup.SERVER.RSYNC, remoteUser = Setup.BACKUP.user, remoteHost = Setup.BACKUP.host, sshCmd = Setup.SERVER.SSH): ''' Run RSYNC on the given files over ssh, with the scrFiles on the local machine and destFiles (must be a 1-for-1 match) on the remote machine. ''' cmd = ( rsyncCmd, '-vcrpt', '--rsh=' + sshCmd, '--delete', '--force', '--files-from=-', srcBaseDir, "%s@%s:%s" % (remoteUser, remoteHost, destBaseDir) ) log(INFO, ("Running ", cmd)) conn = normalPopen(cmd) out = [] # send the list of files we want to sync for x in relFiles: log(VERBOSE, (" - Sending ", x)) conn.tochild.write(x + "\n") conn.tochild.close() # read output from command while conn.poll() == -1: lines = read_write(conn.fromchild, None, None, 0, out) for x in lines: log(VERBOSE, ("rsync: ", x)) # cmd returned ret = conn.poll() # read in the remainder of the input lines = read_write(conn.fromchild, None, None, 0, out) for x in lines: log(VERBOSE, ("rsync: ", x)) log(VERBOSE, ("Stopped rsync: return code = ", ret)) # what to do with the output? return ret class SSH_CMD: ''' Executes a command to a remote computer over SSH. Keeps the command open for reads/writes until either the server disconnects us (the command terminates), or we close the stream connections. ''' def __init__(self, cmdArgs, sshCmd = Setup.SERVER.SSH, remoteUser = Setup.BACKUP.user, remoteHost = Setup.BACKUP.host): ''' Creates a new SSH connection, which, by default, will keep the connection alive and use the remote setup options. ''' self._connection = None self.sshCmd = sshCmd self.cmdArgs = cmdArgs self.host = remoteHost self.user = remoteUser self.stopCmd = None cmd = [sshCmd, "%s@%s" % (self.user, self.host)] for x in cmdArgs: cmd.append(x) self.cmd = cmd def __del__(self): self.disconnect() def connect(self): ''' Attempt to connect to the remote server. It will return the output from the connection attempt (a list of lines returned by the server). If the user is required to enter a password, or if the connection attempt fails, then the connection will be severed, and calls to isAlive() will return False. ''' output = [] if not self.isAlive(): if self._connection != None: self.disconnect() log(INFO, ("Starting SSH connection using [", self.cmd, "] [", self.user, "@", self.host, "]")) self._connection = normalPopen(self.cmd) output = self.readWrite(None, output) # XXXXXXXXXXXXXXXXX # TODO: check if it requests authentication. If so, we should fail. return output def disconnect(self): ''' Attempts to disconnect from the server. If, after a time-out period, the command hasn't stopped, this will attempt to kill the local SSH process. ''' out = [] if self.isAlive(): if self.stopCmd != None and len(self.stopCmd) > 0: self.readWrite(self.stopCmd, None) self._connection.tochild.close() self._connection.tochild = None # We cannot close the fromchild pipe here, because the process # might still have pending data to pass to us. We must continue # to read from this stream until we know the process is complete. import os, time waitCount = 0 self.readWrite(None, out) self._connection.fromchild.close() self._connection.fromchild = None while self._connection.poll() == -1 and waitCount < 3: time.sleep(1) waitCount += 1 if self._connection.poll() == -1: # kill it using the OS, since it didn't die after ~3 seconds log(INFO, "ssh: sending OS kill signal to process") os.kill(self._connection.pid, 9) self._connection.wait() self._connection = None log(INFO, "ssh: Disconnected from server") return out def isAlive(self): ''' Returns true or false, determining if the connection to the server is still alive. ''' if self._connection == None: return False if self._connection.poll() == -1: return True # otherwise, the connection has terminated return False def readWrite(self, outputLines, inputList = None, retryCount = 3): ''' Read and write lines to the server. ''' self._checkAlive() if inputList is None: inputList = [] #log(VERBOSE, ("ssh <- (", len(outputLines), " # lines) [2]")) retries = 0 while retries < retryCount: retries = retries + 1 try: # Potential bug: each attempt re-writes everything # in the lines to the server read_write(self._connection.fromchild, self._connection.tochild, outputLines, 0, inputList) if self._connection.tochild: self._connection.tochild.flush() log(DEBUG, ("ssh: - finished sending lines to server")) return inputList except OSError, err: # check for broken pipe situations if err.errno == 32: # retry the operation. We should be okay with # only resending these requested lines, since the last # set of lines did a flush successfully, meaning that # those lines don't need to be resent. log(WARN, ("SSH connection encountered broken pipe. ", "Retrying operation.")) self.disconnect() self.connect() else: log(ERR, ("Encountered unknown IO error number ", err.errno, " (", err, ")")) raise RemoteConnectionException("SSH", err) except IOError, err: # check for broken pipe situations if err.errno == 32: # retry the operation. We should be okay with # only resending these requested lines, since the last # set of lines did a flush successfully, meaning that # those lines don't need to be resent. log(WARN, ("SSH connection encountered broken pipe. ", "Retrying operation.")) self.disconnect() self.connect() else: log(ERR, ("Encountered unknown IO error number ", err.errno, " (", err, ")")) raise RemoteConnectionException("SSH", err) # ran out of retry attempts. Throw the error. log(ERR, ("SSH encountered broken pipe and ran out of ", "retries.")) raise RemoteConnectionException("SSH", "Too many broken pipe retries. Underlying IOError " + err) def _checkAlive(self): for x in range(1,3): if self.isAlive(): break log(DEBUG, "Connection to server was broken. Retrying.") self.connect() if not self.isAlive(): raise RemoteConnectionException("SSH", "No connection to remote server") class RemoteConnectionException(Exception): ''' General exception for failures in remote connections. ''' def __init__(self, connectionType, value): self.connectionType = connectionType self.value = value def __str__(self): return repr(self.connectionType + ": " + self.value)