P4Transfer.py #9

#!/usr/bin/env python
# -*- coding: utf-8 -*-

#
# Copyright (c) 2011-2014 Sven Erik Knop/Robert Cowham, Perforce Software Ltd
#
# ========================================
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# 1.  Redistributions of source code must retain the above copyright
#     notice, this list of conditions and the following disclaimer.
#
# 2.  Redistributions in binary form must reproduce the above copyright
#     notice, this list of conditions and the following disclaimer in the
#     documentation and/or other materials provided with the
#     distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL PERFORCE
# SOFTWARE, INC. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# User contributed content on the Perforce Public Depot is not supported by Perforce,
# although it may be supported by its author. This applies to all contributions
# even those submitted by Perforce employees.
# ========================================
# P4Transfer.py
#
# This python script will transfer Perforce changelists with all contents
# between independent servers when no remote depots are possible.
#
# This script transfers changes in one direction - from a source server to a target server.
# There is an alternative called PerforceExchange.py which transfers changes
# in both directions.
#
# Usage:
#   python2 P4Transfer.py [options]
#
# The -h/--help option describes all options.
#
# The script requires a config file, normally called transfer.cfg,
# that provides the Perforce connection information for both servers.
# The config file has three sections: [general], [source] and [target].
# See DEFAULT_CONFIG for details. An initial example can be generated, e.g.
#
#   P4Transfer.py --sample_config > transfer.cfg
#
# The script also needs a directory in which it can place the mapped files.
# This directory has to be the root of both servers' workspaces (this will be verified).

from __future__ import print_function

VERSION = """$Id: //guest/perforce_software/p4transfer/P4Transfer.py#9 $"""

import sys, re
from collections import OrderedDict
import P4
import pprint

def logrepr(self):
    return pprint.pformat(self.__dict__, width=240)

# Log messages just once per run
alreadyLogged = {}
def logOnce(logger, *args):
    global alreadyLogged
    msg = ", ".join([str(x) for x in args])
    if not msg in alreadyLogged:
        alreadyLogged[msg] = 1
        logger.debug(msg)

P4.Revision.__repr__ = logrepr
P4.Integration.__repr__ = logrepr
P4.DepotFile.__repr__ = logrepr

python3 = sys.version_info[0] >= 3

# Although this should work with Python 3, it doesn't currently handle Windows Perforce servers
# with filenames containing charaters such as umlauts etc: åäö
if python3:
    from configparser import ConfigParser
else:
    from ConfigParser import ConfigParser

import argparse
import os.path
from datetime import datetime
import logging
import time

import logutils

CONFIG = 'transfer.cfg'
GENERAL_SECTION = 'general'
SOURCE_SECTION = 'source'
TARGET_SECTION = 'target'
LOGGER_NAME = "P4Transfer"

# This is for writing to sample config file - OrderedDict used to preserve order of lines.
DEFAULT_CONFIG = OrderedDict({
    GENERAL_SECTION: OrderedDict([
        ("# counter_name: Unique counter on target server to use for recording source changes processed. No spaces.", None),
        ("#    Name sensibly if you have multiple instances transferring into the same target p4 repository.", None),
        ("#    The counter value represents the last transferred change number - script will start from next change.", None),
        ("#    If not set, or 0 then transfer will start from first change.", None),
        ("counter_name", "p4transfer_counter"),
        ("# instance_name: Name of the instance of P4Transfer - for emails etc. Spaces allowed.", None),
        ("instance_name", "Perforce Transfer from XYZ"),
        ("# For notification - testing purposes only", None),
        ("mail_form_url", ""),
        ("# The mail_* parameters must all be valid (non-blank) to receive email updates during processing. ", None),
        ("# mail_to: One or more valid email addresses - comma separated for multiple values", None),
        ("#     E.g. [email protected],[email protected]", None),
        ("mail_to", ""),
        ("# mail_from: Email address of sender of emails, E.g. [email protected]", None),
        ("mail_from", ""),
        ("# mail_server: The SMTP server to connect to for email sending, E.g. smtpserver.example.com", None),
        ("mail_server", ""),
        ("# sleep_on_error_interval: How long (in minutes) to sleep when error is encountered in the script", None),
        ("sleep_on_error_interval", "60"),
        ("# poll_interval: How long (in minutes) to wait between polling source server for new changes", None),
        ("poll_interval", "60"),
        ("# The following *_interval values result in reports, but only if mail_* values are specified", None),
        ("# report_interval: Interval (in minutes) between regular update emails being sent", None),
        ("report_interval", "30"),
        ("# error_report_interval: Interval (in minutes) between error emails being sent e.g. connection error", None),
        ("#     Usually some value less than report_interval. Useful if transfer being run with --repeat option.", None),
        ("error_report_interval", "15"),
        ("# summary_report_interval: Interval (in minutes) between summary emails being sent e.g. changes processed", None),
        ("#     Typically some value such as 1 week (10080 = 7 * 24 *60). Useful if transfer being run with --repeat option.", None),
        ("summary_report_interval", "10080"),
        ("# sync_progress_size_interval: Size in bytes controlling when syncs are reported to log file. ", None),
        ("#    Useful for keeping an eye on progress for large syncs over slow links. ", None),
        ("sync_progress_size_interval", "500000000")]),
    SOURCE_SECTION: OrderedDict([
        ("# P4PORT to connect to, e.g. some-server:1666", None),
        ("p4port", ""),
        ("# P4USER to use", None),
        ("p4user", ""),
        ("# P4CLIENT to use, e.g. p4-transfer-client", None),
        ("p4client", ""),
        ("# P4PASSWD for the user - valid password. If blank then no login performed.", None),
        ("p4passwd", "")]),
    TARGET_SECTION: OrderedDict([
        ("# P4PORT to connect to, e.g. some-server:1666", None),
        ("p4port", ""),
        ("# P4USER to use", None),
        ("p4user", ""),
        ("# P4CLIENT to use, e.g. p4-transfer-client", None),
        ("p4client", ""),
        ("# P4PASSWD for the user - valid password. If blank then no login performed.", None),
        ("p4passwd", "")]),
    })

def p4time(unixtime):
    "Convert time to Perforce format time"
    return time.strftime("%Y/%m/%d:%H:%M:%S", time.localtime(unixtime))

def printSampleConfig():
    "Print defaults from above dictionary for saving as a base file"
    config = ConfigParser(allow_no_value=True)
    for sec in DEFAULT_CONFIG.keys():
        config.add_section(sec)
        for k in DEFAULT_CONFIG[sec].keys():
            config.set(sec, k, DEFAULT_CONFIG[sec][k])
    print("")
    print("# Save this output to a file to e.g. transfer.cfg and edit it for your configuration")
    print("")
    config.write(sys.stdout)

def fmtsize(num):
    for x in ['bytes','KB','MB','GB','TB']:
        if num < 1024.0:
            return "%3.1f %s" % (num, x)
        num /= 1024.0

class ChangeRevision:
    "Represents a change - created from P4API supplied information and thus encoding"

    def __init__(self, r, a, t, d):
        self.rev = r
        self.action = a
        self.type = t
        self.depotFile = d
        self.localFile = None

    def setIntegrationInfo(self, integ):
        self.integration = integ

    def setLocalFile(self, localFile):
        self.localFile = localFile
        localFile = localFile.replace("%40", "@")
        localFile = localFile.replace("%23", "#")
        localFile = localFile.replace("%2A", "*")
        localFile = localFile.replace("%25", "%")
        localFile = localFile.replace("/", os.sep)
        self.fixedLocalFile = localFile

    def __repr__(self):
        return 'rev = {rev} action = {action} type = {type} depotFile = {depotfile}' .format(
            rev=self.rev,
            action=self.action,
            type=self.type,
            depotfile=self.depotFile,
        )

class ReportProgress(object):
    "Report overall progress"

    def __init__(self, p4, changes, logger):
        self.logger = logger
        self.filesToSync = 0
        self.changesToSync = len(changes)
        self.sizeToSync = 0
        self.filesSynced = 0
        self.changesSynced = 0
        self.sizeSynced = 0
        self.previousSizeSynced = 0
        self.sync_progress_size_interval = None     # Set to integer value to get reports
        for chg in changes:
            sizes = p4.run('sizes', '-s', '@%s,%s' % (chg['change'], chg['change']))
            self.sizeToSync += int(sizes[0]['fileSize'])
            self.filesToSync += int(sizes[0]['fileCount'])
        self.logger.info("Syncing %d changes, files %d, size %s" % (self.changesToSync,
                        self.filesToSync, fmtsize(self.sizeToSync)))

    def SetSyncProgressSizeInterval(self, interval):
        "Set appropriate"
        if interval:
            self.sync_progress_size_interval = int(interval)

    def ReportChangeSync(self):
        self.changesSynced += 1

    def ReportFileSync(self, fileSize):
        self.filesSynced += 1
        self.sizeSynced += fileSize
        if not self.sync_progress_size_interval:
            return
        if self.sizeSynced > self.previousSizeSynced + self.sync_progress_size_interval:
            self.previousSizeSynced = self.sizeSynced
            syncPercent = 100 * float(self.filesSynced) / float(self.filesToSync)
            sizePercent = 100 * float(self.sizeSynced) / float(self.sizeToSync)
            self.logger.info("Synced %d/%d changes, files %d/%d (%2.1f %%), size %s/%s (%2.1f %%)" % ( \
                    self.changesSynced, self.changesToSync,
                    self.filesSynced, self.filesToSync, syncPercent,
                    fmtsize(self.sizeSynced), fmtsize(self.sizeToSync),
                    sizePercent))

class P4Base(object):
    "Processes a config"

    section = None
    P4PORT = None
    P4CLIENT = None
    P4USER = None
    P4PASSWD = None
    counter = 0
    clientLogged = 0

    def __init__(self, section, options, p4id):
        self.section = section
        self.options = options
        self.logger = logging.getLogger(LOGGER_NAME)
        self.p4id = p4id
        self.p4 = None
        self.client_logged = 0

    def __str__(self):
        return '[section = {} P4PORT = {} P4CLIENT = {} P4USER = {} P4PASSWD = {}]'.format( \
            self.section,
            self.P4PORT,
            self.P4CLIENT,
            self.P4USER,
            self.P4PASSWD,
            )

    def connect(self, progname):
        self.p4 = P4.P4()
        self.p4.port = self.P4PORT
        self.p4.client = self.P4CLIENT
        self.p4.user = self.P4USER
        self.p4.prog = progname
        self.p4.exception_level = P4.P4.RAISE_ERROR
        self.p4.connect()
        if not self.P4PASSWD == None:
            self.p4.password = self.P4PASSWD
            self.p4.run_login()

        clientspec = self.p4.fetch_client(self.p4.client)
        logOnce(self.logger, "%s:%s:%s" % (self.p4id, self.p4.client, pprint.pformat(clientspec)))
        self.root = clientspec._root
        self.p4.cwd = self.root
        self.clientmap = P4.Map(clientspec._view)

        ctr = P4.Map('//"'+clientspec._client+'/..."   "' + clientspec._root + '/..."')
        self.localmap = P4.Map.join(self.clientmap, ctr)
        self.depotmap = self.localmap.reverse()

    def p4cmd(self, *args):
        "Execute p4 cmd while logging arguments and results"
        self.logger.debug(self.p4id, args)
        output = self.p4.run(args)
        self.logger.debug(self.p4id, output)
        return output

    def disconnect(self):
        if self.p4:
            self.p4.disconnect()

    def checkWarnings(self, where):
        if self.p4 and self.p4.warnings:
            self.logger.warning('warning in {} : {}'.format(where, str(self.p4.warnings)))

    def resetWorkspace(self):
        self.p4cmd('sync', '...#none')

class TrackedAdd(object):
    """Record class used in MoveTracker"""
    def __init__(self, chRev, deleteDepotFile):
        self.chRev = chRev
        self.deleteDepotFile = deleteDepotFile

class MoveTracker(object):
    """Tracks move/add and move/delete and handles orphans where 
    source or target isn't mapped (either outside source workspace or not
    visible due to permissions)."""
        
    def __init__(self, logger):
        self.logger = logger
        self.adds = {}       # Key is localFile for matching delete
        self.deletes = {}    # Key is localFile for delete
        
    def trackAdd(self, chRev, localFile, deleteDepotFile, deleteLocalFile):
        "Remember the move/add or move/delete so we can match them up"
        self.adds[deleteLocalFile] = TrackedAdd(chRev, deleteDepotFile)
    
    def trackDelete(self, chRev, localFile):
        "Track a move/delete"
        self.deletes[localFile] = chRev
        
    def getMoves(self):
        "Return orphaned moves, or the move/add from add/delete pairs"
        for localFile in self.adds:
            if localFile in self.deletes:
                self.logger.debug("Matched move add/delete '%s'" % localFile)
                del self.deletes[localFile]
            else:
                self.logger.debug("Action move/add changed to add")
                self.adds[localFile].chRev.action = "add"
        result = [self.adds[k].chRev for k in self.adds]
        for k in self.deletes:
            self.logger.debug("Action move/delete changed to delete")
            self.deletes[k].action = 'delete'
        result.extend([self.deletes[k] for k in self.deletes])
        return result

class P4Source(P4Base):
    "Functionality for reading from source Perforce repository"

    def __init__(self, section, options):
        super(P4Source, self).__init__(section, options, 'src')

    def missingChanges(self, counter):
        changes = self.p4cmd('changes', '-l', '...@{rev},#head'.format(rev=counter + 1))
        changes.reverse()
        if self.options.maximum:
            changes = changes[:self.options.maximum]
        return changes

    def getChange(self, change):
        """Expects change number as a string"""

        class SyncOutput(P4.OutputHandler):
            "Log sync progress"

            def __init__(self, progress):
                P4.OutputHandler.__init__(self)
                self.progress = progress

            def outputStat(self, stat):
                if 'fileSize' in stat:
                    self.progress.ReportFileSync(int(stat['fileSize']))
                return P4.OutputHandler.HANDLED

            def outputInfo(self, info):
                return P4.OutputHandler.HANDLED

            def outputMessage(self, msg):
                return P4.OutputHandler.HANDLED

        self.progress.ReportChangeSync()
        callback = SyncOutput(self.progress)
        self.p4.run('sync', '...@{},{}'.format(change, change), handler=callback)
        change = self.p4cmd('describe', change)[0]
        filerevs = []
        movetracker = MoveTracker(self.logger)
        for (n, rev) in enumerate(change['rev']):
            localFile = self.localmap.translate(change['depotFile'][n])
            if localFile and len(localFile) > 0:
                chRev = ChangeRevision(rev, change['action'][n], change['type'][n], change['depotFile'][n])
                chRev.setLocalFile(localFile)
                if chRev.action in ('branch', 'integrate', 'add', 'delete', 'move/add'):
                    filelog = self.p4.run_filelog('-m1', '{}#{}'.format(chRev.depotFile, chRev.rev))
                    if filelog:
                        self.logger.debug('filelog', filelog)
                        depotFile = filelog[0]
                        revision = depotFile.revisions[0]
                        if len(revision.integrations) > 0:
                            for integ in revision.integrations:
                                if 'from' in integ.how or integ.how == "ignored":
                                    chRev.setIntegrationInfo(integ)
                                    integ.localFile = self.localmap.translate(integ.file)
                                    break
                        if chRev.action == 'move/add':
                            self.logger.debug('integration', chRev.integration)
                            movetracker.trackAdd(chRev, localFile, depotFile.depotFile, chRev.integration.localFile)
                        else:
                            filerevs.append(chRev)
                    else:
                        self.logger.error(u"Failed to retrieve filelog for {}#{}".format(chRev.depotFile, 
                                            chRev.rev))
                elif chRev.action == 'move/delete':
                    movetracker.trackDelete(chRev, localFile)
                else:
                    filerevs.append(chRev)
        filerevs.extend(movetracker.getMoves())
        return filerevs

class P4Target(P4Base):
    "Functionality for transferring changes to target Perforce repository"
    
    def __init__(self, section, options, src):
        super(P4Target, self).__init__(section, options, 'targ')
        self.src = src
        self.re_cant_integ = None

    def replicateChange(self, filerevs, change, sourcePort):
        """This is the heart of it all. Replicate all changes according to their description"""

        for f in filerevs:
            self.logger.debug('targ:', f)

            if not self.options.preview:
                if f.action == 'edit':
                    self.logger.debug('processing:0010 edit')
                    self.p4cmd('sync', '-k', f.localFile)
                    self.p4cmd('edit', '-t', f.type, f.localFile)
                    self.checkWarnings('edit')
                elif f.action == 'add':
                    if 'integration' in f.__dict__:
                        self.replicateBranch(f, dirty=True)
                    else:
                        self.logger.debug('processing:0020 add')
                        self.p4cmd('add', '-ft', f.type, f.fixedLocalFile)
                        self.checkWarnings('add')
                elif f.action == 'delete':
                    if 'integration' in f.__dict__:
                        self.replicateIntegration(f)
                        self.checkWarnings('integrate (delete)')
                    else:
                        self.logger.debug('processing:0030 delete')
                        self.p4cmd('delete', '-v', f.localFile)
                        self.checkWarnings('delete')
                elif f.action == 'purge':
                    # special case. Type of file is +S, and source.sync removed the file
                    # create a temporary file, it will be overwritten again later
                    self.logger.debug('processing:0040 purge')
                    dummy = open(f.localFile, 'w')
                    dummy.write('purged file')
                    dummy.close()
                    self.p4cmd('sync', '-k', f.localFile)
                    self.p4cmd('edit', '-t', f.type, f.localFile)
                    if self.p4.warnings:
                        self.p4cmd('add', '-tf', f.type, f.fixedLocalFile)
                        self.checkWarnings('purge -add')
                elif f.action == 'branch':
                    self.replicateBranch(f, dirty=False)
                    self.checkWarnings('branch')
                elif f.action == 'integrate':
                    self.replicateIntegration(f)
                    self.checkWarnings('integrate')
                elif f.action == 'move/add':
                    self.moveAdd(f)
                else:
                    raise Exception('Unknown action: %s for %s' % (
                                f.action, str(f)))
        newChangeId = None

        opened = self.p4cmd('opened')
        if len(opened) > 0:
            description = change['desc'] \
                + '''

Transferred from p4://%s@%s''' % (sourcePort,
                    change['change'])

            if self.options.nokeywords:
                self.removeKeywords(opened)

            result = self.p4cmd('submit', '-d', description)

            # the submit information can be followed by refreshFile lines
            # need to go backwards to find submittedChange

            a = -1
            while 'submittedChange' not in result[a]:
                a -= 1
            newChangeId = result[a]['submittedChange']
            self.updateChange(change, newChangeId)
            self.reverifyRevisions(result)

        self.logger.info("source = {} : target = {}".format(change['change'], newChangeId))
        return newChangeId
        
    def moveAdd(self, file):
        "Either paired with a move/delete or an orphaned move/add"
        self.logger.debug('processing:0100 move/add')
        if file.integration and file.integration.localFile:
            source = file.integration.localFile
            self.p4cmd('sync', '-f', source)
            self.p4cmd('edit', source)
            if os.path.exists(file.localFile):
                self.p4cmd('move', '-k', source, file.localFile)
            else:
                self.p4cmd('move', source, file.localFile)
        else:
            self.p4cmd('add', '-ft', file.type, file.fixedLocalFile)
            self.checkWarnings('add')
            
    def updateChange(self, change, newChangeId):
        # need to update the user and time stamp
        newChange = self.p4.fetch_change(newChangeId)
        newChange._user = change['user']
        # date in change is in epoch time, we need it in canonical form
        newDate = datetime.utcfromtimestamp(int(change['time'])).strftime("%Y/%m/%d %H:%M:%S")
        newChange._date = newDate
        self.p4.save_change(newChange, '-f')

    def reverifyRevisions(self, result):
        revisionsToVerify = ["{file}#{rev},{rev}".format(file=x['refreshFile'], rev=x['refreshRev'])
                                for x in result if 'refreshFile' in x]
        if revisionsToVerify:
            self.p4cmd('verify', '-qv', revisionsToVerify)

    def removeKeywords(self, opened):
        for openFile in opened:
            if self.hasKeyword(openFile["type"]):
                fileType = self.removeKeyword(openFile["type"])

                self.p4.run_reopen('-t', fileType, openFile["depotFile"])

                self.logger.debug("targ: Changed type from {} to {} for {}".
                                format(openFile["type"], fileType, openFile["depotFile"]))

    KTEXT = re.compile("(.*)\+([^k]*)k([^k]*)")

    def hasKeyword(self, fileType):
        return(fileType in ["ktext", "kxtext"] or self.KTEXT.match(fileType))

    def removeKeyword(self, fileType):
        if fileType == "ktext":
            newType = "text"
        elif fileType == "kxtext":
            newType = "xtext"
        else:
            m = self.KTEXT.match(fileType)
            newType = m.group(1) + "+" + m.group(2) + m.group(3)

        return newType

    def replicateBranch(self, file, dirty=False):
        # An integration where source has been obliterated will not have integrations
        if not self.re_cant_integ:
            self.re_cant_integ = re.compile("can't integrate from .* or use -Di to disregard move")
        self.logger.debug('replicateBranch')
        has_integration = False
        try:
            i = file.integration
            has_integration = True
        except:
            pass
        if has_integration and not self.options.ignore and file.integration.localFile:
            if file.integration.how == 'add from':
                self.logger.debug('processing:0200 add from')
                self.p4cmd('add', '-ft', file.type, file.fixedLocalFile)
            else:
                self.logger.debug('processing:0210 integrate')
                output = self.p4cmd('integrate', '-v', file.integration.localFile, file.localFile)
                if self.re_cant_integ.search(str(output)):
                    output = self.p4cmd('integrate', '-v', '-Di', file.integration.localFile, file.localFile)
                if dirty:
                    self.p4cmd('edit', file.localFile)
        else:
            self.logger.debug('processing:0220 add')
            self.p4cmd('add', '-ft', file.type, file.fixedLocalFile)

    def replicateIntegration(self, file):
        self.logger.debug('replicateIntegration')
        if not self.options.ignore and file.integration.localFile:
            if file.integration.how == 'edit from':
                self.logger.debug('processing:0300 edit from')
                with open(file.localFile) as f:
                    content = f.read()
                self.p4cmd('sync', '-f', file.localFile)    # to avoid tamper checking
                self.p4cmd('integrate', file.integration.localFile, file.localFile)

                class MyResolver(P4.Resolver):
                    "Local resolver to accept edits on merge"
                    def __init__(self, content):
                        self.content = content

                    def resolve(self, mergeData):
                        with open(mergeData.result_path, 'w') as f:
                            f.write(self.content)
                        return 'ae'

                self.p4.run_resolve(resolver=MyResolver(content))
            else:
                self.logger.debug('processing:0310 integrate')
                self.p4cmd('sync', '-f', file.localFile)    # to avoid tamper checking
                self.p4cmd('integrate', file.integration.localFile, file.localFile)
                if file.integration.how == 'copy from':
                    self.logger.debug('processing:0320 copy from')
                    self.p4cmd('resolve', '-at')
                elif file.integration.how == 'ignored':
                    self.logger.debug('processing:0330 ignored')
                    self.p4cmd('resolve', '-ay')
                elif file.integration.how in ('delete', 'delete from'):
                    self.logger.debug('processing:0340 delete')
                    self.p4cmd('resolve', '-at')
                elif file.integration.how == 'merge from':
                    self.logger.debug('processing:0350 merge from')
                    # self.p4cmd('edit(file.localFile) # to overcome tamper check
                    self.p4cmd('resolve', '-am')
                else:
                    self.logger.error('Cannot deal with {}'.format(file.integration))
        else:
            self.logger.debug('processing:0360 ignore integrations')
            if file.integration.how in ('delete', 'delete from'):
                self.logger.debug('processing:0370 delete')
                self.p4cmd('delete', '-v', file.localFile)
            else:
                self.logger.debug('processing:0380 else')
                self.p4cmd('sync', '-k', file.localFile)
                self.p4cmd('edit', file.localFile)

    def getCounter(self):
        "Returns value of counter as integer"
        result = self.p4cmd('counter', self.options.counter_name)
        if result and 'counter' in result[0]:
            return int(result[0]['value'])
        return 0

    def setCounter(self, value):
        "Set's the counter to specified value"
        self.p4cmd('counter', self.options.counter_name, str(value))

class P4Transfer(object):
    "Main transfer class"

    def __init__(self, *args):
        parser = argparse.ArgumentParser(
            description="P4Transfer",
            epilog="Copyright (C) 2012-14 Sven Erik Knop/Robert Cowham, Perforce Software Ltd"
        )

        parser.add_argument('-n', '--preview', action='store_true', help="Preview only, no transfer")
        parser.add_argument('-c', '--config', default=CONFIG, help="Default is " + CONFIG)
        parser.add_argument('-m', '--maximum', default=None, type=int, help="Maximum number of changes to transfer")
        parser.add_argument('-k', '--nokeywords', action='store_true', help="Do not expand keywords and remove +k from filetype")
        parser.add_argument('-p', '--preflight', action='store_true', help="Run a sanity check first to ensure target is empty")
        parser.add_argument('-r', '--repeat', action='store_true', help="Repeat transfer in a loop - for continuous transfer")
        parser.add_argument('-s', '--stoponerror', action='store_true', help="Stop on any error even if --repeat has been specified")
        parser.add_argument('--sample_config', action='store_true', help="Print an example config file and exit")
        parser.add_argument('-i', '--ignore', action='store_true', help="Treat integrations as adds and edits")

        self.options = parser.parse_args(list(args))
        self.options.sync_progress_size_interval = None

        self.logger = logutils.getLogger(LOGGER_NAME)
        self.previous_target_change_counter = 0 # Current value

    def getOption(self, section, option_name, default=None):
        result = default
        try:
            result = self.parser.get(section, option_name)
        except:
            pass
        return result

    def getIntOption(self, section, option_name, default=None):
        result = default
        strval = self.getOption(section, option_name, default)
        if strval:
            try:
                result = int(eval(strval))
            except:
                pass
        return result

    def readConfig(self):
        self.parser = ConfigParser()
        self.options.parser = self.parser    # for later use
        try:
            with open(self.options.config) as f:
                if python3:
                    self.parser.read_file(f)
                else:
                    self.parser.readfp(f)
        except Exception as e:
            raise Exception('Could not read %s: %s' % (self.options.config, str(e)))

        self.options.counter_name = self.getOption(GENERAL_SECTION, "counter_name")
        if not self.options.counter_name:
            raise Exception("Option counter_name in the [general] section must be specified")
        self.options.instance_name = self.getOption(GENERAL_SECTION, "instance_name", self.options.counter_name)
        self.options.mail_form_url = self.getOption(GENERAL_SECTION, "mail_form_url")
        self.options.mail_to = self.getOption(GENERAL_SECTION, "mail_to")
        self.options.mail_from = self.getOption(GENERAL_SECTION, "mail_from")
        self.options.mail_server = self.getOption(GENERAL_SECTION, "mail_server")
        self.options.sleep_on_error_interval = self.getIntOption(GENERAL_SECTION, "sleep_on_error_interval", 60)
        self.options.poll_interval = self.getIntOption(GENERAL_SECTION, "poll_interval", 60)
        self.options.report_interval = self.getIntOption(GENERAL_SECTION, "report_interval", 30)
        self.options.error_report_interval = self.getIntOption(GENERAL_SECTION, "error_report_interval", 30)
        self.options.summary_report_interval = self.getIntOption(GENERAL_SECTION, "summary_report_interval", 10080)
        self.options.sync_progress_size_interval = self.getIntOption(GENERAL_SECTION,
                    "sync_progress_size_interval")

        self.source = P4Source(SOURCE_SECTION, self.options)
        self.target = P4Target(TARGET_SECTION, self.options, self.source)

        self.readSection(self.source)
        self.readSection(self.target)

    def readSection(self, p4config):
        if self.parser.has_section(p4config.section):
            self.readOptions(p4config)
        else:
            raise Exception('Config file needs section %s' % p4config.section)

    def readOptions(self, p4config):
        self.readOption('P4CLIENT', p4config)
        self.readOption('P4USER', p4config)
        self.readOption('P4PORT', p4config)
        self.readOption('P4PASSWD', p4config, optional=True)

    def readOption(self, option, p4config, optional=False):
        if self.parser.has_option(p4config.section, option):
            p4config.__dict__[option] = self.parser.get(p4config.section, option)
        elif not optional:
            raise Exception('Required option %s not found in section %s' % (option, p4config.section))

    def replicate_changes(self):
        "Perform a replication loop"
        self.source.connect('source replicate')
        self.target.connect('target replicate')
        changes = self.source.missingChanges(self.target.getCounter())
        self.logger.info("Transferring %d changes" % len(changes))
        if len(changes) > 0:
            self.save_previous_target_change_counter()
            self.source.progress = ReportProgress(self.source.p4, changes, self.logger)
            self.source.progress.SetSyncProgressSizeInterval(self.options.sync_progress_size_interval)
            for change in changes:
                msg = 'Processing change: {} "{}"'.format(change['change'], change['desc'].strip())
                self.logger.info(msg)
                filerevs = self.source.getChange(change['change'])
                self.target.replicateChange(filerevs, change, self.source.p4.port)
                self.target.setCounter(change['change'])
                # Tidy up the workspaces after successful transfer
                self.source.p4cmd("sync", "#0")
                self.target.p4cmd("sync", "#0")
        self.source.disconnect()
        self.target.disconnect()
        return len(changes)

    def log_exception(self, e):
        "Log exceptions appropriately"
        etext = str(e)
        if re.search("WSAETIMEDOUT", etext, re.MULTILINE) or re.search("WSAECONNREFUSED", etext, re.MULTILINE):
            self.logger.error(etext)
        else:
            self.logger.exception(e)

    def save_previous_target_change_counter(self):
        "Save the latest change transferred to the target"
        chg = self.target.p4cmd('changes', '-m1', '-ssubmitted', '//{client}/...'.format(client=self.target.P4CLIENT))
        if chg:
            self.previous_target_change_counter = int(chg[0]['change']) + 1

    def send_summary_email(self, time_last_summary_sent, change_last_summary_sent):
        "Send an email summarising changes transferred"
        time_str = p4time(time_last_summary_sent)
        self.target.connect('target replicate')
        # Combine changes reported by time or since last changelist transferred
        changes = self.target.p4cmd('changes', '-l', '//{client}/...@{rev},#head'.format(
                client=self.target.P4CLIENT, rev=time_str))
        chgnums = [chg['change'] for chg in changes]
        counter_changes = self.target.p4cmd('changes', '-l', '//{client}/...@{rev},#head'.format(
                client=self.target.P4CLIENT, rev=change_last_summary_sent))
        for chg in counter_changes:
            if chg['change'] not in chgnums:
                changes.append(chg)
        changes.reverse()
        lines = []
        lines.append(["Date", "Time", "Changelist", "File Revisions", "Size (bytes)", "Size"])
        total_changes = 0
        total_rev_count = 0
        total_file_sizes = 0
        for chg in changes:
            sizes = self.target.p4cmd('sizes', '-s', '@%s,%s' % (chg['change'], chg['change']))
            lines.append([time.strftime("%Y/%m/%d", time.localtime(int(chg['time']))),
                         time.strftime("%H:%M:%S", time.localtime(int(chg['time']))),
                         chg['change'], sizes[0]['fileCount'], sizes[0]['fileSize'],
                         fmtsize(int(sizes[0]['fileSize']))])
            total_changes += 1
            total_rev_count += int(sizes[0]['fileCount'])
            total_file_sizes += int(sizes[0]['fileSize'])
        lines.append([])
        lines.append(['Totals', '', str(total_changes), str(total_rev_count), str(total_file_sizes), fmtsize(total_file_sizes)])
        report = "Changes transferred since %s\n%s" % (time_str,
                "\n".join(["\t".join(line) for line in lines]))
        self.logger.debug("Transfer summary report:\n%s" % report)
        self.logger.info("Sending Transfer summary report")
        self.logger.notify("Transfer summary report", report, include_output=False)
        self.save_previous_target_change_counter()
        self.target.disconnect()

    #
    # This is the central method
    # It provides the replication process
    # Algorithm:
    #     Read the config file
    #     Connect to server1 and server
    #     Determine if counter is there
    def replicate(self):
        """Central method that performs the replication between server1 and server2"""
        if self.options.sample_config:
            printSampleConfig()
            return 0
        try:
            logOnce(self.logger, VERSION)
            logOnce(self.logger, "Reading config file")
            self.readConfig()
            self.source.connect('source replicate')
            self.target.connect('target replicate')
            self.logger.debug("connected to source and target")
            if not self.source.root == self.target.root:
                raise Exception('server1 and server2 workspace root directories must be the same')
        except Exception as e:
            self.log_exception(e)
            logging.shutdown()
            return 1
        # self.source.resetWorkspace()

        if self.options.preflight:
            print("Running pre-flight check first ...")
            targetFiles = self.target.p4cmd('fstat', '-T clientFile', '...')
            sourceFiles = self.source.p4cmd('fstat', '-T clientFile', '...')

            for f in targetFiles:
                if f in sourceFiles:
                    depotFile = self.target.p4cmd('fstat', f['clientFile'])[0]
                    raise Exception("Failed pre-flight check, file '{}' in source and target".format(depotFile['depotFile']))
            print("Finished pre-flight check ...")

        time_last_summary_sent = time.time()
        change_last_summary_sent = 0
        self.logger.debug("Time last summary sent: %s" % p4time(time_last_summary_sent))
        time_last_error_occurred = 0
        error_encountered = False # Flag to indicate error encountered which may require reporting
        error_notified = False
        finished = False
        while not finished:
            try:
                self.readConfig()       # Read every time to allow user to change them
                self.logger.setReportingOptions(instance_name=self.options.instance_name,
                                mail_form_url=self.options.mail_form_url, mail_to=self.options.mail_to,
                                mail_from=self.options.mail_from, mail_server=self.options.mail_server,
                                report_interval=self.options.report_interval)
                logOnce(self.logger, self.source.options)
                logOnce(self.logger, self.target.options)
                self.source.disconnect()
                self.target.disconnect()
                num_changes = self.replicate_changes()
                if num_changes > 0:
                    self.logger.info("Transferred %d changes successfully" % num_changes)
                if change_last_summary_sent == 0:
                    change_last_summary_sent = self.previous_target_change_counter
                if not self.options.repeat:
                    finished = True
                else:
                    if error_encountered:
                        self.logger.info("Logging - reset error interval")
                        self.logger.notify("Cleared error", "Previous error has now been cleared")
                        error_encountered = False
                        error_notified = False
                    if time.time() - time_last_summary_sent > self.options.summary_report_interval * 60:
                        time_last_summary_sent = time.time()
                        self.send_summary_email(time_last_summary_sent, change_last_summary_sent)
                    self.logger.info("Sleeping for %d minutes" % self.options.poll_interval)
                    time.sleep(self.options.poll_interval * 60)
            except Exception as e:
                self.log_exception(e)
                if self.options.stoponerror:
                    self.logger.notify("Error", "Exception encountered and --stoponerror specified")
                    logging.shutdown()
                    return 1
                else:
                    # Decide whether to report an error
                    if not error_encountered:
                        error_encountered = True
                        time_last_error_occurred = time.time()
                    elif not error_notified:
                        if time.time() - time_last_error_occurred > self.options.error_report_interval * 60:
                            error_notified = True
                            self.logger.info("Logging - Notifying recurring error")
                            self.logger.notify("Recurring error", "Multiple errors seen")
                    self.logger.info("Sleeping on error for %d minutes" % self.options.sleep_on_error_interval)
                    time.sleep(self.options.sleep_on_error_interval * 60)
        self.logger.notify("Changes transferred", "Completed successfully")
        logging.shutdown()
        return 0

if __name__ == '__main__':
    result = 0
    try:
        prog = P4Transfer(*sys.argv[1:])
        result = prog.replicate()
    except Exception as e:
        print(str(e))
        result = 1
    sys.exit(result)