# Perforce source for convert extension.
#
# Copyright 2009, Frank Kingswood <frank@kingswood-consulting.co.uk>
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
#
# Jan 24, 2011 sknop@perforce.com: adding P4 as destination (p4_sink)
# $Id: //depot/dev/sknop/mercurial/hgext/convert/p4.py#46 $
from mercurial import util, commands, hg
from mercurial.i18n import _
from common import commit, converter_source, converter_sink, checktool, NoRepo
import marshal
import re, os, os.path
from datetime import datetime
import P4
def loaditer(f):
"Yield the dictionary objects generated by p4"
try:
while True:
d = marshal.load(f)
if not d:
break
yield d
except EOFError:
pass
class p4_source(converter_source):
def __init__(self, ui, path, rev=None):
super(p4_source, self).__init__(ui, path, rev=rev)
if "/" in path and not path.startswith('//'):
raise NoRepo(_('%s does not look like a P4 repository') % path)
checktool('p4', abort=False)
self.p4changes = {}
self.heads = {}
self.changeset = {}
self.files = {}
self.tags = {}
self.lastbranch = {}
self.parent = {}
self.encoding = "latin_1"
self.depotname = {} # mapping from local name to depot name
self.re_type = re.compile(
"([a-z]+)?(text|binary|symlink|apple|resource|unicode|utf\d+)"
"(\+\w+)?$")
self.re_keywords = re.compile(
r"\$(Id|Header|Date|DateTime|Change|File|Revision|Author)"
r":[^$\n]*\$")
self.re_keywords_old = re.compile("\$(Id|Header):[^$\n]*\$")
self._parse(ui, path)
def _parse_view(self, path):
"Read changes affecting the path"
cmd = 'p4 -G changes -s submitted %s' % util.shellquote(path)
stdout = util.popen(cmd, mode='rb')
for d in loaditer(stdout):
c = d.get("change", None)
if c:
self.p4changes[c] = True
def _parse(self, ui, path):
"Prepare list of P4 filenames and revisions to import"
ui.status(_('reading p4 views\n'))
# read client spec or view
if "/" in path:
self._parse_view(path)
if path.startswith("//") and path.endswith("/..."):
views = {path[:-3]:""}
else:
views = {"//": ""}
else:
cmd = 'p4 -G client -o %s' % util.shellquote(path)
clientspec = marshal.load(util.popen(cmd, mode='rb'))
views = {}
for client in clientspec:
if client.startswith("View"):
sview, cview = clientspec[client].split()
self._parse_view(sview)
if sview.endswith("...") and cview.endswith("..."):
sview = sview[:-3]
cview = cview[:-3]
cview = cview[2:]
cview = cview[cview.find("/") + 1:]
views[sview] = cview
# list of changes that affect our source files
self.p4changes = self.p4changes.keys()
self.p4changes.sort(key=int)
# list with depot pathnames, longest first
vieworder = views.keys()
vieworder.sort(key=len, reverse=True)
# handle revision limiting
startrev = self.ui.config('convert', 'p4.startrev', default=0)
self.p4changes = [x for x in self.p4changes
if ((not startrev or int(x) >= int(startrev)) and
(not self.rev or int(x) <= int(self.rev)))]
# now read the full changelists to get the list of file revisions
ui.status(_('collecting p4 changelists\n'))
lastid = None
for change in self.p4changes:
cmd = "p4 -G describe -s %s" % change
stdout = util.popen(cmd, mode='rb')
d = marshal.load(stdout)
desc = self.recode(d["desc"])
shortdesc = desc.split("\n", 1)[0]
t = '%s %s' % (d["change"], repr(shortdesc)[1:-1])
ui.status(util.ellipsis(t, 80) + '\n')
if lastid:
parents = [lastid]
else:
parents = []
date = (int(d["time"]), 0) # timezone not set
c = commit(author=self.recode(d["user"]), date=util.datestr(date),
parents=parents, desc=desc, branch='',
extra={"p4": change})
files = []
i = 0
while ("depotFile%d" % i) in d and ("rev%d" % i) in d:
oldname = d["depotFile%d" % i]
filename = None
for v in vieworder:
if oldname.startswith(v):
filename = views[v] + oldname[len(v):]
break
if filename:
files.append((filename, d["rev%d" % i]))
self.depotname[filename] = oldname
i += 1
self.changeset[change] = c
self.files[change] = files
lastid = change
if lastid:
self.heads = [lastid]
def getheads(self):
return self.heads
def getfile(self, name, rev):
cmd = 'p4 -G print %s' \
% util.shellquote("%s#%s" % (self.depotname[name], rev))
stdout = util.popen(cmd, mode='rb')
mode = None
contents = ""
keywords = None
for d in loaditer(stdout):
code = d["code"]
data = d.get("data")
if code == "error":
raise IOError(d["generic"], data)
elif code == "stat":
p4type = self.re_type.match(d["type"])
if p4type:
mode = ""
flags = (p4type.group(1) or "") + (p4type.group(3) or "")
if "x" in flags:
mode = "x"
if p4type.group(2) == "symlink":
mode = "l"
if "ko" in flags:
keywords = self.re_keywords_old
elif "k" in flags:
keywords = self.re_keywords
elif code == "text" or code == "binary":
contents += data
if mode is None:
raise IOError(0, "bad stat")
if keywords:
contents = keywords.sub("$\\1$", contents)
if mode == "l" and contents.endswith("\n"):
contents = contents[:-1]
return contents, mode
def getchanges(self, rev):
return self.files[rev], {}
def getcommit(self, rev):
return self.changeset[rev]
def gettags(self):
return self.tags
def getchangedfiles(self, rev, i):
return sorted([x[0] for x in self.files[rev]])
class p4_sink(converter_sink):
def __init__(self, ui, path):
converter_sink.__init__(self, ui, path)
self.p4 = P4.P4()
self.repo = hg.repository(ui, path.replace('-hg', ''))
#ui.pushbuffer()
#commands.log(self.ui, self.repo, date=None, rev=None, user=None)
#print(ui.popbuffer())
def getheads(self):
print( "Called getheads()")
return []
def revmapfile(self):
print( "Called revmapfile()")
return "revmapfile"
def authorfile(self):
print( "Called authorfile()")
return "authorfile"
def join(self, branch, file):
return os.path.join(self.root, branch, file)
def fixFilename( self, name ):
result = name.replace('%', '%25')
result = result.replace('@', '%40')
result = result.replace('#', '%23')
result = result.replace('*', '%2A')
return result
def fixBranchname( self, name ):
result = name.replace('%', '_PERCENT_')
result = result.replace('@', '_AT_')
result = result.replace('#', '_HASH_')
result = result.replace('*', '_STAR_')
return result
def putfile( self, file, data, mode ):
# find the directories first
targetPath,basename = os.path.split(file)
if not os.path.isdir( targetPath ):
os.makedirs( targetPath )
with open(file, 'w') as f:
f.write(data)
# ignore mode for now
def findMergedFiles(self, search, sortkey):
self.ui.pushbuffer()
commands.diff(self.ui, self.repo, rev=[str(search), str(sortkey)], change=None, stat=True, reverse=None)
result = self.ui.popbuffer()
print(result)
resultLines = result.split('\n')[:-2] # cut off the last two lines containing summary and a blank line
files = [ line.split('|')[0].strip() for line in resultLines ]
return files
def putcommit(self, files, copies, parents, commit, source, revmap):
print( "Called putcommit() with")
print( "*** files", [x[0] for x in files] )
print( "*** copies", copies )
print( "*** parents", parents )
print( "*** commit.author", commit.author )
print( "*** commit.date", commit.date )
print( "*** commit.desc", commit.desc )
print( "*** commit.parents", commit.parents )
print( "*** commit.branch", commit.branch )
# print( "*** commit.rev", commit.rev )
print( "*** commit.extra", commit.extra )
print( "*** commit.sortkey", commit.sortkey )
edited = []
added = []
deleted = []
copied = []
copySource = []
renamed = []
copyData = {}
change = ''
revs = set()
date = commit.date[:-6] # cut the timezone off, Python2.X cannot deal with it yet
timestamp = datetime.strptime(date, '%a %b %d %H:%M:%S %Y').strftime('%Y/%m/%d %H:%M:%S')
if commit.author in self.users:
self.p4.user = commit.author
self.p4.password = self.users[commit.author]
else:
self.p4.user = self.superuser
self.p4.password = self.superticket
r = self.p4.run_login('-p', commit.author)
self.p4.user = commit.author # users are mapped via authorfile to legit Perforce users
self.p4.password = r[0]
self.users[self.p4.user] = self.p4.password
# now create branches if necessary
cbranch = self.fixBranchname(commit.branch)
branch = self.fixBranchname(self.next_branch)
pbranches = [ (x[0], self.fixBranchname(x[1])) for x in self.next_pbranches ]
if not pbranches:
pass
elif len(pbranches) == 1:
change, parent = pbranches[0]
if parent != branch:
print("creating branch %s from %s @ %s" % (branch, parent, change))
with self.p4.at_exception_level(1):
if change and change != '0':
self.p4.run_integrate('%s/...@%s' % (parent, change), '%s/...' % branch)
else:
self.p4.run_integrate('%s/...' % parent, '%s/...' % branch)
print( "after integrate; warnings : ", self.p4.warnings )
# might need a resolve here, just in case the target exists
self.p4.run_resolve('-at')
change = self.p4.fetch_change()
if 'Files' in change:
change._description = 'Created branch %s from %s @ %s' % (branch, parent, change)
submitted = self.p4.run_submit(change)
# remember the change if this is just a branch with no edited changes.
# Otherwise the next submit will override this number
for s in reversed( submitted ):
if 'submittedChange' in s:
change = s['submittedChange'] # the last entry in the array contains the change number
break
else:
print("Nothing to integrate. Maybe a rebase?")
else:
# in some cases Mercurial does not tell us all the files that have changed
# if the change originates from a push or pull between repos
# find the missing files here
searchRev = commit.parents[0]
mergedFiles = self.findMergedFiles(searchRev, commit.sortkey)
officiallyEditedFiles = [ x[0] for x in files ]
for m in mergedFiles:
if m not in officiallyEditedFiles:
files.append( (m, str(commit.sortkey)) )
print files
elif len(pbranches) == 2:
targetChange, tgt = pbranches[0]
sourceChange, src = pbranches[1]
if tgt == branch and src == branch:
# merge between two repositories in the same branch
mergedFiles = []
for search in commit.parents:
mergedFiles += self.findMergedFiles(search, commit.sortkey)
# add all the merged files to the list of files
officiallyEditedFiles = [ x[0] for x in files ]
for m in mergedFiles:
if m not in officiallyEditedFiles:
files.append( (m, str(commit.sortkey)) )
print files
elif tgt == branch:
print("merging to %s@%s with %s@%s" % (tgt, targetChange, src, sourceChange) )
if sourceChange:
with self.p4.at_exception_level(1):
for f,v in files:
copied = self.p4.run_copy('%s/%s@%s' % (src, f, sourceChange), '%s/%s' % (tgt, f) )
else:
with self.p4.at_exception_level(1):
for f,v in files:
copied = self.p4.run_copy('%s/%s' % (src, f), '%s/%s' % (tgt,f) )
elif src == branch:
print("merging from %s@%s with %s@%s" % (tgt, targetChange, src, sourceChange))
if targetChange:
with self.p4.at_exception_level(1):
for f,v in files:
self.p4.run_copy('%s/%s@%s' % (tgt, f, targetChange), '%s/%s' % (src,f) )
else:
with self.p4.at_exception_level(1):
for f,v in files:
self.p4.run_copy('%s/%s' % (tgt,f), '%s/%s' % (src,f) )
else:
print("Serious trouble in setbranch: parent branches do not match target branch")
else:
print( "Serious problem in setbranches: more than 2 parent branches ???")
executables = []
for f, v in files:
target = self.join(cbranch, f)
fixedTarget = self.fixFilename(target)
revs.add(v)
try:
data, mode = source.getfile(f, v)
if 'x' in mode:
executables.append(fixedTarget)
except IOError:
deleted.append(fixedTarget)
else:
print("* %s mode: %s" % (f, mode))
if f in copies:
print("*** file %s is a copy from %s" % (f, copies[f]))
s = self.fixFilename( self.join(cbranch, copies[f]) )
copied.append( (fixedTarget, s) )
copySource.append( (s, fixedTarget) )
copyData[fixedTarget] = (data, mode)
else:
with self.p4.at_exception_level(1):
fileinfo = self.p4.run_fstat(fixedTarget) # warning if file does not exist
print( fileinfo, self.p4.warnings, self.p4.errors )
if (len(self.p4.warnings) > 0) \
or ('headAction' in fileinfo[0] and 'delete' in fileinfo[0]['headAction'] ) \
or ('action' in fileinfo[0] and 'branch' in fileinfo[0]['action'] ):
added.append(target)
print("*a %s" % target)
else:
self.p4.run_sync(fixedTarget) # ignore warning that file already synced
edited.append(fixedTarget)
print("*e %s" % target)
self.putfile(target, data, mode)
# time to do some processing
print( "Total added ", len(added))
print( "Total edited ", len(edited))
if len(added) > 0 :
self.p4.run_add('-f', added)
if len(edited) > 0 :
self.p4.run_edit(edited)
# if a file was the source of a copy and deleted, it was a rename
trueCopies = []
for i in copySource:
if i[0] in deleted:
renamed.append( i )
deleted.remove(i[0])
else:
trueCopies.append( i )
# now process renames, copies and deletes
if len(deleted) > 0 :
with self.p4.at_exception_level(1):
self.p4.run_delete('-v', deleted)
print("warnings after delete: ", self.p4.warnings)
with self.p4.at_exception_level(1):
for i in renamed:
print("moving %s to %s" % (i[0], i[1]))
self.p4.run_sync('-f', i[0], i[1]) # in case we are renaming into an existing target
fileinfo = self.p4.run_fstat(fixedTarget) # warning if file does not exist
if len(self.p4.warnings) > 0:
# target does not exist yet ==> move it
self.p4.run_edit(i[0])
self.p4.run_move('-k', i[0], i[1]) # file content is already in workspace
print("warnings after move: ", self.p4.warnings)
else:
# downgrade the move to an integrate and delete, it might have been a
# forced rename in Mercurial
self.p4.run_integrate(i[0], i[1])
self.p4.run_resolve('-at', i[1])
self.p4.run_edit(i[1])
self.p4.run_delete(i[0])
(data, mode) = copyData[i[1]]
self.putfile(i[1], data, mode) # need to overwrite the file, in case it is a dirty edit
for i in trueCopies:
print(">>> Trying to copy %s to %s" % (i[0], i[1]))
with self.p4.at_exception_level(0):
self.p4.run_copy(i[0], i[1])
if self.p4.errors or self.p4.warnings:
print("Copy errors ", self.p4.errors, " warnings ", self.p4.warnings)
with self.p4.at_exception_level(1):
self.p4.run_integrate(i[0], i[1])
self.p4.run_resolve('-at')
self.p4.run_sync(i[1]) # if copy and integrate say the file is already integrated without syncing it
if self.p4.warnings:
print(">>>Sync in trueCopies warning: ", self.p4.warnings)
self.p4.run_edit(i[1])
if self.p4.warnings:
print(">>>Edit in trueCopies warning, downgrading to add: ", self.p4.warnings)
self.p4.run_add(i[1])
if self.p4.warnings:
print(">>>Add in trueCopies warning: ", self.p4.warnings)
(data, mode) = copyData[i[1]]
self.putfile(i[1], data, mode) # need to overwrite the file, in case it is a dirty edit
# if we have any files of type +x, change the type here
if len(executables) > 0:
for x in self.p4.run_opened():
print('open files: ', x)
print('changing file type to +x for %s' % str(executables))
self.p4.run_reopen('-t+x', executables)
# time to submit the lot
l = len( self.p4.run_opened() )
if l > 0: # the first entry in OpenOffice_DEV300 is empty
print("*** Open files : ", l)
description = 'Comment: ' + commit.desc
description += '\nDate: ' + commit.date
description += '\nBranch: ' + cbranch
description += '\nUser: ' + commit.author
description += '\nRev:'
for i in revs: # not sure if there can be more than one. Better be safe
description += ' ' + i
description += '\nParents: ' + str(commit.parents)
description += '\nSortKey: ' + str(commit.sortkey)
ch = self.p4.fetch_change()
ch._description = description
with self.p4.at_exception_level(0):
submitted = self.p4.run_submit(ch, '-frevertunchanged')
print( submitted )
if self.p4.errors:
print( "*** putcommit(): nothing to submit. Errors : ", self.p4.errors, " Warnings: ", self.p4.warnings )
self.p4.delete_change(submitted[0]['change'])
if self.p4.errors or self.p4.warnings:
print( "delete change had problems. Errors =:", self.p4.errors, " Warnings : ", self.p4.warnings)
return '0'
else:
for s in reversed( submitted ):
if 'submittedChange' in s:
change = s['submittedChange'] # the last entry in the array contains the change number
break
# fixing the timestamp to resemble original timestamp from Mercurial
# need to be super user to do that
if change:
self.p4.user = self.superuser
self.p4.password = self.superticket
ch = self.p4.fetch_change(change)
ch._date = timestamp
self.p4.save_change(ch, '-f')
print( "*** End of putcommit(%s) ***" % change)
return change
def puttags(self, tags):
print( "Called puttags() with")
print( "***", tags )
with self.p4.at_exception_level(1):
for name, change in tags.items():
self.p4.run_tag('-l'+name, '//%s/...@%s' % (self.p4.client, change))
if self.p4.warnings:
print( '>>>', self.p4.warnings)
print( "*** End of puttags() ***")
return None, None
def setbranch(self, branch, pbranches):
print( "Called setbranch() with")
print( "***", branch )
print( "***", pbranches )
self.next_branch = branch
self.next_pbranches = pbranches
print( "*** End of setbranch() ***")
def setfilemapmode(self, active):
print( "Called setfilemapmode() with")
print( "***", active )
print( "*** End of setfilemapmode() ***")
def before(self):
print( "Called before")
self.p4.port = self.ui.config('convert', 'p4.port', default=self.p4.port)
self.p4.client = self.ui.config('convert', 'p4.client', default=self.p4.client)
self.p4.user = self.ui.config('convert', 'p4.user', default=self.p4.user)
password = self.ui.config('convert', 'p4.password', default=self.p4.password)
self.superuser = self.p4.user
self.p4.connect()
print( self.p4 )
print( self.p4.ticket_file )
if password != self.p4.password:
print( 'logging on ...')
self.p4.password = password
self.p4.run_login()
self.superticket = self.p4.password
self.users = {}
client = self.p4.fetch_client()
self.root = client._root
self.p4.cwd = self.root
def after(self):
print( "Called after")
self.p4.disconnect()