# Version 2.5.0 # # 2.4.0 changes (performance): # Several of the internal path-prefix helpers (getDirList, getFileDirList, # and the fallback path in searchCache) rebuilt each ancestor-directory # prefix from scratch via list-slicing + "/".join() at every path depth, # and getFileDirList additionally recomputed an invariant (the file's # parent directory) at every depth level despite it not changing. None of # this affected correctness, but it adds up on very large changelists -- # e.g. Unreal Engine 5 "One File Per Actor" workflows routinely produce # changelists of 100K-400K+ files, and this trigger runs on every submit. # These helpers now build each prefix incrementally instead. filterRenames # was also changed from an O(N*M) list-membership scan (and O(N) list # .remove() per match) to a single O(N) pass using a set, with an early # exit when a changelist contains no deletes at all (the common case for # bulk-add changelists). All of the above are pure refactors -- verified # against the previous implementation across randomized and edge-case # inputs -- with no change in trigger behavior. Additionally, # getFileDirList() now returns a set instead of a dict, since its values # were write-only (never read by any caller); this required a matching # update to TestCheckCaseTrigger.py's testBasic(), which asserted directly # on the old dict shape. # tag::includeManual[] """ CaseCheckTrigger.py This trigger ensures users are not adding new files (directly or by branching) which only differ in case (either filename or a directory element of their path) from existing depot paths. It is useful for both case-sensitive and case-insensitive servers, although most used for the former. Example 1: Typical usage from the Helix Core server triggers table: Triggers: CheckCaseTrigger change-submit //... "/usr/bin/python3 /p4/common/bin/triggers/CheckCaseTrigger.py %changelist% myuser=%user%" SAMPLE OUTPUT: Submit validation failed -- fix problems then use 'p4 submit -c 1234'. 'CheckCaseTrigger' validation failed: Your submission has been rejected because the following files are inconsistent in their use of case with respect to existing directories: Your file: '//depot/dir/test' existing file/dir: '//depot/DIR' CASE CONFLICTS WITH DELETED FILES: By default (as of version 2.3.0), this trigger treats a depot path as "existing" for case-comparison purposes even if its head revision action is 'delete'. This closes a gap where, e.g., '//depot/foo.txt' is deleted in one changelist and '//depot/Foo.txt' is added in a later changelist: the case conflict against the (deleted) history of foo.txt is now caught, because the earlier submitted revisions of foo.txt still exist in the depot and can still cause problems for case-insensitive clients (see 'p4 help protect' / SDP documentation on case sensitivity for details). This is deliberately NOT the same thing as a same-changelist case-only rename (delete old-case path + add/branch/move-add new-case path in one submit) -- that pattern is still recognized and allowed automatically; see filterRenames() below. The stricter check above only applies across separate changelists. Sites that need to phase in this stricter behavior (e.g. because existing depot history already contains such conflicts from before this fix was deployed) can temporarily restore the old, more permissive behavior with 'includedeletedpaths=no' on the trigger command line. This is intended as a rollout aid, not a long-term setting, since it re-opens the original gap. BYPASS LOGIC: By default, this trigger can be bypassed by any user by adding the token BYPASS_CASE_CHECK to the changelist description. Specify 'allowbypass=no' on the command line to disable the ability to bypass this trigger As an exception, if the user is 'git-fusion-user', the case check is always bypassed if 'myuser' is defined. NOTE: With the stricter deleted-path checking described above, this bypass is the intended, auditable way to perform a deliberate case-only rename across two separate changelists (e.g. delete '//depot/foo.txt' in one submit, then later add '//depot/Foo.txt'). Because it requires an explicit token in the changelist description, it leaves a record of who chose to override the check and when, rather than requiring a blanket, site-wide relaxation of the rule. DEPENDENCIES: This trigger requires P4Triggers.py and the P4Python API. SEE ALSO: See 'p4 help triggers'. """ # end::includeManual[] from __future__ import print_function import logging import os import platform import re import subprocess import sys import P4 import P4Triggers # Method canonical # IN: string, utf8 compatible # OUT: unicode string, all lower case def canonical(aString): return aString.lower() def getDepot(path): return path[2:].split("/")[0] class CheckCaseTrigger(P4Triggers.P4Trigger): # tag::includeManual[] """CaseCheckTrigger is a subclass of P4Trigger. Use this trigger to ensure that your depot does not contain two filenames or directories that only differ in case. Having files with different case spelling will cause problems in mixed environments where both case-insensitive clients like Windows and case- sensitive clients like UNIX access the same server. """ # end::includeManual[] def __init__(self, *args, **kwargs): kwargs['charset'] = 'none' kwargs['api_level'] = 71 self.allowBypass=AllowBypass self.includeDeletedPaths=IncludeDeletedPaths fileFilter = None if 'filefilter' in kwargs: fileFilter = kwargs['filefilter'] del kwargs['filefilter'] P4Triggers.P4Trigger.__init__(self, **kwargs) self.parse_args(__doc__, args) # Ensure that -ztag global option is used. self.p4.tagged = True # need to reset the args in case a p4config file overwrote them for (k, v) in kwargs.items(): if k != "log": try: setattr(self.p4, k, v) except: self.logger.error("error setting p4 property: '%s' to '%s'" % (k, v)) self.map = None if fileFilter: try: with open(fileFilter) as f: self.map = P4.Map() for line in f: self.map.insert(line.strip()) except IOError: self.logger.error("Could not open filter file %s" % fileFilter) self.depotCache = {} self.masterCache = {} self.maxWildcards = 9 self.loggingEnabled = self.logger.isEnabledFor(logging.DEBUG) self.caseSensitive = platform.system() == "Linux" # Default - will be checked later def add_parse_args(self, parser): """Specific args for this trigger - also calls super class to add common trigger args""" parser.add_argument('change', help="Change to validate - %%change%% argument from triggers entry.") parser.add_argument('-m', '--max-errors', default=10, help="Max no of errors before aborting submit. Default 10.") super(CheckCaseTrigger, self).add_parse_args(parser) def setUp(self): info = self.p4.run_info()[0] if "unicode" in info and info["unicode"] == "enabled": self.p4.charset = "utf8" self.p4.exception_level = 1 # ignore WARNINGS like "no such file" self.p4.prog = "CheckCaseTrigger" if self.allowBypass: self.USER_MESSAGE=""" Your changelist submit attempt has been rejected because one or more file paths opened for add vary only by case from existing files/directory paths. Creating file/folders that vary only in case from existing paths causes inconsistent behavior across platforms with different case handling behaviors (e.g. Windows, Linux/UNIX, Mac OSX). Thus, adding case-only variations of existing paths is strongly discouraged. If you are certain the files to be added will only be accessed from workspaces on case-sensitive platforms (like UNIX/Linux), then this trigger can be bypassed by adding the token BYPASS_CASE_CHECK to the changelist description and attempting the submit again. Alternately, you can revert any files opened for add in your changelists that vary only in case from existing files, or move them to new names that don't conflict with existing files. Offending files: """ else: self.USER_MESSAGE=""" Your changelist submit attempt has been rejected because one or more file paths opened for add vary only by case from existing files/directory paths. Creating file/folders that vary only in case from existing paths causes inconsistent behavior across platforms with different case handling behaviors (e.g. Windows, Linux/UNIX, Mac OSX). Thus, adding case-only variations of existing paths is disallowed. To move forward, you can revert any files opened for add in your changelists that vary only in case from existing files, or move them to new names that don't conflict with existing files. Offending files: """ self.BADFILE_FORMAT=""" Your file: '%s' existing file/dir: '%s' """ def validate(self): """Here the fun begins. This method overrides P4Trigger.validate()""" badlist = {} info = self.p4.run_info() if "caseHandling" in info[0]: self.caseSensitive = "insensitive" != info[0]["caseHandling"] self.logger.debug("validate: p4d caseSensitive %s", self.caseSensitive) files = self.change.files if self.loggingEnabled: self.logger.debug("validate: Files to submit: %s", files) self.filterRenames(files) # Determine valid file list validFiles = [] for file in files: action = file.revisions[0].action if self.map and self.map.includes(file.depotFile): continue if not action in ("add", "branch", "move/add"): continue path = file.depotFile[2:] if self.loggingEnabled: self.logger.debug("validate: path = %s", path) validFiles.append(file.depotFile) if self.loggingEnabled: self.logger.debug("validate: file.depotFile = %s", file.depotFile) if self.loggingEnabled: self.logger.debug("validate: validFiles = %s", validFiles) # Build cache for each unique depot. self.buildCache(validFiles) if self.loggingEnabled: self.logger.debug("validate: masterCache_1 = %s", self.masterCache) # Look for files in cache. This includes looking for directories in file path along the way. self.searchCache(validFiles, badlist) if self.loggingEnabled: self.logger.debug("validate: badlist = %s", badlist) self.logger.debug("validate: masterCache_2 = %s", self.masterCache) if len(badlist) > 0: self.report(badlist) return len(badlist) == 0 # This method returns a list of all dirs between root and lowest level in the filelist # Can then run "p4 dirs -i a/* a/b/*" against this list to find any other potential conflicts at each level # IN: filelist # OUT: dirlist for dirs command def getDirList(self, fileList): # Files: # //D/a/f.txt # //D/a/b/c/f.txt # Output: # //D # //D/a # //D/a/b # //D/a/b/c # We don't need to go any deeper than max path of files in list # # PERF NOTE: builds each ancestor prefix incrementally (appending one # path segment at a time) rather than re-slicing parts[:i] and calling # "/".join() from scratch at every depth level i. On case-sensitive # servers this also avoids computing the identical "/".join(parts[:i]) # twice per level (once for the key, once for the value) as the prior # implementation did. Verified to produce identical output to the # previous slicing-based implementation. dirList = {} for f in fileList: cf = canonical(f) parts = f[2:].split('/') # Original case cparts = cf[2:].split('/') # Lower case prefix = "" cprefix = "" for i in range(1, len(cparts)): # Process up to the parent dir of the file seg = parts[i - 1] cseg = cparts[i - 1] prefix = seg if i == 1 else prefix + "/" + seg cprefix = cseg if i == 1 else cprefix + "/" + cseg p = prefix if self.caseSensitive else cprefix if not p in dirList: dirList[p] = prefix return dirList # This method returns a list of dirs containing files in the filelist (includes intermediat dirs to allow for # dir and filename collision # Can then run "p4 files -i a/b/c/* a/b/d/*" against this list to find any other potential conflicts at each level # IN: filelist # OUT: dirlist (as a set) for files command # # PERF/TYPE NOTE (2.4.0): this used to return a dict whose values were # always just the file's own parent directory, recomputed (uselessly) # at every depth level. The only caller (buildCache) only ever iterates # 'for d in fileDirList', i.e. only the keys were ever used -- the values # were write-only and never read anywhere. This now returns a plain set # of directory paths instead, which is both cheaper (no value string to # build at all) and a more honest representation of what this data # actually is: a set of directories to query, not a mapping. # # NOTE: this is a return-type change from the previous dict-returning # implementation. buildCache() only ever does 'for d in fileDirList' and # 'if not fileDirList', both of which work identically for a set, so # trigger behavior is unaffected. TestCheckCaseTrigger.py's testBasic() # asserted directly on this method's dict shape (including values) and # has been updated to match; see that file for details. def getFileDirList(self, fileList): # Files: # //D/a/c.txt # //D/a/b/c/d.txt # Output: # //D/a # //D/a/b # //D/a/b/C - sensitive # //D/a/b/c - insensitive # We don't need to go any deeper than max path of files in list dirSet = set() for f in fileList: parts = f[2:].split('/') if self.caseSensitive: prefix = "" for i in range(1, len(parts)): # Process up to the parent dir of the file seg = parts[i - 1] prefix = seg if i == 1 else prefix + "/" + seg dirSet.add(prefix) else: cf = canonical(f) cparts = cf[2:].split('/') cprefix = "" for i in range(1, len(cparts)): # Process up to the parent dir of the file cseg = cparts[i - 1] cprefix = cseg if i == 1 else cprefix + "/" + cseg dirSet.add(cprefix) return dirSet # Builds a global cache to use for mismatch searches. # IN: depots to use # fileList to parse # OUT: None def buildCache(self, fileList): if self.loggingEnabled: self.logger.debug("buildCache: fileList = %s", fileList) depots = self.p4.run_depots() for d in depots: dname = d["name"] cd = canonical(dname) if self.caseSensitive: self.depotCache[dname] = dname else: self.depotCache[cd] = dname dirList = self.getDirList(fileList) if self.loggingEnabled: self.logger.debug("buildCache: dirList = %s", dirList) # Note depots will exist in the list but we need to ensure correct case is used if not self.caseSensitive: for d in self.depotCache: if not d in dirList: dirList[d] = d if self.caseSensitive: dirParams = ["//" + d + "/*" for d in dirList] else: dirParams = ["//" + dirList[d] + "/*" for d in dirList] cdirs = {} if dirParams: for d in self.p4.run_dirs(*dirParams): d = d["dir"] # result is in tagged mode, single entry "dir"=>directory name cd = d.lower() self.masterCache[cd] = d if self.caseSensitive: if not d in dirList and getDepot(cd) in self.depotCache: cdirs[cd] = d else: if cd != d and not d in dirList: cdirs[cd] = d # If necessary, repeat the dirs command on case sensitive systems with any extra dirs found from # previous call if self.caseSensitive and len(cdirs) > 0: dirParams = [d + "/*" for d in cdirs.keys()] for d in self.p4.run_dirs(*dirParams): d = d["dir"] # result is in tagged mode, single entry "dir"=>directory name cd = d.lower() self.masterCache[cd] = d fileDirList = self.getFileDirList(fileList) if not fileDirList: return fileParams = ["//" + d + "/*" for d in fileDirList] for f in self.p4.run_files(*fileParams): # NOTE: Prior versions of this trigger unconditionally skipped any # file whose head revision action is 'delete' when building this # cache. That meant a path could be deleted in one changelist and # a case-differing path added in a later changelist without ever # being caught, even though the deleted file's earlier revisions # still exist in the depot and can still cause problems for # case-insensitive clients. As of 2.3.0 we include deleted paths # by default so this history is caught too; a same-changelist # case-only rename (delete + add/branch/move-add together) is # still allowed via filterRenames() below, and a deliberate # cross-changelist rename can use BYPASS_CASE_CHECK. Sites that # need to temporarily restore the old behavior (e.g. to phase in # this stricter check against existing depot history) can pass # 'includedeletedpaths=no' on the trigger command line. if self.includeDeletedPaths or not "delete" in f["action"]: f = f["depotFile"] cf = f.lower() if not cf in self.masterCache: self.masterCache[cf] = f # Method filterRenames: # Removes files opened for add, branch, or move/add that are paired in # this SAME changelist with a delete of a case-differing path. This # represents an intentional, atomic case-only rename/fix (e.g. deleting # '//depot/foo.txt' and adding '//depot/Foo.txt' in one submit) and # should not be blocked, regardless of which of the three actions was # used to add the new path back. # # This is distinct from -- and does not affect -- the deleted-path # history check in buildCache(), which only looks at PREVIOUSLY # COMMITTED changelists, since files in the changelist currently being # submitted are not yet reflected by 'p4 files' at change-submit time. def filterRenames(self, files): # PERF NOTE: the previous implementation checked membership against # 'deletes' while it was still a list, which is O(len(deletes)) per # check, and called files.remove(f) once per match, which is itself # O(len(files)) per call. For a large changelist that mixes many # deletes with many adds (a common shape for bulk reorganizations -- # exactly the case this broader add/branch/move-add check targets), # that combination can get expensive. 'deletes' is now a set for O(1) # membership checks, and 'files' is rebuilt in a single O(N) pass # instead of being mutated one .remove() call at a time. There is # also an early exit when the changelist contains no deletes at all # (the common case for pure-add changelists), which skips the # filtering pass entirely. deletes = {x.depotFile.lower() for x in files if x.revisions[0].action == 'delete'} if not deletes: return keep = [] for x in files: if x.revisions[0].action in ('add', 'branch', 'move/add') and x.depotFile.lower() in deletes: continue # part of an in-changelist case-only rename; not a conflict keep.append(x) files[:] = keep def report(self, badfiles): msg = self.USER_MESSAGE for (n, (file, mismatch)) in enumerate(badfiles.items()): if n >= self.options.max_errors: break msg += self.BADFILE_FORMAT % (file, mismatch) self.message(msg) def run(self): """Runs trigger""" try: self.logger.debug("CheckCaseTrigger firing") self.setupP4() return self.parseChange(self.options.change) except Exception: return self.reportException() # This method searches a global cache to find case mismatches for changelist files. # File subdirectories and file itself are added to cache if no mismatches are found. # IN: List of changelist files for which we want verify there are no case mismatches. # Mismatch dictionary that records mismatches. # OUT: May modify mismatches parameter def searchCache(self, cfiles, mismatches): if self.loggingEnabled: self.logger.debug("searchCache: changelist files = %s, mismatches = %s", cfiles, mismatches) self.logger.debug("searchCache: masterCache = %s", self.masterCache) for f in cfiles: if self.loggingEnabled: self.logger.debug("searchCache: f = %s", f) mismatch = "" # Check depot depot = getDepot(f) cdepot = canonical(depot) # Search on depots - require depot to be in cache if self.caseSensitive: if not depot in self.depotCache: mismatch = self.depotCache[cdepot] if self.loggingEnabled: self.logger.debug("depot not found: %s", depot) mismatches[f] = mismatch continue else: if depot != self.depotCache[cdepot]: mismatch = self.depotCache[cdepot] if self.loggingEnabled: self.logger.debug("mismatch2: sd = %s, f = %s, m = %s", cdepot, depot, mismatch) mismatches[f] = mismatch continue # Look for file and continue if it's in the cache. It's already been added. cf = canonical(f) if cf in self.masterCache: if self.loggingEnabled: self.logger.debug("searchCache: found %s: %s", f, self.masterCache[cf]) if f != self.masterCache[cf]: mismatch = self.masterCache[cf] if self.loggingEnabled: self.logger.debug("mismatch3: cf = %s, f = %s, m = %s", cf, f, mismatch) mismatches[f] = mismatch continue # Need to check for mismatch of file path components. # If none, add components to cache. # # PERF NOTE: this is the path taken for any file not already in # masterCache -- i.e. essentially every file in a changelist # dominated by brand-new adds (the common shape for large Unreal # Engine "One File Per Actor" changelists). Prefixes are built # incrementally (appending one segment at a time) instead of # re-slicing parts[:i]/cparts[:i] and calling "/".join() from # scratch at every depth level. Verified to produce identical # cp/p values to the previous slicing-based implementation. parts = f[2:].split('/') cparts = cf[2:].split('/') cprefix = "" prefix = "" for i in range(1, len(cparts) + 1): cseg = cparts[i - 1] seg = parts[i - 1] cprefix = cseg if i == 1 else cprefix + "/" + cseg prefix = seg if i == 1 else prefix + "/" + seg cp = "//" + cprefix p = "//" + prefix if not cp in self.masterCache: # Save in cache self.masterCache[cp] = p if self.loggingEnabled: self.logger.debug("adding to cache: %s", self.masterCache[cp]) else: m = self.masterCache[cp] if m != p: mismatch = m if self.loggingEnabled: self.logger.debug("mismatch4: cp = %s, p = %s, mismatch = %s", cp, p, mismatch) break if mismatch: mismatches[f] = mismatch else: self.masterCache[cf] = f if __name__ == "__main__": # Generate new args - parsing out port=123 style way of specifying # parameters intended for p4 properties kwargs = {} args = [] for arg in sys.argv[1:]: p = arg.split("=", 1) if len(p) == 1: args.append(arg) else: kwargs[p[0]] = p[1] # Example of how to exclude the 'git-fusion-user' # Note: Need to remove 'myuser' after test as it's not a valid P4 argument. if 'myuser' in kwargs: if kwargs['myuser'] == 'git-fusion-user': sys.exit(0) else: del kwargs['myuser'] AllowBypass = 1 if 'allowbypass' in kwargs: if kwargs['allowbypass'] == 'no': AllowBypass = 0 # Remove 'allowbypass' after test as it's not a valid P4 argument. del kwargs['allowbypass'] # Controls whether files whose head revision action is 'delete' are # still treated as "existing" for case-conflict purposes (see the # CASE CONFLICTS WITH DELETED FILES section of the module docstring). # Defaults to on (the stricter, corrected behavior). Sites phasing in # this change against existing depot history can pass # 'includedeletedpaths=no' to temporarily restore the old behavior. IncludeDeletedPaths = 1 if 'includedeletedpaths' in kwargs: if kwargs['includedeletedpaths'] == 'no': IncludeDeletedPaths = 0 # Remove after test as it's not a valid P4 argument. del kwargs['includedeletedpaths'] if AllowBypass: # Grab the changelist description, and scan for the bypass token string. # If the token is detected, silently and immediately exit with a happy 0 # exit code. changelist = sys.argv[1] cmd = "%s -ztag -F %%desc%% describe -f -s %s" % (os.getenv('P4BIN','p4'), changelist) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) (changeDesc, err) = p.communicate() p_status = p.wait() # If the changelist description contains the text BYPASS_CASE_CHECK, # bypass the case check logic. if (re.search (b'BYPASS_CASE_CHECK', changeDesc, re.MULTILINE)): sys.exit(0) ct = CheckCaseTrigger(*args, **kwargs) sys.exit(ct.run())