#!/p4/common/python/bin/python3
# -*- coding: utf-8 -*-
"""@SSTemplateUpdate.py
|------------------------------------------------------------------------------|
| Copyright (c) 2008-2016 Perforce Software, Inc. |
|------------------------------------------------------------------------------|
SSTemplateUpdate.py - Stream Spec Template Update, a part of the CBD system.
Environment Dependencies:
This script assumes a complete Perforce environment including a valid,
non-expiring ticket in the P4TICKETS file. This script is called by
the Perforce server, so the environment should be reliable.
"""
# Python 2.7/3.3 compatibility.
from __future__ import print_function
import sys
# Python 2.7/3.3 compatibility.
if sys.version_info[0] >= 3:
from configparser import ConfigParser
else:
from ConfigParser import ConfigParser
import argparse
import textwrap
import os
import re
from datetime import datetime
import logging
from P4 import P4, P4Exception
# If working on a server with the SDP, the 'LOGS' environment variable contains
# the path the standard loggin directory. The '-L <logfile>' argument shoudl be
# specified in non-SDP environments.
LOGDIR = os.getenv ('LOGS', '/p4/1/logs')
P4PORT = os.getenv ('P4PORT', 'UNDEFINED_P4PORT_VALUE')
P4USER = os.getenv ('P4USER', 'UNDEFINED_P4USER_VALUE')
P4CLIENT = os.getenv ('P4CLIENT', 'UNDEFINED_P4CLIENT_VALUE')
P4CONFIG = os.getenv ('P4CONFIG', 'UNDEFINED_P4CONFIG_VALUE')
P4BIN = os.getenv("P4BIN", "p4")
if (P4CONFIG != 'UNDEFINED_P4CONFIG_VALUE'):
del os.environ[ 'P4CONFIG']
DEFAULT_LOG_FILE = '%s/SSTemplateUpdate.log' % LOGDIR
DEFAULT_VERBOSITY = 'DEBUG'
LOGGER_NAME = 'SSTemplateUpdateTrigger'
VERSION = '2.1.7'
class Main:
""" SSTemplateUpdate
"""
def __init__(self, *argv):
""" Initialization. Process command line argument and initialize logging.
"""
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=textwrap.dedent('''\
NAME
SSTemplateUpdate.py
VERSION
2.1.7
DESCRIPTION
Stream Spec template update.
TRIGGER CONFIGURATION
This script is intended to be configured as a Perforce server trigger.
The entry in the 'Triggers:' table looks like the following:
SSTemplateUpdate change-content //....cbdsst "/p4/common/bin/cbd/triggers/SSTemplateUpdate.py %changelist%"
Use of two entries is intended to enhance robustness for scenarios where 'p4 populate' is used.
EXIT CODES
Zero indicates normal completion. Non-zero indicates an error.
'''),
epilog="Copyright (c) 2008-2016 Perforce Software, Inc."
)
parser.add_argument('changelist', help="Changelist containing an update to an versioned stream spec template (*.cbdsst) file.")
parser.add_argument('-L', '--log', default=DEFAULT_LOG_FILE, help="Default: " + DEFAULT_LOG_FILE)
parser.add_argument('-v', '--verbosity',
nargs='?',
const="INFO",
default=DEFAULT_VERBOSITY,
choices=('DEBUG', 'WARNING', 'INFO', 'ERROR', 'FATAL') ,
help="Output verbosity level. Default is: " + DEFAULT_VERBOSITY)
self.myOptions = parser.parse_args()
self.log = logging.getLogger(LOGGER_NAME)
self.log.setLevel(self.myOptions.verbosity)
format='%(levelname)s [%(asctime)s] [%(funcName)s : %(lineno)d] - %(message)s'
logging.basicConfig(format=format, filename=self.myOptions.log, level=self.myOptions.verbosity)
self.logHandler = logging.FileHandler(self.myOptions.log, mode='a')
df = logging.Formatter('%(asctime)s %(levelname)s: %(message)s', datefmt='%m/%d/%Y %H:%M:%S')
bf = logging.Formatter('%(levelname)s: %(message)s')
self.logHandler.setFormatter(bf)
self.log.addHandler (self.logHandler)
self.log.debug ("Command Line Options: %s\n" % self.myOptions)
# Connect to Perforce
def initP4(self):
"""Connect to Perforce."""
self.p4 = P4()
self.p4.prog = LOGGER_NAME
self.p4.version = VERSION
self.p4.port = P4PORT
self.p4.user = P4USER
self.p4.client = P4CLIENT
self.p4.ticket_file = os.getenv ('P4TICKETS', 'UNDEFINED_P4TICKETS_VALUE')
# API Levels are defined here: http://answers.perforce.com/articles/KB/3197
# Ensure this matches the P4Python version used.
# API Level 79 is for p4d 2015.2.
self.p4.api_level = 79
self.stream = None
self.streamDepot = None
self.streamDepth = None
self.streamType = None
self.streamShortName = None
self.VersionRemappedField = False
self.VersionIgnoredField = False
try:
self.p4.connect()
except P4Exception:
self.log.fatal("Unable to connect to Perforce at P4PORT=%s.\n\nThe error from the Perforce server is:" % P4PORT)
for e in self.p4.errors:
print (e)
self.log.error("Errors: " + e)
for w in self.p4.warnings:
print (w)
self.log.warn("Warnings: " + w)
return False
try:
self.log.debug("Doing 'p4 login -s' login status check.")
self.p4.run_login('-s')
except P4Exception:
userMessage = "Your attempt to submit changelist %s has been rejected because the CBD system user [%s] is not logged into the server [P4PORT=%s]. Please contact your Perforce Administrator for help.\n\nThe error from the Perforce server is:\n" % (self.myOptions.changelist, P4USER, P4PORT)
print (userMessage)
self.log.fatal("CBD admin user not logged in. User error message is:\n%s\n" % userMessage)
for e in self.p4.errors:
print (e)
self.log.error("Errors: " + e)
for w in self.p4.warnings:
print (w)
self.log.warn("Warnings: " + w)
return False
return True
def get_sst_files(self):
"""Find all *.cbdsst files submitted in the given changelist.
Set set.workspace, self.sst_files[].
Return sst_file_count, the number of *.cbdsst files associated with the changelist.
"""
sst_file_count = 0
if (not re.match ('\d+$', self.myOptions.changelist)):
self.log.fatal ("The value supplied as a changelist number [%s] must be purely numeric." % self.myOptions.changelist)
return 0
self.log.info ("Processing changelist %s." % self.myOptions.changelist)
try:
cData = self.p4.run_describe('-s', self.myOptions.changelist)
except P4Exception:
userMessage = "Your attempt to submit changelist %s has been rejected because the CBD system cannot get information about that changelist. Please contact your Perforce Administrator for help.\n\nThe error from the Perforce server is:\n" % (self.myOptions.changelist)
print (userMessage)
self.log.fatal("Failed to describe changlist. User error message is:\n%s\n" % userMessage)
for e in self.p4.errors:
print (e)
self.log.error("Errors: " + e)
for w in self.p4.warnings:
print (w)
self.log.warn("Warnings: " + w)
sys.exit(1)
self.log.debug ("Data: %s" % cData)
self.p4.client = cData[0]['client']
self.sst_files = []
index = -1
sst_file_count = 0
self.cspec = self.p4.fetch_client (self.p4.client)
self.log.debug ("CDATA: %s" % self.cspec)
# Since this script is called as a trigger firing on changelists containing *.cbdsst files,
# we can assume that at least one file will be associated with the changelist. However, if
# a task stream, those files will not be visibile to us. So we detect if no *.cbdsst files
# are associated with the given changelist, and bail if so.
isTaskStream = 0
try:
if cData[0]['depotFile']:
isTaskStream = 0
except KeyError:
isTaskStream = 1
if not isTaskStream:
for file in cData[0]['depotFile']:
index = index + 1
if (re.search ('\.cbdsst$', file)):
action = cData[0]['action'][index]
rev = cData[0]['rev'][index]
# Ignore actions other than add, move/add, branch, edit, and integrate.
# The delete and move/delete actions are ignored, as well as the
# purged and archive actions.
if (not re.search ('add|branch|edit|integrate', action)):
self.log.warn ("Ignored '%s' action on %s#%s." % (action, file, rev))
continue
self.sst_files.append(file)
sst_file_count = sst_file_count + 1
if not self.streamDepot:
# Take something like //fgs/main/fgs.cbdsst, and derive values from it,
# streamDepot = fgs
# streamShortName = main
# streamName = //fgs/main
self.streamDepot = re.sub (r'^//', '', file)
self.streamDepot = re.sub ('/.*$', '', self.streamDepot)
dData = self.p4.fetch_depot (self.streamDepot)
if (dData['StreamDepth']):
self.streamDepth = re.sub ('^.*/', '', dData['StreamDepth'])
self.streamDepth = int(self.streamDepth)
else:
self.streamDepth = 1
# Split the path into elements using the '/' delimter.
# Elements 0 and 1 are always empty (due to Perforce paths starting
# with '//'. Element 2 is the stream depot name.
# The stream name will look like //fgs/main, or //components/fgs/main.
# The streamDepth tells how many levels of directory to count.
# The streamShortName (e.g. 'main') is the n'th element, dependent on
# the streamDepth (e.g. element 3 for //fgs/main, element 4 for
# //components/fgs/main.
pathElements = file.split ('/')
self.streamShortName = pathElements[self.streamDepth+2]
self.stream = '//%s/%s' % (self.streamDepot, pathElements[3])
# For stream depots with StreamDepth> 1,
i = 1
while (i < self.streamDepth):
self.stream = "%s/%s" % (self.stream, pathElements[i+3])
i = i + 1
self.log.debug ("Stream for CBDSST files is: %s" % self.stream)
self.log.debug ("Found %d *.cbdsst files in changelist %s." % (sst_file_count, self.myOptions.changelist))
return sst_file_count
def update_stream_spec_and_keys (self, file):
"""Update the stream spec on the live server from the template.
Paths:
share Source/...
import Framework/... //Framework/main/...@5036
"""
# When called as a 'submit-content' trigger, the content of not-yet-submitted files can be accessed
# with '@=' syntax, symilar to referencing content of files in a shelved changelist. This allows us
# to verify that the stream spec can be updated on the live server before allowing the submit of the
# *.cbdsst file to proceed.
fileWithRevSpec = "%s%s%s" % (file, r'@=', self.myOptions.changelist)
self.log.info ("Updating live stream spec from: [%s]." % fileWithRevSpec)
sData = self.p4.fetch_stream (self.stream)
self.log.debug ("SDATA1: %s" % sData)
try:
sstFileContents = self.p4.run_print ('-q', fileWithRevSpec)
except P4Exception:
userMessage = "Your attempt to submit changelist %s failed because the CBD automation was unable to get the contents of [%s] from the Perforce server. This could happen if no changes were made to the *.cbdsst file.\n\nThe error from the Perforce server is:\n" % (self.myOptions.changelist, fileWithRevSpec)
print (userMessage)
self.log.fatal("Failed to save stream spec. User error message is:\n%s\n" % userMessage)
for e in self.p4.errors:
print (e)
self.log.error("Errors: " + e)
for w in self.p4.warnings:
print (w)
self.log.warn("Warnings: " + w)
sys.exit (1)
oldDesc = sstFileContents[1]
###oldDesc = sstFileContents
newDesc = ''
# Parse 'Description:' field value from the *.cbdsst file, substituting the
# stream name tag.
for line in oldDesc.split('\n'):
if (re.match ('^\s*#', line) or re.match('^Description\:', line)):
continue
if (re.match ('(Stream|Options|Owner|Type|Parent)\:', line)):
continue
if (re.match ('Paths\:', line)):
break
line = re.sub ('__EDITME_STREAM__', self.stream, line)
line.strip()
if not newDesc:
newDesc = line
else:
newDesc = newDesc + '\n' + line
newDesc = re.sub ('^\t', '', newDesc)
newDesc = re.sub ('\n\t', '\n', newDesc)
sData['Description'] = newDesc
# The paths() array includes data from the 'Paths:' field of the stream
# spec, augmented by revision specifiers extracted from the stream spec
# template file.
paths = list()
# The newPaths() list is similar to paths, but excludes the revsion
# specifiers, as they're not valid for P4D 2013.2 and lower servers.
# It is in a form suitalbe for feeding directly to the 'Paths' field
# of the stream spec using the P4Python API.
newPaths = list()
# This is a count/index for both the paths() and newPaths() arrays,
# as they both have the name number of elements.
pathsCount = 0
# Parse import path entries from the 'Paths:' field in the *.cbdsst file.
# If we find revision specifiers on the depot paths, strip them off
# before feeding them to the server, but also preserve the revision
# specifiers for writing to the 'p4 keys' store.
pathEntriesStarted = False
for line in oldDesc.split('\n'):
if (re.match ('Paths\:', line)):
pathEntriesStarted = True
continue
if (pathEntriesStarted == False):
continue
line = line.strip()
if (re.match ('share ', line)):
# shortPath is just what follows the 'share' token in the
# 'Paths:' field entry of a stream spec.
# sharePath is the fully-qualified form of the shortPath following
# the 'share' token in values in the 'Paths:' field of the stream
# spec. It is obtained by prefixing shortPath with the stream
# name and '/'. So for 'share src/...' in the //Jam/MAIN
# stream, sharePath would be "//Jam/MAIN/src/...".
shortPath = re.sub ('^share\s+', '', line)
sharePath = "%s/%s" % (self.stream, re.sub ('^share\s+', '', line))
paths.append (('share', sharePath))
newPaths.append ("share %s" % shortPath)
self.log.debug ("PATH DATA %s %s" % paths[pathsCount])
if (re.match ('import\s+', line)):
# If a revsion specifier was found on an 'import' line, store it.
revSpec = '#head'
if (re.search('#', line)):
revSpec = line
revSpec = re.sub('^.*#', '#', line)
if (re.search('@', line)):
revSpec = line
revSpec = re.sub('^.*@', '@', line)
localPath = re.sub ('import\s+', '', line)
localPath = re.sub (' //.*$', '', localPath)
depotPath = re.sub ('^.*//', '//', line)
depotPath = re.sub ('(#|@).*$', '', depotPath)
paths.append (('import', localPath, depotPath, revSpec))
# The newPaths() list excludes the revision specifier, since P4D servers
# 2013.2 and lower cannot handle it.
newPaths.append ("import %s %s" % (localPath, depotPath))
self.log.debug ("PATH DATA %s L:[%s] D:[%s] R:[%s]" % paths[pathsCount])
if (re.match ('import\+ ', line)):
# If a revsion specifier was found on an 'import+' line, store it.
revSpec = '#head'
if (re.search('#', line)):
revSpec = line
revSpec = re.sub('^.*#', '#', line)
if (re.search('@', line)):
revSpec = line
revSpec = re.sub('^.*@', '@', line)
localPath = re.sub ('import\+\s+', '', line)
localPath = re.sub (' //.*$', '', localPath)
depotPath = re.sub ('^.*//', '//', line)
depotPath = re.sub ('(#|@).*$', '', depotPath)
paths.append (('import+', localPath, depotPath, revSpec))
# The newPaths() list excludes the revision specifier, since P4D servers
# 2013.2 and lower cannot handle it.
newPaths.append ("import+ %s %s" % (localPath, depotPath))
self.log.debug ("PATH DATA %s L:[%s] D:[%s] R:[%s]" % paths[pathsCount])
pathsCount = pathsCount + 1
self.log.debug ("== Path Entries ==")
for pathEntry in paths:
self.log.debug ("RAW PATH ENTRY[0]: %s" % pathEntry[0])
if (pathEntry[0] == 'share'):
self.log.debug ("PATH ENTRY: share %s" % pathEntry[1])
if (pathEntry[0] == 'import'):
self.log.debug ("PATH ENTRY: import %s %s%s" % (pathEntry[1], pathEntry[2], pathEntry[3]))
if (pathEntry[0] == 'import+'):
self.log.debug ("PATH ENTRY: import+ %s %s%s" % (pathEntry[1], pathEntry[2], pathEntry[3]))
sData['Paths'] = newPaths
if self.VersionRemappedField:
# The newRemapped() array includes data from the 'Remapped:' field of the stream
# spec.
newRemapped = list()
# This is a count/index for the newRemapped() array.
remappedCount = 0
# Parse entries from the 'Remapped:' field in the *.cbdsst file.
remappedEntriesStarted = False
for line in oldDesc.split('\n'):
if (re.match ('Remapped\:', line)):
remappedEntriesStarted = True
continue
if (remappedEntriesStarted == False):
continue
if (not re.match ('\t', line)):
break
line = line.strip()
line = line.rstrip()
self.log.debug ("Remapped Entry: %s" % line)
newRemapped.append(line)
remappedCount = remappedCount + 1
if (remappedCount > 0):
self.log.debug ("Adding these Remapped field entries: %s" % newRemapped)
sData['Remapped'] = newRemapped
if self.VersionIgnoredField:
# The newIgnored() array includes data from the 'Ignored:' field of the stream
# spec.
newIgnored = list()
# This is a count/index for the newIgnored() array.
ignoredCount = 0
# Parse entries from the 'Ignored:' field in the *.cbdsst file.
ignoredEntriesStarted = False
for line in oldDesc.split('\n'):
if (re.match ('Ignored\:', line)):
ignoredEntriesStarted = True
continue
if (ignoredEntriesStarted == False):
continue
if (not re.match ('\t', line)):
break
line = line.strip()
line = line.rstrip()
self.log.debug ("Ignored Entry: %s" % line)
newIgnored.append(line)
ignoredCount = ignoredCount + 1
if (ignoredCount > 0):
self.log.debug ("Adding these Ignored field entries: %s" % newIgnored)
sData['Ignored'] = newIgnored
self.log.debug ("SDATA2: %s" % sData)
try:
self.log.debug ("Saving stream spec data %s: %s" % (self.stream, sData))
self.p4.save_stream(sData)
except P4Exception:
userMessage = "Your attempt to submit changelist %s has been rejected because the stream spec failed to update on the server. It may have been rejected by the Perforce server.\n\nThe error from the Perforce server is:\n" % self.myOptions.changelist
print (userMessage)
self.log.fatal("Failed to save stream spec. User error message is:\n%s\n" % userMessage)
for e in self.p4.errors:
print (e)
self.log.error("Errors: " + e)
for w in self.p4.warnings:
print (w)
self.log.warn("Warnings: " + w)
return False
# Next, update the 'p4 keys' on the server.
# First, generate a list of existing keys for this stream to remove.
keyNameBase = 'cbd_stream_%s' % self.stream
keyNameBase = re.sub ('//', '', keyNameBase)
keyNameBase = re.sub ('/', '_', keyNameBase)
pathKeySearch = keyNameBase
vSpecKeySearch = keyNameBase
pathKeySearch = '%s_path*' % pathKeySearch
vSpecKeySearch = '%s_vspec*' % vSpecKeySearch
pathKeysData = self.p4.run_keys('-e', pathKeySearch)
vSpecKeysData = self.p4.run_keys('-e', vSpecKeySearch)
for keyData in pathKeysData:
self.log.debug ("Running: p4 key -d %s" % keyData['key'])
try:
self.p4.run_key('-d', keyData['key'])
except P4Exception:
self.log.fatal("Failed to delete key [%s] from server." % keyData['key'])
for e in self.p4.errors:
self.log.error("Errors: " + e)
for w in self.p4.warnings:
self.log.warn("Warnings: " + w)
for keyData in vSpecKeysData:
self.log.debug ("Running: p4 key -d %s" % keyData['key'])
try:
self.p4.run_key('-d', keyData['key'])
except P4Exception:
self.log.fatal("Failed to delete key [%s] from server." % keyData['key'])
for e in self.p4.errors:
self.log.error("Errors: " + e)
for w in self.p4.warnings:
self.log.warn("Warnings: " + w)
# Finally, generate the new keys.
i = 0
for pathEntry in paths:
pathKeyName = "%s_path%d" % (keyNameBase, i)
vSpecKeyName = "%s_vspec%d" % (keyNameBase, i)
if (pathEntry[0] == 'share'):
pathKeyValue = pathEntry[1]
vSpecKeyValue = '#head'
if (pathEntry[0] == 'import'):
pathKeyValue = pathEntry[2]
vSpecKeyValue = pathEntry[3]
if (pathEntry[0] == 'import+'):
pathKeyValue = pathEntry[2]
vSpecKeyValue = pathEntry[3]
self.log.debug ("Running: p4 key %s %s" % (pathKeyName, pathKeyValue))
try:
self.p4.run_key(pathKeyName, pathKeyValue)
except P4Exception:
self.log.fatal("Failed to create path key [%s] with value [%s]." % (pathKeyName, pathKeyValue))
for e in self.p4.errors:
self.log.error("Errors: " + e)
for w in self.p4.warnings:
self.log.warn("Warnings: " + w)
self.log.debug ("Running p4 key %s %s" % (vSpecKeyName, vSpecKeyValue))
try:
self.p4.run_key(vSpecKeyName, vSpecKeyValue)
except P4Exception:
self.log.fatal("Failed to create vspec key [%s] with value [%s]." % (vSpecKeyName, vSpecKeyValue))
for e in self.p4.errors:
self.log.error("Errors: " + e)
for w in self.p4.warnings:
self.log.warn("Warnings: " + w)
i = i + 1
return True
def update_modified_stream_specs (self):
"""Update stream specs for the given changelist."""
if (self.get_sst_files()):
for file in self.sst_files:
if (not self.update_stream_spec_and_keys (file)):
self.log.debug ("Processing complete. Changelist %s REJECTED." % self.myOptions.changelist)
return False
self.log.debug ("Processing complete. Changelist %s ACCEPTED." % self.myOptions.changelist)
return True
else:
self.log.warn ("No actionable *.cbdsst files found in changelist %s. Allowing change to submit." % self.myOptions.changelist)
self.log.debug ("Processing complete. Changelist %s ACCEPTED." % self.myOptions.changelist)
return True
if __name__ == '__main__':
""" Main Program
"""
main = Main(*sys.argv[1:])
if (not Main.initP4(main)):
sys.exit (1)
if (Main.update_modified_stream_specs(main)):
sys.exit (0)
else:
sys.exit (1)
| # | Change | User | Description | Committed | |
|---|---|---|---|---|---|
| #1 | 21706 | Sven Erik Knop |
Populate //cbd/main/... .... |