SSTemplateUpdate.py #1

#!/p4/common/python/bin/python3
# -*- coding: utf-8 -*-
"""@SSTemplateUpdate.py
|------------------------------------------------------------------------------|
| Copyright (c) 2008-2016 Perforce Software, Inc.                              |
|------------------------------------------------------------------------------|
SSTemplateUpdate.py - Stream Spec Template Update, a part of the CBD system.

Environment Dependencies:
   This script assumes a complete Perforce environment including a valid,
   non-expiring ticket in the P4TICKETS file.  This script is called by
   the Perforce server, so the environment should be reliable.

"""

# Python 2.7/3.3 compatibility.
from __future__ import print_function

import sys

# Python 2.7/3.3 compatibility.
if sys.version_info[0] >= 3:
   from configparser import ConfigParser
else:
   from ConfigParser import ConfigParser

import argparse
import textwrap
import os
import re
from datetime import datetime
import logging
from P4 import P4, P4Exception

# If working on a server with the SDP, the 'LOGS' environment variable contains
# the path the standard loggin directory.  The '-L <logfile>' argument shoudl be
# specified in non-SDP environments.
LOGDIR = os.getenv ('LOGS', '/p4/1/logs')
P4PORT = os.getenv ('P4PORT', 'UNDEFINED_P4PORT_VALUE')
P4USER = os.getenv ('P4USER', 'UNDEFINED_P4USER_VALUE')
P4CLIENT = os.getenv ('P4CLIENT', 'UNDEFINED_P4CLIENT_VALUE')
P4CONFIG = os.getenv ('P4CONFIG', 'UNDEFINED_P4CONFIG_VALUE')
P4BIN = os.getenv("P4BIN", "p4")

if (P4CONFIG != 'UNDEFINED_P4CONFIG_VALUE'):
    del os.environ[ 'P4CONFIG']

DEFAULT_LOG_FILE = '%s/SSTemplateUpdate.log' % LOGDIR
DEFAULT_VERBOSITY = 'DEBUG'

LOGGER_NAME = 'SSTemplateUpdateTrigger'

VERSION = '2.1.7'

class Main:
   """ SSTemplateUpdate
   """

   def __init__(self, *argv):
      """ Initialization.  Process command line argument and initialize logging.
      """
      parser = argparse.ArgumentParser(
         formatter_class=argparse.RawDescriptionHelpFormatter,
         description=textwrap.dedent('''\
            NAME
         
            SSTemplateUpdate.py
            
            VERSION
            
            2.1.7
            
            DESCRIPTION
            
            Stream Spec template update.
            
            TRIGGER CONFIGURATION
               This script is intended to be configured as a Perforce server trigger.

               The entry in the 'Triggers:' table looks like the following: 

               SSTemplateUpdate change-content //....cbdsst "/p4/common/bin/cbd/triggers/SSTemplateUpdate.py %changelist%"

               Use of two entries is intended to enhance robustness for scenarios where 'p4 populate' is used.
            
            EXIT CODES
            Zero indicates normal completion.  Non-zero indicates an error.
            
            '''),
         epilog="Copyright (c) 2008-2016 Perforce Software, Inc."
      )
      
      parser.add_argument('changelist', help="Changelist containing an update to an versioned stream spec template (*.cbdsst) file.")
      parser.add_argument('-L', '--log', default=DEFAULT_LOG_FILE, help="Default: " + DEFAULT_LOG_FILE)
      parser.add_argument('-v', '--verbosity', 
         nargs='?', 
         const="INFO",
         default=DEFAULT_VERBOSITY,
         choices=('DEBUG', 'WARNING', 'INFO', 'ERROR', 'FATAL') ,
         help="Output verbosity level. Default is: " + DEFAULT_VERBOSITY)
      
      self.myOptions = parser.parse_args()
      
      self.log = logging.getLogger(LOGGER_NAME)
      self.log.setLevel(self.myOptions.verbosity)
      format='%(levelname)s [%(asctime)s] [%(funcName)s : %(lineno)d] - %(message)s'
      logging.basicConfig(format=format, filename=self.myOptions.log, level=self.myOptions.verbosity)

      self.logHandler = logging.FileHandler(self.myOptions.log, mode='a')

      df = logging.Formatter('%(asctime)s %(levelname)s: %(message)s', datefmt='%m/%d/%Y %H:%M:%S')
      bf = logging.Formatter('%(levelname)s: %(message)s')
      self.logHandler.setFormatter(bf)
      self.log.addHandler (self.logHandler)

      self.log.debug ("Command Line Options: %s\n" % self.myOptions)
      
   # Connect to Perforce
   def initP4(self):
      """Connect to Perforce."""
      self.p4 = P4()
      self.p4.prog = LOGGER_NAME
      self.p4.version = VERSION
      self.p4.port = P4PORT
      self.p4.user = P4USER
      self.p4.client = P4CLIENT
      self.p4.ticket_file = os.getenv ('P4TICKETS', 'UNDEFINED_P4TICKETS_VALUE')

      # API Levels are defined here: http://answers.perforce.com/articles/KB/3197
      # Ensure this matches the P4Python version used.
      # API Level 79 is for p4d 2015.2.
      self.p4.api_level = 79
      self.stream = None
      self.streamDepot = None
      self.streamDepth = None
      self.streamType = None
      self.streamShortName = None
      self.VersionRemappedField = False
      self.VersionIgnoredField = False

      try:
         self.p4.connect()

      except P4Exception:
         self.log.fatal("Unable to connect to Perforce at P4PORT=%s.\n\nThe error from the Perforce server is:" % P4PORT)
         for e in self.p4.errors:
            print (e)
            self.log.error("Errors: " + e)
         for w in self.p4.warnings:
            print (w)
            self.log.warn("Warnings: " + w)
         return False

      try:
         self.log.debug("Doing 'p4 login -s' login status check.")
         self.p4.run_login('-s')

      except P4Exception:
         userMessage = "Your attempt to submit changelist %s has been rejected because the CBD system user [%s] is not logged into the server [P4PORT=%s].  Please contact your Perforce Administrator for help.\n\nThe error from the Perforce server is:\n" % (self.myOptions.changelist, P4USER, P4PORT)
         print (userMessage)
         self.log.fatal("CBD admin user not logged in.  User error message is:\n%s\n" % userMessage)
         for e in self.p4.errors:
            print (e)
            self.log.error("Errors: " + e)
         for w in self.p4.warnings:
            print (w)
            self.log.warn("Warnings: " + w)
         return False

      return True

   def get_sst_files(self):
      """Find all *.cbdsst files submitted in the given changelist.
         Set set.workspace, self.sst_files[].
         Return sst_file_count, the number of *.cbdsst files associated with the changelist.
      """

      sst_file_count = 0

      if (not re.match ('\d+$', self.myOptions.changelist)):
         self.log.fatal ("The value supplied as a changelist number [%s] must be purely numeric." % self.myOptions.changelist)
         return 0

      self.log.info ("Processing changelist %s." % self.myOptions.changelist)

      try:
         cData = self.p4.run_describe('-s', self.myOptions.changelist)

      except P4Exception:
         userMessage = "Your attempt to submit changelist %s has been rejected because the CBD system cannot get information about that changelist.  Please contact your Perforce Administrator for help.\n\nThe error from the Perforce server is:\n" % (self.myOptions.changelist)
         print (userMessage)
         self.log.fatal("Failed to describe changlist.  User error message is:\n%s\n" % userMessage)

         for e in self.p4.errors:
            print (e)
            self.log.error("Errors: " + e)
         for w in self.p4.warnings:
            print (w)
            self.log.warn("Warnings: " + w)

         sys.exit(1)

      self.log.debug ("Data: %s" % cData)
      self.p4.client = cData[0]['client']

      self.sst_files = []
      index = -1
      sst_file_count = 0

      self.cspec = self.p4.fetch_client (self.p4.client)
      self.log.debug ("CDATA: %s" % self.cspec)

      # Since this script is called as a trigger firing on changelists containing *.cbdsst files,
      # we can assume that at least one file will be associated with the changelist.  However, if
      # a task stream, those files will not be visibile to us.  So we detect if no *.cbdsst files
      # are associated with the given changelist, and bail if so.
      isTaskStream = 0

      try:
         if cData[0]['depotFile']:
            isTaskStream = 0

      except KeyError:
         isTaskStream = 1

      if not isTaskStream:
         for file in cData[0]['depotFile']:
            index = index + 1

            if (re.search ('\.cbdsst$', file)):
               action = cData[0]['action'][index]
               rev = cData[0]['rev'][index]

               # Ignore actions other than add, move/add, branch, edit, and integrate.
               # The delete and move/delete actions are ignored, as well as the
               # purged and archive actions.
               if (not re.search ('add|branch|edit|integrate', action)):
                  self.log.warn ("Ignored '%s' action on %s#%s." % (action, file, rev))
                  continue

               self.sst_files.append(file)
               sst_file_count = sst_file_count + 1

               if not self.streamDepot:
                  # Take something like //fgs/main/fgs.cbdsst, and derive values from it,
                  # streamDepot = fgs
                  # streamShortName = main
                  # streamName = //fgs/main
                  self.streamDepot = re.sub (r'^//', '', file)
                  self.streamDepot = re.sub ('/.*$', '', self.streamDepot)
                  dData = self.p4.fetch_depot (self.streamDepot)
                  if (dData['StreamDepth']):
                     self.streamDepth = re.sub ('^.*/', '', dData['StreamDepth'])
                     self.streamDepth = int(self.streamDepth)
                  else:
                     self.streamDepth = 1

                  # Split the path into elements using the '/' delimter.
                  # Elements 0 and 1 are always empty (due to Perforce paths starting
                  # with '//'.  Element 2 is the stream depot name.
                  # The stream name will look like //fgs/main, or //components/fgs/main.
                  # The streamDepth tells how many levels of directory to count.
                  # The streamShortName (e.g. 'main') is the n'th element, dependent on
                  # the streamDepth (e.g. element 3 for //fgs/main, element 4 for 
                  # //components/fgs/main.
                  pathElements = file.split ('/')
                  self.streamShortName = pathElements[self.streamDepth+2]
                  self.stream = '//%s/%s' % (self.streamDepot, pathElements[3])

                  # For stream depots with  StreamDepth> 1,
                  i = 1
                  while (i < self.streamDepth):
                     self.stream = "%s/%s" % (self.stream, pathElements[i+3])
                     i = i + 1

                  self.log.debug ("Stream for CBDSST files is: %s" % self.stream)

      self.log.debug ("Found %d *.cbdsst files in changelist %s." % (sst_file_count, self.myOptions.changelist))
      return sst_file_count

   def update_stream_spec_and_keys (self, file):
      """Update the stream spec on the live server from the template.
         Paths:
         share Source/...
         import Framework/... //Framework/main/...@5036

       """

      # When called as a 'submit-content' trigger, the content of not-yet-submitted files can be accessed
      # with '@=' syntax, symilar to referencing content of files in a shelved changelist.  This allows us
      # to verify that the stream spec can be updated on the live server before allowing the submit of the
      # *.cbdsst file to proceed.
      fileWithRevSpec = "%s%s%s" % (file, r'@=', self.myOptions.changelist)
      self.log.info ("Updating live stream spec from: [%s]." % fileWithRevSpec)

      sData = self.p4.fetch_stream (self.stream)
      self.log.debug ("SDATA1: %s" % sData)

      try:
         sstFileContents = self.p4.run_print ('-q', fileWithRevSpec)

      except P4Exception:
         userMessage = "Your attempt to submit changelist %s failed because the CBD automation was unable to get the contents of [%s] from the Perforce server.  This could happen if no changes were made to the *.cbdsst file.\n\nThe error from the Perforce server is:\n" % (self.myOptions.changelist, fileWithRevSpec)
         print (userMessage)
         self.log.fatal("Failed to save stream spec.  User error message is:\n%s\n" % userMessage)

         for e in self.p4.errors:
            print (e)
            self.log.error("Errors: " + e)
         for w in self.p4.warnings:
            print (w)
            self.log.warn("Warnings: " + w)

         sys.exit (1)

      oldDesc = sstFileContents[1]
      ###oldDesc = sstFileContents
      newDesc = ''

      # Parse 'Description:' field value from the *.cbdsst file, substituting the
      # stream name tag.
      for line in oldDesc.split('\n'):
         if (re.match ('^\s*#', line) or re.match('^Description\:', line)):
            continue
         if (re.match ('(Stream|Options|Owner|Type|Parent)\:', line)):
            continue
         if (re.match ('Paths\:', line)):
            break
         line = re.sub ('__EDITME_STREAM__', self.stream, line)
         line.strip()

         if not newDesc:
            newDesc = line
         else:
            newDesc = newDesc + '\n' + line

      newDesc = re.sub ('^\t', '', newDesc)
      newDesc = re.sub ('\n\t', '\n', newDesc)

      sData['Description'] = newDesc

      # The paths() array includes data from the 'Paths:' field of the stream
      # spec, augmented by revision specifiers extracted from the stream spec
      # template file.
      paths = list()

      # The newPaths() list is similar to paths, but excludes the revsion
      # specifiers, as they're not valid for P4D 2013.2 and lower servers.
      # It is in a form suitalbe for feeding directly to the 'Paths' field
      # of the stream spec using the P4Python API.
      newPaths = list()

      # This is a count/index for both the paths() and newPaths() arrays,
      # as they both have the name number of elements.
      pathsCount = 0

      # Parse import path entries from the 'Paths:' field in the *.cbdsst file.
      # If we find revision specifiers on the depot paths, strip them off
      # before feeding them to the server, but also preserve the revision
      # specifiers for writing  to the 'p4 keys' store.
      pathEntriesStarted = False

      for line in oldDesc.split('\n'):
         if (re.match ('Paths\:', line)):
            pathEntriesStarted = True
            continue
         if (pathEntriesStarted == False):
            continue

         line = line.strip()

         if (re.match ('share ', line)):
            # shortPath is just what follows the 'share' token in the
            # 'Paths:' field entry of a stream spec.
            # sharePath is the fully-qualified form of the shortPath following
            # the 'share' token in values in the 'Paths:' field of the stream
            # spec.  It is obtained by prefixing shortPath with the stream
            # name and '/'.  So for 'share src/...' in the //Jam/MAIN
            # stream, sharePath would be "//Jam/MAIN/src/...".
            shortPath = re.sub ('^share\s+', '', line)
            sharePath = "%s/%s" % (self.stream, re.sub ('^share\s+', '', line))
            paths.append (('share', sharePath))
            newPaths.append ("share %s" % shortPath)
            self.log.debug ("PATH DATA %s %s" % paths[pathsCount])

         if (re.match ('import\s+', line)):
            # If a revsion specifier was found on an 'import' line, store it.
            revSpec = '#head'
            if (re.search('#', line)):
               revSpec = line
               revSpec = re.sub('^.*#', '#', line)
            if (re.search('@', line)):
               revSpec = line
               revSpec = re.sub('^.*@', '@', line)
            localPath = re.sub ('import\s+', '', line)
            localPath = re.sub (' //.*$', '', localPath)
            depotPath = re.sub ('^.*//', '//', line)
            depotPath = re.sub ('(#|@).*$', '', depotPath)
            paths.append (('import', localPath, depotPath, revSpec))
            # The newPaths() list excludes the revision specifier, since P4D servers
            #  2013.2 and lower cannot handle it.
            newPaths.append ("import %s %s" % (localPath, depotPath))
            self.log.debug ("PATH DATA %s L:[%s] D:[%s] R:[%s]" % paths[pathsCount])

         if (re.match ('import\+ ', line)):
            # If a revsion specifier was found on an 'import+' line, store it.
            revSpec = '#head'
            if (re.search('#', line)):
               revSpec = line
               revSpec = re.sub('^.*#', '#', line)
            if (re.search('@', line)):
               revSpec = line
               revSpec = re.sub('^.*@', '@', line)
            localPath = re.sub ('import\+\s+', '', line)
            localPath = re.sub (' //.*$', '', localPath)
            depotPath = re.sub ('^.*//', '//', line)
            depotPath = re.sub ('(#|@).*$', '', depotPath)
            paths.append (('import+', localPath, depotPath, revSpec))
            # The newPaths() list excludes the revision specifier, since P4D servers
            # 2013.2 and lower cannot handle it.
            newPaths.append ("import+ %s %s" % (localPath, depotPath))
            self.log.debug ("PATH DATA %s L:[%s] D:[%s] R:[%s]" % paths[pathsCount])

         pathsCount = pathsCount + 1

      self.log.debug ("== Path Entries ==")
      for pathEntry in paths:
         self.log.debug ("RAW PATH ENTRY[0]: %s" % pathEntry[0])
         if (pathEntry[0] == 'share'):
            self.log.debug ("PATH ENTRY: share %s" % pathEntry[1])
         if (pathEntry[0] == 'import'):
            self.log.debug ("PATH ENTRY: import %s %s%s" % (pathEntry[1], pathEntry[2], pathEntry[3]))
         if (pathEntry[0] == 'import+'):
            self.log.debug ("PATH ENTRY: import+ %s %s%s" % (pathEntry[1], pathEntry[2], pathEntry[3]))

      sData['Paths'] = newPaths

      if self.VersionRemappedField:
         # The newRemapped() array includes data from the 'Remapped:' field of the stream
         # spec.
         newRemapped = list()

         # This is a count/index for the newRemapped() array.
         remappedCount = 0

         # Parse entries from the 'Remapped:' field in the *.cbdsst file.
         remappedEntriesStarted = False

         for line in oldDesc.split('\n'):
            if (re.match ('Remapped\:', line)):
               remappedEntriesStarted = True
               continue
            if (remappedEntriesStarted == False):
               continue

            if (not re.match ('\t', line)):
                break

            line = line.strip()
            line = line.rstrip()
            self.log.debug ("Remapped Entry: %s" % line)
            newRemapped.append(line)

            remappedCount = remappedCount + 1

         if (remappedCount > 0):
             self.log.debug ("Adding these Remapped field entries: %s" % newRemapped)
             sData['Remapped'] = newRemapped

      if self.VersionIgnoredField:
         # The newIgnored() array includes data from the 'Ignored:' field of the stream
         # spec.
         newIgnored = list()

         # This is a count/index for the newIgnored() array.
         ignoredCount = 0

         # Parse entries from the 'Ignored:' field in the *.cbdsst file.
         ignoredEntriesStarted = False

         for line in oldDesc.split('\n'):
            if (re.match ('Ignored\:', line)):
               ignoredEntriesStarted = True
               continue
            if (ignoredEntriesStarted == False):
               continue

            if (not re.match ('\t', line)):
                break

            line = line.strip()
            line = line.rstrip()
            self.log.debug ("Ignored Entry: %s" % line)
            newIgnored.append(line)

            ignoredCount = ignoredCount + 1

         if (ignoredCount > 0):
             self.log.debug ("Adding these Ignored field entries: %s" % newIgnored)
             sData['Ignored'] = newIgnored

      self.log.debug ("SDATA2: %s" % sData)

      try:
         self.log.debug ("Saving stream spec data %s: %s" % (self.stream, sData))
         self.p4.save_stream(sData)

      except P4Exception:
         userMessage = "Your attempt to submit changelist %s has been rejected because the stream spec failed to update on the server.  It may have been rejected by the Perforce server.\n\nThe error from the Perforce server is:\n" % self.myOptions.changelist
         print (userMessage)
         self.log.fatal("Failed to save stream spec.  User error message is:\n%s\n" % userMessage)

         for e in self.p4.errors:
            print (e)
            self.log.error("Errors: " + e)
         for w in self.p4.warnings:
            print (w)
            self.log.warn("Warnings: " + w)

         return False

      # Next, update the 'p4 keys' on the server.
      # First, generate a list of existing keys for this stream to remove.
      keyNameBase = 'cbd_stream_%s' % self.stream
      keyNameBase = re.sub ('//', '', keyNameBase)
      keyNameBase = re.sub ('/', '_', keyNameBase)
      pathKeySearch = keyNameBase
      vSpecKeySearch = keyNameBase
      pathKeySearch = '%s_path*' % pathKeySearch
      vSpecKeySearch = '%s_vspec*' % vSpecKeySearch
      pathKeysData = self.p4.run_keys('-e', pathKeySearch)
      vSpecKeysData = self.p4.run_keys('-e', vSpecKeySearch)

      for keyData in pathKeysData:
         self.log.debug ("Running: p4 key -d %s" % keyData['key'])

         try:
            self.p4.run_key('-d', keyData['key'])

         except P4Exception:
            self.log.fatal("Failed to delete key [%s] from server." % keyData['key'])

            for e in self.p4.errors:
               self.log.error("Errors: " + e)
            for w in self.p4.warnings:
               self.log.warn("Warnings: " + w)

      for keyData in vSpecKeysData:
         self.log.debug ("Running: p4 key -d %s" % keyData['key'])

         try:
            self.p4.run_key('-d', keyData['key'])

         except P4Exception:
            self.log.fatal("Failed to delete key [%s] from server." % keyData['key'])

            for e in self.p4.errors:
               self.log.error("Errors: " + e)
            for w in self.p4.warnings:
               self.log.warn("Warnings: " + w)

      # Finally, generate the new keys.
      i = 0
      for pathEntry in paths:
         pathKeyName = "%s_path%d" % (keyNameBase, i)
         vSpecKeyName = "%s_vspec%d" % (keyNameBase, i)
         if (pathEntry[0] == 'share'):
            pathKeyValue = pathEntry[1]
            vSpecKeyValue = '#head'
         if (pathEntry[0] == 'import'):
            pathKeyValue = pathEntry[2]
            vSpecKeyValue = pathEntry[3]
         if (pathEntry[0] == 'import+'):
            pathKeyValue = pathEntry[2]
            vSpecKeyValue = pathEntry[3]

         self.log.debug ("Running: p4 key %s %s" % (pathKeyName, pathKeyValue))

         try:
            self.p4.run_key(pathKeyName, pathKeyValue)

         except P4Exception:
            self.log.fatal("Failed to create path key [%s] with value [%s]." % (pathKeyName, pathKeyValue))

            for e in self.p4.errors:
               self.log.error("Errors: " + e)
            for w in self.p4.warnings:
               self.log.warn("Warnings: " + w)

         self.log.debug ("Running p4 key %s %s" % (vSpecKeyName, vSpecKeyValue))

         try:
            self.p4.run_key(vSpecKeyName, vSpecKeyValue)

         except P4Exception:
            self.log.fatal("Failed to create vspec key [%s] with value [%s]." % (vSpecKeyName, vSpecKeyValue))

            for e in self.p4.errors:
               self.log.error("Errors: " + e)
            for w in self.p4.warnings:
               self.log.warn("Warnings: " + w)

         i = i + 1

      return True

   def update_modified_stream_specs (self):
      """Update stream specs for the given changelist."""

      if (self.get_sst_files()):
         for file in self.sst_files:
            if (not self.update_stream_spec_and_keys (file)):
               self.log.debug ("Processing complete.  Changelist %s REJECTED." % self.myOptions.changelist)
               return False

         self.log.debug ("Processing complete.  Changelist %s ACCEPTED." % self.myOptions.changelist)
         return True
      else:
         self.log.warn ("No actionable *.cbdsst files found in changelist %s.  Allowing change to submit." % self.myOptions.changelist)
         self.log.debug ("Processing complete.  Changelist %s ACCEPTED." % self.myOptions.changelist)
         return True

if __name__ == '__main__':
   """ Main Program
   """
   main = Main(*sys.argv[1:])

   if (not Main.initP4(main)):
      sys.exit (1)

   if (Main.update_modified_stream_specs(main)):
      sys.exit (0)
   else:
      sys.exit (1)