p4genreview.py #1

#!/usr/bin/python
#
# p4genreview
# Perforce generic review daemon
#
# Copyright (c) 2000 by Thomas Quinot
# Derived from the Perforce review daemon:
#   Copyright (c) 1999, 2000 by Perforce Software (?)
#
# $Id: //guest/thomas_quinot/perforce/utils/genreview/p4genreview.py#1 $
#
# This script determines new changelists and jobs and emails interested users.
# Users express interest in reviewing changes and/or jobs by setting the
# "Reviews:" entry on their user form.  Users are notified of changes if they
# review any file involved in that change.  Users are notified of jobs if
# they review //depot/jobs (configurable - see jobpath, below).
#
# NOTE: the job review function requires a 98.2 or later server.
# NOTE: the "p4 counter" function used in this daemon requires
# a 99.1 or later server.
#
# If run directly with "repeat=1" (see below) the script will sleep for
# "sleeptime" seconds and then run again.  On UNIX you can run the script from
# cron by setting "repeat=0" and adding the following line with crontab -e:
# * * * * * /path/to/p4review.py
# This will run the script every minute.  Note that if you use cron you
# should be sure that the script will complete within the time allotted.
#
# See the CONFIGURATION VARIABLES section below for other values which may
# need to be customized for your site.  In particular, be sure to set
# administrator, mailhost, repeat, p4, P4PORT and P4USER.
#
# Common pitfalls and debugging tips
# - "command not found" (Windows) or "name: not found" (UNIX) errors
#   -> check that "p4" is on your PATH or set
#      p4='"c:/program files/perforce/p4"' or as appropriate (Windows)
#         (NOTE the use of " inside the string to prevent interpretation
#         of the command as "run c:/program with arguments files/perforce/p4...)
#      p4='/usr/local/bin/p4' or as appropriate (UNIX)
# - "You don't have permission for this operation"
#   -> check that the user you set os.environ['P4USER'] to (see below)
#      has "review" or "super" permission (use "p4 protect")
#      You should be able to run "p4 -u username review -c 42 -t test"
#      - this sets the value of a counter named "test" to 42
# - "Unable to connect to SMTP host"
#   -> check that the mailhost is set correctly - try "telnet mailhost 25"
#      and see if you connect to an SMTP server.  Type "quit" to exit
# - it seems to run but you don't get email
#   -> check the output of "p4 counters" - you should see a counter named
#      "review"
#   -> check the output of "p4 reviews -c changenum" for a recent change;
#      if no one is reviewing the change then no email will be sent.
#      Check the setting of "Reviews:" on the user form with "p4 user"
#   -> check that your email address is set correctly on the user form
#      (run "p4 reviews" to see email addresses for all reviewers,
#       run "p4 user" to set email address)
# - multiple job notifications are sent
#   -> the script should be run on the same machine as the Perforce server;
#      otherwise time differences between the two machines can cause problems
#      (the job review mechanism uses a timestamp; the change review mechanism
#       uses change numbers, so it's not affected by this problem)

import sys, os, string, re, time, rfc822, smtplib, p4wrap

# CONFIGURATION VARIABLES
notify_changes = 1     # set to 0 to disable change notification completely
notify_jobs    = 0     # set to 0 to disable job notification completely
bcc_admin      = 1     # set to 0 to disable Bcc: of all email to administrator
send_to_author = 1     # set to 1 to have mail sent to author of change or job
reply_to_admin = 0     # set to 1 to enable Reply-To: administrator
administrator  = 'perforce'  # If you set this to an email address then you will be
                       # notified of problems with the script (e.g. invalid
                       # email addresses for users) and, if bcc_admin is set,
                       # you will get a copy of all email the script generates
#mailhost = 'localhost' # set to hostname of machine running an SMTP server
p4='p4'                # set to path of p4 executable, or just 'p4' if the
                       # executable is on your path (use forward slashes even
                       # on Windows! Backslash has a special meaning in Python)
repeat = 1             # set to 0 to run just once (do this if running from
                       # cron!), set to 1 to repeat every sleeptime seconds
sleeptime = 60         # number of seconds to sleep between invocations
                       # (irrelevant if repeat == 0)
limit_emails = 10      # don't send more than this many emails of each type
                       # (job and change) at a time
datefield = 'Date'     # field used to determine job updates
                       # ***Currently you must set this to a field which has the
                       # last modified date rather than creation date (which
                       # means you'll be notified of changed jobs as well as
                       # new ones), since Perforce sets the creation date when
                       # the editor is launched, not when the job is stored.
                       # (Hopefully this will be fixed in future).
jobpath = '//depot/jobs' # send job review mail to users reviewing jobpath
counter = "greview"    # The Perforce counter used to keep track of which
                       # changes have been reviewed.
jobcounter = "gjobreview"
config_file = "genreview.conf"
#os.environ['P4PORT'] = 'melchior:1666'
#os.environ['P4USER'] = 'quinot' # user must have Perforce review privileges
# END OF CONFIGURATION VARIABLES

class Fileset:

  filesets = {}
  
  def __init__ (self, name):
    self.name = name;
    # The name of the file set.
    
    self.filespecs = []
    # A list of compiled regexps corresponding
    # to the file specifications for this file set.
    
    self.action = ""
    # The action to be performed when reviewing a
    # change that affects any file matched by any
    # of the filespecs. A string executed with 'system'.
    # All occurrences of %changelist% are replaced
    # with the changelist number.
    
    self.mailto = rfc822.AddressList (None)
    # A list of destinations to mail the output of
    # the action to.

    self.include_reviewers = []
    # A list of strings suitable for use as argument of
    # p4 reviews or "-c " followed by a change list number.
    # Keyword expansion is performed before the command
    # is executed.
    
    self.options = {
    }
    # Options. Nothing yet.

    Fileset.filesets [name] = self
    
  def add_filespec (self, filespec):
    self.filespecs.append (p4wrap.fs2re (string.strip (filespec)))

  def add_action (self, action):
    self.action = self.action + string.lstrip (action)

  def add_mailto (self, address):
    m = re.match ("^reviewers(\(.*\))?", address)
    if m:
      if not m.group (1):
        self.include_reviewers.append ("-c %change%")
        # Include the reviewers of the current change in
        # the recipient list.
      else:
        self.include_reviewers.append (m.group (1))
        # Include the reviewers of the specified file spec
        # in the recipients list.
    else:
      self.mailto = self.mailto + rfc822.AddressList (address)

  def set_option (self, name, value):
    self.options [name] = value
    
  def in_fileset (self, file):
    for fs in self.filespecs:
      if fs.match (file):
        return 1
    return 0
  
def parse_config (f):
  global mailhost, administrator
  
  current = None
  
  input = open (f, "r").readlines()
  line = 0
  for i in input:
    line = line + 1
    if re.match ("^#", i) or re.match ("^\s*$", i):
      continue
    m = re.match ("(?i)^fileset\s+(.*):$", i)
    if m:
      current = Fileset (m.group (1))
      state = 'fileset'
      continue
    m = re.match ("(?i)^action:$", i)
    if m:
      state = 'action'
      continue
    m = re.match ("(?i)^mailto:$", i)
    if m:
      state = 'mailto'
      continue
    m = re.match ("(?i)^mailhost:$", i)
    if m:
      state = 'mailhost'
      continue
    m = re.match ("(?i)^administrator:$", i)
    if m:
      state = 'admin'
      continue
    m = re.match ("(?i)^environment:$", i)
    if m:
      state = 'env'
      continue
    if not re.match ("^\t", i):
      print i
      print "Syntax error at line %i: tabulation expected" % (line)
      sys.exit (1)
    if state == 'fileset':
      current.add_filespec (i)
    elif state == 'action':
      current.add_action (i)
    elif state == 'mailto':
      current.add_mailto (string.strip (i))
    # Global options
    elif state == 'mailhost':
      mailhost = string.strip (i)
    elif state == 'admin':
      administrator = string.strip (i)
    elif state == 'env':
      m = re.match ("^\t([^\s=]*)=(.*)$", i)
      if not m:
        print i
        print "Environment variable setting expected at line %i." % (line)
        sys.exit (1)
      os.environ[m.group(1)] = m.group (2)
    else:
      print i
      print "Unexpected input at line %i." % (line)
      sys.exit (1)

def desc2files (desc):
  files = []
  for f in desc['files']:
    files.append (f[0])
  return files

def files_intersect_fileset (files, fileset):
  for f in files:
    if fileset.in_fileset (f):
      return 1
  return 0

def make_recipients (fs, desc, ignore_author=None):
  res = fs.mailto
  for rev in fs.include_reviewers:
    res = res + reviewers (p4wrap.sub_keywords (rev, desc),
                           ignore_author)
  return res

def dest_addrs (recipients):
  dests = []

  for dfn, d in recipients.addresslist:
    dests.append (d)
  return dests

bcc_admin = bcc_admin and administrator # don't Bcc: None!
if administrator and reply_to_admin:
  reply_to = administrator
else:
  reply_to = None


def complain(mailport,complaint):
  '''Send a plaintive message to the human looking after this script if we
  have any difficulties.  If no email address for such a human is given,
  send the complaint to stderr.
  '''
  complaint = complaint + '\n'
  if administrator:
    mailport.sendmail('PerforceReviewDaemon',[administrator],\
      'Subject: Perforce Review Daemon Problem\n\n' + complaint)
  else:
    sys.stderr.write(complaint)

def parse_action_output (output):
  headers = {}
  current_header = ""
  state = 'hdr'
  body = []
  for i in output:
    if state == 'hdr':
      m = re.match ('^\s+(.*)$', i)
      if m:
        if current_header == '':
          state = 'body'
          body.append (i)
          continue
        else:
          headers [current_header] =\
            headers [current_header] + ' ' + m.group (1)
          continue
      m = re.match ('^([\S]+):\s*(.*)$', i)
      if m:
        current_header = string.lower (m.group (1))
        headers [current_header] = m.group (2)
        continue
      state = 'body'
      if not i == '':
        body.append (i)
    else:
      body.append (i)
  return headers, body

def capitalize (s):
  result = ''
  tab = re.split ('(\W)', s)
  for t in tab:
    result = result + string.capitalize (t)
  return result

def make_headers (headers, default_headers):
  s = ''
  for d in default_headers.keys():
    if not headers.has_key (d):
      headers[d] = default_headers[d]
  for h in headers.keys():
    s = s + capitalize (h) + ': ' + headers [h] + '\n'
  return s

def mailit(mailport, sender, recipients, message):
  '''Try to mail message from sender to list of recipients using SMTP object
  mailport.  complain() if there are any problems.
  '''
  try:
    failed = mailport.sendmail(sender, recipients, message)
  except:
    failed = 'Exception ' + repr(sys.exc_info()[0]) + ' raised.'

  if failed:
    complain( mailport, 'The following errors\n' +\
               repr(failed) +\
              '\noccurred while trying to email from\n' + repr(sender) + '\nto ' +\
               repr(recipients) + '\nwith body\n\n' + message)

def reviewers(what,ignore_author=None):
  '''For a given object (file spec, or change list number preceded
  with "-c ", return list of reviewers' email addresses.
  If ignore_author is given then the given user will not be included
  in the lists.
  '''
  return p4wrap.parse_p4_review \
    (os.popen (p4 + ' reviews ' + what).readlines(),\
     ignore_author)


def review_changes(mailport,limit_emails=100):
  '''For each change which hasn't been reviewed yet send email to users
  interested in reviewing the change.  Update the "review" counter to
  reflect the last change reviewed.  Note that the number of emails sent
  is limited by the variable "limit_emails"
  '''
  change = None

  for line in os.popen(p4 + ' review -t ' + counter,'r').readlines():
    # sample line: Change 119424 james <james@perforce.com> (James Strickland)
    #                     change author email                fullname
    (change,author,email,fullname) = \
      re.match( r'^Change (\d+) (\S+) <(\S+)> \(([^\)]+)\)', line).groups()

    desc = p4wrap.parse_p4_describe \
      (os.popen(p4 + ' describe -s ' + change, 'r').readlines ())
    files = desc2files (desc)

    for fsk in Fileset.filesets.keys():
      fs = Fileset.filesets[fsk]
      if files_intersect_fileset (files, fs):
        action = p4wrap.sub_keywords (string.rstrip (fs.action), desc)
        action = re.sub ("%fileset%", fs.name, action)
        output = os.popen (action, 'r').readlines ()
        
        recipients = make_recipients (fs, desc)
        if bcc_admin: recipients = recipients + rfc822.AddressList(administrator)
        if not (recipients and output): continue
        # no one is interested, or action produced no output.
        
        headers, body = parse_action_output (output)
        
        default_headers = { \
          'from': fullname + ' <' + email + '>',
          'to': recipients.__str__ (),
          'subject': 'PERFORCE change ' + change + ' for review',
          'x-p4genreview-fileset': fs.name }
          
        if reply_to:
          default_headers['reply-to'] = reply_to
          
        message = make_headers (headers, default_headers) +\
          '\n' +\
          string.join (body, "")

        mailit(mailport, email, dest_addrs (recipients), message)
        
        limit_emails = limit_emails - 1
        if limit_emails <= 0:
          complain( mailport, 'email limit exceeded in job review - extra jobs dropped!')
          break

    # if there were change(s) reviewed in the above loop, update the counter
    if change:
      # NOTE: the use of "p4 review -c" is for backwards compatibility with
      # pre-99.1 servers; with 99.1 or later servers use "p4 counter"
      if os.system(p4 + ' counter ' + counter + ' ' + change + " > /dev/null") !=0:
        complain(mailport,'Unable to set ' + counter + ' counter - check user "' +\
        os.environ['P4USER'] + '" has review privileges\n(use p4 protect)')

def job_reviewers(jobname,ignore_author=None):
  '''For a given job, return list of reviewers' email addresses,
  plus a list of email addresses + full names.
  If ignore_author is given then the given user will not be included
  in the lists.
  '''
  return reviewers (jobpath, ignore_author)

def review_jobs(mailport,limit_emails=100):
  '''For each job which hasn't been reviewed yet send email to users
  interested in reviewing the job.  Update the "jobreview" counter to
  reflect the last time this function was evaluated.  Note that the number
  of emails sent is limited by the variable "limit_emails" - ***currently
  this causes extra job notifications to be dropped...not optimal...
  '''
  start_time = 0
  for line in os.popen(p4 + ' counters').readlines():
    if line[:len(jobcounter)] == jobcounter:
      start_time = int(line[len(jobcounter)+3:])
  query_time = int(time.time())
  query = datefield + '>' +\
          time.strftime('%Y/%m/%d:%H:%M:%S',time.localtime(start_time)) + '&' +\
          datefield + '<=' +\
          time.strftime('%Y/%m/%d:%H:%M:%S',time.localtime(query_time))

  for line in os.popen(p4 + ' jobs -e "' + query + '"','r').readlines():
    # sample line: job000001 on 1998/08/10 by james *closed* 'comment'
    #              jobname      date          author
    (jobname,author) = re.match( r'^(\S+) on \S+ by (\S+)', line).groups()
    (email,fullname) = re.match( r'^\S+ <(\S+)> \(([^\)]+)\)', \
      os.popen(p4 + ' users ' + author,'r').read() ).groups()

    if send_to_author:
      recipients = job_reviewers(jobname)
    else:
      recipients = job_reviewers(jobname,author)

    if bcc_admin: recipients.append(administrator)

    if not recipients: continue  # no one is interested

    message = 'From: ' + fullname + ' <' + email + '>\n' +\
              'To: ' + recipients + '\n' +\
              'Subject: PERFORCE job ' + jobname + ' for review\n' +\
              replyto_line +\
              '\n'
    for line in os.popen(p4 + ' job -o ' + jobname,'r').readlines():
      if line[0] != '#': message = message + line

    mailit(mailport, email, recipients, message)
    limit_emails = limit_emails - 1
    if limit_emails <= 0: break

  # NOTE: the use of "p4 review -c" is for backwards compatibility with
  # pre-99.1 servers; with 99.1 or later servers use "p4 counter"
  if os.system(p4 + ' review -c ' + repr(query_time) + ' -t ' + jobcounter) !=0:
    complain(mailport,'Unable to set ' + jobcounter + ' counter - check user "' +\
      os.environ['P4USER'] + '" has review privileges\n(use p4 protect)')

def format_exception(type=None, value=None, tb=None, limit=None):
    if type is None:
        type, value, tb = sys.exc_info()
    import traceback
    list = traceback.format_tb(tb, limit) + \
           traceback.format_exception_only(type, value)
    s = string.join(list[:-1], "") + list[-1]
    del tb
    return s

def loop_body(mailhost):
  # Note: there's a try: wrapped around everything so that the program won't
  # halt.  Unfortunately, as a result you don't get the full traceback.
  # If you're debugging this script, strip off the special exception handlers
  # to get the real traceback, or try figuring out how to get a real traceback,
  # by importing the traceback module and defining a file object that
  # will take the output of traceback.print_exc(file=mailfileobject)
  # and mail it (see the example in cgi.py)
  try:
    mailport=smtplib.SMTP(mailhost)
  except:
    sys.stderr.write('Unable to connect to SMTP host "' + mailhost + '"!\n')
    if repeat:
      sys.stderr.write ('Will try again in ' + repr(sleeptime) + ' seconds.\n')
  else:
    try:
      if notify_changes: review_changes(mailport,limit_emails)
      if notify_jobs: review_jobs(mailport,limit_emails)
    except:
      complain(mailport,'Exception raised:' + format_exception())
    try:
      mailport.quit()
    except:
      sys.stderr.write('Error while doing SMTP quit command (ignore).\n')


if __name__ == '__main__':
  if len (sys.argv) == 2:
    config_file = sys.argv[1]
  parse_config (config_file)
  while(repeat):
    loop_body(mailhost)
    time.sleep(sleeptime)
  else:
    loop_body(mailhost)