mkdepot.pl #8

#!/usr/bin/perl -w
#
# $Id: //guest/jeffery_g_smith/perforce/utils/sttop4/main/mkdepot.pl#8 $
#
# StarTeam to Perforce Converter, phase III: construct Perforce depot
#
# Copyright 1998 Perforce Software.  All rights reserved.
# Based on VSStoP4:
#   Written by James Strickland, April 1998
#   Maintained by Robert Cowham, since 2000
# Updated to support StarTeam conversions:
#   Jeffery G. Smith, MedPlus, Inc. 2004-2005
#
# This script uses the metadata produced by earlier phases to direct a loop
# which extracts StarTeam revisions and performs the required Perforce commands
# to construct a Perforce depot corresponding to the (improved) StarTeam data.
#
# RHGC - Modifed to use new depot and depot_root as specified in config.
#      - Modified to use DB_File for speed with large data sets (and improve label handling)
#      - Modified to use P4Perl interface and VSS OLE Automation for speed.
# RV   - Modified to speed up label processing for large changelists

require 5.0;
use strict;
use integer;
use lib '.';
use convert;
use Change;
use Env;
use File::Path;

use Win32::OLE;
use Win32::OLE qw(in);
use Win32::OLE::Const;
use Win32::OLE::Variant;

convert::openlog('open');

# Check for P4 module installed
my $p4;
my $p4form;

# Check that P4 Module is installed
eval{require P4; import P4;};
if ($@) {
  print "\nP4Perl module (P4) is required for the converter to run.

Install P4Perl from http://public.perforce.com//guest/tony_smith/perforce/API/Perl/index.html.

The latest versions of P4Perl may work fine, but the following are known to work just
double-click on Windows installer and run it (no real need to download and save). E.g.:

http://public.perforce.com/guest/tony_smith/perforce/API/Perl/released/p4perl56-setup-2.4320.exe
  or
http://public.perforce.com/guest/tony_smith/perforce/API/Perl/released/p4perl58-setup-2.4320.exe
\n";

  exit(1);
}

$p4 = new P4;
$p4->SetPort($convert::p4port);
$p4->SetClient($convert::p4client);
$p4->SetUser($convert::p4user);
$p4->MergeErrors(1);

$p4->Connect() or die( "Failed to connect to Perforce Server: $convert::p4port $!\n" );

$p4form = new P4;
$p4form->SetPort($convert::p4port);
$p4form->SetClient($convert::p4client);
$p4form->SetUser($convert::p4user);
$p4form->ParseForms();
$p4form->MergeErrors(1);

$p4form->Connect() or die( "Failed to connect to Perforce Server: $convert::p4port $!\n" );

# See if we want to and can initialise VSS OLE interface
my $VSSDB;
if ($convert::vss_use_ole) {
  eval {$VSSDB = Win32::OLE->new('SourceSafe', 'Quit');};
  if (!$@) {
    print "Using VSS OLE Automation interface\n";

    my $SrcSafeIni = $ENV{"SSDIR"}."\\SRCSAFE.INI";
    Win32::OLE->Option(Warn => 3);  # Die on exceptions (with error message), e.g. if password wrong
    $VSSDB->Open($SrcSafeIni, $convert::vss_user, $convert::vss_password);
  }
}


# open all our input files
my $msg="can't open for read";
open(FILES,   "<$convert::metadata_dir/files")      or die $msg;
open(LABELS_SUMMARY,  "<$convert::metadata_dir/labels_summary")     or die $msg;
open(CHANGES, "<$convert::metadata_dir/changes")    or die $msg;
open(MAPPING, ">$convert::metadata_dir/mapping.ns") or die "can't open for write: $!";

# variables to be initialized with metadata read from files
my (%file_type,    # maps archive            -> file type
                   # e.g. "$/foo"            -> "text"
                   # Now a DB_File

    %labels,       # List of all labels

    %label_files,  # maps "revision archive" -> list of labels
                   # e.g. "1 $/foo"          -> "itworks","bobsyouruncle"
                   # Now a DB_File

    %added,        # maps depot file         -> true/false
                   # e.g. "//depot/foo"      -> 1
                   # Now a DB_File
    );

# Check for DB_File module installed - much faster for larger repositories if so
my $USEDB_FILE = 0;
eval{require DB_File; import DB_File;};
if (!$@) {
  $USEDB_FILE = 1;
  # Local read/write ones DB files
  tie %added, "DB_File", "$convert::metadata_dir/db.added", O_RDWR()|O_CREAT(), 0666, $DB_File::DB_HASH
          or die "Cannot open file '$convert::metadata_dir/db.added': $!\n";
  tie %file_type, "DB_File", "$convert::metadata_dir/db.file_type", O_RDWR()|O_CREAT(), 0666, $DB_File::DB_HASH
          or die "Cannot open file '$convert::metadata_dir/db.file_type': $!\n";

  # This one is created by readhist.pl
  tie %label_files, "DB_File", "$convert::metadata_dir/db.labels", O_RDONLY(), $DB_File::DB_HASH
          or die "Cannot open file '$convert::metadata_dir/db.labels': $!\n";
}
else {
  # We have to read the contents of the label file
  open(LF,"<$convert::metadata_dir/db.labels") or die $msg;
  while (<LF>) {
    chomp;
    my ($key,$val) = split(/#/,$_,2);
    $label_files{$key} = $val;
  }
  close(LF);
}

print "Reading file list\n";
# initialize %file_type
while(<FILES>) {
  chomp;
  my ($file_type,$file) = split(/ /,$_,2);
  $file_type{$file} = $file_type;
}
close(FILES);


print "Creating labels\n";

# initialize %label and run p4 label once for each label which is not a
# "delete" label
while(<LABELS_SUMMARY>) {
  chomp;
  my ($label,$timestamp) = split(/ /,$_,2);

  my $comment = '';
  $comment .= <LABELS_SUMMARY> while ($comment !~ /\*{10}\n/);
  $comment =~ s/\*{10}\n$//;

  $comment = "STtoP4 label" if ($comment =~ /^\s*$/);  # Set default comment.

  my $label_view = "\"//$convert::depot/$convert::depot_root/...\"";

  # adjust the update time of the label
  my (@tm,$date);
  @tm=localtime($timestamp);
  $date=sprintf("%4d/%02d/%02d %02d:%02d:%02d",(($tm[5]>=70) ? $tm[5]+1900 : $tm[5]+2000),
                                                $tm[4]+1,$tm[3],$tm[2],$tm[1],$tm[0]);

  my ($form, @view, @result);

  $form = $p4form->FetchLabel($label);
  $form->{'Description'} = $comment;
  push @view, $label_view;
  $form->{'View'} = \@view;
  $form->{'Update'} = $date;
  @result = $p4form->SaveLabel($form);
  convert::log("Label update:\n".join("\n", @result));
  print_p4errors($p4form, "Updating label $label");

  $labels{$label} = 1;
  unlink ("$convert::metadata_dir/labels/$label");
}
close(LABELS_SUMMARY);

# print timestamp
print "Depot creation started         " . scalar(localtime()) . "\n";

# For each change,
# - retrieve revisions from VSS using 'get'
# - mark files as added or edited with 'p4 add' or 'p4 edit',
#   followed by 'p4 submit'
# - update list of file#rev associated with any pertinent labels

my $start_time = $convert::start_time;
my ($c,$op);
my %label_hash;
CHG: while( $c = get Change(\*CHANGES) ) {

  %label_hash = ();
  my ($change_number,@checkins);

  # Ignore change if a start_time is specified
  if ($start_time > $c->timestamp)
  {
    print "Ignoring changelist with timestamp: ". $c->datetime ."\n";
    next CHG;
  }

  my $index;
  foreach $index (@{$c->changelist}) {

    my $p4rev;

    my ($revision,$vss_file) = split(/ /,$index,2);

    my ($client_rel_dir,$client_file) = convert::p4dir_and_file( $vss_file);

    my $client_dir       = convert::join_paths( $convert::client_root,
                                                $client_rel_dir );

    # create all needed directories on the path
    mkpath($client_dir);

    # for p4 - could use $client_file, but I'd rather make sure we use
    # the same string that's written out to the mapping file
    my $depot_file       = convert::join_paths("//$convert::depot",
                                               $client_rel_dir,
                                               $client_file );

    # get the file from VSS
    if ($VSSDB) {
      unlink("${client_dir}/${client_file}");
      my $item = $VSSDB->VSSItem($vss_file, 0);
      my $version = $item->Version(($file_type{$vss_file} eq "tempobj") ? "" : "$revision");
      $version->Get("${client_dir}/${client_file}");
    }
    else {
      convert::get_vss_file($vss_file,($file_type{$vss_file} eq "tempobj") ? "" : $revision,$client_dir,$client_file);
    }

    # see if the file is indeed there now
    if (-f "${client_dir}/${client_file}") {

      # schedule p4 add or edit operation
      # depending on whether file has already been added

      push(@checkins, [ $vss_file,$revision,$depot_file ] );
      # RHGC - because of restartability, add extra check
      if($added{$depot_file}) {
        p4exec($depot_file, "edit") =~ /#(\d+) - opened for edit/ or die "p4 edit $depot_file failed";
        $p4rev=$1 +1;
      }
      elsif (p4exec($depot_file, "files") !~ /no such file\(s\)/) {
        p4exec($depot_file, "edit") =~ /#(\d+) - opened for edit/ or die "p4 edit $depot_file failed";
        $added{$depot_file}=1;
        $p4rev=$1 +1;
      } else {
        my @addcmd = ("add", "-t", "$file_type{$vss_file}");
        if ($convert::typemap_regexp) {
          # don't specify type when file is handled with p4 typemap
          if ($depot_file =~ /$convert::typemap_regexp/) {
            @addcmd = ("add");
          } else {
            print "file $depot_file not matched by typemap.\n";
          }
        }
        p4exec($depot_file, @addcmd) =~ /opened for add/ or die "p4 add $depot_file failed";
        $added{$depot_file}=1;
        $p4rev=1;
      }

      # check if this revision is labelled
      if(exists($label_files{$index})) {
        my $labels = $label_files{$index};
        my @label_list = split(/ /, $labels);
        for (@label_list) {
          $label_hash{"$convert::metadata_dir/labels/$_"} .= "$depot_file#$p4rev\n";
        }
      }

    } else {  # the file isn't there - the ss get failed

      die "get_vss_file() revision $revision to $client_dir/$client_file failed";

    }
  }

  # update the label files with the revision information collected above
#  print "\nUpdating label metadata        " . scalar(localtime()) . "\n";
  foreach my $label_file (keys %label_hash) {
  	open(LABELFILE,">>$label_file") or die "can't open label file $label_file: $!\n";
  	print LABELFILE $label_hash{$label_file};
  	close(LABELFILE);
  }

  # submit the change, and write out the association between VSS
  # archive and revision number and Perforce file and change number
  $change_number = p4submit($c); # submit the change

  my $checkin;
  foreach $checkin (@checkins) {
    my ($vss_file,$r,$depot_filename) = @$checkin;
    # write change number#filename to avoid confusion with filename#revision
    print MAPPING "$r#$change_number#$depot_filename#$vss_file\n";
    # NOTE: $vss_file has to go last since it can contain # characters
  }
}

# now do the labelling
print "\nLabelling started              " . scalar(localtime()) . "\n";
for (keys(%labels)) {
  convert::p4run(" -x $convert::metadata_dir/labels/$_ labelsync -l $_");
}

if ($USEDB_FILE) {
  untie %label_files;
  untie %added;
  untie %file_type;
}

$p4->Final();
$p4form->Final();

# Run appropriate version of the command depending on if P4Perl is installed.
sub p4exec
{
  # Need to distinguish filename parameter as it might need quoting in case of spaces
  my $fname = shift;
  my @cmd = @_;
  my ($r, @result);

  push @cmd, $fname;
  @result = $p4->Run(@cmd);
  if ($p4->ErrorCount()) {
    foreach my $e ($p4->Errors()){
      if (scalar(@$e)) {
        push @result, @$e;
      }
      else {
        push @result, $e;
      }
    }
  }
  if($convert::debug_level > 0) {
    convert::log("p4 cmd: ".join(" ", @cmd));
    if (!@result || $#result < 0) {
      convert::log("result blank");
    }
  }
  return @result if (!@result);
  $r = join(" ", @result);
  if($convert::debug_level > 0) {
    convert::log("result: ".$r);
  }
  return $r;
}

# Run appropriate version of submit
sub p4submit
{
  my $c = shift;
  my ($change_number);

  my $change_description = $c->change_description;
  my ($form,$output,@result);

  $form = $p4form->FetchChange();
  if ($form->{'Files'}) {
    $form->{'Description'} = $change_description;
    @result = $p4form->SaveSubmit($form);
    $output = join("\n", @result);
    convert::log("Submit result:\n$output\n");
    print_p4errors($p4form, "Submitting change");

    # 2 forms of result - check for which one and extract the resulting change number.
    if( $output =~ m/Change ([0-9]+) submitted./si ) {
      $change_number = $1;
    } elsif ( $output =~ m/Change [0-9]+ renamed change ([0-9]+) and submitted./si ) {
      $change_number = $1;
    }
    else {
      die "p4 submit aborted - conversion terminated. Output was:\n$output";
    }


    # fix date, user on submitted change
    my $user = $c->author;
    my $date = $c->datetime;

    $form = $p4form->FetchChange($change_number);
    $form->{'Date'} = $date;
    $form->{'User'} = $user;
    @result = $p4form->SaveChange($form, "-f");
    print_p4errors($p4form, "Updating change");

    print "Change $change_number submitted.\r"; # running total..
  }
  else {
    print "WARNING: Change $change_number empty.\r";
  }
  return $change_number; # returns the change number
}

sub print_p4errors
{
  my ($p4, $msg) = @_;
  if ($p4->ErrorCount()) {
    print "**p4errors - $msg:\n";
    foreach my $e ($p4->Errors()){
      print "$e->[0]\n";
    }
  }
}