deviceSync #1

#!/usr/bin/env ruby
# NOTE: This was a prototype script created by Alan Teague for validating the
# 		  Helix Sync have table algorithm.

require 'P4'
require 'pp'
require 'json'
require 'digest'
require 'set'

$have_table_file = '.have_table'
$device_client = ''
$shelf_client = ENV['SHELF_CLIENT']
$shelf_change = ''
$shelf_gen = ''
$shelf_working = {}
$shelf_contents = {}
$shelf_submitted = false
$have_table = {}
$last_change = ''
$depot_sync = {}
$client_sync = {}
$depot_rec = {}

def spew( msg, obj )
  if obj != nil and !obj.empty? then
    puts(msg)
    pp(obj)
  end
end

def local_path( p4, f )
  where = p4.run_where([f])
  if where != nil then
    return where[0]['path']
  else
    return nil
  end
end

def grab_shelf( p4 )
  # take over shelf ownership
  if $shelf_change != '' then
    change = p4.fetch_change($shelf_change)
    change["Client"] = $device_client
    p4.save_change( change )
  end
end

def release_shelf( p4 )
  # take over shelf ownership
  if $shelf_change != '' then
    change = p4.fetch_change($shelf_change)
    change["Client"] = $shelf_client
    p4.save_change( change )
  end
end

def create_shelf( p4 )
  change = p4.fetch_change
  change['Description'] = '{"shelfGen":"0", "working":{}}'
  change["Client"] = $shelf_client
  msg = p4.save_change( change )
  changes = p4.run_changes( ['-m1', "-c", "#{$shelf_client}"] )
  $shelf_change = changes[0]['change']
end

def load_have_table()
  if File.exist?($have_table_file) then
    File.open($have_table_file, "r") do |f|
      $have_table = JSON.load(f)
      $shelf_change = $have_table['shelfNum']
    end
  end
end

def get_last_change( p4 )
  changes = p4.run_changes(['-m1', "//#{$device_client}/..."])
  if !changes.empty? then
    $last_change = changes[0]['change']
  else
    $last_change = '0'
  end
end

def load_shelf( p4 )
  if $shelf_change == '' then
    changes = p4.run_changes( ["-m1", "-s", "shelved","-c", "#{$shelf_client}"] )
    if !changes.empty? then
      $shelf_change = changes[0]['change']
    else
      return
    end
  end
  shelf = p4.run_describe( ['-S', '-s', "#{$shelf_change}"] )
  if shelf[0] == nil then
    # p4 change -o -O 2 -s
    puts("Searching for shelf")
    change = p4.run_change( ["-o", "-s", "-O", "#{$shelf_change}"] )
    pp(change)
    if change != nil and !change.empty? then
      $shelf_change = change[0]['Change']
      puts("...found #{$shelf_change}")
      shelf = p4.run_describe( ['-S', '-s', "#{$shelf_change}"] )
    else
      puts("Bad news, cannot resolve previous shelf")
    end
  end
  shelf = shelf[0]

  # Extract details from shelf desc
  if shelf['status'] == "submitted" then
    $shelf_submitted = true
  end
  shelf_desc = shelf['desc']
  s = JSON.parse(shelf_desc)
  $shelf_gen = s['shelfGen']
  $shelf_working = s['working']

  ['depotFile', 'action', 'type', 'rev', 'fileSize', 'digest', 'fromFile', 'fromRev'].each {
    |k|
    if !shelf.has_key?(k) then
      shelf[k] = Array.new
    end
  }
  shelf_raw = shelf['depotFile'].zip(
    shelf['action'],
    shelf['type'],
    shelf['rev'],
  shelf['fileSize'],
    shelf['digest'],
    shelf['fromFile'],
    shelf['fromRev'])

  $shelf_contents = Hash.new
  shelf_raw.each {
  |e|
    $shelf_contents[e[0]] = e[1..-1]
  }
end

def load_sync_files( p4 )
  files = p4.run_sync(['-n', "//#{$device_client}/...@#{$last_change}"])
  $depot_sync = Hash.new
  $client_sync = Hash.new
  files.each {
  |f|
    $depot_sync[f['depotFile']] = f
    $client_sync[f['clientFile']] = f
  }
end

def load_rec_files( p4 )
  files = p4.run_reconcile(['-n', "//#{$device_client}/..."])
  files.delete_if { |entry| entry.is_a? String }
  $depot_rec = Hash.new
  $depot_to_client_map = Hash.new
  files.each {
  |f|
    $depot_rec[f['depotFile']] = f
    if f['action'] == 'move/add' then
      f['action'] = 'add'
    elsif f['action'] == 'move/delete' then
      f['action'] = 'delete'
    end
    $depot_to_client_map[f['depotFile']] = f['clientFile']
  }
end

def save_have_table()
  have_table = Hash.new
  have_table['shelfGen'] = $shelf_gen
  have_table['shelfNum'] = $shelf_change
  have_table['lastChange'] = $last_change
  have_table['have'] = $shelf_working
  File.open($have_table_file, "w") do |f|
    f.write(JSON.pretty_generate(have_table))
  end
end

def update_disk( p4, updates )
  if updates.empty? then
    return
  end

  grab_shelf( p4 )
  args = ["-s", "#{$shelf_change}"]
  args.concat(updates.to_a)
  stuff = p4.run_unshelve(args)
  args = ["-k"]
  args.concat(updates.to_a)
  stuff = p4.run_revert(args)
  release_shelf( p4 )

  save_have_table
end

def update_shelf( p4, update_to_shelf, reverting )
  if update_to_shelf.empty? and reverting.empty?
    return
  end
  if $shelf_change == '' then
    create_shelf( p4 )
  end
  grab_shelf( p4 )

  updated_working = false
  updated_gen_number = false
  update_to_shelf.each {
  |f|
    stuff = p4.run([$depot_rec[f]['action'], "-c", $shelf_change, f])
  }
  if !update_to_shelf.empty? then
    stuff = p4.run_shelve(["-c", "#{$shelf_change}", "-f"])
    stuff = p4.run_revert(["-k", "//#{$device_client}/..."])
    load_shelf(p4)
    if !updated_gen_number then
      $shelf_gen = Integer($shelf_gen) + 1
      updated_gen_number = true
    end
    update_to_shelf.each {
    |f|
      fileSize = $shelf_contents[f][3]
      fileDigest = $shelf_contents[f][4]
      $shelf_working[f] = ["#{$shelf_gen}", "#{fileSize}", "#{fileDigest}", nil, Time.now.to_i]
    }
    updated_working = true
  end

  if !reverting.empty? then
    reverting.each {
    |f|
      if !$shelf_working[f][3] then
        if !updated_gen_number then
          $shelf_gen = Integer($shelf_gen) + 1
          updated_gen_number = true
        end
        $shelf_working[f][0] = $shelf_gen
        $shelf_working[f][3] = true
        p4.run_shelve(["-d", "-c", "#{$shelf_change}", f])
      end
    }
    updated_working = true
  end

  if updated_working then
    new_desc = Hash.new
    new_desc["shelfGen"] = $shelf_gen
    new_desc["working"] = $shelf_working
    change = p4.fetch_change($shelf_change)
    change["Description"] = JSON.generate(new_desc)
    p4.save_change( change )

    save_have_table
  end

  release_shelf( p4 )
end

p4 = P4.new
$device_client = ENV['P4CLIENT']

puts("pwd: #{Dir.pwd}")
puts("\nDeviceSync:\nDevice: #{$device_client}\nShelf: #{$shelf_client}")

begin
  p4.connect
  p4.exception_level = P4::RAISE_ERRORS

  info = p4.run_info
  puts("info: #{info}")

  # Get HAVE, WORKING(shelf), CONTENTS(shelf), SYNC-N, REC-N, and LAST_CHANGE

  # Run rec -n, then last_change, then sync -n
  #   rec is not bound by change number so run it first
  #   last_change is needed by sync -n
  load_rec_files( p4 )
  get_last_change( p4 )
  load_sync_files( p4 )

  # These two can run in either order and before or after the rec/sync loading
  load_have_table
  load_shelf( p4 )

  if $shelf_submitted and ($last_change < $shelf_change) then
    puts("Start over again, submit happened midstream")
    exit
  end

  # Review disk for changes from server or from shelf
  changed_on_disk = Set.new
  not_changed_on_disk = Set.new
  have = $have_table['have']
  if have == nil then
    have = Hash.new
  end
  $depot_rec.each {
  |f,e|
    if have.key?(f) then
      if have[f][1] == '' then
        if File.exist?(e['clientFile']) then
          puts("#{f}: exists now")
          changed_on_disk.add(f)
        else
          not_changed_on_disk.add(f)
        end
      elsif !File.exist?(e['clientFile']) then
        puts("#{f}: not exists")
        changed_on_disk.add(f)
      elsif Integer(have[f][1]) != File.size(e['clientFile']) or
        have[f][2] != String(Digest::MD5.file(e['clientFile'])).upcase! then
        if Integer(have[f][1]) != File.size(e['clientFile']) then
          puts("#{f}: filesize")
          pp(e['clientFile'])
          pp(File.size(e['clientFile']))
        else
          puts("#{f}: digest")
          pp(e['clientFile'])
          pp(String(Digest::MD5.file(e['clientFile'])))
        end
        changed_on_disk.add( f )
      else
        not_changed_on_disk.add( f )
      end
    else
      changed_on_disk.add( f )
    end
  }
  spew("\nChanged on disk:", changed_on_disk)
  spew("\nNot changed on disk:", not_changed_on_disk)

  changed_on_shelf = Set.new
  not_changed_on_shelf = Set.new
  $shelf_working.each {
  |f,e|
    if have.key?(f) then
      if e == have[f] then
        not_changed_on_shelf.add( f )
      else
        changed_on_shelf.add( f )
      end
    else
      changed_on_shelf.add( f )
    end
  }
  spew("\nChanged on shelf:", changed_on_shelf)
  spew("\nNot changed on shelf:", not_changed_on_shelf)

  update_to_shelf = Set.new
  shelf_conflict = Set.new
  changed_on_disk.each {
  |f|
    if not_changed_on_shelf.include?(f) then
      update_to_shelf.add(f)
    elsif changed_on_shelf.include?(f) then
      shelf_conflict.add(f)
    else
      update_to_shelf.add(f)
    end
  }
  spew("\nUpdate shelf with:", update_to_shelf)
  spew("\nShelf conflicts:", shelf_conflict)

  update_from_shelf = Set.new
  reverted_shelf = Set.new
  changed_on_shelf.each {
  |f|
    if not_changed_on_disk.include?(f) then
      if $shelf_working[f][3] then
        reverted_shelf.add(f)
      else
        update_from_shelf.add(f)
      end
    elsif !changed_on_disk.include?(f) then
      update_from_shelf.add(f)
    # else already in shelf_conflict
    end
  }
  spew("\nUpdate from shelf:", update_from_shelf)

  reverted_disk = Set.new
  reverted_conflicts = Set.new
  have.each {
  |f,e|
    if !$depot_rec.key?(f) then
      if not_changed_on_shelf.include?(f) then
        if !e[3] then
          reverted_disk.add(f)
        end
      elsif changed_on_shelf.include?(f) then
        reverted_conflicts.add(f)
      else
        # should never happen: HAVE record but no WORKING record
        reverted_disk.add(f)
      end
    end
  }
  spew("\nReverted shelf files:", reverted_shelf)
  spew("\nReverted disk files:", reverted_disk)
  spew("\nReverted conflicts:", reverted_conflicts)

  shelf_conflict.each {
  |f|
    puts("Conflict for #{f}")
    local_file = local_path(p4, f)
    if File.exist?(local_file) then
      conflict_name = "#{f}.conflict"
      File.rename(local_file, local_path(p4, conflict_name))
      update_to_shelf.add(conflict_name)
      $depot_rec[conflict_name] = Hash.new
      $depot_rec[conflict_name]['action'] = 'add'
    end
    update_from_shelf.add( f )
  }

  reverted_conflicts.each {
  |f|
    puts("Revert conflict for #{f}")
    update_from_shelf.add( f )
  }

  # if shelf was submitted
  if $shelf_submitted then
    puts("DRAFT - Handle resolving submitted shelf")
    # update_to_shelf CHANGED ON DISK @ MY KNOWN SHELF
    # reverted_disk NEED NEW VERSION, CLEAR HAVE
    # update_from_shelf NEED NEW VERSION, CLEAR HAVE
    # reverted_shelf NEED NEW VERSION, CLEAR HAVE
    # $depot_sync NEED THESE MINUS update_to_shelf

    spew("\nNeed to shelve:", update_to_shelf)
    File.delete($have_table_file)
    $depot_sync.delete_if{|k,v| update_to_shelf.include?(k)}
    # Need to rerun because there may already be a shelf
    # that needs to be resolved against
    puts("\nRerun deviceSync after this run completes")
  else
    update_shelf( p4, update_to_shelf, reverted_disk )
    update_disk( p4, update_from_shelf )

    $depot_sync.delete_if{|k,v| $shelf_working.include?(k) and !$shelf_working[k][3]}

    if reverted_shelf.empty? then
      puts("\nNo files to force sync")
    else
      puts("\nForce sync files:(if not in sync)")
      reverted_shelf.each {
      |f|
        if !$depot_sync.include?(f) then
          puts(f)
          lf = local_path(p4, f)
          if lf != nil and File.exist?(lf) then
            File.delete(lf)
          end
          stuff = p4.run_sync(["-f", "#{f}@#{$last_change}"])
        end
      }
      save_have_table
    end
  end

  if $depot_sync.empty? then
    puts("\nNo files to sync")
  else
    spew("\nSync files:", $depot_sync)
    $depot_sync.each {
    |f,v|
      p4.run_sync(["#{v['depotFile']}@#{$last_change}"])
    }
  end

rescue P4Exception
  p4.errors.each { |e| pp( e ) }
ensure
  p4.disconnect
end