MirrorPending.rb #3

#!/usr/bin/env ruby

require 'P4'
require 'securerandom'

BRANCH_PREFIX = "temp-"

class MirrorPending
    def initialize(p4, source, target)
        @p4 = p4
        @source_ws = source
        @target_ws = target
        
        @p4.connect unless @p4.connected?
        
        @source_ws_root = @p4.fetch_client(@source_ws)._root
        @target_ws_root = @p4.fetch_client(@target_ws)._root
    end

    def read_have
        have = Array.new
        @p4.at_exception_level(P4::RAISE_ERRORS) do
            @p4.run_have.each do |h|
                have << h['depotFile'] + "#" + h['haveRev']
            end
        end
        return have
    end

    class ResolveState
        attr_accessor :type
        attr_accessor :action
        attr_accessor :baseFile
        attr_accessor :baseRev
        attr_accessor :fromFile
        attr_accessor :startFromRev
        attr_accessor :endFromRev
        
        def to_s
            "\tResolveState:\n" \
            "\t\ttype         = #{@type}\n" \
            "\t\taction       = #{@action}\n" \
            "\t\tbaseFile     = #{@baseFile}\n" \
            "\t\tbaseRev      = #{@baseRev}\n" \
            "\t\tfromFile     = #{@fromFile}\n" \
            "\t\tstartFromRev = #{@startFromRev}\n" \
            "\t\tendFromRev   = #{@endFromRev}\n" 
        end
    end
    
    class Attribute
        attr_accessor :permanent
        
        def initialize(key, value)
            @key = key
            @value = value
            @permanent = false
        end
        
        def apply(p4, file)
            if @permanent
                p4.run_attribute('-p', '-n', @key, '-v', @value, file)
            else
                p4.run_attribute('-n', @key, '-v', @value, file)
            end
        end
    end
    
    class IntegrationRecord 
        attr_reader :record
        attr_reader :resolves
        attr_reader :fromFile
        attr_reader :startFromRev
        attr_reader :endFromRev
        attr_reader :baseRev
        
        def initialize(r)
            @record = "#{r.fromFile}##{r.startFromRev},#{r.endFromRev}"
            @resolves = Array.new << r
            @fromFile = r.fromFile
            @startFromRev = r.startFromRev
            @endFromRev = r.endFromRev
            @baseRev = r.baseRev
        end
        
        def addResolve(r)
            @resolves << r
        end
        
        def == (another)
            self.record == another.record
        end
        
        def hash
            @record.hash
        end
    end
    
    class OpenFile
        def initialize(p4, fstat, source_ws_root, target_ws_root)
            # ==============================================================
            # Pre-sorted resolve information used by the individual commands
            # ==============================================================
            # p4, the Perforce connection
            @p4 = p4
            # info : all information for this open file 
            @info = Hash.new
            # resolve       : true content integrate resolves 
            @resolve = Array.new
            # editResolve   : resolves coming from edit/sync
            @editResolve = Array.new
            # branchResolve : separate store for integ -Rbd resolves for branching
            @branchResolve = nil
            # deleteResolve : separate store for integ -Rbd resolves for deleting
            @deleteResolve = nil
            # moveResolve : we detected an integrated move, the most complicated case
            @moveResolve = Array.new
            # if we find any integrated moves, we need to create a branch mapping
            @branchMapping = nil
            # branch        : the one resolve entry representing a true branch
            @branched = nil
            # move          : the one resolve entry representing a true move
            @moved = nil
            # movedSource   : in case of a move, this is the source
            @movedSource = nil
            @movedRev = nil
            # afterMoveResolve  : resolves that go into the new moved file 
            @afterMoveResolve = Array.new
            @attributes = Hash.new
            
            @source_ws_root = source_ws_root
            @target_ws_root = target_ws_root
            
            attrprop = Array.new
            
            # store all entries but the resolve arrays in @info 
            fstat.each do |k,v|
                if k.start_with?("openattr-")
                    key = k.sub("openattr-","")
                    @attributes[key] = Attribute.new(key, v)
                elsif k.start_with?("openattrProp-")
                    key = k.sub("openattrProp-", "")
                    attrprop << key # P4Ruby uses an unsorted hash, so I cannot rely on the order
                end
                    
                if not v.kind_of?(Array)
                    if k == "clientFile"
                        @info["localFile"] = v.sub(@source_ws_root, @target_ws_root)
                    end
                    @info[k] = v
                end
                
                if k == "movedFile"
                    @movedSource = v
                elsif k == "movedRev"
                    @movedRev = v
                end
            end
            
            attrprop.each do |key|
                @attributes[key].permanent = true
            end
            
            # if we have resolveAction, we have all other entries as well
            if fstat.has_key?("resolveAction")
                fstat["resolveAction"].length.times do |i|
                    r = ResolveState.new
                    r.action = fstat["resolveAction"][i]
                    r.type = fstat["resolveType"][i]
                    r.baseFile = fstat["resolveBaseFile"][i]
                    r.baseRev = fstat["resolveBaseRev"][i]
                    r.fromFile = fstat["resolveFromFile"][i]
                    r.startFromRev = fstat["resolveStartFromRev"][i]
                    r.endFromRev = fstat["resolveEndFromRev"][i]
                   
                    # pre-sort the resolves here to make sure we perform the correct action
                      
                    if r.action == "moved from" # there can be only one, and no resolve necessary 
                        @moved = r
                    elsif r.type == "branch"
                        @branchResolve = r # there can be only one
                    elsif r.type == "delete"
                        @deleteResolve = r # there can be only one
                    elsif r.type == "move"
                        @moveResolve << r
                    elsif @info["depotFile"] == r.fromFile
                        @editResolve << r
                    elsif @movedSource and @movedSource == r.fromFile
                        if r.endFromRev.to_i < @movedRev.to_i
                            @editResolve << r
                        else
                            @afterMoveResolve << r
                        end
                    elsif r.action == "branch from" # there can be only one
                        @branched = r
                    else
                        @resolve << r
                    end
                end
                
                if @moveResolve.length > 0
                    analyse_move_resolves
                end
            end
            
            if File.exists?(@info["clientFile"])
                @content = File.open(@info["clientFile"],"rb") { |f| f.read }
            end
        end

        def analyse_move_resolves()
            # need to find the content resolves associated with this merge
            # there might be one in @editResolve, the rest is in @resolve 
            
            moveHint = @moveResolve[0] # there is only one to start with
            targetFile = moveHint.baseFile
            targetRev = moveHint.baseRev
            finalTargetFile = moveHint.fromFile
            
            candidates = Array.new
            @resolve.delete_if do |r|
                if r.baseFile == targetFile and 
                   r.baseRev == targetRev
                   candidates << r
                   true
                else
                    false
                end
            end
            
            if candidates.length > 0
                # overwriting movedSource and movedRev.
                # In the resolve data these are separate artifacts tracing the
                # move from oldTarget to movedTarget. Instead replace them with 
                # the actual integration source that causes the move
                # We use the first candidate for the source. If there additional candidates
                # they will be for filetypes and attributes
                
                @movedSource = candidates[0].fromFile
                @movedRev = candidates[0].endFromRev
                
                # Now we need to determine the mapping from source to target
                # and store it in a temporary branch spec
                
                # better to break it down into directories and compare directories
                # assuming that branches live in directories
                
                # try to determine the underlying branch structure and create a general purpose branch

                bottom = nil                          
                catch :done do
                    filelog = @p4.run_filelog("#{@movedSource}##{@movedRev}")
                    filelog.each do |f|     
                        f.revisions.each do |rev|
                            rev.integrations.each do |integ|
                                if integ.how == "branch into" and integ.file == targetFile
                                    bottom = f.depot_file
                                    throw :done
                                end
                            end
                        end
                    end
                end
                
                sourcePath = common_directory(@movedSource, bottom)
                targetPath = common_directory(finalTargetFile, targetFile)
                
                @moveResolve += candidates
            else
                puts ">>>> We have no candidates for the move <<<<"
                raise "No candidates for move resolve"
            end
            
            # We need a unique branch name to avoid overlap with any other branches
            # These branches need to be excluded in the specmap of a spec depot to avoid overloading
            # TODO could be optimized by checking an existing mapping first and reusing it
             
            uniqueBranchname = BRANCH_PREFIX + SecureRandom.uuid.upcase
            
            @branchMapping = @p4.fetch_branch(uniqueBranchname)
            @branchMapping._view = [ "#{sourcePath}/...  #{targetPath}/..." ]
            @p4.save_branch(@branchMapping)
        end        

        # calculate the common directory for two files as the basis for a branch mapping
        
        def common_directory(s1,s2)
            def directory_list(s)
                s.slice(2..s.length).split("/")            
            end
            
            common = Array.new
            
            a1 = directory_list(s1)
            a2 = directory_list(s2)
            
            (0..[a1.length, a2.length].min - 1).each do |i|
                if a1[i] == a2[i]
                    common << a1[i]
                else
                    break
                end
            end
            
            return "//" + common.join('/')
        end
        
        def to_s
            "OpenFile:\n\t#{@info.inspect}\n#{@resolve}\n#{@editResolve}"
        end
        
        @@commands = {
            "add"         => :add_file,
            "edit"        => :edit_file,
            "delete"      => :delete_file,
            "branch"      => :branch_file,
            "move/delete" => :move_delete_file,
            "move/add"    => :move_add_file,
            "integrate"   => :integrate_file
        }
        
        @@response = {
            "unresolved"    =>  "s",
            "copy from"     =>  "at",
            "ignored"       =>  "ay",
            "delete from"   =>  "at",
            "merge from"    =>  "am",
            "edit from"     =>  "ae",
            "resolved"      =>  "hint"
        }
        
        # keep a cache of move/add,move/delete pairs for setting the file type
        
        @@moved = Hash.new
        
        def apply_resolve(file, resolveRecords) 
            @p4.run_resolve(file) do |mergedata|
                mergeResponse = "s" # skip by default
                 
                # will be invoked for every resolve step
                # need to find the correct entry in the resolve records
                
                info = mergedata.info
                fromFile = info[0]["fromFile"]
                startFromRev = info[0]["startFromRev"]
                # startFromRev in resolve records is "off by one" compared to fstat output
                if startFromRev == "none" 
                    startFromRev = "1"
                else
                    startFromRev = (startFromRev.to_i + 1).to_s
                end
                endFromRev = info[0]["endFromRev"]
                resolveType = info[0]["resolveType"]
                
                found = nil
                index = resolveRecords.find_index do |r|
                    #puts "@@> #{mergedata}"
                    #puts "@@> #{r.fromFile} == #{fromFile}"
                    #puts "@@> #{r.startFromRev} == #{startFromRev}"
                    #puts "@@> #{r.endFromRev} == #{endFromRev}"
                    #puts "@@> #{r.type} == #{resolveType}"
                    r.fromFile == fromFile and
                    r.startFromRev == startFromRev and
                    r.endFromRev == endFromRev and
                    ((r.type == resolveType) or 
                     (r.type == "content" and resolveType == "branch") or
                     (r.type == "content" and resolveType == "delete"))
                end
                
                # if we find a record like this, remove it from the resolve array, so that
                # successive integrates do not pick it up
                
                if index
                    found = resolveRecords.delete_at(index)
                end

                if mergedata.action_resolve?        
                    if found
                        if @@response.include?( found.action )
                            response = @@response[ found.action ]
                            if response == "hint"
                                mergeResponse = mergedata.merge_hint
                            else
                                mergeResponse = response
                            end
                        else
                            puts ">>>>>>>>>>>> Unknown action #{found.action} <<<<<<<<<<"
                            # raise exception?
                        end
                    else
                        # Previously skipped resolve. Nothing to do here.
                        #puts "(action) Not found : " + resolveRecords.inspect
                    end
                        
                elsif mergedata.content_resolve?                    
                    if found
                        if @@response.include?( found.action )
                            mergeResponse = @@response[ found.action ]
                        else
                            puts ">>>>>>>>>>>> Unknown action #{found.action} <<<<<<<<<<"
                            # raise exception?
                        end 
                    else
                        # Previously skipped resolve. Nothing to do here.
                        #puts "(content) Not found : " + resolveRecords.inspect
                    end
                else
                    puts ">>>>>>>>> Neither content nor action ???? <<<<<<<<<<<<<<<"
                end
                
                mergeResponse # this is what the block returns to the resolve process
            end
        end
        
        def find_and_remove_resolves(pattern, resolves)
            resolves.delete_if do |p|
                p.fromFile == pattern.fromFile and
                p.startFromRev == pattern.startFromRev and
                p.endFromRev == pattern.endFromRev and
                p.baseFile == pattern.baseFile and
                p.baseRev == pattern.baseRev
            end    
        end
        
        def apply_integ_resolve(file, resolve)
            if @moveResolve.length > 0
                @p4.run_integ("-b", @branchMapping._branch, "-s", "#{@movedSource}##{@movedRev}")
                
                apply_resolve(@moveResolve[0].baseFile, @moveResolve)           
            else
        
                # sort them into unique integrates
                unique = Array.new
                
                resolve.each do |r|
                    intrec = IntegrationRecord.new( r )
                    # use Integration Record as a probe as well as the real record
                    if record = unique.find { |i| i == intrec }
                        record.addResolve(r) # there can be more than one resolve for an integrate
                    else
                        unique << intrec
                    end
                end
                
                # run the integrates
                unique.each do |integRecord|
                    @p4.run_integrate("-f", integRecord.record, file) # -f includes '-i'. We know the range, so this is safe
                    
                    apply_resolve(file, integRecord.resolves)
                end
            end                        
        end
        
        # =======================================
        def add_file()
            file = @info["localFile"]
            
            if @branched
                s = "#{@branched.fromFile}##{@branched.startFromRev},#{@branched.endFromRev}"
                @p4.run_integrate( s, file )
                    
                @p4.run_add(file)
                @p4.run_reopen('-t', @info["type"], file) # ignored if no file type change
                
                File.open(file,"wb") do |f|
                    f.write(@content)
                end
            else
                File.open(file,"wb") do |f|
                    f.write(@content)
                end
                @p4.run_add('-t', @info["type"], file)
            end
            
            # any outstanding resolve records are integrations dealt with here
            
            apply_integ_resolve(file, @resolve)
            
            # because we do not have the actual record of which type is chosen
            # in each resolve, we simply reset the type here
                                    
            @p4.run_reopen('-t', @info["type"], file) # ignored if no file type change

            # again, we do not have the record when which attribute is set
            # but resetting is not a problem, so we do that here
            
            @attributes.each do |key, attribute|
                attribute.apply(@p4, file)
            end
        end
        # =======================================
        
        def apply_edit_resolve(file, type, resolve)
            # sort them into unique syncs, we are reusing the IntegrationRecord object for it
            unique = Array.new
            
            resolve.each do |r|
                intrec = IntegrationRecord.new( r )
                # use Integration Record as a probe as well as the real record
                if record = unique.find { |i| i == intrec }
                    record.addResolve(r) # there can be more than one resolve for an integrate
                else
                    unique << intrec
                end
            end

            unique.each do |resyncRecords|
                
                @p4.run_sync("#{file}##{resyncRecords.baseRev}")
                @p4.run_edit(file) # ignore the "file already open"
                @p4.run_reopen("-t", type, file) # separate in case file type changed already
                @p4.run_sync("#{file}##{resyncRecords.endFromRev}")
                
                apply_resolve(file, resyncRecords.resolves)
            end
            
        end
        
        def edit_file()
            file = @info["localFile"]

            if @resolve.length > 0 # any integrations or downgrades to edits?
                apply_integ_resolve(file, @resolve)
            end    

            if @editResolve.length > 0 # any edits of previous revisions?
                apply_edit_resolve(file, @info["type"], @editResolve)
            else
                @p4.run_edit(@info["depotFile"])
                @p4.run_reopen('-t', @info["type"],@info["depotFile"]) # separate in case of file type change
            end

            # set the real content of the file here

            if File.writable?(file)     
                File.open(file,"wb") do |f|
                    f.write(@content)
                end
            else
                raise "File #{file} (#{@info['depotFile']}) Not Writable => odd"
            end

            # because we do not have the actual record of which type is chosen
            # in each resolve, we simply reset the type here
                                    
            @p4.run_reopen('-t', @info["type"], file) # ignored if no file type change

            @attributes.each do |key, attribute|
                attribute.apply(p4, file)
            end
        end

        def delete_file()
            file = @info["depotFile"]

            if @branchResolve
                # this is not a delete, this is a ignored add
                s = "#{@branchResolve.fromFile}##{@branchResolve.startFromRev},#{@branchResolve.endFromRev}"
                @p4.run_merge( s, file )
                
                # if the merge has not been resolved yet, the action is still unresolved
                # need to distinguish or the remaining resolve record will trigger a false positive
                
                if @branchResolve.action == "unresolved"
                    find_and_remove_resolves(@branchResolve, @resolve)
                else
                    apply_resolve(file, @resolve)
                end
                
            elsif @deleteResolve
                # this is not a delete, this is a ignored add
                s = "#{@deleteResolve.fromFile}##{@deleteResolve.startFromRev},#{@deleteResolve.endFromRev}"
                @p4.run_merge( s, file )
                
                # if the merge has not been resolved yet, the action is still unresolved
                # need to distinguish or the remaining resolve record will trigger a false positive
                
                # TODO : do we need to remove the resolve records here?
                apply_resolve(file, @resolve) unless @deleteResolve.action == "unresolved"

            else
                @p4.run_delete( @info["depotFile"] )
            end
        end
        
        def branch_file()
            file = @info["localFile"]
            
            if @branched
                s = "#{@branched.fromFile}##{@branched.startFromRev},#{@branched.endFromRev}"
                if @branchResolve
                    @p4.run_merge( s, file )
                    apply_resolve(file, [@branchResolve])
                else
                    @p4.run_integrate( s, file )
                end
            else
                puts ">>>>>>>> Branched file without branched record? <<<<<<<<<<<"
            end
            
            # any outstanding resolve records are integrations dealt with here
            
            apply_integ_resolve(file, @resolve)
            
            # because we do not have the actual record of which type is chosen
            # in each resolve, we simply reset the type here
                                    
            @p4.run_reopen('-t', @info["type"], file) # ignored if no file type change

            # again, we do not have the record when which attribute is set
            # but resetting is not a problem, so we do that here
            
            @attributes.each do |key, attribute|
                attribute.apply(p4, file)
            end
        end
        
        def move_delete_file()
            file = @info["depotFile"]
            if @@moved.include?(file)
                @@moved.delete(file)
                @p4.run_reopen("-t", @info["type"], file)
            else
                @@moved[file] = @info["type"]
            end
        end
        
        def move_add_file()
            file = @info["localFile"]

            if @moveResolve.length > 0
                @p4.run_integ("-b", @branchMapping._branch, "-s", "#{@movedSource}##{@movedRev}")
                
                apply_resolve(@moveResolve[0].baseFile, @moveResolve)           
            else
                if @editResolve.length > 0 # any edits of previous revisions?
                    apply_edit_resolve(@movedSource, @info["type"], @editResolve)
                else
                    @p4.run_edit(@movedSource)
                end
        
                @p4.run_move(@movedSource, file)
        
                if @afterMoveResolve.length > 0
                    @p4.run_sync(file)
                    apply_resolve(file, @afterMoveResolve)
                end

                File.open(file,"wb") do |f|
                    f.write(@content)
                end
            end
            
            # we perform all integrations after the move, even if they happened 
            # before the move in the source. The worst thing that can happen is
            # that the file type of the move/delete file revision is incorrect,
            # which we sort out afterwards with a 'p4 reopen'
            
            apply_integ_resolve(file, @resolve)

            @p4.run_reopen('-t', @info["type"], file) # ignored if no file type change
            if @@moved.include?(@movedSource)
                type = @@moved.delete(@movedSource)
                @p4.run_reopen("-t", type, @movedSource)
            else
                @@moved[@movedSource] = ""
            end
            
            @attributes.each do |key, attribute|
                attribute.apply(p4, file)
            end
        end
        
        def integrate_file()
            file = @info["localFile"]
            
            if @deleteResolve
                s = "#{@deleteResolve.fromFile}##{@deleteResolve.endFromRev}"
                @p4.run_merge( s, file )
                
                # if the merge has not been resolved yet, the action is still unresolved
                # need to distinguish or the remaining resolve record will trigger a false positive

                if @deleteResolve.action == "unresolved"
                    find_and_remove_resolves(@deleteResolve, @resolve)
                else
                    apply_resolve(file, @resolve)
                end
            end
            
            apply_integ_resolve(file, @resolve)
            
            # because we do not have the actual record of which type is chosen
            # in each resolve, we simply reset the type here
                                    
            @p4.run_reopen('-t', @info["type"], file) # ignored if no file type change

            # again, we do not have the record when which attribute is set
            # but resetting is not a problem, so we do that here
            
            @attributes.each do |key, attribute|
                attribute.apply(p4, file)
            end
        end
                
        def apply()
            send(@@commands[@info["action"]])
            
            # if we used a branch spec for integrated moves, delete it here
            if @branchMapping
                @p4.delete_branch(@branchMapping._branch)
            end
        end
    end

    def read_open
        result = []
        @p4.run_fstat("-Oar", "-Ro", "//#{@source_ws}/...").each do |fstat|
            result << OpenFile.new(@p4, fstat, @source_ws_root, @target_ws_root)
        end
        
        result
    end

    def read_entries
        # one big fstat to get all the data we need 
        # might be useful as an OutputHandler to deal with the large amount of data

        @p4.client = @source_ws

        have = read_have()

        openFiles = read_open
        
        return have, openFiles
    end

    def restore_entries(have, open)
        @p4.client = @target_ws
        
        # This is just for paranoia. Making sure the target ws is truly empty
        # ... and I do not want any exceptions unless it is truly serious
        
        @p4.at_exception_level(P4::RAISE_ERRORS) do
            @p4.run_revert('//...') # no more open files
            @p4.run_sync('#0') # clear out workspace

            @p4.run_sync(have)    
        end
        
        open.each do |o|
            o.apply
        end
    end
    
    def mirror
        # read and analyze all entries from source
        # restore data to target
    
        have, open = read_entries()
        restore_entries(have, open)
    end

end

# Both source and target file must have the same mapping, or this will fail
# A proper paranoid test would check this first, but alas, we do not have time 

if __FILE__ == $0
    p4 = P4.new()
    
    if ARGV.length < 2
        abort("Usage MirrorPending <source-ws> <target-ws>")
    end
    
    m = MirrorPending.new(p4, ARGV[0], ARGV[1])
    m.mirror
end