methods.rb #10

require 'hws_strings'
require 'helix_sync/errors/lock_failed'
require 'helix_sync/errors/submit_failed'
require 'hws_settings'
require 'p4_util'

module HelixSync
  module Methods

    # List HVE Projects as configured in the system.
    #
    # See the Appendix in the documentation for details on values.
    def list(details: false, extension: nil)
      return if extension and (extension != HVE_ID or extension != HVE_CONTENT_TYPE)

      project_dirs = list_project_names
      project_names = project_dirs.map { |d| File.basename(d) }

      if details
        project_names.map { |n| fetch_by_name(n) }
      else
        project_names.map { |n| encode_name(n) }
      end
    end

    def list_project_names
      if !Cloud::Settings.cloud_enabled?
        pattern = "#{hve_projects_path}/*"
        results = p4.run_dirs(pattern)
        results.map { |r| r['dir'] }
      else
        project_service = Cloud::Projects.new(env:env)
        project_service.list
      end
    end

    # The ID is a URL encoded version of the directory name under
    # HVE_PROJECTS_PATH.
    #
    # This will unencode the ID and fetch by name.
    def fetch(id)
      name = unencode_name(id)
      fetch_by_name(name)
    end

    # Returns the project's "details" based on the project name.
    #
    # No validation is done to ensure this directory actually exists in the
    # system.
    def fetch_by_name(name)
      id = encode_name(name)

      {
          'id': id,
          'name': name,
          'server': server_uri_for_id(id),
          HVE_ID => {
              'depotPath': depot_path_for_name(name)
          }
      }
    end

    def create_shelf_client(project_id, root)
      client_name = create_shelf_client_name(project_id)
      create_client(client_name, project_id, root)
    end

    def create_device_client(project_id, device, root)
      client_name = create_device_client_name(project_id, device)
      create_client(client_name, project_id, root)
    end

    def create_client(client_name, project_id, root)
      if !Cloud::Settings.cloud_enabled?
        return create_classic_onedir_client(client_name, project_id, root)
      else
        project_service = Cloud::Projects.new(env:env)
        return project_service.create_client(client_name, project_id, root)
      end
    end

    def delete_shelf_client(project_id)
      client_name = create_shelf_client_name(project_id)
      delete_client(client_name)
    end

    def delete_device_client(project_id, device_id)
      client_name = create_device_client_name(project_id, device_id)
      delete_client(client_name)
    end

    # Deletes the client and ignores "client doesn't exist" errors
    def delete_client(client_name)
      p4.at_exception_level(P4::RAISE_NONE) do
        p4.run_client('-d', client_name)
      end
      unless p4.messages.all?{ |m| m.msgid == 6178 || m.severity < P4::E_FAILED }
        fail P4Util.make_p4_error(p4)
      end
    end

    # Generate a new client that only contains the project mapping.
    #
    # The client name is a combination of user, project, and device. We prefix
    # it with "_hve" just for clarity.
    #
    # We do not host lock the client.
    #
    # @param project_id {String} Our encoded project name
    # @param device {String} A device ID, like a hostname
    # @param root {String} The `Root` value for the client parameter
    def create_classic_onedir_client(client_name, project_id, root)
      return nil if env['hws_settings'].HVE_PROJECTS_PATH.nil?
      client_spec = p4.fetch_client(client_name)

      client_spec._root = root;
      client_spec._host = nil;

      client_spec._options = 'allwrite clobber nocompress unlocked nomodtime rmdir';

      project_name = unencode_name(project_id)

      client_spec._view = [
          %Q|"#{depot_path_for_name(project_name)}/..." "//#{client_name}/..."|
      ]

      p4.save_client(client_spec)

      client_name
    end

    def create_device_client_name(project_id, device)
      "_hve_#{user}_#{project_id}_#{device}"
    end

    def create_shelf_client_name(project_id)
      "_hve_#{user}_#{project_id}_shelf"
    end

    def create_lock_client_name(project_id)
      "#{create_shelf_client_name(project_id)}_lock"
    end

    # Find the latest submitted change for the project restricted to the shelf
    # client.
    #
    # Uses the command: 'p4 changes -m 1 -s submitted [shelf client]'
    #
    # This has subtle surprises:
    #
    # * If a the client has not been used or does not exist, this does not
    #   return a changelist.
    #
    # @param project_id [String] The encoded project ID
    def find_latest_change_for_project(project_id)
      client_name = create_shelf_client_name(project_id)
      begin
        p4.client = client_name
        results = p4.run_changes('-m', '1', '-s', 'submitted', "//#{client_name}/...")
        results.first['change'] unless results.empty?
      ensure
        p4.client = 'INVALID'
      end
    end

    # The HVE project 'changelist' is a shelved changelist whose client is
    # the shelf client name
    def find_pending_change_for_project(project_id)
      client_name = create_shelf_client_name(project_id)
      results = p4.run_changes('-m', '1', '-u', user, '-s', 'shelved', '-c', client_name)
      results.first['change'] unless results.empty?
    end

    # Remove the HVE project changelist (if it exists).
    #
    # The general algorithm:
    #
    # 1. Take over ownership of our shelf client
    # 2. Revert everything
    # 3. Delete any shelved files
    # 4. Delete the changelist
    def delete_pending_change_for_project(project_id)
      Dir.mktmpdir('delete_pending_change') do |dir|
        shelf_client = create_shelf_client_name(project_id)
        assume_client_ownership(shelf_client, dir)
        p4.client = shelf_client

        change = find_pending_change_for_project(project_id)
        if change
          p4.run_revert('-c', change, '//...')
          p4.run_shelve('-c', change, '-d')
          p4.run_change('-d', change)
        end
      end
    end

    # Generates the resolution plan for the shelf
    def preview_pending_change(project_id)
      Dir.mktmpdir('hve_resolve') do |dir|
        shelf_client = create_shelf_client_name(project_id)
        assume_client_ownership(shelf_client, dir)
        p4.client = shelf_client

        begin
          change = find_pending_change_for_project(project_id)

          describe_results = p4.run_describe('-S', '-s', change)
          describe = Describe.new(describe_results.first)

          shelf_meta = shelf_meta_from_json(describe.desc)

          return load_shelf_plan(project_id, change, describe, shelf_meta)
        ensure
          p4.client = 'INVALID'
        end
      end
    end

    # Will attempt to resolve all changes on the shelf and submit it
    #
    # Note: there may be addition
    def resolve_and_submit_shelf(project_id)
      lock_p4 = obtain_client_lock(project_id)

      # All resolves require setting the local client to a local directory,
      # that we delete when done.
      dir = Dir.mktmpdir('hve_resolve')

      shelf_client = create_shelf_client_name(project_id)
      assume_client_ownership(shelf_client, dir)
      p4.client = shelf_client

      change = find_pending_change_for_project(project_id)

      retries = HWSSettings.system.HELIX_SYNC_RECONCILE_RETRIES
      is_resolved = false

      while !is_resolved && retries > 0
        describe_results = p4.run_describe('-S', '-s', change)
        describe = Describe.new(describe_results.first)

        shelf_meta = shelf_meta_from_json(describe.desc)

        shelf_plan = load_shelf_plan(project_id, change, describe, shelf_meta)
        is_resolved = resolve_and_submit_plan(shelf_plan, change)
        retries -= 1
        sleep(1) unless is_resolved && retries > 0
      end

      if !is_resolved
        fail SubmitFailed.new(change)
      end
    ensure
      unless lock_p4.nil?
        lock_p4.run_client('-d', lock_p4.client)
        lock_p4.disconnect
      end

      p4.client = 'INVALID'

      # Our client has the 'rmdir' option set, which can delete the local
      # directory if there's nothing else remaining in the client.
      unless dir.nil?
        FileUtils.rmtree(dir) if Dir.exist?(dir)
      end
    end

    # Will attempt to create a special "-x" client, and if that fails after
    # re-attempting a few times, will throw a
    def obtain_client_lock(project_id)
      lock_p4 = P4Util.open_from_env(env)
      lock_p4.connect
      client_name = create_lock_client_name(project_id)

      locked = false
      retries_left = HWSSettings.system.HELIX_SYNC_LOCK_RETRIES

      lock_client = lock_p4.fetch_client(client_name)
      lock_p4.client = lock_client._client

      while !locked && retries_left > 0
        retries_left -= 1
        lock_p4.at_exception_level(P4::RAISE_NONE) do
          lock_p4.save_client(lock_client, '-x')
        end
        if lock_p4.messages &&
            !lock_p4.messages.empty? &&
            lock_p4.messages.any? {|x| x.msgid == 7748 }
          sleep(2) if retries_left > 0
        elsif lock_p4.messages.all? { |x| x.severity < P4::E_FAILED }
          locked = true
        else
          # This is a different error altogether
          fail P4Util.make_p4_error(lock_p4)
        end
      end

      fail LockFailed.new(client_name) unless locked

      return lock_p4
    end

    def assume_client_ownership(client_name, root)
      client_spec = p4.fetch_client(client_name)
      client_spec._root = root
      p4.save_client(client_spec)
    end

    def resolve_and_submit_plan(plan, change)
      return true if plan.empty?

      if rename_all_locked_by_other(plan, change)
        return false
      end

      unless plan.remove_actions.empty?
        remove_from_shelf_and_plan(plan, change)
      end

      return true if plan.empty?

      unless plan.readd_actions.empty?
        readd(plan.readd_actions, change)
      end

      unless plan.resolve_actions.empty?
        unshelve_and_resolve(plan.resolve_actions, change)
      end

      if !plan.readd_actions.empty? or !plan.resolve_actions.empty?
        shelve_revert_and_sync(plan.readd_actions + plan.resolve_actions, change)
      end

      p4.at_exception_level(P4::RAISE_NONE) do
        p4.run_submit('-e', change)
        return false unless p4.errors.empty?
      end

      return true
    end

    # Will return true if anything actually happened. If that's true, you
    # should rebuild the revised plan.
    def rename_all_locked_by_other(plan, change)
      to_remove = plan.subplans.select {|s| s.locked_by_other_than?(user) }

      unless to_remove.empty?
        depot_files = to_remove.map { |s| s.depotFile }

        p4.run_sync('-f', *depot_files)
        p4.run_unshelve('-c', change, '-s', change, *depot_files)
        p4.run_shelve('-d', '-c', change, *depot_files)
        p4.run_revert('-k', *depot_files)

        to_readd = []

        to_remove.each do |subplan|
          old_name = p4_where_path(subplan.depotFile)
          new_name = "#{old_name}.#{change}.locked"
          File.rename(old_name, new_name)
          to_readd << new_name
        end

        p4.run_add('-c', change, *to_readd)
        p4.run_shelve('-f', '-c', change, *to_readd)
        p4.run_revert(to_readd)

        return true
      end
      return false
    end

    # Determines the local path of the file on disk using "p4 where".
    def p4_where_path(depot_path)
      where_results = p4.run_where(depot_path)
      where_results.nil? ? nil : where_results.first['path']
    end

    # Remove all remove_actions from the shelf, and the plan.
    def remove_from_shelf_and_plan(plan, change)
      files_to_remove = plan.remove_actions.map(&:depotFile)
      p4.run_shelve('-d', '-c', change, *files_to_remove)
      plan.remove_actions.each { |x| plan.subplans.delete(x) }
    end

    def readd(subplans, change)
      depot_files = subplans.map(&:depotFile)
      p4.run_sync('-f', *depot_files)
      p4.run_unshelve('-c', change, '-s', change, *depot_files)
      p4.run_revert('-k', *depot_files)
      p4.run_add('-c', change, *depot_files)
    end

    def unshelve_and_resolve(subplans, change)
      depot_files = subplans.map(&:depotFile)
      p4.run_unshelve('-c', change, '-s', change, *depot_files)
      p4.run_sync(depot_files)
      p4.run_resolve('-c', change, '-ay', *depot_files)
    end

    def shelve_revert_and_sync(subplans, change)
      depot_files = subplans.map(&:depotFile)
      p4.run_shelve('-f', '-c', change, *depot_files)
      p4.run_revert(depot_files)
      df_no_specs = depot_files.map{ |f| "#{f}#none"}
      p4.run_sync(df_no_specs)
    end

    def shelf_meta_from_json(description)
      parsed = JSON.parse(description)
      ShelfMeta.new(parsed)
    end

    def load_shelf_plan(project_id, change, describe, shelf_meta)
      fstat_files = load_fstat_files(project_id, change)

      plan = Plan.new
      plan.subplans = fstat_files.map { |f| Subplan.new(f) }

      plan
    end

    def load_fstat_files(project_id, change)
      fstat_files = p4.run_fstat('-Rs', '-e', change, "//#{create_shelf_client_name(project_id)}/...")
      fstat_files.delete_if { |f| !f['depotFile'] }
      fstat_files
    end

    # If the user doesn't have a current pending change for the project,
    # create one, and return that.
    def create_pending_change(project_id)
      change = find_pending_change_for_project(project_id)

      return change if change

      shelf_client = create_shelf_client(project_id, '/dev/null')

      p4.client = shelf_client

      change_spec = p4.fetch_change
      change_spec._description = "_hws_#{user}_#{project_id}"
      change_spec._client = shelf_client

      save_results = p4.save_change(change_spec)

      change = save_results.first.gsub(/Change (\d+) created./, '\1')

      p4.client = 'INVALID'

      change
    end

    def encode_name(name)
      HWSStrings.component_encode(name)
    end

    def unencode_name(name)
      HWSStrings.component_decode(name)
    end

    def server_uri_for_id(id)
      "p4://#{userinfo}#{server}#{safe_hve_projects_path}/#{id}"
    end

    def depot_path_for_name(name)
      "#{hve_projects_path}/#{name}"
    end

    def hve_projects_path
      env['hws_settings'].HVE_PROJECTS_PATH || fail('HVE_PROJECTS_PATH not set')
    end

    def safe_hve_projects_path
      hve_projects_path.gsub('//', '/')
    end

    # For HVE Projects, it may be interesting to people to see various
    # connection settings for each server URL.
    def userinfo
      data = {}
      if env['hws_settings'].P4CHARSET
        data['P4CHARSET'] = env['hws_settings'].P4CHARSET
      end
      if data.keys.empty?
        ''
      else
        encoded_data = data.map {|k,v| "#{k}=#{v}"}.join(';')
        "#{encoded_data}@"
      end
    end

    def server
      return p4port if p4port.include?(':')

      host = p4host ? p4host : 'localhost'
      port = p4port

      "#{host}:#{port}"
    end

    def p4port
      env['hws_settings'].P4PORT || fail('P4PORT setting not available')
    end

    def p4host
      env['hws_settings'].P4HOST
    end

    # The changelist for the shelf will have a JSON blob stored in the
    # description.
    #
    # This will capture the metadata, and add basic logic along the way.
    class ShelfMeta < OpenStruct
    end

    class Describe < OpenStruct
      def submitted?
        self.status == 'submitted'
      end
    end

    class Plan
      # Array of Subplan instances
      attr_accessor :subplans

      def initialize
        @subplans = []
      end

      def empty?
        @subplans.empty?
      end

      def remove_actions
        @subplans.select { |s| s.sync_conflict == 'remove' }
      end

      def readd_actions
        @subplans.select { |s| s.sync_conflict == 'readd' }
      end

      def resolve_actions
        @subplans.select { |s| s.sync_conflict == 'resolve' }
      end

      def to_json
        @subplans.to_json
      end
    end

    class Subplan < OpenStruct

      # Returns true if the subplan contains an "otherLock" that is *not*
      # the indicated user
      def locked_by_other_than?(user)
        !self.otherLock.nil? &&
            self.otherLock.is_a?(Array) &&
            self.otherLock.any?{ |l| !l.start_with?("#{user}@") }
      end

      def head_delete?
        %w(delete move/delete).include?(self.headAction)
      end

      def sync_conflict
        if self.action == 'delete'
          head_delete? ? 'remove' : 'resolve'
        elsif self.action == 'add' || self.action == 'edit'
          head_delete? ? 'readd' : 'resolve'
        else
          'unknown'
        end
      end

      def to_json(*a)
        to_h.to_json
      end
    end
  end
end