require 'hws_strings' require 'helix_sync/errors/lock_failed' require 'helix_sync/errors/submit_failed' require 'hws_settings' require 'p4_util' module HelixSync module Methods # List HVE Projects as configured in the system. # # See the Appendix in the documentation for details on values. def list(details: false, extension: nil) return if extension and (extension != HVE_ID or extension != HVE_CONTENT_TYPE) project_dirs = list_project_names project_names = project_dirs.map { |d| File.basename(d) } if details project_names.map { |n| fetch_by_name(n) } else project_names.map { |n| encode_name(n) } end end def list_project_names if !Cloud::Settings.cloud_enabled? pattern = "#{hve_projects_path}/*" results = p4.run_dirs(pattern) results.map { |r| r['dir'] } else project_service = Cloud::Projects.new(env:env) project_service.list end end # The ID is a URL encoded version of the directory name under # HVE_PROJECTS_PATH. # # This will unencode the ID and fetch by name. def fetch(id) name = unencode_name(id) fetch_by_name(name) end # Returns the project's "details" based on the project name. # # No validation is done to ensure this directory actually exists in the # system. def fetch_by_name(name) id = encode_name(name) { 'id': id, 'name': name, 'server': server_uri_for_id(id), HVE_ID => { 'depotPath': depot_path_for_name(name) } } end def create_shelf_client(project_id, root) client_name = create_shelf_client_name(project_id) create_client(client_name, project_id, root) end def create_device_client(project_id, device, root) client_name = create_device_client_name(project_id, device) create_client(client_name, project_id, root) end def create_client(client_name, project_id, root) if !Cloud::Settings.cloud_enabled? return create_classic_onedir_client(client_name, project_id, root) else project_service = Cloud::Projects.new(env:env) return project_service.create_client(client_name, project_id, root) end end def delete_shelf_client(project_id) client_name = create_shelf_client_name(project_id) delete_client(client_name) end def delete_device_client(project_id, device_id) client_name = create_device_client_name(project_id, device_id) delete_client(client_name) end # Deletes the client and ignores "client doesn't exist" errors def delete_client(client_name) p4.at_exception_level(P4::RAISE_NONE) do p4.run_client('-d', client_name) end unless p4.messages.all?{ |m| m.msgid == 6178 || m.severity < P4::E_FAILED } fail P4Util.make_p4_error(p4) end end # Generate a new client that only contains the project mapping. # # The client name is a combination of user, project, and device. We prefix # it with "_hve" just for clarity. # # We do not host lock the client. # # @param project_id {String} Our encoded project name # @param device {String} A device ID, like a hostname # @param root {String} The `Root` value for the client parameter def create_classic_onedir_client(client_name, project_id, root) return nil if env['hws_settings'].HVE_PROJECTS_PATH.nil? client_spec = p4.fetch_client(client_name) client_spec._root = root; client_spec._host = nil; client_spec._options = 'allwrite clobber nocompress unlocked nomodtime rmdir'; project_name = unencode_name(project_id) client_spec._view = [ %Q|"#{depot_path_for_name(project_name)}/..." "//#{client_name}/..."| ] p4.save_client(client_spec) client_name end def create_device_client_name(project_id, device) "_hve_#{user}_#{project_id}_#{device}" end def create_shelf_client_name(project_id) "_hve_#{user}_#{project_id}_shelf" end def create_lock_client_name(project_id) "#{create_shelf_client_name(project_id)}_lock" end # Find the latest submitted change for the project restricted to the shelf # client. # # Uses the command: 'p4 changes -m 1 -s submitted [shelf client]' # # This has subtle surprises: # # * If a the client has not been used or does not exist, this does not # return a changelist. # # @param project_id [String] The encoded project ID def find_latest_change_for_project(project_id) client_name = create_shelf_client_name(project_id) begin p4.client = client_name results = p4.run_changes('-m', '1', '-s', 'submitted', "//#{client_name}/...") results.first['change'] unless results.empty? ensure p4.client = 'INVALID' end end # The HVE project 'changelist' is a shelved changelist whose client is # the shelf client name def find_pending_change_for_project(project_id) client_name = create_shelf_client_name(project_id) results = p4.run_changes('-m', '1', '-u', user, '-s', 'shelved', '-c', client_name) results.first['change'] unless results.empty? end # Remove the HVE project changelist (if it exists). # # The general algorithm: # # 1. Take over ownership of our shelf client # 2. Revert everything # 3. Delete any shelved files # 4. Delete the changelist def delete_pending_change_for_project(project_id) Dir.mktmpdir('delete_pending_change') do |dir| shelf_client = create_shelf_client_name(project_id) assume_client_ownership(shelf_client, dir) p4.client = shelf_client change = find_pending_change_for_project(project_id) if change p4.run_revert('-c', change, '//...') p4.run_shelve('-c', change, '-d') p4.run_change('-d', change) end end end # Generates the resolution plan for the shelf def preview_pending_change(project_id) Dir.mktmpdir('hve_resolve') do |dir| shelf_client = create_shelf_client_name(project_id) assume_client_ownership(shelf_client, dir) p4.client = shelf_client begin change = find_pending_change_for_project(project_id) describe_results = p4.run_describe('-S', '-s', change) describe = Describe.new(describe_results.first) shelf_meta = shelf_meta_from_json(describe.desc) return load_shelf_plan(project_id, change, describe, shelf_meta) ensure p4.client = 'INVALID' end end end # Will attempt to resolve all changes on the shelf and submit it # # Note: there may be addition def resolve_and_submit_shelf(project_id) lock_p4 = obtain_client_lock(project_id) # All resolves require setting the local client to a local directory, # that we delete when done. dir = Dir.mktmpdir('hve_resolve') shelf_client = create_shelf_client_name(project_id) assume_client_ownership(shelf_client, dir) p4.client = shelf_client change = find_pending_change_for_project(project_id) retries = HWSSettings.system.HELIX_SYNC_RECONCILE_RETRIES is_resolved = false while !is_resolved && retries > 0 describe_results = p4.run_describe('-S', '-s', change) describe = Describe.new(describe_results.first) shelf_meta = shelf_meta_from_json(describe.desc) shelf_plan = load_shelf_plan(project_id, change, describe, shelf_meta) is_resolved = resolve_and_submit_plan(shelf_plan, change) retries -= 1 sleep(1) unless is_resolved && retries > 0 end if !is_resolved fail SubmitFailed.new(change) end ensure unless lock_p4.nil? lock_p4.run_client('-d', lock_p4.client) lock_p4.disconnect end p4.client = 'INVALID' # Our client has the 'rmdir' option set, which can delete the local # directory if there's nothing else remaining in the client. unless dir.nil? FileUtils.rmtree(dir) if Dir.exist?(dir) end end # Will attempt to create a special "-x" client, and if that fails after # re-attempting a few times, will throw a def obtain_client_lock(project_id) lock_p4 = P4Util.open_from_env(env) lock_p4.connect client_name = create_lock_client_name(project_id) locked = false retries_left = HWSSettings.system.HELIX_SYNC_LOCK_RETRIES lock_client = lock_p4.fetch_client(client_name) lock_p4.client = lock_client._client while !locked && retries_left > 0 retries_left -= 1 lock_p4.at_exception_level(P4::RAISE_NONE) do lock_p4.save_client(lock_client, '-x') end if lock_p4.messages && !lock_p4.messages.empty? && lock_p4.messages.any? {|x| x.msgid == 7748 } sleep(2) if retries_left > 0 elsif lock_p4.messages.all? { |x| x.severity < P4::E_FAILED } locked = true else # This is a different error altogether fail P4Util.make_p4_error(lock_p4) end end fail LockFailed.new(client_name) unless locked return lock_p4 end def assume_client_ownership(client_name, root) client_spec = p4.fetch_client(client_name) client_spec._root = root p4.save_client(client_spec) end def resolve_and_submit_plan(plan, change) return true if plan.empty? if rename_all_locked_by_other(plan, change) return false end unless plan.remove_actions.empty? remove_from_shelf_and_plan(plan, change) end return true if plan.empty? unless plan.readd_actions.empty? readd(plan.readd_actions, change) end unless plan.resolve_actions.empty? unshelve_and_resolve(plan.resolve_actions, change) end if !plan.readd_actions.empty? or !plan.resolve_actions.empty? shelve_revert_and_sync(plan.readd_actions + plan.resolve_actions, change) end p4.at_exception_level(P4::RAISE_NONE) do p4.run_submit('-e', change) return false unless p4.errors.empty? end return true end # Will return true if anything actually happened. If that's true, you # should rebuild the revised plan. def rename_all_locked_by_other(plan, change) to_remove = plan.subplans.select {|s| s.locked_by_other_than?(user) } unless to_remove.empty? depot_files = to_remove.map { |s| s.depotFile } p4.run_sync('-f', *depot_files) p4.run_unshelve('-c', change, '-s', change, *depot_files) p4.run_shelve('-d', '-c', change, *depot_files) p4.run_revert('-k', *depot_files) to_readd = [] to_remove.each do |subplan| old_name = p4_where_path(subplan.depotFile) new_name = "#{old_name}.#{change}.locked" File.rename(old_name, new_name) to_readd << new_name end p4.run_add('-c', change, *to_readd) p4.run_shelve('-f', '-c', change, *to_readd) p4.run_revert(to_readd) return true end return false end # Determines the local path of the file on disk using "p4 where". def p4_where_path(depot_path) where_results = p4.run_where(depot_path) where_results.nil? ? nil : where_results.first['path'] end # Remove all remove_actions from the shelf, and the plan. def remove_from_shelf_and_plan(plan, change) files_to_remove = plan.remove_actions.map(&:depotFile) p4.run_shelve('-d', '-c', change, *files_to_remove) plan.remove_actions.each { |x| plan.subplans.delete(x) } end def readd(subplans, change) depot_files = subplans.map(&:depotFile) p4.run_sync('-f', *depot_files) p4.run_unshelve('-c', change, '-s', change, *depot_files) p4.run_revert('-k', *depot_files) p4.run_add('-c', change, *depot_files) end def unshelve_and_resolve(subplans, change) depot_files = subplans.map(&:depotFile) p4.run_unshelve('-c', change, '-s', change, *depot_files) p4.run_sync(depot_files) p4.run_resolve('-c', change, '-ay', *depot_files) end def shelve_revert_and_sync(subplans, change) depot_files = subplans.map(&:depotFile) p4.run_shelve('-f', '-c', change, *depot_files) p4.run_revert(depot_files) df_no_specs = depot_files.map{ |f| "#{f}#none"} p4.run_sync(df_no_specs) end def shelf_meta_from_json(description) parsed = JSON.parse(description) ShelfMeta.new(parsed) end def load_shelf_plan(project_id, change, describe, shelf_meta) fstat_files = load_fstat_files(project_id, change) plan = Plan.new plan.subplans = fstat_files.map { |f| Subplan.new(f) } plan end def load_fstat_files(project_id, change) fstat_files = p4.run_fstat('-Rs', '-e', change, "//#{create_shelf_client_name(project_id)}/...") fstat_files.delete_if { |f| !f['depotFile'] } fstat_files end # If the user doesn't have a current pending change for the project, # create one, and return that. def create_pending_change(project_id) change = find_pending_change_for_project(project_id) return change if change shelf_client = create_shelf_client(project_id, '/dev/null') p4.client = shelf_client change_spec = p4.fetch_change change_spec._description = "_hws_#{user}_#{project_id}" change_spec._client = shelf_client save_results = p4.save_change(change_spec) change = save_results.first.gsub(/Change (\d+) created./, '\1') p4.client = 'INVALID' change end def encode_name(name) HWSStrings.component_encode(name) end def unencode_name(name) HWSStrings.component_decode(name) end def server_uri_for_id(id) "p4://#{userinfo}#{server}#{safe_hve_projects_path}/#{id}" end def depot_path_for_name(name) "#{hve_projects_path}/#{name}" end def hve_projects_path env['hws_settings'].HVE_PROJECTS_PATH || fail('HVE_PROJECTS_PATH not set') end def safe_hve_projects_path hve_projects_path.gsub('//', '/') end # For HVE Projects, it may be interesting to people to see various # connection settings for each server URL. def userinfo data = {} if env['hws_settings'].P4CHARSET data['P4CHARSET'] = env['hws_settings'].P4CHARSET end if data.keys.empty? '' else encoded_data = data.map {|k,v| "#{k}=#{v}"}.join(';') "#{encoded_data}@" end end def server return p4port if p4port.include?(':') host = p4host ? p4host : 'localhost' port = p4port "#{host}:#{port}" end def p4port env['hws_settings'].P4PORT || fail('P4PORT setting not available') end def p4host env['hws_settings'].P4HOST end # The changelist for the shelf will have a JSON blob stored in the # description. # # This will capture the metadata, and add basic logic along the way. class ShelfMeta < OpenStruct end class Describe < OpenStruct def submitted? self.status == 'submitted' end end class Plan # Array of Subplan instances attr_accessor :subplans def initialize @subplans = [] end def empty? @subplans.empty? end def remove_actions @subplans.select { |s| s.sync_conflict == 'remove' } end def readd_actions @subplans.select { |s| s.sync_conflict == 'readd' } end def resolve_actions @subplans.select { |s| s.sync_conflict == 'resolve' } end def to_json @subplans.to_json end end class Subplan < OpenStruct # Returns true if the subplan contains an "otherLock" that is *not* # the indicated user def locked_by_other_than?(user) !self.otherLock.nil? && self.otherLock.is_a?(Array) && self.otherLock.any?{ |l| !l.start_with?("#{user}@") } end def head_delete? %w(delete move/delete).include?(self.headAction) end def sync_conflict if self.action == 'delete' head_delete? ? 'remove' : 'resolve' elsif self.action == 'add' || self.action == 'edit' head_delete? ? 'readd' : 'resolve' else 'unknown' end end def to_json(*a) to_h.to_json end end end end