debian-mirror-gitlab/lib/gitlab/import_export/project_tree_restorer.rb

# frozen_string_literal: true

module Gitlab
  module ImportExport
    class ProjectTreeRestorer
      # Relations which cannot be saved at project level (and have a group assigned)
      GROUP_MODELS = [GroupLabel, Milestone].freeze

      attr_reader :user
      attr_reader :shared
      attr_reader :project

      def initialize(user:, shared:, project:)
        @path = File.join(shared.export_path, 'project.json')
        @user = user
        @shared = shared
        @project = project
        @saved = true
      end

      def restore
        begin
          @tree_hash = read_tree_hash
        rescue => e
          Rails.logger.error("Import/Export error: #{e.message}") # rubocop:disable Gitlab/RailsLogger
          raise Gitlab::ImportExport::Error.new('Incorrect JSON format')
        end

        @project_members = @tree_hash.delete('project_members')

        RelationRenameService.rename(@tree_hash)

        ActiveRecord::Base.uncached do
          ActiveRecord::Base.no_touching do
            update_project_params!
            create_relations
          end
        end

        # ensure that we have latest version of the restore
        @project.reload # rubocop:disable Cop/ActiveRecordAssociationReload

        true
      rescue => e
        @shared.error(e)
        false
      end

      private

      def read_tree_hash
        json = IO.read(@path)
        ActiveSupport::JSON.decode(json)
      end

      def members_mapper
        @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members,
                                                                    user: @user,
                                                                    project: @project)
      end

      # A Hash of the imported merge request ID -> imported ID.
      def merge_requests_mapping
        @merge_requests_mapping ||= {}
      end

      # Loops through the tree of models defined in import_export.yml and
      # finds them in the imported JSON so they can be instantiated and saved
      # in the DB. The structure and relationships between models are guessed from
      # the configuration yaml file too.
      # Finally, it updates each attribute in the newly imported project.
      def create_relations
        project_relations.each do |relation_key, relation_definition|
          relation_key_s = relation_key.to_s

          if relation_definition.present?
            create_sub_relations(relation_key_s, relation_definition, @tree_hash)
          elsif @tree_hash[relation_key_s].present?
            save_relation_hash(relation_key_s, @tree_hash[relation_key_s])
          end
        end

        @project.merge_requests.set_latest_merge_request_diff_ids!

        @saved
      end

      def save_relation_hash(relation_key, relation_hash_batch)
        relation_hash = create_relation(relation_key, relation_hash_batch)

        remove_group_models(relation_hash) if relation_hash.is_a?(Array)

        @saved = false unless @project.append_or_update_attribute(relation_key, relation_hash)

        save_id_mappings(relation_key, relation_hash_batch, relation_hash)

        @project.reset
      end

      # Older, serialized CI pipeline exports may only have a
      # merge_request_id and not the full hash of the merge request. To
      # import these pipelines, we need to preserve the mapping between
      # the old and new the merge request ID.
      def save_id_mappings(relation_key, relation_hash_batch, relation_hash)
        return unless relation_key == 'merge_requests'

        relation_hash = Array(relation_hash)

        Array(relation_hash_batch).each_with_index do |raw_data, index|
          merge_requests_mapping[raw_data['id']] = relation_hash[index]['id']
        end
      end

      # Remove project models that became group models as we found them at group level.
      # This no longer required saving them at the root project level.
      # For example, in the case of an existing group label that matched the title.
      def remove_group_models(relation_hash)
        relation_hash.reject! do |value|
          GROUP_MODELS.include?(value.class) && value.group_id
        end
      end

      def remove_feature_dependent_sub_relations!(_relation_item)
        # no-op
      end

      def project_relations
        @project_relations ||= reader.attributes_finder.find_relations_tree(:project)
      end

      def update_project_params!
        Gitlab::Timeless.timeless(@project) do
          project_params = @tree_hash.reject do |key, value|
            project_relations.include?(key.to_sym)
          end

          project_params = project_params.merge(present_project_override_params)

          # Cleaning all imported and overridden params
          project_params = Gitlab::ImportExport::AttributeCleaner.clean(
            relation_hash: project_params,
            relation_class: Project,
            excluded_keys: excluded_keys_for_relation(:project))

          @project.assign_attributes(project_params)
          @project.drop_visibility_level!
          @project.save!
        end
      end

      def present_project_override_params
        # we filter out the empty strings from the overrides
        # keeping the default values configured
        project_override_params.transform_values do |value|
          value.is_a?(String) ? value.presence : value
        end.compact
      end

      def project_override_params
        @project_override_params ||= @project.import_data&.data&.fetch('override_params', nil) || {}
      end

      # Given a relation hash containing one or more models and its relationships,
      # loops through each model and each object from a model type and
      # and assigns its correspondent attributes hash from +tree_hash+
      # Example:
      # +relation_key+ issues, loops through the list of *issues* and for each individual
      # issue, finds any subrelations such as notes, creates them and assign them back to the hash
      #
      # Recursively calls this method if the sub-relation is a hash containing more sub-relations
      def create_sub_relations(relation_key, relation_definition, tree_hash, save: true)
        return if tree_hash[relation_key].blank?

        tree_array = [tree_hash[relation_key]].flatten

        # Avoid keeping a possible heavy object in memory once we are done with it
        while relation_item = tree_array.shift
          remove_feature_dependent_sub_relations!(relation_item)

          # The transaction at this level is less speedy than one single transaction
          # But we can't have it in the upper level or GC won't get rid of the AR objects
          # after we save the batch.
          Project.transaction do
            process_sub_relation(relation_key, relation_definition, relation_item)

            # For every subrelation that hangs from Project, save the associated records altogether
            # This effectively batches all records per subrelation item, only keeping those in memory
            # We have to keep in mind that more batch granularity << Memory, but >> Slowness
            if save
              save_relation_hash(relation_key, [relation_item])
              tree_hash[relation_key].delete(relation_item)
            end
          end
        end

        tree_hash.delete(relation_key) if save
      end

      def process_sub_relation(relation_key, relation_definition, relation_item)
        relation_definition.each do |sub_relation_key, sub_relation_definition|
          # We just use author to get the user ID, do not attempt to create an instance.
          next if sub_relation_key == :author

          sub_relation_key_s = sub_relation_key.to_s

          # create dependent relations if present
          if sub_relation_definition.present?
            create_sub_relations(sub_relation_key_s, sub_relation_definition, relation_item, save: false)
          end

          # transform relation hash to actual object
          sub_relation_hash = relation_item[sub_relation_key_s]
          if sub_relation_hash.present?
            relation_item[sub_relation_key_s] = create_relation(sub_relation_key, sub_relation_hash)
          end
        end
      end

      def create_relation(relation_key, relation_hash_list)
        relation_array = [relation_hash_list].flatten.map do |relation_hash|
          Gitlab::ImportExport::RelationFactory.create(
            relation_sym: relation_key.to_sym,
            relation_hash: relation_hash,
            members_mapper: members_mapper,
            merge_requests_mapping: merge_requests_mapping,
            user: @user,
            project: @project,
            excluded_keys: excluded_keys_for_relation(relation_key))
        end.compact

        relation_hash_list.is_a?(Array) ? relation_array : relation_array.first
      end

      def reader
        @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared)
      end

      def excluded_keys_for_relation(relation)
        reader.attributes_finder.find_excluded_keys(relation)
      end
    end
  end
end

Gitlab::ImportExport::ProjectTreeRestorer.prepend_if_ee('::EE::Gitlab::ImportExport::ProjectTreeRestorer')