2019-02-15 15:39:39 +05:30
|
|
|
# frozen_string_literal: true
|
|
|
|
|
2016-06-22 15:30:34 +05:30
|
|
|
module Gitlab
|
|
|
|
module ImportExport
|
|
|
|
class ProjectTreeRestorer
|
2018-11-08 19:23:39 +05:30
|
|
|
# Relations which cannot be saved at project level (and have a group assigned)
|
|
|
|
GROUP_MODELS = [GroupLabel, Milestone].freeze
|
2017-09-10 17:25:29 +05:30
|
|
|
|
2019-12-21 20:55:43 +05:30
|
|
|
attr_reader :user
|
|
|
|
attr_reader :shared
|
|
|
|
attr_reader :project
|
|
|
|
|
2016-06-22 15:30:34 +05:30
|
|
|
def initialize(user:, shared:, project:)
|
|
|
|
@path = File.join(shared.export_path, 'project.json')
|
|
|
|
@user = user
|
|
|
|
@shared = shared
|
|
|
|
@project = project
|
|
|
|
end
|
|
|
|
|
|
|
|
def restore
|
2016-11-03 12:29:30 +05:30
|
|
|
begin
|
2019-12-21 20:55:43 +05:30
|
|
|
@tree_hash = read_tree_hash
|
2016-11-03 12:29:30 +05:30
|
|
|
rescue => e
|
2019-09-30 21:07:59 +05:30
|
|
|
Rails.logger.error("Import/Export error: #{e.message}") # rubocop:disable Gitlab/RailsLogger
|
2016-11-03 12:29:30 +05:30
|
|
|
raise Gitlab::ImportExport::Error.new('Incorrect JSON format')
|
|
|
|
end
|
|
|
|
|
2016-06-22 15:30:34 +05:30
|
|
|
@project_members = @tree_hash.delete('project_members')
|
2016-08-24 12:49:21 +05:30
|
|
|
|
2019-02-15 15:39:39 +05:30
|
|
|
RelationRenameService.rename(@tree_hash)
|
|
|
|
|
2018-03-17 18:26:18 +05:30
|
|
|
ActiveRecord::Base.uncached do
|
|
|
|
ActiveRecord::Base.no_touching do
|
2019-12-21 20:55:43 +05:30
|
|
|
update_project_params!
|
2019-12-26 22:10:19 +05:30
|
|
|
create_project_relations!
|
|
|
|
post_import!
|
2018-03-17 18:26:18 +05:30
|
|
|
end
|
2016-08-24 12:49:21 +05:30
|
|
|
end
|
2019-12-21 20:55:43 +05:30
|
|
|
|
|
|
|
# ensure that we have latest version of the restore
|
|
|
|
@project.reload # rubocop:disable Cop/ActiveRecordAssociationReload
|
|
|
|
|
|
|
|
true
|
2016-06-22 15:30:34 +05:30
|
|
|
rescue => e
|
|
|
|
@shared.error(e)
|
|
|
|
false
|
|
|
|
end
|
|
|
|
|
2019-12-21 20:55:43 +05:30
|
|
|
private
|
2018-05-09 12:01:36 +05:30
|
|
|
|
2019-12-21 20:55:43 +05:30
|
|
|
def read_tree_hash
|
|
|
|
json = IO.read(@path)
|
|
|
|
ActiveSupport::JSON.decode(json)
|
2016-06-22 15:30:34 +05:30
|
|
|
end
|
|
|
|
|
|
|
|
def members_mapper
|
|
|
|
@members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members,
|
|
|
|
user: @user,
|
2019-12-21 20:55:43 +05:30
|
|
|
project: @project)
|
|
|
|
end
|
|
|
|
|
|
|
|
# A Hash of the imported merge request ID -> imported ID.
|
|
|
|
def merge_requests_mapping
|
|
|
|
@merge_requests_mapping ||= {}
|
2016-06-22 15:30:34 +05:30
|
|
|
end
|
|
|
|
|
|
|
|
# Loops through the tree of models defined in import_export.yml and
|
|
|
|
# finds them in the imported JSON so they can be instantiated and saved
|
|
|
|
# in the DB. The structure and relationships between models are guessed from
|
|
|
|
# the configuration yaml file too.
|
|
|
|
# Finally, it updates each attribute in the newly imported project.
|
2019-12-26 22:10:19 +05:30
|
|
|
def create_project_relations!
|
|
|
|
project_relations.each(&method(
|
|
|
|
:process_project_relation!))
|
|
|
|
end
|
2016-06-22 15:30:34 +05:30
|
|
|
|
2019-12-26 22:10:19 +05:30
|
|
|
def post_import!
|
2018-03-17 18:26:18 +05:30
|
|
|
@project.merge_requests.set_latest_merge_request_diff_ids!
|
|
|
|
end
|
2017-08-17 22:00:37 +05:30
|
|
|
|
2019-12-26 22:10:19 +05:30
|
|
|
def process_project_relation!(relation_key, relation_definition)
|
|
|
|
data_hashes = @tree_hash.delete(relation_key)
|
|
|
|
return unless data_hashes
|
2017-08-17 22:00:37 +05:30
|
|
|
|
2019-12-26 22:10:19 +05:30
|
|
|
# we do not care if we process array or hash
|
|
|
|
data_hashes = [data_hashes] unless data_hashes.is_a?(Array)
|
2018-11-08 19:23:39 +05:30
|
|
|
|
2019-12-26 22:10:19 +05:30
|
|
|
# consume and remove objects from memory
|
|
|
|
while data_hash = data_hashes.shift
|
|
|
|
process_project_relation_item!(relation_key, relation_definition, data_hash)
|
|
|
|
end
|
|
|
|
end
|
2018-03-17 18:26:18 +05:30
|
|
|
|
2019-12-26 22:10:19 +05:30
|
|
|
def process_project_relation_item!(relation_key, relation_definition, data_hash)
|
|
|
|
relation_object = build_relation(relation_key, relation_definition, data_hash)
|
|
|
|
return unless relation_object
|
|
|
|
return if group_model?(relation_object)
|
2019-12-21 20:55:43 +05:30
|
|
|
|
2019-12-26 22:10:19 +05:30
|
|
|
relation_object.project = @project
|
|
|
|
relation_object.save!
|
|
|
|
|
|
|
|
save_id_mapping(relation_key, data_hash, relation_object)
|
2019-12-21 20:55:43 +05:30
|
|
|
end
|
|
|
|
|
|
|
|
# Older, serialized CI pipeline exports may only have a
|
|
|
|
# merge_request_id and not the full hash of the merge request. To
|
|
|
|
# import these pipelines, we need to preserve the mapping between
|
|
|
|
# the old and new the merge request ID.
|
2019-12-26 22:10:19 +05:30
|
|
|
def save_id_mapping(relation_key, data_hash, relation_object)
|
2019-12-21 20:55:43 +05:30
|
|
|
return unless relation_key == 'merge_requests'
|
|
|
|
|
2019-12-26 22:10:19 +05:30
|
|
|
merge_requests_mapping[data_hash['id']] = relation_object.id
|
2019-12-04 20:38:33 +05:30
|
|
|
end
|
|
|
|
|
|
|
|
def project_relations
|
2019-12-26 22:10:19 +05:30
|
|
|
@project_relations ||=
|
|
|
|
reader
|
|
|
|
.attributes_finder
|
|
|
|
.find_relations_tree(:project)
|
|
|
|
.deep_stringify_keys
|
2016-06-22 15:30:34 +05:30
|
|
|
end
|
|
|
|
|
2019-12-21 20:55:43 +05:30
|
|
|
def update_project_params!
|
2019-12-26 22:10:19 +05:30
|
|
|
project_params = @tree_hash.reject do |key, value|
|
|
|
|
project_relations.include?(key)
|
|
|
|
end
|
2016-06-22 15:30:34 +05:30
|
|
|
|
2019-12-26 22:10:19 +05:30
|
|
|
project_params = project_params.merge(
|
|
|
|
present_project_override_params)
|
2018-06-03 19:52:53 +05:30
|
|
|
|
2019-12-26 22:10:19 +05:30
|
|
|
# Cleaning all imported and overridden params
|
|
|
|
project_params = Gitlab::ImportExport::AttributeCleaner.clean(
|
|
|
|
relation_hash: project_params,
|
|
|
|
relation_class: Project,
|
|
|
|
excluded_keys: excluded_keys_for_relation(:project))
|
2019-12-21 20:55:43 +05:30
|
|
|
|
2019-12-26 22:10:19 +05:30
|
|
|
@project.assign_attributes(project_params)
|
|
|
|
@project.drop_visibility_level!
|
|
|
|
|
|
|
|
Gitlab::Timeless.timeless(@project) do
|
2019-12-21 20:55:43 +05:30
|
|
|
@project.save!
|
2016-10-01 15:18:49 +05:30
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2019-12-21 20:55:43 +05:30
|
|
|
def present_project_override_params
|
|
|
|
# we filter out the empty strings from the overrides
|
|
|
|
# keeping the default values configured
|
|
|
|
project_override_params.transform_values do |value|
|
|
|
|
value.is_a?(String) ? value.presence : value
|
|
|
|
end.compact
|
2019-02-02 18:00:53 +05:30
|
|
|
end
|
|
|
|
|
2019-12-21 20:55:43 +05:30
|
|
|
def project_override_params
|
|
|
|
@project_override_params ||= @project.import_data&.data&.fetch('override_params', nil) || {}
|
2019-12-04 20:38:33 +05:30
|
|
|
end
|
|
|
|
|
2019-12-26 22:10:19 +05:30
|
|
|
def build_relations(relation_key, relation_definition, data_hashes)
|
|
|
|
data_hashes.map do |data_hash|
|
|
|
|
build_relation(relation_key, relation_definition, data_hash)
|
|
|
|
end.compact
|
2018-03-17 18:26:18 +05:30
|
|
|
end
|
2016-08-24 12:49:21 +05:30
|
|
|
|
2019-12-26 22:10:19 +05:30
|
|
|
def build_relation(relation_key, relation_definition, data_hash)
|
|
|
|
# TODO: This is hack to not create relation for the author
|
|
|
|
# Rather make `RelationFactory#set_note_author` to take care of that
|
|
|
|
return data_hash if relation_key == 'author'
|
2016-08-24 12:49:21 +05:30
|
|
|
|
2019-12-26 22:10:19 +05:30
|
|
|
# create relation objects recursively for all sub-objects
|
|
|
|
relation_definition.each do |sub_relation_key, sub_relation_definition|
|
|
|
|
transform_sub_relations!(data_hash, sub_relation_key, sub_relation_definition)
|
|
|
|
end
|
2018-03-17 18:26:18 +05:30
|
|
|
|
2019-12-26 22:10:19 +05:30
|
|
|
Gitlab::ImportExport::RelationFactory.create(
|
|
|
|
relation_sym: relation_key.to_sym,
|
|
|
|
relation_hash: data_hash,
|
|
|
|
members_mapper: members_mapper,
|
|
|
|
merge_requests_mapping: merge_requests_mapping,
|
|
|
|
user: @user,
|
|
|
|
project: @project,
|
|
|
|
excluded_keys: excluded_keys_for_relation(relation_key))
|
|
|
|
end
|
|
|
|
|
|
|
|
def transform_sub_relations!(data_hash, sub_relation_key, sub_relation_definition)
|
|
|
|
sub_data_hash = data_hash[sub_relation_key]
|
|
|
|
return unless sub_data_hash
|
|
|
|
|
|
|
|
# if object is a hash we can create simple object
|
|
|
|
# as it means that this is 1-to-1 vs 1-to-many
|
|
|
|
sub_data_hash =
|
|
|
|
if sub_data_hash.is_a?(Array)
|
|
|
|
build_relations(
|
|
|
|
sub_relation_key,
|
|
|
|
sub_relation_definition,
|
|
|
|
sub_data_hash).presence
|
|
|
|
else
|
|
|
|
build_relation(
|
|
|
|
sub_relation_key,
|
|
|
|
sub_relation_definition,
|
|
|
|
sub_data_hash)
|
2019-12-04 20:38:33 +05:30
|
|
|
end
|
2016-06-22 15:30:34 +05:30
|
|
|
|
2019-12-26 22:10:19 +05:30
|
|
|
# persist object(s) or delete from relation
|
|
|
|
if sub_data_hash
|
|
|
|
data_hash[sub_relation_key] = sub_data_hash
|
|
|
|
else
|
|
|
|
data_hash.delete(sub_relation_key)
|
2016-06-22 15:30:34 +05:30
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2019-12-26 22:10:19 +05:30
|
|
|
def group_model?(relation_object)
|
|
|
|
GROUP_MODELS.include?(relation_object.class) && relation_object.group_id
|
2016-06-22 15:30:34 +05:30
|
|
|
end
|
2016-11-03 12:29:30 +05:30
|
|
|
|
2018-05-09 12:01:36 +05:30
|
|
|
def reader
|
|
|
|
@reader ||= Gitlab::ImportExport::Reader.new(shared: @shared)
|
|
|
|
end
|
2018-06-03 19:52:53 +05:30
|
|
|
|
|
|
|
def excluded_keys_for_relation(relation)
|
2018-11-20 20:47:30 +05:30
|
|
|
reader.attributes_finder.find_excluded_keys(relation)
|
|
|
|
end
|
2016-06-22 15:30:34 +05:30
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|