debian-mirror-gitlab/lib/gitlab/github_import/bulk_importing.rb
2023-03-04 22:38:38 +05:30

102 lines
2.8 KiB
Ruby

# frozen_string_literal: true
module Gitlab
module GithubImport
module BulkImporting
attr_reader :project, :client
# project - An instance of `Project`.
# client - An instance of `Gitlab::GithubImport::Client`.
def initialize(project, client)
@project = project
@client = client
@validation_errors = []
end
# Builds and returns an Array of objects to bulk insert into the
# database and array of validation errors if object is invalid.
#
# enum - An Enumerable that returns the objects to turn into database
# rows.
def build_database_rows(enum)
errors = []
rows = enum.each_with_object([]) do |(object, _), result|
next if already_imported?(object)
attrs = build_attributes(object)
build_record = model.new(attrs)
if build_record.invalid?
log_error(object[:id], build_record.errors.full_messages)
errors << build_record.errors
next
end
result << attrs
end
log_and_increment_counter(rows.size, :fetched)
[rows, errors]
end
# Bulk inserts the given rows into the database.
def bulk_insert(rows, batch_size: 100)
rows.each_slice(batch_size) do |slice|
ApplicationRecord.legacy_bulk_insert(model.table_name, slice) # rubocop:disable Gitlab/BulkInsert
log_and_increment_counter(slice.size, :imported)
end
end
def object_type
raise NotImplementedError
end
def bulk_insert_failures(validation_errors)
rows = validation_errors.map do |error|
correlation_id_value = Labkit::Correlation::CorrelationId.current_or_new_id
{
source: self.class.name,
exception_class: 'ActiveRecord::RecordInvalid',
exception_message: error.full_messages.first.truncate(255),
correlation_id_value: correlation_id_value,
retry_count: nil,
created_at: Time.zone.now
}
end
project.import_failures.insert_all(rows)
end
private
def log_and_increment_counter(value, operation)
Gitlab::Import::Logger.info(
import_type: :github,
project_id: project.id,
importer: self.class.name,
message: "#{value} #{object_type.to_s.pluralize} #{operation}"
)
Gitlab::GithubImport::ObjectCounter.increment(
project,
object_type,
operation,
value: value
)
end
def log_error(object_id, messages)
Gitlab::Import::Logger.error(
import_type: :github,
project_id: project.id,
importer: self.class.name,
message: messages,
github_identifier: object_id
)
end
end
end
end