2020-04-08 14:13:33 +05:30
|
|
|
# frozen_string_literal: true
|
|
|
|
|
|
|
|
module Gitlab
|
|
|
|
module ImportExport
|
2021-09-04 01:27:46 +05:30
|
|
|
module Json
|
2020-04-08 14:13:33 +05:30
|
|
|
class StreamingSerializer
|
|
|
|
include Gitlab::ImportExport::CommandLineUtil
|
|
|
|
|
|
|
|
BATCH_SIZE = 100
|
2020-07-28 23:09:34 +05:30
|
|
|
SMALLER_BATCH_SIZE = 2
|
2020-06-23 00:09:42 +05:30
|
|
|
|
|
|
|
def self.batch_size(exportable)
|
|
|
|
if Feature.enabled?(:export_reduce_relation_batch_size, exportable)
|
|
|
|
SMALLER_BATCH_SIZE
|
|
|
|
else
|
|
|
|
BATCH_SIZE
|
|
|
|
end
|
|
|
|
end
|
2020-04-08 14:13:33 +05:30
|
|
|
|
|
|
|
class Raw < String
|
|
|
|
def to_json(*_args)
|
|
|
|
to_s
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2020-04-22 19:07:51 +05:30
|
|
|
def initialize(exportable, relations_schema, json_writer, exportable_path:)
|
2020-04-08 14:13:33 +05:30
|
|
|
@exportable = exportable
|
2020-04-22 19:07:51 +05:30
|
|
|
@exportable_path = exportable_path
|
2020-04-08 14:13:33 +05:30
|
|
|
@relations_schema = relations_schema
|
|
|
|
@json_writer = json_writer
|
|
|
|
end
|
|
|
|
|
|
|
|
def execute
|
2021-10-27 15:23:28 +05:30
|
|
|
read_from_replica_if_available do
|
|
|
|
serialize_root
|
2020-04-08 14:13:33 +05:30
|
|
|
|
2021-10-27 15:23:28 +05:30
|
|
|
includes.each do |relation_definition|
|
|
|
|
serialize_relation(relation_definition)
|
|
|
|
end
|
2020-04-08 14:13:33 +05:30
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
def serialize_relation(definition)
|
|
|
|
raise ArgumentError, 'definition needs to be Hash' unless definition.is_a?(Hash)
|
|
|
|
raise ArgumentError, 'definition needs to have exactly one Hash element' unless definition.one?
|
|
|
|
|
|
|
|
key, options = definition.first
|
|
|
|
|
|
|
|
record = exportable.public_send(key) # rubocop: disable GitlabSecurity/PublicSend
|
|
|
|
if record.is_a?(ActiveRecord::Relation)
|
|
|
|
serialize_many_relations(key, record, options)
|
2020-04-22 19:07:51 +05:30
|
|
|
elsif record.respond_to?(:each) # this is to support `project_members` that return an Array
|
|
|
|
serialize_many_each(key, record, options)
|
2020-04-08 14:13:33 +05:30
|
|
|
else
|
|
|
|
serialize_single_relation(key, record, options)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2021-06-08 01:23:25 +05:30
|
|
|
private
|
|
|
|
|
|
|
|
attr_reader :json_writer, :relations_schema, :exportable
|
|
|
|
|
|
|
|
def serialize_root
|
|
|
|
attributes = exportable.as_json(
|
|
|
|
relations_schema.merge(include: nil, preloads: nil))
|
|
|
|
json_writer.write_attributes(@exportable_path, attributes)
|
|
|
|
end
|
|
|
|
|
2020-04-08 14:13:33 +05:30
|
|
|
def serialize_many_relations(key, records, options)
|
2020-04-22 19:07:51 +05:30
|
|
|
enumerator = Enumerator.new do |items|
|
|
|
|
key_preloads = preloads&.dig(key)
|
2020-04-08 14:13:33 +05:30
|
|
|
|
2021-06-08 01:23:25 +05:30
|
|
|
batch(records, key) do |batch|
|
|
|
|
batch = batch.preload(key_preloads) if key_preloads
|
2020-07-28 23:09:34 +05:30
|
|
|
|
|
|
|
batch.each do |record|
|
|
|
|
items << Raw.new(record.to_json(options))
|
|
|
|
end
|
2020-04-22 19:07:51 +05:30
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
json_writer.write_relation_array(@exportable_path, key, enumerator)
|
|
|
|
end
|
2020-04-08 14:13:33 +05:30
|
|
|
|
2021-06-08 01:23:25 +05:30
|
|
|
def batch(relation, key)
|
|
|
|
opts = { of: batch_size }
|
|
|
|
order_by = reorders(relation, key)
|
|
|
|
|
|
|
|
# we need to sort issues by non primary key column(relative_position)
|
|
|
|
# and `in_batches` does not support that
|
|
|
|
if order_by
|
|
|
|
scope = relation.reorder(order_by)
|
|
|
|
|
|
|
|
Gitlab::Pagination::Keyset::Iterator.new(scope: scope, use_union_optimization: true).each_batch(**opts) do |batch|
|
|
|
|
yield batch
|
|
|
|
end
|
|
|
|
else
|
|
|
|
relation.in_batches(**opts) do |batch| # rubocop:disable Cop/InBatches
|
|
|
|
# order each batch by its primary key to ensure
|
|
|
|
# consistent and predictable ordering of each exported relation
|
|
|
|
# as additional `WHERE` clauses can impact the order in which data is being
|
|
|
|
# returned by database when no `ORDER` is specified
|
|
|
|
yield batch.reorder(batch.klass.primary_key)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2020-04-22 19:07:51 +05:30
|
|
|
def serialize_many_each(key, records, options)
|
|
|
|
enumerator = Enumerator.new do |items|
|
|
|
|
records.each do |record|
|
|
|
|
items << Raw.new(record.to_json(options))
|
|
|
|
end
|
2020-04-08 14:13:33 +05:30
|
|
|
end
|
2020-04-22 19:07:51 +05:30
|
|
|
|
|
|
|
json_writer.write_relation_array(@exportable_path, key, enumerator)
|
2020-04-08 14:13:33 +05:30
|
|
|
end
|
|
|
|
|
|
|
|
def serialize_single_relation(key, record, options)
|
|
|
|
json = Raw.new(record.to_json(options))
|
|
|
|
|
2020-04-22 19:07:51 +05:30
|
|
|
json_writer.write_relation(@exportable_path, key, json)
|
2020-04-08 14:13:33 +05:30
|
|
|
end
|
|
|
|
|
|
|
|
def includes
|
|
|
|
relations_schema[:include]
|
|
|
|
end
|
|
|
|
|
|
|
|
def preloads
|
|
|
|
relations_schema[:preload]
|
|
|
|
end
|
2020-06-23 00:09:42 +05:30
|
|
|
|
|
|
|
def batch_size
|
|
|
|
@batch_size ||= self.class.batch_size(@exportable)
|
|
|
|
end
|
2021-06-08 01:23:25 +05:30
|
|
|
|
|
|
|
def reorders(relation, key)
|
|
|
|
export_reorder = relations_schema[:export_reorder]&.dig(key)
|
|
|
|
return unless export_reorder
|
|
|
|
|
|
|
|
custom_reorder(relation.klass, export_reorder)
|
|
|
|
end
|
|
|
|
|
|
|
|
def custom_reorder(klass, order_by)
|
|
|
|
arel_table = klass.arel_table
|
|
|
|
column = order_by[:column] || klass.primary_key
|
|
|
|
direction = order_by[:direction] || :asc
|
|
|
|
nulls_position = order_by[:nulls_position] || :nulls_last
|
|
|
|
|
|
|
|
arel_order_classes = ::Gitlab::Pagination::Keyset::ColumnOrderDefinition::AREL_ORDER_CLASSES.invert
|
|
|
|
reverse_direction = ::Gitlab::Pagination::Keyset::ColumnOrderDefinition::REVERSED_ORDER_DIRECTIONS[direction]
|
|
|
|
reverse_nulls_position = ::Gitlab::Pagination::Keyset::ColumnOrderDefinition::REVERSED_NULL_POSITIONS[nulls_position]
|
|
|
|
order_expression = ::Gitlab::Database.nulls_order(column, direction, nulls_position)
|
|
|
|
reverse_order_expression = ::Gitlab::Database.nulls_order(column, reverse_direction, reverse_nulls_position)
|
|
|
|
|
|
|
|
::Gitlab::Pagination::Keyset::Order.build([
|
|
|
|
::Gitlab::Pagination::Keyset::ColumnOrderDefinition.new(
|
|
|
|
attribute_name: column,
|
|
|
|
column_expression: arel_table[column],
|
|
|
|
order_expression: order_expression,
|
|
|
|
reversed_order_expression: reverse_order_expression,
|
|
|
|
order_direction: direction,
|
|
|
|
nullable: nulls_position,
|
|
|
|
distinct: false
|
|
|
|
),
|
|
|
|
::Gitlab::Pagination::Keyset::ColumnOrderDefinition.new(
|
|
|
|
attribute_name: klass.primary_key,
|
|
|
|
order_expression: arel_order_classes[direction].new(arel_table[klass.primary_key.to_sym])
|
|
|
|
)
|
|
|
|
])
|
|
|
|
end
|
2021-10-27 15:23:28 +05:30
|
|
|
|
|
|
|
def read_from_replica_if_available(&block)
|
|
|
|
return yield unless ::Feature.enabled?(:load_balancing_for_export_workers, type: :development, default_enabled: :yaml)
|
|
|
|
|
|
|
|
::Gitlab::Database::LoadBalancing::Session.current.use_replicas_for_read_queries(&block)
|
|
|
|
end
|
2020-04-08 14:13:33 +05:30
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|