debian-mirror-gitlab/lib/gitlab/database/background_migration/batched_migration_runner.rb

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

139 lines
5.3 KiB
Ruby
Raw Normal View History

2021-04-17 20:07:23 +05:30
# frozen_string_literal: true
module Gitlab
module Database
module BackgroundMigration
2021-04-29 21:17:54 +05:30
class BatchedMigrationRunner
2021-09-30 23:02:18 +05:30
FailedToFinalize = Class.new(RuntimeError)
2022-06-21 17:19:12 +05:30
def self.finalize(job_class_name, table_name, column_name, job_arguments, connection:)
2022-05-07 20:08:51 +05:30
new(connection: connection).finalize(job_class_name, table_name, column_name, job_arguments)
2021-09-30 23:02:18 +05:30
end
2022-06-21 17:19:12 +05:30
def initialize(connection:, migration_wrapper: BatchedMigrationWrapper.new(connection: connection))
2022-05-07 20:08:51 +05:30
@connection = connection
2022-06-21 17:19:12 +05:30
@migration_wrapper = migration_wrapper
2021-04-29 21:17:54 +05:30
end
2021-04-17 20:07:23 +05:30
2021-04-29 21:17:54 +05:30
# Runs the next batched_job for a batched_background_migration.
#
# The batch bounds of the next job are calculated at runtime, based on the migration
# configuration and the bounds of the most recently created batched_job. Updating the
# migration configuration will cause future jobs to use the updated batch sizes.
#
# The job instance will automatically receive a set of arguments based on the migration
# configuration. For more details, see the BatchedMigrationWrapper class.
#
# Note that this method is primarily intended to called by a scheduled worker.
def run_migration_job(active_migration)
2021-06-08 01:23:25 +05:30
if next_batched_job = find_or_create_next_batched_job(active_migration)
2021-04-17 20:07:23 +05:30
migration_wrapper.perform(next_batched_job)
2021-06-08 01:23:25 +05:30
active_migration.optimize!
2022-06-21 17:19:12 +05:30
active_migration.failure! if next_batched_job.failed? && active_migration.should_stop?
2021-04-17 20:07:23 +05:30
else
finish_active_migration(active_migration)
end
end
2021-04-29 21:17:54 +05:30
# Runs all remaining batched_jobs for a batched_background_migration.
#
# This method is intended to be used in a test/dev environment to execute the background
# migration inline. It should NOT be used in a real environment for any non-trivial migrations.
def run_entire_migration(migration)
unless Rails.env.development? || Rails.env.test?
raise 'this method is not intended for use in real environments'
end
2021-09-30 23:02:18 +05:30
run_migration_while(migration, :active)
end
2021-04-29 21:17:54 +05:30
2021-09-30 23:02:18 +05:30
# Finalize migration for given configuration.
#
# If the migration is already finished, do nothing. Otherwise change its status to `finalizing`
# in order to prevent it being picked up by the background worker. Perform all pending jobs,
# then keep running until migration is finished.
def finalize(job_class_name, table_name, column_name, job_arguments)
migration = BatchedMigration.find_for_configuration(job_class_name, table_name, column_name, job_arguments)
configuration = {
job_class_name: job_class_name,
table_name: table_name,
column_name: column_name,
job_arguments: job_arguments
}
if migration.nil?
Gitlab::AppLogger.warn "Could not find batched background migration for the given configuration: #{configuration}"
elsif migration.finished?
Gitlab::AppLogger.warn "Batched background migration for the given configuration is already finished: #{configuration}"
else
2022-06-21 17:19:12 +05:30
migration.finalize!
2022-04-04 11:22:00 +05:30
migration.batched_jobs.with_status(:pending).each { |job| migration_wrapper.perform(job) }
2021-09-30 23:02:18 +05:30
run_migration_while(migration, :finalizing)
raise FailedToFinalize unless migration.finished?
2021-04-29 21:17:54 +05:30
end
end
2021-04-17 20:07:23 +05:30
private
2022-06-21 17:19:12 +05:30
attr_reader :connection, :migration_wrapper
2021-04-29 21:17:54 +05:30
2021-06-08 01:23:25 +05:30
def find_or_create_next_batched_job(active_migration)
if next_batch_range = find_next_batch_range(active_migration)
active_migration.create_batched_job!(next_batch_range.min, next_batch_range.max)
else
active_migration.batched_jobs.retriable.first
end
2021-04-17 20:07:23 +05:30
end
def find_next_batch_range(active_migration)
2022-05-07 20:08:51 +05:30
batching_strategy = active_migration.batch_class.new(connection: connection)
2021-04-17 20:07:23 +05:30
batch_min_value = active_migration.next_min_value
next_batch_bounds = batching_strategy.next_batch(
active_migration.table_name,
active_migration.column_name,
batch_min_value: batch_min_value,
2022-04-04 11:22:00 +05:30
batch_size: active_migration.batch_size,
job_arguments: active_migration.job_arguments)
2021-04-17 20:07:23 +05:30
return if next_batch_bounds.nil?
clamped_batch_range(active_migration, next_batch_bounds)
end
def clamped_batch_range(active_migration, next_bounds)
min_value, max_value = next_bounds
return if min_value > active_migration.max_value
max_value = max_value.clamp(min_value, active_migration.max_value)
(min_value..max_value)
end
def finish_active_migration(active_migration)
2021-06-08 01:23:25 +05:30
return if active_migration.batched_jobs.active.exists?
2022-04-04 11:22:00 +05:30
if active_migration.batched_jobs.with_status(:failed).exists?
2022-06-21 17:19:12 +05:30
active_migration.failure!
2021-06-08 01:23:25 +05:30
else
2022-06-21 17:19:12 +05:30
active_migration.finish!
2021-06-08 01:23:25 +05:30
end
2021-04-17 20:07:23 +05:30
end
2021-09-30 23:02:18 +05:30
def run_migration_while(migration, status)
2022-06-21 17:19:12 +05:30
while migration.status_name == status
2021-09-30 23:02:18 +05:30
run_migration_job(migration)
migration.reload_last_job
end
end
2021-04-17 20:07:23 +05:30
end
end
end
end