80 lines
2.5 KiB
Ruby
80 lines
2.5 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
module Pages
|
|
class MigrateFromLegacyStorageService
|
|
def initialize(logger, migration_threads:, batch_size:, ignore_invalid_entries:)
|
|
@logger = logger
|
|
@migration_threads = migration_threads
|
|
@batch_size = batch_size
|
|
@ignore_invalid_entries = ignore_invalid_entries
|
|
|
|
@migrated = 0
|
|
@errored = 0
|
|
@counters_lock = Mutex.new
|
|
end
|
|
|
|
def execute
|
|
@queue = SizedQueue.new(1)
|
|
|
|
threads = start_migration_threads
|
|
|
|
ProjectPagesMetadatum.only_on_legacy_storage.each_batch(of: @batch_size) do |batch|
|
|
@queue.push(batch)
|
|
end
|
|
|
|
@queue.close
|
|
|
|
@logger.info("Waiting for threads to finish...")
|
|
threads.each(&:join)
|
|
|
|
{ migrated: @migrated, errored: @errored }
|
|
end
|
|
|
|
def start_migration_threads
|
|
Array.new(@migration_threads) do
|
|
Thread.new do
|
|
while batch = @queue.pop
|
|
Rails.application.executor.wrap do
|
|
process_batch(batch)
|
|
end
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
def process_batch(batch)
|
|
batch.with_project_route_and_deployment.each do |metadatum|
|
|
project = metadatum.project
|
|
|
|
migrate_project(project)
|
|
end
|
|
|
|
@logger.info("#{@migrated} projects are migrated successfully, #{@errored} projects failed to be migrated")
|
|
rescue => e
|
|
# This method should never raise exception otherwise all threads might be killed
|
|
# and this will result in queue starving (and deadlock)
|
|
Gitlab::ErrorTracking.track_exception(e)
|
|
@logger.error("failed processing a batch: #{e.message}")
|
|
end
|
|
|
|
def migrate_project(project)
|
|
result = nil
|
|
time = Benchmark.realtime do
|
|
result = ::Pages::MigrateLegacyStorageToDeploymentService.new(project, ignore_invalid_entries: @ignore_invalid_entries).execute
|
|
end
|
|
|
|
if result[:status] == :success
|
|
@logger.info("project_id: #{project.id} #{project.pages_path} has been migrated in #{time.round(2)} seconds")
|
|
@counters_lock.synchronize { @migrated += 1 }
|
|
else
|
|
@logger.error("project_id: #{project.id} #{project.pages_path} failed to be migrated in #{time.round(2)} seconds: #{result[:message]}")
|
|
@counters_lock.synchronize { @errored += 1 }
|
|
end
|
|
rescue => e
|
|
@counters_lock.synchronize { @errored += 1 }
|
|
@logger.error("project_id: #{project&.id} #{project&.pages_path} failed to be migrated: #{e.message}")
|
|
Gitlab::ErrorTracking.track_exception(e, project_id: project&.id)
|
|
end
|
|
end
|
|
end
|