2021-03-11 19:13:27 +05:30
# frozen_string_literal: true
module Pages
class MigrateFromLegacyStorageService
2021-04-29 21:17:54 +05:30
def initialize ( logger , ignore_invalid_entries : , mark_projects_as_not_deployed : )
2021-03-11 19:13:27 +05:30
@logger = logger
@ignore_invalid_entries = ignore_invalid_entries
2021-04-29 21:17:54 +05:30
@mark_projects_as_not_deployed = mark_projects_as_not_deployed
2021-03-11 19:13:27 +05:30
@migrated = 0
@errored = 0
@counters_lock = Mutex . new
end
2021-04-29 21:17:54 +05:30
def execute_with_threads ( threads : , batch_size : )
2021-03-11 19:13:27 +05:30
@queue = SizedQueue . new ( 1 )
2021-04-29 21:17:54 +05:30
migration_threads = start_migration_threads ( threads )
2021-03-11 19:13:27 +05:30
2021-04-29 21:17:54 +05:30
ProjectPagesMetadatum . only_on_legacy_storage . each_batch ( of : batch_size ) do | batch |
2021-03-11 19:13:27 +05:30
@queue . push ( batch )
end
@queue . close
2021-04-29 21:17:54 +05:30
@logger . info ( message : " Pages legacy storage migration: Waiting for threads to finish... " )
migration_threads . each ( & :join )
2021-03-11 19:13:27 +05:30
{ migrated : @migrated , errored : @errored }
end
2021-04-29 21:17:54 +05:30
def execute_for_batch ( project_ids )
batch = ProjectPagesMetadatum . only_on_legacy_storage . where ( project_id : project_ids ) # rubocop: disable CodeReuse/ActiveRecord
process_batch ( batch )
{ migrated : @migrated , errored : @errored }
end
private
def start_migration_threads ( count )
Array . new ( count ) do
2021-03-11 19:13:27 +05:30
Thread . new do
while batch = @queue . pop
Rails . application . executor . wrap do
process_batch ( batch )
end
end
end
end
end
def process_batch ( batch )
batch . with_project_route_and_deployment . each do | metadatum |
project = metadatum . project
migrate_project ( project )
end
2021-04-29 21:17:54 +05:30
@logger . info ( message : " Pages legacy storage migration: batch processed " , migrated : @migrated , errored : @errored )
2021-06-08 01:23:25 +05:30
rescue StandardError = > e
2021-03-11 19:13:27 +05:30
# This method should never raise exception otherwise all threads might be killed
# and this will result in queue starving (and deadlock)
Gitlab :: ErrorTracking . track_exception ( e )
2021-04-29 21:17:54 +05:30
@logger . error ( message : " Pages legacy storage migration: failed processing a batch: #{ e . message } " )
2021-03-11 19:13:27 +05:30
end
def migrate_project ( project )
result = nil
time = Benchmark . realtime do
2021-04-29 21:17:54 +05:30
result = :: Pages :: MigrateLegacyStorageToDeploymentService . new ( project ,
ignore_invalid_entries : @ignore_invalid_entries ,
mark_projects_as_not_deployed : @mark_projects_as_not_deployed ) . execute
2021-03-11 19:13:27 +05:30
end
if result [ :status ] == :success
2021-04-29 21:17:54 +05:30
@logger . info ( message : " Pages legacy storage migration: project migrated: #{ result [ :message ] } " , project_id : project . id , pages_path : project . pages_path , duration : time . round ( 2 ) )
2021-03-11 19:13:27 +05:30
@counters_lock . synchronize { @migrated += 1 }
else
2021-04-29 21:17:54 +05:30
@logger . error ( message : " Pages legacy storage migration: project failed to be migrated: #{ result [ :message ] } " , project_id : project . id , pages_path : project . pages_path , duration : time . round ( 2 ) )
2021-03-11 19:13:27 +05:30
@counters_lock . synchronize { @errored += 1 }
end
2021-06-08 01:23:25 +05:30
rescue StandardError = > e
2021-03-11 19:13:27 +05:30
@counters_lock . synchronize { @errored += 1 }
2021-04-29 21:17:54 +05:30
@logger . error ( message : " Pages legacy storage migration: project failed to be migrated: #{ result [ :message ] } " , project_id : project & . id , pages_path : project & . pages_path )
2021-03-11 19:13:27 +05:30
Gitlab :: ErrorTracking . track_exception ( e , project_id : project & . id )
end
end
end