debian-mirror-gitlab/lib/gitlab/hashed_storage/migrator.rb

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

126 lines
4.4 KiB
Ruby
Raw Normal View History

2019-02-15 15:39:39 +05:30
# frozen_string_literal: true
2018-11-08 19:23:39 +05:30
module Gitlab
module HashedStorage
# Hashed Storage Migrator
#
# This is responsible for scheduling and flagging projects
# to be migrated from Legacy to Hashed storage, either one by one or in bulk.
class Migrator
BATCH_SIZE = 100
# Schedule a range of projects to be bulk migrated with #bulk_migrate asynchronously
#
2019-03-02 22:35:43 +05:30
# @param [Integer] start first project id for the range
# @param [Integer] finish last project id for the range
2019-07-07 11:18:12 +05:30
def bulk_schedule_migration(start:, finish:)
2019-03-02 22:35:43 +05:30
::HashedStorage::MigratorWorker.perform_async(start, finish)
2018-11-08 19:23:39 +05:30
end
2019-07-07 11:18:12 +05:30
# Schedule a range of projects to be bulk rolledback with #bulk_rollback asynchronously
#
# @param [Integer] start first project id for the range
# @param [Integer] finish last project id for the range
def bulk_schedule_rollback(start:, finish:)
::HashedStorage::RollbackerWorker.perform_async(start, finish)
end
2018-11-08 19:23:39 +05:30
# Start migration of projects from specified range
#
2019-03-02 22:35:43 +05:30
# Flagging a project to be migrated is a synchronous action
2018-11-08 19:23:39 +05:30
# but the migration runs through async jobs
#
2019-03-02 22:35:43 +05:30
# @param [Integer] start first project id for the range
# @param [Integer] finish last project id for the range
2018-12-05 23:21:45 +05:30
# rubocop: disable CodeReuse/ActiveRecord
2019-03-02 22:35:43 +05:30
def bulk_migrate(start:, finish:)
2018-11-08 19:23:39 +05:30
projects = build_relation(start, finish)
projects.with_route.find_each(batch_size: BATCH_SIZE) do |project|
migrate(project)
end
end
2018-12-05 23:21:45 +05:30
# rubocop: enable CodeReuse/ActiveRecord
2018-11-08 19:23:39 +05:30
2019-07-07 11:18:12 +05:30
# Start rollback of projects from specified range
#
# Flagging a project to be rolled back is a synchronous action
# but the rollback runs through async jobs
#
# @param [Integer] start first project id for the range
# @param [Integer] finish last project id for the range
# rubocop: disable CodeReuse/ActiveRecord
def bulk_rollback(start:, finish:)
projects = build_relation(start, finish)
projects.with_route.find_each(batch_size: BATCH_SIZE) do |project|
rollback(project)
end
end
# rubocop: enable CodeReuse/ActiveRecord
2019-03-02 22:35:43 +05:30
# Flag a project to be migrated to Hashed Storage
2018-11-08 19:23:39 +05:30
#
2019-03-02 22:35:43 +05:30
# @param [Project] project that will be migrated
2018-11-08 19:23:39 +05:30
def migrate(project)
2020-11-24 15:15:51 +05:30
Gitlab::AppLogger.info "Starting storage migration of #{project.full_path} (ID=#{project.id})..."
2018-11-08 19:23:39 +05:30
project.migrate_to_hashed_storage!
2021-06-08 01:23:25 +05:30
rescue StandardError => err
2020-11-24 15:15:51 +05:30
Gitlab::AppLogger.error("#{err.message} migrating storage of #{project.full_path} (ID=#{project.id}), trace - #{err.backtrace}")
2018-11-08 19:23:39 +05:30
end
2019-07-07 11:18:12 +05:30
# Flag a project to be rolled-back to Legacy Storage
#
# @param [Project] project that will be rolled-back
2019-03-02 22:35:43 +05:30
def rollback(project)
2020-11-24 15:15:51 +05:30
Gitlab::AppLogger.info "Starting storage rollback of #{project.full_path} (ID=#{project.id})..."
2019-07-07 11:18:12 +05:30
project.rollback_to_legacy_storage!
2021-06-08 01:23:25 +05:30
rescue StandardError => err
2020-11-24 15:15:51 +05:30
Gitlab::AppLogger.error("#{err.message} rolling-back storage of #{project.full_path} (ID=#{project.id}), trace - #{err.backtrace}")
2019-07-07 11:18:12 +05:30
end
# Returns whether we have any pending storage migration
#
def migration_pending?
any_non_empty_queue?(::HashedStorage::MigratorWorker, ::HashedStorage::ProjectMigrateWorker)
end
# Returns whether we have any pending storage rollback
#
def rollback_pending?
any_non_empty_queue?(::HashedStorage::RollbackerWorker, ::HashedStorage::ProjectRollbackWorker)
2019-03-02 22:35:43 +05:30
end
2020-11-24 15:15:51 +05:30
# Remove all remaining scheduled rollback operations
#
def abort_rollback!
[::HashedStorage::RollbackerWorker, ::HashedStorage::ProjectRollbackWorker].each do |worker|
Sidekiq::Queue.new(worker.queue).clear
end
end
2018-11-08 19:23:39 +05:30
private
2019-07-07 11:18:12 +05:30
def any_non_empty_queue?(*workers)
workers.any? do |worker|
2020-10-24 23:57:45 +05:30
Sidekiq::Queue.new(worker.queue).size != 0 # rubocop:disable Style/ZeroLengthPredicate
2019-07-07 11:18:12 +05:30
end
end
2018-12-05 23:21:45 +05:30
# rubocop: disable CodeReuse/ActiveRecord
2018-11-08 19:23:39 +05:30
def build_relation(start, finish)
relation = Project
table = Project.arel_table
relation = relation.where(table[:id].gteq(start)) if start
relation = relation.where(table[:id].lteq(finish)) if finish
relation
end
2018-12-05 23:21:45 +05:30
# rubocop: enable CodeReuse/ActiveRecord
2018-11-08 19:23:39 +05:30
end
end
end