debian-mirror-gitlab/lib/tasks/gitlab/storage.rake

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

187 lines
7.4 KiB
Ruby
Raw Normal View History

2021-03-11 19:13:27 +05:30
# frozen_string_literal: true
2018-03-17 18:26:18 +05:30
namespace :gitlab do
namespace :storage do
desc 'GitLab | Storage | Migrate existing projects to Hashed Storage'
task migrate_to_hashed: :environment do
2019-02-15 15:39:39 +05:30
if Gitlab::Database.read_only?
2019-09-30 21:07:59 +05:30
abort 'This task requires database write access. Exiting.'
2019-02-15 15:39:39 +05:30
end
2018-11-08 19:23:39 +05:30
storage_migrator = Gitlab::HashedStorage::Migrator.new
helper = Gitlab::HashedStorage::RakeHelper
2019-07-07 11:18:12 +05:30
if storage_migrator.rollback_pending?
2019-09-30 21:07:59 +05:30
abort "There is already a rollback operation in progress, " \
2019-07-07 11:18:12 +05:30
"running a migration at the same time may have unexpected consequences."
end
2018-11-08 19:23:39 +05:30
if helper.range_single_item?
project = Project.with_unmigrated_storage.find_by(id: helper.range_from)
unless project
2019-09-30 21:07:59 +05:30
abort "There are no projects requiring storage migration with ID=#{helper.range_from}"
2018-11-08 19:23:39 +05:30
end
puts "Enqueueing storage migration of #{project.full_path} (ID=#{project.id})..."
storage_migrator.migrate(project)
2019-09-30 21:07:59 +05:30
else
legacy_projects_count = if helper.using_ranges?
Project.with_unmigrated_storage.id_in(helper.range_from..helper.range_to).count
else
Project.with_unmigrated_storage.count
end
if legacy_projects_count == 0
abort 'There are no projects requiring storage migration. Nothing to do!'
end
2018-11-08 19:23:39 +05:30
2019-09-30 21:07:59 +05:30
print "Enqueuing migration of #{legacy_projects_count} projects in batches of #{helper.batch_size}"
2018-03-17 18:26:18 +05:30
2019-09-30 21:07:59 +05:30
helper.project_id_batches_migration do |start, finish|
storage_migrator.bulk_schedule_migration(start: start, finish: finish)
2019-07-07 11:18:12 +05:30
2019-09-30 21:07:59 +05:30
print '.'
end
2019-07-07 11:18:12 +05:30
end
puts ' Done!'
end
desc 'GitLab | Storage | Rollback existing projects to Legacy Storage'
task rollback_to_legacy: :environment do
if Gitlab::Database.read_only?
2019-09-30 21:07:59 +05:30
abort 'This task requires database write access. Exiting.'
2019-07-07 11:18:12 +05:30
end
storage_migrator = Gitlab::HashedStorage::Migrator.new
helper = Gitlab::HashedStorage::RakeHelper
if storage_migrator.migration_pending?
2019-09-30 21:07:59 +05:30
abort "There is already a migration operation in progress, " \
2019-07-07 11:18:12 +05:30
"running a rollback at the same time may have unexpected consequences."
end
if helper.range_single_item?
project = Project.with_storage_feature(:repository).find_by(id: helper.range_from)
unless project
2019-09-30 21:07:59 +05:30
abort "There are no projects that can be rolledback with ID=#{helper.range_from}"
2019-07-07 11:18:12 +05:30
end
puts "Enqueueing storage rollback of #{project.full_path} (ID=#{project.id})..."
storage_migrator.rollback(project)
2019-09-30 21:07:59 +05:30
else
hashed_projects_count = if helper.using_ranges?
Project.with_storage_feature(:repository).id_in(helper.range_from..helper.range_to).count
else
Project.with_storage_feature(:repository).count
end
if hashed_projects_count == 0
abort 'There are no projects that can have storage rolledback. Nothing to do!'
end
2019-07-07 11:18:12 +05:30
2019-09-30 21:07:59 +05:30
print "Enqueuing rollback of #{hashed_projects_count} projects in batches of #{helper.batch_size}"
2019-07-07 11:18:12 +05:30
2019-09-30 21:07:59 +05:30
helper.project_id_batches_rollback do |start, finish|
storage_migrator.bulk_schedule_rollback(start: start, finish: finish)
2018-03-17 18:26:18 +05:30
2019-09-30 21:07:59 +05:30
print '.'
end
2018-03-17 18:26:18 +05:30
end
puts ' Done!'
end
desc 'Gitlab | Storage | Summary of existing projects using Legacy Storage'
task legacy_projects: :environment do
2021-09-04 01:27:46 +05:30
# Required to prevent Docker upgrade to 14.0 if there data on legacy storage
# See: https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/5311#note_590454698
wait_until_database_is_ready do
helper = Gitlab::HashedStorage::RakeHelper
helper.relation_summary('projects using Legacy Storage', Project.without_storage_feature(:repository))
end
2018-03-17 18:26:18 +05:30
end
desc 'Gitlab | Storage | List existing projects using Legacy Storage'
task list_legacy_projects: :environment do
2018-11-08 19:23:39 +05:30
helper = Gitlab::HashedStorage::RakeHelper
helper.projects_list('projects using Legacy Storage', Project.without_storage_feature(:repository))
2018-03-17 18:26:18 +05:30
end
desc 'Gitlab | Storage | Summary of existing projects using Hashed Storage'
task hashed_projects: :environment do
2018-11-08 19:23:39 +05:30
helper = Gitlab::HashedStorage::RakeHelper
helper.relation_summary('projects using Hashed Storage', Project.with_storage_feature(:repository))
2018-03-17 18:26:18 +05:30
end
desc 'Gitlab | Storage | List existing projects using Hashed Storage'
task list_hashed_projects: :environment do
2018-11-08 19:23:39 +05:30
helper = Gitlab::HashedStorage::RakeHelper
helper.projects_list('projects using Hashed Storage', Project.with_storage_feature(:repository))
2018-03-17 18:26:18 +05:30
end
2021-03-08 18:12:59 +05:30
desc 'Gitlab | Storage | Prune projects using Hashed Storage. Remove all hashed directories that do not have a project associated'
task prune_hashed_projects: [:environment, :gitlab_environment] do
if Rails.env.production?
abort('This destructive action may only be run in development')
end
helper = Gitlab::HashedStorage::RakeHelper
name = 'projects using Hashed Storage'
relation = Project.with_storage_feature(:repository)
root = Gitlab.config.repositories.storages['default'].legacy_disk_path
dry_run = !ENV['FORCE'].present?
helper.prune(name, relation, dry_run: dry_run, root: root)
end
2018-03-17 18:26:18 +05:30
desc 'Gitlab | Storage | Summary of project attachments using Legacy Storage'
task legacy_attachments: :environment do
2021-09-04 01:27:46 +05:30
# Required to prevent Docker upgrade to 14.0 if there data on legacy storage
# See: https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/5311#note_590454698
wait_until_database_is_ready do
helper = Gitlab::HashedStorage::RakeHelper
helper.relation_summary('attachments using Legacy Storage', helper.legacy_attachments_relation)
end
2018-03-17 18:26:18 +05:30
end
desc 'Gitlab | Storage | List existing project attachments using Legacy Storage'
task list_legacy_attachments: :environment do
2018-11-08 19:23:39 +05:30
helper = Gitlab::HashedStorage::RakeHelper
helper.attachments_list('attachments using Legacy Storage', helper.legacy_attachments_relation)
2018-03-17 18:26:18 +05:30
end
desc 'Gitlab | Storage | Summary of project attachments using Hashed Storage'
task hashed_attachments: :environment do
2018-11-08 19:23:39 +05:30
helper = Gitlab::HashedStorage::RakeHelper
helper.relation_summary('attachments using Hashed Storage', helper.hashed_attachments_relation)
2018-03-17 18:26:18 +05:30
end
desc 'Gitlab | Storage | List existing project attachments using Hashed Storage'
task list_hashed_attachments: :environment do
2018-11-08 19:23:39 +05:30
helper = Gitlab::HashedStorage::RakeHelper
helper.attachments_list('attachments using Hashed Storage', helper.hashed_attachments_relation)
2018-03-17 18:26:18 +05:30
end
2021-09-04 01:27:46 +05:30
def wait_until_database_is_ready
attempts = (ENV['MAX_DATABASE_CONNECTION_CHECKS'] || 1).to_i
inverval = (ENV['MAX_DATABASE_CONNECTION_CHECK_INTERVAL'] || 10).to_f
attempts.to_i.times do
2021-12-11 22:18:48 +05:30
unless ApplicationRecord.database.exists?
2021-09-04 01:27:46 +05:30
puts "Waiting until database is ready before continuing...".color(:yellow)
sleep inverval
end
end
yield
rescue ActiveRecord::ConnectionNotEstablished => ex
puts "Failed to connect to the database...".color(:red)
puts "Error: #{ex}"
exit 1
end
2018-03-17 18:26:18 +05:30
end
end