debian-mirror-gitlab/lib/backup/repositories.rb

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

135 lines
3.8 KiB
Ruby
Raw Normal View History

2021-01-03 14:25:43 +05:30
# frozen_string_literal: true
require 'yaml'
module Backup
2022-07-23 23:45:48 +05:30
# Backup and restores repositories by querying the database
2022-05-07 20:08:51 +05:30
class Repositories < Task
extend ::Gitlab::Utils::Override
2022-07-23 23:45:48 +05:30
# @param [IO] progress IO interface to output progress
# @param [Object] :strategy Fetches backups from gitaly
# @param [Array<String>] :storages Filter by specified storage names. Empty means all storages.
# @param [Array<String>] :paths Filter by specified project paths. Empty means all projects, groups and snippets.
def initialize(progress, strategy:, storages: [], paths: [])
2022-05-07 20:08:51 +05:30
super(progress)
2021-09-04 01:27:46 +05:30
@strategy = strategy
2022-07-16 23:28:13 +05:30
@storages = storages
2022-07-23 23:45:48 +05:30
@paths = paths
2021-01-03 14:25:43 +05:30
end
2022-05-07 20:08:51 +05:30
override :dump
2022-07-23 23:45:48 +05:30
def dump(destination_path, backup_id)
strategy.start(:create, destination_path, backup_id: backup_id)
2022-06-21 17:19:12 +05:30
enqueue_consecutive
2021-01-03 14:25:43 +05:30
2021-09-04 01:27:46 +05:30
ensure
2022-03-02 08:16:31 +05:30
strategy.finish!
2021-01-03 14:25:43 +05:30
end
2022-05-07 20:08:51 +05:30
override :restore
2022-07-23 23:45:48 +05:30
def restore(destination_path)
strategy.start(:restore, destination_path)
2021-09-04 01:27:46 +05:30
enqueue_consecutive
ensure
2022-03-02 08:16:31 +05:30
strategy.finish!
2021-03-08 18:12:59 +05:30
2021-09-04 01:27:46 +05:30
cleanup_snippets_without_repositories
2021-03-08 18:12:59 +05:30
restore_object_pools
end
private
2022-07-23 23:45:48 +05:30
attr_reader :strategy, :storages, :paths
2021-03-08 18:12:59 +05:30
2021-09-04 01:27:46 +05:30
def enqueue_consecutive
enqueue_consecutive_projects
enqueue_consecutive_snippets
2021-01-03 14:25:43 +05:30
end
2021-09-04 01:27:46 +05:30
def enqueue_consecutive_projects
2021-01-03 14:25:43 +05:30
project_relation.find_each(batch_size: 1000) do |project|
2021-09-04 01:27:46 +05:30
enqueue_project(project)
2021-01-03 14:25:43 +05:30
end
end
2021-09-04 01:27:46 +05:30
def enqueue_consecutive_snippets
2022-07-16 23:28:13 +05:30
snippet_relation.find_each(batch_size: 1000) { |snippet| enqueue_snippet(snippet) }
2021-01-03 14:25:43 +05:30
end
2021-09-04 01:27:46 +05:30
def enqueue_project(project)
strategy.enqueue(project, Gitlab::GlRepository::PROJECT)
strategy.enqueue(project, Gitlab::GlRepository::WIKI)
strategy.enqueue(project, Gitlab::GlRepository::DESIGN)
2021-01-03 14:25:43 +05:30
end
2021-09-04 01:27:46 +05:30
def enqueue_snippet(snippet)
strategy.enqueue(snippet, Gitlab::GlRepository::SNIPPET)
2021-01-03 14:25:43 +05:30
end
def project_relation
2022-07-16 23:28:13 +05:30
scope = Project.includes(:route, :group, namespace: :owner)
scope = scope.id_in(ProjectRepository.for_repository_storage(storages).select(:project_id)) if storages.any?
2022-07-23 23:45:48 +05:30
if paths.any?
scope = scope.where_full_path_in(paths).or(
Project.where(namespace_id: Namespace.where_full_path_in(paths).self_and_descendants)
)
end
2022-07-16 23:28:13 +05:30
scope
end
def snippet_relation
scope = Snippet.all
scope = scope.id_in(SnippetRepository.for_repository_storage(storages).select(:snippet_id)) if storages.any?
2022-07-23 23:45:48 +05:30
if paths.any?
scope = scope.joins(:project).merge(
Project.where_full_path_in(paths).or(
Project.where(namespace_id: Namespace.where_full_path_in(paths).self_and_descendants)
)
)
end
2022-07-16 23:28:13 +05:30
scope
2021-01-03 14:25:43 +05:30
end
def restore_object_pools
PoolRepository.includes(:source_project).find_each do |pool|
progress.puts " - Object pool #{pool.disk_path}..."
2021-04-17 20:07:23 +05:30
unless pool.source_project
progress.puts " - Object pool #{pool.disk_path}... " + "[SKIPPED]".color(:cyan)
next
end
2021-01-03 14:25:43 +05:30
pool.state = 'none'
pool.save
pool.schedule
end
end
# Snippets without a repository should be removed because they failed to import
# due to having invalid repositories
2021-09-04 01:27:46 +05:30
def cleanup_snippets_without_repositories
invalid_snippets = []
2021-01-03 14:25:43 +05:30
2022-07-16 23:28:13 +05:30
snippet_relation.find_each(batch_size: 1000).each do |snippet|
2021-09-04 01:27:46 +05:30
response = Snippets::RepositoryValidationService.new(nil, snippet).execute
next if response.success?
2021-01-03 14:25:43 +05:30
2021-09-04 01:27:46 +05:30
snippet.repository.remove
progress.puts("Snippet #{snippet.full_path} can't be restored: #{response.message}")
2021-01-03 14:25:43 +05:30
2021-09-04 01:27:46 +05:30
invalid_snippets << snippet.id
2021-01-03 14:25:43 +05:30
end
2021-09-04 01:27:46 +05:30
Snippet.id_in(invalid_snippets).delete_all
2021-01-03 14:25:43 +05:30
end
end
end
2021-03-08 18:12:59 +05:30
2021-06-08 01:23:25 +05:30
Backup::Repositories.prepend_mod_with('Backup::Repositories')