debian-mirror-gitlab/spec/tasks/gitlab/backup_rake_spec.rb

496 lines
18 KiB
Ruby
Raw Normal View History

2019-12-21 20:55:43 +05:30
# frozen_string_literal: true
2014-09-02 18:07:02 +05:30
require 'spec_helper'
require 'rake'
2020-07-28 23:09:34 +05:30
RSpec.describe 'gitlab:app namespace rake task', :delete do
2016-06-02 11:05:42 +05:30
let(:enable_registry) { true }
2018-03-17 18:26:18 +05:30
def tars_glob
Dir.glob(File.join(Gitlab.config.backup.path, '*_gitlab_backup.tar'))
end
def backup_tar
tars_glob.first
end
2020-04-08 14:13:33 +05:30
def backup_files
%w(backup_information.yml artifacts.tar.gz builds.tar.gz lfs.tar.gz pages.tar.gz)
end
def backup_directories
%w(db repositories)
end
2018-03-17 18:26:18 +05:30
before(:all) do
2020-04-22 19:07:51 +05:30
Rake.application.rake_require 'active_record/railties/databases'
2017-08-17 22:00:37 +05:30
Rake.application.rake_require 'tasks/gitlab/helpers'
2016-06-02 11:05:42 +05:30
Rake.application.rake_require 'tasks/gitlab/backup'
Rake.application.rake_require 'tasks/gitlab/shell'
Rake.application.rake_require 'tasks/gitlab/db'
2017-08-17 22:00:37 +05:30
Rake.application.rake_require 'tasks/cache'
2016-06-02 11:05:42 +05:30
2014-09-02 18:07:02 +05:30
# empty task as env is already loaded
Rake::Task.define_task :environment
2016-06-02 11:05:42 +05:30
end
before do
2018-03-17 18:26:18 +05:30
stub_env('force', 'yes')
FileUtils.rm(tars_glob, force: true)
2020-04-08 14:13:33 +05:30
FileUtils.rm(backup_files, force: true)
FileUtils.rm_rf(backup_directories, secure: true)
2018-03-17 18:26:18 +05:30
reenable_backup_sub_tasks
2016-06-02 11:05:42 +05:30
stub_container_registry_config(enabled: enable_registry)
2014-09-02 18:07:02 +05:30
end
2018-03-17 18:26:18 +05:30
after do
FileUtils.rm(tars_glob, force: true)
2020-04-08 14:13:33 +05:30
FileUtils.rm(backup_files, force: true)
FileUtils.rm_rf(backup_directories, secure: true)
2018-03-17 18:26:18 +05:30
end
2015-04-26 12:48:37 +05:30
def run_rake_task(task_name)
2019-07-07 11:18:12 +05:30
FileUtils.mkdir_p('tmp/tests/public/uploads')
2015-04-26 12:48:37 +05:30
Rake::Task[task_name].reenable
Rake.application.invoke_task task_name
end
2015-09-11 14:41:01 +05:30
def reenable_backup_sub_tasks
2017-08-17 22:00:37 +05:30
%w{db repo uploads builds artifacts pages lfs registry}.each do |subtask|
2015-09-11 14:41:01 +05:30
Rake::Task["gitlab:backup:#{subtask}:create"].reenable
end
end
2014-09-02 18:07:02 +05:30
describe 'backup_restore' do
context 'gitlab version' do
before do
2017-08-17 22:00:37 +05:30
allow(Dir).to receive(:glob).and_return(['1_gitlab_backup.tar'])
2016-09-13 17:45:13 +05:30
allow(File).to receive(:exist?).and_return(true)
2015-09-11 14:41:01 +05:30
allow(Kernel).to receive(:system).and_return(true)
allow(FileUtils).to receive(:cp_r).and_return(true)
allow(FileUtils).to receive(:mv).and_return(true)
2017-09-10 17:25:29 +05:30
allow(Rake::Task["gitlab:shell:setup"])
.to receive(:invoke).and_return(true)
2014-09-02 18:07:02 +05:30
end
let(:gitlab_version) { Gitlab::VERSION }
2019-03-02 22:35:43 +05:30
context 'restore with matching gitlab version' do
before do
allow(YAML).to receive(:load_file)
.and_return({ gitlab_version: gitlab_version })
expect(Rake::Task['gitlab:db:drop_tables']).to receive(:invoke)
expect(Rake::Task['gitlab:backup:db:restore']).to receive(:invoke)
expect(Rake::Task['gitlab:backup:repo:restore']).to receive(:invoke)
expect(Rake::Task['gitlab:backup:builds:restore']).to receive(:invoke)
expect(Rake::Task['gitlab:backup:uploads:restore']).to receive(:invoke)
expect(Rake::Task['gitlab:backup:artifacts:restore']).to receive(:invoke)
expect(Rake::Task['gitlab:backup:pages:restore']).to receive(:invoke)
expect(Rake::Task['gitlab:backup:lfs:restore']).to receive(:invoke)
expect(Rake::Task['gitlab:backup:registry:restore']).to receive(:invoke)
expect(Rake::Task['gitlab:shell:setup']).to receive(:invoke)
end
it 'invokes restoration on match' do
expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout
end
it 'prints timestamps on messages' do
expect { run_rake_task('gitlab:backup:restore') }.to output(/.*\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}\s[-+]\d{4}\s--\s.*/).to_stdout
end
2014-09-02 18:07:02 +05:30
end
end
2018-11-18 11:00:15 +05:30
context 'when the restore directory is not empty' do
before do
# We only need a backup of the repositories for this test
stub_env('SKIP', 'db,uploads,builds,artifacts,lfs,registry')
end
it 'removes stale data' do
expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout
excluded_project = create(:project, :repository, name: 'mepmep')
expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout
raw_repo = excluded_project.repository.raw
# The restore will not find the repository in the backup, but will create
# an empty one in its place
expect(raw_repo.empty?).to be(true)
end
end
2019-12-21 20:55:43 +05:30
context 'when the backup is restored' do
let!(:included_project) { create(:project, :repository) }
before do
expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout
backup_tar = Dir.glob(File.join(Gitlab.config.backup.path, '*_gitlab_backup.tar')).last
allow(Dir).to receive(:glob).and_return([backup_tar])
allow(File).to receive(:exist?).and_return(true)
allow(Kernel).to receive(:system).and_return(true)
allow(FileUtils).to receive(:cp_r).and_return(true)
allow(FileUtils).to receive(:mv).and_return(true)
allow(YAML).to receive(:load_file)
.and_return({ gitlab_version: Gitlab::VERSION })
expect(Rake::Task['gitlab:db:drop_tables']).to receive(:invoke)
expect(Rake::Task['gitlab:backup:db:restore']).to receive(:invoke)
expect(Rake::Task['gitlab:backup:repo:restore']).to receive(:invoke)
expect(Rake::Task['gitlab:backup:builds:restore']).to receive(:invoke)
expect(Rake::Task['gitlab:backup:uploads:restore']).to receive(:invoke)
expect(Rake::Task['gitlab:backup:artifacts:restore']).to receive(:invoke)
expect(Rake::Task['gitlab:backup:pages:restore']).to receive(:invoke)
expect(Rake::Task['gitlab:backup:lfs:restore']).to receive(:invoke)
expect(Rake::Task['gitlab:backup:registry:restore']).to receive(:invoke)
expect(Rake::Task['gitlab:shell:setup']).to receive(:invoke)
# We only need a backup of the repositories for this test
stub_env('SKIP', 'db,uploads,builds,artifacts,lfs,registry')
end
it 'restores the data' do
expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout
raw_repo = included_project.repository.raw
expect(raw_repo.empty?).to be(false)
end
end
2020-11-24 15:15:51 +05:30
end
# backup_restore task
2015-04-26 12:48:37 +05:30
2017-08-17 22:00:37 +05:30
describe 'backup' do
2018-03-17 18:26:18 +05:30
before do
2017-08-17 22:00:37 +05:30
# This reconnect makes our project fixture disappear, breaking the restore. Stub it out.
allow(ActiveRecord::Base.connection).to receive(:reconnect!)
end
describe 'backup creation and deletion using custom_hooks' do
let(:project) { create(:project, :repository) }
2017-09-10 17:25:29 +05:30
let(:user_backup_path) { "repositories/#{project.disk_path}" }
2017-08-17 22:00:37 +05:30
2018-03-17 18:26:18 +05:30
before do
stub_env('SKIP', 'db')
2018-11-08 19:23:39 +05:30
path = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
File.join(project.repository.path_to_repo, 'custom_hooks')
end
2017-08-17 22:00:37 +05:30
FileUtils.mkdir_p(path)
FileUtils.touch(File.join(path, "dummy.txt"))
end
context 'project uses custom_hooks and successfully creates backup' do
it 'creates custom_hooks.tar and project bundle' do
2018-03-17 18:26:18 +05:30
expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout
tar_contents, exit_status = Gitlab::Popen.popen(%W{tar -tvf #{backup_tar}})
2017-08-17 22:00:37 +05:30
expect(exit_status).to eq(0)
expect(tar_contents).to match(user_backup_path)
expect(tar_contents).to match("#{user_backup_path}/custom_hooks.tar")
expect(tar_contents).to match("#{user_backup_path}.bundle")
end
it 'restores files correctly' do
2018-03-17 18:26:18 +05:30
expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout
expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout
2017-08-17 22:00:37 +05:30
2018-11-08 19:23:39 +05:30
repo_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
project.repository.path
end
expect(Dir.entries(File.join(repo_path, 'custom_hooks'))).to include("dummy.txt")
end
end
context 'specific backup tasks' do
let(:task_list) { %w(db repo uploads builds artifacts pages lfs registry) }
it 'prints a progress message to stdout' do
task_list.each do |task|
expect { run_rake_task("gitlab:backup:#{task}:create") }.to output(/Dumping /).to_stdout
end
2017-08-17 22:00:37 +05:30
end
end
end
2016-08-24 12:49:21 +05:30
context 'tar creation' do
context 'archive file permissions' do
2016-09-13 17:45:13 +05:30
it 'sets correct permissions on the tar file' do
2018-03-17 18:26:18 +05:30
expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout
expect(File.exist?(backup_tar)).to be_truthy
expect(File::Stat.new(backup_tar).mode.to_s(8)).to eq('100600')
2015-09-11 14:41:01 +05:30
end
2016-08-24 12:49:21 +05:30
context 'with custom archive_permissions' do
before do
allow(Gitlab.config.backup).to receive(:archive_permissions).and_return(0651)
end
it 'uses the custom permissions' do
2018-03-17 18:26:18 +05:30
expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout
expect(File::Stat.new(backup_tar).mode.to_s(8)).to eq('100651')
2016-08-24 12:49:21 +05:30
end
2015-09-11 14:41:01 +05:30
end
end
2015-04-26 12:48:37 +05:30
2016-09-13 17:45:13 +05:30
it 'sets correct permissions on the tar contents' do
2018-03-17 18:26:18 +05:30
expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout
2016-08-24 12:49:21 +05:30
tar_contents, exit_status = Gitlab::Popen.popen(
2018-03-17 18:26:18 +05:30
%W{tar -tvf #{backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz registry.tar.gz}
2016-08-24 12:49:21 +05:30
)
2018-03-17 18:26:18 +05:30
2016-08-24 12:49:21 +05:30
expect(exit_status).to eq(0)
2020-04-08 14:13:33 +05:30
expect(tar_contents).to match('db')
2016-08-24 12:49:21 +05:30
expect(tar_contents).to match('uploads.tar.gz')
expect(tar_contents).to match('repositories/')
expect(tar_contents).to match('builds.tar.gz')
expect(tar_contents).to match('artifacts.tar.gz')
2017-08-17 22:00:37 +05:30
expect(tar_contents).to match('pages.tar.gz')
2016-08-24 12:49:21 +05:30
expect(tar_contents).to match('lfs.tar.gz')
expect(tar_contents).to match('registry.tar.gz')
2018-03-17 18:26:18 +05:30
expect(tar_contents).not_to match(%r{^.{4,9}[rwx].* (database.sql.gz|uploads.tar.gz|repositories|builds.tar.gz|pages.tar.gz|artifacts.tar.gz|registry.tar.gz)/$})
2016-08-24 12:49:21 +05:30
end
2016-09-13 17:45:13 +05:30
it 'deletes temp directories' do
2018-03-17 18:26:18 +05:30
expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout
2016-08-24 12:49:21 +05:30
temp_dirs = Dir.glob(
2017-08-17 22:00:37 +05:30
File.join(Gitlab.config.backup.path, '{db,repositories,uploads,builds,artifacts,pages,lfs,registry}')
2016-08-24 12:49:21 +05:30
)
2015-04-26 12:48:37 +05:30
2016-08-24 12:49:21 +05:30
expect(temp_dirs).to be_empty
end
context 'registry disabled' do
let(:enable_registry) { false }
2015-04-26 12:48:37 +05:30
2016-09-13 17:45:13 +05:30
it 'does not create registry.tar.gz' do
2018-03-17 18:26:18 +05:30
expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout
2016-08-24 12:49:21 +05:30
tar_contents, exit_status = Gitlab::Popen.popen(
2018-03-17 18:26:18 +05:30
%W{tar -tvf #{backup_tar}}
2016-08-24 12:49:21 +05:30
)
2018-03-17 18:26:18 +05:30
2016-08-24 12:49:21 +05:30
expect(exit_status).to eq(0)
expect(tar_contents).not_to match('registry.tar.gz')
end
end
2015-04-26 12:48:37 +05:30
end
2016-06-02 11:05:42 +05:30
2016-08-24 12:49:21 +05:30
context 'multiple repository storages' do
2021-01-03 14:25:43 +05:30
include StubConfiguration
let(:default_storage_name) { 'default' }
let(:second_storage_name) { 'test_second_storage' }
2018-05-09 12:01:36 +05:30
2018-03-17 18:26:18 +05:30
before do
# We only need a backup of the repositories for this test
stub_env('SKIP', 'db,uploads,builds,artifacts,lfs,registry')
2021-01-03 14:25:43 +05:30
stub_storage_settings( second_storage_name => {
'gitaly_address' => Gitlab.config.repositories.storages.default.gitaly_address,
'path' => TestEnv::SECOND_STORAGE_PATH
})
2020-10-24 23:57:45 +05:30
end
2021-01-03 14:25:43 +05:30
shared_examples 'includes repositories in all repository storages' do
specify :aggregate_failures do
project_a = create(:project, :repository)
project_a.track_project_repository
project_snippet_a = create(:project_snippet, :repository, project: project_a, author: project_a.owner)
project_b = create(:project, :repository, repository_storage: second_storage_name)
project_b.track_project_repository
project_snippet_b = create(:project_snippet, :repository, project: project_b, author: project_b.owner)
project_snippet_b.snippet_repository.update!(shard: project_b.project_repository.shard)
create(:wiki_page, container: project_a)
create(:design, :with_file, issue: create(:issue, project: project_a))
move_repository_to_secondary(project_b)
move_repository_to_secondary(project_snippet_b)
2020-10-24 23:57:45 +05:30
expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout
tar_contents, exit_status = Gitlab::Popen.popen(
%W{tar -tvf #{backup_tar} repositories}
)
2021-01-03 14:25:43 +05:30
tar_lines = tar_contents.lines.grep(/\.bundle/)
2020-10-24 23:57:45 +05:30
expect(exit_status).to eq(0)
2021-01-03 14:25:43 +05:30
[
"#{project_a.disk_path}.bundle",
"#{project_a.disk_path}.wiki.bundle",
"#{project_a.disk_path}.design.bundle",
"#{project_b.disk_path}.bundle",
"#{project_snippet_a.disk_path}.bundle",
"#{project_snippet_b.disk_path}.bundle"
].each do |repo_name|
expect(tar_lines.grep(/#{repo_name}/).size).to eq 1
end
2020-10-24 23:57:45 +05:30
end
2021-01-03 14:25:43 +05:30
def move_repository_to_secondary(record)
Gitlab::GitalyClient::StorageSettings.allow_disk_access do
default_shard_legacy_path = Gitlab.config.repositories.storages.default.legacy_disk_path
secondary_legacy_path = Gitlab.config.repositories.storages[second_storage_name].legacy_disk_path
dst_dir = File.join(secondary_legacy_path, File.dirname(record.disk_path))
FileUtils.mkdir_p(dst_dir) unless Dir.exist?(dst_dir)
FileUtils.mv(
File.join(default_shard_legacy_path, record.disk_path + '.git'),
File.join(secondary_legacy_path, record.disk_path + '.git')
)
end
end
end
context 'no concurrency' do
it_behaves_like 'includes repositories in all repository storages'
2020-10-24 23:57:45 +05:30
end
context 'with concurrency' do
before do
stub_env('GITLAB_BACKUP_MAX_CONCURRENCY', 4)
end
2021-01-03 14:25:43 +05:30
it_behaves_like 'includes repositories in all repository storages'
2020-10-24 23:57:45 +05:30
end
end
context 'concurrency settings' do
before do
# We only need a backup of the repositories for this test
stub_env('SKIP', 'db,uploads,builds,artifacts,lfs,registry')
create(:project, :repository)
end
it 'has defaults' do
2021-01-03 14:25:43 +05:30
expect_next_instance_of(::Backup::Repositories) do |instance|
2020-10-24 23:57:45 +05:30
expect(instance).to receive(:dump)
.with(max_concurrency: 1, max_storage_concurrency: 1)
.and_call_original
end
2018-10-15 14:42:47 +05:30
2018-03-17 18:26:18 +05:30
expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout
2020-10-24 23:57:45 +05:30
end
2018-03-17 18:26:18 +05:30
2020-10-24 23:57:45 +05:30
it 'passes through concurrency environment variables' do
stub_env('GITLAB_BACKUP_MAX_CONCURRENCY', 5)
stub_env('GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY', 2)
2018-03-17 18:26:18 +05:30
2021-01-03 14:25:43 +05:30
expect_next_instance_of(::Backup::Repositories) do |instance|
2020-10-24 23:57:45 +05:30
expect(instance).to receive(:dump)
.with(max_concurrency: 5, max_storage_concurrency: 2)
.and_call_original
end
expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout
2016-06-02 11:05:42 +05:30
end
end
2020-11-24 15:15:51 +05:30
end
# backup_create task
2015-04-26 12:48:37 +05:30
describe "Skipping items" do
2018-03-17 18:26:18 +05:30
before do
stub_env('SKIP', 'repositories,uploads')
2015-04-26 12:48:37 +05:30
end
it "does not contain skipped item" do
2018-03-17 18:26:18 +05:30
expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout
2015-10-24 18:46:33 +05:30
tar_contents, _exit_status = Gitlab::Popen.popen(
2018-03-17 18:26:18 +05:30
%W{tar -tvf #{backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz registry.tar.gz}
2015-04-26 12:48:37 +05:30
)
expect(tar_contents).to match('db/')
2015-11-26 14:37:03 +05:30
expect(tar_contents).to match('uploads.tar.gz')
expect(tar_contents).to match('builds.tar.gz')
expect(tar_contents).to match('artifacts.tar.gz')
expect(tar_contents).to match('lfs.tar.gz')
2017-08-17 22:00:37 +05:30
expect(tar_contents).to match('pages.tar.gz')
2016-06-02 11:05:42 +05:30
expect(tar_contents).to match('registry.tar.gz')
2015-04-26 12:48:37 +05:30
expect(tar_contents).not_to match('repositories/')
end
it 'does not invoke repositories restore' do
2018-03-17 18:26:18 +05:30
expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout
2017-09-10 17:25:29 +05:30
allow(Rake::Task['gitlab:shell:setup'])
.to receive(:invoke).and_return(true)
2015-04-26 12:48:37 +05:30
2016-06-02 11:05:42 +05:30
expect(Rake::Task['gitlab:db:drop_tables']).to receive :invoke
expect(Rake::Task['gitlab:backup:db:restore']).to receive :invoke
expect(Rake::Task['gitlab:backup:repo:restore']).not_to receive :invoke
expect(Rake::Task['gitlab:backup:uploads:restore']).not_to receive :invoke
expect(Rake::Task['gitlab:backup:builds:restore']).to receive :invoke
expect(Rake::Task['gitlab:backup:artifacts:restore']).to receive :invoke
2017-08-17 22:00:37 +05:30
expect(Rake::Task['gitlab:backup:pages:restore']).to receive :invoke
2016-06-02 11:05:42 +05:30
expect(Rake::Task['gitlab:backup:lfs:restore']).to receive :invoke
expect(Rake::Task['gitlab:backup:registry:restore']).to receive :invoke
expect(Rake::Task['gitlab:shell:setup']).to receive :invoke
2018-03-17 18:26:18 +05:30
expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout
2015-04-26 12:48:37 +05:30
end
end
2017-08-17 22:00:37 +05:30
2020-04-08 14:13:33 +05:30
describe 'skipping tar archive creation' do
before do
stub_env('SKIP', 'tar')
end
it 'created files with backup content and no tar archive' do
expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout
dir_contents = Dir.children(Gitlab.config.backup.path)
expect(dir_contents).to contain_exactly(
'backup_information.yml',
'db',
'uploads.tar.gz',
'builds.tar.gz',
'artifacts.tar.gz',
'lfs.tar.gz',
'pages.tar.gz',
'registry.tar.gz',
'repositories',
'tmp'
)
end
it 'those component files can be restored from' do
expect { run_rake_task("gitlab:backup:create") }.to output.to_stdout
allow(Rake::Task['gitlab:shell:setup'])
.to receive(:invoke).and_return(true)
expect(Rake::Task['gitlab:db:drop_tables']).to receive :invoke
expect(Rake::Task['gitlab:backup:db:restore']).to receive :invoke
expect(Rake::Task['gitlab:backup:repo:restore']).to receive :invoke
expect(Rake::Task['gitlab:backup:uploads:restore']).to receive :invoke
expect(Rake::Task['gitlab:backup:builds:restore']).to receive :invoke
expect(Rake::Task['gitlab:backup:artifacts:restore']).to receive :invoke
expect(Rake::Task['gitlab:backup:pages:restore']).to receive :invoke
expect(Rake::Task['gitlab:backup:lfs:restore']).to receive :invoke
expect(Rake::Task['gitlab:backup:registry:restore']).to receive :invoke
expect(Rake::Task['gitlab:shell:setup']).to receive :invoke
expect { run_rake_task("gitlab:backup:restore") }.to output.to_stdout
end
end
2017-08-17 22:00:37 +05:30
describe "Human Readable Backup Name" do
it 'name has human readable time' do
2018-03-17 18:26:18 +05:30
expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout
expect(backup_tar).to match(/\d+_\d{4}_\d{2}_\d{2}_\d+\.\d+\.\d+.*_gitlab_backup.tar$/)
2017-08-17 22:00:37 +05:30
end
end
2020-11-24 15:15:51 +05:30
end
# gitlab:app namespace