debian-mirror-gitlab/spec/tasks/gitlab/backup_rake_spec.rb

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

701 lines
26 KiB
Ruby
Raw Permalink Normal View History

2019-12-21 20:55:43 +05:30
# frozen_string_literal: true
2021-09-04 01:27:46 +05:30
require 'rake_helper'
2014-09-02 18:07:02 +05:30
2023-04-23 21:23:45 +05:30
RSpec.describe 'gitlab:backup namespace rake tasks', :delete, feature_category: :backup_restore do
2016-06-02 11:05:42 +05:30
let(:enable_registry) { true }
2022-11-25 23:54:43 +05:30
let(:backup_restore_pid_path) { "#{Rails.application.root}/tmp/backup_restore.pid" }
let(:backup_tasks) { %w[db repo uploads builds artifacts pages lfs terraform_state registry packages] }
2022-08-27 11:52:29 +05:30
let(:backup_types) do
2023-04-23 21:23:45 +05:30
%w[db repositories uploads builds artifacts pages lfs terraform_state registry packages]
2022-08-27 11:52:29 +05:30
end
2016-06-02 11:05:42 +05:30
2018-03-17 18:26:18 +05:30
def tars_glob
Dir.glob(File.join(Gitlab.config.backup.path, '*_gitlab_backup.tar'))
end
def backup_tar
tars_glob.first
end
2020-04-08 14:13:33 +05:30
def backup_files
2022-11-25 23:54:43 +05:30
%w[
backup_information.yml
artifacts.tar.gz
builds.tar.gz
lfs.tar.gz
terraform_state.tar.gz
pages.tar.gz
packages.tar.gz
]
2020-04-08 14:13:33 +05:30
end
def backup_directories
2022-11-25 23:54:43 +05:30
%w[db repositories]
2020-04-08 14:13:33 +05:30
end
2018-03-17 18:26:18 +05:30
before(:all) do
2020-04-22 19:07:51 +05:30
Rake.application.rake_require 'active_record/railties/databases'
2016-06-02 11:05:42 +05:30
Rake.application.rake_require 'tasks/gitlab/backup'
Rake.application.rake_require 'tasks/gitlab/shell'
Rake.application.rake_require 'tasks/gitlab/db'
2017-08-17 22:00:37 +05:30
Rake.application.rake_require 'tasks/cache'
2016-06-02 11:05:42 +05:30
end
before do
2018-03-17 18:26:18 +05:30
stub_env('force', 'yes')
FileUtils.rm(tars_glob, force: true)
2020-04-08 14:13:33 +05:30
FileUtils.rm(backup_files, force: true)
FileUtils.rm_rf(backup_directories, secure: true)
2021-09-04 01:27:46 +05:30
FileUtils.mkdir_p('tmp/tests/public/uploads')
2018-03-17 18:26:18 +05:30
reenable_backup_sub_tasks
2016-06-02 11:05:42 +05:30
stub_container_registry_config(enabled: enable_registry)
2014-09-02 18:07:02 +05:30
end
2018-03-17 18:26:18 +05:30
after do
FileUtils.rm(tars_glob, force: true)
2020-04-08 14:13:33 +05:30
FileUtils.rm(backup_files, force: true)
2023-07-09 08:55:56 +05:30
FileUtils.rm(backup_restore_pid_path, force: true)
2020-04-08 14:13:33 +05:30
FileUtils.rm_rf(backup_directories, secure: true)
2021-09-04 01:27:46 +05:30
FileUtils.rm_rf('tmp/tests/public/uploads', secure: true)
2015-04-26 12:48:37 +05:30
end
2015-09-11 14:41:01 +05:30
def reenable_backup_sub_tasks
2022-04-04 11:22:00 +05:30
backup_tasks.each do |subtask|
2015-09-11 14:41:01 +05:30
Rake::Task["gitlab:backup:#{subtask}:create"].reenable
end
end
2022-11-25 23:54:43 +05:30
describe 'lock parallel backups' do
2023-07-09 08:55:56 +05:30
let(:progress) { $stdout }
let(:delete_message) { /-- Deleting backup and restore PID file/ }
let(:pid_file) do
File.open(backup_restore_pid_path, File::RDWR | File::CREAT)
end
2022-11-25 23:54:43 +05:30
2023-07-09 08:55:56 +05:30
before do
allow(Kernel).to receive(:system).and_return(true)
allow(YAML).to receive(:safe_load_file).and_return({ gitlab_version: Gitlab::VERSION })
end
context 'when a process is running in parallel' do
before do
File.open(backup_restore_pid_path, 'wb') do |file|
file.write('123456')
file.close
end
end
2022-11-25 23:54:43 +05:30
it 'exits the new process' do
allow(File).to receive(:open).and_call_original
allow(File).to receive(:open).with(backup_restore_pid_path, any_args).and_yield(pid_file)
2023-07-09 08:55:56 +05:30
allow(Process).to receive(:getpgid).with(123456).and_return(123456)
2022-11-25 23:54:43 +05:30
expect { run_rake_task('gitlab:backup:create') }.to raise_error(SystemExit).and output(
2023-07-09 08:55:56 +05:30
<<~MESSAGE
2022-11-25 23:54:43 +05:30
Backup and restore in progress:
2023-07-09 08:55:56 +05:30
There is a backup and restore task in progress (PID 123456).
Try to run the current task once the previous one ends.
MESSAGE
2022-11-25 23:54:43 +05:30
).to_stdout
end
end
2023-07-09 08:55:56 +05:30
context 'when no process is running in parallel but a PID file exists' do
let(:rewritten_message) do
<<~MESSAGE
The PID file #{backup_restore_pid_path} exists and contains 123456, but the process is not running.
The PID file will be rewritten with the current process ID #{Process.pid}.
MESSAGE
end
2022-11-25 23:54:43 +05:30
2023-07-09 08:55:56 +05:30
before do
File.open(backup_restore_pid_path, 'wb') do |file|
file.write('123456')
file.close
end
end
it 'rewrites, locks and deletes the PID file while logging a message' do
allow(File).to receive(:open).and_call_original
allow(File).to receive(:open).with(backup_restore_pid_path, any_args).and_yield(pid_file)
allow(Process).to receive(:getpgid).with(123456).and_raise(Errno::ESRCH)
allow(progress).to receive(:puts).with(delete_message).once
allow(progress).to receive(:puts).with(rewritten_message).once
allow_next_instance_of(::Backup::Manager) do |instance|
allow(instance).to receive(:run_restore_task).with('db')
end
expect(pid_file).to receive(:flock).with(File::LOCK_EX)
expect(pid_file).to receive(:flock).with(File::LOCK_UN)
expect(File).to receive(:delete).with(backup_restore_pid_path)
expect(progress).to receive(:puts).with(rewritten_message).once
expect(progress).to receive(:puts).with(delete_message).once
run_rake_task('gitlab:backup:db:restore')
end
end
context 'when no process is running in parallel' do
using RSpec::Parameterized::TableSyntax
where(:task_name, :rake_task) do
2023-04-23 21:23:45 +05:30
'db' | 'gitlab:backup:db:restore'
2022-11-25 23:54:43 +05:30
'repositories' | 'gitlab:backup:repo:restore'
'builds' | 'gitlab:backup:builds:restore'
'uploads' | 'gitlab:backup:uploads:restore'
'artifacts' | 'gitlab:backup:artifacts:restore'
'pages' | 'gitlab:backup:pages:restore'
'lfs' | 'gitlab:backup:lfs:restore'
'terraform_state' | 'gitlab:backup:terraform_state:restore'
'registry' | 'gitlab:backup:registry:restore'
'packages' | 'gitlab:backup:packages:restore'
end
with_them do
before do
allow(File).to receive(:open).and_call_original
allow(File).to receive(:open).with(backup_restore_pid_path, any_args).and_yield(pid_file)
2023-07-09 08:55:56 +05:30
allow(File).to receive(:delete).with(backup_restore_pid_path)
2022-11-25 23:54:43 +05:30
allow(progress).to receive(:puts).at_least(:once)
allow_next_instance_of(::Backup::Manager) do |instance|
2023-07-09 08:55:56 +05:30
Array(task_name).each do |task|
2022-11-25 23:54:43 +05:30
allow(instance).to receive(:run_restore_task).with(task)
end
end
end
2023-07-09 08:55:56 +05:30
it 'locks and deletes the PID file while logging a message' do
2022-11-25 23:54:43 +05:30
expect(pid_file).to receive(:flock).with(File::LOCK_EX)
expect(pid_file).to receive(:flock).with(File::LOCK_UN)
expect(File).to receive(:delete).with(backup_restore_pid_path)
2023-07-09 08:55:56 +05:30
expect(progress).to receive(:puts).with(delete_message)
2022-11-25 23:54:43 +05:30
run_rake_task(rake_task)
end
end
end
end
2014-09-02 18:07:02 +05:30
describe 'backup_restore' do
2022-11-25 23:54:43 +05:30
context 'with gitlab version' do
2014-09-02 18:07:02 +05:30
before do
2017-08-17 22:00:37 +05:30
allow(Dir).to receive(:glob).and_return(['1_gitlab_backup.tar'])
2016-09-13 17:45:13 +05:30
allow(File).to receive(:exist?).and_return(true)
2022-11-25 23:54:43 +05:30
allow(File).to receive(:exist?).with(backup_restore_pid_path).and_return(false)
2015-09-11 14:41:01 +05:30
allow(Kernel).to receive(:system).and_return(true)
allow(FileUtils).to receive(:cp_r).and_return(true)
allow(FileUtils).to receive(:mv).and_return(true)
2017-09-10 17:25:29 +05:30
allow(Rake::Task["gitlab:shell:setup"])
.to receive(:invoke).and_return(true)
2014-09-02 18:07:02 +05:30
end
let(:gitlab_version) { Gitlab::VERSION }
2022-11-25 23:54:43 +05:30
context 'when restore matches gitlab version' do
2019-03-02 22:35:43 +05:30
before do
2023-06-20 00:43:36 +05:30
allow(YAML).to receive(:safe_load_file)
2019-03-02 22:35:43 +05:30
.and_return({ gitlab_version: gitlab_version })
2022-04-04 11:22:00 +05:30
expect_next_instance_of(::Backup::Manager) do |instance|
backup_types.each do |subtask|
expect(instance).to receive(:run_restore_task).with(subtask).ordered
end
expect(instance).not_to receive(:run_restore_task)
2022-03-02 08:16:31 +05:30
end
2019-03-02 22:35:43 +05:30
expect(Rake::Task['gitlab:shell:setup']).to receive(:invoke)
end
it 'invokes restoration on match' do
2021-09-04 01:27:46 +05:30
expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout_from_any_process
2019-03-02 22:35:43 +05:30
end
2014-09-02 18:07:02 +05:30
end
end
2018-11-18 11:00:15 +05:30
context 'when the restore directory is not empty' do
before do
# We only need a backup of the repositories for this test
2022-03-02 08:16:31 +05:30
stub_env('SKIP', 'db,uploads,builds,artifacts,lfs,terraform_state,registry')
2021-09-04 01:27:46 +05:30
create(:project, :repository)
2018-11-18 11:00:15 +05:30
end
it 'removes stale data' do
2021-09-04 01:27:46 +05:30
expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
2018-11-18 11:00:15 +05:30
excluded_project = create(:project, :repository, name: 'mepmep')
2021-09-04 01:27:46 +05:30
expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout_from_any_process
2018-11-18 11:00:15 +05:30
raw_repo = excluded_project.repository.raw
# The restore will not find the repository in the backup, but will create
# an empty one in its place
expect(raw_repo.empty?).to be(true)
end
end
2019-12-21 20:55:43 +05:30
context 'when the backup is restored' do
let!(:included_project) { create(:project, :repository) }
2021-09-04 01:27:46 +05:30
let!(:original_checksum) { included_project.repository.checksum }
2019-12-21 20:55:43 +05:30
before do
2021-09-04 01:27:46 +05:30
expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
2019-12-21 20:55:43 +05:30
backup_tar = Dir.glob(File.join(Gitlab.config.backup.path, '*_gitlab_backup.tar')).last
allow(Dir).to receive(:glob).and_return([backup_tar])
allow(File).to receive(:exist?).and_return(true)
2022-11-25 23:54:43 +05:30
allow(File).to receive(:exist?).with(backup_restore_pid_path).and_return(false)
2019-12-21 20:55:43 +05:30
allow(Kernel).to receive(:system).and_return(true)
allow(FileUtils).to receive(:cp_r).and_return(true)
allow(FileUtils).to receive(:mv).and_return(true)
2023-06-20 00:43:36 +05:30
allow(YAML).to receive(:safe_load_file)
2019-12-21 20:55:43 +05:30
.and_return({ gitlab_version: Gitlab::VERSION })
2022-04-04 11:22:00 +05:30
expect_next_instance_of(::Backup::Manager) do |instance|
backup_types.each do |subtask|
expect(instance).to receive(:run_restore_task).with(subtask).ordered
end
expect(instance).not_to receive(:run_restore_task)
end
2019-12-21 20:55:43 +05:30
expect(Rake::Task['gitlab:shell:setup']).to receive(:invoke)
end
it 'restores the data' do
2021-09-04 01:27:46 +05:30
expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout_from_any_process
2019-12-21 20:55:43 +05:30
raw_repo = included_project.repository.raw
expect(raw_repo.empty?).to be(false)
2021-09-04 01:27:46 +05:30
expect(included_project.repository.checksum).to eq(original_checksum)
2019-12-21 20:55:43 +05:30
end
end
2020-11-24 15:15:51 +05:30
end
# backup_restore task
2015-04-26 12:48:37 +05:30
2017-08-17 22:00:37 +05:30
describe 'backup' do
2018-03-17 18:26:18 +05:30
before do
2017-08-17 22:00:37 +05:30
# This reconnect makes our project fixture disappear, breaking the restore. Stub it out.
2022-08-27 11:52:29 +05:30
allow(ApplicationRecord.connection).to receive(:reconnect!)
allow(Ci::ApplicationRecord.connection).to receive(:reconnect!)
2017-08-17 22:00:37 +05:30
end
2021-09-04 01:27:46 +05:30
let!(:project) { create(:project, :repository) }
2022-11-25 23:54:43 +05:30
context 'with specific backup tasks' do
2018-03-17 18:26:18 +05:30
before do
stub_env('SKIP', 'db')
2017-08-17 22:00:37 +05:30
end
2022-11-25 23:54:43 +05:30
it 'prints a progress message to stdout' do
backup_tasks.each do |task|
expect { run_rake_task("gitlab:backup:#{task}:create") }.to output(/Dumping /).to_stdout_from_any_process
2018-11-08 19:23:39 +05:30
end
end
2022-11-25 23:54:43 +05:30
it 'logs the progress to log file' do
2023-04-23 21:23:45 +05:30
expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping database ... [SKIPPED]")
2022-11-25 23:54:43 +05:30
expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping repositories ... ")
expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping repositories ... done")
expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping uploads ... ")
expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping uploads ... done")
expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping builds ... ")
expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping builds ... done")
expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping artifacts ... ")
expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping artifacts ... done")
expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping pages ... ")
expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping pages ... done")
expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping lfs objects ... ")
expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping lfs objects ... done")
expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping terraform states ... ")
expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping terraform states ... done")
expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping container registry images ... ")
expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping container registry images ... done")
expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping packages ... ")
expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping packages ... done")
backup_tasks.each do |task|
run_rake_task("gitlab:backup:#{task}:create")
2021-09-30 23:02:18 +05:30
end
2017-08-17 22:00:37 +05:30
end
end
2022-03-02 08:16:31 +05:30
describe 'backup create fails' do
using RSpec::Parameterized::TableSyntax
file_backup_error = Backup::FileBackupError.new('/tmp', '/tmp/backup/uploads')
config = ActiveRecord::Base.configurations.find_db_config(Rails.env).configuration_hash
db_file_name = File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz')
db_backup_error = Backup::DatabaseBackupError.new(config, db_file_name)
where(:backup_class, :rake_task, :error) do
2022-06-21 17:19:12 +05:30
Backup::Database | 'gitlab:backup:db:create' | db_backup_error
Backup::Files | 'gitlab:backup:builds:create' | file_backup_error
Backup::Files | 'gitlab:backup:uploads:create' | file_backup_error
Backup::Files | 'gitlab:backup:artifacts:create' | file_backup_error
Backup::Files | 'gitlab:backup:pages:create' | file_backup_error
Backup::Files | 'gitlab:backup:lfs:create' | file_backup_error
Backup::Files | 'gitlab:backup:registry:create' | file_backup_error
2022-03-02 08:16:31 +05:30
end
with_them do
before do
2022-06-21 17:19:12 +05:30
allow_next_instance_of(backup_class) do |instance|
allow(instance).to receive(:dump).and_raise(error)
2022-03-02 08:16:31 +05:30
end
end
it "raises an error with message" do
expect { run_rake_task(rake_task) }.to output(Regexp.new(error.message)).to_stdout_from_any_process
end
end
end
2022-11-25 23:54:43 +05:30
context 'with tar creation' do
context 'with archive file permissions' do
2016-09-13 17:45:13 +05:30
it 'sets correct permissions on the tar file' do
2021-09-04 01:27:46 +05:30
expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
2018-03-17 18:26:18 +05:30
2022-11-25 23:54:43 +05:30
expect(File).to exist(backup_tar)
2018-03-17 18:26:18 +05:30
expect(File::Stat.new(backup_tar).mode.to_s(8)).to eq('100600')
2015-09-11 14:41:01 +05:30
end
2016-08-24 12:49:21 +05:30
context 'with custom archive_permissions' do
before do
2022-11-25 23:54:43 +05:30
allow(Gitlab.config.backup).to receive(:archive_permissions).and_return(0o651)
2016-08-24 12:49:21 +05:30
end
it 'uses the custom permissions' do
2021-09-04 01:27:46 +05:30
expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
2018-03-17 18:26:18 +05:30
expect(File::Stat.new(backup_tar).mode.to_s(8)).to eq('100651')
2016-08-24 12:49:21 +05:30
end
2015-09-11 14:41:01 +05:30
end
end
2015-04-26 12:48:37 +05:30
2016-09-13 17:45:13 +05:30
it 'sets correct permissions on the tar contents' do
2021-09-04 01:27:46 +05:30
expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
2018-03-17 18:26:18 +05:30
2016-08-24 12:49:21 +05:30
tar_contents, exit_status = Gitlab::Popen.popen(
2022-11-25 23:54:43 +05:30
%W[
tar -tvf #{backup_tar}
db
uploads.tar.gz
repositories
builds.tar.gz
artifacts.tar.gz
pages.tar.gz
lfs.tar.gz
terraform_state.tar.gz
registry.tar.gz
packages.tar.gz
]
2016-08-24 12:49:21 +05:30
)
2018-03-17 18:26:18 +05:30
2016-08-24 12:49:21 +05:30
expect(exit_status).to eq(0)
2020-04-08 14:13:33 +05:30
expect(tar_contents).to match('db')
2016-08-24 12:49:21 +05:30
expect(tar_contents).to match('uploads.tar.gz')
expect(tar_contents).to match('repositories/')
expect(tar_contents).to match('builds.tar.gz')
expect(tar_contents).to match('artifacts.tar.gz')
2017-08-17 22:00:37 +05:30
expect(tar_contents).to match('pages.tar.gz')
2016-08-24 12:49:21 +05:30
expect(tar_contents).to match('lfs.tar.gz')
2022-03-02 08:16:31 +05:30
expect(tar_contents).to match('terraform_state.tar.gz')
2016-08-24 12:49:21 +05:30
expect(tar_contents).to match('registry.tar.gz')
2022-03-02 08:16:31 +05:30
expect(tar_contents).to match('packages.tar.gz')
2022-11-25 23:54:43 +05:30
expect(tar_contents).not_to match(%r{^.{4,9}[rwx].* (database.sql.gz|uploads.tar.gz|repositories|builds.tar.gz|
pages.tar.gz|artifacts.tar.gz|registry.tar.gz)/$})
2016-08-24 12:49:21 +05:30
end
2016-09-13 17:45:13 +05:30
it 'deletes temp directories' do
2021-09-04 01:27:46 +05:30
expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
2018-03-17 18:26:18 +05:30
2016-08-24 12:49:21 +05:30
temp_dirs = Dir.glob(
2022-11-25 23:54:43 +05:30
File.join(
Gitlab.config.backup.path,
'{db,repositories,uploads,builds,artifacts,pages,lfs,terraform_state,registry,packages}'
)
2016-08-24 12:49:21 +05:30
)
2015-04-26 12:48:37 +05:30
2016-08-24 12:49:21 +05:30
expect(temp_dirs).to be_empty
end
2022-11-25 23:54:43 +05:30
context 'when registry is disabled' do
2016-08-24 12:49:21 +05:30
let(:enable_registry) { false }
2015-04-26 12:48:37 +05:30
2016-09-13 17:45:13 +05:30
it 'does not create registry.tar.gz' do
2021-09-04 01:27:46 +05:30
expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
2018-03-17 18:26:18 +05:30
2016-08-24 12:49:21 +05:30
tar_contents, exit_status = Gitlab::Popen.popen(
2022-11-25 23:54:43 +05:30
%W[tar -tvf #{backup_tar}]
2016-08-24 12:49:21 +05:30
)
2018-03-17 18:26:18 +05:30
2016-08-24 12:49:21 +05:30
expect(exit_status).to eq(0)
expect(tar_contents).not_to match('registry.tar.gz')
end
end
2015-04-26 12:48:37 +05:30
end
2016-06-02 11:05:42 +05:30
2022-11-25 23:54:43 +05:30
context 'with multiple repository storages' do
2021-01-03 14:25:43 +05:30
include StubConfiguration
let(:default_storage_name) { 'default' }
let(:second_storage_name) { 'test_second_storage' }
2018-05-09 12:01:36 +05:30
2018-03-17 18:26:18 +05:30
before do
# We only need a backup of the repositories for this test
2022-03-02 08:16:31 +05:30
stub_env('SKIP', 'db,uploads,builds,artifacts,lfs,terraform_state,registry')
2022-11-25 23:54:43 +05:30
stub_storage_settings(second_storage_name => {
'gitaly_address' => Gitlab.config.repositories.storages.default.gitaly_address,
'path' => TestEnv::SECOND_STORAGE_PATH
})
2020-10-24 23:57:45 +05:30
end
2021-01-03 14:25:43 +05:30
shared_examples 'includes repositories in all repository storages' do
specify :aggregate_failures do
project_a = create(:project, :repository)
2022-04-04 11:22:00 +05:30
project_snippet_a = create(:project_snippet, :repository, project: project_a, author: project_a.first_owner)
2021-01-03 14:25:43 +05:30
project_b = create(:project, :repository, repository_storage: second_storage_name)
2022-08-13 15:12:31 +05:30
project_snippet_b = create(
:project_snippet,
:repository,
project: project_b,
author: project_b.first_owner,
repository_storage: second_storage_name
)
2021-01-03 14:25:43 +05:30
create(:wiki_page, container: project_a)
create(:design, :with_file, issue: create(:issue, project: project_a))
2021-09-04 01:27:46 +05:30
expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
2020-10-24 23:57:45 +05:30
tar_contents, exit_status = Gitlab::Popen.popen(
2022-11-25 23:54:43 +05:30
%W[tar -tvf #{backup_tar} repositories]
2020-10-24 23:57:45 +05:30
)
2021-01-03 14:25:43 +05:30
tar_lines = tar_contents.lines.grep(/\.bundle/)
2020-10-24 23:57:45 +05:30
expect(exit_status).to eq(0)
2021-01-03 14:25:43 +05:30
2022-11-25 23:54:43 +05:30
%W[
#{project_a.disk_path}/.+/001.bundle
#{project_a.disk_path}.wiki/.+/001.bundle
#{project_a.disk_path}.design/.+/001.bundle
#{project_b.disk_path}/.+/001.bundle
#{project_snippet_a.disk_path}/.+/001.bundle
#{project_snippet_b.disk_path}/.+/001.bundle
2021-01-03 14:25:43 +05:30
].each do |repo_name|
2022-05-07 20:08:51 +05:30
expect(tar_lines).to include(a_string_matching(repo_name))
2021-01-03 14:25:43 +05:30
end
2020-10-24 23:57:45 +05:30
end
2021-01-03 14:25:43 +05:30
end
2022-11-25 23:54:43 +05:30
context 'with no concurrency' do
2021-01-03 14:25:43 +05:30
it_behaves_like 'includes repositories in all repository storages'
2020-10-24 23:57:45 +05:30
end
context 'with concurrency' do
before do
stub_env('GITLAB_BACKUP_MAX_CONCURRENCY', 4)
end
2021-01-03 14:25:43 +05:30
it_behaves_like 'includes repositories in all repository storages'
2020-10-24 23:57:45 +05:30
end
2022-07-16 23:28:13 +05:30
2022-11-25 23:54:43 +05:30
context 'when REPOSITORIES_STORAGES is set' do
2022-07-16 23:28:13 +05:30
before do
stub_env('REPOSITORIES_STORAGES', default_storage_name)
end
it 'includes repositories in default repository storage', :aggregate_failures do
project_a = create(:project, :repository)
project_snippet_a = create(:project_snippet, :repository, project: project_a, author: project_a.first_owner)
project_b = create(:project, :repository, repository_storage: second_storage_name)
2022-08-13 15:12:31 +05:30
project_snippet_b = create(
:project_snippet,
:repository,
project: project_b,
author: project_b.first_owner,
repository_storage: second_storage_name
)
2022-07-16 23:28:13 +05:30
create(:wiki_page, container: project_a)
create(:design, :with_file, issue: create(:issue, project: project_a))
expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
tar_contents, exit_status = Gitlab::Popen.popen(
2022-11-25 23:54:43 +05:30
%W[tar -tvf #{backup_tar} repositories]
2022-07-16 23:28:13 +05:30
)
tar_lines = tar_contents.lines.grep(/\.bundle/)
expect(exit_status).to eq(0)
2022-11-25 23:54:43 +05:30
%W[
#{project_a.disk_path}/.+/001.bundle
#{project_a.disk_path}.wiki/.+/001.bundle
#{project_a.disk_path}.design/.+/001.bundle
#{project_snippet_a.disk_path}/.+/001.bundle
2022-07-16 23:28:13 +05:30
].each do |repo_name|
expect(tar_lines).to include(a_string_matching(repo_name))
end
2022-11-25 23:54:43 +05:30
%W[
#{project_b.disk_path}/.+/001.bundle
#{project_snippet_b.disk_path}/.+/001.bundle
2022-07-16 23:28:13 +05:30
].each do |repo_name|
expect(tar_lines).not_to include(a_string_matching(repo_name))
end
end
end
2020-10-24 23:57:45 +05:30
end
2022-11-25 23:54:43 +05:30
context 'with concurrency settings' do
2020-10-24 23:57:45 +05:30
before do
# We only need a backup of the repositories for this test
2022-03-02 08:16:31 +05:30
stub_env('SKIP', 'db,uploads,builds,artifacts,lfs,terraform_state,registry')
2020-10-24 23:57:45 +05:30
create(:project, :repository)
end
it 'passes through concurrency environment variables' do
stub_env('GITLAB_BACKUP_MAX_CONCURRENCY', 5)
stub_env('GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY', 2)
2018-03-17 18:26:18 +05:30
2022-04-04 11:22:00 +05:30
expect(::Backup::Repositories).to receive(:new)
2022-07-23 23:45:48 +05:30
.with(anything, strategy: anything, storages: [], paths: [])
2022-04-04 11:22:00 +05:30
.and_call_original
2022-11-25 23:54:43 +05:30
expect(::Backup::GitalyBackup).to receive(:new).with(
anything,
max_parallelism: 5,
storage_parallelism: 2,
incremental: false
).and_call_original
2020-10-24 23:57:45 +05:30
2021-09-04 01:27:46 +05:30
expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
2016-06-02 11:05:42 +05:30
end
end
2021-10-27 15:23:28 +05:30
2022-11-25 23:54:43 +05:30
context 'when CRON env is set' do
2021-10-27 15:23:28 +05:30
before do
stub_env('CRON', '1')
end
it 'does not output to stdout' do
expect { run_rake_task('gitlab:backup:create') }.not_to output.to_stdout_from_any_process
end
end
2020-11-24 15:15:51 +05:30
end
# backup_create task
2015-04-26 12:48:37 +05:30
2022-11-25 23:54:43 +05:30
describe "skipping items in a backup" do
2018-03-17 18:26:18 +05:30
before do
2022-03-02 08:16:31 +05:30
stub_env('SKIP', 'an-unknown-type,repositories,uploads,anotherunknowntype')
2021-09-04 01:27:46 +05:30
create(:project, :repository)
2015-04-26 12:48:37 +05:30
end
2022-03-02 08:16:31 +05:30
it "does not contain repositories and uploads" do
2021-09-04 01:27:46 +05:30
expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
2018-03-17 18:26:18 +05:30
2015-10-24 18:46:33 +05:30
tar_contents, _exit_status = Gitlab::Popen.popen(
2022-11-25 23:54:43 +05:30
%W[
tar -tvf #{backup_tar}
db
uploads.tar.gz
repositories
builds.tar.gz
artifacts.tar.gz
pages.tar.gz
lfs.tar.gz
terraform_state.tar.gz
registry.tar.gz
packages.tar.gz
]
2015-04-26 12:48:37 +05:30
)
expect(tar_contents).to match('db/')
2022-03-02 08:16:31 +05:30
expect(tar_contents).to match('uploads.tar.gz: Not found in archive')
2015-11-26 14:37:03 +05:30
expect(tar_contents).to match('builds.tar.gz')
expect(tar_contents).to match('artifacts.tar.gz')
expect(tar_contents).to match('lfs.tar.gz')
2022-03-02 08:16:31 +05:30
expect(tar_contents).to match('terraform_state.tar.gz')
2017-08-17 22:00:37 +05:30
expect(tar_contents).to match('pages.tar.gz')
2016-06-02 11:05:42 +05:30
expect(tar_contents).to match('registry.tar.gz')
2022-03-02 08:16:31 +05:30
expect(tar_contents).to match('packages.tar.gz')
2015-04-26 12:48:37 +05:30
expect(tar_contents).not_to match('repositories/')
2022-03-02 08:16:31 +05:30
expect(tar_contents).to match('repositories: Not found in archive')
2015-04-26 12:48:37 +05:30
end
2022-03-02 08:16:31 +05:30
it 'does not invoke restore of repositories and uploads' do
2021-09-04 01:27:46 +05:30
expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
2018-03-17 18:26:18 +05:30
2017-09-10 17:25:29 +05:30
allow(Rake::Task['gitlab:shell:setup'])
.to receive(:invoke).and_return(true)
2015-04-26 12:48:37 +05:30
2022-04-04 11:22:00 +05:30
expect_next_instance_of(::Backup::Manager) do |instance|
2022-11-25 23:54:43 +05:30
(backup_types - %w[repositories uploads]).each do |subtask|
2022-04-04 11:22:00 +05:30
expect(instance).to receive(:run_restore_task).with(subtask).ordered
end
expect(instance).not_to receive(:run_restore_task)
end
2016-06-02 11:05:42 +05:30
expect(Rake::Task['gitlab:shell:setup']).to receive :invoke
2021-09-04 01:27:46 +05:30
expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout_from_any_process
2015-04-26 12:48:37 +05:30
end
end
2017-08-17 22:00:37 +05:30
2020-04-08 14:13:33 +05:30
describe 'skipping tar archive creation' do
before do
stub_env('SKIP', 'tar')
2021-09-04 01:27:46 +05:30
create(:project, :repository)
2020-04-08 14:13:33 +05:30
end
it 'created files with backup content and no tar archive' do
2021-09-04 01:27:46 +05:30
expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
2020-04-08 14:13:33 +05:30
dir_contents = Dir.children(Gitlab.config.backup.path)
expect(dir_contents).to contain_exactly(
'backup_information.yml',
'db',
'uploads.tar.gz',
'builds.tar.gz',
'artifacts.tar.gz',
'lfs.tar.gz',
2022-03-02 08:16:31 +05:30
'terraform_state.tar.gz',
2020-04-08 14:13:33 +05:30
'pages.tar.gz',
'registry.tar.gz',
2022-03-02 08:16:31 +05:30
'packages.tar.gz',
2021-10-27 15:23:28 +05:30
'repositories'
2020-04-08 14:13:33 +05:30
)
end
it 'those component files can be restored from' do
2021-09-04 01:27:46 +05:30
expect { run_rake_task("gitlab:backup:create") }.to output.to_stdout_from_any_process
2020-04-08 14:13:33 +05:30
allow(Rake::Task['gitlab:shell:setup'])
.to receive(:invoke).and_return(true)
2022-04-04 11:22:00 +05:30
expect_next_instance_of(::Backup::Manager) do |instance|
backup_types.each do |subtask|
expect(instance).to receive(:run_restore_task).with(subtask).ordered
end
expect(instance).not_to receive(:run_restore_task)
2022-03-02 08:16:31 +05:30
end
2020-04-08 14:13:33 +05:30
expect(Rake::Task['gitlab:shell:setup']).to receive :invoke
2021-09-04 01:27:46 +05:30
expect { run_rake_task("gitlab:backup:restore") }.to output.to_stdout_from_any_process
2020-04-08 14:13:33 +05:30
end
end
2017-08-17 22:00:37 +05:30
describe "Human Readable Backup Name" do
it 'name has human readable time' do
2021-09-04 01:27:46 +05:30
expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
2018-03-17 18:26:18 +05:30
expect(backup_tar).to match(/\d+_\d{4}_\d{2}_\d{2}_\d+\.\d+\.\d+.*_gitlab_backup.tar$/)
2017-08-17 22:00:37 +05:30
end
end
2020-11-24 15:15:51 +05:30
end
# gitlab:app namespace