debian-mirror-gitlab/lib/backup/manager.rb

290 lines
8.9 KiB
Ruby
Raw Normal View History

2018-12-05 23:21:45 +05:30
# frozen_string_literal: true
2014-09-02 18:07:02 +05:30
module Backup
class Manager
2017-08-17 22:00:37 +05:30
ARCHIVES_TO_BACKUP = %w[uploads builds artifacts pages lfs registry].freeze
FOLDERS_TO_BACKUP = %w[repositories db].freeze
2019-12-04 20:38:33 +05:30
FILE_NAME_SUFFIX = '_gitlab_backup.tar'
2016-06-02 11:05:42 +05:30
2018-11-08 19:23:39 +05:30
attr_reader :progress
def initialize(progress)
@progress = progress
end
2020-04-08 14:13:33 +05:30
def write_info
2016-04-02 18:10:28 +05:30
# Make sure there is a connection
ActiveRecord::Base.connection.reconnect!
2017-08-17 22:00:37 +05:30
Dir.chdir(backup_path) do
File.open("#{backup_path}/backup_information.yml", "w+") do |file|
2017-09-10 17:25:29 +05:30
file << backup_information.to_yaml.gsub(/^---\n/, '')
2015-04-26 12:48:37 +05:30
end
2020-04-08 14:13:33 +05:30
end
end
2015-04-26 12:48:37 +05:30
2020-04-08 14:13:33 +05:30
def pack
Dir.chdir(backup_path) do
2015-04-26 12:48:37 +05:30
# create archive
2018-11-08 19:23:39 +05:30
progress.print "Creating backup archive: #{tar_file} ... "
2015-09-11 14:41:01 +05:30
# Set file permissions on open to prevent chmod races.
2017-08-17 22:00:37 +05:30
tar_system_options = { out: [tar_file, 'w', Gitlab.config.backup.archive_permissions] }
2015-09-11 14:41:01 +05:30
if Kernel.system('tar', '-cf', '-', *backup_contents, tar_system_options)
2018-11-08 19:23:39 +05:30
progress.puts "done".color(:green)
2015-04-26 12:48:37 +05:30
else
puts "creating archive #{tar_file} failed".color(:red)
2018-11-08 19:23:39 +05:30
raise Backup::Error, 'Backup failed'
2015-04-26 12:48:37 +05:30
end
2014-09-02 18:07:02 +05:30
end
2015-04-26 12:48:37 +05:30
end
2017-09-10 17:25:29 +05:30
def upload
2018-11-08 19:23:39 +05:30
progress.print "Uploading backup archive to remote storage #{remote_directory} ... "
2014-09-02 18:07:02 +05:30
2015-04-26 12:48:37 +05:30
connection_settings = Gitlab.config.backup.upload.connection
if connection_settings.blank?
2018-11-08 19:23:39 +05:30
progress.puts "skipped".color(:yellow)
2015-04-26 12:48:37 +05:30
return
end
2020-11-24 15:15:51 +05:30
directory = connect_to_remote_directory(Gitlab.config.backup.upload)
2015-04-26 12:48:37 +05:30
2020-01-01 13:55:28 +05:30
if directory.files.create(create_attributes)
2018-11-08 19:23:39 +05:30
progress.puts "done".color(:green)
2014-09-02 18:07:02 +05:30
else
puts "uploading backup to #{remote_directory} failed".color(:red)
2018-11-08 19:23:39 +05:30
raise Backup::Error, 'Backup failed'
2014-09-02 18:07:02 +05:30
end
end
def cleanup
2018-11-08 19:23:39 +05:30
progress.print "Deleting tmp directories ... "
2015-10-24 18:46:33 +05:30
2015-04-26 12:48:37 +05:30
backup_contents.each do |dir|
2017-08-17 22:00:37 +05:30
next unless File.exist?(File.join(backup_path, dir))
2015-04-26 12:48:37 +05:30
2017-08-17 22:00:37 +05:30
if FileUtils.rm_rf(File.join(backup_path, dir))
2018-11-08 19:23:39 +05:30
progress.puts "done".color(:green)
2015-04-26 12:48:37 +05:30
else
puts "deleting tmp directory '#{dir}' failed".color(:red)
2018-11-08 19:23:39 +05:30
raise Backup::Error, 'Backup failed'
2015-04-26 12:48:37 +05:30
end
2014-09-02 18:07:02 +05:30
end
end
def remove_old
# delete backups
2018-11-08 19:23:39 +05:30
progress.print "Deleting old backups ... "
2014-09-02 18:07:02 +05:30
keep_time = Gitlab.config.backup.keep_time.to_i
if keep_time > 0
removed = 0
2015-10-24 18:46:33 +05:30
2017-08-17 22:00:37 +05:30
Dir.chdir(backup_path) do
backup_file_list.each do |file|
# For backward compatibility, there are 3 names the backups can have:
# - 1495527122_gitlab_backup.tar
# - 1495527068_2017_05_23_gitlab_backup.tar
# - 1495527097_2017_05_23_9.3.0-pre_gitlab_backup.tar
2018-03-17 18:26:18 +05:30
next unless file =~ /^(\d{10})(?:_\d{4}_\d{2}_\d{2}(_\d+\.\d+\.\d+((-|\.)(pre|rc\d))?(-ee)?)?)?_gitlab_backup\.tar$/
2017-08-17 22:00:37 +05:30
2020-11-24 15:15:51 +05:30
timestamp = Regexp.last_match(1).to_i
2017-08-17 22:00:37 +05:30
2015-04-26 12:48:37 +05:30
if Time.at(timestamp) < (Time.now - keep_time)
2017-08-17 22:00:37 +05:30
begin
FileUtils.rm(file)
2015-04-26 12:48:37 +05:30
removed += 1
2017-08-17 22:00:37 +05:30
rescue => e
2018-11-08 19:23:39 +05:30
progress.puts "Deleting #{file} failed: #{e.message}".color(:red)
2015-04-26 12:48:37 +05:30
end
2014-09-02 18:07:02 +05:30
end
end
end
2015-04-26 12:48:37 +05:30
2018-11-08 19:23:39 +05:30
progress.puts "done. (#{removed} removed)".color(:green)
2014-09-02 18:07:02 +05:30
else
2018-11-08 19:23:39 +05:30
progress.puts "skipping".color(:yellow)
2014-09-02 18:07:02 +05:30
end
end
2020-04-08 14:13:33 +05:30
def verify_backup_version
Dir.chdir(backup_path) do
# restoring mismatching backups can lead to unexpected problems
if settings[:gitlab_version] != Gitlab::VERSION
progress.puts(<<~HEREDOC.color(:red))
GitLab version mismatch:
Your current GitLab version (#{Gitlab::VERSION}) differs from the GitLab version in the backup!
Please switch to the following version and try again:
version: #{settings[:gitlab_version]}
HEREDOC
progress.puts
progress.puts "Hint: git checkout v#{settings[:gitlab_version]}"
exit 1
end
end
end
2014-09-02 18:07:02 +05:30
def unpack
2020-04-08 14:13:33 +05:30
if ENV['BACKUP'].blank? && non_tarred_backup?
progress.puts "Non tarred backup found in #{backup_path}, using that"
return false
end
2018-03-17 18:26:18 +05:30
Dir.chdir(backup_path) do
# check for existing backups in the backup dir
if backup_file_list.empty?
2018-11-08 19:23:39 +05:30
progress.puts "No backups found in #{backup_path}"
progress.puts "Please make sure that file name ends with #{FILE_NAME_SUFFIX}"
2018-03-17 18:26:18 +05:30
exit 1
elsif backup_file_list.many? && ENV["BACKUP"].nil?
2018-11-08 19:23:39 +05:30
progress.puts 'Found more than one backup:'
2018-03-17 18:26:18 +05:30
# print list of available backups
2018-11-08 19:23:39 +05:30
progress.puts " " + available_timestamps.join("\n ")
progress.puts 'Please specify which one you want to restore:'
progress.puts 'rake gitlab:backup:restore BACKUP=timestamp_of_backup'
2018-03-17 18:26:18 +05:30
exit 1
end
2014-09-02 18:07:02 +05:30
2018-03-17 18:26:18 +05:30
tar_file = if ENV['BACKUP'].present?
2019-12-21 20:55:43 +05:30
File.basename(ENV['BACKUP']) + FILE_NAME_SUFFIX
2018-03-17 18:26:18 +05:30
else
backup_file_list.first
end
2014-09-02 18:07:02 +05:30
2018-03-17 18:26:18 +05:30
unless File.exist?(tar_file)
2018-11-08 19:23:39 +05:30
progress.puts "The backup file #{tar_file} does not exist!"
2018-03-17 18:26:18 +05:30
exit 1
end
2014-09-02 18:07:02 +05:30
2018-11-08 19:23:39 +05:30
progress.print 'Unpacking backup ... '
2015-04-26 12:48:37 +05:30
2020-03-13 15:44:24 +05:30
if Kernel.system(*%W(tar -xf #{tar_file}))
progress.puts 'done'.color(:green)
else
2018-11-08 19:23:39 +05:30
progress.puts 'unpacking backup failed'.color(:red)
2018-03-17 18:26:18 +05:30
exit 1
end
2014-09-02 18:07:02 +05:30
end
end
def tar_version
2017-08-17 22:00:37 +05:30
tar_version, _ = Gitlab::Popen.popen(%w(tar --version))
2018-12-13 13:39:08 +05:30
tar_version.dup.force_encoding('locale').split("\n").first
2014-09-02 18:07:02 +05:30
end
2015-04-26 12:48:37 +05:30
def skipped?(item)
2016-06-02 11:05:42 +05:30
settings[:skipped] && settings[:skipped].include?(item) || disabled_features.include?(item)
2015-04-26 12:48:37 +05:30
end
private
2020-04-08 14:13:33 +05:30
def non_tarred_backup?
File.exist?(File.join(backup_path, 'backup_information.yml'))
end
2017-08-17 22:00:37 +05:30
def backup_path
Gitlab.config.backup.path
end
def backup_file_list
@backup_file_list ||= Dir.glob("*#{FILE_NAME_SUFFIX}")
end
2018-03-17 18:26:18 +05:30
def available_timestamps
@backup_file_list.map {|item| item.gsub("#{FILE_NAME_SUFFIX}", "")}
end
2020-11-24 15:15:51 +05:30
def connect_to_remote_directory(options)
config = ObjectStorage::Config.new(options)
config.load_provider
connection = ::Fog::Storage.new(config.credentials)
# We only attempt to create the directory for local backups. For AWS
# and other cloud providers, we cannot guarantee the user will have
# permission to create the bucket.
if connection.service == ::Fog::Storage::Local
connection.directories.create(key: remote_directory)
else
2019-02-15 15:39:39 +05:30
connection.directories.new(key: remote_directory)
end
end
def remote_directory
Gitlab.config.backup.upload.remote_directory
end
2017-09-10 17:25:29 +05:30
def remote_target
if ENV['DIRECTORY']
File.join(ENV['DIRECTORY'], tar_file)
else
tar_file
end
end
2015-04-26 12:48:37 +05:30
def backup_contents
2015-11-26 14:37:03 +05:30
folders_to_backup + archives_to_backup + ["backup_information.yml"]
2015-04-26 12:48:37 +05:30
end
2015-11-26 14:37:03 +05:30
def archives_to_backup
2018-03-17 18:26:18 +05:30
ARCHIVES_TO_BACKUP.map { |name| (name + ".tar.gz") unless skipped?(name) }.compact
2015-11-26 14:37:03 +05:30
end
2015-04-26 12:48:37 +05:30
2015-11-26 14:37:03 +05:30
def folders_to_backup
2018-03-17 18:26:18 +05:30
FOLDERS_TO_BACKUP.reject { |name| skipped?(name) }
2016-06-02 11:05:42 +05:30
end
def disabled_features
features = []
features << 'registry' unless Gitlab.config.registry.enabled
features
2015-04-26 12:48:37 +05:30
end
def settings
@settings ||= YAML.load_file("backup_information.yml")
end
2017-09-10 17:25:29 +05:30
def tar_file
2019-12-21 20:55:43 +05:30
@tar_file ||= if ENV['BACKUP'].present?
File.basename(ENV['BACKUP']) + FILE_NAME_SUFFIX
2019-07-07 11:18:12 +05:30
else
"#{backup_information[:backup_created_at].strftime('%s_%Y_%m_%d_')}#{backup_information[:gitlab_version]}#{FILE_NAME_SUFFIX}"
end
2017-09-10 17:25:29 +05:30
end
def backup_information
@backup_information ||= {
db_version: ActiveRecord::Migrator.current_version.to_s,
backup_created_at: Time.now,
gitlab_version: Gitlab::VERSION,
tar_version: tar_version,
2018-12-05 23:21:45 +05:30
installation_type: Gitlab::INSTALLATION_TYPE,
2017-09-10 17:25:29 +05:30
skipped: ENV["SKIP"]
}
end
2020-01-01 13:55:28 +05:30
def create_attributes
attrs = {
key: remote_target,
2020-04-08 14:13:33 +05:30
body: File.open(File.join(backup_path, tar_file)),
2020-01-01 13:55:28 +05:30
multipart_chunk_size: Gitlab.config.backup.upload.multipart_chunk_size,
encryption: Gitlab.config.backup.upload.encryption,
encryption_key: Gitlab.config.backup.upload.encryption_key,
storage_class: Gitlab.config.backup.upload.storage_class
}
# Google bucket-only policies prevent setting an ACL. In any case, by default,
# all objects are set to the default ACL, which is project-private:
# https://cloud.google.com/storage/docs/json_api/v1/defaultObjectAccessControls
attrs[:public] = false unless google_provider?
attrs
end
def google_provider?
Gitlab.config.backup.upload.connection&.provider&.downcase == 'google'
end
2014-09-02 18:07:02 +05:30
end
end