debian-mirror-gitlab/app/workers/git_garbage_collect_worker.rb

119 lines
3.5 KiB
Ruby
Raw Normal View History

2018-11-08 19:23:39 +05:30
# frozen_string_literal: true
2020-04-08 14:13:33 +05:30
class GitGarbageCollectWorker # rubocop:disable Scalability/IdempotentWorker
2018-03-17 18:26:18 +05:30
include ApplicationWorker
2016-08-24 12:49:21 +05:30
2016-11-03 12:29:30 +05:30
sidekiq_options retry: false
2019-12-21 20:55:43 +05:30
feature_category :gitaly
2020-06-23 00:09:42 +05:30
loggable_arguments 1, 2, 3
2016-08-24 12:49:21 +05:30
2018-03-17 18:26:18 +05:30
# Timeout set to 24h
LEASE_TIMEOUT = 86400
2017-08-17 22:00:37 +05:30
def perform(project_id, task = :gc, lease_key = nil, lease_uuid = nil)
2020-10-24 23:57:45 +05:30
lease_key ||= "git_gc:#{task}:#{project_id}"
2016-08-24 12:49:21 +05:30
project = Project.find(project_id)
2018-03-17 18:26:18 +05:30
active_uuid = get_lease_uuid(lease_key)
if active_uuid
return unless active_uuid == lease_uuid
renew_lease(lease_key, active_uuid)
else
lease_uuid = try_obtain_lease(lease_key)
return unless lease_uuid
end
2017-08-17 22:00:37 +05:30
2018-03-17 18:26:18 +05:30
task = task.to_sym
2019-09-04 21:01:54 +05:30
2020-10-24 23:57:45 +05:30
if task == :gc
::Projects::GitDeduplicationService.new(project).execute
cleanup_orphan_lfs_file_references(project)
end
2019-09-04 21:01:54 +05:30
2018-11-08 19:23:39 +05:30
gitaly_call(task, project.repository.raw_repository)
2016-08-24 12:49:21 +05:30
# Refresh the branch cache in case garbage collection caused a ref lookup to fail
2017-08-17 22:00:37 +05:30
flush_ref_caches(project) if task == :gc
2018-03-27 19:54:05 +05:30
2020-11-24 15:15:51 +05:30
update_repository_statistics(project) if task != :pack_refs
2019-02-15 15:39:39 +05:30
2018-03-27 19:54:05 +05:30
# In case pack files are deleted, release libgit2 cache and open file
# descriptors ASAP instead of waiting for Ruby garbage collection
project.cleanup
2017-08-17 22:00:37 +05:30
ensure
2018-03-17 18:26:18 +05:30
cancel_lease(lease_key, lease_uuid) if lease_key.present? && lease_uuid.present?
2017-08-17 22:00:37 +05:30
end
private
2018-03-17 18:26:18 +05:30
def try_obtain_lease(key)
::Gitlab::ExclusiveLease.new(key, timeout: LEASE_TIMEOUT).try_obtain
end
def renew_lease(key, uuid)
::Gitlab::ExclusiveLease.new(key, uuid: uuid, timeout: LEASE_TIMEOUT).renew
end
def cancel_lease(key, uuid)
::Gitlab::ExclusiveLease.cancel(key, uuid)
end
def get_lease_uuid(key)
::Gitlab::ExclusiveLease.get_uuid(key)
end
2017-09-10 17:25:29 +05:30
## `repository` has to be a Gitlab::Git::Repository
def gitaly_call(task, repository)
2019-07-31 22:56:46 +05:30
client = if task == :pack_refs
Gitlab::GitalyClient::RefService.new(repository)
else
Gitlab::GitalyClient::RepositoryService.new(repository)
end
2017-09-10 17:25:29 +05:30
case task
when :gc
client.garbage_collect(bitmaps_enabled?)
when :full_repack
client.repack_full(bitmaps_enabled?)
when :incremental_repack
client.repack_incremental
2019-07-31 22:56:46 +05:30
when :pack_refs
client.pack_refs
2017-09-10 17:25:29 +05:30
end
2018-11-08 19:23:39 +05:30
rescue GRPC::NotFound => e
Gitlab::GitLogger.error("#{__method__} failed:\nRepository not found")
raise Gitlab::Git::Repository::NoRepository.new(e)
rescue GRPC::BadStatus => e
Gitlab::GitLogger.error("#{__method__} failed:\n#{e}")
raise Gitlab::Git::CommandError.new(e)
2017-08-17 22:00:37 +05:30
end
2020-10-24 23:57:45 +05:30
def cleanup_orphan_lfs_file_references(project)
return if Gitlab::Database.read_only? # GitGarbageCollectWorker may be run on a Geo secondary
::Gitlab::Cleanup::OrphanLfsFileReferences.new(project, dry_run: false, logger: logger).run!
2021-01-03 14:25:43 +05:30
rescue => err
Gitlab::GitLogger.warn(message: "Cleaning up orphan LFS objects files failed", error: err.message)
Gitlab::ErrorTracking.track_and_raise_for_dev_exception(err)
2020-10-24 23:57:45 +05:30
end
2017-08-17 22:00:37 +05:30
def flush_ref_caches(project)
2021-01-03 14:25:43 +05:30
project.repository.expire_branches_cache
2016-08-24 12:49:21 +05:30
project.repository.branch_names
project.repository.has_visible_content?
end
2017-08-17 22:00:37 +05:30
2020-11-24 15:15:51 +05:30
def update_repository_statistics(project)
project.repository.expire_statistics_caches
return if Gitlab::Database.read_only? # GitGarbageCollectWorker may be run on a Geo secondary
Projects::UpdateStatisticsService.new(project, nil, statistics: [:repository_size, :lfs_objects_size]).execute
end
2017-08-17 22:00:37 +05:30
def bitmaps_enabled?
2018-03-17 18:26:18 +05:30
Gitlab::CurrentSettings.housekeeping_bitmaps_enabled
2017-08-17 22:00:37 +05:30
end
2016-08-24 12:49:21 +05:30
end