debian-mirror-gitlab/app/workers/container_expiration_policy_worker.rb

113 lines
3.5 KiB
Ruby
Raw Normal View History

2020-03-13 15:44:24 +05:30
# frozen_string_literal: true
2020-04-08 14:13:33 +05:30
class ContainerExpirationPolicyWorker # rubocop:disable Scalability/IdempotentWorker
2020-03-13 15:44:24 +05:30
include ApplicationWorker
2021-06-08 01:23:25 +05:30
2021-10-27 15:23:28 +05:30
data_consistency :always
2020-03-13 15:44:24 +05:30
include CronjobQueue
2021-01-29 00:20:46 +05:30
include ExclusiveLeaseGuard
2020-03-13 15:44:24 +05:30
feature_category :container_registry
2021-01-29 00:20:46 +05:30
InvalidPolicyError = Class.new(StandardError)
2021-04-29 21:17:54 +05:30
BATCH_SIZE = 1000
2021-01-29 00:20:46 +05:30
2020-03-13 15:44:24 +05:30
def perform
2021-09-04 01:27:46 +05:30
process_stale_ongoing_cleanups
disable_policies_without_container_repositories
2021-01-29 00:20:46 +05:30
throttling_enabled? ? perform_throttled : perform_unthrottled
2021-09-30 23:02:18 +05:30
log_counts
2021-01-29 00:20:46 +05:30
end
private
2021-09-04 01:27:46 +05:30
def disable_policies_without_container_repositories
ContainerExpirationPolicy.active.each_batch(of: BATCH_SIZE) do |policies|
policies.without_container_repositories
.update_all(enabled: false)
end
end
2021-09-30 23:02:18 +05:30
def log_counts
use_replica_if_available do
required_count = ContainerRepository.requiring_cleanup.count
unfinished_count = ContainerRepository.with_unfinished_cleanup.count
log_extra_metadata_on_done(:cleanup_required_count, required_count)
log_extra_metadata_on_done(:cleanup_unfinished_count, unfinished_count)
log_extra_metadata_on_done(:cleanup_total_count, required_count + unfinished_count)
end
end
# data_consistency :delayed not used as this is a cron job and those jobs are
# not perfomed with a delay
# https://gitlab.com/gitlab-org/gitlab/-/merge_requests/63635#note_603771207
def use_replica_if_available(&blk)
::Gitlab::Database::LoadBalancing::Session.current.use_replicas_for_read_queries(&blk)
end
2021-09-04 01:27:46 +05:30
def process_stale_ongoing_cleanups
threshold = delete_tags_service_timeout.seconds + 30.minutes
ContainerRepository.with_stale_ongoing_cleanup(threshold.ago)
.update_all(expiration_policy_cleanup_status: :cleanup_unfinished)
end
2021-01-29 00:20:46 +05:30
def perform_unthrottled
with_runnable_policy(preloaded: true) do |policy|
with_context(project: policy.project,
user: policy.project.owner) do |project:, user:|
ContainerExpirationPolicyService.new(project, user)
.execute(policy)
end
end
end
def perform_throttled
try_obtain_lease do
ContainerExpirationPolicies::CleanupContainerRepositoryWorker.perform_with_capacity
2020-03-13 15:44:24 +05:30
end
end
2021-01-29 00:20:46 +05:30
# TODO : remove the preload option when cleaning FF container_registry_expiration_policies_throttling
def with_runnable_policy(preloaded: false)
ContainerExpirationPolicy.runnable_schedules.each_batch(of: BATCH_SIZE) do |policies|
# rubocop: disable CodeReuse/ActiveRecord
cte = Gitlab::SQL::CTE.new(:batched_policies, policies.limit(BATCH_SIZE))
# rubocop: enable CodeReuse/ActiveRecord
scope = cte.apply_to(ContainerExpirationPolicy.all).with_container_repositories
scope = scope.preloaded if preloaded
scope.each do |policy|
if policy.valid?
yield policy
else
disable_invalid_policy!(policy)
end
end
end
end
def disable_invalid_policy!(policy)
policy.disable!
Gitlab::ErrorTracking.log_exception(
::ContainerExpirationPolicyWorker::InvalidPolicyError.new,
container_expiration_policy_id: policy.id
)
end
def throttling_enabled?
Feature.enabled?(:container_registry_expiration_policies_throttling)
end
def lease_timeout
5.hours
end
2021-09-04 01:27:46 +05:30
def delete_tags_service_timeout
::Gitlab::CurrentSettings.current_application_settings.container_registry_delete_tags_service_timeout || 0
end
2020-03-13 15:44:24 +05:30
end