151 lines
5.1 KiB
Ruby
151 lines
5.1 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
module Ci
|
|
module JobArtifacts
|
|
class DestroyBatchService
|
|
include BaseServiceUtility
|
|
include ::Gitlab::Utils::StrongMemoize
|
|
|
|
# Danger: Private - Should only be called in Ci Services that pass a batch of job artifacts
|
|
# Not for use outside of the Ci:: namespace
|
|
|
|
# Adds the passed batch of job artifacts to the `ci_deleted_objects` table
|
|
# for asyncronous destruction of the objects in Object Storage via the `Ci::DeleteObjectsService`
|
|
# and then deletes the batch of related `ci_job_artifacts` records.
|
|
# Params:
|
|
# +job_artifacts+:: A relation of job artifacts to destroy (fewer than MAX_JOB_ARTIFACT_BATCH_SIZE)
|
|
# +pick_up_at+:: When to pick up for deletion of files
|
|
# Returns:
|
|
# +Hash+:: A hash with status and destroyed_artifacts_count keys
|
|
def initialize(job_artifacts, pick_up_at: nil, skip_projects_on_refresh: false)
|
|
@job_artifacts = job_artifacts.with_destroy_preloads.to_a
|
|
@pick_up_at = pick_up_at
|
|
@skip_projects_on_refresh = skip_projects_on_refresh
|
|
end
|
|
|
|
# rubocop: disable CodeReuse/ActiveRecord
|
|
def execute(update_stats: true)
|
|
if @skip_projects_on_refresh
|
|
exclude_artifacts_undergoing_stats_refresh
|
|
else
|
|
track_artifacts_undergoing_stats_refresh
|
|
end
|
|
|
|
exclude_trace_artifacts
|
|
|
|
return success(destroyed_artifacts_count: 0, statistics_updates: {}) if @job_artifacts.empty?
|
|
|
|
destroy_related_records(@job_artifacts)
|
|
|
|
destroy_around_hook(@job_artifacts) do
|
|
Ci::DeletedObject.transaction do
|
|
Ci::DeletedObject.bulk_import(@job_artifacts, @pick_up_at)
|
|
Ci::JobArtifact.id_in(@job_artifacts.map(&:id)).delete_all
|
|
end
|
|
end
|
|
|
|
after_batch_destroy_hook(@job_artifacts)
|
|
|
|
update_project_statistics! if update_stats
|
|
|
|
increment_monitoring_statistics(artifacts_count, artifacts_bytes)
|
|
|
|
Gitlab::Ci::Artifacts::Logger.log_deleted(@job_artifacts, 'Ci::JobArtifacts::DestroyBatchService#execute')
|
|
|
|
success(destroyed_artifacts_count: artifacts_count, statistics_updates: statistics_updates_per_project)
|
|
end
|
|
# rubocop: enable CodeReuse/ActiveRecord
|
|
|
|
private
|
|
|
|
# Overriden in EE
|
|
# :nocov:
|
|
def destroy_around_hook(artifacts)
|
|
yield
|
|
end
|
|
# :nocov:
|
|
|
|
# Overriden in EE
|
|
def destroy_related_records(artifacts); end
|
|
|
|
# Overriden in EE
|
|
def after_batch_destroy_hook(artifacts); end
|
|
|
|
# using ! here since this can't be called inside a transaction
|
|
def update_project_statistics!
|
|
statistics_updates_per_project.each do |project, increments|
|
|
ProjectStatistics.bulk_increment_statistic(project, Ci::JobArtifact.project_statistics_name, increments)
|
|
end
|
|
end
|
|
|
|
def statistics_updates_per_project
|
|
strong_memoize(:statistics_updates_per_project) do
|
|
result = Hash.new { |updates, project| updates[project] = [] }
|
|
|
|
@job_artifacts.each_with_object(result) do |job_artifact, result|
|
|
increment = Gitlab::Counters::Increment.new(amount: -job_artifact.size.to_i, ref: job_artifact.id)
|
|
result[job_artifact.project] << increment
|
|
end
|
|
end
|
|
end
|
|
|
|
def increment_monitoring_statistics(size, bytes)
|
|
metrics.increment_destroyed_artifacts_count(size)
|
|
metrics.increment_destroyed_artifacts_bytes(bytes)
|
|
end
|
|
|
|
def metrics
|
|
@metrics ||= ::Gitlab::Ci::Artifacts::Metrics.new
|
|
end
|
|
|
|
def artifacts_count
|
|
strong_memoize(:artifacts_count) do
|
|
@job_artifacts.count
|
|
end
|
|
end
|
|
|
|
def artifacts_bytes
|
|
strong_memoize(:artifacts_bytes) do
|
|
@job_artifacts.sum { |artifact| artifact.try(:size) || 0 }
|
|
end
|
|
end
|
|
|
|
# Traces should never be destroyed.
|
|
def exclude_trace_artifacts
|
|
_trace_artifacts, @job_artifacts = @job_artifacts.partition(&:trace?)
|
|
end
|
|
|
|
def track_artifacts_undergoing_stats_refresh
|
|
project_ids = @job_artifacts.find_all do |artifact|
|
|
artifact.project.refreshing_build_artifacts_size?
|
|
end.map(&:project_id).uniq
|
|
|
|
project_ids.each do |project_id|
|
|
Gitlab::ProjectStatsRefreshConflictsLogger.warn_artifact_deletion_during_stats_refresh(
|
|
method: 'Ci::JobArtifacts::DestroyBatchService#execute',
|
|
project_id: project_id
|
|
)
|
|
end
|
|
end
|
|
|
|
def exclude_artifacts_undergoing_stats_refresh
|
|
project_ids = Set.new
|
|
|
|
@job_artifacts.reject! do |artifact|
|
|
next unless artifact.project.refreshing_build_artifacts_size?
|
|
|
|
project_ids << artifact.project_id
|
|
end
|
|
|
|
if project_ids.any?
|
|
Gitlab::ProjectStatsRefreshConflictsLogger.warn_skipped_artifact_deletion_during_stats_refresh(
|
|
method: 'Ci::JobArtifacts::DestroyBatchService#execute',
|
|
project_ids: project_ids
|
|
)
|
|
end
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
Ci::JobArtifacts::DestroyBatchService.prepend_mod_with('Ci::JobArtifacts::DestroyBatchService')
|