debian-mirror-gitlab/app/services/ci/job_artifacts/destroy_batch_service.rb

# frozen_string_literal: true

module Ci
  module JobArtifacts
    class DestroyBatchService
      include BaseServiceUtility
      include ::Gitlab::Utils::StrongMemoize

      # Danger: Private - Should only be called in Ci Services that pass a batch of job artifacts
      # Not for use outside of the Ci:: namespace

      # Adds the passed batch of job artifacts to the `ci_deleted_objects` table
      # for asyncronous destruction of the objects in Object Storage via the `Ci::DeleteObjectsService`
      # and then deletes the batch of related `ci_job_artifacts` records.
      # Params:
      # +job_artifacts+:: A relation of job artifacts to destroy (fewer than MAX_JOB_ARTIFACT_BATCH_SIZE)
      # +pick_up_at+:: When to pick up for deletion of files
      # Returns:
      # +Hash+:: A hash with status and destroyed_artifacts_count keys
      def initialize(job_artifacts, pick_up_at: nil, skip_projects_on_refresh: false)
        @job_artifacts = job_artifacts.with_destroy_preloads.to_a
        @pick_up_at = pick_up_at
        @skip_projects_on_refresh = skip_projects_on_refresh
      end

      # rubocop: disable CodeReuse/ActiveRecord
      def execute(update_stats: true)
        if @skip_projects_on_refresh
          exclude_artifacts_undergoing_stats_refresh
        else
          track_artifacts_undergoing_stats_refresh
        end

        exclude_trace_artifacts

        return success(destroyed_artifacts_count: 0, statistics_updates: {}) if @job_artifacts.empty?

        destroy_related_records(@job_artifacts)

        destroy_around_hook(@job_artifacts) do
          Ci::DeletedObject.transaction do
            Ci::DeletedObject.bulk_import(@job_artifacts, @pick_up_at)
            Ci::JobArtifact.id_in(@job_artifacts.map(&:id)).delete_all
          end
        end

        after_batch_destroy_hook(@job_artifacts)

        # This is executed outside of the transaction because it depends on Redis
        update_project_statistics! if update_stats
        increment_monitoring_statistics(artifacts_count, artifacts_bytes)

        Gitlab::Ci::Artifacts::Logger.log_deleted(@job_artifacts, 'Ci::JobArtifacts::DestroyBatchService#execute')

        success(destroyed_artifacts_count: artifacts_count,
                statistics_updates: affected_project_statistics)
      end
      # rubocop: enable CodeReuse/ActiveRecord

      private

      # Overriden in EE
      # :nocov:
      def destroy_around_hook(artifacts)
        yield
      end
      # :nocov:

      # Overriden in EE
      def destroy_related_records(artifacts); end

      # Overriden in EE
      def after_batch_destroy_hook(artifacts); end

      # using ! here since this can't be called inside a transaction
      def update_project_statistics!
        affected_project_statistics.each do |project, delta|
          project.increment_statistic_value(Ci::JobArtifact.project_statistics_name, delta)
        end
      end

      def affected_project_statistics
        strong_memoize(:affected_project_statistics) do
          artifacts_by_project = @job_artifacts.group_by(&:project)
          artifacts_by_project.each.with_object({}) do |(project, artifacts), accumulator|
            delta = -artifacts.sum { |artifact| artifact.size.to_i }
            accumulator[project] = delta
          end
        end
      end

      def increment_monitoring_statistics(size, bytes)
        metrics.increment_destroyed_artifacts_count(size)
        metrics.increment_destroyed_artifacts_bytes(bytes)
      end

      def metrics
        @metrics ||= ::Gitlab::Ci::Artifacts::Metrics.new
      end

      def artifacts_count
        strong_memoize(:artifacts_count) do
          @job_artifacts.count
        end
      end

      def artifacts_bytes
        strong_memoize(:artifacts_bytes) do
          @job_artifacts.sum { |artifact| artifact.try(:size) || 0 }
        end
      end

      # Traces should never be destroyed.
      def exclude_trace_artifacts
        _trace_artifacts, @job_artifacts = @job_artifacts.partition(&:trace?)
      end

      def track_artifacts_undergoing_stats_refresh
        project_ids = @job_artifacts.find_all do |artifact|
          artifact.project.refreshing_build_artifacts_size?
        end.map(&:project_id).uniq

        project_ids.each do |project_id|
          Gitlab::ProjectStatsRefreshConflictsLogger.warn_artifact_deletion_during_stats_refresh(
            method: 'Ci::JobArtifacts::DestroyBatchService#execute',
            project_id: project_id
          )
        end
      end

      def exclude_artifacts_undergoing_stats_refresh
        project_ids = Set.new

        @job_artifacts.reject! do |artifact|
          next unless artifact.project.refreshing_build_artifacts_size?

          project_ids << artifact.project_id
        end

        if project_ids.any?
          Gitlab::ProjectStatsRefreshConflictsLogger.warn_skipped_artifact_deletion_during_stats_refresh(
            method: 'Ci::JobArtifacts::DestroyBatchService#execute',
            project_ids: project_ids
          )
        end
      end
    end
  end
end

Ci::JobArtifacts::DestroyBatchService.prepend_mod_with('Ci::JobArtifacts::DestroyBatchService')