debian-mirror-gitlab/app/services/ci/archive_trace_service.rb

# frozen_string_literal: true

module Ci
  class ArchiveTraceService
    include ::Gitlab::ExclusiveLeaseHelpers

    EXCLUSIVE_LOCK_KEY = 'archive_trace_service:batch_execute:lock'
    LOCK_TIMEOUT = 56.minutes
    LOOP_TIMEOUT = 55.minutes
    LOOP_LIMIT = 2000
    BATCH_SIZE = 100

    # rubocop: disable CodeReuse/ActiveRecord
    def batch_execute(worker_name:)
      start_time = Time.current
      in_lock(EXCLUSIVE_LOCK_KEY, ttl: LOCK_TIMEOUT, retries: 1) do
        Ci::Build.with_stale_live_trace.find_each(batch_size: BATCH_SIZE).with_index do |build, index|
          break if Time.current - start_time > LOOP_TIMEOUT

          if index > LOOP_LIMIT
            Sidekiq.logger.warn(class: worker_name, message: 'Loop limit reached.', job_id: build.id)
            break
          end

          begin
            execute(build, worker_name: worker_name)
          rescue StandardError
            next
          end
        end
      end
    end
    # rubocop: enable CodeReuse/ActiveRecord

    def execute(job, worker_name:)
      unless job.trace.archival_attempts_available?
        Sidekiq.logger.warn(class: worker_name, message: 'The job is out of archival attempts.', job_id: job.id)

        job.trace.attempt_archive_cleanup!
        return
      end

      unless job.trace.can_attempt_archival_now?
        Sidekiq.logger.warn(class: worker_name, message: 'The job can not be archived right now.', job_id: job.id)
        return
      end

      # TODO: Remove this logging once we confirmed new live trace architecture is functional.
      # See https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/4667.
      unless job.has_live_trace?
        Sidekiq.logger.warn(class: worker_name,
                            message: 'The job does not have live trace but going to be archived.',
                            job_id: job.id)
        return
      end

      job.trace.archive!
      job.remove_pending_state!

      if job.job_artifacts_trace.present?
        job.project.execute_integrations(Gitlab::DataBuilder::ArchiveTrace.build(job), :archive_trace_hooks)
      end

      # TODO: Remove this logging once we confirmed new live trace architecture is functional.
      # See https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/4667.
      unless job.has_archived_trace?
        Sidekiq.logger.warn(class: worker_name,
                            message: 'The job does not have archived trace after archiving.',
                            job_id: job.id)
      end
    rescue ::Gitlab::Ci::Trace::AlreadyArchivedError
      # It's already archived, thus we can safely ignore this exception.
    rescue StandardError => e
      job.trace.increment_archival_attempts!

      # Tracks this error with application logs, Sentry, and Prometheus.
      # If `archive!` keeps failing for over a week, that could incur data loss.
      # (See more https://docs.gitlab.com/ee/administration/job_logs.html#new-incremental-logging-architecture)
      # In order to avoid interrupting the system, we do not raise an exception here.
      archive_error(e, job, worker_name)
    end

    private

    def failed_archive_counter
      @failed_archive_counter ||=
        Gitlab::Metrics.counter(:job_trace_archive_failed_total,
                                "Counter of failed attempts of trace archiving")
    end

    def archive_error(error, job, worker_name)
      failed_archive_counter.increment

      Sidekiq.logger.warn(class: worker_name,
                          message: "Failed to archive trace. message: #{error.message}.",
                          job_id: job.id)

      Gitlab::ErrorTracking
        .track_and_raise_for_dev_exception(error,
                          issue_url: 'https://gitlab.com/gitlab-org/gitlab-foss/issues/51502',
                          job_id: job.id)
    end
  end
end