97 lines
3 KiB
Ruby
97 lines
3 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
module Gitlab
|
|
module Import
|
|
module StuckImportJob
|
|
extend ActiveSupport::Concern
|
|
|
|
StuckImportJobError = Class.new(StandardError)
|
|
|
|
IMPORT_JOBS_EXPIRATION = 24.hours.seconds.to_i
|
|
|
|
included do
|
|
include ApplicationWorker
|
|
|
|
# rubocop:disable Scalability/CronWorkerContext
|
|
# This worker updates several import states inline and does not schedule
|
|
# other jobs. So no context needed
|
|
include CronjobQueue
|
|
# rubocop:enable Scalability/CronWorkerContext
|
|
|
|
feature_category :importers
|
|
worker_resource_boundary :cpu
|
|
end
|
|
|
|
def perform
|
|
stuck_imports_without_jid_count = mark_imports_without_jid_as_failed!
|
|
stuck_imports_with_jid_count = mark_imports_with_jid_as_failed!
|
|
|
|
track_metrics(stuck_imports_with_jid_count, stuck_imports_without_jid_count)
|
|
end
|
|
|
|
private
|
|
|
|
def track_metrics(with_jid_count, without_jid_count)
|
|
raise NotImplementedError
|
|
end
|
|
|
|
def mark_imports_without_jid_as_failed!
|
|
enqueued_import_states_without_jid
|
|
.each(&method(:mark_as_failed))
|
|
.size
|
|
end
|
|
|
|
def mark_imports_with_jid_as_failed!
|
|
jids_and_ids = enqueued_import_states_with_jid.pluck(:jid, :id).to_h # rubocop: disable CodeReuse/ActiveRecord
|
|
|
|
# Find the jobs that aren't currently running or that exceeded the threshold.
|
|
completed_jids = Gitlab::SidekiqStatus.completed_jids(jids_and_ids.keys)
|
|
return 0 unless completed_jids.any?
|
|
|
|
completed_import_state_ids = jids_and_ids.values_at(*completed_jids)
|
|
|
|
# We select the import states again, because they may have transitioned from
|
|
# scheduled/started to finished/failed while we were looking up their Sidekiq status.
|
|
completed_import_states = enqueued_import_states_with_jid.id_in(completed_import_state_ids)
|
|
completed_import_state_jids = completed_import_states.map { |import_state| import_state.jid }.join(', ')
|
|
|
|
Gitlab::Import::Logger.info(
|
|
message: 'Marked stuck import jobs as failed',
|
|
job_ids: completed_import_state_jids
|
|
)
|
|
|
|
completed_import_states
|
|
.each(&method(:mark_as_failed))
|
|
.size
|
|
end
|
|
|
|
def mark_as_failed(import_state)
|
|
raise StuckImportJobError, error_message
|
|
rescue StuckImportJobError => e
|
|
Gitlab::Import::ImportFailureService.track(
|
|
import_state: import_state,
|
|
exception: e,
|
|
error_source: self.class.name,
|
|
fail_import: true
|
|
)
|
|
end
|
|
|
|
def enqueued_import_states
|
|
raise NotImplementedError
|
|
end
|
|
|
|
def enqueued_import_states_with_jid
|
|
enqueued_import_states.with_jid
|
|
end
|
|
|
|
def enqueued_import_states_without_jid
|
|
enqueued_import_states.without_jid
|
|
end
|
|
|
|
def error_message
|
|
_("Import timed out. Import took longer than %{import_jobs_expiration} seconds") % { import_jobs_expiration: IMPORT_JOBS_EXPIRATION }
|
|
end
|
|
end
|
|
end
|
|
end
|