2018-11-08 19:23:39 +05:30
# frozen_string_literal: true
2017-08-17 22:00:37 +05:30
class StuckImportJobsWorker
2018-03-17 18:26:18 +05:30
include ApplicationWorker
2017-08-17 22:00:37 +05:30
include CronjobQueue
2018-03-17 18:26:18 +05:30
IMPORT_JOBS_EXPIRATION = 15 . hours . to_i
2017-08-17 22:00:37 +05:30
def perform
2018-03-17 18:26:18 +05:30
projects_without_jid_count = mark_projects_without_jid_as_failed!
projects_with_jid_count = mark_projects_with_jid_as_failed!
Gitlab :: Metrics . add_event ( :stuck_import_jobs ,
projects_without_jid_count : projects_without_jid_count ,
projects_with_jid_count : projects_with_jid_count )
end
private
def mark_projects_without_jid_as_failed!
2018-03-27 19:54:05 +05:30
enqueued_projects_without_jid . each do | project |
2018-03-17 18:26:18 +05:30
project . mark_import_as_failed ( error_message )
end . count
end
2018-12-05 23:21:45 +05:30
# rubocop: disable CodeReuse/ActiveRecord
2018-03-17 18:26:18 +05:30
def mark_projects_with_jid_as_failed!
2018-10-15 14:42:47 +05:30
# TODO: Rollback this change to use SQL through #pluck
jids_and_ids = enqueued_projects_with_jid . map { | project | [ project . import_jid , project . id ] } . to_h
2018-03-17 18:26:18 +05:30
2018-03-27 19:54:05 +05:30
# Find the jobs that aren't currently running or that exceeded the threshold.
completed_jids = Gitlab :: SidekiqStatus . completed_jids ( jids_and_ids . keys )
return unless completed_jids . any?
2017-08-17 22:00:37 +05:30
2018-03-27 19:54:05 +05:30
completed_project_ids = jids_and_ids . values_at ( * completed_jids )
2017-08-17 22:00:37 +05:30
2018-03-27 19:54:05 +05:30
# We select the projects again, because they may have transitioned from
# scheduled/started to finished/failed while we were looking up their Sidekiq status.
completed_projects = enqueued_projects_with_jid . where ( id : completed_project_ids )
2017-08-17 22:00:37 +05:30
2018-03-27 19:54:05 +05:30
Rails . logger . info ( " Marked stuck import jobs as failed. JIDs: #{ completed_projects . map ( & :import_jid ) . join ( ', ' ) } " )
2017-08-17 22:00:37 +05:30
2018-03-27 19:54:05 +05:30
completed_projects . each do | project |
project . mark_import_as_failed ( error_message )
end . count
2018-03-17 18:26:18 +05:30
end
2018-12-05 23:21:45 +05:30
# rubocop: enable CodeReuse/ActiveRecord
2017-08-17 22:00:37 +05:30
2018-12-05 23:21:45 +05:30
# rubocop: disable CodeReuse/ActiveRecord
2018-03-27 19:54:05 +05:30
def enqueued_projects
2018-10-15 14:42:47 +05:30
Project . joins_import_state . where ( " (import_state.status = 'scheduled' OR import_state.status = 'started') OR (projects.import_status = 'scheduled' OR projects.import_status = 'started') " )
2017-08-17 22:00:37 +05:30
end
2018-12-05 23:21:45 +05:30
# rubocop: enable CodeReuse/ActiveRecord
2017-08-17 22:00:37 +05:30
2018-12-05 23:21:45 +05:30
# rubocop: disable CodeReuse/ActiveRecord
2018-03-27 19:54:05 +05:30
def enqueued_projects_with_jid
2018-10-15 14:42:47 +05:30
enqueued_projects . where . not ( " import_state.jid IS NULL AND projects.import_jid IS NULL " )
2018-03-17 18:26:18 +05:30
end
2018-12-05 23:21:45 +05:30
# rubocop: enable CodeReuse/ActiveRecord
2017-08-17 22:00:37 +05:30
2018-12-05 23:21:45 +05:30
# rubocop: disable CodeReuse/ActiveRecord
2018-03-27 19:54:05 +05:30
def enqueued_projects_without_jid
2018-10-15 14:42:47 +05:30
enqueued_projects . where ( " import_state.jid IS NULL AND projects.import_jid IS NULL " )
2017-08-17 22:00:37 +05:30
end
2018-12-05 23:21:45 +05:30
# rubocop: enable CodeReuse/ActiveRecord
2017-08-17 22:00:37 +05:30
def error_message
2018-03-17 18:26:18 +05:30
" Import timed out. Import took longer than #{ IMPORT_JOBS_EXPIRATION } seconds "
2017-08-17 22:00:37 +05:30
end
end