debian-mirror-gitlab/lib/gitlab/sidekiq_config.rb

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

205 lines
6.7 KiB
Ruby
Raw Normal View History

2018-12-13 13:39:08 +05:30
# frozen_string_literal: true
2018-03-17 18:26:18 +05:30
require 'yaml'
module Gitlab
module SidekiqConfig
2020-03-13 15:44:24 +05:30
FOSS_QUEUE_CONFIG_PATH = 'app/workers/all_queues.yml'
EE_QUEUE_CONFIG_PATH = 'ee/app/workers/all_queues.yml'
2021-12-11 22:18:48 +05:30
JH_QUEUE_CONFIG_PATH = 'jh/app/workers/all_queues.yml'
2020-03-13 15:44:24 +05:30
SIDEKIQ_QUEUES_PATH = 'config/sidekiq_queues.yml'
2022-01-26 12:08:38 +05:30
JH_SIDEKIQ_QUEUES_PATH = 'jh/config/sidekiq_queues.yml'
2020-03-13 15:44:24 +05:30
QUEUE_CONFIG_PATHS = [
FOSS_QUEUE_CONFIG_PATH,
2021-12-11 22:18:48 +05:30
(EE_QUEUE_CONFIG_PATH if Gitlab.ee?),
(JH_QUEUE_CONFIG_PATH if Gitlab.jh?)
2020-03-13 15:44:24 +05:30
].compact.freeze
2021-04-29 21:17:54 +05:30
# This maps workers not in our application code to queues. We need
# these queues in our YAML files to ensure we don't accidentally
# miss jobs from these queues.
#
# The default queue should be unused, which is why it maps to an
# invalid class name. We keep it in the YAML file for safety, just
# in case anything does get scheduled to run there.
DEFAULT_WORKERS = {
2021-06-08 01:23:25 +05:30
'_' => DummyWorker.new(
queue: 'default',
2021-11-18 22:05:49 +05:30
weight: 1,
tags: []
2021-06-08 01:23:25 +05:30
),
'ActionMailer::MailDeliveryJob' => DummyWorker.new(
name: 'ActionMailer::MailDeliveryJob',
queue: 'mailers',
urgency: 'low',
weight: 2,
tags: []
)
2021-12-11 22:18:48 +05:30
}.transform_values { |worker| Gitlab::SidekiqConfig::Worker.new(worker, ee: false, jh: false) }.freeze
2020-03-13 15:44:24 +05:30
class << self
include Gitlab::SidekiqConfig::CliMethods
def redis_queues
# Not memoized, because this can change during the life of the application
Sidekiq::Queue.all.map(&:name)
end
def config_queues
@config_queues ||= begin
config = YAML.load_file(Rails.root.join(SIDEKIQ_QUEUES_PATH))
config[:queues].map(&:first)
end
end
2022-11-25 23:54:43 +05:30
def cron_jobs
@cron_jobs ||= begin
Gitlab.config.load_dynamic_cron_schedules!
# Load recurring jobs from gitlab.yml
# UGLY Hack to get nested hash from settingslogic
jobs = Gitlab::Json.parse(Gitlab.config.cron_jobs.to_json)
jobs.delete('poll_interval') # Would be interpreted as a job otherwise
# UGLY hack: Settingslogic doesn't allow 'class' key
required_keys = %w[job_class cron]
jobs.each do |k, v|
if jobs[k] && required_keys.all? { |s| jobs[k].key?(s) }
jobs[k]['class'] = jobs[k].delete('job_class')
else
jobs.delete(k)
Gitlab::AppLogger.error("Invalid cron_jobs config key: '#{k}'. Check your gitlab config file.")
end
end
jobs
end
end
2020-03-13 15:44:24 +05:30
def cron_workers
2022-11-25 23:54:43 +05:30
@cron_workers ||= cron_jobs.map { |job_name, options| options['class'].constantize }
2020-03-13 15:44:24 +05:30
end
2019-02-15 15:39:39 +05:30
2020-03-13 15:44:24 +05:30
def workers
@workers ||= begin
result = []
2021-04-29 21:17:54 +05:30
result.concat(DEFAULT_WORKERS.values)
2021-12-11 22:18:48 +05:30
result.concat(find_workers(Rails.root.join('app', 'workers'), ee: false, jh: false))
2019-02-15 15:39:39 +05:30
2020-03-13 15:44:24 +05:30
if Gitlab.ee?
2021-12-11 22:18:48 +05:30
result.concat(find_workers(Rails.root.join('ee', 'app', 'workers'), ee: true, jh: false))
end
if Gitlab.jh?
result.concat(find_workers(Rails.root.join('jh', 'app', 'workers'), ee: false, jh: true))
2020-03-13 15:44:24 +05:30
end
2019-02-15 15:39:39 +05:30
2020-03-13 15:44:24 +05:30
result
end
end
def workers_for_all_queues_yml
2021-12-11 22:18:48 +05:30
workers.each_with_object([[], [], []]) do |worker, array|
if worker.jh?
array[2].push(worker)
elsif worker.ee?
array[1].push(worker)
else
array[0].push(worker)
end
end.map(&:sort)
2019-02-15 15:39:39 +05:30
end
2018-03-17 18:26:18 +05:30
2020-03-13 15:44:24 +05:30
# YAML.load_file is OK here as we control the file contents
def all_queues_yml_outdated?
2021-12-11 22:18:48 +05:30
foss_workers, ee_workers, jh_workers = workers_for_all_queues_yml
2018-03-17 18:26:18 +05:30
2020-03-13 15:44:24 +05:30
return true if foss_workers != YAML.load_file(FOSS_QUEUE_CONFIG_PATH)
2018-03-17 18:26:18 +05:30
2021-12-11 22:18:48 +05:30
return true if Gitlab.ee? && ee_workers != YAML.load_file(EE_QUEUE_CONFIG_PATH)
Gitlab.jh? && File.exist?(JH_QUEUE_CONFIG_PATH) && jh_workers != YAML.load_file(JH_QUEUE_CONFIG_PATH)
2018-03-17 18:26:18 +05:30
end
2020-03-13 15:44:24 +05:30
def queues_for_sidekiq_queues_yml
namespaces_with_equal_weights =
workers
2022-01-26 12:08:38 +05:30
.reject { |worker| worker.jh? }
2020-03-13 15:44:24 +05:30
.group_by(&:queue_namespace)
.map(&:last)
.select { |workers| workers.map(&:get_weight).uniq.count == 1 }
.map(&:first)
namespaces = namespaces_with_equal_weights.map(&:queue_namespace).to_set
2022-01-26 12:08:38 +05:30
remaining_queues = workers.reject { |worker| worker.jh? }.reject { |worker| namespaces.include?(worker.queue_namespace) }
2018-03-17 18:26:18 +05:30
2020-03-13 15:44:24 +05:30
(namespaces_with_equal_weights.map(&:namespace_and_weight) +
remaining_queues.map(&:queue_and_weight)).sort
2018-03-17 18:26:18 +05:30
end
2022-01-26 12:08:38 +05:30
# Override in JH repo
def jh_queues_for_sidekiq_queues_yml
[]
end
2020-03-13 15:44:24 +05:30
# YAML.load_file is OK here as we control the file contents
def sidekiq_queues_yml_outdated?
config_queues = YAML.load_file(SIDEKIQ_QUEUES_PATH)[:queues]
2018-03-17 18:26:18 +05:30
2020-03-13 15:44:24 +05:30
queues_for_sidekiq_queues_yml != config_queues
2019-12-21 20:55:43 +05:30
end
2018-03-17 18:26:18 +05:30
2021-09-30 23:02:18 +05:30
# Returns a hash of worker class name => mapped queue name
def worker_queue_mappings
workers
.reject { |worker| worker.klass.is_a?(Gitlab::SidekiqConfig::DummyWorker) }
.to_h { |worker| [worker.klass.to_s, ::Gitlab::SidekiqConfig::WorkerRouter.global.route(worker.klass)] }
end
# Like worker_queue_mappings, but only for the queues running in
# the current Sidekiq process
def current_worker_queue_mappings
worker_queue_mappings
.select { |worker, queue| Sidekiq.options[:queues].include?(queue) }
.to_h
end
2022-07-16 23:28:13 +05:30
# Get the list of queues from all available workers following queue
# routing rules. Sidekiq::Queue.all fetches the list of queues from Redis.
# It may contain some redundant, obsolete queues from previous iterations
# of GitLab.
def routing_queues
@routing_queues ||= workers.map do |worker|
if worker.klass.is_a?(Gitlab::SidekiqConfig::DummyWorker)
worker.queue
else
::Gitlab::SidekiqConfig::WorkerRouter.global.route(worker.klass)
end
end.uniq.sort
end
2020-03-13 15:44:24 +05:30
private
2021-12-11 22:18:48 +05:30
def find_workers(root, ee:, jh:)
2020-03-13 15:44:24 +05:30
concerns = root.join('concerns').to_s
2018-03-17 18:26:18 +05:30
2020-03-13 15:44:24 +05:30
Dir[root.join('**', '*.rb')]
.reject { |path| path.start_with?(concerns) }
.map { |path| worker_from_path(path, root) }
.select { |worker| worker < Sidekiq::Worker }
2021-12-11 22:18:48 +05:30
.map { |worker| Gitlab::SidekiqConfig::Worker.new(worker, ee: ee, jh: jh) }
2020-03-13 15:44:24 +05:30
end
2018-03-17 18:26:18 +05:30
2020-03-13 15:44:24 +05:30
def worker_from_path(path, root)
2018-03-17 18:26:18 +05:30
ns = Pathname.new(path).relative_path_from(root).to_s.gsub('.rb', '')
ns.camelize.constantize
end
end
end
end
2022-01-26 12:08:38 +05:30
Gitlab::SidekiqConfig.prepend_mod