debian-mirror-gitlab/config/initializers/sidekiq.rb

109 lines
4 KiB
Ruby
Raw Normal View History

2020-04-08 14:13:33 +05:30
# frozen_string_literal: true
2021-01-29 00:20:46 +05:30
module SidekiqLogArguments
def self.enabled?
Gitlab::Utils.to_boolean(ENV['SIDEKIQ_LOG_ARGUMENTS'], default: true)
end
end
2020-04-08 14:13:33 +05:30
2019-07-31 22:56:46 +05:30
def enable_reliable_fetch?
return true unless Feature::FlipperFeature.table_exists?
2021-01-03 14:25:43 +05:30
Feature.enabled?(:gitlab_sidekiq_reliable_fetcher, type: :ops, default_enabled: true)
2019-07-31 22:56:46 +05:30
end
def enable_semi_reliable_fetch_mode?
return true unless Feature::FlipperFeature.table_exists?
2021-01-03 14:25:43 +05:30
Feature.enabled?(:gitlab_sidekiq_enable_semi_reliable_fetcher, type: :ops, default_enabled: true)
2019-07-31 22:56:46 +05:30
end
2017-09-10 17:25:29 +05:30
# Custom Queues configuration
queues_config_hash = Gitlab::Redis::Queues.params
queues_config_hash[:namespace] = Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE
2016-09-13 17:45:13 +05:30
2018-05-09 12:01:36 +05:30
enable_json_logs = Gitlab.config.sidekiq.log_format == 'json'
2019-12-21 20:55:43 +05:30
enable_sidekiq_memory_killer = ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'].to_i.nonzero?
2020-11-24 15:15:51 +05:30
use_sidekiq_daemon_memory_killer = ENV.fetch("SIDEKIQ_DAEMON_MEMORY_KILLER", 1).to_i.nonzero?
2019-12-21 20:55:43 +05:30
use_sidekiq_legacy_memory_killer = !use_sidekiq_daemon_memory_killer
2018-05-09 12:01:36 +05:30
2014-09-02 18:07:02 +05:30
Sidekiq.configure_server do |config|
2018-05-09 12:01:36 +05:30
if enable_json_logs
Sidekiq.logger.formatter = Gitlab::SidekiqLogging::JSONFormatter.new
config.options[:job_logger] = Gitlab::SidekiqLogging::StructuredLogger
2019-12-21 20:55:43 +05:30
# Remove the default-provided handler
config.error_handlers.reject! { |handler| handler.is_a?(Sidekiq::ExceptionHandler::Logger) }
config.error_handlers << Gitlab::SidekiqLogging::ExceptionHandler.new
2018-05-09 12:01:36 +05:30
end
2020-04-08 14:13:33 +05:30
config.redis = queues_config_hash
config.server_middleware(&Gitlab::SidekiqMiddleware.server_configurator({
metrics: Settings.monitoring.sidekiq_exporter,
2021-01-29 00:20:46 +05:30
arguments_logger: SidekiqLogArguments.enabled? && !enable_json_logs,
2020-05-24 23:13:21 +05:30
memory_killer: enable_sidekiq_memory_killer && use_sidekiq_legacy_memory_killer
2020-04-08 14:13:33 +05:30
}))
2020-01-01 13:55:28 +05:30
config.client_middleware(&Gitlab::SidekiqMiddleware.client_configurator)
2017-08-17 22:00:37 +05:30
config.on :startup do
# Clear any connections that might have been obtained before starting
# Sidekiq (e.g. in an initializer).
ActiveRecord::Base.clear_all_connections!
2019-12-04 20:38:33 +05:30
2019-12-21 20:55:43 +05:30
# Start monitor to track running jobs. By default, cancel job is not enabled
# To cancel job, it requires `SIDEKIQ_MONITOR_WORKER=1` to enable notification channel
Gitlab::SidekiqDaemon::Monitor.instance.start
Gitlab::SidekiqDaemon::MemoryKiller.instance.start if enable_sidekiq_memory_killer && use_sidekiq_daemon_memory_killer
2014-09-02 18:07:02 +05:30
end
2015-12-23 02:04:40 +05:30
2019-07-31 22:56:46 +05:30
if enable_reliable_fetch?
config.options[:semi_reliable_fetch] = enable_semi_reliable_fetch_mode?
2019-02-15 15:39:39 +05:30
Sidekiq::ReliableFetch.setup_reliable_fetch!(config)
2018-12-05 23:21:45 +05:30
end
2020-04-08 14:13:33 +05:30
Gitlab.config.load_dynamic_cron_schedules!
# Sidekiq-cron: load recurring jobs from gitlab.yml
# UGLY Hack to get nested hash from settingslogic
2020-05-24 23:13:21 +05:30
cron_jobs = Gitlab::Json.parse(Gitlab.config.cron_jobs.to_json)
# UGLY hack: Settingslogic doesn't allow 'class' key
2016-08-24 12:49:21 +05:30
cron_jobs_required_keys = %w(job_class cron)
cron_jobs.each do |k, v|
if cron_jobs[k] && cron_jobs_required_keys.all? { |s| cron_jobs[k].key?(s) }
cron_jobs[k]['class'] = cron_jobs[k].delete('job_class')
else
cron_jobs.delete(k)
2020-11-24 15:15:51 +05:30
Gitlab::AppLogger.error("Invalid cron_jobs config key: '#{k}'. Check your gitlab config file.")
2016-08-24 12:49:21 +05:30
end
end
Sidekiq::Cron::Job.load_from_hash! cron_jobs
2015-12-23 02:04:40 +05:30
2018-03-17 18:26:18 +05:30
Gitlab::SidekiqVersioning.install!
2019-09-30 21:07:59 +05:30
Gitlab.ee do
Gitlab::Mirror.configure_cron_job!
Gitlab::Geo.configure_cron_jobs!
end
2016-06-22 15:30:34 +05:30
# Avoid autoload issue such as 'Mail::Parsers::AddressStruct'
# https://github.com/mikel/mail/issues/912#issuecomment-214850355
Mail.eager_autoload!
2019-07-07 11:18:12 +05:30
# Ensure the whole process group is terminated if possible
Gitlab::SidekiqSignals.install!(Sidekiq::CLI::SIGNAL_HANDLERS)
2014-09-02 18:07:02 +05:30
end
Sidekiq.configure_client do |config|
2017-09-10 17:25:29 +05:30
config.redis = queues_config_hash
2020-04-08 14:13:33 +05:30
# We only need to do this for other clients. If Sidekiq-server is the
# client scheduling jobs, we have access to the regular sidekiq logger that
# writes to STDOUT
Sidekiq.logger = Gitlab::SidekiqLogging::ClientLogger.build
Sidekiq.logger.formatter = Gitlab::SidekiqLogging::JSONFormatter.new if enable_json_logs
2017-08-17 22:00:37 +05:30
2020-01-01 13:55:28 +05:30
config.client_middleware(&Gitlab::SidekiqMiddleware.client_configurator)
2017-08-17 22:00:37 +05:30
end