debian-mirror-gitlab/config/initializers/7_prometheus_metrics.rb

127 lines
4.6 KiB
Ruby
Raw Normal View History

2021-03-11 19:13:27 +05:30
# frozen_string_literal: true
2022-06-21 17:19:12 +05:30
require Rails.root.join('metrics_server', 'metrics_server')
2022-05-07 20:08:51 +05:30
2019-09-30 21:07:59 +05:30
# Keep separate directories for separate processes
2022-06-21 17:19:12 +05:30
def metrics_temp_dir
2019-09-30 21:07:59 +05:30
return unless Rails.env.development? || Rails.env.test?
2020-03-13 15:44:24 +05:30
if Gitlab::Runtime.sidekiq?
2019-09-30 21:07:59 +05:30
Rails.root.join('tmp/prometheus_multiproc_dir/sidekiq')
2020-03-13 15:44:24 +05:30
elsif Gitlab::Runtime.puma?
2019-09-30 21:07:59 +05:30
Rails.root.join('tmp/prometheus_multiproc_dir/puma')
else
Rails.root.join('tmp/prometheus_multiproc_dir')
end
end
2022-06-21 17:19:12 +05:30
def prometheus_metrics_dir
ENV['prometheus_multiproc_dir'] || metrics_temp_dir
end
def puma_master?
2022-04-04 11:22:00 +05:30
Prometheus::PidProvider.worker_id == 'puma_master'
end
2022-06-21 17:19:12 +05:30
# Whether a dedicated process should run that serves Rails application metrics, as opposed
# to using a Rails controller.
def puma_dedicated_metrics_server?
Settings.monitoring.web_exporter.enabled
2022-04-04 11:22:00 +05:30
end
2022-06-21 17:19:12 +05:30
if puma_master?
2022-04-04 11:22:00 +05:30
# The following is necessary to ensure stale Prometheus metrics don't accumulate over time.
2022-06-21 17:19:12 +05:30
# It needs to be done as early as possible to ensure new metrics aren't being deleted.
#
# Note that this should not happen for Sidekiq. Since Sidekiq workers are spawned from the
# sidekiq-cluster script, we perform this cleanup in `sidekiq_cluster/cli.rb` instead,
# since it must happen prior to any worker processes or the metrics server starting up.
Prometheus::CleanupMultiprocDirService.new(prometheus_metrics_dir).execute
2022-04-04 11:22:00 +05:30
::Prometheus::Client.reinitialize_on_pid_change(force: true)
end
2021-09-30 23:02:18 +05:30
::Prometheus::Client.configure do |config|
2020-11-24 15:15:51 +05:30
config.logger = Gitlab::AppLogger
2017-09-10 17:25:29 +05:30
2022-06-21 17:19:12 +05:30
config.multiprocess_files_dir = prometheus_metrics_dir
2017-09-10 17:25:29 +05:30
2021-09-30 23:02:18 +05:30
config.pid_provider = ::Prometheus::PidProvider.method(:worker_id)
2017-09-10 17:25:29 +05:30
end
2018-03-17 18:26:18 +05:30
Gitlab::Application.configure do |config|
# 0 should be Sentry to catch errors in this middleware
2021-09-04 01:27:46 +05:30
config.middleware.insert_after(Labkit::Middleware::Rack, Gitlab::Metrics::RequestsRackMiddleware)
2018-03-17 18:26:18 +05:30
end
2022-05-07 20:08:51 +05:30
# Any actions beyond this check should only execute outside of tests, when running in an application
# context (i.e. not in the Rails console or rspec) and when users have enabled metrics.
return if Rails.env.test? || !Gitlab::Runtime.application? || !Gitlab::Metrics.prometheus_metrics_enabled?
Gitlab::Cluster::LifecycleEvents.on_master_start do
Gitlab::Metrics.gauge(:deployments, 'GitLab Version', {}, :max).set({ version: Gitlab::VERSION, revision: Gitlab.revision }, 1)
2021-03-11 19:13:27 +05:30
2022-05-07 20:08:51 +05:30
if Gitlab::Runtime.puma?
2022-07-23 23:45:48 +05:30
[
Gitlab::Metrics::Samplers::RubySampler,
Gitlab::Metrics::Samplers::ThreadsSampler
].each { |sampler| sampler.instance(logger: Gitlab::AppLogger).start }
2022-05-07 20:08:51 +05:30
Gitlab::Metrics::Samplers::PumaSampler.instance.start
2021-03-11 19:13:27 +05:30
2022-06-21 17:19:12 +05:30
MetricsServer.start_for_puma if puma_dedicated_metrics_server?
2021-03-11 19:13:27 +05:30
end
2022-05-07 20:08:51 +05:30
Gitlab::Ci::Parsers.instrument!
rescue IOError => e
Gitlab::ErrorTracking.track_exception(e)
Gitlab::Metrics.error_detected!
end
2020-11-24 15:15:51 +05:30
2022-05-07 20:08:51 +05:30
Gitlab::Cluster::LifecycleEvents.on_worker_start do
defined?(::Prometheus::Client.reinitialize_on_pid_change) && ::Prometheus::Client.reinitialize_on_pid_change
logger = Gitlab::AppLogger
2022-07-23 23:45:48 +05:30
# Since we also run these samplers in the Puma primary, we need to re-create them each time we fork.
# For Sidekiq, this does not make any difference, since there is no primary.
[
Gitlab::Metrics::Samplers::RubySampler,
Gitlab::Metrics::Samplers::ThreadsSampler
].each { |sampler| sampler.initialize_instance(logger: logger, recreate: true).start }
2022-05-07 20:08:51 +05:30
Gitlab::Metrics::Samplers::DatabaseSampler.initialize_instance(logger: logger).start
if Gitlab::Runtime.puma?
# Since we are observing a metrics server from the Puma primary, we would inherit
# this supervision thread after forking into workers, so we need to explicitly stop it here.
2022-06-21 17:19:12 +05:30
::MetricsServer::PumaProcessSupervisor.instance.stop if puma_dedicated_metrics_server?
2019-12-21 20:55:43 +05:30
2022-05-07 20:08:51 +05:30
Gitlab::Metrics::Samplers::ActionCableSampler.instance(logger: logger).start
2019-12-21 20:55:43 +05:30
end
2022-05-07 20:08:51 +05:30
if Gitlab.ee? && Gitlab::Runtime.sidekiq?
Gitlab::Metrics::Samplers::GlobalSearchSampler.instance(logger: logger).start
2019-12-21 20:55:43 +05:30
end
2022-05-07 20:08:51 +05:30
Gitlab::Ci::Parsers.instrument!
rescue IOError => e
Gitlab::ErrorTracking.track_exception(e)
Gitlab::Metrics.error_detected!
end
2022-06-21 17:19:12 +05:30
if Gitlab::Runtime.puma? && puma_dedicated_metrics_server?
2019-12-26 22:10:19 +05:30
Gitlab::Cluster::LifecycleEvents.on_before_graceful_shutdown do
# We need to ensure that before we re-exec or shutdown server
2022-05-07 20:08:51 +05:30
# we also stop the metrics server
2022-06-21 17:19:12 +05:30
::MetricsServer::PumaProcessSupervisor.instance.shutdown
2019-12-21 20:55:43 +05:30
end
Gitlab::Cluster::LifecycleEvents.on_before_master_restart do
# We need to ensure that before we re-exec server
2022-05-07 20:08:51 +05:30
# we also stop the metrics server
2019-12-21 20:55:43 +05:30
#
# We do it again, for being extra safe,
# but it should not be needed
2022-06-21 17:19:12 +05:30
::MetricsServer::PumaProcessSupervisor.instance.shutdown
2019-09-04 21:01:54 +05:30
end
2018-12-13 13:39:08 +05:30
end