debian-mirror-gitlab/metrics_server/metrics_server.rb

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

176 lines
5.6 KiB
Ruby
Raw Normal View History

2022-01-26 12:08:38 +05:30
# frozen_string_literal: true
require_relative '../config/boot'
class MetricsServer # rubocop:disable Gitlab/NamespacedClass
2022-05-07 20:08:51 +05:30
# The singleton instance used to supervise the Puma metrics server.
PumaProcessSupervisor = Class.new(Gitlab::ProcessSupervisor)
2022-01-26 12:08:38 +05:30
class << self
2022-07-16 23:28:13 +05:30
def version
Rails.root.join('GITLAB_METRICS_EXPORTER_VERSION').read.chomp
end
2022-05-07 20:08:51 +05:30
def start_for_puma
metrics_dir = ::Prometheus::Client.configuration.multiprocess_files_dir
start_server = proc do
MetricsServer.spawn('puma', metrics_dir: metrics_dir).tap do |pid|
Gitlab::AppLogger.info("Starting Puma metrics server with pid #{pid}")
end
end
supervisor = PumaProcessSupervisor.instance
supervisor.supervise(start_server.call) do
Gitlab::AppLogger.info('Puma metrics server terminated, restarting...')
start_server.call
end
end
2022-07-16 23:28:13 +05:30
def start_for_sidekiq(**options)
if new_metrics_server?
self.spawn('sidekiq', **options)
else
self.fork('sidekiq', **options)
end
end
def spawn(target, metrics_dir:, **options)
return spawn_ruby_server(target, metrics_dir: metrics_dir, **options) unless new_metrics_server?
2023-03-17 16:20:25 +05:30
settings = settings_value(target)
2022-07-16 23:28:13 +05:30
path = options[:path]&.then { |p| Pathname.new(p) } || Pathname.new('')
cmd = path.join('gitlab-metrics-exporter').to_path
env = {
2023-03-04 22:38:38 +05:30
'GOGC' => '10', # Set Go GC heap goal to 10% to curb memory growth.
2022-07-16 23:28:13 +05:30
'GME_MMAP_METRICS_DIR' => metrics_dir.to_s,
2023-03-17 16:20:25 +05:30
'GME_PROBES' => 'self,mmap,mmap_stats',
2022-07-16 23:28:13 +05:30
'GME_SERVER_HOST' => settings['address'],
'GME_SERVER_PORT' => settings['port'].to_s
}
if settings['log_enabled']
2023-03-17 16:20:25 +05:30
env['GME_LOG_FILE'] = File.join(Rails.root, 'log', "#{name(target)}.log")
2022-07-16 23:28:13 +05:30
env['GME_LOG_LEVEL'] = 'info'
else
env['GME_LOG_LEVEL'] = 'quiet'
end
2022-08-13 15:12:31 +05:30
if settings['tls_enabled']
env['GME_CERT_FILE'] = settings['tls_cert_path']
env['GME_CERT_KEY'] = settings['tls_key_path']
end
2022-07-16 23:28:13 +05:30
Process.spawn(env, cmd, err: $stderr, out: $stdout, pgroup: true).tap do |pid|
Process.detach(pid)
end
end
def spawn_ruby_server(target, metrics_dir:, wipe_metrics_dir: false, **options)
2022-04-04 11:22:00 +05:30
ensure_valid_target!(target)
cmd = "#{Rails.root}/bin/metrics-server"
env = {
'METRICS_SERVER_TARGET' => target,
2022-07-16 23:28:13 +05:30
'WIPE_METRICS_DIR' => wipe_metrics_dir ? '1' : '0',
'GITLAB_CONFIG' => ENV['GITLAB_CONFIG']
2022-04-04 11:22:00 +05:30
}
Process.spawn(env, cmd, err: $stderr, out: $stdout, pgroup: true).tap do |pid|
Process.detach(pid)
end
end
def fork(target, metrics_dir:, wipe_metrics_dir: false, reset_signals: [])
ensure_valid_target!(target)
2022-01-26 12:08:38 +05:30
pid = Process.fork
if pid.nil? # nil means we're inside the fork
# Remove any custom signal handlers the parent process had registered, since we do
# not want to inherit them, and Ruby forks with a `clone` that has the `CLONE_SIGHAND`
# flag set.
2022-04-04 11:22:00 +05:30
Gitlab::ProcessManagement.modify_signals(reset_signals, 'DEFAULT')
2022-01-26 12:08:38 +05:30
server = MetricsServer.new(target, metrics_dir, wipe_metrics_dir)
# This rewrites /proc/cmdline, since otherwise tools like `top` will show the
# parent process `cmdline` which is really confusing.
$0 = server.name
server.start
else
Process.detach(pid)
end
pid
end
2022-04-04 11:22:00 +05:30
2023-03-17 16:20:25 +05:30
def name(target)
case target
when 'puma' then 'web_exporter'
when 'sidekiq' then 'sidekiq_exporter'
else ensure_valid_target!(target)
end
end
2022-04-04 11:22:00 +05:30
private
2023-03-17 16:20:25 +05:30
# We need to use `.` (dot) notation to access the updates we did in `config/initializers/1_settings.rb`
# For that reason, avoid using `[]` ("optional/dynamic settings notation") to resolve it dynamically.
# Refer to https://gitlab.com/gitlab-org/gitlab/-/issues/386865
def settings_value(target)
case target
when 'puma' then ::Settings.monitoring.web_exporter
when 'sidekiq' then ::Settings.monitoring.sidekiq_exporter
else ensure_valid_target!(target)
end
end
2022-07-16 23:28:13 +05:30
def new_metrics_server?
Gitlab::Utils.to_boolean(ENV['GITLAB_GOLANG_METRICS_SERVER'])
end
2022-04-04 11:22:00 +05:30
def ensure_valid_target!(target)
raise "Target must be one of [puma,sidekiq]" unless %w(puma sidekiq).include?(target)
end
2022-01-26 12:08:38 +05:30
end
def initialize(target, metrics_dir, wipe_metrics_dir)
@target = target
@metrics_dir = metrics_dir
@wipe_metrics_dir = wipe_metrics_dir
end
def start
::Prometheus::Client.configure do |config|
config.multiprocess_files_dir = @metrics_dir
2022-04-04 11:22:00 +05:30
config.pid_provider = proc { name }
2022-01-26 12:08:38 +05:30
end
FileUtils.mkdir_p(@metrics_dir, mode: 0700)
2022-06-21 17:19:12 +05:30
::Prometheus::CleanupMultiprocDirService.new(@metrics_dir).execute if @wipe_metrics_dir
2022-01-26 12:08:38 +05:30
2022-03-02 08:16:31 +05:30
# We need to `warmup: true` since otherwise the sampler and exporter threads enter
# a race where not all Prometheus db files will be visible to the exporter, resulting
# in missing metrics.
# Warming up ensures that these files exist prior to the exporter starting up.
2022-04-04 11:22:00 +05:30
Gitlab::Metrics::Samplers::RubySampler.initialize_instance(prefix: name, warmup: true).start
2022-01-26 12:08:38 +05:30
2022-04-04 11:22:00 +05:30
default_opts = { gc_requests: true, synchronous: true }
exporter =
case @target
when 'puma'
Gitlab::Metrics::Exporter::WebExporter.instance(**default_opts)
when 'sidekiq'
2023-07-09 08:55:56 +05:30
settings = GitlabSettings::Options.build(Settings.monitoring.sidekiq_exporter)
2022-04-04 11:22:00 +05:30
Gitlab::Metrics::Exporter::SidekiqExporter.instance(settings, **default_opts)
end
2022-01-26 12:08:38 +05:30
2022-04-04 11:22:00 +05:30
exporter.start
2022-01-26 12:08:38 +05:30
end
def name
2023-03-17 16:20:25 +05:30
self.class.name(@target)
2022-01-26 12:08:38 +05:30
end
end