debian-mirror-gitlab/lib/gitlab/memory/watchdog.rb
2022-08-27 11:52:29 +05:30

190 lines
6.2 KiB
Ruby

# frozen_string_literal: true
module Gitlab
module Memory
# A background thread that observes Ruby heap fragmentation and calls
# into a handler when the Ruby heap has been fragmented for an extended
# period of time.
#
# See Gitlab::Metrics::Memory for how heap fragmentation is defined.
#
# To decide whether a given fragmentation level is being exceeded,
# the watchdog regularly polls the GC. Whenever a violation occurs
# a strike is issued. If the maximum number of strikes are reached,
# a handler is invoked to deal with the situation.
#
# The duration for which a process may be above a given fragmentation
# threshold is computed as `max_strikes * sleep_time_seconds`.
class Watchdog
DEFAULT_SLEEP_TIME_SECONDS = 60
DEFAULT_HEAP_FRAG_THRESHOLD = 0.5
DEFAULT_MAX_STRIKES = 5
# This handler does nothing. It returns `false` to indicate to the
# caller that the situation has not been dealt with so it will
# receive calls repeatedly if fragmentation remains high.
#
# This is useful for "dress rehearsals" in production since it allows
# us to observe how frequently the handler is invoked before taking action.
class NullHandler
include Singleton
def on_high_heap_fragmentation(value)
# NOP
false
end
end
# This handler sends SIGTERM and considers the situation handled.
class TermProcessHandler
def initialize(pid = $$)
@pid = pid
end
def on_high_heap_fragmentation(value)
Process.kill(:TERM, @pid)
true
end
end
# This handler invokes Puma's graceful termination handler, which takes
# into account a configurable grace period during which a process may
# remain unresponsive to a SIGTERM.
class PumaHandler
def initialize(puma_options = ::Puma.cli_config.options)
@worker = ::Puma::Cluster::WorkerHandle.new(0, $$, 0, puma_options)
end
def on_high_heap_fragmentation(value)
@worker.term
true
end
end
# max_heap_fragmentation:
# The degree to which the Ruby heap is allowed to be fragmented. Range [0,1].
# max_strikes:
# How many times the process is allowed to be above max_heap_fragmentation before
# a handler is invoked.
# sleep_time_seconds:
# Used to control the frequency with which the watchdog will wake up and poll the GC.
def initialize(
handler: NullHandler.instance,
logger: Logger.new($stdout),
max_heap_fragmentation: ENV['GITLAB_MEMWD_MAX_HEAP_FRAG']&.to_f || DEFAULT_HEAP_FRAG_THRESHOLD,
max_strikes: ENV['GITLAB_MEMWD_MAX_STRIKES']&.to_i || DEFAULT_MAX_STRIKES,
sleep_time_seconds: ENV['GITLAB_MEMWD_SLEEP_TIME_SEC']&.to_i || DEFAULT_SLEEP_TIME_SECONDS,
**options)
super(**options)
@handler = handler
@logger = logger
@max_heap_fragmentation = max_heap_fragmentation
@sleep_time_seconds = sleep_time_seconds
@max_strikes = max_strikes
@alive = true
@strikes = 0
init_prometheus_metrics(max_heap_fragmentation)
end
attr_reader :strikes, :max_heap_fragmentation, :max_strikes, :sleep_time_seconds
def call
@logger.info(log_labels.merge(message: 'started'))
while @alive
sleep(@sleep_time_seconds)
monitor_heap_fragmentation if Feature.enabled?(:gitlab_memory_watchdog, type: :ops)
end
@logger.info(log_labels.merge(message: 'stopped'))
end
def stop
@alive = false
end
private
def monitor_heap_fragmentation
heap_fragmentation = Gitlab::Metrics::Memory.gc_heap_fragmentation
if heap_fragmentation > @max_heap_fragmentation
@strikes += 1
@heap_frag_violations.increment
else
@strikes = 0
end
if @strikes > @max_strikes
# If the handler returns true, it means the event is handled and we can shut down.
@alive = !handle_heap_fragmentation_limit_exceeded(heap_fragmentation)
@strikes = 0
end
end
def handle_heap_fragmentation_limit_exceeded(value)
@logger.warn(
log_labels.merge(
message: 'heap fragmentation limit exceeded',
memwd_cur_heap_frag: value
))
@heap_frag_violations_handled.increment
handler.on_high_heap_fragmentation(value)
end
def handler
# This allows us to keep the watchdog running but turn it into "friendly mode" where
# all that happens is we collect logs and Prometheus events for fragmentation violations.
return NullHandler.instance unless Feature.enabled?(:enforce_memory_watchdog, type: :ops)
@handler
end
def log_labels
{
pid: $$,
worker_id: worker_id,
memwd_handler_class: handler.class.name,
memwd_sleep_time_s: @sleep_time_seconds,
memwd_max_heap_frag: @max_heap_fragmentation,
memwd_max_strikes: @max_strikes,
memwd_cur_strikes: @strikes,
memwd_rss_bytes: process_rss_bytes
}
end
def worker_id
::Prometheus::PidProvider.worker_id
end
def process_rss_bytes
Gitlab::Metrics::System.memory_usage_rss
end
def init_prometheus_metrics(max_heap_fragmentation)
@heap_frag_limit = Gitlab::Metrics.gauge(
:gitlab_memwd_heap_frag_limit,
'The configured limit for how fragmented the Ruby heap is allowed to be'
)
@heap_frag_limit.set({}, max_heap_fragmentation)
default_labels = { pid: worker_id }
@heap_frag_violations = Gitlab::Metrics.counter(
:gitlab_memwd_heap_frag_violations_total,
'Total number of times heap fragmentation in a Ruby process exceeded its allowed maximum',
default_labels
)
@heap_frag_violations_handled = Gitlab::Metrics.counter(
:gitlab_memwd_heap_frag_violations_handled_total,
'Total number of times heap fragmentation violations in a Ruby process were handled',
default_labels
)
end
end
end
end