2018-12-13 13:39:08 +05:30
|
|
|
# frozen_string_literal: true
|
|
|
|
|
2021-03-11 19:13:27 +05:30
|
|
|
require_relative '../utils' # Gitlab::Utils
|
|
|
|
|
2018-12-13 13:39:08 +05:30
|
|
|
module Gitlab
|
|
|
|
module Cluster
|
2022-10-11 01:57:18 +05:30
|
|
|
# We take advantage of the fact that the application is pre-loaded in the primary
|
|
|
|
# process. If it's a pre-fork server like Puma, this will be the Puma master process.
|
|
|
|
# Otherwise it is the worker itself such as for Sidekiq.
|
|
|
|
PRIMARY_PID = $$
|
|
|
|
|
2018-12-13 13:39:08 +05:30
|
|
|
#
|
|
|
|
# LifecycleEvents lets Rails initializers register application startup hooks
|
|
|
|
# that are sensitive to forking. For example, to defer the creation of
|
|
|
|
# watchdog threads. This lets us abstract away the Unix process
|
2021-09-04 01:27:46 +05:30
|
|
|
# lifecycles of Sidekiq, Puma, Puma Cluster, etc.
|
2018-12-13 13:39:08 +05:30
|
|
|
#
|
2019-12-21 20:55:43 +05:30
|
|
|
# We have the following lifecycle events.
|
2018-12-13 13:39:08 +05:30
|
|
|
#
|
2019-12-26 22:10:19 +05:30
|
|
|
# - on_before_fork (on master process):
|
2019-12-21 20:55:43 +05:30
|
|
|
#
|
2021-09-04 01:27:46 +05:30
|
|
|
# Puma Cluster: This will be called exactly once,
|
2019-12-21 20:55:43 +05:30
|
|
|
# on startup, before the workers are forked. This is
|
|
|
|
# called in the PARENT/MASTER process.
|
|
|
|
#
|
2019-12-26 22:10:19 +05:30
|
|
|
# Sidekiq/Puma Single: This is not called.
|
2019-12-21 20:55:43 +05:30
|
|
|
#
|
2019-12-26 22:10:19 +05:30
|
|
|
# - on_master_start (on master process):
|
2019-12-21 20:55:43 +05:30
|
|
|
#
|
2021-09-04 01:27:46 +05:30
|
|
|
# Puma Cluster: This will be called exactly once,
|
2019-12-21 20:55:43 +05:30
|
|
|
# on startup, before the workers are forked. This is
|
|
|
|
# called in the PARENT/MASTER process.
|
|
|
|
#
|
2019-12-26 22:10:19 +05:30
|
|
|
# Sidekiq/Puma Single: This is called immediately.
|
2019-12-21 20:55:43 +05:30
|
|
|
#
|
2019-12-26 22:10:19 +05:30
|
|
|
# - on_before_blackout_period (on master process):
|
2019-12-21 20:55:43 +05:30
|
|
|
#
|
2021-09-04 01:27:46 +05:30
|
|
|
# Puma Cluster: This will be called before a blackout
|
2019-12-26 22:10:19 +05:30
|
|
|
# period when performing graceful shutdown of master.
|
|
|
|
# This is called on `master` process.
|
2019-12-21 20:55:43 +05:30
|
|
|
#
|
2019-12-26 22:10:19 +05:30
|
|
|
# Sidekiq/Puma Single: This is not called.
|
2019-12-21 20:55:43 +05:30
|
|
|
#
|
2019-12-26 22:10:19 +05:30
|
|
|
# - on_before_graceful_shutdown (on master process):
|
2019-12-21 20:55:43 +05:30
|
|
|
#
|
2021-09-04 01:27:46 +05:30
|
|
|
# Puma Cluster: This will be called before a graceful
|
2019-12-26 22:10:19 +05:30
|
|
|
# shutdown of workers starts happening, but after blackout period.
|
2019-12-21 20:55:43 +05:30
|
|
|
# This is called on `master` process.
|
|
|
|
#
|
|
|
|
# Sidekiq/Puma Single: This is not called.
|
|
|
|
#
|
2019-12-26 22:10:19 +05:30
|
|
|
# - on_before_master_restart (on master process):
|
2019-12-21 20:55:43 +05:30
|
|
|
#
|
|
|
|
# Puma Cluster: This will be called before a new master is spun up.
|
|
|
|
# This is called on `master` process.
|
|
|
|
#
|
|
|
|
# Sidekiq/Puma Single: This is not called.
|
2018-12-13 13:39:08 +05:30
|
|
|
#
|
2019-12-26 22:10:19 +05:30
|
|
|
# - on_worker_start (on worker process):
|
|
|
|
#
|
2021-09-04 01:27:46 +05:30
|
|
|
# Puma Cluster: This is called in the worker process
|
2019-12-26 22:10:19 +05:30
|
|
|
# exactly once before processing requests.
|
|
|
|
#
|
|
|
|
# Sidekiq/Puma Single: This is called immediately.
|
|
|
|
#
|
2018-12-13 13:39:08 +05:30
|
|
|
# Blocks will be executed in the order in which they are registered.
|
|
|
|
#
|
|
|
|
class LifecycleEvents
|
2021-03-11 19:13:27 +05:30
|
|
|
FatalError = Class.new(Exception) # rubocop:disable Lint/InheritException
|
|
|
|
|
|
|
|
USE_FATAL_LIFECYCLE_EVENTS = Gitlab::Utils.to_boolean(ENV.fetch('GITLAB_FATAL_LIFECYCLE_EVENTS', 'true'))
|
|
|
|
|
2018-12-13 13:39:08 +05:30
|
|
|
class << self
|
|
|
|
#
|
|
|
|
# Hook registration methods (called from initializers)
|
|
|
|
#
|
|
|
|
def on_worker_start(&block)
|
2022-04-04 11:22:00 +05:30
|
|
|
if in_clustered_puma?
|
2018-12-13 13:39:08 +05:30
|
|
|
# Defer block execution
|
|
|
|
(@worker_start_hooks ||= []) << block
|
|
|
|
else
|
|
|
|
yield
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
def on_before_fork(&block)
|
|
|
|
# Defer block execution
|
|
|
|
(@before_fork_hooks ||= []) << block
|
|
|
|
end
|
|
|
|
|
2019-12-21 20:55:43 +05:30
|
|
|
# Read the config/initializers/cluster_events_before_phased_restart.rb
|
2019-12-26 22:10:19 +05:30
|
|
|
def on_before_blackout_period(&block)
|
2019-12-21 20:55:43 +05:30
|
|
|
# Defer block execution
|
2019-12-26 22:10:19 +05:30
|
|
|
(@master_blackout_period ||= []) << block
|
|
|
|
end
|
|
|
|
|
|
|
|
# Read the config/initializers/cluster_events_before_phased_restart.rb
|
|
|
|
def on_before_graceful_shutdown(&block)
|
|
|
|
# Defer block execution
|
|
|
|
(@master_graceful_shutdown ||= []) << block
|
2019-12-21 20:55:43 +05:30
|
|
|
end
|
2018-12-13 13:39:08 +05:30
|
|
|
|
2019-12-21 20:55:43 +05:30
|
|
|
def on_before_master_restart(&block)
|
2018-12-13 13:39:08 +05:30
|
|
|
# Defer block execution
|
|
|
|
(@master_restart_hooks ||= []) << block
|
|
|
|
end
|
|
|
|
|
2019-09-04 21:01:54 +05:30
|
|
|
def on_master_start(&block)
|
2022-04-04 11:22:00 +05:30
|
|
|
if in_clustered_puma?
|
2019-09-04 21:01:54 +05:30
|
|
|
on_before_fork(&block)
|
|
|
|
else
|
|
|
|
on_worker_start(&block)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2018-12-13 13:39:08 +05:30
|
|
|
#
|
2021-09-04 01:27:46 +05:30
|
|
|
# Lifecycle integration methods (called from puma.rb, etc.)
|
2018-12-13 13:39:08 +05:30
|
|
|
#
|
|
|
|
def do_worker_start
|
2021-03-11 19:13:27 +05:30
|
|
|
call(:worker_start_hooks, @worker_start_hooks)
|
2018-12-13 13:39:08 +05:30
|
|
|
end
|
|
|
|
|
|
|
|
def do_before_fork
|
2021-03-11 19:13:27 +05:30
|
|
|
call(:before_fork_hooks, @before_fork_hooks)
|
2018-12-13 13:39:08 +05:30
|
|
|
end
|
|
|
|
|
2019-12-26 22:10:19 +05:30
|
|
|
def do_before_graceful_shutdown
|
2021-03-11 19:13:27 +05:30
|
|
|
call(:master_blackout_period, @master_blackout_period)
|
2019-12-26 22:10:19 +05:30
|
|
|
|
|
|
|
blackout_seconds = ::Settings.shutdown.blackout_seconds.to_i
|
|
|
|
sleep(blackout_seconds) if blackout_seconds > 0
|
|
|
|
|
2021-03-11 19:13:27 +05:30
|
|
|
call(:master_graceful_shutdown, @master_graceful_shutdown)
|
2018-12-13 13:39:08 +05:30
|
|
|
end
|
|
|
|
|
2019-12-21 20:55:43 +05:30
|
|
|
def do_before_master_restart
|
2021-03-11 19:13:27 +05:30
|
|
|
call(:master_restart_hooks, @master_restart_hooks)
|
2019-12-21 20:55:43 +05:30
|
|
|
end
|
|
|
|
|
|
|
|
# DEPRECATED
|
|
|
|
alias_method :do_master_restart, :do_before_master_restart
|
|
|
|
|
2018-12-13 13:39:08 +05:30
|
|
|
# Puma doesn't use singletons (which is good) but
|
|
|
|
# this means we need to pass through whether the
|
|
|
|
# puma server is running in single mode or cluster mode
|
|
|
|
def set_puma_options(options)
|
|
|
|
@puma_options = options
|
|
|
|
end
|
|
|
|
|
|
|
|
private
|
|
|
|
|
2021-03-11 19:13:27 +05:30
|
|
|
def call(name, hooks)
|
|
|
|
return unless hooks
|
|
|
|
|
|
|
|
hooks.each do |hook|
|
|
|
|
hook.call
|
2021-06-08 01:23:25 +05:30
|
|
|
rescue StandardError => e
|
2021-03-11 19:13:27 +05:30
|
|
|
Gitlab::ErrorTracking.track_exception(e, type: 'LifecycleEvents', hook: hook)
|
|
|
|
warn("ERROR: The hook #{name} failed with exception (#{e.class}) \"#{e.message}\".")
|
|
|
|
|
|
|
|
# we consider lifecycle hooks to be fatal errors
|
|
|
|
raise FatalError, e if USE_FATAL_LIFECYCLE_EVENTS
|
|
|
|
end
|
2019-12-26 22:10:19 +05:30
|
|
|
end
|
|
|
|
|
2018-12-13 13:39:08 +05:30
|
|
|
def in_clustered_puma?
|
2022-04-04 11:22:00 +05:30
|
|
|
Gitlab::Runtime.puma? && @puma_options && @puma_options[:workers] && @puma_options[:workers] > 0
|
2018-12-13 13:39:08 +05:30
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|