debian-mirror-gitlab/spec/lib/gitlab/process_supervisor_spec.rb
2022-05-07 20:08:51 +05:30

170 lines
5.9 KiB
Ruby

# frozen_string_literal: true
require_relative '../../../lib/gitlab/process_supervisor'
RSpec.describe Gitlab::ProcessSupervisor do
let(:health_check_interval_seconds) { 0.1 }
let(:check_terminate_interval_seconds) { 1 }
let(:forwarded_signals) { [] }
let(:process_ids) { [spawn_process, spawn_process] }
def spawn_process
Process.spawn('while true; do sleep 1; done').tap do |pid|
Process.detach(pid)
end
end
subject(:supervisor) do
described_class.new(
health_check_interval_seconds: health_check_interval_seconds,
check_terminate_interval_seconds: check_terminate_interval_seconds,
terminate_timeout_seconds: 1 + check_terminate_interval_seconds,
forwarded_signals: forwarded_signals
)
end
after do
process_ids.each do |pid|
Process.kill('KILL', pid)
rescue Errno::ESRCH
# Ignore if a process wasn't actually alive.
end
end
describe '#supervise' do
context 'while supervised processes are alive' do
it 'does not invoke callback' do
expect(Gitlab::ProcessManagement.all_alive?(process_ids)).to be(true)
pids_killed = []
supervisor.supervise(process_ids) do |dead_pids|
pids_killed = dead_pids
[]
end
# Wait several times the poll frequency of the supervisor.
sleep health_check_interval_seconds * 10
expect(pids_killed).to be_empty
expect(Gitlab::ProcessManagement.all_alive?(process_ids)).to be(true)
end
end
context 'when a supervised process dies' do
it 'triggers callback with the dead PIDs and adds new PIDs to supervised PIDs' do
expect(Gitlab::ProcessManagement.all_alive?(process_ids)).to be(true)
pids_killed = []
supervisor.supervise(process_ids) do |dead_pids|
pids_killed = dead_pids
[42] # Fake starting a new process in place of the terminated one.
end
# Terminate the supervised process.
Process.kill('TERM', process_ids.first)
await_condition(sleep_sec: health_check_interval_seconds) do
pids_killed == [process_ids.first]
end
expect(Gitlab::ProcessManagement.process_alive?(process_ids.first)).to be(false)
expect(Gitlab::ProcessManagement.process_alive?(process_ids.last)).to be(true)
expect(supervisor.supervised_pids).to match_array([process_ids.last, 42])
end
end
context 'signal handling' do
before do
allow(supervisor).to receive(:sleep)
allow(Gitlab::ProcessManagement).to receive(:trap_signals)
allow(Gitlab::ProcessManagement).to receive(:all_alive?).and_return(false)
allow(Gitlab::ProcessManagement).to receive(:signal_processes).with(process_ids, anything)
end
context 'termination signals' do
context 'when TERM results in timely shutdown of processes' do
it 'forwards them to observed processes without waiting for grace period to expire' do
allow(Gitlab::ProcessManagement).to receive(:any_alive?).and_return(false)
expect(Gitlab::ProcessManagement).to receive(:trap_signals).ordered.with(%i(INT TERM)).and_yield(:TERM)
expect(Gitlab::ProcessManagement).to receive(:signal_processes).ordered.with(process_ids, :TERM)
expect(supervisor).not_to receive(:sleep).with(check_terminate_interval_seconds)
supervisor.supervise(process_ids) { [] }
end
end
context 'when TERM does not result in timely shutdown of processes' do
it 'issues a KILL signal after the grace period expires' do
expect(Gitlab::ProcessManagement).to receive(:trap_signals).with(%i(INT TERM)).and_yield(:TERM)
expect(Gitlab::ProcessManagement).to receive(:signal_processes).ordered.with(process_ids, :TERM)
expect(supervisor).to receive(:sleep).ordered.with(check_terminate_interval_seconds).at_least(:once)
expect(Gitlab::ProcessManagement).to receive(:signal_processes).ordered.with(process_ids, '-KILL')
supervisor.supervise(process_ids) { [] }
end
end
end
context 'forwarded signals' do
let(:forwarded_signals) { %i(USR1) }
it 'forwards given signals to the observed processes' do
expect(Gitlab::ProcessManagement).to receive(:trap_signals).with(%i(USR1)).and_yield(:USR1)
expect(Gitlab::ProcessManagement).to receive(:signal_processes).ordered.with(process_ids, :USR1)
supervisor.supervise(process_ids) { [] }
end
end
end
end
describe '#shutdown' do
context 'when supervisor is supervising processes' do
before do
supervisor.supervise(process_ids)
end
context 'when supervisor is alive' do
it 'signals TERM then KILL to all supervised processes' do
expect(Gitlab::ProcessManagement).to receive(:signal_processes).ordered.with(process_ids, :TERM)
expect(Gitlab::ProcessManagement).to receive(:signal_processes).ordered.with(process_ids, '-KILL')
supervisor.shutdown
end
it 'stops the supervisor' do
expect { supervisor.shutdown }.to change { supervisor.alive }.from(true).to(false)
end
end
context 'when supervisor has already shut down' do
before do
supervisor.shutdown
end
it 'does nothing' do
expect(supervisor.alive).to be(false)
expect(Gitlab::ProcessManagement).not_to receive(:signal_processes)
supervisor.shutdown
end
end
end
context 'when supervisor never started' do
it 'does nothing' do
expect(supervisor.alive).to be(false)
expect(Gitlab::ProcessManagement).not_to receive(:signal_processes)
supervisor.shutdown
end
end
end
def await_condition(timeout_sec: 5, sleep_sec: 0.1)
Timeout.timeout(timeout_sec) do
sleep sleep_sec until yield
end
end
end