2021-09-04 01:27:46 +05:30
|
|
|
# frozen_string_literal: true
|
|
|
|
|
|
|
|
require 'spec_helper'
|
|
|
|
|
|
|
|
RSpec.describe Gitlab::Database::LoadBalancing::Sticking, :redis do
|
2021-11-18 22:05:49 +05:30
|
|
|
let(:sticking) do
|
2021-12-11 22:18:48 +05:30
|
|
|
described_class.new(ActiveRecord::Base.load_balancer)
|
2021-11-18 22:05:49 +05:30
|
|
|
end
|
|
|
|
|
2021-09-04 01:27:46 +05:30
|
|
|
after do
|
|
|
|
Gitlab::Database::LoadBalancing::Session.clear_session
|
|
|
|
end
|
|
|
|
|
2021-11-18 22:05:49 +05:30
|
|
|
describe '#stick_or_unstick_request' do
|
|
|
|
it 'sticks or unsticks a single object and updates the Rack environment' do
|
|
|
|
expect(sticking)
|
|
|
|
.to receive(:unstick_or_continue_sticking)
|
|
|
|
.with(:user, 42)
|
2021-09-04 01:27:46 +05:30
|
|
|
|
2021-11-18 22:05:49 +05:30
|
|
|
env = {}
|
|
|
|
|
|
|
|
sticking.stick_or_unstick_request(env, :user, 42)
|
|
|
|
|
|
|
|
expect(env[Gitlab::Database::LoadBalancing::RackMiddleware::STICK_OBJECT].to_a)
|
2021-12-11 22:18:48 +05:30
|
|
|
.to eq([[sticking, :user, 42]])
|
2021-09-04 01:27:46 +05:30
|
|
|
end
|
|
|
|
|
2021-11-18 22:05:49 +05:30
|
|
|
it 'sticks or unsticks multiple objects and updates the Rack environment' do
|
|
|
|
expect(sticking)
|
|
|
|
.to receive(:unstick_or_continue_sticking)
|
|
|
|
.with(:user, 42)
|
|
|
|
.ordered
|
2021-09-04 01:27:46 +05:30
|
|
|
|
2021-11-18 22:05:49 +05:30
|
|
|
expect(sticking)
|
|
|
|
.to receive(:unstick_or_continue_sticking)
|
|
|
|
.with(:runner, '123456789')
|
|
|
|
.ordered
|
2021-09-04 01:27:46 +05:30
|
|
|
|
2021-11-18 22:05:49 +05:30
|
|
|
env = {}
|
2021-09-04 01:27:46 +05:30
|
|
|
|
2021-11-18 22:05:49 +05:30
|
|
|
sticking.stick_or_unstick_request(env, :user, 42)
|
|
|
|
sticking.stick_or_unstick_request(env, :runner, '123456789')
|
2021-09-04 01:27:46 +05:30
|
|
|
|
2021-11-18 22:05:49 +05:30
|
|
|
expect(env[Gitlab::Database::LoadBalancing::RackMiddleware::STICK_OBJECT].to_a).to eq([
|
2021-12-11 22:18:48 +05:30
|
|
|
[sticking, :user, 42],
|
|
|
|
[sticking, :runner, '123456789']
|
2021-11-18 22:05:49 +05:30
|
|
|
])
|
|
|
|
end
|
|
|
|
end
|
2021-09-04 01:27:46 +05:30
|
|
|
|
2021-11-18 22:05:49 +05:30
|
|
|
describe '#stick_if_necessary' do
|
|
|
|
it 'does not stick if no write was performed' do
|
|
|
|
allow(Gitlab::Database::LoadBalancing::Session.current)
|
|
|
|
.to receive(:performed_write?)
|
|
|
|
.and_return(false)
|
2021-09-04 01:27:46 +05:30
|
|
|
|
2021-11-18 22:05:49 +05:30
|
|
|
expect(sticking).not_to receive(:stick)
|
|
|
|
|
|
|
|
sticking.stick_if_necessary(:user, 42)
|
|
|
|
end
|
|
|
|
|
|
|
|
it 'sticks to the primary if a write was performed' do
|
|
|
|
allow(Gitlab::Database::LoadBalancing::Session.current)
|
|
|
|
.to receive(:performed_write?)
|
|
|
|
.and_return(true)
|
|
|
|
|
|
|
|
expect(sticking)
|
|
|
|
.to receive(:stick)
|
|
|
|
.with(:user, 42)
|
|
|
|
|
|
|
|
sticking.stick_if_necessary(:user, 42)
|
2021-09-04 01:27:46 +05:30
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2021-11-18 22:05:49 +05:30
|
|
|
describe '#all_caught_up?' do
|
2021-12-11 22:18:48 +05:30
|
|
|
let(:lb) { ActiveRecord::Base.load_balancer }
|
2021-09-30 23:02:18 +05:30
|
|
|
let(:last_write_location) { 'foo' }
|
2021-09-04 01:27:46 +05:30
|
|
|
|
|
|
|
before do
|
2021-11-18 22:05:49 +05:30
|
|
|
allow(sticking)
|
|
|
|
.to receive(:last_write_location_for)
|
2021-09-04 01:27:46 +05:30
|
|
|
.with(:user, 42)
|
2021-09-30 23:02:18 +05:30
|
|
|
.and_return(last_write_location)
|
|
|
|
end
|
|
|
|
|
|
|
|
context 'when no write location could be found' do
|
|
|
|
let(:last_write_location) { nil }
|
2021-09-04 01:27:46 +05:30
|
|
|
|
2021-09-30 23:02:18 +05:30
|
|
|
it 'returns true' do
|
|
|
|
expect(lb).not_to receive(:select_up_to_date_host)
|
2021-09-04 01:27:46 +05:30
|
|
|
|
2021-11-18 22:05:49 +05:30
|
|
|
expect(sticking.all_caught_up?(:user, 42)).to eq(true)
|
2021-09-30 23:02:18 +05:30
|
|
|
end
|
2021-09-04 01:27:46 +05:30
|
|
|
end
|
|
|
|
|
2021-09-30 23:02:18 +05:30
|
|
|
context 'when all secondaries have caught up' do
|
|
|
|
before do
|
|
|
|
allow(lb).to receive(:select_up_to_date_host).with('foo').and_return(true)
|
|
|
|
end
|
2021-09-04 01:27:46 +05:30
|
|
|
|
2021-09-30 23:02:18 +05:30
|
|
|
it 'returns true, and unsticks' do
|
2021-11-18 22:05:49 +05:30
|
|
|
expect(sticking)
|
|
|
|
.to receive(:unstick)
|
|
|
|
.with(:user, 42)
|
2021-09-04 01:27:46 +05:30
|
|
|
|
2021-11-18 22:05:49 +05:30
|
|
|
expect(sticking.all_caught_up?(:user, 42)).to eq(true)
|
2021-09-30 23:02:18 +05:30
|
|
|
end
|
|
|
|
|
|
|
|
it 'notifies with the proper event payload' do
|
|
|
|
expect(ActiveSupport::Notifications)
|
|
|
|
.to receive(:instrument)
|
|
|
|
.with('caught_up_replica_pick.load_balancing', { result: true })
|
|
|
|
.and_call_original
|
2021-09-04 01:27:46 +05:30
|
|
|
|
2021-11-18 22:05:49 +05:30
|
|
|
sticking.all_caught_up?(:user, 42)
|
2021-09-30 23:02:18 +05:30
|
|
|
end
|
2021-09-04 01:27:46 +05:30
|
|
|
end
|
|
|
|
|
2021-09-30 23:02:18 +05:30
|
|
|
context 'when the secondaries have not yet caught up' do
|
|
|
|
before do
|
|
|
|
allow(lb).to receive(:select_up_to_date_host).with('foo').and_return(false)
|
|
|
|
end
|
|
|
|
|
|
|
|
it 'returns false' do
|
2021-11-18 22:05:49 +05:30
|
|
|
expect(sticking.all_caught_up?(:user, 42)).to eq(false)
|
2021-09-30 23:02:18 +05:30
|
|
|
end
|
2021-09-04 01:27:46 +05:30
|
|
|
|
2021-09-30 23:02:18 +05:30
|
|
|
it 'notifies with the proper event payload' do
|
|
|
|
expect(ActiveSupport::Notifications)
|
|
|
|
.to receive(:instrument)
|
|
|
|
.with('caught_up_replica_pick.load_balancing', { result: false })
|
|
|
|
.and_call_original
|
2021-09-04 01:27:46 +05:30
|
|
|
|
2021-11-18 22:05:49 +05:30
|
|
|
sticking.all_caught_up?(:user, 42)
|
2021-09-30 23:02:18 +05:30
|
|
|
end
|
2021-09-04 01:27:46 +05:30
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2021-11-18 22:05:49 +05:30
|
|
|
describe '#unstick_or_continue_sticking' do
|
2021-12-11 22:18:48 +05:30
|
|
|
let(:lb) { ActiveRecord::Base.load_balancer }
|
2021-09-04 01:27:46 +05:30
|
|
|
|
|
|
|
it 'simply returns if no write location could be found' do
|
2021-11-18 22:05:49 +05:30
|
|
|
allow(sticking)
|
|
|
|
.to receive(:last_write_location_for)
|
2021-09-04 01:27:46 +05:30
|
|
|
.with(:user, 42)
|
|
|
|
.and_return(nil)
|
|
|
|
|
2021-09-30 23:02:18 +05:30
|
|
|
expect(lb).not_to receive(:select_up_to_date_host)
|
2021-09-04 01:27:46 +05:30
|
|
|
|
2021-11-18 22:05:49 +05:30
|
|
|
sticking.unstick_or_continue_sticking(:user, 42)
|
2021-09-04 01:27:46 +05:30
|
|
|
end
|
|
|
|
|
|
|
|
it 'unsticks if all secondaries have caught up' do
|
2021-11-18 22:05:49 +05:30
|
|
|
allow(sticking)
|
|
|
|
.to receive(:last_write_location_for)
|
2021-09-04 01:27:46 +05:30
|
|
|
.with(:user, 42)
|
|
|
|
.and_return('foo')
|
|
|
|
|
2021-09-30 23:02:18 +05:30
|
|
|
allow(lb).to receive(:select_up_to_date_host).with('foo').and_return(true)
|
2021-09-04 01:27:46 +05:30
|
|
|
|
2021-11-18 22:05:49 +05:30
|
|
|
expect(sticking)
|
|
|
|
.to receive(:unstick)
|
|
|
|
.with(:user, 42)
|
2021-09-04 01:27:46 +05:30
|
|
|
|
2021-11-18 22:05:49 +05:30
|
|
|
sticking.unstick_or_continue_sticking(:user, 42)
|
2021-09-04 01:27:46 +05:30
|
|
|
end
|
|
|
|
|
|
|
|
it 'continues using the primary if the secondaries have not yet caught up' do
|
2021-11-18 22:05:49 +05:30
|
|
|
allow(sticking)
|
|
|
|
.to receive(:last_write_location_for)
|
2021-09-04 01:27:46 +05:30
|
|
|
.with(:user, 42)
|
|
|
|
.and_return('foo')
|
|
|
|
|
2021-09-30 23:02:18 +05:30
|
|
|
allow(lb).to receive(:select_up_to_date_host).with('foo').and_return(false)
|
2021-09-04 01:27:46 +05:30
|
|
|
|
|
|
|
expect(Gitlab::Database::LoadBalancing::Session.current)
|
|
|
|
.to receive(:use_primary!)
|
|
|
|
|
2021-11-18 22:05:49 +05:30
|
|
|
sticking.unstick_or_continue_sticking(:user, 42)
|
2021-09-04 01:27:46 +05:30
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
RSpec.shared_examples 'sticking' do
|
2021-11-18 22:05:49 +05:30
|
|
|
before do
|
2021-12-11 22:18:48 +05:30
|
|
|
allow(ActiveRecord::Base.load_balancer)
|
2021-11-18 22:05:49 +05:30
|
|
|
.to receive(:primary_write_location)
|
|
|
|
.and_return('foo')
|
2021-09-04 01:27:46 +05:30
|
|
|
end
|
|
|
|
|
2021-11-18 22:05:49 +05:30
|
|
|
it 'sticks an entity to the primary', :aggregate_failures do
|
2021-12-11 22:18:48 +05:30
|
|
|
allow(ActiveRecord::Base.load_balancer)
|
2021-11-18 22:05:49 +05:30
|
|
|
.to receive(:primary_only?)
|
|
|
|
.and_return(false)
|
2021-09-04 01:27:46 +05:30
|
|
|
|
2021-11-18 22:05:49 +05:30
|
|
|
ids.each do |id|
|
|
|
|
expect(sticking)
|
|
|
|
.to receive(:set_write_location_for)
|
|
|
|
.with(:user, id, 'foo')
|
2021-09-04 01:27:46 +05:30
|
|
|
end
|
|
|
|
|
2021-11-18 22:05:49 +05:30
|
|
|
expect(Gitlab::Database::LoadBalancing::Session.current)
|
|
|
|
.to receive(:use_primary!)
|
2021-09-04 01:27:46 +05:30
|
|
|
|
2021-11-18 22:05:49 +05:30
|
|
|
subject
|
|
|
|
end
|
2021-09-04 01:27:46 +05:30
|
|
|
|
2021-11-18 22:05:49 +05:30
|
|
|
it 'does not update the write location when no replicas are used' do
|
|
|
|
expect(sticking).not_to receive(:set_write_location_for)
|
|
|
|
|
|
|
|
subject
|
2021-09-04 01:27:46 +05:30
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2021-11-18 22:05:49 +05:30
|
|
|
describe '#stick' do
|
2021-09-04 01:27:46 +05:30
|
|
|
it_behaves_like 'sticking' do
|
|
|
|
let(:ids) { [42] }
|
2021-11-18 22:05:49 +05:30
|
|
|
subject { sticking.stick(:user, ids.first) }
|
2021-09-04 01:27:46 +05:30
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2021-11-18 22:05:49 +05:30
|
|
|
describe '#bulk_stick' do
|
2021-09-04 01:27:46 +05:30
|
|
|
it_behaves_like 'sticking' do
|
|
|
|
let(:ids) { [42, 43] }
|
2021-11-18 22:05:49 +05:30
|
|
|
subject { sticking.bulk_stick(:user, ids) }
|
2021-09-04 01:27:46 +05:30
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2021-11-18 22:05:49 +05:30
|
|
|
describe '#mark_primary_write_location' do
|
|
|
|
it 'updates the write location with the load balancer' do
|
2021-12-11 22:18:48 +05:30
|
|
|
allow(ActiveRecord::Base.load_balancer)
|
2021-11-18 22:05:49 +05:30
|
|
|
.to receive(:primary_write_location)
|
|
|
|
.and_return('foo')
|
2021-09-04 01:27:46 +05:30
|
|
|
|
2021-12-11 22:18:48 +05:30
|
|
|
allow(ActiveRecord::Base.load_balancer)
|
2021-11-18 22:05:49 +05:30
|
|
|
.to receive(:primary_only?)
|
|
|
|
.and_return(false)
|
2021-09-04 01:27:46 +05:30
|
|
|
|
2021-11-18 22:05:49 +05:30
|
|
|
expect(sticking)
|
|
|
|
.to receive(:set_write_location_for)
|
|
|
|
.with(:user, 42, 'foo')
|
2021-09-04 01:27:46 +05:30
|
|
|
|
2021-11-18 22:05:49 +05:30
|
|
|
sticking.mark_primary_write_location(:user, 42)
|
2021-09-04 01:27:46 +05:30
|
|
|
end
|
|
|
|
|
2021-11-18 22:05:49 +05:30
|
|
|
it 'does nothing when no replicas are used' do
|
|
|
|
expect(sticking).not_to receive(:set_write_location_for)
|
2021-09-04 01:27:46 +05:30
|
|
|
|
2021-11-18 22:05:49 +05:30
|
|
|
sticking.mark_primary_write_location(:user, 42)
|
|
|
|
end
|
|
|
|
end
|
2021-09-04 01:27:46 +05:30
|
|
|
|
2021-11-18 22:05:49 +05:30
|
|
|
describe '#unstick' do
|
|
|
|
it 'removes the sticking data from Redis' do
|
|
|
|
sticking.set_write_location_for(:user, 4, 'foo')
|
|
|
|
sticking.unstick(:user, 4)
|
2021-09-04 01:27:46 +05:30
|
|
|
|
2021-11-18 22:05:49 +05:30
|
|
|
expect(sticking.last_write_location_for(:user, 4)).to be_nil
|
2021-09-04 01:27:46 +05:30
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2021-11-18 22:05:49 +05:30
|
|
|
describe '#last_write_location_for' do
|
|
|
|
it 'returns the last WAL write location for a user' do
|
|
|
|
sticking.set_write_location_for(:user, 4, 'foo')
|
2021-09-04 01:27:46 +05:30
|
|
|
|
2021-11-18 22:05:49 +05:30
|
|
|
expect(sticking.last_write_location_for(:user, 4)).to eq('foo')
|
2021-09-04 01:27:46 +05:30
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2021-11-18 22:05:49 +05:30
|
|
|
describe '#redis_key_for' do
|
2021-09-04 01:27:46 +05:30
|
|
|
it 'returns a String' do
|
2021-11-18 22:05:49 +05:30
|
|
|
expect(sticking.redis_key_for(:user, 42))
|
|
|
|
.to eq('database-load-balancing/write-location/main/user/42')
|
2021-09-04 01:27:46 +05:30
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2021-11-18 22:05:49 +05:30
|
|
|
describe '#select_caught_up_replicas' do
|
2021-12-11 22:18:48 +05:30
|
|
|
let(:lb) { ActiveRecord::Base.load_balancer }
|
2021-09-04 01:27:46 +05:30
|
|
|
|
|
|
|
context 'with no write location' do
|
|
|
|
before do
|
2021-11-18 22:05:49 +05:30
|
|
|
allow(sticking)
|
|
|
|
.to receive(:last_write_location_for)
|
|
|
|
.with(:project, 42)
|
|
|
|
.and_return(nil)
|
2021-09-04 01:27:46 +05:30
|
|
|
end
|
|
|
|
|
|
|
|
it 'returns false and does not try to find caught up hosts' do
|
2021-10-27 15:23:28 +05:30
|
|
|
expect(lb).not_to receive(:select_up_to_date_host)
|
2021-11-18 22:05:49 +05:30
|
|
|
expect(sticking.select_caught_up_replicas(:project, 42)).to be false
|
2021-09-04 01:27:46 +05:30
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
context 'with write location' do
|
|
|
|
before do
|
2021-11-18 22:05:49 +05:30
|
|
|
allow(sticking)
|
|
|
|
.to receive(:last_write_location_for)
|
|
|
|
.with(:project, 42)
|
|
|
|
.and_return('foo')
|
2021-09-04 01:27:46 +05:30
|
|
|
end
|
|
|
|
|
|
|
|
it 'returns true, selects hosts, and unsticks if any secondary has caught up' do
|
2021-09-30 23:02:18 +05:30
|
|
|
expect(lb).to receive(:select_up_to_date_host).and_return(true)
|
2021-11-18 22:05:49 +05:30
|
|
|
expect(sticking)
|
|
|
|
.to receive(:unstick)
|
|
|
|
.with(:project, 42)
|
|
|
|
expect(sticking.select_caught_up_replicas(:project, 42)).to be true
|
2021-09-04 01:27:46 +05:30
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|