debian-mirror-gitlab/spec/lib/gitlab/database/with_lock_retries_spec.rb

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

278 lines
9.8 KiB
Ruby
Raw Normal View History

2020-03-13 15:44:24 +05:30
# frozen_string_literal: true
require 'spec_helper'
2020-07-28 23:09:34 +05:30
RSpec.describe Gitlab::Database::WithLockRetries do
2020-03-13 15:44:24 +05:30
let(:env) { {} }
let(:logger) { Gitlab::Database::WithLockRetries::NULL_LOGGER }
2022-04-04 11:22:00 +05:30
let(:subject) { described_class.new(connection: connection, env: env, logger: logger, allow_savepoints: allow_savepoints, timing_configuration: timing_configuration) }
2021-11-11 11:23:49 +05:30
let(:allow_savepoints) { true }
2021-11-18 22:05:49 +05:30
let(:connection) { ActiveRecord::Base.retrieve_connection }
2020-03-13 15:44:24 +05:30
let(:timing_configuration) do
[
[1.second, 1.second],
[1.second, 1.second],
[1.second, 1.second],
[1.second, 1.second],
[1.second, 1.second]
]
end
describe '#run' do
it 'requires block' do
expect { subject.run }.to raise_error(StandardError, 'no block given')
end
context 'when DISABLE_LOCK_RETRIES is set' do
let(:env) { { 'DISABLE_LOCK_RETRIES' => 'true' } }
it 'executes the passed block without retrying' do
object = double
expect(object).to receive(:method).once
subject.run { object.method }
end
end
context 'when lock retry is enabled' do
let(:lock_fiber) do
Fiber.new do
2021-10-27 15:23:28 +05:30
# Initiating a separate DB connection for the lock
conn = ActiveRecord::Base.connection_pool.checkout
2020-03-13 15:44:24 +05:30
conn.transaction do
conn.execute("LOCK TABLE #{Project.table_name} in exclusive mode")
Fiber.yield
end
2021-10-27 15:23:28 +05:30
# Releasing the connection we requested
ActiveRecord::Base.connection_pool.checkin(conn)
2020-03-13 15:44:24 +05:30
end
end
before do
lock_fiber.resume # start the transaction and lock the table
end
2021-03-11 19:13:27 +05:30
after do
lock_fiber.resume if lock_fiber.alive?
end
2020-03-13 15:44:24 +05:30
context 'lock_fiber' do
it 'acquires lock successfully' do
check_exclusive_lock_query = """
SELECT 1
FROM pg_locks l
JOIN pg_class t ON l.relation = t.oid
WHERE t.relkind = 'r' AND l.mode = 'ExclusiveLock' AND t.relname = '#{Project.table_name}'
"""
2021-11-11 11:23:49 +05:30
expect(connection.execute(check_exclusive_lock_query).to_a).to be_present
2020-03-13 15:44:24 +05:30
end
end
shared_examples 'retriable exclusive lock on `projects`' do
it 'succeeds executing the given block' do
lock_attempts = 0
lock_acquired = false
2021-06-08 01:23:25 +05:30
# the actual number of attempts to run_block_with_lock_timeout can never exceed the number of
2020-10-24 23:57:45 +05:30
# timings_configurations, so here we limit the retry_count if it exceeds that value
#
# also, there is no call to sleep after the final attempt, which is why it will always be one less
expected_runs_with_timeout = [retry_count, timing_configuration.size].min
expect(subject).to receive(:sleep).exactly(expected_runs_with_timeout - 1).times
2021-06-08 01:23:25 +05:30
expect(subject).to receive(:run_block_with_lock_timeout).exactly(expected_runs_with_timeout).times.and_wrap_original do |method|
2020-03-13 15:44:24 +05:30
lock_fiber.resume if lock_attempts == retry_count
method.call
end
subject.run do
lock_attempts += 1
2020-05-24 23:13:21 +05:30
if lock_attempts == retry_count # we reached the last retry iteration, if we kill the thread, the last try (no lock_timeout) will succeed
2020-03-13 15:44:24 +05:30
lock_fiber.resume
end
2021-11-11 11:23:49 +05:30
connection.transaction do
connection.execute("LOCK TABLE #{Project.table_name} in exclusive mode")
2020-03-13 15:44:24 +05:30
lock_acquired = true
end
end
expect(lock_attempts).to eq(retry_count)
expect(lock_acquired).to eq(true)
end
end
context 'after 3 iterations' do
2021-01-03 14:25:43 +05:30
it_behaves_like 'retriable exclusive lock on `projects`' do
let(:retry_count) { 4 }
end
context 'setting the idle transaction timeout' do
context 'when there is no outer transaction: disable_ddl_transaction! is set in the migration' do
it 'does not disable the idle transaction timeout' do
2021-11-11 11:23:49 +05:30
allow(connection).to receive(:transaction_open?).and_return(false)
2021-06-08 01:23:25 +05:30
allow(subject).to receive(:run_block_with_lock_timeout).once.and_raise(ActiveRecord::LockWaitTimeout)
allow(subject).to receive(:run_block_with_lock_timeout).once
2021-01-03 14:25:43 +05:30
expect(subject).not_to receive(:disable_idle_in_transaction_timeout)
subject.run {}
end
end
context 'when there is outer transaction: disable_ddl_transaction! is not set in the migration' do
it 'disables the idle transaction timeout so the code can sleep and retry' do
2021-11-11 11:23:49 +05:30
allow(connection).to receive(:transaction_open?).and_return(true)
2021-01-03 14:25:43 +05:30
n = 0
2021-06-08 01:23:25 +05:30
allow(subject).to receive(:run_block_with_lock_timeout).twice do
2021-01-03 14:25:43 +05:30
n += 1
raise(ActiveRecord::LockWaitTimeout) if n == 1
end
expect(subject).to receive(:disable_idle_in_transaction_timeout).once
subject.run {}
end
end
end
end
context 'after the retries are exhausted' do
let(:timing_configuration) do
[
[1.second, 1.second]
]
end
2020-03-13 15:44:24 +05:30
2021-01-03 14:25:43 +05:30
context 'when there is no outer transaction: disable_ddl_transaction! is set in the migration' do
it 'does not disable the lock_timeout' do
2021-11-11 11:23:49 +05:30
allow(connection).to receive(:transaction_open?).and_return(false)
2021-06-08 01:23:25 +05:30
allow(subject).to receive(:run_block_with_lock_timeout).once.and_raise(ActiveRecord::LockWaitTimeout)
2021-01-03 14:25:43 +05:30
expect(subject).not_to receive(:disable_lock_timeout)
subject.run {}
end
end
context 'when there is outer transaction: disable_ddl_transaction! is not set in the migration' do
it 'disables the lock_timeout' do
2021-11-11 11:23:49 +05:30
allow(connection).to receive(:transaction_open?).and_return(true)
2021-06-08 01:23:25 +05:30
allow(subject).to receive(:run_block_with_lock_timeout).once.and_raise(ActiveRecord::LockWaitTimeout)
2021-01-03 14:25:43 +05:30
expect(subject).to receive(:disable_lock_timeout)
subject.run {}
end
end
2020-03-13 15:44:24 +05:30
end
context 'after the retries, without setting lock_timeout' do
2020-05-24 23:13:21 +05:30
let(:retry_count) { timing_configuration.size + 1 }
2020-03-13 15:44:24 +05:30
2020-05-24 23:13:21 +05:30
it_behaves_like 'retriable exclusive lock on `projects`' do
before do
expect(subject).to receive(:run_block_without_lock_timeout).and_call_original
end
end
2020-03-13 15:44:24 +05:30
end
2020-10-24 23:57:45 +05:30
context 'after the retries, when requested to raise an error' do
let(:expected_attempts_with_timeout) { timing_configuration.size }
let(:retry_count) { timing_configuration.size + 1 }
it 'raises an error instead of waiting indefinitely for the lock' do
lock_attempts = 0
lock_acquired = false
expect(subject).to receive(:sleep).exactly(expected_attempts_with_timeout - 1).times
2021-06-08 01:23:25 +05:30
expect(subject).to receive(:run_block_with_lock_timeout).exactly(expected_attempts_with_timeout).times.and_call_original
2020-10-24 23:57:45 +05:30
expect do
subject.run(raise_on_exhaustion: true) do
lock_attempts += 1
2021-11-11 11:23:49 +05:30
connection.transaction do
connection.execute("LOCK TABLE #{Project.table_name} in exclusive mode")
2020-10-24 23:57:45 +05:30
lock_acquired = true
end
end
end.to raise_error(described_class::AttemptsExhaustedError)
expect(lock_attempts).to eq(retry_count - 1)
expect(lock_acquired).to eq(false)
end
end
2020-03-13 15:44:24 +05:30
context 'when statement timeout is reached' do
it 'raises QueryCanceled error' do
lock_acquired = false
2021-11-11 11:23:49 +05:30
connection.execute("SET LOCAL statement_timeout='100ms'")
2020-03-13 15:44:24 +05:30
expect do
subject.run do
2021-11-11 11:23:49 +05:30
connection.execute("SELECT 1 FROM pg_sleep(0.11)") # 110ms
2020-03-13 15:44:24 +05:30
lock_acquired = true
end
end.to raise_error(ActiveRecord::QueryCanceled)
expect(lock_acquired).to eq(false)
end
end
end
end
2020-05-24 23:13:21 +05:30
context 'restore local database variables' do
it do
2021-11-11 11:23:49 +05:30
expect { subject.run {} }.not_to change { connection.execute("SHOW lock_timeout").to_a }
2020-05-24 23:13:21 +05:30
end
it do
2021-11-11 11:23:49 +05:30
expect { subject.run {} }.not_to change { connection.execute("SHOW idle_in_transaction_session_timeout").to_a }
2020-05-24 23:13:21 +05:30
end
end
2020-03-13 15:44:24 +05:30
context 'casting durations correctly' do
let(:timing_configuration) { [[0.015.seconds, 0.025.seconds], [0.015.seconds, 0.025.seconds]] } # 15ms, 25ms
it 'executes `SET LOCAL lock_timeout` using the configured timeout value in milliseconds' do
2021-11-11 11:23:49 +05:30
expect(connection).to receive(:execute).with("RESET idle_in_transaction_session_timeout; RESET lock_timeout").and_call_original
expect(connection).to receive(:execute).with("SAVEPOINT active_record_1", "TRANSACTION").and_call_original
expect(connection).to receive(:execute).with("SET LOCAL lock_timeout TO '15ms'").and_call_original
expect(connection).to receive(:execute).with("RELEASE SAVEPOINT active_record_1", "TRANSACTION").and_call_original
2020-03-13 15:44:24 +05:30
subject.run { }
end
it 'calls `sleep` after the first iteration fails, using the configured sleep time' do
2021-06-08 01:23:25 +05:30
expect(subject).to receive(:run_block_with_lock_timeout).and_raise(ActiveRecord::LockWaitTimeout).twice
2020-03-13 15:44:24 +05:30
expect(subject).to receive(:sleep).with(0.025)
subject.run { }
end
end
2021-11-11 11:23:49 +05:30
context 'Stop using subtransactions - allow_savepoints: false' do
let(:allow_savepoints) { false }
it 'prevents running inside already open transaction' do
allow(connection).to receive(:transaction_open?).and_return(true)
expect { subject.run { } }.to raise_error(/should not run inside already open transaction/)
end
it 'does not raise the error if not inside open transaction' do
allow(connection).to receive(:transaction_open?).and_return(false)
expect { subject.run { } }.not_to raise_error
end
end
2020-03-13 15:44:24 +05:30
end