debian-mirror-gitlab/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb

115 lines
3.7 KiB
Ruby
Raw Normal View History

2021-09-04 01:27:46 +05:30
# frozen_string_literal: true
module Gitlab
module Database
module LoadBalancing
class SidekiqServerMiddleware
JobReplicaNotUpToDate = Class.new(StandardError)
2022-01-26 12:08:38 +05:30
MINIMUM_DELAY_INTERVAL_SECONDS = 0.8
2021-09-04 01:27:46 +05:30
def call(worker, job, _queue)
2021-09-30 23:02:18 +05:30
worker_class = worker.class
strategy = select_load_balancing_strategy(worker_class, job)
job['load_balancing_strategy'] = strategy.to_s
if use_primary?(strategy)
2021-12-11 22:18:48 +05:30
::Gitlab::Database::LoadBalancing::Session.current.use_primary!
2021-09-30 23:02:18 +05:30
elsif strategy == :retry
raise JobReplicaNotUpToDate, "Sidekiq job #{worker_class} JID-#{job['jid']} couldn't use the replica."\
" Replica was not up to date."
else
# this means we selected an up-to-date replica, but there is nothing to do in this case.
2021-09-04 01:27:46 +05:30
end
yield
ensure
clear
end
private
def clear
2021-12-11 22:18:48 +05:30
::Gitlab::Database::LoadBalancing.release_hosts
::Gitlab::Database::LoadBalancing::Session.clear_session
2021-09-04 01:27:46 +05:30
end
2021-09-30 23:02:18 +05:30
def use_primary?(strategy)
strategy.start_with?('primary')
end
2021-09-04 01:27:46 +05:30
2021-09-30 23:02:18 +05:30
def select_load_balancing_strategy(worker_class, job)
return :primary unless load_balancing_available?(worker_class)
2021-09-04 01:27:46 +05:30
2021-11-11 11:23:49 +05:30
wal_locations = get_wal_locations(job)
2022-01-26 12:08:38 +05:30
return :primary_no_wal if wal_locations.blank?
# Happy case: we can read from a replica.
return replica_strategy(worker_class, job) if databases_in_sync?(wal_locations)
sleep_if_needed(job)
2021-09-04 01:27:46 +05:30
2021-11-18 22:05:49 +05:30
if databases_in_sync?(wal_locations)
2022-01-26 12:08:38 +05:30
replica_strategy(worker_class, job)
2021-09-30 23:02:18 +05:30
elsif can_retry?(worker_class, job)
# Optimistic case: The worker allows retries and we have retries left.
:retry
2021-09-04 01:27:46 +05:30
else
2021-09-30 23:02:18 +05:30
# Sad case: we need to fall back to the primary.
:primary
2021-09-04 01:27:46 +05:30
end
end
2022-01-26 12:08:38 +05:30
def sleep_if_needed(job)
remaining_delay = MINIMUM_DELAY_INTERVAL_SECONDS - (Time.current.to_f - job['created_at'].to_f)
2021-11-11 11:23:49 +05:30
2022-01-26 12:08:38 +05:30
sleep remaining_delay if remaining_delay > 0 && remaining_delay < MINIMUM_DELAY_INTERVAL_SECONDS
end
2021-11-11 11:23:49 +05:30
2022-01-26 12:08:38 +05:30
def get_wal_locations(job)
job['dedup_wal_locations'] || job['wal_locations']
2021-11-11 11:23:49 +05:30
end
2021-09-30 23:02:18 +05:30
def load_balancing_available?(worker_class)
worker_class.include?(::ApplicationWorker) &&
worker_class.utilizes_load_balancing_capabilities? &&
worker_class.get_data_consistency_feature_flag_enabled?
end
def can_retry?(worker_class, job)
worker_class.get_data_consistency == :delayed && not_yet_retried?(job)
end
2022-01-26 12:08:38 +05:30
def replica_strategy(worker_class, job)
retried_before?(worker_class, job) ? :replica_retried : :replica
end
2021-09-30 23:02:18 +05:30
def retried_before?(worker_class, job)
worker_class.get_data_consistency == :delayed && !not_yet_retried?(job)
end
2021-09-04 01:27:46 +05:30
def not_yet_retried?(job)
# if `retry_count` is `nil` it indicates that this job was never retried
# the `0` indicates that this is a first retry
job['retry_count'].nil?
end
2021-11-18 22:05:49 +05:30
def databases_in_sync?(wal_locations)
2021-12-11 22:18:48 +05:30
::Gitlab::Database::LoadBalancing.each_load_balancer.all? do |lb|
2021-11-18 22:05:49 +05:30
if (location = wal_locations[lb.name])
lb.select_up_to_date_host(location)
else
# If there's no entry for a load balancer it means the Sidekiq
# job doesn't care for it. In this case we'll treat the load
# balancer as being in sync.
true
end
2021-11-11 11:23:49 +05:30
end
2021-09-04 01:27:46 +05:30
end
end
end
end
end