121 lines
4 KiB
Ruby
121 lines
4 KiB
Ruby
|
module Gitlab
|
||
|
|
module Git
|
||
|
|
module Storage
|
||
|
|
class Checker
|
||
|
|
include CircuitBreakerSettings
|
||
|
|
|
||
|
|
attr_reader :storage_path, :storage, :hostname, :logger
|
||
|
|
METRICS_MUTEX = Mutex.new
|
||
|
|
STORAGE_TIMING_BUCKETS = [0.1, 0.15, 0.25, 0.33, 0.5, 1, 1.5, 2.5, 5, 10, 15].freeze
|
||
|
|
|
||
|
|
def self.check_all(logger = Rails.logger)
|
||
|
|
threads = Gitlab.config.repositories.storages.keys.map do |storage_name|
|
||
|
|
Thread.new do
|
||
|
|
Thread.current[:result] = new(storage_name, logger).check_with_lease
|
||
|
|
end
|
||
|
|
end
|
||
|
|
|
||
|
|
threads.map do |thread|
|
||
|
|
thread.join
|
||
|
|
thread[:result]
|
||
|
|
end
|
||
|
|
end
|
||
|
|
|
||
|
|
def self.check_histogram
|
||
|
|
@check_histogram ||=
|
||
|
|
METRICS_MUTEX.synchronize do
|
||
|
|
@check_histogram || Gitlab::Metrics.histogram(:circuitbreaker_storage_check_duration_seconds,
|
||
|
|
'Storage check time in seconds',
|
||
|
|
{},
|
||
|
|
STORAGE_TIMING_BUCKETS
|
||
|
|
)
|
||
|
|
end
|
||
|
|
end
|
||
|
|
|
||
|
|
def initialize(storage, logger = Rails.logger)
|
||
|
|
@storage = storage
|
||
|
|
config = Gitlab.config.repositories.storages[@storage]
|
||
|
|
@storage_path = config['path']
|
||
|
|
@logger = logger
|
||
|
|
|
||
|
|
@hostname = Gitlab::Environment.hostname
|
||
|
|
end
|
||
|
|
|
||
|
|
def check_with_lease
|
||
|
|
lease_key = "storage_check:#{cache_key}"
|
||
|
|
lease = Gitlab::ExclusiveLease.new(lease_key, timeout: storage_timeout)
|
||
|
|
result = { storage: storage, success: nil }
|
||
|
|
|
||
|
|
if uuid = lease.try_obtain
|
||
|
|
result[:success] = check
|
||
|
|
|
||
|
|
Gitlab::ExclusiveLease.cancel(lease_key, uuid)
|
||
|
|
else
|
||
|
|
logger.warn("#{hostname}: #{storage}: Skipping check, previous check still running")
|
||
|
|
end
|
||
|
|
|
||
|
|
result
|
||
|
|
end
|
||
|
|
|
||
|
|
def check
|
||
|
|
if perform_access_check
|
||
|
|
track_storage_accessible
|
||
|
|
true
|
||
|
|
else
|
||
|
|
track_storage_inaccessible
|
||
|
|
logger.error("#{hostname}: #{storage}: Not accessible.")
|
||
|
|
false
|
||
|
|
end
|
||
|
|
end
|
||
|
|
|
||
|
|
private
|
||
|
|
|
||
|
|
def perform_access_check
|
||
|
|
start_time = Gitlab::Metrics::System.monotonic_time
|
||
|
|
|
||
|
|
Gitlab::Git::Storage::ForkedStorageCheck.storage_available?(storage_path, storage_timeout, access_retries)
|
||
|
|
ensure
|
||
|
|
execution_time = Gitlab::Metrics::System.monotonic_time - start_time
|
||
|
|
self.class.check_histogram.observe({ storage: storage }, execution_time)
|
||
|
|
end
|
||
|
|
|
||
|
|
def track_storage_inaccessible
|
||
|
|
first_failure = current_failure_info.first_failure || Time.now
|
||
|
|
last_failure = Time.now
|
||
|
|
|
||
|
|
Gitlab::Git::Storage.redis.with do |redis|
|
||
|
|
redis.pipelined do
|
||
|
|
redis.hset(cache_key, :first_failure, first_failure.to_i)
|
||
|
|
redis.hset(cache_key, :last_failure, last_failure.to_i)
|
||
|
|
redis.hincrby(cache_key, :failure_count, 1)
|
||
|
|
redis.expire(cache_key, failure_reset_time)
|
||
|
|
maintain_known_keys(redis)
|
||
|
|
end
|
||
|
|
end
|
||
|
|
end
|
||
|
|
|
||
|
|
def track_storage_accessible
|
||
|
|
Gitlab::Git::Storage.redis.with do |redis|
|
||
|
|
redis.pipelined do
|
||
|
|
redis.hset(cache_key, :first_failure, nil)
|
||
|
|
redis.hset(cache_key, :last_failure, nil)
|
||
|
|
redis.hset(cache_key, :failure_count, 0)
|
||
|
|
maintain_known_keys(redis)
|
||
|
|
end
|
||
|
|
end
|
||
|
|
end
|
||
|
|
|
||
|
|
def maintain_known_keys(redis)
|
||
|
|
expire_time = Time.now.to_i + failure_reset_time
|
||
|
|
redis.zadd(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, expire_time, cache_key)
|
||
|
|
redis.zremrangebyscore(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, '-inf', Time.now.to_i)
|
||
|
|
end
|
||
|
|
|
||
|
|
def current_failure_info
|
||
|
|
FailureInfo.load(cache_key)
|
||
|
|
end
|
||
|
|
end
|
||
|
|
end
|
||
|
|
end
|
||
|
|
end
|