debian-mirror-gitlab/lib/gitlab/gitaly_client.rb

509 lines
18 KiB
Ruby
Raw Normal View History

2018-12-13 13:39:08 +05:30
# frozen_string_literal: true
2017-09-10 17:25:29 +05:30
require 'base64'
2017-08-17 22:00:37 +05:30
require 'gitaly'
2018-03-17 18:26:18 +05:30
require 'grpc/health/v1/health_pb'
require 'grpc/health/v1/health_services_pb'
2017-08-17 22:00:37 +05:30
module Gitlab
module GitalyClient
2018-03-17 18:26:18 +05:30
class TooManyInvocationsError < StandardError
attr_reader :call_site, :invocation_count, :max_call_stack
def initialize(call_site, invocation_count, max_call_stack, most_invoked_stack)
@call_site = call_site
@invocation_count = invocation_count
@max_call_stack = max_call_stack
stacks = most_invoked_stack.join('\n') if most_invoked_stack
msg = "GitalyClient##{call_site} called #{invocation_count} times from single request. Potential n+1?"
2018-12-13 13:39:08 +05:30
msg = "#{msg}\nThe following call site called into Gitaly #{max_call_stack} times:\n#{stacks}\n" if stacks
2018-03-17 18:26:18 +05:30
super(msg)
end
end
2019-07-31 22:56:46 +05:30
PEM_REGEX = /\-+BEGIN CERTIFICATE\-+.+?\-+END CERTIFICATE\-+/m.freeze
2019-02-15 15:39:39 +05:30
SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION'
2019-07-07 11:18:12 +05:30
MAXIMUM_GITALY_CALLS = 30
2020-03-13 15:44:24 +05:30
CLIENT_NAME = (Gitlab::Runtime.sidekiq? ? 'gitlab-sidekiq' : 'gitlab-web').freeze
2019-09-30 21:07:59 +05:30
GITALY_METADATA_FILENAME = '.gitaly-metadata'
2017-08-17 22:00:37 +05:30
MUTEX = Mutex.new
2018-03-17 18:26:18 +05:30
2017-08-17 22:00:37 +05:30
def self.stub(name, storage)
MUTEX.synchronize do
@stubs ||= {}
@stubs[storage] ||= {}
@stubs[storage][name] ||= begin
2018-03-17 18:26:18 +05:30
klass = stub_class(name)
addr = stub_address(storage)
2019-02-15 15:39:39 +05:30
creds = stub_creds(storage)
2020-04-08 14:13:33 +05:30
klass.new(addr, creds, interceptors: interceptors, channel_args: channel_args)
2017-08-17 22:00:37 +05:30
end
end
end
2019-03-02 22:35:43 +05:30
def self.interceptors
2019-07-31 22:56:46 +05:30
return [] unless Labkit::Tracing.enabled?
2019-03-02 22:35:43 +05:30
2019-12-04 20:38:33 +05:30
[Labkit::Tracing::GRPC::ClientInterceptor.instance]
2019-03-02 22:35:43 +05:30
end
private_class_method :interceptors
2020-04-08 14:13:33 +05:30
def self.channel_args
# These values match the go Gitaly client
# https://gitlab.com/gitlab-org/gitaly/-/blob/bf9f52bc/client/dial.go#L78
{
'grpc.keepalive_time_ms': 20000,
'grpc.keepalive_permit_without_calls': 1
}
end
private_class_method :channel_args
2019-02-15 15:39:39 +05:30
def self.stub_cert_paths
cert_paths = Dir["#{OpenSSL::X509::DEFAULT_CERT_DIR}/*"]
cert_paths << OpenSSL::X509::DEFAULT_CERT_FILE if File.exist? OpenSSL::X509::DEFAULT_CERT_FILE
cert_paths
end
def self.stub_certs
return @certs if @certs
@certs = stub_cert_paths.flat_map do |cert_file|
File.read(cert_file).scan(PEM_REGEX).map do |cert|
2019-07-07 11:18:12 +05:30
OpenSSL::X509::Certificate.new(cert).to_pem
rescue OpenSSL::OpenSSLError => e
2020-01-01 13:55:28 +05:30
Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, cert_file: cert_file)
2019-07-07 11:18:12 +05:30
nil
2019-02-15 15:39:39 +05:30
end.compact
end.uniq.join("\n")
end
def self.stub_creds(storage)
if URI(address(storage)).scheme == 'tls'
GRPC::Core::ChannelCredentials.new stub_certs
else
:this_channel_is_insecure
end
end
2018-03-17 18:26:18 +05:30
def self.stub_class(name)
if name == :health_check
Grpc::Health::V1::Health::Stub
else
2019-12-21 20:55:43 +05:30
Gitaly.const_get(name.to_s.camelcase.to_sym, false).const_get(:Stub, false)
2018-03-17 18:26:18 +05:30
end
end
def self.stub_address(storage)
2019-02-15 15:39:39 +05:30
address(storage).sub(%r{^tcp://|^tls://}, '')
2018-03-17 18:26:18 +05:30
end
2017-08-17 22:00:37 +05:30
def self.clear_stubs!
MUTEX.synchronize do
@stubs = nil
end
end
2018-05-09 12:01:36 +05:30
def self.random_storage
Gitlab.config.repositories.storages.keys.sample
end
2017-08-17 22:00:37 +05:30
def self.address(storage)
params = Gitlab.config.repositories.storages[storage]
raise "storage not found: #{storage.inspect}" if params.nil?
address = params['gitaly_address']
unless address.present?
raise "storage #{storage.inspect} is missing a gitaly_address"
end
2021-09-30 23:02:18 +05:30
unless %w(tcp unix tls).include?(URI(address).scheme)
2019-02-15 15:39:39 +05:30
raise "Unsupported Gitaly address: #{address.inspect} does not use URL scheme 'tcp' or 'unix' or 'tls'"
2017-08-17 22:00:37 +05:30
end
address
end
2018-03-17 18:26:18 +05:30
def self.address_metadata(storage)
2020-05-24 23:13:21 +05:30
Base64.strict_encode64(Gitlab::Json.dump(storage => connection_data(storage)))
2019-03-02 22:35:43 +05:30
end
def self.connection_data(storage)
{ 'address' => address(storage), 'token' => token(storage) }
2018-03-17 18:26:18 +05:30
end
2017-09-10 17:25:29 +05:30
# All Gitaly RPC call sites should use GitalyClient.call. This method
# makes sure that per-request authentication headers are set.
2018-03-17 18:26:18 +05:30
#
# This method optionally takes a block which receives the keyword
# arguments hash 'kwargs' that will be passed to gRPC. This allows the
# caller to modify or augment the keyword arguments. The block must
# return a hash.
#
# For example:
#
# GitalyClient.call(storage, service, rpc, request) do |kwargs|
# kwargs.merge(deadline: Time.now + 10)
# end
#
2020-04-08 14:13:33 +05:30
# The optional remote_storage keyword argument is used to enable
# inter-gitaly calls. Say you have an RPC that needs to pull data from
# one repository to another. For example, to fetch a branch from a
# (non-deduplicated) fork into the fork parent. In that case you would
# send an RPC call to the Gitaly server hosting the fork parent, and in
# the request, you would tell that Gitaly server to pull Git data from
# the fork. How does that Gitaly server connect to the Gitaly server the
# forked repo lives on? This is the problem `remote_storage:` solves: it
# adds address and authentication information to the call, as gRPC
# metadata (under the `gitaly-servers` header). The request would say
# "pull from repo X on gitaly-2". In the Ruby code you pass
# `remote_storage: 'gitaly-2'`. And then the metadata would say
# "gitaly-2 is at network address tcp://10.0.1.2:8075".
#
2019-12-26 22:10:19 +05:30
def self.call(storage, service, rpc, request, remote_storage: nil, timeout: default_timeout, &block)
2020-07-28 23:09:34 +05:30
Gitlab::GitalyClient::Call.new(storage, service, rpc, request, remote_storage, timeout).call(&block)
2019-12-26 22:10:19 +05:30
end
def self.execute(storage, service, rpc, request, remote_storage:, timeout:)
2018-03-17 18:26:18 +05:30
enforce_gitaly_request_limits(:call)
2020-03-13 15:44:24 +05:30
Gitlab::RequestContext.instance.ensure_deadline_not_exceeded!
2018-03-17 18:26:18 +05:30
2019-12-21 20:55:43 +05:30
kwargs = request_kwargs(storage, timeout: timeout.to_f, remote_storage: remote_storage)
2018-03-17 18:26:18 +05:30
kwargs = yield(kwargs) if block_given?
stub(service, storage).__send__(rpc, request, kwargs) # rubocop:disable GitlabSecurity/PublicSend
2019-12-26 22:10:19 +05:30
end
2019-07-07 11:18:12 +05:30
def self.query_time
2020-06-23 00:09:42 +05:30
query_time = Gitlab::SafeRequestStore[:gitaly_query_time] || 0
2020-05-24 23:13:21 +05:30
query_time.round(Gitlab::InstrumentationHelper::DURATION_PRECISION)
2019-07-07 11:18:12 +05:30
end
2018-03-17 18:26:18 +05:30
2020-06-23 00:09:42 +05:30
def self.add_query_time(duration)
return unless Gitlab::SafeRequestStore.active?
Gitlab::SafeRequestStore[:gitaly_query_time] ||= 0
Gitlab::SafeRequestStore[:gitaly_query_time] += duration
2019-07-07 11:18:12 +05:30
end
2019-05-30 16:15:17 +05:30
2018-12-05 23:21:45 +05:30
# For some time related tasks we can't rely on `Time.now` since it will be
# affected by Timecop in some tests, and the clock of some gitaly-related
# components (grpc's c-core and gitaly server) use system time instead of
# timecop's time, so tests will fail.
# `Time.at(Process.clock_gettime(Process::CLOCK_REALTIME))` will circumvent
# timecop.
def self.real_time
Time.at(Process.clock_gettime(Process::CLOCK_REALTIME))
end
private_class_method :real_time
def self.authorization_token(storage)
token = token(storage).to_s
issued_at = real_time.to_i.to_s
2021-03-11 19:13:27 +05:30
hmac = OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new('SHA256'), token, issued_at)
2018-12-05 23:21:45 +05:30
"v2.#{hmac}.#{issued_at}"
end
private_class_method :authorization_token
2019-12-21 20:55:43 +05:30
def self.request_kwargs(storage, timeout:, remote_storage: nil)
2018-03-17 18:26:18 +05:30
metadata = {
2018-12-05 23:21:45 +05:30
'authorization' => "Bearer #{authorization_token(storage)}",
2018-03-17 18:26:18 +05:30
'client_name' => CLIENT_NAME
}
2021-04-29 21:17:54 +05:30
context_data = Gitlab::ApplicationContext.current
2021-03-08 18:12:59 +05:30
2018-03-17 18:26:18 +05:30
feature_stack = Thread.current[:gitaly_feature_stack]
feature = feature_stack && feature_stack[0]
metadata['call_site'] = feature.to_s if feature
metadata['gitaly-servers'] = address_metadata(remote_storage) if remote_storage
2019-07-31 22:56:46 +05:30
metadata['x-gitlab-correlation-id'] = Labkit::Correlation::CorrelationId.current_id if Labkit::Correlation::CorrelationId.current_id
2019-09-30 21:07:59 +05:30
metadata['gitaly-session-id'] = session_id
2021-03-08 18:12:59 +05:30
metadata['username'] = context_data['meta.user'] if context_data&.fetch('meta.user', nil)
metadata['remote_ip'] = context_data['meta.remote_ip'] if context_data&.fetch('meta.remote_ip', nil)
2019-09-04 21:01:54 +05:30
metadata.merge!(Feature::Gitaly.server_feature_flags)
2021-03-11 19:13:27 +05:30
metadata.merge!(route_to_primary)
2018-11-08 19:23:39 +05:30
2020-03-13 15:44:24 +05:30
deadline_info = request_deadline(timeout)
metadata.merge!(deadline_info.slice(:deadline_type))
2018-03-17 18:26:18 +05:30
2020-03-13 15:44:24 +05:30
{ metadata: metadata, deadline: deadline_info[:deadline] }
2017-08-17 22:00:37 +05:30
end
2021-03-11 19:13:27 +05:30
# Gitlab::Git::HookEnv will set the :gitlab_git_env variable in case we're
# running in the context of a Gitaly hook call, which may make use of
# quarantined object directories. We thus need to pass along the path of
# the quarantined object directory to Gitaly, otherwise it won't be able to
# find these quarantined objects. Given that the quarantine directory is
# generated with a random name, they'll have different names when multiple
# Gitaly nodes take part in a single transaction. As a result, we are
# forced to route all requests to the primary node which has injected the
# quarantine object directory to us.
def self.route_to_primary
return {} unless Gitlab::SafeRequestStore.active?
2021-04-17 20:07:23 +05:30
return {} if Gitlab::SafeRequestStore[:gitlab_git_env].blank?
2021-03-11 19:13:27 +05:30
{ 'gitaly-route-repository-accessor-policy' => 'primary-only' }
end
private_class_method :route_to_primary
2020-03-13 15:44:24 +05:30
def self.request_deadline(timeout)
# timeout being 0 means the request is allowed to run indefinitely.
# We can't allow that inside a request, but this won't count towards Gitaly
# error budgets
regular_deadline = real_time.to_i + timeout if timeout > 0
return { deadline: regular_deadline } if Sidekiq.server?
return { deadline: regular_deadline } unless Gitlab::RequestContext.instance.request_deadline
limited_deadline = [regular_deadline, Gitlab::RequestContext.instance.request_deadline].compact.min
limited = limited_deadline < regular_deadline
{ deadline: limited_deadline, deadline_type: limited ? "limited" : "regular" }
end
private_class_method :request_deadline
2019-07-31 22:56:46 +05:30
def self.session_id
Gitlab::SafeRequestStore[:gitaly_session_id] ||= SecureRandom.uuid
end
2018-11-08 19:23:39 +05:30
2017-09-10 17:25:29 +05:30
def self.token(storage)
params = Gitlab.config.repositories.storages[storage]
raise "storage not found: #{storage.inspect}" if params.nil?
params['gitaly_token'].presence || Gitlab.config.gitaly['token']
end
2018-03-17 18:26:18 +05:30
# Ensures that Gitaly is not being abuse through n+1 misuse etc
def self.enforce_gitaly_request_limits(call_site)
2019-10-12 21:52:04 +05:30
# Only count limits in request-response environments
2018-12-05 23:21:45 +05:30
return unless Gitlab::SafeRequestStore.active?
2018-03-17 18:26:18 +05:30
# This is this actual number of times this call was made. Used for information purposes only
actual_call_count = increment_call_count("gitaly_#{call_site}_actual")
2019-07-07 11:18:12 +05:30
return unless enforce_gitaly_request_limits?
2018-03-17 18:26:18 +05:30
# Check if this call is nested within a allow_n_plus_1_calls
# block and skip check if it is
return if get_call_count(:gitaly_call_count_exception_block_depth) > 0
# This is the count of calls outside of a `allow_n_plus_1_calls` block
# It is used for enforcement but not statistics
permitted_call_count = increment_call_count("gitaly_#{call_site}_permitted")
count_stack
return if permitted_call_count <= MAXIMUM_GITALY_CALLS
raise TooManyInvocationsError.new(call_site, actual_call_count, max_call_count, max_stacks)
end
2019-07-07 11:18:12 +05:30
def self.enforce_gitaly_request_limits?
# We typically don't want to enforce request limits in production
# However, we have some production-like test environments, i.e., ones
# where `Rails.env.production?` returns `true`. We do want to be able to
# check if the limit is being exceeded while testing in those environments
# In that case we can use a feature flag to indicate that we do want to
# enforce request limits.
2019-09-04 21:01:54 +05:30
return true if Feature::Gitaly.enabled?('enforce_requests_limits')
2019-07-07 11:18:12 +05:30
!(Rails.env.production? || ENV["GITALY_DISABLE_REQUEST_LIMITS"])
end
private_class_method :enforce_gitaly_request_limits?
2018-03-17 18:26:18 +05:30
def self.allow_n_plus_1_calls
2018-12-05 23:21:45 +05:30
return yield unless Gitlab::SafeRequestStore.active?
2018-03-17 18:26:18 +05:30
begin
increment_call_count(:gitaly_call_count_exception_block_depth)
yield
ensure
decrement_call_count(:gitaly_call_count_exception_block_depth)
end
end
2019-07-07 11:18:12 +05:30
# Normally a FindCommit RPC will cache the commit with its SHA
# instead of a ref name, since it's possible the branch is mutated
# afterwards. However, for read-only requests that never mutate the
# branch, this method allows caching of the ref name directly.
def self.allow_ref_name_caching
return yield unless Gitlab::SafeRequestStore.active?
return yield if ref_name_caching_allowed?
begin
Gitlab::SafeRequestStore[:allow_ref_name_caching] = true
yield
ensure
Gitlab::SafeRequestStore[:allow_ref_name_caching] = false
end
end
def self.ref_name_caching_allowed?
Gitlab::SafeRequestStore[:allow_ref_name_caching]
end
2018-03-17 18:26:18 +05:30
def self.get_call_count(key)
2018-12-05 23:21:45 +05:30
Gitlab::SafeRequestStore[key] || 0
2018-03-17 18:26:18 +05:30
end
private_class_method :get_call_count
def self.increment_call_count(key)
2018-12-05 23:21:45 +05:30
Gitlab::SafeRequestStore[key] ||= 0
Gitlab::SafeRequestStore[key] += 1
2018-03-17 18:26:18 +05:30
end
private_class_method :increment_call_count
def self.decrement_call_count(key)
2018-12-05 23:21:45 +05:30
Gitlab::SafeRequestStore[key] -= 1
2018-03-17 18:26:18 +05:30
end
private_class_method :decrement_call_count
2019-02-15 15:39:39 +05:30
# Returns the of the number of Gitaly calls made for this request
2018-03-17 18:26:18 +05:30
def self.get_request_count
2019-02-15 15:39:39 +05:30
get_call_count("gitaly_call_actual")
2018-03-17 18:26:18 +05:30
end
def self.reset_counts
2018-12-05 23:21:45 +05:30
return unless Gitlab::SafeRequestStore.active?
2018-03-17 18:26:18 +05:30
2019-02-15 15:39:39 +05:30
Gitlab::SafeRequestStore["gitaly_call_actual"] = 0
Gitlab::SafeRequestStore["gitaly_call_permitted"] = 0
2017-08-17 22:00:37 +05:30
end
2019-07-31 22:56:46 +05:30
def self.add_call_details(details)
2019-02-15 15:39:39 +05:30
Gitlab::SafeRequestStore['gitaly_call_details'] ||= []
Gitlab::SafeRequestStore['gitaly_call_details'] << details
2018-05-09 12:01:36 +05:30
end
def self.list_call_details
2019-12-04 20:38:33 +05:30
return [] unless Gitlab::PerformanceBar.enabled_for_request?
2018-05-09 12:01:36 +05:30
2019-02-15 15:39:39 +05:30
Gitlab::SafeRequestStore['gitaly_call_details'] || []
2018-05-09 12:01:36 +05:30
end
2017-08-17 22:00:37 +05:30
def self.expected_server_version
path = Rails.root.join(SERVER_VERSION_FILE)
path.read.chomp
end
2017-09-10 17:25:29 +05:30
2018-11-18 11:00:15 +05:30
def self.timestamp(time)
Google::Protobuf::Timestamp.new(seconds: time.to_i)
2018-03-17 18:26:18 +05:30
end
# The default timeout on all Gitaly calls
def self.default_timeout
timeout(:gitaly_timeout_default)
end
def self.fast_timeout
timeout(:gitaly_timeout_fast)
end
def self.medium_timeout
timeout(:gitaly_timeout_medium)
end
2019-12-21 20:55:43 +05:30
def self.long_timeout
2020-03-13 15:44:24 +05:30
if Gitlab::Runtime.web_server?
2019-12-21 20:55:43 +05:30
default_timeout
else
6.hours
end
end
2019-09-30 21:07:59 +05:30
def self.storage_metadata_file_path(storage)
Gitlab::GitalyClient::StorageSettings.allow_disk_access do
File.join(
Gitlab.config.repositories.storages[storage].legacy_disk_path, GITALY_METADATA_FILENAME
)
end
end
def self.can_use_disk?(storage)
cached_value = MUTEX.synchronize do
@can_use_disk ||= {}
@can_use_disk[storage]
end
return cached_value unless cached_value.nil?
gitaly_filesystem_id = filesystem_id(storage)
direct_filesystem_id = filesystem_id_from_disk(storage)
MUTEX.synchronize do
@can_use_disk[storage] = gitaly_filesystem_id.present? &&
gitaly_filesystem_id == direct_filesystem_id
end
end
def self.filesystem_id(storage)
2020-03-13 15:44:24 +05:30
Gitlab::GitalyClient::ServerService.new(storage).storage_info&.filesystem_id
2019-09-30 21:07:59 +05:30
end
def self.filesystem_id_from_disk(storage)
metadata_file = File.read(storage_metadata_file_path(storage))
2020-05-24 23:13:21 +05:30
metadata_hash = Gitlab::Json.parse(metadata_file)
2019-09-30 21:07:59 +05:30
metadata_hash['gitaly_filesystem_id']
rescue Errno::ENOENT, Errno::EACCES, JSON::ParserError
nil
end
2020-03-13 15:44:24 +05:30
def self.filesystem_disk_available(storage)
Gitlab::GitalyClient::ServerService.new(storage).storage_disk_statistics&.available
end
def self.filesystem_disk_used(storage)
Gitlab::GitalyClient::ServerService.new(storage).storage_disk_statistics&.used
end
2018-03-17 18:26:18 +05:30
def self.timeout(timeout_name)
Gitlab::CurrentSettings.current_application_settings[timeout_name]
end
private_class_method :timeout
# Count a stack. Used for n+1 detection
def self.count_stack
2018-12-05 23:21:45 +05:30
return unless Gitlab::SafeRequestStore.active?
2018-03-17 18:26:18 +05:30
2020-03-13 15:44:24 +05:30
stack_string = Gitlab::BacktraceCleaner.clean_backtrace(caller).drop(1).join("\n")
2018-03-17 18:26:18 +05:30
2020-11-24 15:15:51 +05:30
Gitlab::SafeRequestStore[:stack_counter] ||= {}
2018-03-17 18:26:18 +05:30
2018-12-05 23:21:45 +05:30
count = Gitlab::SafeRequestStore[:stack_counter][stack_string] || 0
Gitlab::SafeRequestStore[:stack_counter][stack_string] = count + 1
2018-03-17 18:26:18 +05:30
end
private_class_method :count_stack
# Returns a count for the stack which called Gitaly the most times. Used for n+1 detection
def self.max_call_count
2018-12-05 23:21:45 +05:30
return 0 unless Gitlab::SafeRequestStore.active?
2018-03-17 18:26:18 +05:30
2018-12-05 23:21:45 +05:30
stack_counter = Gitlab::SafeRequestStore[:stack_counter]
2018-03-17 18:26:18 +05:30
return 0 unless stack_counter
stack_counter.values.max
end
private_class_method :max_call_count
# Returns the stacks that calls Gitaly the most times. Used for n+1 detection
def self.max_stacks
2019-07-07 11:18:12 +05:30
return unless Gitlab::SafeRequestStore.active?
2018-03-17 18:26:18 +05:30
2018-12-05 23:21:45 +05:30
stack_counter = Gitlab::SafeRequestStore[:stack_counter]
2019-07-07 11:18:12 +05:30
return unless stack_counter
2018-03-17 18:26:18 +05:30
max = max_call_count
2020-10-24 23:57:45 +05:30
return if max == 0
2018-03-17 18:26:18 +05:30
stack_counter.select { |_, v| v == max }.keys
2017-09-10 17:25:29 +05:30
end
2018-03-17 18:26:18 +05:30
private_class_method :max_stacks
2017-08-17 22:00:37 +05:30
end
end