2019-02-15 15:39:39 +05:30
|
|
|
# frozen_string_literal: true
|
|
|
|
|
2017-09-10 17:25:29 +05:30
|
|
|
module Gitlab
|
|
|
|
module Metrics
|
|
|
|
class RequestsRackMiddleware
|
2021-01-29 00:20:46 +05:30
|
|
|
HTTP_METHODS = {
|
|
|
|
"delete" => %w(200 202 204 303 400 401 403 404 500 503),
|
|
|
|
"get" => %w(200 204 301 302 303 304 307 400 401 403 404 410 422 429 500 503),
|
|
|
|
"head" => %w(200 204 301 302 303 401 403 404 410 500),
|
|
|
|
"options" => %w(200 404),
|
|
|
|
"patch" => %w(200 202 204 400 403 404 409 416 500),
|
|
|
|
"post" => %w(200 201 202 204 301 302 303 304 400 401 403 404 406 409 410 412 422 429 500 503),
|
|
|
|
"put" => %w(200 202 204 400 401 403 404 405 406 409 410 422 500)
|
|
|
|
}.freeze
|
2019-12-21 20:55:43 +05:30
|
|
|
|
2021-10-27 15:23:28 +05:30
|
|
|
HEALTH_ENDPOINT = %r{^/-/(liveness|readiness|health|metrics)/?$}.freeze
|
2020-04-22 19:07:51 +05:30
|
|
|
|
2021-11-18 22:05:49 +05:30
|
|
|
FEATURE_CATEGORY_DEFAULT = ::Gitlab::FeatureCategories::FEATURE_CATEGORY_DEFAULT
|
|
|
|
ENDPOINT_MISSING = 'unknown'
|
2021-01-03 14:25:43 +05:30
|
|
|
|
2021-01-29 00:20:46 +05:30
|
|
|
# These were the top 5 categories at a point in time, chosen as a
|
|
|
|
# reasonable default. If we initialize every category we'll end up
|
|
|
|
# with an explosion in unused metric combinations, but we want the
|
|
|
|
# most common ones to be always present.
|
|
|
|
FEATURE_CATEGORIES_TO_INITIALIZE = ['authentication_and_authorization',
|
|
|
|
'code_review', 'continuous_integration',
|
|
|
|
'not_owned', 'source_code_management',
|
|
|
|
FEATURE_CATEGORY_DEFAULT].freeze
|
|
|
|
|
2017-09-10 17:25:29 +05:30
|
|
|
def initialize(app)
|
|
|
|
@app = app
|
|
|
|
end
|
|
|
|
|
2021-01-29 00:20:46 +05:30
|
|
|
def self.http_requests_total
|
|
|
|
::Gitlab::Metrics.counter(:http_requests_total, 'Request count')
|
2017-09-10 17:25:29 +05:30
|
|
|
end
|
|
|
|
|
|
|
|
def self.rack_uncaught_errors_count
|
2021-01-29 00:20:46 +05:30
|
|
|
::Gitlab::Metrics.counter(:rack_uncaught_errors_total, 'Request handling uncaught errors count')
|
2017-09-10 17:25:29 +05:30
|
|
|
end
|
|
|
|
|
|
|
|
def self.http_request_duration_seconds
|
2021-01-29 00:20:46 +05:30
|
|
|
::Gitlab::Metrics.histogram(:http_request_duration_seconds, 'Request handling execution time',
|
|
|
|
{}, [0.05, 0.1, 0.25, 0.5, 0.7, 1, 2.5, 5, 10, 25])
|
2017-09-10 17:25:29 +05:30
|
|
|
end
|
|
|
|
|
2020-04-22 19:07:51 +05:30
|
|
|
def self.http_health_requests_total
|
2021-01-29 00:20:46 +05:30
|
|
|
::Gitlab::Metrics.counter(:http_health_requests_total, 'Health endpoint request count')
|
2020-04-22 19:07:51 +05:30
|
|
|
end
|
|
|
|
|
2021-01-29 00:20:46 +05:30
|
|
|
def self.initialize_metrics
|
|
|
|
# This initialization is done to avoid gaps in scraped metrics after
|
|
|
|
# restarts. It makes sure all counters/histograms are available at
|
|
|
|
# process start.
|
|
|
|
#
|
|
|
|
# For example `rate(http_requests_total{status="500"}[1m])` would return
|
|
|
|
# no data until the first 500 error would occur.
|
|
|
|
HTTP_METHODS.each do |method, statuses|
|
2021-01-03 14:25:43 +05:30
|
|
|
http_request_duration_seconds.get({ method: method })
|
2021-01-29 00:20:46 +05:30
|
|
|
|
|
|
|
statuses.product(FEATURE_CATEGORIES_TO_INITIALIZE) do |status, feature_category|
|
|
|
|
http_requests_total.get({ method: method, status: status, feature_category: feature_category })
|
|
|
|
end
|
2019-12-21 20:55:43 +05:30
|
|
|
end
|
2022-04-04 11:22:00 +05:30
|
|
|
|
|
|
|
Gitlab::Metrics::RailsSlis.initialize_request_slis!
|
2019-12-21 20:55:43 +05:30
|
|
|
end
|
|
|
|
|
2017-09-10 17:25:29 +05:30
|
|
|
def call(env)
|
|
|
|
method = env['REQUEST_METHOD'].downcase
|
2021-01-29 00:20:46 +05:30
|
|
|
method = 'INVALID' unless HTTP_METHODS.key?(method)
|
2021-10-27 15:23:28 +05:30
|
|
|
started = ::Gitlab::Metrics::System.monotonic_time
|
2021-01-03 14:25:43 +05:30
|
|
|
health_endpoint = health_endpoint?(env['PATH_INFO'])
|
|
|
|
status = 'undefined'
|
2020-04-22 19:07:51 +05:30
|
|
|
|
2017-09-10 17:25:29 +05:30
|
|
|
begin
|
|
|
|
status, headers, body = @app.call(env)
|
2023-03-04 22:38:38 +05:30
|
|
|
return [status, headers, body] if health_endpoint
|
2017-09-10 17:25:29 +05:30
|
|
|
|
2023-03-04 22:38:38 +05:30
|
|
|
urgency = urgency_for_env(env)
|
|
|
|
if ::Gitlab::Metrics.record_duration_for_status?(status)
|
|
|
|
elapsed = ::Gitlab::Metrics::System.monotonic_time - started
|
2021-10-27 15:23:28 +05:30
|
|
|
self.class.http_request_duration_seconds.observe({ method: method }, elapsed)
|
2023-03-04 22:38:38 +05:30
|
|
|
record_apdex(urgency, elapsed)
|
2021-01-03 14:25:43 +05:30
|
|
|
end
|
2017-09-10 17:25:29 +05:30
|
|
|
|
2023-03-04 22:38:38 +05:30
|
|
|
record_error(urgency, status)
|
|
|
|
|
2017-09-10 17:25:29 +05:30
|
|
|
[status, headers, body]
|
2021-06-08 01:23:25 +05:30
|
|
|
rescue StandardError
|
2021-10-27 15:23:28 +05:30
|
|
|
self.class.rack_uncaught_errors_count.increment
|
2017-09-10 17:25:29 +05:30
|
|
|
raise
|
2021-01-03 14:25:43 +05:30
|
|
|
ensure
|
|
|
|
if health_endpoint
|
2021-10-27 15:23:28 +05:30
|
|
|
self.class.http_health_requests_total.increment(status: status.to_s, method: method)
|
2021-01-03 14:25:43 +05:30
|
|
|
else
|
2021-10-27 15:23:28 +05:30
|
|
|
self.class.http_requests_total.increment(
|
2021-01-29 00:20:46 +05:30
|
|
|
status: status.to_s,
|
|
|
|
method: method,
|
|
|
|
feature_category: feature_category.presence || FEATURE_CATEGORY_DEFAULT
|
|
|
|
)
|
2021-01-03 14:25:43 +05:30
|
|
|
end
|
2017-09-10 17:25:29 +05:30
|
|
|
end
|
|
|
|
end
|
2020-04-22 19:07:51 +05:30
|
|
|
|
|
|
|
def health_endpoint?(path)
|
|
|
|
return false if path.blank?
|
|
|
|
|
|
|
|
HEALTH_ENDPOINT.match?(CGI.unescape(path))
|
|
|
|
end
|
2021-09-04 01:27:46 +05:30
|
|
|
|
|
|
|
def feature_category
|
|
|
|
::Gitlab::ApplicationContext.current_context_attribute(:feature_category)
|
|
|
|
end
|
2021-11-18 22:05:49 +05:30
|
|
|
|
|
|
|
def endpoint_id
|
|
|
|
::Gitlab::ApplicationContext.current_context_attribute(:caller_id)
|
|
|
|
end
|
|
|
|
|
2023-03-04 22:38:38 +05:30
|
|
|
def record_apdex(urgency, elapsed)
|
2021-11-18 22:05:49 +05:30
|
|
|
Gitlab::Metrics::RailsSlis.request_apdex.increment(
|
2021-12-11 22:18:48 +05:30
|
|
|
labels: labels_from_context.merge(request_urgency: urgency.name),
|
|
|
|
success: elapsed < urgency.duration
|
2021-11-18 22:05:49 +05:30
|
|
|
)
|
|
|
|
end
|
|
|
|
|
2023-03-04 22:38:38 +05:30
|
|
|
def record_error(urgency, status)
|
|
|
|
return unless Feature.enabled?(:gitlab_metrics_error_rate_sli, type: :development)
|
|
|
|
|
|
|
|
Gitlab::Metrics::RailsSlis.request_error_rate.increment(
|
|
|
|
labels: labels_from_context.merge(request_urgency: urgency.name),
|
|
|
|
error: ::Gitlab::Metrics.server_error?(status)
|
|
|
|
)
|
|
|
|
end
|
|
|
|
|
2021-11-18 22:05:49 +05:30
|
|
|
def labels_from_context
|
|
|
|
{
|
|
|
|
feature_category: feature_category.presence || FEATURE_CATEGORY_DEFAULT,
|
|
|
|
endpoint_id: endpoint_id.presence || ENDPOINT_MISSING
|
|
|
|
}
|
|
|
|
end
|
|
|
|
|
2021-12-11 22:18:48 +05:30
|
|
|
def urgency_for_env(env)
|
|
|
|
endpoint_urgency =
|
2021-11-18 22:05:49 +05:30
|
|
|
if env['api.endpoint'].present?
|
|
|
|
env['api.endpoint'].options[:for].try(:urgency_for_app, env['api.endpoint'])
|
|
|
|
elsif env['action_controller.instance'].present? && env['action_controller.instance'].respond_to?(:urgency)
|
|
|
|
env['action_controller.instance'].urgency
|
|
|
|
end
|
|
|
|
|
2021-12-11 22:18:48 +05:30
|
|
|
endpoint_urgency || Gitlab::EndpointAttributes::DEFAULT_URGENCY
|
2021-11-18 22:05:49 +05:30
|
|
|
end
|
2017-09-10 17:25:29 +05:30
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|