# frozen_string_literal: true module Gitlab module Metrics module Subscribers # Class for tracking the total query duration of a transaction. class ActiveRecord < ActiveSupport::Subscriber attach_to :active_record IGNORABLE_SQL = %w{BEGIN COMMIT}.freeze DB_COUNTERS = %i{count write_count cached_count}.freeze SQL_COMMANDS_WITH_COMMENTS_REGEX = %r{\A(/\*.*\*/\s)?((?!(.*[^\w'"](DELETE|UPDATE|INSERT INTO)[^\w'"])))(WITH.*)?(SELECT)((?!(FOR UPDATE|FOR SHARE)).)*$}i.freeze SQL_DURATION_BUCKET = [0.05, 0.1, 0.25].freeze TRANSACTION_DURATION_BUCKET = [0.1, 0.25, 1].freeze DB_LOAD_BALANCING_ROLES = %i{replica primary}.freeze DB_LOAD_BALANCING_COUNTERS = %i{count cached_count wal_count wal_cached_count}.freeze DB_LOAD_BALANCING_DURATIONS = %i{duration_s}.freeze SQL_WAL_LOCATION_REGEX = /(pg_current_wal_insert_lsn\(\)::text|pg_last_wal_replay_lsn\(\)::text)/.freeze # This event is published from ActiveRecordBaseTransactionMetrics and # used to record a database transaction duration when calling # ActiveRecord::Base.transaction {} block. def transaction(event) observe(:gitlab_database_transaction_seconds, event) do buckets TRANSACTION_DURATION_BUCKET end end def sql(event) # Mark this thread as requiring a database connection. This is used # by the Gitlab::Metrics::Samplers::ThreadsSampler to count threads # using a connection. Thread.current[:uses_db_connection] = true payload = event.payload return if ignored_query?(payload) db_config_name = db_config_name(event.payload) increment(:count, db_config_name: db_config_name) increment(:cached_count, db_config_name: db_config_name) if cached_query?(payload) increment(:write_count, db_config_name: db_config_name) unless select_sql_command?(payload) observe(:gitlab_sql_duration_seconds, event) do buckets SQL_DURATION_BUCKET end if ::Gitlab::Database::LoadBalancing.enable? db_role = ::Gitlab::Database::LoadBalancing.db_role_for_connection(payload[:connection]) return if db_role.blank? increment_db_role_counters(db_role, payload) observe_db_role_duration(db_role, event) end end def self.db_counter_payload return {} unless Gitlab::SafeRequestStore.active? {}.tap do |payload| db_counter_keys.each do |key| payload[key] = Gitlab::SafeRequestStore[key].to_i end if ::Gitlab::SafeRequestStore.active? && ::Gitlab::Database::LoadBalancing.enable? load_balancing_metric_counter_keys.each do |counter| payload[counter] = ::Gitlab::SafeRequestStore[counter].to_i end load_balancing_metric_duration_keys.each do |duration| payload[duration] = ::Gitlab::SafeRequestStore[duration].to_f.round(3) end end end end private def wal_command?(payload) payload[:sql].match(SQL_WAL_LOCATION_REGEX) end def increment_db_role_counters(db_role, payload) cached = cached_query?(payload) db_config_name = db_config_name(payload) increment(:count, db_role: db_role, db_config_name: db_config_name) increment(:cached_count, db_role: db_role, db_config_name: db_config_name) if cached if wal_command?(payload) increment(:wal_count, db_role: db_role, db_config_name: db_config_name) increment(:wal_cached_count, db_role: db_role, db_config_name: db_config_name) if cached end end def observe_db_role_duration(db_role, event) observe("gitlab_sql_#{db_role}_duration_seconds".to_sym, event) do buckets ::Gitlab::Metrics::Subscribers::ActiveRecord::SQL_DURATION_BUCKET end return unless ::Gitlab::SafeRequestStore.active? duration = event.duration / 1000.0 duration_key = compose_metric_key(:duration_s, db_role) ::Gitlab::SafeRequestStore[duration_key] = (::Gitlab::SafeRequestStore[duration_key].presence || 0) + duration # Per database metrics db_config_name = db_config_name(event.payload) duration_key = compose_metric_key(:duration_s, db_role, db_config_name) ::Gitlab::SafeRequestStore[duration_key] = (::Gitlab::SafeRequestStore[duration_key].presence || 0) + duration end def ignored_query?(payload) payload[:name] == 'SCHEMA' || IGNORABLE_SQL.include?(payload[:sql]) end def cached_query?(payload) payload.fetch(:cached, payload[:name] == 'CACHE') end def select_sql_command?(payload) payload[:sql].match(SQL_COMMANDS_WITH_COMMENTS_REGEX) end def increment(counter, db_config_name:, db_role: nil) log_key = compose_metric_key(counter, db_role) prometheus_key = if db_role :"gitlab_transaction_db_#{db_role}_#{counter}_total" else :"gitlab_transaction_db_#{counter}_total" end if ENV['GITLAB_MULTIPLE_DATABASE_METRICS'] current_transaction&.increment(prometheus_key, 1, { db_config_name: db_config_name }) else current_transaction&.increment(prometheus_key, 1) end Gitlab::SafeRequestStore[log_key] = Gitlab::SafeRequestStore[log_key].to_i + 1 # To avoid confusing log keys we only log the db_config_name metrics # when we are also logging the db_role. Otherwise it will be hard to # tell if the log key is referring to a db_role OR a db_config_name. if db_role.present? && db_config_name.present? log_key = compose_metric_key(counter, db_role, db_config_name) Gitlab::SafeRequestStore[log_key] = Gitlab::SafeRequestStore[log_key].to_i + 1 end end def observe(histogram, event, &block) db_config_name = db_config_name(event.payload) if ENV['GITLAB_MULTIPLE_DATABASE_METRICS'] current_transaction&.observe(histogram, event.duration / 1000.0, { db_config_name: db_config_name }, &block) else current_transaction&.observe(histogram, event.duration / 1000.0, &block) end end def current_transaction ::Gitlab::Metrics::WebTransaction.current || ::Gitlab::Metrics::BackgroundTransaction.current end def db_config_name(payload) ::Gitlab::Database.db_config_name(payload[:connection]) end def self.db_counter_keys DB_COUNTERS.map { |c| compose_metric_key(c) } end def self.load_balancing_metric_counter_keys load_balancing_metric_keys(DB_LOAD_BALANCING_COUNTERS) end def self.load_balancing_metric_duration_keys load_balancing_metric_keys(DB_LOAD_BALANCING_DURATIONS) end def self.load_balancing_metric_keys(metrics) [].tap do |counters| DB_LOAD_BALANCING_ROLES.each do |role| metrics.each do |metric| counters << compose_metric_key(metric, role) next unless ENV['GITLAB_MULTIPLE_DATABASE_METRICS'] ::Gitlab::Database.db_config_names.each do |config_name| counters << compose_metric_key(metric, role, config_name) end end end end end def compose_metric_key(metric, db_role = nil, db_config_name = nil) self.class.compose_metric_key(metric, db_role, db_config_name) end def self.compose_metric_key(metric, db_role = nil, db_config_name = nil) [:db, db_role, db_config_name, metric].compact.join("_").to_sym end end end end end