debian-mirror-gitlab/lib/gitlab/github_import/object_counter.rb

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

96 lines
3.2 KiB
Ruby
Raw Normal View History

2021-09-30 23:02:18 +05:30
# frozen_string_literal: true
# Count objects fetched or imported from Github.
module Gitlab
module GithubImport
class ObjectCounter
OPERATIONS = %w[fetched imported].freeze
PROJECT_COUNTER_LIST_KEY = 'github-importer/object-counters-list/%{project}/%{operation}'
PROJECT_COUNTER_KEY = 'github-importer/object-counter/%{project}/%{operation}/%{object_type}'
GLOBAL_COUNTER_KEY = 'github_importer_%{operation}_%{object_type}'
GLOBAL_COUNTER_DESCRIPTION = 'The number of %{operation} Github %{object_type}'
CACHING = Gitlab::Cache::Import::Caching
class << self
2021-10-27 15:23:28 +05:30
# Increments the project and the global counters if the given value is >= 1
def increment(project, object_type, operation, value: 1)
integer = value.to_i
return if integer <= 0
2021-09-30 23:02:18 +05:30
validate_operation!(operation)
2021-10-27 15:23:28 +05:30
increment_project_counter(project, object_type, operation, integer)
increment_global_counter(object_type, operation, integer)
2022-06-21 17:19:12 +05:30
project.import_state&.expire_etag_cache
2021-09-30 23:02:18 +05:30
end
def summary(project)
OPERATIONS.each_with_object({}) do |operation, result|
result[operation] = {}
CACHING
.values_from_set(counter_list_key(project, operation))
.sort
.each do |counter|
object_type = counter.split('/').last
result[operation][object_type] = CACHING.read_integer(counter)
end
end
end
private
# Global counters are long lived, in Prometheus,
# and it's used to report the health of the Github Importer
# in the Grafana Dashboard
# https://dashboards.gitlab.net/d/2zgM_rImz/github-importer?orgId=1
2021-10-27 15:23:28 +05:30
def increment_global_counter(object_type, operation, value)
2021-09-30 23:02:18 +05:30
key = GLOBAL_COUNTER_KEY % {
operation: operation,
object_type: object_type
}
description = GLOBAL_COUNTER_DESCRIPTION % {
operation: operation,
object_type: object_type.to_s.humanize
}
2021-10-27 15:23:28 +05:30
Gitlab::Metrics.counter(key.to_sym, description).increment(by: value)
2021-09-30 23:02:18 +05:30
end
# Project counters are short lived, in Redis,
# and it's used to report how successful a project
# import was with the #summary method.
2021-10-27 15:23:28 +05:30
def increment_project_counter(project, object_type, operation, value)
counter_key = PROJECT_COUNTER_KEY % {
project: project.id,
operation: operation,
object_type: object_type
}
2021-09-30 23:02:18 +05:30
add_counter_to_list(project, operation, counter_key)
2022-04-04 11:22:00 +05:30
CACHING.increment_by(counter_key, value)
2021-09-30 23:02:18 +05:30
end
def add_counter_to_list(project, operation, key)
CACHING.set_add(counter_list_key(project, operation), key)
end
def counter_list_key(project, operation)
PROJECT_COUNTER_LIST_KEY % { project: project.id, operation: operation }
end
def validate_operation!(operation)
unless operation.to_s.presence_in(OPERATIONS)
2021-10-27 15:23:28 +05:30
raise ArgumentError, "operation must be #{OPERATIONS.join(' or ')}"
2021-09-30 23:02:18 +05:30
end
end
end
end
end
end