106 lines
3.8 KiB
Ruby
106 lines
3.8 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
# Count objects fetched or imported from Github.
|
|
module Gitlab
|
|
module GithubImport
|
|
class ObjectCounter
|
|
OPERATIONS = %w[fetched imported].freeze
|
|
PROJECT_COUNTER_LIST_KEY = 'github-importer/object-counters-list/%{project}/%{operation}'
|
|
PROJECT_COUNTER_KEY = 'github-importer/object-counter/%{project}/%{operation}/%{object_type}'
|
|
EMPTY_SUMMARY = OPERATIONS.index_with { |operation| {} }
|
|
|
|
GLOBAL_COUNTER_KEY = 'github_importer_%{operation}_%{object_type}'
|
|
GLOBAL_COUNTER_DESCRIPTION = 'The number of %{operation} Github %{object_type}'
|
|
|
|
CACHING = Gitlab::Cache::Import::Caching
|
|
|
|
class << self
|
|
# Increments the project and the global counters if the given value is >= 1
|
|
def increment(project, object_type, operation, value: 1)
|
|
integer = value.to_i
|
|
|
|
return if integer <= 0
|
|
|
|
validate_operation!(operation)
|
|
|
|
increment_project_counter(project, object_type, operation, integer)
|
|
increment_global_counter(object_type, operation, integer)
|
|
|
|
project.import_state&.expire_etag_cache
|
|
end
|
|
|
|
def summary(project)
|
|
cached_summary = cashed_summary(project)
|
|
# Actual information about objects that have already been imported is stored
|
|
# in the Redis Cache until Redis key is expired.
|
|
# After import is completed we store this information in project's import_checksums
|
|
return cached_summary if cached_summary != EMPTY_SUMMARY || project.import_state.blank?
|
|
|
|
project.import_state.in_progress? ? cached_summary : project.import_checksums
|
|
end
|
|
|
|
private
|
|
|
|
def cashed_summary(project)
|
|
OPERATIONS.each_with_object({}) do |operation, result|
|
|
result[operation] = {}
|
|
|
|
CACHING
|
|
.values_from_set(counter_list_key(project, operation))
|
|
.sort
|
|
.each do |counter|
|
|
object_type = counter.split('/').last
|
|
result[operation][object_type] = CACHING.read_integer(counter)
|
|
end
|
|
end
|
|
end
|
|
|
|
# Global counters are long lived, in Prometheus,
|
|
# and it's used to report the health of the Github Importer
|
|
# in the Grafana Dashboard
|
|
# https://dashboards.gitlab.net/d/2zgM_rImz/github-importer?orgId=1
|
|
def increment_global_counter(object_type, operation, value)
|
|
key = GLOBAL_COUNTER_KEY % {
|
|
operation: operation,
|
|
object_type: object_type
|
|
}
|
|
description = GLOBAL_COUNTER_DESCRIPTION % {
|
|
operation: operation,
|
|
object_type: object_type.to_s.humanize
|
|
}
|
|
|
|
Gitlab::Metrics.counter(key.to_sym, description).increment(by: value)
|
|
end
|
|
|
|
# Project counters are short lived, in Redis,
|
|
# and it's used to report how successful a project
|
|
# import was with the #summary method.
|
|
def increment_project_counter(project, object_type, operation, value)
|
|
counter_key = PROJECT_COUNTER_KEY % {
|
|
project: project.id,
|
|
operation: operation,
|
|
object_type: object_type
|
|
}
|
|
|
|
add_counter_to_list(project, operation, counter_key)
|
|
|
|
CACHING.increment_by(counter_key, value)
|
|
end
|
|
|
|
def add_counter_to_list(project, operation, key)
|
|
CACHING.set_add(counter_list_key(project, operation), key)
|
|
end
|
|
|
|
def counter_list_key(project, operation)
|
|
PROJECT_COUNTER_LIST_KEY % { project: project.id, operation: operation }
|
|
end
|
|
|
|
def validate_operation!(operation)
|
|
unless operation.to_s.presence_in(OPERATIONS)
|
|
raise ArgumentError, "operation must be #{OPERATIONS.join(' or ')}"
|
|
end
|
|
end
|
|
end
|
|
end
|
|
end
|
|
end
|