# frozen_string_literal: true
module Gitlab
module GithubImport
# HTTP client for interacting with the GitHub API.
#
# This class is basically a fancy wrapped around Octokit while adding some
# functionality to deal with rate limiting and parallel imports. Usage is
# mostly the same as Octokit, for example:
#
# client = GithubImport::Client.new('hunter2')
#
# client.labels.each do |label|
# puts label.name
# end
class Client
include ::Gitlab::Utils::StrongMemoize
attr_reader :octokit
SEARCH_MAX_REQUESTS_PER_MINUTE = 30
DEFAULT_PER_PAGE = 100
LOWER_PER_PAGE = 50
CLIENT_CONNECTION_ERROR = ::Faraday::ConnectionFailed # used/set in sawyer agent which octokit uses
# A single page of data and the corresponding page number.
Page = Struct.new(:objects, :number)
# The minimum number of requests we want to keep available.
#
# We don't use a value of 0 as multiple threads may be using the same
# token in parallel. This could result in all of them hitting the GitHub
# rate limit at once. The threshold is put in place to not hit the limit
# in most cases.
RATE_LIMIT_THRESHOLD = 50
SEARCH_RATE_LIMIT_THRESHOLD = 3
# token - The GitHub API token to use.
#
# host - The GitHub hostname. If nil, github.com will be used.
#
# per_page - The number of objects that should be displayed per page.
#
# parallel - When set to true hitting the rate limit will result in a
# dedicated error being raised. When set to `false` we will
# instead just `sleep()` until the rate limit is reset. Setting
# this value to `true` for parallel importing is crucial as
# otherwise hitting the rate limit will result in a thread
# being blocked in a `sleep()` call for up to an hour.
def initialize(token, host: nil, per_page: DEFAULT_PER_PAGE, parallel: true)
@host = host
@octokit = ::Octokit::Client.new(
access_token: token,
per_page: per_page,
api_endpoint: api_endpoint,
web_endpoint: web_endpoint
)
@octokit.connection_options[:ssl] = { verify: verify_ssl }
@parallel = parallel
end
def parallel?
@parallel
end
# Returns the details of a GitHub user.
#
# username - The username of the user.
def user(username)
with_rate_limit { octokit.user(username) }
end
def pull_request_reviews(repo_name, iid)
each_object(:pull_request_reviews, repo_name, iid)
end
# Returns the details of a GitHub repository.
#
# name - The path (in the form `owner/repository`) of the repository.
def repository(name)
with_rate_limit { octokit.repo(name) }
end
def pull_request(repo_name, iid)
with_rate_limit { octokit.pull_request(repo_name, iid) }
end
def labels(*args)
each_object(:labels, *args)
end
def milestones(*args)
each_object(:milestones, *args)
end
def releases(*args)
each_object(:releases, *args)
end
# Fetches data from the GitHub API and yields a Page object for every page
# of data, without loading all of them into memory.
#
# method - The Octokit method to use for getting the data.
# args - Arguments to pass to the Octokit method.
#
# rubocop: disable GitlabSecurity/PublicSend
def each_page(method, *args, &block)
return to_enum(__method__, method, *args) unless block_given?
page =
if args.last.is_a?(Hash) && args.last[:page]
args.last[:page]
else
1
end
collection = with_rate_limit { octokit.public_send(method, *args) }
next_url = octokit.last_response.rels[:next]
yield Page.new(collection, page)
while next_url
response = with_rate_limit { next_url.get }
next_url = response.rels[:next]
yield Page.new(response.data, page += 1)
end
end
# Iterates over all of the objects for the given method (e.g. `:labels`).
#
# method - The method to send to Octokit for querying data.
# args - Any arguments to pass to the Octokit method.
def each_object(method, *args, &block)
return to_enum(__method__, method, *args) unless block_given?
each_page(method, *args) do |page|
page.objects.each do |object|
yield object
end
end
end
# Yields the supplied block, responding to any rate limit errors.
#
# The exact strategy used for handling rate limiting errors depends on
# whether we are running in parallel mode or not. For more information see
# `#rate_or_wait_for_rate_limit`.
def with_rate_limit
return with_retry { yield } unless rate_limiting_enabled?
request_count_counter.increment
raise_or_wait_for_rate_limit unless requests_remaining?
begin
with_retry { yield }
rescue ::Octokit::TooManyRequests
raise_or_wait_for_rate_limit
# This retry will only happen when running in sequential mode as we'll
# raise an error in parallel mode.
retry
end
end
def search_repos_by_name(name, options = {})
with_retry { octokit.search_repositories(search_query(str: name, type: :name), options) }
end
def search_query(str:, type:, include_collaborations: true, include_orgs: true)
query = "#{str} in:#{type} is:public,private user:#{octokit.user.login}"
query = [query, collaborations_subquery].join(' ') if include_collaborations
query = [query, organizations_subquery].join(' ') if include_orgs
query
end
# Returns `true` if we're still allowed to perform API calls.
# Search API has rate limit of 30, use lowered threshold when search is used.
def requests_remaining?
if requests_limit == SEARCH_MAX_REQUESTS_PER_MINUTE
return remaining_requests > SEARCH_RATE_LIMIT_THRESHOLD
end
remaining_requests > RATE_LIMIT_THRESHOLD
end
def remaining_requests
octokit.rate_limit.remaining
end
def requests_limit
octokit.rate_limit.limit
end
def raise_or_wait_for_rate_limit
rate_limit_counter.increment
if parallel?
raise RateLimitError
else
sleep(rate_limit_resets_in)
end
end
def rate_limit_resets_in
# We add a few seconds to the rate limit so we don't _immediately_
# resume when the rate limit resets as this may result in us performing
# a request before GitHub has a chance to reset the limit.
octokit.rate_limit.resets_in + 5
end
def rate_limiting_enabled?
strong_memoize(:rate_limiting_enabled) do
api_endpoint.include?('.github.com')
end
end
def api_endpoint
@host || custom_api_endpoint || default_api_endpoint
end
def web_endpoint
@host || custom_api_endpoint || ::Octokit::Default.web_endpoint
end
def custom_api_endpoint
github_omniauth_provider.dig('args', 'client_options', 'site')
end
def default_api_endpoint
OmniAuth::Strategies::GitHub.default_options[:client_options][:site] || ::Octokit::Default.api_endpoint
end
def verify_ssl
github_omniauth_provider.fetch('verify_ssl', true)
end
def github_omniauth_provider
@github_omniauth_provider ||= Gitlab::Auth::OAuth::Provider.config_for('github').to_h
end
def rate_limit_counter
@rate_limit_counter ||= Gitlab::Metrics.counter(
:github_importer_rate_limit_hits,
'The number of times we hit the GitHub rate limit when importing projects'
)
end
def request_count_counter
@request_counter ||= Gitlab::Metrics.counter(
:github_importer_request_count,
'The number of GitHub API calls performed when importing projects'
)
end
private
def collaborations_subquery
each_object(:repos, nil, { affiliation: 'collaborator' })
.map { |repo| "repo:#{repo.full_name}" }
.join(' ')
end
def organizations_subquery
each_object(:organizations)
.map { |org| "org:#{org.login}" }
.join(' ')
end
def with_retry
Retriable.retriable(on: CLIENT_CONNECTION_ERROR, on_retry: on_retry) do
yield
end
end
def on_retry
proc do |exception, try, elapsed_time, next_interval|
Gitlab::Import::Logger.info(
message: "GitHub connection retry triggered",
'error.class': exception.class,
'error.message': exception.message,
try_count: try,
elapsed_time_s: elapsed_time,
wait_to_retry_s: next_interval
)
end
end
end
end
end