debian-mirror-gitlab/app/models/remote_mirror.rb

343 lines
8.1 KiB
Ruby
Raw Normal View History

2018-11-18 11:00:15 +05:30
# frozen_string_literal: true
2019-07-07 11:18:12 +05:30
class RemoteMirror < ApplicationRecord
2018-10-15 14:42:47 +05:30
include AfterCommitQueue
2019-02-15 15:39:39 +05:30
include MirrorAuthentication
2020-01-01 13:55:28 +05:30
include SafeUrl
2018-10-15 14:42:47 +05:30
2019-10-12 21:52:04 +05:30
MAX_FIRST_RUNTIME = 3.hours
MAX_INCREMENTAL_RUNTIME = 1.hour
2018-10-15 14:42:47 +05:30
PROTECTED_BACKOFF_DELAY = 1.minute
UNPROTECTED_BACKOFF_DELAY = 5.minutes
attr_encrypted :credentials,
2018-11-08 19:23:39 +05:30
key: Settings.attr_encrypted_db_key_base,
2018-10-15 14:42:47 +05:30
marshal: true,
encode: true,
mode: :per_attribute_iv_and_salt,
insecure_mode: true,
algorithm: 'aes-256-cbc'
belongs_to :project, inverse_of: :remote_mirrors
2019-07-31 22:56:46 +05:30
validates :url, presence: true, public_url: { schemes: %w(ssh git http https), allow_blank: true, enforce_user: true }
2018-10-15 14:42:47 +05:30
before_save :set_new_remote_name, if: :mirror_url_changed?
after_save :set_override_remote_mirror_available, unless: -> { Gitlab::CurrentSettings.current_application_settings.mirror_available }
2019-07-31 22:56:46 +05:30
after_save :refresh_remote, if: :saved_change_to_mirror_url?
after_update :reset_fields, if: :saved_change_to_mirror_url?
2018-10-15 14:42:47 +05:30
after_commit :remove_remote, on: :destroy
2019-02-15 15:39:39 +05:30
before_validation :store_credentials
2018-10-15 14:42:47 +05:30
scope :enabled, -> { where(enabled: true) }
scope :started, -> { with_update_status(:started) }
2019-10-12 21:52:04 +05:30
scope :stuck, -> do
started
.where('(last_update_started_at < ? AND last_update_at IS NOT NULL)',
MAX_INCREMENTAL_RUNTIME.ago)
.or(where('(last_update_started_at < ? AND last_update_at IS NULL)',
MAX_FIRST_RUNTIME.ago))
end
2018-10-15 14:42:47 +05:30
state_machine :update_status, initial: :none do
event :update_start do
2019-10-12 21:52:04 +05:30
transition any => :started
2018-10-15 14:42:47 +05:30
end
event :update_finish do
transition started: :finished
end
event :update_fail do
transition started: :failed
end
2019-10-12 21:52:04 +05:30
event :update_retry do
transition started: :to_retry
end
2018-10-15 14:42:47 +05:30
state :started
state :finished
state :failed
2019-10-12 21:52:04 +05:30
state :to_retry
2018-10-15 14:42:47 +05:30
after_transition any => :started do |remote_mirror, _|
Gitlab::Metrics.add_event(:remote_mirrors_running)
2020-06-23 00:09:42 +05:30
remote_mirror.update(last_update_started_at: Time.current)
2018-10-15 14:42:47 +05:30
end
after_transition started: :finished do |remote_mirror, _|
Gitlab::Metrics.add_event(:remote_mirrors_finished)
2020-06-23 00:09:42 +05:30
timestamp = Time.current
2018-11-18 11:00:15 +05:30
remote_mirror.update!(
2019-02-15 15:39:39 +05:30
last_update_at: timestamp,
last_successful_update_at: timestamp,
last_error: nil,
error_notification_sent: false
2018-10-15 14:42:47 +05:30
)
end
2019-02-15 15:39:39 +05:30
after_transition started: :failed do |remote_mirror|
2021-04-29 21:17:54 +05:30
remote_mirror.send_failure_notifications
2018-10-15 14:42:47 +05:30
end
end
def remote_name
super || fallback_remote_name
end
def update_failed?
update_status == 'failed'
end
def update_in_progress?
update_status == 'started'
end
2020-05-24 23:13:21 +05:30
def update_repository
Gitlab::Git::RemoteMirror.new(
project.repository.raw,
remote_name,
**options_for_update
).update
end
def options_for_update
options = {
keep_divergent_refs: keep_divergent_refs?
}
if only_protected_branches?
options[:only_branches_matching] = project.protected_branches.pluck(:name)
end
2019-02-15 15:39:39 +05:30
if ssh_mirror_url?
if ssh_key_auth? && ssh_private_key.present?
options[:ssh_key] = ssh_private_key
end
if ssh_known_hosts.present?
options[:known_hosts] = ssh_known_hosts
end
end
2020-05-24 23:13:21 +05:30
options
2018-10-15 14:42:47 +05:30
end
def sync?
enabled?
end
def sync
return unless sync?
if recently_scheduled?
2020-06-23 00:09:42 +05:30
RepositoryUpdateRemoteMirrorWorker.perform_in(backoff_delay, self.id, Time.current)
2018-10-15 14:42:47 +05:30
else
2020-06-23 00:09:42 +05:30
RepositoryUpdateRemoteMirrorWorker.perform_async(self.id, Time.current)
2018-10-15 14:42:47 +05:30
end
end
def enabled
return false unless project && super
return false unless project.remote_mirror_available?
return false unless project.repository_exists?
return false if project.pending_delete?
true
end
alias_method :enabled?, :enabled
2019-07-31 22:56:46 +05:30
def disabled?
!enabled?
end
2018-10-15 14:42:47 +05:30
def updated_since?(timestamp)
2019-10-12 21:52:04 +05:30
return false if failed?
last_update_started_at && last_update_started_at > timestamp
2018-10-15 14:42:47 +05:30
end
def mark_for_delete_if_blank_url
mark_for_destruction if url.blank?
end
2019-10-12 21:52:04 +05:30
def update_error_message(error_message)
self.last_error = Gitlab::UrlSanitizer.sanitize(error_message)
end
def mark_for_retry!(error_message)
update_error_message(error_message)
update_retry!
end
def mark_as_failed!(error_message)
update_error_message(error_message)
update_fail!
2018-10-15 14:42:47 +05:30
end
2021-04-29 21:17:54 +05:30
# Force the mrror into the retry state
def hard_retry!(error_message)
update_error_message(error_message)
self.update_status = :to_retry
save!(validate: false)
end
# Force the mirror into the failed state
def hard_fail!(error_message)
update_error_message(error_message)
self.update_status = :failed
save!(validate: false)
send_failure_notifications
end
2018-10-15 14:42:47 +05:30
def url=(value)
super(value) && return unless Gitlab::UrlSanitizer.valid?(value)
mirror_url = Gitlab::UrlSanitizer.new(value)
2019-02-15 15:39:39 +05:30
self.credentials ||= {}
self.credentials = self.credentials.merge(mirror_url.credentials)
2018-10-15 14:42:47 +05:30
super(mirror_url.sanitized_url)
end
def url
if super
Gitlab::UrlSanitizer.new(super, credentials: credentials).full_url
end
2021-06-08 01:23:25 +05:30
rescue StandardError
2018-10-15 14:42:47 +05:30
super
end
def safe_url
2021-04-29 21:17:54 +05:30
super(allowed_usernames: %w[git])
2018-10-15 14:42:47 +05:30
end
2020-11-24 15:15:51 +05:30
def bare_url
Gitlab::UrlSanitizer.new(read_attribute(:url)).full_url
end
2018-11-20 20:47:30 +05:30
def ensure_remote!
return unless project
2019-02-15 15:39:39 +05:30
return unless remote_name && remote_url
2018-11-20 20:47:30 +05:30
# If this fails or the remote already exists, we won't know due to
# https://gitlab.com/gitlab-org/gitaly/issues/1317
2019-02-15 15:39:39 +05:30
project.repository.add_remote(remote_name, remote_url)
end
def after_sent_notification
update_column(:error_notification_sent, true)
2018-11-20 20:47:30 +05:30
end
2019-10-12 21:52:04 +05:30
def backoff_delay
if self.only_protected_branches
PROTECTED_BACKOFF_DELAY
else
UNPROTECTED_BACKOFF_DELAY
end
end
def max_runtime
last_update_at.present? ? MAX_INCREMENTAL_RUNTIME : MAX_FIRST_RUNTIME
end
2021-04-29 21:17:54 +05:30
def send_failure_notifications
Gitlab::Metrics.add_event(:remote_mirrors_failed)
run_after_commit do
RemoteMirrorNotificationWorker.perform_async(id)
end
self.last_update_at = Time.current
save!(validate: false)
end
2018-10-15 14:42:47 +05:30
private
2019-02-15 15:39:39 +05:30
def store_credentials
# This is a necessary workaround for attr_encrypted, which doesn't otherwise
# notice that the credentials have changed
self.credentials = self.credentials
end
# The remote URL omits any password if SSH public-key authentication is in use
def remote_url
return url unless ssh_key_auth? && password.present?
Gitlab::UrlSanitizer.new(read_attribute(:url), credentials: { user: user }).full_url
2021-06-08 01:23:25 +05:30
rescue StandardError
2019-02-15 15:39:39 +05:30
super
2018-10-15 14:42:47 +05:30
end
def fallback_remote_name
return unless id
"remote_mirror_#{id}"
end
def recently_scheduled?
return false unless self.last_update_started_at
2020-06-23 00:09:42 +05:30
self.last_update_started_at >= Time.current - backoff_delay
2018-10-15 14:42:47 +05:30
end
def reset_fields
update_columns(
last_error: nil,
last_update_at: nil,
last_successful_update_at: nil,
2019-02-15 15:39:39 +05:30
update_status: 'finished',
error_notification_sent: false
2018-10-15 14:42:47 +05:30
)
end
def set_override_remote_mirror_available
enabled = read_attribute(:enabled)
project.update(remote_mirror_available_overridden: enabled)
end
def set_new_remote_name
self.remote_name = "remote_mirror_#{SecureRandom.hex}"
end
def refresh_remote
return unless project
# Before adding a new remote we have to delete the data from
# the previous remote name
2019-07-31 22:56:46 +05:30
prev_remote_name = remote_name_before_last_save || fallback_remote_name
2018-10-15 14:42:47 +05:30
run_after_commit do
project.repository.async_remove_remote(prev_remote_name)
end
2019-02-15 15:39:39 +05:30
project.repository.add_remote(remote_name, remote_url)
2018-10-15 14:42:47 +05:30
end
def remove_remote
return unless project # could be pending to delete so don't need to touch the git repository
project.repository.async_remove_remote(remote_name)
end
def mirror_url_changed?
2021-03-08 18:12:59 +05:30
url_changed? || attribute_changed?(:credentials)
2018-10-15 14:42:47 +05:30
end
2019-07-31 22:56:46 +05:30
def saved_change_to_mirror_url?
saved_change_to_url? || saved_change_to_credentials?
end
2018-10-15 14:42:47 +05:30
end
2019-12-04 20:38:33 +05:30
2021-06-08 01:23:25 +05:30
RemoteMirror.prepend_mod_with('RemoteMirror')