150 lines
4.4 KiB
Ruby
150 lines
4.4 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
module WebHooks
|
|
module AutoDisabling
|
|
extend ActiveSupport::Concern
|
|
|
|
ENABLED_HOOK_TYPES = %w[ProjectHook].freeze
|
|
MAX_FAILURES = 100
|
|
FAILURE_THRESHOLD = 3
|
|
EXCEEDED_FAILURE_THRESHOLD = FAILURE_THRESHOLD + 1
|
|
INITIAL_BACKOFF = 1.minute.freeze
|
|
MAX_BACKOFF = 1.day.freeze
|
|
BACKOFF_GROWTH_FACTOR = 2.0
|
|
|
|
class_methods do
|
|
def auto_disabling_enabled?
|
|
enabled_hook_types.include?(name) &&
|
|
Gitlab::SafeRequestStore.fetch(:auto_disabling_web_hooks) do
|
|
Feature.enabled?(:auto_disabling_web_hooks, type: :ops)
|
|
end
|
|
end
|
|
|
|
private
|
|
|
|
def enabled_hook_types
|
|
ENABLED_HOOK_TYPES
|
|
end
|
|
end
|
|
|
|
included do
|
|
delegate :auto_disabling_enabled?, to: :class, private: true
|
|
|
|
# A hook is disabled if:
|
|
#
|
|
# - we are no longer in the grace-perod (recent_failures > ?)
|
|
# - and either:
|
|
# - disabled_until is nil (i.e. this was set by WebHook#fail!)
|
|
# - or disabled_until is in the future (i.e. this was set by WebHook#backoff!)
|
|
scope :disabled, -> do
|
|
return none unless auto_disabling_enabled?
|
|
|
|
where('recent_failures > ? AND (disabled_until IS NULL OR disabled_until >= ?)',
|
|
FAILURE_THRESHOLD, Time.current)
|
|
end
|
|
|
|
# A hook is executable if:
|
|
#
|
|
# - we are still in the grace-period (recent_failures <= ?)
|
|
# - OR we have exceeded the grace period and neither of the following is true:
|
|
# - disabled_until is nil (i.e. this was set by WebHook#fail!)
|
|
# - disabled_until is in the future (i.e. this was set by WebHook#backoff!)
|
|
scope :executable, -> do
|
|
return all unless auto_disabling_enabled?
|
|
|
|
where('recent_failures <= ? OR (recent_failures > ? AND (disabled_until IS NOT NULL) AND (disabled_until < ?))',
|
|
FAILURE_THRESHOLD, FAILURE_THRESHOLD, Time.current)
|
|
end
|
|
end
|
|
|
|
def executable?
|
|
return true unless auto_disabling_enabled?
|
|
|
|
!temporarily_disabled? && !permanently_disabled?
|
|
end
|
|
|
|
def temporarily_disabled?
|
|
return false unless auto_disabling_enabled?
|
|
|
|
disabled_until.present? && disabled_until >= Time.current && recent_failures > FAILURE_THRESHOLD
|
|
end
|
|
|
|
def permanently_disabled?
|
|
return false unless auto_disabling_enabled?
|
|
|
|
recent_failures > FAILURE_THRESHOLD && disabled_until.blank?
|
|
end
|
|
|
|
def disable!
|
|
return if !auto_disabling_enabled? || permanently_disabled?
|
|
|
|
update_attribute(:recent_failures, EXCEEDED_FAILURE_THRESHOLD)
|
|
end
|
|
|
|
def enable!
|
|
return unless auto_disabling_enabled?
|
|
return if recent_failures == 0 && disabled_until.nil? && backoff_count == 0
|
|
|
|
assign_attributes(recent_failures: 0, disabled_until: nil, backoff_count: 0)
|
|
save(validate: false)
|
|
end
|
|
|
|
# Don't actually back-off until FAILURE_THRESHOLD failures have been seen
|
|
# we mark the grace-period using the recent_failures counter
|
|
def backoff!
|
|
return unless auto_disabling_enabled?
|
|
return if permanently_disabled? || (backoff_count >= MAX_FAILURES && temporarily_disabled?)
|
|
|
|
attrs = { recent_failures: next_failure_count }
|
|
|
|
if recent_failures >= FAILURE_THRESHOLD
|
|
attrs[:backoff_count] = next_backoff_count
|
|
attrs[:disabled_until] = next_backoff.from_now
|
|
end
|
|
|
|
assign_attributes(attrs)
|
|
save(validate: false) if changed?
|
|
end
|
|
|
|
def failed!
|
|
return unless auto_disabling_enabled?
|
|
return unless recent_failures < MAX_FAILURES
|
|
|
|
assign_attributes(disabled_until: nil, backoff_count: 0, recent_failures: next_failure_count)
|
|
save(validate: false)
|
|
end
|
|
|
|
def next_backoff
|
|
return MAX_BACKOFF if backoff_count >= 8 # optimization to prevent expensive exponentiation and possible overflows
|
|
|
|
(INITIAL_BACKOFF * (BACKOFF_GROWTH_FACTOR**backoff_count))
|
|
.clamp(INITIAL_BACKOFF, MAX_BACKOFF)
|
|
.seconds
|
|
end
|
|
|
|
def alert_status
|
|
return :executable unless auto_disabling_enabled?
|
|
|
|
if temporarily_disabled?
|
|
:temporarily_disabled
|
|
elsif permanently_disabled?
|
|
:disabled
|
|
else
|
|
:executable
|
|
end
|
|
end
|
|
|
|
private
|
|
|
|
def next_failure_count
|
|
recent_failures.succ.clamp(1, MAX_FAILURES)
|
|
end
|
|
|
|
def next_backoff_count
|
|
backoff_count.succ.clamp(1, MAX_FAILURES)
|
|
end
|
|
end
|
|
end
|
|
|
|
WebHooks::AutoDisabling.prepend_mod
|
|
WebHooks::AutoDisabling::ClassMethods.prepend_mod
|