debian-mirror-gitlab/app/workers/concerns/worker_attributes.rb

205 lines
6.9 KiB
Ruby
Raw Normal View History

2019-12-21 20:55:43 +05:30
# frozen_string_literal: true
module WorkerAttributes
extend ActiveSupport::Concern
2020-07-28 23:09:34 +05:30
include Gitlab::ClassAttributes
2019-12-21 20:55:43 +05:30
2019-12-26 22:10:19 +05:30
# Resource boundaries that workers can declare through the
2020-04-08 14:13:33 +05:30
# `resource_boundary` attribute
2019-12-26 22:10:19 +05:30
VALID_RESOURCE_BOUNDARIES = [:memory, :cpu, :unknown].freeze
2020-04-08 14:13:33 +05:30
# Urgencies that workers can declare through the `urgencies` attribute
VALID_URGENCIES = [:high, :low, :throttled].freeze
2021-04-29 21:17:54 +05:30
VALID_DATA_CONSISTENCIES = [:always, :sticky, :delayed].freeze
2021-09-30 23:02:18 +05:30
DEFAULT_DATA_CONSISTENCY = :always
2021-04-29 21:17:54 +05:30
2020-03-13 15:44:24 +05:30
NAMESPACE_WEIGHTS = {
auto_devops: 2,
auto_merge: 3,
chaos: 2,
deployment: 3,
mail_scheduler: 2,
notifications: 2,
pipeline_cache: 3,
pipeline_creation: 4,
pipeline_default: 3,
pipeline_hooks: 2,
pipeline_processing: 5,
# EE-specific
epics: 2,
incident_management: 2,
security_scans: 2
}.stringify_keys.freeze
2019-12-21 20:55:43 +05:30
class_methods do
2020-07-28 23:09:34 +05:30
def feature_category(value, *extras)
2019-12-21 20:55:43 +05:30
raise "Invalid category. Use `feature_category_not_owned!` to mark a worker as not owned" if value == :not_owned
2021-06-08 01:23:25 +05:30
set_class_attribute(:feature_category, value)
2019-12-21 20:55:43 +05:30
end
# Special case: mark this work as not associated with a feature category
# this should be used for cross-cutting concerns, such as mailer workers.
def feature_category_not_owned!
2021-06-08 01:23:25 +05:30
set_class_attribute(:feature_category, :not_owned)
2019-12-21 20:55:43 +05:30
end
def get_feature_category
2020-07-28 23:09:34 +05:30
get_class_attribute(:feature_category)
2019-12-21 20:55:43 +05:30
end
def feature_category_not_owned?
2020-07-28 23:09:34 +05:30
get_feature_category == :not_owned
2019-12-21 20:55:43 +05:30
end
2020-04-08 14:13:33 +05:30
# This should be set to :high for jobs that need to be run
# immediately, or, if they are delayed, risk creating
# inconsistencies in the application that could being perceived by
# the user as incorrect behavior (ie, a bug)
#
# See
# doc/development/sidekiq_style_guide.md#urgency
2019-12-26 22:10:19 +05:30
# for details
2020-04-08 14:13:33 +05:30
def urgency(urgency)
raise "Invalid urgency: #{urgency}" unless VALID_URGENCIES.include?(urgency)
2021-06-08 01:23:25 +05:30
set_class_attribute(:urgency, urgency)
2019-12-26 22:10:19 +05:30
end
2020-04-08 14:13:33 +05:30
def get_urgency
2020-07-28 23:09:34 +05:30
class_attributes[:urgency] || :low
2019-12-26 22:10:19 +05:30
end
2021-09-04 01:27:46 +05:30
# Allows configuring worker's data_consistency.
#
# Worker can utilize Sidekiq readonly database replicas capabilities by setting data_consistency attribute.
# Workers with data_consistency set to :delayed or :sticky, calling #perform_async
# will be delayed in order to give replication process enough time to complete.
#
# - *data_consistency* values:
# - 'always' - The job is required to use the primary database (default).
# - 'sticky' - The uses a replica as long as possible. It switches to primary either on write or long replication lag.
# - 'delayed' - The job would switch to primary only on write. It would use replica always.
# If there's a long replication lag the job will be delayed, and only if the replica is not up to date on the next retry,
# it will switch to the primary.
# - *feature_flag* - allows you to toggle a job's `data_consistency, which permits you to safely toggle load balancing capabilities for a specific job.
# If disabled, job will default to `:always`, which means that the job will always use the primary.
2021-04-29 21:17:54 +05:30
def data_consistency(data_consistency, feature_flag: nil)
raise ArgumentError, "Invalid data consistency: #{data_consistency}" unless VALID_DATA_CONSISTENCIES.include?(data_consistency)
raise ArgumentError, 'Data consistency is already set' if class_attributes[:data_consistency]
2021-06-08 01:23:25 +05:30
set_class_attribute(:data_consistency_feature_flag, feature_flag) if feature_flag
set_class_attribute(:data_consistency, data_consistency)
2021-04-29 21:17:54 +05:30
validate_worker_attributes!
end
def validate_worker_attributes!
# Since the deduplication should always take into account the latest binary replication pointer into account,
# not the first one, the deduplication will not work with sticky or delayed.
# Follow up issue to improve this: https://gitlab.com/gitlab-org/gitlab/-/issues/325291
2021-09-04 01:27:46 +05:30
if idempotent? && utilizes_load_balancing_capabilities?
2021-04-29 21:17:54 +05:30
raise ArgumentError, "Class can't be marked as idempotent if data_consistency is not set to :always"
end
end
2021-09-04 01:27:46 +05:30
# If data_consistency is not set to :always, worker will try to utilize load balancing capabilities and use the replica
def utilizes_load_balancing_capabilities?
get_data_consistency != :always
end
2021-04-29 21:17:54 +05:30
def get_data_consistency
2021-09-30 23:02:18 +05:30
class_attributes[:data_consistency] || DEFAULT_DATA_CONSISTENCY
2021-04-29 21:17:54 +05:30
end
def get_data_consistency_feature_flag_enabled?
return true unless class_attributes[:data_consistency_feature_flag]
Feature.enabled?(class_attributes[:data_consistency_feature_flag], default_enabled: :yaml)
end
2019-12-26 22:10:19 +05:30
# Set this attribute on a job when it will call to services outside of the
# application, such as 3rd party applications, other k8s clusters etc See
2020-08-09 01:23:26 +05:30
# doc/development/sidekiq_style_guide.md#jobs-with-external-dependencies for
2019-12-26 22:10:19 +05:30
# details
def worker_has_external_dependencies!
2021-06-08 01:23:25 +05:30
set_class_attribute(:external_dependencies, true)
2019-12-26 22:10:19 +05:30
end
# Returns a truthy value if the worker has external dependencies.
2020-08-09 01:23:26 +05:30
# See doc/development/sidekiq_style_guide.md#jobs-with-external-dependencies
2019-12-26 22:10:19 +05:30
# for details
def worker_has_external_dependencies?
2020-07-28 23:09:34 +05:30
class_attributes[:external_dependencies]
2019-12-26 22:10:19 +05:30
end
def worker_resource_boundary(boundary)
raise "Invalid boundary" unless VALID_RESOURCE_BOUNDARIES.include? boundary
2021-06-08 01:23:25 +05:30
set_class_attribute(:resource_boundary, boundary)
2019-12-26 22:10:19 +05:30
end
def get_worker_resource_boundary
2020-07-28 23:09:34 +05:30
class_attributes[:resource_boundary] || :unknown
2019-12-26 22:10:19 +05:30
end
2020-04-08 14:13:33 +05:30
def idempotent!
2021-06-08 01:23:25 +05:30
set_class_attribute(:idempotent, true)
2021-04-29 21:17:54 +05:30
validate_worker_attributes!
2020-04-08 14:13:33 +05:30
end
def idempotent?
2020-07-28 23:09:34 +05:30
class_attributes[:idempotent]
2020-04-08 14:13:33 +05:30
end
2020-03-13 15:44:24 +05:30
def weight(value)
2021-06-08 01:23:25 +05:30
set_class_attribute(:weight, value)
2020-03-13 15:44:24 +05:30
end
def get_weight
2020-07-28 23:09:34 +05:30
class_attributes[:weight] ||
2020-03-13 15:44:24 +05:30
NAMESPACE_WEIGHTS[queue_namespace] ||
1
end
2020-06-23 00:09:42 +05:30
def tags(*values)
2021-06-08 01:23:25 +05:30
set_class_attribute(:tags, values)
2020-06-23 00:09:42 +05:30
end
def get_tags
2020-07-28 23:09:34 +05:30
Array(class_attributes[:tags])
2020-06-23 00:09:42 +05:30
end
def deduplicate(strategy, options = {})
2021-06-08 01:23:25 +05:30
set_class_attribute(:deduplication_strategy, strategy)
set_class_attribute(:deduplication_options, options)
2020-06-23 00:09:42 +05:30
end
def get_deduplicate_strategy
2020-07-28 23:09:34 +05:30
class_attributes[:deduplication_strategy] ||
2020-06-23 00:09:42 +05:30
Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob::DEFAULT_STRATEGY
end
def get_deduplication_options
2020-07-28 23:09:34 +05:30
class_attributes[:deduplication_options] || {}
2019-12-21 20:55:43 +05:30
end
2021-04-17 20:07:23 +05:30
2021-09-04 01:27:46 +05:30
def deduplication_enabled?
return true unless get_deduplication_options[:feature_flag]
Feature.enabled?(get_deduplication_options[:feature_flag], default_enabled: :yaml)
end
2021-04-17 20:07:23 +05:30
def big_payload!
2021-06-08 01:23:25 +05:30
set_class_attribute(:big_payload, true)
2021-04-17 20:07:23 +05:30
end
def big_payload?
class_attributes[:big_payload]
end
2019-12-21 20:55:43 +05:30
end
end