# frozen_string_literal: true module Ci class Runner < Ci::ApplicationRecord prepend Ci::BulkInsertableTags include Gitlab::SQL::Pattern include RedisCacheable include ChronicDurationAttribute include FromUnion include TokenAuthenticatable include FeatureGate include Gitlab::Utils::StrongMemoize include TaggableQueries include Presentable include EachBatch add_authentication_token_field :token, encrypted: :optional, expires_at: :compute_token_expiration enum access_level: { not_protected: 0, ref_protected: 1 } enum runner_type: { instance_type: 1, group_type: 2, project_type: 3 } enum executor_type: { unknown: 0, custom: 1, shell: 2, docker: 3, docker_windows: 4, docker_ssh: 5, ssh: 6, parallels: 7, virtualbox: 8, docker_machine: 9, docker_ssh_machine: 10, kubernetes: 11 }, _suffix: true # This `ONLINE_CONTACT_TIMEOUT` needs to be larger than # `RUNNER_QUEUE_EXPIRY_TIME+UPDATE_CONTACT_COLUMN_EVERY` # ONLINE_CONTACT_TIMEOUT = 2.hours # The `RUNNER_QUEUE_EXPIRY_TIME` indicates the longest interval that # Runner request needs to be refreshed by Rails instead of being handled # by Workhorse RUNNER_QUEUE_EXPIRY_TIME = 1.hour # The `UPDATE_CONTACT_COLUMN_EVERY` defines how often the Runner DB entry can be updated UPDATE_CONTACT_COLUMN_EVERY = (40.minutes..55.minutes).freeze # The `STALE_TIMEOUT` constant defines the how far past the last contact or creation date a runner will be considered stale STALE_TIMEOUT = 3.months AVAILABLE_TYPES_LEGACY = %w[specific shared].freeze AVAILABLE_TYPES = runner_types.keys.freeze AVAILABLE_STATUSES = %w[active paused online offline never_contacted stale].freeze # TODO: Remove in %16.0: active, paused. Relevant issue: https://gitlab.com/gitlab-org/gitlab/-/issues/344648 AVAILABLE_SCOPES = (AVAILABLE_TYPES_LEGACY + AVAILABLE_TYPES + AVAILABLE_STATUSES).freeze FORM_EDITABLE = %i[description tag_list active run_untagged locked access_level maximum_timeout_human_readable].freeze MINUTES_COST_FACTOR_FIELDS = %i[public_projects_minutes_cost_factor private_projects_minutes_cost_factor].freeze TAG_LIST_MAX_LENGTH = 50 has_many :builds has_many :runner_projects, inverse_of: :runner, autosave: true, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :projects, through: :runner_projects, disable_joins: true has_many :runner_namespaces, inverse_of: :runner, autosave: true has_many :groups, through: :runner_namespaces, disable_joins: true has_one :last_build, -> { order('id DESC') }, class_name: 'Ci::Build' has_one :runner_version, primary_key: :version, foreign_key: :version, class_name: 'Ci::RunnerVersion' before_save :ensure_token scope :active, -> (value = true) { where(active: value) } scope :paused, -> { active(false) } scope :online, -> { where('contacted_at > ?', online_contact_time_deadline) } scope :recent, -> { where('ci_runners.created_at >= :date OR ci_runners.contacted_at >= :date', date: stale_deadline) } scope :stale, -> { where('ci_runners.created_at < :date AND (ci_runners.contacted_at IS NULL OR ci_runners.contacted_at < :date)', date: stale_deadline) } scope :offline, -> { where(arel_table[:contacted_at].lteq(online_contact_time_deadline)) } scope :never_contacted, -> { where(contacted_at: nil) } scope :ordered, -> { order(id: :desc) } scope :with_recent_runner_queue, -> { where('contacted_at > ?', recent_queue_deadline) } # BACKWARD COMPATIBILITY: There are needed to maintain compatibility with `AVAILABLE_SCOPES` used by `lib/api/runners.rb` scope :deprecated_shared, -> { instance_type } scope :deprecated_specific, -> { project_type.or(group_type) } scope :belonging_to_project, -> (project_id) { joins(:runner_projects).where(ci_runner_projects: { project_id: project_id }) } scope :belonging_to_group, -> (group_id) { joins(:runner_namespaces).where(ci_runner_namespaces: { namespace_id: group_id }) } scope :belonging_to_group_or_project_descendants, -> (group_id) { group_ids = Ci::NamespaceMirror.by_group_and_descendants(group_id).select(:namespace_id) project_ids = Ci::ProjectMirror.by_namespace_id(group_ids).select(:project_id) group_runners = belonging_to_group(group_ids) project_runners = belonging_to_project(project_ids).distinct from_union( [group_runners, project_runners], remove_duplicates: false ) } scope :belonging_to_group_and_ancestors, -> (group_id) { group_self_and_ancestors_ids = ::Group.find_by(id: group_id)&.self_and_ancestor_ids belonging_to_group(group_self_and_ancestors_ids) } scope :belonging_to_parent_group_of_project, -> (project_id) { raise ArgumentError, "only 1 project_id allowed for performance reasons" unless project_id.is_a?(Integer) project_groups = ::Group.joins(:projects).where(projects: { id: project_id }) belonging_to_group(project_groups.self_and_ancestors.pluck(:id)) } scope :owned_or_instance_wide, -> (project_id) do project = project_id.respond_to?(:shared_runners) ? project_id : Project.find(project_id) from_union( [ belonging_to_project(project_id), project.group_runners_enabled? ? belonging_to_parent_group_of_project(project_id) : nil, project.shared_runners ].compact, remove_duplicates: false ) end scope :group_or_instance_wide, -> (group) do from_union( [ belonging_to_group_and_ancestors(group.id), group.shared_runners ], remove_duplicates: false ) end scope :usable_from_scope, -> (group) do from_union( [ belonging_to_group(group.ancestor_ids), belonging_to_group_or_project_descendants(group.id), group.shared_runners ], remove_duplicates: false ) end scope :assignable_for, ->(project) do # FIXME: That `to_sql` is needed to workaround a weird Rails bug. # Without that, placeholders would miss one and couldn't match. # # We use "unscoped" here so that any current Ci::Runner filters don't # apply to the inner query, which is not necessary. exclude_runners = unscoped { project.runners.select(:id) }.to_sql where(locked: false) .where.not("ci_runners.id IN (#{exclude_runners})") .project_type end scope :order_contacted_at_asc, -> { order(contacted_at: :asc) } scope :order_contacted_at_desc, -> { order(contacted_at: :desc) } scope :order_created_at_asc, -> { order(created_at: :asc) } scope :order_created_at_desc, -> { order(created_at: :desc) } scope :order_token_expires_at_asc, -> { order(token_expires_at: :asc) } scope :order_token_expires_at_desc, -> { order(token_expires_at: :desc) } scope :with_tags, -> { preload(:tags) } validate :tag_constraints validates :access_level, presence: true validates :runner_type, presence: true validate :no_projects, unless: :project_type? validate :no_groups, unless: :group_type? validate :any_project, if: :project_type? validate :exactly_one_group, if: :group_type? acts_as_taggable after_destroy :cleanup_runner_queue cached_attr_reader :version, :revision, :platform, :architecture, :ip_address, :contacted_at, :executor_type chronic_duration_attr :maximum_timeout_human_readable, :maximum_timeout, error_message: 'Maximum job timeout has a value which could not be accepted' validates :maximum_timeout, allow_nil: true, numericality: { greater_than_or_equal_to: 600, message: 'needs to be at least 10 minutes' } validates :public_projects_minutes_cost_factor, :private_projects_minutes_cost_factor, allow_nil: false, numericality: { greater_than_or_equal_to: 0.0, message: 'needs to be non-negative' } validates :config, json_schema: { filename: 'ci_runner_config' } validates :maintenance_note, length: { maximum: 1024 } alias_attribute :maintenance_note, :maintainer_note # NOTE: Need to keep until REST v5 is implemented # Searches for runners matching the given query. # # This method uses ILIKE on PostgreSQL for the description field and performs a full match on tokens. # # query - The search query as a String. # # Returns an ActiveRecord::Relation. def self.search(query) where(token: query).or(fuzzy_search(query, [:description])) end def self.online_contact_time_deadline ONLINE_CONTACT_TIMEOUT.ago end def self.stale_deadline STALE_TIMEOUT.ago end def self.recent_queue_deadline # we add queue expiry + online # - contacted_at can be updated at any time within this interval # we have always accurate `contacted_at` but it is stored in Redis # and not persisted in database (ONLINE_CONTACT_TIMEOUT + RUNNER_QUEUE_EXPIRY_TIME).ago end def self.order_by(order) case order when 'contacted_asc' order_contacted_at_asc when 'contacted_desc' order_contacted_at_desc when 'created_at_asc' order_created_at_asc when 'token_expires_at_asc' order_token_expires_at_asc when 'token_expires_at_desc' order_token_expires_at_desc else order_created_at_desc end end def self.runner_matchers unique_params = [ :runner_type, :public_projects_minutes_cost_factor, :private_projects_minutes_cost_factor, :run_untagged, :access_level, Arel.sql("(#{arel_tag_names_array.to_sql})") ] group(*unique_params).pluck('array_agg(ci_runners.id)', *unique_params).map do |values| Gitlab::Ci::Matching::RunnerMatcher.new({ runner_ids: values[0], runner_type: values[1], public_projects_minutes_cost_factor: values[2], private_projects_minutes_cost_factor: values[3], run_untagged: values[4], access_level: values[5], tag_list: values[6] }) end end def runner_matcher strong_memoize(:runner_matcher) do Gitlab::Ci::Matching::RunnerMatcher.new({ runner_ids: [id], runner_type: runner_type, public_projects_minutes_cost_factor: public_projects_minutes_cost_factor, private_projects_minutes_cost_factor: private_projects_minutes_cost_factor, run_untagged: run_untagged, access_level: access_level, tag_list: tag_list }) end end def assign_to(project, current_user = nil) if instance_type? raise ArgumentError, 'Transitioning an instance runner to a project runner is not supported' elsif group_type? raise ArgumentError, 'Transitioning a group runner to a project runner is not supported' end begin transaction do self.runner_projects << ::Ci::RunnerProject.new(project: project, runner: self) self.save! end rescue ActiveRecord::RecordInvalid => e self.errors.add(:assign_to, e.message) false end end def display_name return short_sha if description.blank? description end def online? contacted_at && contacted_at > self.class.online_contact_time_deadline end def stale? return false unless created_at [created_at, contacted_at].compact.max < self.class.stale_deadline end def status(legacy_mode = nil) # TODO Deprecate legacy_mode in %16.0 and make it a no-op # (see https://gitlab.com/gitlab-org/gitlab/-/issues/360545) # TODO Remove legacy_mode in %17.0 return deprecated_rest_status if legacy_mode == '14.5' return :stale if stale? return :never_contacted unless contacted_at online? ? :online : :offline end # DEPRECATED # TODO Remove in v5 in favor of `status` for REST calls, see https://gitlab.com/gitlab-org/gitlab/-/issues/344648 def deprecated_rest_status return :stale if stale? if contacted_at.nil? :never_contacted elsif active? online? ? :online : :offline else :paused end end def owner_project return unless project_type? runner_projects.order(:id).first.project end def belongs_to_one_project? runner_projects.count == 1 end def belongs_to_more_than_one_project? runner_projects.limit(2).count(:all) > 1 end def match_build_if_online?(build) active? && online? && matches_build?(build) end def only_for?(project) !runner_projects.where.not(project_id: project.id).exists? end def short_sha token[0...8] if token end def tag_list if tags.loaded? tags.map(&:name) else super end end def has_tags? tag_list.any? end def predefined_variables Gitlab::Ci::Variables::Collection.new .append(key: 'CI_RUNNER_ID', value: id.to_s) .append(key: 'CI_RUNNER_DESCRIPTION', value: description) .append(key: 'CI_RUNNER_TAGS', value: tag_list.to_s) end def tick_runner_queue ## # We only stick a runner to primary database to be able to detect the # replication lag in `EE::Ci::RegisterJobService#execute`. The # intention here is not to execute `Ci::RegisterJobService#execute` on # the primary database. # ::Ci::Runner.sticking.stick(:runner, id) SecureRandom.hex.tap do |new_update| ::Gitlab::Workhorse.set_key_and_notify(runner_queue_key, new_update, expire: RUNNER_QUEUE_EXPIRY_TIME, overwrite: true) end end def ensure_runner_queue_value new_value = SecureRandom.hex ::Gitlab::Workhorse.set_key_and_notify(runner_queue_key, new_value, expire: RUNNER_QUEUE_EXPIRY_TIME, overwrite: false) end def runner_queue_value_latest?(value) ensure_runner_queue_value == value if value.present? end def heartbeat(values) ## # We can safely ignore writes performed by a runner heartbeat. We do # not want to upgrade database connection proxy to use the primary # database after heartbeat write happens. # ::Gitlab::Database::LoadBalancing::Session.without_sticky_writes do values = values&.slice(:version, :revision, :platform, :architecture, :ip_address, :config, :executor) || {} values[:contacted_at] = Time.current values[:executor_type] = EXECUTOR_NAME_TO_TYPES.fetch(values.delete(:executor), :unknown) cache_attributes(values) # We save data without validation, it will always change due to `contacted_at` if persist_cached_data? version_updated = values.include?(:version) && values[:version] != version update_columns(values) schedule_runner_version_update if version_updated end end end def pick_build!(build) tick_runner_queue if matches_build?(build) end def matches_build?(build) runner_matcher.matches?(build.build_matcher) end def uncached_contacted_at read_attribute(:contacted_at) end def namespace_ids strong_memoize(:namespace_ids) do runner_namespaces.pluck(:namespace_id).compact end end def compute_token_expiration case runner_type when 'instance_type' compute_token_expiration_instance when 'group_type' compute_token_expiration_group when 'project_type' compute_token_expiration_project end end private scope :with_upgrade_status, ->(upgrade_status) do joins(:runner_version).where(runner_version: { status: upgrade_status }) end EXECUTOR_NAME_TO_TYPES = { 'unknown' => :unknown, 'custom' => :custom, 'shell' => :shell, 'docker' => :docker, 'docker-windows' => :docker_windows, 'docker-ssh' => :docker_ssh, 'ssh' => :ssh, 'parallels' => :parallels, 'virtualbox' => :virtualbox, 'docker+machine' => :docker_machine, 'docker-ssh+machine' => :docker_ssh_machine, 'kubernetes' => :kubernetes }.freeze EXECUTOR_TYPE_TO_NAMES = EXECUTOR_NAME_TO_TYPES.invert.freeze def compute_token_expiration_instance return unless expiration_interval = Gitlab::CurrentSettings.runner_token_expiration_interval expiration_interval.seconds.from_now end def compute_token_expiration_group ::Group.where(id: runner_namespaces.map(&:namespace_id)).map(&:effective_runner_token_expiration_interval).compact.min&.from_now end def compute_token_expiration_project Project.where(id: runner_projects.map(&:project_id)).map(&:effective_runner_token_expiration_interval).compact.min&.from_now end def cleanup_runner_queue Gitlab::Redis::SharedState.with do |redis| redis.del(runner_queue_key) end end def runner_queue_key "runner:build_queue:#{self.token}" end def persist_cached_data? # Use a random threshold to prevent beating DB updates. contacted_at_max_age = Random.rand(UPDATE_CONTACT_COLUMN_EVERY) real_contacted_at = read_attribute(:contacted_at) real_contacted_at.nil? || (Time.current - real_contacted_at) >= contacted_at_max_age end def tag_constraints unless has_tags? || run_untagged? errors.add(:tags_list, 'can not be empty when runner is not allowed to pick untagged jobs') end if tag_list_changed? && tag_list.count > TAG_LIST_MAX_LENGTH errors.add(:tags_list, "Too many tags specified. Please limit the number of tags to #{TAG_LIST_MAX_LENGTH}") end end def no_projects if runner_projects.any? errors.add(:runner, 'cannot have projects assigned') end end def no_groups if runner_namespaces.any? errors.add(:runner, 'cannot have groups assigned') end end def any_project unless runner_projects.any? errors.add(:runner, 'needs to be assigned to at least one project') end end def exactly_one_group unless runner_namespaces.one? errors.add(:runner, 'needs to be assigned to exactly one group') end end def schedule_runner_version_update return unless version Ci::Runners::ProcessRunnerVersionUpdateWorker.perform_async(version) end end end Ci::Runner.prepend_mod_with('Ci::Runner')