debian-mirror-gitlab/lib/gitlab/job_waiter.rb

103 lines
3.1 KiB
Ruby
Raw Normal View History

2018-12-13 13:39:08 +05:30
# frozen_string_literal: true
2017-08-17 22:00:37 +05:30
module Gitlab
# JobWaiter can be used to wait for a number of Sidekiq jobs to complete.
2018-03-17 18:26:18 +05:30
#
# Its use requires the cooperation of the sidekiq jobs themselves. Set up the
# waiter, then start the jobs, passing them its `key`. Their `perform` methods
# should look like:
#
# def perform(args, notify_key)
# # do work
# ensure
# ::Gitlab::JobWaiter.notify(notify_key, jid)
# end
#
# The JobWaiter blocks popping items from a Redis array. All the sidekiq jobs
# push to that array when done. Once the waiter has popped `count` items, it
# knows all the jobs are done.
2017-08-17 22:00:37 +05:30
class JobWaiter
2019-12-04 20:38:33 +05:30
KEY_PREFIX = "gitlab:job_waiter"
2018-03-27 19:54:05 +05:30
2020-04-08 14:13:33 +05:30
STARTED_METRIC = :gitlab_job_waiter_started_total
TIMEOUTS_METRIC = :gitlab_job_waiter_timeouts_total
2018-03-17 18:26:18 +05:30
def self.notify(key, jid)
2021-01-03 14:25:43 +05:30
Gitlab::Redis::SharedState.with do |redis|
# Use a Redis MULTI transaction to ensure we always set an expiry
redis.multi do |multi|
multi.lpush(key, jid)
# This TTL needs to be long enough to allow whichever Sidekiq job calls
# JobWaiter#wait to reach BLPOP.
multi.expire(key, 6.hours.to_i)
end
end
2018-03-17 18:26:18 +05:30
end
2018-03-27 19:54:05 +05:30
def self.key?(key)
2022-08-27 11:52:29 +05:30
key.is_a?(String) && key =~ /\A#{KEY_PREFIX}:\h{8}-\h{4}-\h{4}-\h{4}-\h{12}\z/o
2018-03-27 19:54:05 +05:30
end
2020-04-08 14:13:33 +05:30
attr_reader :key, :finished, :worker_label
2018-03-17 18:26:18 +05:30
attr_accessor :jobs_remaining
2017-08-17 22:00:37 +05:30
2018-03-17 18:26:18 +05:30
# jobs_remaining - the number of jobs left to wait for
# key - The key of this waiter.
2020-04-08 14:13:33 +05:30
def initialize(jobs_remaining = 0, key = "#{KEY_PREFIX}:#{SecureRandom.uuid}", worker_label: nil)
2018-03-17 18:26:18 +05:30
@key = key
@jobs_remaining = jobs_remaining
@finished = []
2020-04-08 14:13:33 +05:30
@worker_label = worker_label
2017-08-17 22:00:37 +05:30
end
# Waits for all the jobs to be completed.
#
# timeout - The maximum amount of seconds to block the caller for. This
# ensures we don't indefinitely block a caller in case a job takes
# long to process, or is never processed.
2017-09-10 17:25:29 +05:30
def wait(timeout = 10)
2018-03-17 18:26:18 +05:30
deadline = Time.now.utc + timeout
2020-04-08 14:13:33 +05:30
increment_counter(STARTED_METRIC)
2018-03-17 18:26:18 +05:30
Gitlab::Redis::SharedState.with do |redis|
while jobs_remaining > 0
# Redis will not take fractional seconds. Prefer waiting too long over
# not waiting long enough
seconds_left = (deadline - Time.now.utc).ceil
2017-08-17 22:00:37 +05:30
2018-03-17 18:26:18 +05:30
# Redis interprets 0 as "wait forever", so skip the final `blpop` call
break if seconds_left <= 0
2017-08-17 22:00:37 +05:30
2018-03-17 18:26:18 +05:30
list, jid = redis.blpop(key, timeout: seconds_left)
2020-04-08 14:13:33 +05:30
# timed out
unless list && jid
increment_counter(TIMEOUTS_METRIC)
break
end
2018-03-17 18:26:18 +05:30
@finished << jid
@jobs_remaining -= 1
end
2017-08-17 22:00:37 +05:30
end
2018-03-17 18:26:18 +05:30
finished
2017-08-17 22:00:37 +05:30
end
2020-04-08 14:13:33 +05:30
private
def increment_counter(metric)
return unless worker_label
metrics[metric].increment(worker: worker_label)
end
def metrics
@metrics ||= {
STARTED_METRIC => Gitlab::Metrics.counter(STARTED_METRIC, 'JobWaiter attempts started'),
TIMEOUTS_METRIC => Gitlab::Metrics.counter(TIMEOUTS_METRIC, 'JobWaiter attempts timed out')
}
end
2017-08-17 22:00:37 +05:30
end
end