debian-mirror-gitlab/lib/gitlab/ci/trace.rb

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

324 lines
8 KiB
Ruby
Raw Normal View History

2018-12-13 13:39:08 +05:30
# frozen_string_literal: true
2017-08-17 22:00:37 +05:30
module Gitlab
module Ci
class Trace
2019-02-15 15:39:39 +05:30
include ::Gitlab::ExclusiveLeaseHelpers
2021-11-11 11:23:49 +05:30
include ::Gitlab::Utils::StrongMemoize
2019-12-21 20:55:43 +05:30
include Checksummable
2018-11-08 19:23:39 +05:30
2019-09-30 21:07:59 +05:30
LOCK_TTL = 10.minutes
2019-02-15 15:39:39 +05:30
LOCK_RETRIES = 2
LOCK_SLEEP = 0.001.seconds
2020-03-13 15:44:24 +05:30
WATCH_FLAG_TTL = 10.seconds
2021-04-29 21:17:54 +05:30
UPDATE_FREQUENCY_DEFAULT = 60.seconds
2020-03-13 15:44:24 +05:30
UPDATE_FREQUENCY_WHEN_BEING_WATCHED = 3.seconds
2018-11-08 19:23:39 +05:30
2021-09-04 01:27:46 +05:30
LOAD_BALANCING_STICKING_NAMESPACE = 'ci/build/trace'
2018-03-27 19:54:05 +05:30
ArchiveError = Class.new(StandardError)
2018-11-08 19:23:39 +05:30
AlreadyArchivedError = Class.new(StandardError)
2021-01-03 14:25:43 +05:30
LockedError = Class.new(StandardError)
2018-03-27 19:54:05 +05:30
2017-08-17 22:00:37 +05:30
attr_reader :job
delegate :old_trace, to: :job
2021-11-11 11:23:49 +05:30
delegate :can_attempt_archival_now?, :increment_archival_attempts!,
2021-11-18 22:05:49 +05:30
:archival_attempts_message, :archival_attempts_available?, to: :trace_metadata
2017-08-17 22:00:37 +05:30
def initialize(job)
@job = job
end
def html(last_lines: nil)
read do |stream|
stream.html(last_lines: last_lines)
end
end
def raw(last_lines: nil)
read do |stream|
stream.raw(last_lines: last_lines)
end
end
def extract_coverage(regex)
read do |stream|
stream.extract_coverage(regex)
end
end
2018-03-17 18:26:18 +05:30
def extract_sections
read do |stream|
stream.extract_sections
end
end
2017-08-17 22:00:37 +05:30
def set(data)
2018-10-15 14:42:47 +05:30
write('w+b') do |stream|
2017-08-17 22:00:37 +05:30
data = job.hide_secrets(data)
stream.set(data)
end
end
def append(data, offset)
2018-10-15 14:42:47 +05:30
write('a+b') do |stream|
2017-08-17 22:00:37 +05:30
current_length = stream.size
2018-10-15 14:42:47 +05:30
break current_length unless current_length == offset
2017-08-17 22:00:37 +05:30
data = job.hide_secrets(data)
stream.append(data, offset)
stream.size
end
end
def exist?
2019-10-12 21:52:04 +05:30
archived_trace_exist? || live_trace_exist?
end
def archived_trace_exist?
2021-12-11 22:18:48 +05:30
archived?
2019-10-12 21:52:04 +05:30
end
def live_trace_exist?
job.trace_chunks.any? || current_path.present? || old_trace.present?
2017-08-17 22:00:37 +05:30
end
2021-01-03 14:25:43 +05:30
def read(&block)
2020-11-24 15:15:51 +05:30
read_stream(&block)
2021-01-03 14:25:43 +05:30
rescue Errno::ENOENT, ChunkedIO::FailedToGetChunkError
2020-11-24 15:15:51 +05:30
job.reset
read_stream(&block)
2017-08-17 22:00:37 +05:30
end
2019-02-15 15:39:39 +05:30
def write(mode, &blk)
in_write_lock do
unsafe_write!(mode, &blk)
2017-08-17 22:00:37 +05:30
end
end
2021-04-29 21:17:54 +05:30
def erase_trace_chunks!
job.trace_chunks.fast_destroy_all # Destroy chunks of a live trace
end
2017-08-17 22:00:37 +05:30
def erase!
2018-11-08 19:23:39 +05:30
##
# Erase the archived trace
trace_artifact&.destroy!
##
# Erase the live trace
2021-04-29 21:17:54 +05:30
erase_trace_chunks!
2018-11-08 19:23:39 +05:30
FileUtils.rm_f(current_path) if current_path # Remove a trace file of a live trace
job.erase_old_trace! if job.has_old_trace? # Remove a trace in database of a live trace
ensure
@current_path = nil
end
2018-03-17 18:26:18 +05:30
2018-11-08 19:23:39 +05:30
def archive!
2019-02-15 15:39:39 +05:30
in_write_lock do
2018-11-08 19:23:39 +05:30
unsafe_archive!
2017-08-17 22:00:37 +05:30
end
end
2021-11-18 22:05:49 +05:30
def attempt_archive_cleanup!
destroy_any_orphan_trace_data!
end
2020-03-13 15:44:24 +05:30
def update_interval
2021-04-29 21:17:54 +05:30
if being_watched?
UPDATE_FREQUENCY_WHEN_BEING_WATCHED
else
UPDATE_FREQUENCY_DEFAULT
end
2020-03-13 15:44:24 +05:30
end
def being_watched!
Gitlab::Redis::SharedState.with do |redis|
redis.set(being_watched_cache_key, true, ex: WATCH_FLAG_TTL)
end
end
def being_watched?
Gitlab::Redis::SharedState.with do |redis|
redis.exists(being_watched_cache_key)
end
end
2021-01-03 14:25:43 +05:30
def lock(&block)
in_write_lock(&block)
rescue FailedToObtainLockError
raise LockedError, "build trace `#{job.id}` is locked"
end
2018-11-08 19:23:39 +05:30
private
2020-11-24 15:15:51 +05:30
def read_stream
stream = Gitlab::Ci::Trace::Stream.new do
2021-12-11 22:18:48 +05:30
if archived?
2020-11-24 15:15:51 +05:30
trace_artifact.open
elsif job.trace_chunks.any?
Gitlab::Ci::Trace::ChunkedIO.new(job)
elsif current_path
File.open(current_path, "rb")
elsif old_trace
StringIO.new(old_trace)
end
end
yield stream
ensure
stream&.close
end
2019-02-15 15:39:39 +05:30
def unsafe_write!(mode, &blk)
stream = Gitlab::Ci::Trace::Stream.new do
2021-12-11 22:18:48 +05:30
if archived?
2019-02-15 15:39:39 +05:30
raise AlreadyArchivedError, 'Could not write to the archived trace'
elsif current_path
File.open(current_path, mode)
2020-06-23 00:09:42 +05:30
elsif Feature.enabled?(:ci_enable_live_trace, job.project)
2019-02-15 15:39:39 +05:30
Gitlab::Ci::Trace::ChunkedIO.new(job)
else
File.open(ensure_path, mode)
end
end
yield(stream).tap do
job.touch if job.needs_touch?
end
ensure
stream&.close
end
2018-11-08 19:23:39 +05:30
def unsafe_archive!
2018-03-27 19:54:05 +05:30
raise ArchiveError, 'Job is not finished yet' unless job.complete?
2021-12-11 22:18:48 +05:30
archived?.tap do |archived|
2021-11-18 22:05:49 +05:30
destroy_any_orphan_trace_data!
raise AlreadyArchivedError, 'Could not archive again' if archived
end
2021-04-29 21:17:54 +05:30
2018-10-15 14:42:47 +05:30
if job.trace_chunks.any?
Gitlab::Ci::Trace::ChunkedIO.new(job) do |stream|
archive_stream!(stream)
2021-03-11 19:13:27 +05:30
destroy_stream(job) { stream.destroy! }
2018-10-15 14:42:47 +05:30
end
elsif current_path
2018-03-27 19:54:05 +05:30
File.open(current_path) do |stream|
archive_stream!(stream)
FileUtils.rm(current_path)
end
elsif old_trace
StringIO.new(old_trace, 'rb').tap do |stream|
archive_stream!(stream)
job.erase_old_trace!
end
end
end
2021-12-11 22:18:48 +05:30
def archived?
2021-11-11 11:23:49 +05:30
# TODO check checksum to ensure archive completed successfully
# See https://gitlab.com/gitlab-org/gitlab/-/issues/259619
2021-11-18 22:05:49 +05:30
trace_artifact&.archived_trace_exists?
2021-11-11 11:23:49 +05:30
end
2021-11-18 22:05:49 +05:30
def destroy_any_orphan_trace_data!
2021-04-29 21:17:54 +05:30
return unless trace_artifact
2021-12-11 22:18:48 +05:30
if archived?
# An archive file exists, so remove the trace chunks
2021-04-29 21:17:54 +05:30
erase_trace_chunks!
else
2021-12-11 22:18:48 +05:30
# A trace artifact record exists with no archive file
# but an archive was attempted, so cleanup the associated record
2021-04-29 21:17:54 +05:30
trace_artifact.destroy!
end
end
2019-02-15 15:39:39 +05:30
def in_write_lock(&blk)
lock_key = "trace:write:lock:#{job.id}"
in_lock(lock_key, ttl: LOCK_TTL, retries: LOCK_RETRIES, sleep_sec: LOCK_SLEEP, &blk)
end
2018-03-27 19:54:05 +05:30
def archive_stream!(stream)
2021-11-18 22:05:49 +05:30
::Gitlab::Ci::Trace::Archive.new(job, trace_metadata).execute!(stream)
2021-11-11 11:23:49 +05:30
end
def trace_metadata
strong_memoize(:trace_metadata) do
job.ensure_trace_metadata!
2018-03-27 19:54:05 +05:30
end
end
2017-08-17 22:00:37 +05:30
def ensure_path
return current_path if current_path
ensure_directory
default_path
end
def ensure_directory
unless Dir.exist?(default_directory)
FileUtils.mkdir_p(default_directory)
end
end
def current_path
@current_path ||= paths.find do |trace_path|
File.exist?(trace_path)
end
end
def paths
2019-07-31 22:56:46 +05:30
[default_path]
2017-08-17 22:00:37 +05:30
end
def default_directory
File.join(
Settings.gitlab_ci.builds_path,
job.created_at.utc.strftime("%Y_%m"),
job.project_id.to_s
)
end
def default_path
File.join(default_directory, "#{job.id}.log")
end
2018-03-17 18:26:18 +05:30
def trace_artifact
2021-03-11 19:13:27 +05:30
read_trace_artifact(job) { job.job_artifacts_trace }
end
2021-09-04 01:27:46 +05:30
def destroy_stream(build)
if consistent_archived_trace?(build)
2021-11-18 22:05:49 +05:30
::Ci::Build
.sticking
2021-09-04 01:27:46 +05:30
.stick(LOAD_BALANCING_STICKING_NAMESPACE, build.id)
end
2021-03-11 19:13:27 +05:30
yield
end
2021-09-04 01:27:46 +05:30
def read_trace_artifact(build)
if consistent_archived_trace?(build)
2021-11-18 22:05:49 +05:30
::Ci::Build
.sticking
2021-09-04 01:27:46 +05:30
.unstick_or_continue_sticking(LOAD_BALANCING_STICKING_NAMESPACE, build.id)
end
2021-03-11 19:13:27 +05:30
yield
2018-03-17 18:26:18 +05:30
end
2020-03-13 15:44:24 +05:30
2021-09-04 01:27:46 +05:30
def consistent_archived_trace?(build)
::Feature.enabled?(:gitlab_ci_archived_trace_consistent_reads, build.project, default_enabled: false)
end
2020-03-13 15:44:24 +05:30
def being_watched_cache_key
"gitlab:ci:trace:#{job.id}:watched"
end
2017-08-17 22:00:37 +05:30
end
end
end