debian-mirror-gitlab/app/services/ci/update_build_state_service.rb

228 lines
5.9 KiB
Ruby
Raw Normal View History

2020-11-24 15:15:51 +05:30
# frozen_string_literal: true
module Ci
class UpdateBuildStateService
2021-01-03 14:25:43 +05:30
include ::Gitlab::Utils::StrongMemoize
include ::Gitlab::ExclusiveLeaseHelpers
Result = Struct.new(:status, :backoff, keyword_init: true)
InvalidTraceError = Class.new(StandardError)
2020-11-24 15:15:51 +05:30
ACCEPT_TIMEOUT = 5.minutes.freeze
attr_reader :build, :params, :metrics
def initialize(build, params, metrics = ::Gitlab::Ci::Trace::Metrics.new)
@build = build
@params = params
@metrics = metrics
end
def execute
2021-01-03 14:25:43 +05:30
unless accept_available?
return update_build_state!
end
ensure_pending_state!
in_build_trace_lock do
process_build_state!
2020-11-24 15:15:51 +05:30
end
end
private
2021-01-03 14:25:43 +05:30
def ensure_pending_state!
pending_state.created_at
end
def process_build_state!
if live_chunks_pending?
if pending_state_outdated?
discard_build_trace!
update_build_state!
else
accept_build_state!
end
else
validate_build_trace!
update_build_state!
2020-11-24 15:15:51 +05:30
end
2021-01-03 14:25:43 +05:30
end
2020-11-24 15:15:51 +05:30
2021-01-03 14:25:43 +05:30
def accept_build_state!
2020-11-24 15:15:51 +05:30
build.trace_chunks.live.find_each do |chunk|
chunk.schedule_to_persist!
end
metrics.increment_trace_operation(operation: :accepted)
2021-01-03 14:25:43 +05:30
::Gitlab::Ci::Runner::Backoff.new(pending_state.created_at).then do |backoff|
Result.new(status: 202, backoff: backoff.to_seconds)
end
2020-11-24 15:15:51 +05:30
end
2021-01-03 14:25:43 +05:30
def validate_build_trace!
return unless has_chunks?
2020-11-24 15:15:51 +05:30
2021-01-03 14:25:43 +05:30
unless live_chunks_pending?
metrics.increment_trace_operation(operation: :finalized)
metrics.observe_migration_duration(pending_state_seconds)
end
2020-11-24 15:15:51 +05:30
2021-01-03 14:25:43 +05:30
::Gitlab::Ci::Trace::Checksum.new(build).then do |checksum|
unless checksum.valid?
metrics.increment_trace_operation(operation: :invalid)
2021-12-11 22:18:48 +05:30
metrics.increment_error_counter(error_reason: :chunks_invalid_checksum)
2020-11-24 15:15:51 +05:30
2021-02-22 17:27:13 +05:30
if checksum.corrupted?
metrics.increment_trace_operation(operation: :corrupted)
2021-12-11 22:18:48 +05:30
metrics.increment_error_counter(error_reason: :chunks_invalid_size)
2021-02-22 17:27:13 +05:30
end
2021-01-03 14:25:43 +05:30
next unless log_invalid_chunks?
::Gitlab::ErrorTracking.log_exception(InvalidTraceError.new,
project_path: build.project.full_path,
build_id: build.id,
state_crc32: checksum.state_crc32,
chunks_crc32: checksum.chunks_crc32,
2021-02-22 17:27:13 +05:30
chunks_count: checksum.chunks_count,
chunks_corrupted: checksum.corrupted?
2021-01-03 14:25:43 +05:30
)
end
2020-11-24 15:15:51 +05:30
end
end
def update_build_state!
case build_state
when 'running'
build.touch if build.needs_touch?
Result.new(status: 200)
when 'success'
build.success!
Result.new(status: 200)
when 'failed'
2021-03-08 18:12:59 +05:30
build.drop_with_exit_code!(params[:failure_reason] || :unknown_failure, params[:exit_code])
2020-11-24 15:15:51 +05:30
Result.new(status: 200)
else
Result.new(status: 400)
end
end
2021-01-03 14:25:43 +05:30
def discard_build_trace!
metrics.increment_trace_operation(operation: :discarded)
end
2020-11-24 15:15:51 +05:30
def accept_available?
!build_running? && has_checksum? && chunks_migration_enabled?
end
2021-01-03 14:25:43 +05:30
def live_chunks_pending?
build.trace_chunks.live.any?
end
def has_chunks?
build.trace_chunks.any?
end
def pending_state_outdated?
pending_state_duration > ACCEPT_TIMEOUT
end
def pending_state_duration
Time.current - pending_state.created_at
end
def pending_state_seconds
pending_state_duration.seconds
2020-11-24 15:15:51 +05:30
end
def build_state
params.dig(:state).to_s
end
def has_checksum?
2021-02-22 17:27:13 +05:30
trace_checksum.present?
2020-11-24 15:15:51 +05:30
end
def build_running?
build_state == 'running'
end
2021-02-22 17:27:13 +05:30
def trace_checksum
params.dig(:output, :checksum) || params.dig(:checksum)
end
def trace_bytesize
params.dig(:output, :bytesize)
end
2021-01-03 14:25:43 +05:30
def pending_state
strong_memoize(:pending_state) { ensure_pending_state }
end
2020-11-24 15:15:51 +05:30
def ensure_pending_state
2021-01-29 00:20:46 +05:30
build_state = Ci::BuildPendingState.safe_find_or_create_by(
2020-11-24 15:15:51 +05:30
build_id: build.id,
state: params.fetch(:state),
2021-02-22 17:27:13 +05:30
trace_checksum: trace_checksum,
trace_bytesize: trace_bytesize,
2021-09-04 01:27:46 +05:30
failure_reason: failure_reason
2020-11-24 15:15:51 +05:30
)
2021-01-29 00:20:46 +05:30
unless build_state.present?
metrics.increment_trace_operation(operation: :conflict)
end
build_state || build.pending_state
2020-11-24 15:15:51 +05:30
end
2021-09-04 01:27:46 +05:30
def failure_reason
reason = params.dig(:failure_reason)
return unless reason
Ci::BuildPendingState.failure_reasons.fetch(reason.to_s, 'unknown_failure')
end
2021-01-03 14:25:43 +05:30
##
# This method is releasing an exclusive lock on a build trace the moment we
# conclude that build status has been written and the build state update
# has been committed to the database.
#
# Because a build state machine schedules a bunch of workers to run after
# build status transition to complete, we do not want to keep the lease
# until all the workers are scheduled because it opens a possibility of
# race conditions happening.
#
# Instead of keeping the lease until the transition is fully done and
# workers are scheduled, we immediately release the lock after the database
# commit happens.
#
def in_build_trace_lock(&block)
build.trace.lock do |_, lease| # rubocop:disable CodeReuse/ActiveRecord
build.run_on_status_commit { lease.cancel }
yield
end
rescue ::Gitlab::Ci::Trace::LockedError
metrics.increment_trace_operation(operation: :locked)
accept_build_state!
end
2020-11-24 15:15:51 +05:30
def chunks_migration_enabled?
2022-01-26 12:08:38 +05:30
::Feature.enabled?(:ci_enable_live_trace, build.project) &&
::Feature.enabled?(:ci_accept_trace, build.project, type: :ops, default_enabled: true)
2020-11-24 15:15:51 +05:30
end
2021-01-03 14:25:43 +05:30
def log_invalid_chunks?
2022-01-26 12:08:38 +05:30
::Feature.enabled?(:ci_trace_log_invalid_chunks, build.project, type: :ops, default_enabled: false)
2021-01-03 14:25:43 +05:30
end
2020-11-24 15:15:51 +05:30
end
end