debian-mirror-gitlab/app/models/ci/build_trace_chunk.rb

317 lines
8.4 KiB
Ruby
Raw Normal View History

2018-11-18 11:00:15 +05:30
# frozen_string_literal: true
2018-10-15 14:42:47 +05:30
module Ci
2021-10-27 15:23:28 +05:30
class BuildTraceChunk < Ci::ApplicationRecord
2021-01-03 14:25:43 +05:30
include ::Comparable
2020-11-24 15:15:51 +05:30
include ::FastDestroyAll
include ::Checksummable
2018-11-08 19:23:39 +05:30
include ::Gitlab::ExclusiveLeaseHelpers
2021-01-03 14:25:43 +05:30
include ::Gitlab::OptimisticLocking
2021-06-08 01:23:25 +05:30
2018-10-15 14:42:47 +05:30
belongs_to :build, class_name: "Ci::Build", foreign_key: :build_id
2023-01-13 00:05:48 +05:30
attribute :data_store, default: :redis_trace_chunks
2018-10-15 14:42:47 +05:30
2020-11-24 15:15:51 +05:30
after_create { metrics.increment_trace_operation(operation: :chunked) }
2018-10-15 14:42:47 +05:30
CHUNK_SIZE = 128.kilobytes
WRITE_LOCK_RETRY = 10
WRITE_LOCK_SLEEP = 0.01.seconds
WRITE_LOCK_TTL = 1.minute
2019-02-15 15:39:39 +05:30
FailedToPersistDataError = Class.new(StandardError)
2021-01-29 00:20:46 +05:30
DATA_STORES = {
2018-10-15 14:42:47 +05:30
redis: 1,
2018-11-08 19:23:39 +05:30
database: 2,
2021-09-04 01:27:46 +05:30
fog: 3,
redis_trace_chunks: 4
2021-01-29 00:20:46 +05:30
}.freeze
2021-04-29 21:17:54 +05:30
STORE_TYPES = DATA_STORES.keys.to_h do |store|
2021-09-04 01:27:46 +05:30
[store, "Ci::BuildTraceChunks::#{store.to_s.camelize}".constantize]
2021-04-29 21:17:54 +05:30
end.freeze
2021-09-04 01:27:46 +05:30
LIVE_STORES = %i[redis redis_trace_chunks].freeze
2021-01-29 00:20:46 +05:30
enum data_store: DATA_STORES
2018-10-15 14:42:47 +05:30
2021-09-04 01:27:46 +05:30
scope :live, -> { where(data_store: LIVE_STORES) }
scope :persisted, -> { where.not(data_store: LIVE_STORES).order(:chunk_index) }
2020-11-24 15:15:51 +05:30
2018-10-15 14:42:47 +05:30
class << self
2018-11-08 19:23:39 +05:30
def all_stores
2021-01-29 00:20:46 +05:30
STORE_TYPES.keys
2018-10-15 14:42:47 +05:30
end
2018-11-08 19:23:39 +05:30
def persistable_store
2021-09-04 01:27:46 +05:30
STORE_TYPES[:fog].available? ? :fog : :database
2018-10-15 14:42:47 +05:30
end
2018-11-08 19:23:39 +05:30
def get_store_class(store)
2021-01-29 00:20:46 +05:30
store = store.to_sym
2021-01-03 14:25:43 +05:30
2021-01-29 00:20:46 +05:30
raise "Unknown store type: #{store}" unless STORE_TYPES.key?(store)
2021-01-03 14:25:43 +05:30
2021-01-29 00:20:46 +05:30
STORE_TYPES[store].new
2018-10-15 14:42:47 +05:30
end
##
# FastDestroyAll concerns
def begin_fast_destroy
2018-11-08 19:23:39 +05:30
all_stores.each_with_object({}) do |store, result|
relation = public_send(store) # rubocop:disable GitlabSecurity/PublicSend
keys = get_store_class(store).keys(relation)
result[store] = keys if keys.present?
end
2018-10-15 14:42:47 +05:30
end
##
# FastDestroyAll concerns
def finalize_fast_destroy(keys)
2018-11-08 19:23:39 +05:30
keys.each do |store, value|
get_store_class(store).delete_keys(value)
end
2018-10-15 14:42:47 +05:30
end
2021-01-03 14:25:43 +05:30
2021-03-11 19:13:27 +05:30
##
# Sometime we need to ensure that the first read goes to a primary
# database, what is especially important in EE. This method does not
# change the behavior in CE.
#
def with_read_consistency(build, &block)
::Gitlab::Database::Consistency
.with_read_consistency(&block)
end
2021-01-03 14:25:43 +05:30
##
# Sometimes we do not want to read raw data. This method makes it easier
# to find attributes that are just metadata excluding raw data.
#
def metadata_attributes
attribute_names - %w[raw_data]
end
2018-10-15 14:42:47 +05:30
end
def data
@data ||= get_data.to_s
end
2021-01-03 14:25:43 +05:30
def crc32
checksum.to_i
end
2018-10-15 14:42:47 +05:30
def truncate(offset = 0)
raise ArgumentError, 'Offset is out of range' if offset > size || offset < 0
return if offset == size # Skip the following process as it doesn't affect anything
2021-09-30 23:02:18 +05:30
self.append(+"", offset)
2018-10-15 14:42:47 +05:30
end
def append(new_data, offset)
2018-11-08 19:23:39 +05:30
raise ArgumentError, 'New data is missing' unless new_data
2020-10-24 23:57:45 +05:30
raise ArgumentError, 'Offset is out of range' if offset < 0 || offset > size
2018-10-15 14:42:47 +05:30
raise ArgumentError, 'Chunk size overflow' if CHUNK_SIZE < (offset + new_data.bytesize)
2021-01-29 00:20:46 +05:30
in_lock(lock_key, **lock_params) { unsafe_append_data!(new_data, offset) }
2018-11-08 19:23:39 +05:30
2020-11-24 15:15:51 +05:30
schedule_to_persist! if full?
2018-10-15 14:42:47 +05:30
end
def size
2020-11-24 15:15:51 +05:30
@size ||= @data&.bytesize || current_store.size(self) || data&.bytesize
2018-10-15 14:42:47 +05:30
end
def start_offset
chunk_index * CHUNK_SIZE
end
def end_offset
start_offset + size
end
def range
(start_offset...end_offset)
end
2020-11-24 15:15:51 +05:30
def schedule_to_persist!
2021-01-03 14:25:43 +05:30
return if flushed?
2020-11-24 15:15:51 +05:30
Ci::BuildTraceChunkFlushWorker.perform_async(id)
end
2021-01-03 14:25:43 +05:30
##
# It is possible that we run into two concurrent migrations. It might
# happen that a chunk gets migrated after being loaded by another worker
# but before the worker acquires a lock to perform the migration.
#
# We are using Redis locking to ensure that we perform this operation
# inside an exclusive lock, but this does not prevent us from running into
# race conditions related to updating a model representation in the
# database. Optimistic locking is another mechanism that help here.
#
# We are using optimistic locking combined with Redis locking to ensure
# that a chunk gets migrated properly.
#
2021-01-29 00:20:46 +05:30
# We are using until_executed deduplication strategy for workers,
# which should prevent duplicated workers running in parallel for the same build trace,
# and causing an exception related to an exclusive lock not being
# acquired
2021-01-03 14:25:43 +05:30
#
def persist_data!
2021-01-29 00:20:46 +05:30
in_lock(lock_key, **lock_params) do # exclusive Redis lock is acquired first
2021-01-03 14:25:43 +05:30
raise FailedToPersistDataError, 'Modifed build trace chunk detected' if has_changes_to_save?
2020-11-24 15:15:51 +05:30
2021-03-11 19:13:27 +05:30
self.class.with_read_consistency(build) do
self.reset.then { |chunk| chunk.unsafe_persist_data! }
2021-01-03 14:25:43 +05:30
end
end
rescue FailedToObtainLockError
metrics.increment_trace_operation(operation: :stalled)
2021-01-29 00:20:46 +05:30
raise FailedToPersistDataError, 'Data migration failed due to a worker duplication'
2021-01-03 14:25:43 +05:30
rescue ActiveRecord::StaleObjectError
raise FailedToPersistDataError, <<~MSG
Data migration race condition detected
store: #{data_store}
build: #{build.id}
index: #{chunk_index}
MSG
2020-11-24 15:15:51 +05:30
end
##
# Build trace chunk is final (the last one that we do not expect to ever
# become full) when a runner submitted a build pending state and there is
# no chunk with higher index in the database.
#
def final?
2021-01-03 14:25:43 +05:30
build.pending_state.present? && chunks_max_index == chunk_index
2018-10-15 14:42:47 +05:30
end
2021-01-03 14:25:43 +05:30
def flushed?
2021-09-04 01:27:46 +05:30
!live?
2021-01-03 14:25:43 +05:30
end
def migrated?
flushed?
end
def live?
2021-09-04 01:27:46 +05:30
LIVE_STORES.include?(data_store.to_sym)
2021-01-03 14:25:43 +05:30
end
def <=>(other)
return unless self.build_id == other.build_id
self.chunk_index <=> other.chunk_index
end
protected
2018-10-15 14:42:47 +05:30
2020-11-24 15:15:51 +05:30
def get_data
# Redis / database return UTF-8 encoded string by default
current_store.data(self)&.force_encoding(Encoding::BINARY)
end
def unsafe_persist_data!(new_store = self.class.persistable_store)
2018-11-08 19:23:39 +05:30
return if data_store == new_store.to_s
2018-10-15 14:42:47 +05:30
2020-11-24 15:15:51 +05:30
current_data = data
old_store_class = current_store
current_size = current_data&.bytesize.to_i
2018-12-23 12:14:25 +05:30
2020-11-24 15:15:51 +05:30
unless current_size == CHUNK_SIZE || final?
2021-01-03 14:25:43 +05:30
raise FailedToPersistDataError, <<~MSG
data is not fulfilled in a bucket
size: #{current_size}
state: #{pending_state?}
max: #{chunks_max_index}
index: #{chunk_index}
MSG
2019-01-03 12:48:30 +05:30
end
2018-12-23 12:14:25 +05:30
2019-02-15 15:39:39 +05:30
self.raw_data = nil
self.data_store = new_store
2021-01-03 14:25:43 +05:30
self.checksum = self.class.crc32(current_data)
2020-11-24 15:15:51 +05:30
##
# We need to so persist data then save a new store identifier before we
# remove data from the previous store to make this operation
# trasnaction-safe. `unsafe_set_data! calls `save!` because of this
# reason.
#
# TODO consider using callbacks and state machine to remove old data
#
2019-02-15 15:39:39 +05:30
unsafe_set_data!(current_data)
2018-11-08 19:23:39 +05:30
old_store_class.delete_data(self)
2018-10-15 14:42:47 +05:30
end
2018-11-08 19:23:39 +05:30
def unsafe_set_data!(value)
raise ArgumentError, 'New data size exceeds chunk size' if value.bytesize > CHUNK_SIZE
2018-10-15 14:42:47 +05:30
2020-10-24 23:57:45 +05:30
current_store.set_data(self, value)
2018-11-08 19:23:39 +05:30
@data = value
2020-10-24 23:57:45 +05:30
@size = value.bytesize
save! if changed?
end
def unsafe_append_data!(value, offset)
new_size = value.bytesize + offset
if new_size > CHUNK_SIZE
raise ArgumentError, 'New data size exceeds chunk size'
end
current_store.append_data(self, value, offset).then do |stored|
2020-11-24 15:15:51 +05:30
metrics.increment_trace_operation(operation: :appended)
2020-10-24 23:57:45 +05:30
raise ArgumentError, 'Trace appended incorrectly' if stored != new_size
end
@data = nil
@size = new_size
2018-10-15 14:42:47 +05:30
2018-11-08 19:23:39 +05:30
save! if changed?
2018-10-15 14:42:47 +05:30
end
2018-11-08 19:23:39 +05:30
def full?
size == CHUNK_SIZE
end
2018-10-15 14:42:47 +05:30
2021-01-03 14:25:43 +05:30
private
def pending_state?
build.pending_state.present?
end
2020-10-24 23:57:45 +05:30
def current_store
self.class.get_store_class(data_store)
end
2021-01-03 14:25:43 +05:30
def chunks_max_index
build.trace_chunks.maximum(:chunk_index).to_i
end
2021-01-29 00:20:46 +05:30
def lock_key
"trace_write:#{build_id}:chunks:#{chunk_index}"
end
2018-11-08 19:23:39 +05:30
def lock_params
2021-01-29 00:20:46 +05:30
{
ttl: WRITE_LOCK_TTL,
retries: WRITE_LOCK_RETRY,
sleep_sec: WRITE_LOCK_SLEEP
}
2018-10-15 14:42:47 +05:30
end
2020-11-24 15:15:51 +05:30
def metrics
@metrics ||= ::Gitlab::Ci::Trace::Metrics.new
end
2018-10-15 14:42:47 +05:30
end
end