debian-mirror-gitlab/app/models/ci/build_trace_chunk.rb

167 lines
4.3 KiB
Ruby
Raw Normal View History

2018-11-18 11:00:15 +05:30
# frozen_string_literal: true
2018-10-15 14:42:47 +05:30
module Ci
2019-05-18 00:54:41 +05:30
class BuildTraceChunk < ApplicationRecord
2018-10-15 14:42:47 +05:30
include FastDestroyAll
2018-11-08 19:23:39 +05:30
include ::Gitlab::ExclusiveLeaseHelpers
2018-10-15 14:42:47 +05:30
extend Gitlab::Ci::Model
belongs_to :build, class_name: "Ci::Build", foreign_key: :build_id
default_value_for :data_store, :redis
CHUNK_SIZE = 128.kilobytes
WRITE_LOCK_RETRY = 10
WRITE_LOCK_SLEEP = 0.01.seconds
WRITE_LOCK_TTL = 1.minute
2019-02-15 15:39:39 +05:30
FailedToPersistDataError = Class.new(StandardError)
2018-11-08 19:23:39 +05:30
# Note: The ordering of this enum is related to the precedence of persist store.
2019-05-18 00:54:41 +05:30
# The bottom item takes the highest precedence, and the top item takes the lowest precedence.
2018-10-15 14:42:47 +05:30
enum data_store: {
redis: 1,
2018-11-08 19:23:39 +05:30
database: 2,
fog: 3
2018-10-15 14:42:47 +05:30
}
class << self
2018-11-08 19:23:39 +05:30
def all_stores
@all_stores ||= self.data_stores.keys
2018-10-15 14:42:47 +05:30
end
2018-11-08 19:23:39 +05:30
def persistable_store
# get first available store from the back of the list
all_stores.reverse.find { |store| get_store_class(store).available? }
2018-10-15 14:42:47 +05:30
end
2018-11-08 19:23:39 +05:30
def get_store_class(store)
@stores ||= {}
@stores[store] ||= "Ci::BuildTraceChunks::#{store.capitalize}".constantize.new
2018-10-15 14:42:47 +05:30
end
##
# FastDestroyAll concerns
def begin_fast_destroy
2018-11-08 19:23:39 +05:30
all_stores.each_with_object({}) do |store, result|
relation = public_send(store) # rubocop:disable GitlabSecurity/PublicSend
keys = get_store_class(store).keys(relation)
result[store] = keys if keys.present?
end
2018-10-15 14:42:47 +05:30
end
##
# FastDestroyAll concerns
def finalize_fast_destroy(keys)
2018-11-08 19:23:39 +05:30
keys.each do |store, value|
get_store_class(store).delete_keys(value)
end
2018-10-15 14:42:47 +05:30
end
end
##
# Data is memoized for optimizing #size and #end_offset
def data
@data ||= get_data.to_s
end
def truncate(offset = 0)
raise ArgumentError, 'Offset is out of range' if offset > size || offset < 0
return if offset == size # Skip the following process as it doesn't affect anything
self.append("", offset)
end
def append(new_data, offset)
2018-11-08 19:23:39 +05:30
raise ArgumentError, 'New data is missing' unless new_data
2018-10-15 14:42:47 +05:30
raise ArgumentError, 'Offset is out of range' if offset > size || offset < 0
raise ArgumentError, 'Chunk size overflow' if CHUNK_SIZE < (offset + new_data.bytesize)
2019-02-15 15:39:39 +05:30
in_lock(*lock_params) do # Write operation is atomic
2018-11-08 19:23:39 +05:30
unsafe_set_data!(data.byteslice(0, offset) + new_data)
end
schedule_to_persist if full?
2018-10-15 14:42:47 +05:30
end
def size
data&.bytesize.to_i
end
def start_offset
chunk_index * CHUNK_SIZE
end
def end_offset
start_offset + size
end
def range
(start_offset...end_offset)
end
2018-11-08 19:23:39 +05:30
def persist_data!
2019-02-15 15:39:39 +05:30
in_lock(*lock_params) do # Write operation is atomic
2018-11-08 19:23:39 +05:30
unsafe_persist_to!(self.class.persistable_store)
2018-10-15 14:42:47 +05:30
end
end
private
2018-11-08 19:23:39 +05:30
def unsafe_persist_to!(new_store)
return if data_store == new_store.to_s
2018-10-15 14:42:47 +05:30
2019-02-15 15:39:39 +05:30
current_data = get_data
2018-12-23 12:14:25 +05:30
2019-02-15 15:39:39 +05:30
unless current_data&.bytesize.to_i == CHUNK_SIZE
2019-05-18 00:54:41 +05:30
raise FailedToPersistDataError, 'Data is not fulfilled in a bucket'
2019-01-03 12:48:30 +05:30
end
2018-12-23 12:14:25 +05:30
2019-02-15 15:39:39 +05:30
old_store_class = self.class.get_store_class(data_store)
self.raw_data = nil
self.data_store = new_store
unsafe_set_data!(current_data)
2018-11-08 19:23:39 +05:30
old_store_class.delete_data(self)
2018-10-15 14:42:47 +05:30
end
2018-11-08 19:23:39 +05:30
def get_data
self.class.get_store_class(data_store).data(self)&.force_encoding(Encoding::BINARY) # Redis/Database return UTF-8 string as default
rescue Excon::Error::NotFound
# If the data store is :fog and the file does not exist in the object storage, this method returns nil.
2018-10-15 14:42:47 +05:30
end
2018-11-08 19:23:39 +05:30
def unsafe_set_data!(value)
raise ArgumentError, 'New data size exceeds chunk size' if value.bytesize > CHUNK_SIZE
2018-10-15 14:42:47 +05:30
2018-11-08 19:23:39 +05:30
self.class.get_store_class(data_store).set_data(self, value)
@data = value
2018-10-15 14:42:47 +05:30
2018-11-08 19:23:39 +05:30
save! if changed?
2018-10-15 14:42:47 +05:30
end
2018-11-08 19:23:39 +05:30
def schedule_to_persist
return if data_persisted?
2018-10-15 14:42:47 +05:30
2018-11-08 19:23:39 +05:30
Ci::BuildTraceChunkFlushWorker.perform_async(id)
end
2018-10-15 14:42:47 +05:30
2018-11-08 19:23:39 +05:30
def data_persisted?
!redis?
end
2018-10-15 14:42:47 +05:30
2018-11-08 19:23:39 +05:30
def full?
size == CHUNK_SIZE
end
2018-10-15 14:42:47 +05:30
2018-11-08 19:23:39 +05:30
def lock_params
["trace_write:#{build_id}:chunks:#{chunk_index}",
{ ttl: WRITE_LOCK_TTL,
retries: WRITE_LOCK_RETRY,
sleep_sec: WRITE_LOCK_SLEEP }]
2018-10-15 14:42:47 +05:30
end
end
end