debian-mirror-gitlab/lib/gitlab/database/partitioning/partition_manager.rb

136 lines
4.8 KiB
Ruby
Raw Normal View History

2020-07-28 23:09:34 +05:30
# frozen_string_literal: true
module Gitlab
module Database
module Partitioning
2021-09-30 23:02:18 +05:30
class PartitionManager
2021-10-27 15:23:28 +05:30
UnsafeToDetachPartitionError = Class.new(StandardError)
2020-07-28 23:09:34 +05:30
LEASE_TIMEOUT = 1.minute
2021-09-30 23:02:18 +05:30
MANAGEMENT_LEASE_KEY = 'database_partition_management_%s'
2021-10-27 15:23:28 +05:30
RETAIN_DETACHED_PARTITIONS_FOR = 1.week
2020-07-28 23:09:34 +05:30
2021-11-11 11:23:49 +05:30
def initialize(model)
@model = model
2022-04-04 11:22:00 +05:30
@connection_name = model.connection.pool.db_config.name
2020-07-28 23:09:34 +05:30
end
2021-09-30 23:02:18 +05:30
def sync_partitions
2022-04-04 11:22:00 +05:30
Gitlab::AppLogger.info(
message: "Checking state of dynamic postgres partitions",
table_name: model.table_name,
connection_name: @connection_name
)
2020-07-28 23:09:34 +05:30
2021-11-11 11:23:49 +05:30
only_with_exclusive_lease(model, lease_key: MANAGEMENT_LEASE_KEY) do
2022-07-23 23:45:48 +05:30
model.partitioning_strategy.validate_and_fix
2020-07-28 23:09:34 +05:30
2022-07-23 23:45:48 +05:30
partitions_to_create = missing_partitions
2022-01-26 12:08:38 +05:30
partitions_to_detach = extra_partitions
2022-07-23 23:45:48 +05:30
create(partitions_to_create) unless partitions_to_create.empty?
2022-01-26 12:08:38 +05:30
detach(partitions_to_detach) unless partitions_to_detach.empty?
2020-07-28 23:09:34 +05:30
end
2021-11-11 11:23:49 +05:30
rescue StandardError => e
2022-04-04 11:22:00 +05:30
Gitlab::AppLogger.error(
message: "Failed to create / detach partition(s)",
table_name: model.table_name,
exception_class: e.class,
exception_message: e.message,
connection_name: @connection_name
)
2020-07-28 23:09:34 +05:30
end
private
2021-11-11 11:23:49 +05:30
attr_reader :model
2022-05-07 20:08:51 +05:30
2021-11-11 11:23:49 +05:30
delegate :connection, to: :model
def missing_partitions
2020-07-28 23:09:34 +05:30
return [] unless connection.table_exists?(model.table_name)
model.partitioning_strategy.missing_partitions
end
2021-11-11 11:23:49 +05:30
def extra_partitions
2021-09-30 23:02:18 +05:30
return [] unless connection.table_exists?(model.table_name)
model.partitioning_strategy.extra_partitions
end
def only_with_exclusive_lease(model, lease_key:)
lease = Gitlab::ExclusiveLease.new(lease_key % model.table_name, timeout: LEASE_TIMEOUT)
2020-07-28 23:09:34 +05:30
yield if lease.try_obtain
ensure
lease&.cancel
end
2021-09-30 23:02:18 +05:30
def create(partitions)
2021-11-11 11:23:49 +05:30
# with_lock_retries starts a requires_new transaction most of the time, but not on the last iteration
with_lock_retries do
connection.transaction(requires_new: false) do # so we open a transaction here if not already in progress
2022-03-02 08:16:31 +05:30
# Partitions might not get created (IF NOT EXISTS) so explicit locking will not happen.
# This LOCK TABLE ensures to have exclusive lock as the first step.
connection.execute "LOCK TABLE #{connection.quote_table_name(model.table_name)} IN ACCESS EXCLUSIVE MODE"
2020-07-28 23:09:34 +05:30
partitions.each do |partition|
connection.execute partition.to_sql
2021-10-27 15:23:28 +05:30
Gitlab::AppLogger.info(message: "Created partition",
partition_name: partition.partition_name,
table_name: partition.table)
2020-07-28 23:09:34 +05:30
end
2022-01-26 12:08:38 +05:30
model.partitioning_strategy.after_adding_partitions
2020-07-28 23:09:34 +05:30
end
end
end
2021-09-30 23:02:18 +05:30
def detach(partitions)
2021-11-11 11:23:49 +05:30
# with_lock_retries starts a requires_new transaction most of the time, but not on the last iteration
with_lock_retries do
connection.transaction(requires_new: false) do # so we open a transaction here if not already in progress
2021-09-30 23:02:18 +05:30
partitions.each { |p| detach_one_partition(p) }
end
end
end
def detach_one_partition(partition)
2021-10-27 15:23:28 +05:30
assert_partition_detachable!(partition)
connection.execute partition.to_detach_sql
Postgresql::DetachedPartition.create!(table_name: partition.partition_name,
drop_after: RETAIN_DETACHED_PARTITIONS_FOR.from_now)
2022-04-04 11:22:00 +05:30
Gitlab::AppLogger.info(
message: "Detached Partition",
partition_name: partition.partition_name,
table_name: partition.table,
connection_name: @connection_name
)
2021-10-27 15:23:28 +05:30
end
def assert_partition_detachable!(partition)
parent_table_identifier = "#{connection.current_schema}.#{partition.table}"
if (example_fk = PostgresForeignKey.by_referenced_table_identifier(parent_table_identifier).first)
2022-07-23 23:45:48 +05:30
raise UnsafeToDetachPartitionError, "Cannot detach #{partition.partition_name}, it would block while " \
"checking foreign key #{example_fk.name} on #{example_fk.constrained_table_identifier}"
2021-10-27 15:23:28 +05:30
end
2021-09-30 23:02:18 +05:30
end
2020-07-28 23:09:34 +05:30
def with_lock_retries(&block)
2021-01-03 14:25:43 +05:30
Gitlab::Database::WithLockRetries.new(
2020-07-28 23:09:34 +05:30
klass: self.class,
2021-11-11 11:23:49 +05:30
logger: Gitlab::AppLogger,
connection: connection
2021-01-03 14:25:43 +05:30
).run(&block)
2020-07-28 23:09:34 +05:30
end
end
end
end
end