255 lines
11 KiB
Ruby
255 lines
11 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
require 'spec_helper'
|
|
|
|
RSpec.describe Gitlab::BackgroundMigration::ProjectNamespaces::BackfillProjectNamespaces, :migration do
|
|
include MigrationsHelpers
|
|
|
|
context 'when migrating data', :aggregate_failures do
|
|
let(:projects) { table(:projects) }
|
|
let(:namespaces) { table(:namespaces) }
|
|
|
|
let(:parent_group1) { namespaces.create!(name: 'parent_group1', path: 'parent_group1', visibility_level: 20, type: 'Group') }
|
|
let(:parent_group2) { namespaces.create!(name: 'test1', path: 'test1', runners_token: 'my-token1', project_creation_level: 1, visibility_level: 20, type: 'Group') }
|
|
|
|
let(:parent_group1_project) { projects.create!(name: 'parent_group1_project', path: 'parent_group1_project', namespace_id: parent_group1.id, visibility_level: 20) }
|
|
let(:parent_group2_project) { projects.create!(name: 'parent_group2_project', path: 'parent_group2_project', namespace_id: parent_group2.id, visibility_level: 20) }
|
|
|
|
let(:child_nodes_count) { 2 }
|
|
let(:tree_depth) { 3 }
|
|
|
|
let(:backfilled_namespace) { nil }
|
|
|
|
before do
|
|
BackfillProjectNamespaces::TreeGenerator.new(namespaces, projects, [parent_group1, parent_group2], child_nodes_count, tree_depth).build_tree
|
|
end
|
|
|
|
describe '#up' do
|
|
shared_examples 'back-fill project namespaces' do
|
|
it 'back-fills all project namespaces' do
|
|
start_id = ::Project.minimum(:id)
|
|
end_id = ::Project.maximum(:id)
|
|
projects_count = ::Project.count
|
|
batches_count = (projects_count / described_class::BATCH_SIZE.to_f).ceil
|
|
project_namespaces_count = ::Namespace.where(type: 'Project').count
|
|
migration = described_class.new
|
|
|
|
expect(projects_count).not_to eq(project_namespaces_count)
|
|
expect(migration).to receive(:batch_insert_namespaces).exactly(batches_count).and_call_original
|
|
expect(migration).to receive(:batch_update_projects).exactly(batches_count).and_call_original
|
|
expect(migration).to receive(:batch_update_project_namespaces_traversal_ids).exactly(batches_count).and_call_original
|
|
|
|
expect { migration.perform(start_id, end_id, nil, 'up') }.to change(Namespace.where(type: 'Project'), :count)
|
|
|
|
expect(projects_count).to eq(::Namespace.where(type: 'Project').count)
|
|
check_projects_in_sync_with(Namespace.where(type: 'Project'))
|
|
end
|
|
|
|
context 'when passing specific group as parameter' do
|
|
let(:backfilled_namespace) { parent_group1 }
|
|
|
|
it 'back-fills project namespaces for the specified group hierarchy' do
|
|
backfilled_namespace_projects = base_ancestor(backfilled_namespace).first.all_projects
|
|
start_id = backfilled_namespace_projects.minimum(:id)
|
|
end_id = backfilled_namespace_projects.maximum(:id)
|
|
group_projects_count = backfilled_namespace_projects.count
|
|
batches_count = (group_projects_count / described_class::BATCH_SIZE.to_f).ceil
|
|
project_namespaces_in_hierarchy = project_namespaces_in_hierarchy(base_ancestor(backfilled_namespace))
|
|
|
|
migration = described_class.new
|
|
|
|
expect(project_namespaces_in_hierarchy.count).to eq(0)
|
|
expect(migration).to receive(:batch_insert_namespaces).exactly(batches_count).and_call_original
|
|
expect(migration).to receive(:batch_update_projects).exactly(batches_count).and_call_original
|
|
expect(migration).to receive(:batch_update_project_namespaces_traversal_ids).exactly(batches_count).and_call_original
|
|
|
|
expect(group_projects_count).to eq(14)
|
|
expect(project_namespaces_in_hierarchy.count).to eq(0)
|
|
|
|
migration.perform(start_id, end_id, backfilled_namespace.id, 'up')
|
|
|
|
expect(project_namespaces_in_hierarchy.count).to eq(14)
|
|
check_projects_in_sync_with(project_namespaces_in_hierarchy)
|
|
end
|
|
end
|
|
|
|
context 'when projects already have project namespaces' do
|
|
before do
|
|
hierarchy1_projects = base_ancestor(parent_group1).first.all_projects
|
|
start_id = hierarchy1_projects.minimum(:id)
|
|
end_id = hierarchy1_projects.maximum(:id)
|
|
|
|
described_class.new.perform(start_id, end_id, parent_group1.id, 'up')
|
|
end
|
|
|
|
it 'does not duplicate project namespaces' do
|
|
# check there are already some project namespaces but not for all
|
|
projects_count = ::Project.count
|
|
start_id = ::Project.minimum(:id)
|
|
end_id = ::Project.maximum(:id)
|
|
batches_count = (projects_count / described_class::BATCH_SIZE.to_f).ceil
|
|
project_namespaces = ::Namespace.where(type: 'Project')
|
|
migration = described_class.new
|
|
|
|
expect(project_namespaces_in_hierarchy(base_ancestor(parent_group1)).count).to be >= 14
|
|
expect(project_namespaces_in_hierarchy(base_ancestor(parent_group2)).count).to eq(0)
|
|
expect(projects_count).not_to eq(project_namespaces.count)
|
|
|
|
# run migration again to test we do not generate extra project namespaces
|
|
expect(migration).to receive(:batch_insert_namespaces).exactly(batches_count).and_call_original
|
|
expect(migration).to receive(:batch_update_projects).exactly(batches_count).and_call_original
|
|
expect(migration).to receive(:batch_update_project_namespaces_traversal_ids).exactly(batches_count).and_call_original
|
|
|
|
expect { migration.perform(start_id, end_id, nil, 'up') }.to change(project_namespaces, :count).by(14)
|
|
|
|
expect(projects_count).to eq(project_namespaces.count)
|
|
end
|
|
end
|
|
end
|
|
|
|
it 'checks no project namespaces exist in the defined hierarchies' do
|
|
hierarchy1_project_namespaces = project_namespaces_in_hierarchy(base_ancestor(parent_group1))
|
|
hierarchy2_project_namespaces = project_namespaces_in_hierarchy(base_ancestor(parent_group2))
|
|
hierarchy1_projects_count = base_ancestor(parent_group1).first.all_projects.count
|
|
hierarchy2_projects_count = base_ancestor(parent_group2).first.all_projects.count
|
|
|
|
expect(hierarchy1_project_namespaces).to be_empty
|
|
expect(hierarchy2_project_namespaces).to be_empty
|
|
expect(hierarchy1_projects_count).to eq(14)
|
|
expect(hierarchy2_projects_count).to eq(14)
|
|
end
|
|
|
|
context 'back-fill project namespaces in a single batch' do
|
|
it_behaves_like 'back-fill project namespaces'
|
|
end
|
|
|
|
context 'back-fill project namespaces in batches' do
|
|
before do
|
|
stub_const("#{described_class.name}::BATCH_SIZE", 2)
|
|
end
|
|
|
|
it_behaves_like 'back-fill project namespaces'
|
|
end
|
|
end
|
|
|
|
describe '#down' do
|
|
before do
|
|
start_id = ::Project.minimum(:id)
|
|
end_id = ::Project.maximum(:id)
|
|
# back-fill first
|
|
described_class.new.perform(start_id, end_id, nil, 'up')
|
|
end
|
|
|
|
shared_examples 'cleanup project namespaces' do
|
|
it 'removes project namespaces' do
|
|
projects_count = ::Project.count
|
|
start_id = ::Project.minimum(:id)
|
|
end_id = ::Project.maximum(:id)
|
|
migration = described_class.new
|
|
batches_count = (projects_count / described_class::BATCH_SIZE.to_f).ceil
|
|
|
|
expect(projects_count).to be > 0
|
|
expect(projects_count).to eq(::Namespace.where(type: 'Project').count)
|
|
|
|
expect(migration).to receive(:nullify_project_namespaces_in_projects).exactly(batches_count).and_call_original
|
|
expect(migration).to receive(:delete_project_namespace_records).exactly(batches_count).and_call_original
|
|
|
|
migration.perform(start_id, end_id, nil, 'down')
|
|
|
|
expect(::Project.count).to be > 0
|
|
expect(::Namespace.where(type: 'Project').count).to eq(0)
|
|
end
|
|
|
|
context 'when passing specific group as parameter' do
|
|
let(:backfilled_namespace) { parent_group1 }
|
|
|
|
it 'removes project namespaces only for the specific group hierarchy' do
|
|
backfilled_namespace_projects = base_ancestor(backfilled_namespace).first.all_projects
|
|
start_id = backfilled_namespace_projects.minimum(:id)
|
|
end_id = backfilled_namespace_projects.maximum(:id)
|
|
group_projects_count = backfilled_namespace_projects.count
|
|
batches_count = (group_projects_count / described_class::BATCH_SIZE.to_f).ceil
|
|
project_namespaces_in_hierarchy = project_namespaces_in_hierarchy(base_ancestor(backfilled_namespace))
|
|
migration = described_class.new
|
|
|
|
expect(project_namespaces_in_hierarchy.count).to eq(14)
|
|
expect(migration).to receive(:nullify_project_namespaces_in_projects).exactly(batches_count).and_call_original
|
|
expect(migration).to receive(:delete_project_namespace_records).exactly(batches_count).and_call_original
|
|
|
|
migration.perform(start_id, end_id, backfilled_namespace.id, 'down')
|
|
|
|
expect(::Namespace.where(type: 'Project').count).to be > 0
|
|
expect(project_namespaces_in_hierarchy.count).to eq(0)
|
|
end
|
|
end
|
|
end
|
|
|
|
context 'cleanup project namespaces in a single batch' do
|
|
it_behaves_like 'cleanup project namespaces'
|
|
end
|
|
|
|
context 'cleanup project namespaces in batches' do
|
|
before do
|
|
stub_const("#{described_class.name}::BATCH_SIZE", 2)
|
|
end
|
|
|
|
it_behaves_like 'cleanup project namespaces'
|
|
end
|
|
end
|
|
end
|
|
|
|
def base_ancestor(ancestor)
|
|
::Namespace.where(id: ancestor.id)
|
|
end
|
|
|
|
def project_namespaces_in_hierarchy(base_node)
|
|
Gitlab::ObjectHierarchy.new(base_node).base_and_descendants.where(type: 'Project')
|
|
end
|
|
|
|
def check_projects_in_sync_with(namespaces)
|
|
project_namespaces_attrs = namespaces.order(:id).pluck(:id, :name, :path, :parent_id, :visibility_level, :shared_runners_enabled)
|
|
corresponding_projects_attrs = Project.where(project_namespace_id: project_namespaces_attrs.map(&:first))
|
|
.order(:project_namespace_id).pluck(:project_namespace_id, :name, :path, :namespace_id, :visibility_level, :shared_runners_enabled)
|
|
|
|
expect(project_namespaces_attrs).to eq(corresponding_projects_attrs)
|
|
end
|
|
end
|
|
|
|
module BackfillProjectNamespaces
|
|
class TreeGenerator
|
|
def initialize(namespaces, projects, parent_nodes, child_nodes_count, tree_depth)
|
|
parent_nodes_ids = parent_nodes.map(&:id)
|
|
|
|
@namespaces = namespaces
|
|
@projects = projects
|
|
@subgroups_depth = tree_depth
|
|
@resource_count = child_nodes_count
|
|
@all_groups = [parent_nodes_ids]
|
|
end
|
|
|
|
def build_tree
|
|
(1..@subgroups_depth).each do |level|
|
|
parent_level = level - 1
|
|
current_level = level
|
|
parent_groups = @all_groups[parent_level]
|
|
|
|
parent_groups.each do |parent_id|
|
|
@resource_count.times do |i|
|
|
group_path = "child#{i}_level#{level}"
|
|
project_path = "project#{i}_level#{level}"
|
|
sub_group = @namespaces.create!(name: group_path, path: group_path, parent_id: parent_id, visibility_level: 20, type: 'Group')
|
|
@projects.create!(name: project_path, path: project_path, namespace_id: sub_group.id, visibility_level: 20)
|
|
|
|
track_group_id(current_level, sub_group.id)
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
def track_group_id(depth_level, group_id)
|
|
@all_groups[depth_level] ||= []
|
|
@all_groups[depth_level] << group_id
|
|
end
|
|
end
|
|
end
|