debian-mirror-gitlab/lib/tasks/gitlab/db.rake

241 lines
9.5 KiB
Ruby
Raw Normal View History

2021-03-11 19:13:27 +05:30
# frozen_string_literal: true
2016-06-02 11:05:42 +05:30
namespace :gitlab do
namespace :db do
2020-03-13 15:44:24 +05:30
desc 'GitLab | DB | Manually insert schema migration version'
2016-06-02 11:05:42 +05:30
task :mark_migration_complete, [:version] => :environment do |_, args|
unless args[:version]
puts "Must specify a migration version as an argument".color(:red)
2016-06-02 11:05:42 +05:30
exit 1
end
version = args[:version].to_i
if version == 0
puts "Version '#{args[:version]}' must be a non-zero integer".color(:red)
2016-06-02 11:05:42 +05:30
exit 1
end
sql = "INSERT INTO schema_migrations (version) VALUES (#{version})"
begin
ActiveRecord::Base.connection.execute(sql)
puts "Successfully marked '#{version}' as complete".color(:green)
2016-06-02 11:05:42 +05:30
rescue ActiveRecord::RecordNotUnique
puts "Migration version '#{version}' is already marked complete".color(:yellow)
2016-06-02 11:05:42 +05:30
end
end
2020-03-13 15:44:24 +05:30
desc 'GitLab | DB | Drop all tables'
2017-08-17 22:00:37 +05:30
task drop_tables: :environment do
2016-06-02 11:05:42 +05:30
connection = ActiveRecord::Base.connection
2016-08-24 12:49:21 +05:30
2019-10-12 21:52:04 +05:30
# In PostgreSQLAdapter, data_sources returns both views and tables, so use
# #tables instead
tables = connection.tables
2019-07-07 11:18:12 +05:30
2019-03-02 22:35:43 +05:30
# Removes the entry from the array
2016-06-02 11:05:42 +05:30
tables.delete 'schema_migrations'
# Truncate schema_migrations to ensure migrations re-run
2019-10-12 21:52:04 +05:30
connection.execute('TRUNCATE schema_migrations') if connection.table_exists? 'schema_migrations'
2016-06-02 11:05:42 +05:30
2021-01-03 14:25:43 +05:30
# Drop any views
connection.views.each do |view|
connection.execute("DROP VIEW IF EXISTS #{connection.quote_table_name(view)} CASCADE")
end
2016-06-02 11:05:42 +05:30
# Drop tables with cascade to avoid dependent table errors
# PG: http://www.postgresql.org/docs/current/static/ddl-depend.html
# Add `IF EXISTS` because cascade could have already deleted a table.
tables.each { |t| connection.execute("DROP TABLE IF EXISTS #{connection.quote_table_name(t)} CASCADE") }
2020-07-28 23:09:34 +05:30
# Drop all extra schema objects GitLab owns
Gitlab::Database::EXTRA_SCHEMAS.each do |schema|
connection.execute("DROP SCHEMA IF EXISTS #{connection.quote_table_name(schema)}")
end
end
2020-03-13 15:44:24 +05:30
desc 'GitLab | DB | Configures the database by running migrate, or by loading the schema and seeding if needed'
task configure: :environment do
2018-11-08 19:23:39 +05:30
# Check if we have existing db tables
# The schema_migrations table will still exist if drop_tables was called
if ActiveRecord::Base.connection.tables.count > 1
Rake::Task['db:migrate'].invoke
else
2018-12-05 23:21:45 +05:30
# Add post-migrate paths to ensure we mark all migrations as up
Gitlab::Database.add_post_migrate_path_to_rails(force: true)
2020-04-22 19:07:51 +05:30
Rake::Task['db:structure:load'].invoke
Rake::Task['db:seed_fu'].invoke
end
2016-06-02 11:05:42 +05:30
end
2016-08-24 12:49:21 +05:30
2021-01-03 14:25:43 +05:30
desc 'GitLab | DB | Run database migrations and print `unattended_migrations_completed` if action taken'
task unattended: :environment do
no_database = !ActiveRecord::Base.connection.schema_migration.table_exists?
needs_migrations = ActiveRecord::Base.connection.migration_context.needs_migration?
if no_database || needs_migrations
Rake::Task['gitlab:db:configure'].invoke
puts "unattended_migrations_completed"
else
puts "unattended_migrations_static"
end
end
2020-03-13 15:44:24 +05:30
desc 'GitLab | DB | Sets up EE specific database functionality'
2019-10-12 21:52:04 +05:30
if Gitlab.ee?
task setup_ee: %w[geo:db:drop geo:db:create geo:db:schema:load geo:db:migrate]
else
task :setup_ee
end
2020-04-22 19:07:51 +05:30
desc 'This adjusts and cleans db/structure.sql - it runs after db:structure:dump'
task :clean_structure_sql do |task_name|
2021-09-30 23:02:18 +05:30
ActiveRecord::Base.configurations.configs_for(env_name: ActiveRecord::Tasks::DatabaseTasks.env).each do |db_config|
structure_file = ActiveRecord::Tasks::DatabaseTasks.dump_filename(db_config.name)
2020-04-22 19:07:51 +05:30
2021-09-30 23:02:18 +05:30
schema = File.read(structure_file)
2020-04-22 19:07:51 +05:30
2021-09-30 23:02:18 +05:30
File.open(structure_file, 'wb+') do |io|
Gitlab::Database::SchemaCleaner.new(schema).clean(io)
end
end
2020-06-23 00:09:42 +05:30
# Allow this task to be called multiple times, as happens when running db:migrate:redo
Rake::Task[task_name].reenable
end
# Inform Rake that custom tasks should be run every time rake db:structure:dump is run
2021-09-04 01:27:46 +05:30
#
# Rails 6.1 deprecates db:structure:dump in favor of db:schema:dump
2020-04-22 19:07:51 +05:30
Rake::Task['db:structure:dump'].enhance do
Rake::Task['gitlab:db:clean_structure_sql'].invoke
2020-06-23 00:09:42 +05:30
end
2021-09-04 01:27:46 +05:30
# Inform Rake that custom tasks should be run every time rake db:schema:dump is run
Rake::Task['db:schema:dump'].enhance do
Rake::Task['gitlab:db:clean_structure_sql'].invoke
end
2020-07-28 23:09:34 +05:30
desc 'Create missing dynamic database partitions'
2021-09-30 23:02:18 +05:30
task create_dynamic_partitions: :environment do
2021-11-11 11:23:49 +05:30
Gitlab::Database::Partitioning.sync_partitions
2020-07-28 23:09:34 +05:30
end
# This is targeted towards deploys and upgrades of GitLab.
# Since we're running migrations already at this time,
# we also check and create partitions as needed here.
Rake::Task['db:migrate'].enhance do
Rake::Task['gitlab:db:create_dynamic_partitions'].invoke
end
# When we load the database schema from db/structure.sql
# we don't have any dynamic partitions created. We don't really need to
# because application initializers/sidekiq take care of that, too.
# However, the presence of partitions for a table has influence on their
# position in db/structure.sql (which is topologically sorted).
#
# Other than that it's helpful to create partitions early when bootstrapping
# a new installation.
2021-09-04 01:27:46 +05:30
#
# Rails 6.1 deprecates db:structure:load in favor of db:schema:load
2020-07-28 23:09:34 +05:30
Rake::Task['db:structure:load'].enhance do
Rake::Task['gitlab:db:create_dynamic_partitions'].invoke
end
2021-09-04 01:27:46 +05:30
Rake::Task['db:schema:load'].enhance do
Rake::Task['gitlab:db:create_dynamic_partitions'].invoke
end
2020-07-28 23:09:34 +05:30
# During testing, db:test:load restores the database schema from scratch
# which does not include dynamic partitions. We cannot rely on application
# initializers here as the application can continue to run while
# a rake task reloads the database schema.
Rake::Task['db:test:load'].enhance do
2021-11-11 11:23:49 +05:30
# Due to bug in `db:test:load` if many DBs are used
# the `ActiveRecord::Base.connection` might be switched to another one
# This is due to `if should_reconnect`:
# https://github.com/rails/rails/blob/a81aeb63a007ede2fe606c50539417dada9030c7/activerecord/lib/active_record/railties/databases.rake#L622
ActiveRecord::Base.establish_connection :main
2020-07-28 23:09:34 +05:30
Rake::Task['gitlab:db:create_dynamic_partitions'].invoke
end
2020-11-24 15:15:51 +05:30
2021-09-30 23:02:18 +05:30
desc 'reindex a regular index without downtime to eliminate bloat'
2020-11-24 15:15:51 +05:30
task :reindex, [:index_name] => :environment do |_, args|
2021-01-03 14:25:43 +05:30
unless Feature.enabled?(:database_reindexing, type: :ops)
puts "This feature (database_reindexing) is currently disabled.".color(:yellow)
exit
end
2021-10-27 15:23:28 +05:30
indexes = Gitlab::Database::PostgresIndex.reindexing_support
2021-02-22 17:27:13 +05:30
if identifier = args[:index_name]
raise ArgumentError, "Index name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/
indexes = indexes.where(identifier: identifier)
raise "Index not found or not supported: #{args[:index_name]}" if indexes.empty?
end
2021-09-04 01:27:46 +05:30
ActiveRecord::Base.logger = Logger.new($stdout) if Gitlab::Utils.to_boolean(ENV['LOG_QUERIES_TO_CONSOLE'], default: false)
2020-11-24 15:15:51 +05:30
2021-10-27 15:23:28 +05:30
# Cleanup leftover temporary indexes from previous, possibly aborted runs (if any)
Gitlab::Database::Reindexing.cleanup_leftovers!
# Hack: Before we do actual reindexing work, create async indexes
Gitlab::Database::AsyncIndexes.create_pending_indexes! if Feature.enabled?(:database_async_index_creation, type: :ops)
2021-01-03 14:25:43 +05:30
Gitlab::Database::Reindexing.perform(indexes)
2021-06-08 01:23:25 +05:30
rescue StandardError => e
2021-01-03 14:25:43 +05:30
Gitlab::AppLogger.error(e)
raise
2020-11-24 15:15:51 +05:30
end
2021-02-22 17:27:13 +05:30
desc 'Check if there have been user additions to the database'
task active: :environment do
if ActiveRecord::Base.connection.migration_context.needs_migration?
puts "Migrations pending. Database not active"
exit 1
end
# A list of projects that GitLab creates automatically on install/upgrade
# gc = Gitlab::CurrentSettings.current_application_settings
seed_projects = [Gitlab::CurrentSettings.current_application_settings.self_monitoring_project]
if (Project.count - seed_projects.count {|x| !x.nil? }).eql?(0)
puts "No user created projects. Database not active"
exit 1
end
puts "Found user created projects. Database active"
exit 0
end
2021-03-11 19:13:27 +05:30
2021-11-18 22:05:49 +05:30
namespace :migration_testing do
desc 'Run migrations with instrumentation'
task up: :environment do
Gitlab::Database::Migrations::Runner.up.run
2021-03-11 19:13:27 +05:30
end
2021-11-18 22:05:49 +05:30
desc 'Run down migrations in current branch with instrumentation'
task down: :environment do
Gitlab::Database::Migrations::Runner.down.run
2021-03-11 19:13:27 +05:30
end
end
2021-06-08 01:23:25 +05:30
desc 'Run all pending batched migrations'
task execute_batched_migrations: :environment do
Gitlab::Database::BackgroundMigration::BatchedMigration.active.queue_order.each do |migration|
Gitlab::AppLogger.info("Executing batched migration #{migration.id} inline")
Gitlab::Database::BackgroundMigration::BatchedMigrationRunner.new.run_entire_migration(migration)
end
end
# Only for development environments,
# we execute pending data migrations inline for convenience.
Rake::Task['db:migrate'].enhance do
Rake::Task['gitlab:db:execute_batched_migrations'].invoke if Rails.env.development?
end
2016-06-02 11:05:42 +05:30
end
end