2021-03-11 19:13:27 +05:30
|
|
|
# frozen_string_literal: true
|
|
|
|
|
2016-06-02 11:05:42 +05:30
|
|
|
namespace :gitlab do
|
|
|
|
namespace :db do
|
2020-03-13 15:44:24 +05:30
|
|
|
desc 'GitLab | DB | Manually insert schema migration version'
|
2016-06-02 11:05:42 +05:30
|
|
|
task :mark_migration_complete, [:version] => :environment do |_, args|
|
|
|
|
unless args[:version]
|
2016-06-16 23:09:34 +05:30
|
|
|
puts "Must specify a migration version as an argument".color(:red)
|
2016-06-02 11:05:42 +05:30
|
|
|
exit 1
|
|
|
|
end
|
|
|
|
|
|
|
|
version = args[:version].to_i
|
|
|
|
if version == 0
|
2016-06-16 23:09:34 +05:30
|
|
|
puts "Version '#{args[:version]}' must be a non-zero integer".color(:red)
|
2016-06-02 11:05:42 +05:30
|
|
|
exit 1
|
|
|
|
end
|
|
|
|
|
|
|
|
sql = "INSERT INTO schema_migrations (version) VALUES (#{version})"
|
|
|
|
begin
|
|
|
|
ActiveRecord::Base.connection.execute(sql)
|
2016-06-16 23:09:34 +05:30
|
|
|
puts "Successfully marked '#{version}' as complete".color(:green)
|
2016-06-02 11:05:42 +05:30
|
|
|
rescue ActiveRecord::RecordNotUnique
|
2016-06-16 23:09:34 +05:30
|
|
|
puts "Migration version '#{version}' is already marked complete".color(:yellow)
|
2016-06-02 11:05:42 +05:30
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2020-03-13 15:44:24 +05:30
|
|
|
desc 'GitLab | DB | Drop all tables'
|
2017-08-17 22:00:37 +05:30
|
|
|
task drop_tables: :environment do
|
2016-06-02 11:05:42 +05:30
|
|
|
connection = ActiveRecord::Base.connection
|
2016-08-24 12:49:21 +05:30
|
|
|
|
2019-10-12 21:52:04 +05:30
|
|
|
# In PostgreSQLAdapter, data_sources returns both views and tables, so use
|
|
|
|
# #tables instead
|
|
|
|
tables = connection.tables
|
2019-07-07 11:18:12 +05:30
|
|
|
|
2019-03-02 22:35:43 +05:30
|
|
|
# Removes the entry from the array
|
2016-06-02 11:05:42 +05:30
|
|
|
tables.delete 'schema_migrations'
|
|
|
|
# Truncate schema_migrations to ensure migrations re-run
|
2019-10-12 21:52:04 +05:30
|
|
|
connection.execute('TRUNCATE schema_migrations') if connection.table_exists? 'schema_migrations'
|
2016-06-02 11:05:42 +05:30
|
|
|
|
2021-01-03 14:25:43 +05:30
|
|
|
# Drop any views
|
|
|
|
connection.views.each do |view|
|
|
|
|
connection.execute("DROP VIEW IF EXISTS #{connection.quote_table_name(view)} CASCADE")
|
|
|
|
end
|
|
|
|
|
2016-06-02 11:05:42 +05:30
|
|
|
# Drop tables with cascade to avoid dependent table errors
|
|
|
|
# PG: http://www.postgresql.org/docs/current/static/ddl-depend.html
|
|
|
|
# Add `IF EXISTS` because cascade could have already deleted a table.
|
2016-06-16 23:09:34 +05:30
|
|
|
tables.each { |t| connection.execute("DROP TABLE IF EXISTS #{connection.quote_table_name(t)} CASCADE") }
|
2020-07-28 23:09:34 +05:30
|
|
|
|
|
|
|
# Drop all extra schema objects GitLab owns
|
|
|
|
Gitlab::Database::EXTRA_SCHEMAS.each do |schema|
|
|
|
|
connection.execute("DROP SCHEMA IF EXISTS #{connection.quote_table_name(schema)}")
|
|
|
|
end
|
2016-06-16 23:09:34 +05:30
|
|
|
end
|
|
|
|
|
2020-03-13 15:44:24 +05:30
|
|
|
desc 'GitLab | DB | Configures the database by running migrate, or by loading the schema and seeding if needed'
|
2016-06-16 23:09:34 +05:30
|
|
|
task configure: :environment do
|
2018-11-08 19:23:39 +05:30
|
|
|
# Check if we have existing db tables
|
|
|
|
# The schema_migrations table will still exist if drop_tables was called
|
|
|
|
if ActiveRecord::Base.connection.tables.count > 1
|
2016-06-16 23:09:34 +05:30
|
|
|
Rake::Task['db:migrate'].invoke
|
|
|
|
else
|
2018-12-05 23:21:45 +05:30
|
|
|
# Add post-migrate paths to ensure we mark all migrations as up
|
|
|
|
Gitlab::Database.add_post_migrate_path_to_rails(force: true)
|
2020-04-22 19:07:51 +05:30
|
|
|
Rake::Task['db:structure:load'].invoke
|
2016-06-16 23:09:34 +05:30
|
|
|
Rake::Task['db:seed_fu'].invoke
|
|
|
|
end
|
2016-06-02 11:05:42 +05:30
|
|
|
end
|
2016-08-24 12:49:21 +05:30
|
|
|
|
2021-01-03 14:25:43 +05:30
|
|
|
desc 'GitLab | DB | Run database migrations and print `unattended_migrations_completed` if action taken'
|
|
|
|
task unattended: :environment do
|
|
|
|
no_database = !ActiveRecord::Base.connection.schema_migration.table_exists?
|
|
|
|
needs_migrations = ActiveRecord::Base.connection.migration_context.needs_migration?
|
|
|
|
|
|
|
|
if no_database || needs_migrations
|
|
|
|
Rake::Task['gitlab:db:configure'].invoke
|
|
|
|
puts "unattended_migrations_completed"
|
|
|
|
else
|
|
|
|
puts "unattended_migrations_static"
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2020-03-13 15:44:24 +05:30
|
|
|
desc 'GitLab | DB | Checks if migrations require downtime or not'
|
2016-08-24 12:49:21 +05:30
|
|
|
task :downtime_check, [:ref] => :environment do |_, args|
|
|
|
|
abort 'You must specify a Git reference to compare with' unless args[:ref]
|
|
|
|
|
|
|
|
require 'shellwords'
|
|
|
|
|
|
|
|
ref = Shellwords.escape(args[:ref])
|
|
|
|
|
2017-08-17 22:00:37 +05:30
|
|
|
migrations = `git diff #{ref}.. --diff-filter=A --name-only -- db/migrate`.lines
|
|
|
|
.map { |file| Rails.root.join(file.strip).to_s }
|
|
|
|
.select { |file| File.file?(file) }
|
|
|
|
.select { |file| /\A[0-9]+.*\.rb\z/ =~ File.basename(file) }
|
2016-08-24 12:49:21 +05:30
|
|
|
|
|
|
|
Gitlab::DowntimeCheck.new.check_and_print(migrations)
|
|
|
|
end
|
2019-10-12 21:52:04 +05:30
|
|
|
|
2020-03-13 15:44:24 +05:30
|
|
|
desc 'GitLab | DB | Sets up EE specific database functionality'
|
2019-10-12 21:52:04 +05:30
|
|
|
|
|
|
|
if Gitlab.ee?
|
|
|
|
task setup_ee: %w[geo:db:drop geo:db:create geo:db:schema:load geo:db:migrate]
|
|
|
|
else
|
|
|
|
task :setup_ee
|
|
|
|
end
|
2020-04-22 19:07:51 +05:30
|
|
|
|
|
|
|
desc 'This adjusts and cleans db/structure.sql - it runs after db:structure:dump'
|
|
|
|
task :clean_structure_sql do |task_name|
|
|
|
|
structure_file = 'db/structure.sql'
|
|
|
|
schema = File.read(structure_file)
|
|
|
|
|
|
|
|
File.open(structure_file, 'wb+') do |io|
|
|
|
|
Gitlab::Database::SchemaCleaner.new(schema).clean(io)
|
|
|
|
end
|
|
|
|
|
|
|
|
# Allow this task to be called multiple times, as happens when running db:migrate:redo
|
|
|
|
Rake::Task[task_name].reenable
|
|
|
|
end
|
|
|
|
|
2020-06-23 00:09:42 +05:30
|
|
|
desc 'This dumps GitLab specific database details - it runs after db:structure:dump'
|
|
|
|
task :dump_custom_structure do |task_name|
|
|
|
|
Gitlab::Database::CustomStructure.new.dump
|
|
|
|
|
|
|
|
# Allow this task to be called multiple times, as happens when running db:migrate:redo
|
|
|
|
Rake::Task[task_name].reenable
|
|
|
|
end
|
|
|
|
|
|
|
|
desc 'This loads GitLab specific database details - runs after db:structure:dump'
|
|
|
|
task :load_custom_structure do
|
|
|
|
configuration = Rails.application.config_for(:database)
|
|
|
|
|
|
|
|
ENV['PGHOST'] = configuration['host'] if configuration['host']
|
|
|
|
ENV['PGPORT'] = configuration['port'].to_s if configuration['port']
|
|
|
|
ENV['PGPASSWORD'] = configuration['password'].to_s if configuration['password']
|
|
|
|
ENV['PGUSER'] = configuration['username'].to_s if configuration['username']
|
|
|
|
|
|
|
|
command = 'psql'
|
|
|
|
dump_filepath = Gitlab::Database::CustomStructure.custom_dump_filepath.to_path
|
|
|
|
args = ['-v', 'ON_ERROR_STOP=1', '-q', '-X', '-f', dump_filepath, configuration['database']]
|
|
|
|
|
|
|
|
unless Kernel.system(command, *args)
|
|
|
|
raise "failed to execute:\n#{command} #{args.join(' ')}\n\n" \
|
|
|
|
"Please ensure `#{command}` is installed in your PATH and has proper permissions.\n\n"
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
# Inform Rake that custom tasks should be run every time rake db:structure:dump is run
|
2020-04-22 19:07:51 +05:30
|
|
|
Rake::Task['db:structure:dump'].enhance do
|
|
|
|
Rake::Task['gitlab:db:clean_structure_sql'].invoke
|
2020-06-23 00:09:42 +05:30
|
|
|
Rake::Task['gitlab:db:dump_custom_structure'].invoke
|
|
|
|
end
|
|
|
|
|
|
|
|
# Inform Rake that custom tasks should be run every time rake db:structure:load is run
|
|
|
|
Rake::Task['db:structure:load'].enhance do
|
|
|
|
Rake::Task['gitlab:db:load_custom_structure'].invoke
|
2020-04-22 19:07:51 +05:30
|
|
|
end
|
2020-07-28 23:09:34 +05:30
|
|
|
|
|
|
|
desc 'Create missing dynamic database partitions'
|
|
|
|
task :create_dynamic_partitions do
|
|
|
|
Gitlab::Database::Partitioning::PartitionCreator.new.create_partitions
|
|
|
|
end
|
|
|
|
|
|
|
|
# This is targeted towards deploys and upgrades of GitLab.
|
|
|
|
# Since we're running migrations already at this time,
|
|
|
|
# we also check and create partitions as needed here.
|
|
|
|
Rake::Task['db:migrate'].enhance do
|
|
|
|
Rake::Task['gitlab:db:create_dynamic_partitions'].invoke
|
|
|
|
end
|
|
|
|
|
|
|
|
# When we load the database schema from db/structure.sql
|
|
|
|
# we don't have any dynamic partitions created. We don't really need to
|
|
|
|
# because application initializers/sidekiq take care of that, too.
|
|
|
|
# However, the presence of partitions for a table has influence on their
|
|
|
|
# position in db/structure.sql (which is topologically sorted).
|
|
|
|
#
|
|
|
|
# Other than that it's helpful to create partitions early when bootstrapping
|
|
|
|
# a new installation.
|
|
|
|
Rake::Task['db:structure:load'].enhance do
|
|
|
|
Rake::Task['gitlab:db:create_dynamic_partitions'].invoke
|
|
|
|
end
|
|
|
|
|
|
|
|
# During testing, db:test:load restores the database schema from scratch
|
|
|
|
# which does not include dynamic partitions. We cannot rely on application
|
|
|
|
# initializers here as the application can continue to run while
|
|
|
|
# a rake task reloads the database schema.
|
|
|
|
Rake::Task['db:test:load'].enhance do
|
|
|
|
Rake::Task['gitlab:db:create_dynamic_partitions'].invoke
|
|
|
|
end
|
2020-11-24 15:15:51 +05:30
|
|
|
|
|
|
|
desc 'reindex a regular (non-unique) index without downtime to eliminate bloat'
|
|
|
|
task :reindex, [:index_name] => :environment do |_, args|
|
2021-01-03 14:25:43 +05:30
|
|
|
unless Feature.enabled?(:database_reindexing, type: :ops)
|
|
|
|
puts "This feature (database_reindexing) is currently disabled.".color(:yellow)
|
|
|
|
exit
|
|
|
|
end
|
|
|
|
|
2021-02-22 17:27:13 +05:30
|
|
|
indexes = Gitlab::Database::Reindexing.candidate_indexes
|
|
|
|
|
|
|
|
if identifier = args[:index_name]
|
|
|
|
raise ArgumentError, "Index name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/
|
|
|
|
|
|
|
|
indexes = indexes.where(identifier: identifier)
|
|
|
|
|
|
|
|
raise "Index not found or not supported: #{args[:index_name]}" if indexes.empty?
|
|
|
|
end
|
|
|
|
|
|
|
|
ActiveRecord::Base.logger = Logger.new(STDOUT) if Gitlab::Utils.to_boolean(ENV['LOG_QUERIES_TO_CONSOLE'], default: false)
|
2020-11-24 15:15:51 +05:30
|
|
|
|
2021-01-03 14:25:43 +05:30
|
|
|
Gitlab::Database::Reindexing.perform(indexes)
|
|
|
|
rescue => e
|
|
|
|
Gitlab::AppLogger.error(e)
|
|
|
|
raise
|
2020-11-24 15:15:51 +05:30
|
|
|
end
|
2021-02-22 17:27:13 +05:30
|
|
|
|
|
|
|
desc 'Check if there have been user additions to the database'
|
|
|
|
task active: :environment do
|
|
|
|
if ActiveRecord::Base.connection.migration_context.needs_migration?
|
|
|
|
puts "Migrations pending. Database not active"
|
|
|
|
exit 1
|
|
|
|
end
|
|
|
|
|
|
|
|
# A list of projects that GitLab creates automatically on install/upgrade
|
|
|
|
# gc = Gitlab::CurrentSettings.current_application_settings
|
|
|
|
seed_projects = [Gitlab::CurrentSettings.current_application_settings.self_monitoring_project]
|
|
|
|
|
|
|
|
if (Project.count - seed_projects.count {|x| !x.nil? }).eql?(0)
|
|
|
|
puts "No user created projects. Database not active"
|
|
|
|
exit 1
|
|
|
|
end
|
|
|
|
|
|
|
|
puts "Found user created projects. Database active"
|
|
|
|
exit 0
|
|
|
|
end
|
2021-03-11 19:13:27 +05:30
|
|
|
|
|
|
|
desc 'Run migrations with instrumentation'
|
|
|
|
task :migration_testing, [:result_file] => :environment do |_, args|
|
|
|
|
result_file = args[:result_file] || raise("Please specify result_file argument")
|
|
|
|
raise "File exists already, won't overwrite: #{result_file}" if File.exist?(result_file)
|
|
|
|
|
|
|
|
verbose_was, ActiveRecord::Migration.verbose = ActiveRecord::Migration.verbose, true
|
|
|
|
|
|
|
|
ctx = ActiveRecord::Base.connection.migration_context
|
|
|
|
existing_versions = ctx.get_all_versions.to_set
|
|
|
|
|
|
|
|
pending_migrations = ctx.migrations.reject do |migration|
|
|
|
|
existing_versions.include?(migration.version)
|
|
|
|
end
|
|
|
|
|
|
|
|
instrumentation = Gitlab::Database::Migrations::Instrumentation.new
|
|
|
|
|
|
|
|
pending_migrations.each do |migration|
|
|
|
|
instrumentation.observe(migration.version) do
|
|
|
|
ActiveRecord::Migrator.new(:up, ctx.migrations, ctx.schema_migration, migration.version).run
|
|
|
|
end
|
|
|
|
end
|
|
|
|
ensure
|
|
|
|
if instrumentation
|
|
|
|
File.open(result_file, 'wb+') do |io|
|
|
|
|
io << instrumentation.observations.to_json
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
ActiveRecord::Base.clear_cache!
|
|
|
|
ActiveRecord::Migration.verbose = verbose_was
|
|
|
|
end
|
2016-06-02 11:05:42 +05:30
|
|
|
end
|
|
|
|
end
|