debian-mirror-gitlab/lib/tasks/gitlab/cleanup.rake

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

181 lines
5.3 KiB
Ruby
Raw Normal View History

2018-11-20 20:47:30 +05:30
# frozen_string_literal: true
require 'set'
2014-09-02 18:07:02 +05:30
namespace :gitlab do
namespace :cleanup do
2015-09-11 14:41:01 +05:30
desc "GitLab | Cleanup | Block users that have been removed in LDAP"
2018-03-17 18:26:18 +05:30
task block_removed_ldap_users: :gitlab_environment do
2014-09-02 18:07:02 +05:30
warn_user_is_not_gitlab
block_flag = ENV['BLOCK']
2015-04-26 12:48:37 +05:30
User.find_each do |user|
next unless user.ldap_user?
2018-03-17 18:26:18 +05:30
2015-04-26 12:48:37 +05:30
print "#{user.name} (#{user.ldap_identity.extern_uid}) ..."
2018-03-17 18:26:18 +05:30
2020-04-08 14:13:33 +05:30
if Gitlab::Auth::Ldap::Access.allowed?(user)
puts " [OK]".color(:green)
2014-09-02 18:07:02 +05:30
else
if block_flag
2015-04-26 12:48:37 +05:30
user.block! unless user.blocked?
puts " [BLOCKED]".color(:red)
2014-09-02 18:07:02 +05:30
else
puts " [NOT IN LDAP]".color(:yellow)
2014-09-02 18:07:02 +05:30
end
end
end
unless block_flag
puts "To block these users run this command with BLOCK=true".color(:yellow)
2014-09-02 18:07:02 +05:30
end
end
2017-08-17 22:00:37 +05:30
2018-11-18 11:00:15 +05:30
desc "GitLab | Cleanup | Clean orphaned project uploads"
task project_uploads: :gitlab_environment do
warn_user_is_not_gitlab
2017-08-17 22:00:37 +05:30
2018-11-18 11:00:15 +05:30
cleaner = Gitlab::Cleanup::ProjectUploads.new(logger: logger)
cleaner.run!(dry_run: dry_run?)
2017-08-17 22:00:37 +05:30
2018-11-18 11:00:15 +05:30
if dry_run?
logger.info "To clean up these files run this command with DRY_RUN=false".color(:yellow)
end
end
2017-08-17 22:00:37 +05:30
2018-11-18 11:00:15 +05:30
desc 'GitLab | Cleanup | Clean orphan remote upload files that do not exist in the db'
task remote_upload_files: :environment do
cleaner = Gitlab::Cleanup::RemoteUploads.new(logger: logger)
cleaner.run!(dry_run: dry_run?)
2017-08-17 22:00:37 +05:30
2018-11-18 11:00:15 +05:30
if dry_run?
logger.info "To cleanup these files run this command with DRY_RUN=false".color(:yellow)
2017-08-17 22:00:37 +05:30
end
end
2018-11-18 11:00:15 +05:30
2019-09-30 21:07:59 +05:30
desc 'GitLab | Cleanup | Clean orphan job artifact files'
task orphan_job_artifact_files: :gitlab_environment do
warn_user_is_not_gitlab
2021-03-11 19:13:27 +05:30
cleaner = Gitlab::Cleanup::OrphanJobArtifactFiles.new(dry_run: dry_run?, niceness: niceness, logger: logger)
2019-09-30 21:07:59 +05:30
cleaner.run!
if dry_run?
logger.info "To clean up these files run this command with DRY_RUN=false".color(:yellow)
end
end
2020-04-22 19:07:51 +05:30
desc 'GitLab | Cleanup | Clean orphan LFS file references'
task orphan_lfs_file_references: :gitlab_environment do
warn_user_is_not_gitlab
project = find_project
unless project
logger.info "Specify the project with PROJECT_ID={number} or PROJECT_PATH={namespace/project-name}".color(:red)
exit
end
cleaner = Gitlab::Cleanup::OrphanLfsFileReferences.new(
project,
dry_run: dry_run?,
2021-03-11 19:13:27 +05:30
logger: logger
2020-04-22 19:07:51 +05:30
)
cleaner.run!
if dry_run?
logger.info "To clean up these files run this command with DRY_RUN=false".color(:yellow)
end
end
desc 'GitLab | Cleanup | Clean orphan LFS files'
task orphan_lfs_files: :gitlab_environment do
warn_user_is_not_gitlab
2021-09-04 01:27:46 +05:30
number_of_removed_files = RemoveUnreferencedLfsObjectsWorker.new.perform
2020-04-22 19:07:51 +05:30
2021-09-04 01:27:46 +05:30
logger.info "Removed unreferenced LFS files: #{number_of_removed_files}".color(:green)
2020-04-22 19:07:51 +05:30
end
2019-10-12 21:52:04 +05:30
namespace :sessions do
desc "GitLab | Cleanup | Sessions | Clean ActiveSession lookup keys"
task active_sessions_lookup_keys: :gitlab_environment do
2022-01-26 12:08:38 +05:30
session_key_pattern = "#{Gitlab::Redis::Sessions::USER_SESSIONS_LOOKUP_NAMESPACE}:*"
2019-10-12 21:52:04 +05:30
last_save_check = Time.at(0)
wait_time = 10.seconds
cursor = 0
total_users_scanned = 0
2022-03-02 08:16:31 +05:30
Gitlab::Redis::Sessions.with do |redis|
2019-10-12 21:52:04 +05:30
begin
cursor, keys = redis.scan(cursor, match: session_key_pattern)
total_users_scanned += keys.count
if last_save_check < Time.now - 1.second
while redis.info('persistence')['rdb_bgsave_in_progress'] == '1'
puts "BGSAVE in progress, waiting #{wait_time} seconds"
sleep(wait_time)
end
last_save_check = Time.now
end
2022-01-26 12:08:38 +05:30
user = Struct.new(:id)
2019-10-12 21:52:04 +05:30
keys.each do |key|
user_id = key.split(':').last
2022-01-26 12:08:38 +05:30
removed = []
active = ActiveSession.cleaned_up_lookup_entries(redis, user.new(user_id), removed)
2019-10-12 21:52:04 +05:30
2022-01-26 12:08:38 +05:30
if removed.any?
puts "deleted #{removed.count} out of #{active.count + removed.count} lookup keys for User ##{user_id}"
2019-10-12 21:52:04 +05:30
end
end
end while cursor.to_i != 0
puts "--- All done! Total number of scanned users: #{total_users_scanned}"
end
end
end
2018-11-18 11:00:15 +05:30
def remove?
ENV['REMOVE'] == 'true'
end
def dry_run?
ENV['DRY_RUN'] != 'false'
end
2019-09-30 21:07:59 +05:30
def debug?
ENV['DEBUG'].present?
end
def niceness
ENV['NICENESS'].presence
end
2020-04-22 19:07:51 +05:30
def find_project
if ENV['PROJECT_ID']
Project.find_by_id(ENV['PROJECT_ID']&.to_i)
elsif ENV['PROJECT_PATH']
Project.find_by_full_path(ENV['PROJECT_PATH'])
end
end
2019-09-30 21:07:59 +05:30
# rubocop:disable Gitlab/RailsLogger
2018-11-18 11:00:15 +05:30
def logger
return @logger if defined?(@logger)
@logger = if Rails.env.development? || Rails.env.production?
2021-09-04 01:27:46 +05:30
Logger.new($stdout).tap do |stdout_logger|
2018-11-18 11:00:15 +05:30
stdout_logger.extend(ActiveSupport::Logger.broadcast(Rails.logger))
2019-09-30 21:07:59 +05:30
stdout_logger.level = debug? ? Logger::DEBUG : Logger::INFO
2018-11-18 11:00:15 +05:30
end
else
Rails.logger
end
end
2019-09-30 21:07:59 +05:30
# rubocop:enable Gitlab/RailsLogger
2014-09-02 18:07:02 +05:30
end
end