debian-mirror-gitlab/scripts/review_apps/automated_cleanup.rb

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

284 lines
10 KiB
Ruby
Raw Normal View History

2018-12-05 23:21:45 +05:30
# frozen_string_literal: true
require 'gitlab'
2020-07-28 23:09:34 +05:30
require_relative File.expand_path('../../tooling/lib/tooling/helm3_client.rb', __dir__)
require_relative File.expand_path('../../tooling/lib/tooling/kubernetes_client.rb', __dir__)
2018-12-05 23:21:45 +05:30
class AutomatedCleanup
2018-12-13 13:39:08 +05:30
attr_reader :project_path, :gitlab_token
DEPLOYMENTS_PER_PAGE = 100
2021-11-18 22:05:49 +05:30
ENVIRONMENT_PREFIX = {
review_app: 'review/',
docs_review_app: 'review-docs/'
}.freeze
2018-12-13 13:39:08 +05:30
IGNORED_HELM_ERRORS = [
'transport is closing',
2020-04-22 19:07:51 +05:30
'error upgrading connection',
'not found'
2018-12-13 13:39:08 +05:30
].freeze
IGNORED_KUBERNETES_ERRORS = [
'NotFound'
].freeze
def self.ee?
2019-12-04 20:38:33 +05:30
# Support former project name for `dev`
%w[gitlab gitlab-ee].include?(ENV['CI_PROJECT_NAME'])
2018-12-13 13:39:08 +05:30
end
2018-12-05 23:21:45 +05:30
2021-03-11 19:13:27 +05:30
# $GITLAB_PROJECT_REVIEW_APP_CLEANUP_API_TOKEN => `Automated Review App Cleanup` project token
def initialize(project_path: ENV['CI_PROJECT_PATH'], gitlab_token: ENV['GITLAB_PROJECT_REVIEW_APP_CLEANUP_API_TOKEN'])
2018-12-05 23:21:45 +05:30
@project_path = project_path
@gitlab_token = gitlab_token
end
def gitlab
@gitlab ||= begin
Gitlab.configure do |config|
config.endpoint = 'https://gitlab.com/api/v4'
# gitlab-bot's token "GitLab review apps cleanup"
config.private_token = gitlab_token
end
Gitlab
end
end
2018-12-13 13:39:08 +05:30
def review_apps_namespace
2020-07-28 23:09:34 +05:30
'review-apps'
2018-12-13 13:39:08 +05:30
end
2018-12-05 23:21:45 +05:30
def helm
2020-07-28 23:09:34 +05:30
@helm ||= Tooling::Helm3Client.new(namespace: review_apps_namespace)
2018-12-05 23:21:45 +05:30
end
def kubernetes
2020-07-28 23:09:34 +05:30
@kubernetes ||= Tooling::KubernetesClient.new(namespace: review_apps_namespace)
2018-12-05 23:21:45 +05:30
end
def perform_gitlab_environment_cleanup!(days_for_stop:, days_for_delete:)
2020-04-08 14:13:33 +05:30
puts "Checking for Review Apps not updated in the last #{days_for_stop} days..."
2018-12-05 23:21:45 +05:30
checked_environments = []
delete_threshold = threshold_time(days: days_for_delete)
stop_threshold = threshold_time(days: days_for_stop)
2019-12-26 22:10:19 +05:30
deployments_look_back_threshold = threshold_time(days: days_for_delete * 5)
releases_to_delete = []
2021-11-18 22:05:49 +05:30
# Delete environments via deployments
2019-12-26 22:10:19 +05:30
gitlab.deployments(project_path, per_page: DEPLOYMENTS_PER_PAGE, sort: 'desc').auto_paginate do |deployment|
break if Time.parse(deployment.created_at) < deployments_look_back_threshold
2018-12-05 23:21:45 +05:30
2018-12-13 13:39:08 +05:30
environment = deployment.environment
2018-12-05 23:21:45 +05:30
2019-12-26 22:10:19 +05:30
next unless environment
2021-11-18 22:05:49 +05:30
next unless environment.name.start_with?(ENVIRONMENT_PREFIX[:review_app])
2018-12-13 13:39:08 +05:30
next if checked_environments.include?(environment.slug)
last_deploy = deployment.created_at
deployed_at = Time.parse(last_deploy)
2018-12-05 23:21:45 +05:30
if deployed_at < delete_threshold
2020-01-01 13:55:28 +05:30
deleted_environment = delete_environment(environment, deployment)
if deleted_environment
2020-07-28 23:09:34 +05:30
release = Tooling::Helm3Client::Release.new(environment.slug, 1, deployed_at.to_s, nil, nil, review_apps_namespace)
2020-01-01 13:55:28 +05:30
releases_to_delete << release
end
2018-12-05 23:21:45 +05:30
else
2020-04-08 14:13:33 +05:30
if deployed_at >= stop_threshold
print_release_state(subject: 'Review App', release_name: environment.slug, release_date: last_deploy, action: 'leaving')
else
environment_state = fetch_environment(environment)&.state
stop_environment(environment, deployment) if environment_state && environment_state != 'stopped'
end
2018-12-05 23:21:45 +05:30
end
2018-12-13 13:39:08 +05:30
checked_environments << environment.slug
2018-12-05 23:21:45 +05:30
end
2019-12-26 22:10:19 +05:30
2021-11-18 22:05:49 +05:30
delete_stopped_environments(environment_type: :review_app, checked_environments: checked_environments, last_updated_threshold: delete_threshold) do |environment|
releases_to_delete << Tooling::Helm3Client::Release.new(environment.slug, 1, environment.updated_at, nil, nil, review_apps_namespace)
end
2019-12-26 22:10:19 +05:30
delete_helm_releases(releases_to_delete)
2018-12-05 23:21:45 +05:30
end
2021-11-11 11:23:49 +05:30
def perform_gitlab_docs_environment_cleanup!(days_for_stop:, days_for_delete:)
puts "Checking for Docs Review Apps not updated in the last #{days_for_stop} days..."
checked_environments = []
stop_threshold = threshold_time(days: days_for_stop)
delete_threshold = threshold_time(days: days_for_delete)
2021-11-18 22:05:49 +05:30
# Delete environments via deployments
2021-11-11 11:23:49 +05:30
gitlab.deployments(project_path, per_page: DEPLOYMENTS_PER_PAGE, sort: 'desc').auto_paginate do |deployment|
environment = deployment.environment
next unless environment
2021-11-18 22:05:49 +05:30
next unless environment.name.start_with?(ENVIRONMENT_PREFIX[:docs_review_app])
2021-11-11 11:23:49 +05:30
next if checked_environments.include?(environment.slug)
last_deploy = deployment.created_at
deployed_at = Time.parse(last_deploy)
if deployed_at < stop_threshold
environment_state = fetch_environment(environment)&.state
stop_environment(environment, deployment) if environment_state && environment_state != 'stopped'
end
2021-11-18 22:05:49 +05:30
delete_environment(environment, deployment) if deployed_at < delete_threshold
2021-11-11 11:23:49 +05:30
checked_environments << environment.slug
end
2021-11-18 22:05:49 +05:30
delete_stopped_environments(environment_type: :docs_review_app, checked_environments: checked_environments, last_updated_threshold: delete_threshold)
2021-11-11 11:23:49 +05:30
end
2018-12-05 23:21:45 +05:30
def perform_helm_releases_cleanup!(days:)
2020-04-22 19:07:51 +05:30
puts "Checking for Helm releases that are failed or not updated in the last #{days} days..."
2018-12-05 23:21:45 +05:30
2020-04-08 14:13:33 +05:30
threshold = threshold_time(days: days)
2018-12-05 23:21:45 +05:30
2019-12-26 22:10:19 +05:30
releases_to_delete = []
2018-12-13 13:39:08 +05:30
helm_releases.each do |release|
2019-12-26 22:10:19 +05:30
# Prevents deleting `dns-gitlab-review-app` releases or other unrelated releases
next unless release.name.start_with?('review-')
2020-05-24 23:13:21 +05:30
if release.status == 'failed' || release.last_update < threshold
2019-12-26 22:10:19 +05:30
releases_to_delete << release
2018-12-05 23:21:45 +05:30
else
print_release_state(subject: 'Release', release_name: release.name, release_date: release.last_update, action: 'leaving')
end
end
2019-12-26 22:10:19 +05:30
delete_helm_releases(releases_to_delete)
2018-12-05 23:21:45 +05:30
end
2021-09-30 23:02:18 +05:30
def perform_stale_namespace_cleanup!(days:)
kubernetes_client = Tooling::KubernetesClient.new(namespace: nil)
kubernetes_client.cleanup_review_app_namespaces(created_before: threshold_time(days: days), wait: false)
end
2021-03-08 18:12:59 +05:30
def perform_stale_pvc_cleanup!(days:)
kubernetes.cleanup_by_created_at(resource_type: 'pvc', created_before: threshold_time(days: days), wait: false)
end
2018-12-13 13:39:08 +05:30
private
2020-04-08 14:13:33 +05:30
def fetch_environment(environment)
gitlab.environment(project_path, environment.id)
rescue Errno::ETIMEDOUT => ex
puts "Failed to fetch '#{environment.name}' / '#{environment.slug}' (##{environment.id}):\n#{ex.message}"
nil
end
2021-11-18 22:05:49 +05:30
def delete_environment(environment, deployment = nil)
release_date = deployment ? deployment.created_at : environment.updated_at
print_release_state(subject: 'Review app', release_name: environment.slug, release_date: release_date, action: 'deleting')
2018-12-13 13:39:08 +05:30
gitlab.delete_environment(project_path, environment.id)
2020-01-01 13:55:28 +05:30
rescue Gitlab::Error::Forbidden
2020-04-08 14:13:33 +05:30
puts "Review app '#{environment.name}' / '#{environment.slug}' (##{environment.id}) is forbidden: skipping it"
2018-12-13 13:39:08 +05:30
end
def stop_environment(environment, deployment)
print_release_state(subject: 'Review app', release_name: environment.slug, release_date: deployment.created_at, action: 'stopping')
gitlab.stop_environment(project_path, environment.id)
2020-01-01 13:55:28 +05:30
rescue Gitlab::Error::Forbidden
2020-04-08 14:13:33 +05:30
puts "Review app '#{environment.name}' / '#{environment.slug}' (##{environment.id}) is forbidden: skipping it"
2018-12-13 13:39:08 +05:30
end
2021-11-18 22:05:49 +05:30
def delete_stopped_environments(environment_type:, checked_environments:, last_updated_threshold:)
gitlab.environments(project_path, per_page: DEPLOYMENTS_PER_PAGE, sort: 'desc', states: 'stopped', search: ENVIRONMENT_PREFIX[environment_type]).auto_paginate do |environment|
next if skip_environment?(environment: environment, checked_environments: checked_environments, last_updated_threshold: last_updated_threshold, environment_type: environment_type)
yield environment if delete_environment(environment)
checked_environments << environment.slug
end
end
def skip_environment?(environment:, checked_environments:, last_updated_threshold:, environment_type:)
return true unless environment.name.start_with?(ENVIRONMENT_PREFIX[environment_type])
return true if checked_environments.include?(environment.slug)
return true if Time.parse(environment.updated_at) > last_updated_threshold
false
end
2018-12-13 13:39:08 +05:30
def helm_releases
2020-04-22 19:07:51 +05:30
args = ['--all', '--date']
2018-12-13 13:39:08 +05:30
helm.releases(args: args)
end
2019-12-26 22:10:19 +05:30
def delete_helm_releases(releases)
return if releases.empty?
releases.each do |release|
print_release_state(subject: 'Release', release_name: release.name, release_status: release.status, release_date: release.last_update, action: 'cleaning')
end
releases_names = releases.map(&:name)
helm.delete(release_name: releases_names)
2021-03-08 18:12:59 +05:30
kubernetes.cleanup_by_release(release_name: releases_names, wait: false)
2019-12-26 22:10:19 +05:30
2020-07-28 23:09:34 +05:30
rescue Tooling::Helm3Client::CommandFailedError => ex
2018-12-13 13:39:08 +05:30
raise ex unless ignore_exception?(ex.message, IGNORED_HELM_ERRORS)
puts "Ignoring the following Helm error:\n#{ex}\n"
2020-07-28 23:09:34 +05:30
rescue Tooling::KubernetesClient::CommandFailedError => ex
2018-12-13 13:39:08 +05:30
raise ex unless ignore_exception?(ex.message, IGNORED_KUBERNETES_ERRORS)
puts "Ignoring the following Kubernetes error:\n#{ex}\n"
end
2018-12-05 23:21:45 +05:30
def threshold_time(days:)
Time.now - days * 24 * 3600
end
2018-12-13 13:39:08 +05:30
def ignore_exception?(exception_message, exceptions_ignored)
exception_message.match?(/(#{exceptions_ignored})/)
end
def print_release_state(subject:, release_name:, release_date:, action:, release_status: nil)
puts "\n#{subject} '#{release_name}' #{"(#{release_status}) " if release_status}was last deployed on #{release_date}: #{action} it.\n"
2018-12-05 23:21:45 +05:30
end
end
def timed(task)
start = Time.now
yield(self)
puts "#{task} finished in #{Time.now - start} seconds.\n"
end
automated_cleanup = AutomatedCleanup.new
2020-04-08 14:13:33 +05:30
timed('Review Apps cleanup') do
automated_cleanup.perform_gitlab_environment_cleanup!(days_for_stop: 5, days_for_delete: 6)
2018-12-05 23:21:45 +05:30
end
2021-11-11 11:23:49 +05:30
timed('Docs Review Apps cleanup') do
automated_cleanup.perform_gitlab_docs_environment_cleanup!(days_for_stop: 20, days_for_delete: 30)
end
2018-12-05 23:21:45 +05:30
puts
timed('Helm releases cleanup') do
2020-04-08 14:13:33 +05:30
automated_cleanup.perform_helm_releases_cleanup!(days: 7)
2018-12-05 23:21:45 +05:30
end
2021-09-30 23:02:18 +05:30
timed('Stale Namespace cleanup') do
automated_cleanup.perform_stale_namespace_cleanup!(days: 14)
end
2021-03-08 18:12:59 +05:30
timed('Stale PVC cleanup') do
automated_cleanup.perform_stale_pvc_cleanup!(days: 30)
end
2018-12-05 23:21:45 +05:30
exit(0)