debian-mirror-gitlab/scripts/trigger-build.rb

509 lines
15 KiB
Ruby
Raw Normal View History

2018-11-08 19:23:39 +05:30
#!/usr/bin/env ruby
2018-12-13 13:39:08 +05:30
# frozen_string_literal: true
2018-11-08 19:23:39 +05:30
2018-12-05 23:21:45 +05:30
require 'gitlab'
2018-11-08 19:23:39 +05:30
module Trigger
def self.ee?
2019-12-04 20:38:33 +05:30
# Support former project name for `dev`
%w[gitlab gitlab-ee].include?(ENV['CI_PROJECT_NAME'])
2018-11-08 19:23:39 +05:30
end
2020-06-23 00:09:42 +05:30
def self.security?
%r{\Agitlab-org/security(\z|/)}.match?(ENV['CI_PROJECT_NAMESPACE'])
end
2020-04-08 14:13:33 +05:30
def self.non_empty_variable_value(variable)
variable_value = ENV[variable]
return if variable_value.nil? || variable_value.empty?
variable_value
end
2022-04-04 11:22:00 +05:30
def self.variables_for_env_file(variables)
variables.map do |key, value|
%Q(#{key}=#{value})
end.join("\n")
end
2018-12-05 23:21:45 +05:30
class Base
2020-10-24 23:57:45 +05:30
# Can be overridden
def self.access_token
ENV['GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN']
end
2022-07-16 23:28:13 +05:30
def invoke!(downstream_job_name: nil)
2020-03-13 15:44:24 +05:30
pipeline_variables = variables
puts "Triggering downstream pipeline on #{downstream_project_path}"
puts "with variables #{pipeline_variables}"
2022-07-16 23:28:13 +05:30
pipeline = downstream_client.run_trigger(
2018-12-05 23:21:45 +05:30
downstream_project_path,
2018-12-13 13:39:08 +05:30
trigger_token,
2018-12-05 23:21:45 +05:30
ref,
2020-03-13 15:44:24 +05:30
pipeline_variables)
2018-11-08 19:23:39 +05:30
2018-12-13 13:39:08 +05:30
puts "Triggered downstream pipeline: #{pipeline.web_url}\n"
2018-12-05 23:21:45 +05:30
puts "Waiting for downstream pipeline status"
2018-11-08 19:23:39 +05:30
2019-12-26 22:10:19 +05:30
downstream_job =
if downstream_job_name
2022-07-16 23:28:13 +05:30
downstream_client.pipeline_jobs(downstream_project_path, pipeline.id).auto_paginate.find do |potential_job|
2019-12-26 22:10:19 +05:30
potential_job.name == downstream_job_name
end
end
if downstream_job
2022-07-16 23:28:13 +05:30
Trigger::Job.new(downstream_project_path, downstream_job.id, downstream_client)
2019-12-26 22:10:19 +05:30
else
2022-07-16 23:28:13 +05:30
Trigger::Pipeline.new(downstream_project_path, pipeline.id, downstream_client)
2019-12-26 22:10:19 +05:30
end
2018-11-08 19:23:39 +05:30
end
2022-04-04 11:22:00 +05:30
def variables
simple_forwarded_variables.merge(base_variables, extra_variables, version_file_variables)
end
def simple_forwarded_variables
{
'TRIGGER_SOURCE' => ENV['CI_JOB_URL'],
'TOP_UPSTREAM_SOURCE_PROJECT' => ENV['CI_PROJECT_PATH'],
'TOP_UPSTREAM_SOURCE_REF' => ENV['CI_COMMIT_REF_NAME'],
'TOP_UPSTREAM_SOURCE_JOB' => ENV['CI_JOB_URL'],
'TOP_UPSTREAM_MERGE_REQUEST_PROJECT_ID' => ENV['CI_MERGE_REQUEST_PROJECT_ID'],
'TOP_UPSTREAM_MERGE_REQUEST_IID' => ENV['CI_MERGE_REQUEST_IID']
}
end
2018-11-08 19:23:39 +05:30
private
2022-07-16 23:28:13 +05:30
def com_gitlab_client
@com_gitlab_client ||= Gitlab.client(
2021-03-11 19:13:27 +05:30
endpoint: 'https://gitlab.com/api/v4',
private_token: self.class.access_token
)
end
2022-07-16 23:28:13 +05:30
# This client is used for downstream build and pipeline status
# Can be overridden
def downstream_client
com_gitlab_client
end
2018-12-13 13:39:08 +05:30
# Must be overridden
2018-12-05 23:21:45 +05:30
def downstream_project_path
raise NotImplementedError
end
2018-12-13 13:39:08 +05:30
# Must be overridden
2022-07-16 23:28:13 +05:30
def ref_param_name
2018-12-05 23:21:45 +05:30
raise NotImplementedError
end
2022-07-16 23:28:13 +05:30
# Can be overridden
def primary_ref
'main'
end
2020-10-24 23:57:45 +05:30
# Can be overridden
2018-12-13 13:39:08 +05:30
def trigger_token
2020-10-24 23:57:45 +05:30
ENV['CI_JOB_TOKEN']
2018-12-13 13:39:08 +05:30
end
# Can be overridden
2018-12-05 23:21:45 +05:30
def extra_variables
{}
end
2018-12-13 13:39:08 +05:30
# Can be overridden
2018-12-05 23:21:45 +05:30
def version_param_value(version_file)
2020-01-01 13:55:28 +05:30
ENV[version_file]&.strip || File.read(version_file).strip
2018-12-05 23:21:45 +05:30
end
2022-07-16 23:28:13 +05:30
# Can be overridden
def trigger_stable_branch_if_detected?
false
end
def stable_branch?
ENV['CI_COMMIT_REF_NAME'] =~ /^[\d-]+-stable(-ee)?$/
end
def fallback_ref
if trigger_stable_branch_if_detected? && stable_branch?
ENV['CI_COMMIT_REF_NAME'].delete_suffix('-ee')
else
primary_ref
end
end
def ref
ENV.fetch(ref_param_name, fallback_ref)
end
2018-12-05 23:21:45 +05:30
def base_variables
2020-04-08 14:13:33 +05:30
# Use CI_MERGE_REQUEST_SOURCE_BRANCH_SHA for omnibus checkouts due to pipeline for merged results,
# and fallback to CI_COMMIT_SHA for the `detached` pipelines.
2018-11-08 19:23:39 +05:30
{
2019-02-15 15:39:39 +05:30
'GITLAB_REF_SLUG' => ENV['CI_COMMIT_TAG'] ? ENV['CI_COMMIT_REF_NAME'] : ENV['CI_COMMIT_REF_SLUG'],
2018-12-13 13:39:08 +05:30
'TRIGGERED_USER' => ENV['TRIGGERED_USER'] || ENV['GITLAB_USER_NAME'],
2022-04-04 11:22:00 +05:30
'TOP_UPSTREAM_SOURCE_SHA' => Trigger.non_empty_variable_value('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA') || ENV['CI_COMMIT_SHA']
2018-11-08 19:23:39 +05:30
}
end
2018-12-05 23:21:45 +05:30
# Read version files from all components
def version_file_variables
Dir.glob("*_VERSION").each_with_object({}) do |version_file, params|
params[version_file] = version_param_value(version_file)
2018-11-08 19:23:39 +05:30
end
end
end
2018-12-05 23:21:45 +05:30
class Omnibus < Base
2021-10-27 15:23:28 +05:30
def self.access_token
# Default to "Multi-pipeline (from 'gitlab-org/gitlab' 'package-and-qa' job)" at https://gitlab.com/gitlab-org/build/omnibus-gitlab-mirror/-/settings/access_tokens
ENV['OMNIBUS_GITLAB_PROJECT_ACCESS_TOKEN'] || super
end
2018-12-05 23:21:45 +05:30
private
2018-11-08 19:23:39 +05:30
2018-12-05 23:21:45 +05:30
def downstream_project_path
2021-10-27 15:23:28 +05:30
ENV.fetch('OMNIBUS_PROJECT_PATH', 'gitlab-org/build/omnibus-gitlab-mirror')
2018-11-08 19:23:39 +05:30
end
2022-07-16 23:28:13 +05:30
def ref_param_name
'OMNIBUS_BRANCH'
end
def primary_ref
'master'
end
def trigger_stable_branch_if_detected?
true
2018-12-05 23:21:45 +05:30
end
def extra_variables
2021-09-30 23:02:18 +05:30
# Use CI_MERGE_REQUEST_SOURCE_BRANCH_SHA (MR HEAD commit) so that the image is in sync with the assets and QA images.
2021-04-29 21:17:54 +05:30
# See https://docs.gitlab.com/ee/development/testing_guide/end_to_end/index.html#with-pipeline-for-merged-results.
# We also set IMAGE_TAG so the GitLab Docker image is tagged with that SHA.
2020-11-24 15:15:51 +05:30
source_sha = Trigger.non_empty_variable_value('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA') || ENV['CI_COMMIT_SHA']
2021-09-30 23:02:18 +05:30
2018-12-05 23:21:45 +05:30
{
2020-11-24 15:15:51 +05:30
'GITLAB_VERSION' => source_sha,
'IMAGE_TAG' => source_sha,
2021-10-27 15:23:28 +05:30
'QA_IMAGE' => ENV['QA_IMAGE'],
2021-04-29 21:17:54 +05:30
'SKIP_QA_DOCKER' => 'true',
2018-12-05 23:21:45 +05:30
'ALTERNATIVE_SOURCES' => 'true',
2020-06-23 00:09:42 +05:30
'SECURITY_SOURCES' => Trigger.security? ? 'true' : 'false',
2019-02-15 15:39:39 +05:30
'ee' => Trigger.ee? ? 'true' : 'false',
2021-10-27 15:23:28 +05:30
'QA_BRANCH' => ENV['QA_BRANCH'] || 'master',
2021-12-11 22:18:48 +05:30
'CACHE_UPDATE' => ENV['OMNIBUS_GITLAB_CACHE_UPDATE'],
2022-01-26 12:08:38 +05:30
'GITLAB_QA_OPTIONS' => ENV['GITLAB_QA_OPTIONS'],
'QA_TESTS' => ENV['QA_TESTS'],
'ALLURE_JOB_NAME' => ENV['ALLURE_JOB_NAME']
2018-12-05 23:21:45 +05:30
}
end
end
class CNG < Base
2022-04-04 11:22:00 +05:30
def variables
# Delete variables that aren't useful when using native triggers.
super.tap do |hash|
hash.delete('TRIGGER_SOURCE')
hash.delete('TRIGGERED_USER')
end
2021-10-27 15:23:28 +05:30
end
2018-11-08 19:23:39 +05:30
private
2022-07-16 23:28:13 +05:30
def ref_param_name
'CNG_BRANCH'
end
def primary_ref
'master'
end
2018-12-05 23:21:45 +05:30
2022-07-16 23:28:13 +05:30
def trigger_stable_branch_if_detected?
true
2018-12-13 13:39:08 +05:30
end
2018-12-05 23:21:45 +05:30
def extra_variables
2021-09-30 23:02:18 +05:30
# Use CI_MERGE_REQUEST_SOURCE_BRANCH_SHA (MR HEAD commit) so that the image is in sync with the assets and QA images.
source_sha = Trigger.non_empty_variable_value('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA') || ENV['CI_COMMIT_SHA']
2018-12-05 23:21:45 +05:30
{
2022-04-04 11:22:00 +05:30
"TRIGGER_BRANCH" => ref,
2021-09-30 23:02:18 +05:30
"GITLAB_VERSION" => source_sha,
2022-04-04 11:22:00 +05:30
"GITLAB_TAG" => ENV['CI_COMMIT_TAG'], # Always set a value, even an empty string, so that the downstream pipeline can correctly check it.
2021-09-30 23:02:18 +05:30
"GITLAB_ASSETS_TAG" => ENV['CI_COMMIT_TAG'] ? ENV['CI_COMMIT_REF_NAME'] : source_sha,
2019-09-04 21:01:54 +05:30
"FORCE_RAILS_IMAGE_BUILDS" => 'true',
2022-04-04 11:22:00 +05:30
"CE_PIPELINE" => Trigger.ee? ? nil : "true", # Always set a value, even an empty string, so that the downstream pipeline can correctly check it.
"EE_PIPELINE" => Trigger.ee? ? "true" : nil # Always set a value, even an empty string, so that the downstream pipeline can correctly check it.
2018-11-08 19:23:39 +05:30
}
2018-12-05 23:21:45 +05:30
end
2018-11-08 19:23:39 +05:30
2018-12-05 23:21:45 +05:30
def version_param_value(_version_file)
raw_version = super
# if the version matches semver format, treat it as a tag and prepend `v`
if raw_version =~ Regexp.compile(/^\d+\.\d+\.\d+(-rc\d+)?(-ee)?$/)
"v#{raw_version}"
2018-11-08 19:23:39 +05:30
else
2018-12-05 23:21:45 +05:30
raw_version
2018-11-08 19:23:39 +05:30
end
end
2018-12-05 23:21:45 +05:30
end
2018-11-08 19:23:39 +05:30
2020-10-24 23:57:45 +05:30
class Docs < Base
def self.access_token
2021-10-27 15:23:28 +05:30
# Default to "DOCS_PROJECT_API_TOKEN" at https://gitlab.com/gitlab-org/gitlab-docs/-/settings/access_tokens
ENV['DOCS_PROJECT_API_TOKEN'] || super
2020-10-24 23:57:45 +05:30
end
SUCCESS_MESSAGE = <<~MSG
=> You should now be able to preview your changes under the following URL:
%<app_url>s
=> For more information, see the documentation
=> https://docs.gitlab.com/ee/development/documentation/index.html#previewing-the-changes-live
=> If something doesn't work, drop a line in the #docs chat channel.
MSG
def deploy!
invoke!.wait!
display_success_message
end
#
# Remove a remote branch in gitlab-docs.
#
def cleanup!
2022-07-16 23:28:13 +05:30
environment = com_gitlab_client.environments(downstream_project_path, name: downstream_environment).first
2021-04-29 21:17:54 +05:30
return unless environment
2022-07-16 23:28:13 +05:30
environment = com_gitlab_client.stop_environment(downstream_project_path, environment.id)
2021-04-29 21:17:54 +05:30
if environment.state == 'stopped'
2022-07-16 23:28:13 +05:30
puts "=> Downstream environment '#{downstream_environment}' stopped."
2021-04-29 21:17:54 +05:30
else
puts "=> Downstream environment '#{downstream_environment}' failed to stop."
end
2020-10-24 23:57:45 +05:30
end
private
2021-04-29 21:17:54 +05:30
def downstream_environment
"review/#{ref}#{review_slug}"
end
# We prepend the `-` here because we cannot use variable substitution in `environment.name`/`environment.url`
# Some projects (e.g. `omnibus-gitlab`) use this script for branch pipelines, so we fallback to using `CI_COMMIT_REF_SLUG` for those cases.
def review_slug
identifier = ENV['CI_MERGE_REQUEST_IID'] || ENV['CI_COMMIT_REF_SLUG']
"-#{project_slug}-#{identifier}"
end
2020-10-24 23:57:45 +05:30
def downstream_project_path
2021-10-27 15:23:28 +05:30
ENV.fetch('DOCS_PROJECT_PATH', 'gitlab-org/gitlab-docs')
2020-10-24 23:57:45 +05:30
end
2022-07-16 23:28:13 +05:30
def ref_param_name
'DOCS_BRANCH'
2021-04-29 21:17:54 +05:30
end
# `gitlab-org/gitlab-docs` pipeline trigger "Triggered from gitlab-org/gitlab 'review-docs-deploy' job"
def trigger_token
ENV['DOCS_TRIGGER_TOKEN']
2020-10-24 23:57:45 +05:30
end
def extra_variables
{
2021-04-29 21:17:54 +05:30
"BRANCH_#{project_slug.upcase}" => ENV['CI_COMMIT_REF_NAME'],
"REVIEW_SLUG" => review_slug
2020-10-24 23:57:45 +05:30
}
end
2021-04-29 21:17:54 +05:30
def project_slug
2020-10-24 23:57:45 +05:30
case ENV['CI_PROJECT_PATH']
when 'gitlab-org/gitlab-foss'
'ce'
when 'gitlab-org/gitlab'
'ee'
when 'gitlab-org/gitlab-runner'
'runner'
when 'gitlab-org/omnibus-gitlab'
'omnibus'
when 'gitlab-org/charts/gitlab'
'charts'
end
end
2021-04-29 21:17:54 +05:30
# app_url is the URL of the `gitlab-docs` Review App URL defined in
# https://gitlab.com/gitlab-org/gitlab-docs/-/blob/b38038132cf82a24271bbb294dead7c2f529e275/.gitlab-ci.yml#L383
2020-10-24 23:57:45 +05:30
def app_url
2021-04-29 21:17:54 +05:30
"http://#{ref}#{review_slug}.#{ENV['DOCS_REVIEW_APPS_DOMAIN']}/#{project_slug}"
2020-10-24 23:57:45 +05:30
end
def display_success_message
2021-04-29 21:17:54 +05:30
puts format(SUCCESS_MESSAGE, app_url: app_url)
2020-10-24 23:57:45 +05:30
end
end
2021-03-11 19:13:27 +05:30
class DatabaseTesting < Base
2021-09-04 01:27:46 +05:30
IDENTIFIABLE_NOTE_TAG = 'gitlab-org/database-team/gitlab-com-database-testing:identifiable-note'
2022-07-16 23:28:13 +05:30
def invoke!(downstream_job_name: nil)
2021-09-04 01:27:46 +05:30
pipeline = super
2022-05-07 20:08:51 +05:30
project_path = variables['TOP_UPSTREAM_SOURCE_PROJECT']
merge_request_id = variables['TOP_UPSTREAM_MERGE_REQUEST_IID']
2021-09-04 01:27:46 +05:30
comment = "<!-- #{IDENTIFIABLE_NOTE_TAG} --> \nStarted database testing [pipeline](https://ops.gitlab.net/#{downstream_project_path}/-/pipelines/#{pipeline.id}) " \
"(limited access). This comment will be updated once the pipeline has finished running."
2021-09-30 23:02:18 +05:30
# Look for an existing note
2022-07-16 23:28:13 +05:30
db_testing_notes = com_gitlab_client.merge_request_notes(project_path, merge_request_id).auto_paginate.select do |note|
2021-09-04 01:27:46 +05:30
note.body.include?(IDENTIFIABLE_NOTE_TAG)
end
2021-09-30 23:02:18 +05:30
if db_testing_notes.empty?
# This is the first note
2022-07-16 23:28:13 +05:30
note = com_gitlab_client.create_merge_request_note(project_path, merge_request_id, comment)
2021-09-04 01:27:46 +05:30
puts "Posted comment to:\n"
2021-09-30 23:02:18 +05:30
puts "https://gitlab.com/#{project_path}/-/merge_requests/#{merge_request_id}#note_#{note.id}"
2021-09-04 01:27:46 +05:30
end
end
2021-03-11 19:13:27 +05:30
private
2022-07-16 23:28:13 +05:30
def ops_gitlab_client
@ops_gitlab_client ||= Gitlab.client(
endpoint: 'https://ops.gitlab.net/api/v4',
private_token: ENV['GITLABCOM_DATABASE_TESTING_ACCESS_TOKEN']
)
end
2021-03-11 19:13:27 +05:30
2022-07-16 23:28:13 +05:30
def downstream_client
ops_gitlab_client
2021-03-11 19:13:27 +05:30
end
def trigger_token
ENV['GITLABCOM_DATABASE_TESTING_TRIGGER_TOKEN']
end
def downstream_project_path
2021-10-27 15:23:28 +05:30
ENV.fetch('GITLABCOM_DATABASE_TESTING_PROJECT_PATH', 'gitlab-com/database-team/gitlab-com-database-testing')
2021-03-11 19:13:27 +05:30
end
def extra_variables
{
# Use CI_MERGE_REQUEST_SOURCE_BRANCH_SHA for omnibus checkouts due to pipeline for merged results
# and fallback to CI_COMMIT_SHA for the `detached` pipelines.
'GITLAB_COMMIT_SHA' => Trigger.non_empty_variable_value('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA') || ENV['CI_COMMIT_SHA'],
'TRIGGERED_USER_LOGIN' => ENV['GITLAB_USER_LOGIN']
}
end
2022-07-16 23:28:13 +05:30
def ref_param_name
'GITLABCOM_DATABASE_TESTING_TRIGGER_REF'
2021-03-11 19:13:27 +05:30
end
2018-12-13 13:39:08 +05:30
2022-07-16 23:28:13 +05:30
def primary_ref
'master'
2018-11-08 19:23:39 +05:30
end
end
class Pipeline
INTERVAL = 60 # seconds
MAX_DURATION = 3600 * 3 # 3 hours
2021-09-04 01:27:46 +05:30
attr_reader :id
2019-12-26 22:10:19 +05:30
def self.unscoped_class_name
name.split('::').last
end
def self.gitlab_api_method_name
unscoped_class_name.downcase
end
2021-03-11 19:13:27 +05:30
def initialize(project, id, gitlab_client)
2018-12-05 23:21:45 +05:30
@project = project
@id = id
2021-03-11 19:13:27 +05:30
@gitlab_client = gitlab_client
2020-11-24 15:15:51 +05:30
@start_time = Time.now.to_i
2018-11-08 19:23:39 +05:30
end
def wait!
2020-11-24 15:15:51 +05:30
(MAX_DURATION / INTERVAL).times do
2018-11-08 19:23:39 +05:30
case status
when :created, :pending, :running
print "."
sleep INTERVAL
when :success
2019-12-26 22:10:19 +05:30
puts "#{self.class.unscoped_class_name} succeeded in #{duration} minutes!"
2020-11-24 15:15:51 +05:30
return
2018-11-08 19:23:39 +05:30
else
2019-12-26 22:10:19 +05:30
raise "#{self.class.unscoped_class_name} did not succeed!"
2018-11-08 19:23:39 +05:30
end
2021-09-04 01:27:46 +05:30
$stdout.flush
2018-11-08 19:23:39 +05:30
end
2020-11-24 15:15:51 +05:30
raise "#{self.class.unscoped_class_name} timed out after waiting for #{duration} minutes!"
2018-11-08 19:23:39 +05:30
end
def duration
2020-11-24 15:15:51 +05:30
(Time.now.to_i - start_time) / 60
2018-11-08 19:23:39 +05:30
end
def status
2021-03-11 19:13:27 +05:30
gitlab_client.public_send(self.class.gitlab_api_method_name, project, id).status.to_sym # rubocop:disable GitlabSecurity/PublicSend
2018-12-05 23:21:45 +05:30
rescue Gitlab::Error::Error => error
puts "Ignoring the following error: #{error}"
2018-11-08 19:23:39 +05:30
# Ignore GitLab API hiccups. If GitLab is really down, we'll hit the job
# timeout anyway.
:running
end
2020-11-24 15:15:51 +05:30
private
2021-09-04 01:27:46 +05:30
attr_reader :project, :gitlab_client, :start_time
2018-11-08 19:23:39 +05:30
end
2019-12-26 22:10:19 +05:30
Job = Class.new(Pipeline)
2018-11-08 19:23:39 +05:30
end
2022-04-04 11:22:00 +05:30
if $0 == __FILE__
case ARGV[0]
when 'omnibus'
2022-07-16 23:28:13 +05:30
Trigger::Omnibus.new.invoke!(downstream_job_name: 'Trigger:qa-test').wait!
2022-04-04 11:22:00 +05:30
when 'cng'
Trigger::CNG.new.invoke!.wait!
when 'gitlab-com-database-testing'
Trigger::DatabaseTesting.new.invoke!
when 'docs'
docs_trigger = Trigger::Docs.new
case ARGV[1]
when 'deploy'
docs_trigger.deploy!
when 'cleanup'
docs_trigger.cleanup!
else
puts 'usage: trigger-build docs <deploy|cleanup>'
exit 1
end
2020-10-24 23:57:45 +05:30
else
2022-04-04 11:22:00 +05:30
puts "Please provide a valid option:
omnibus - Triggers a pipeline that builds the omnibus-gitlab package
cng - Triggers a pipeline that builds images used by the GitLab helm chart
gitlab-com-database-testing - Triggers a pipeline that tests database changes on GitLab.com data"
2020-10-24 23:57:45 +05:30
end
2018-11-08 19:23:39 +05:30
end