debian-mirror-gitlab/scripts/trigger-build

#!/usr/bin/env ruby
# frozen_string_literal: true

require 'gitlab'

module Trigger
  def self.ee?
    # Support former project name for `dev`
    %w[gitlab gitlab-ee].include?(ENV['CI_PROJECT_NAME'])
  end

  def self.security?
    %r{\Agitlab-org/security(\z|/)}.match?(ENV['CI_PROJECT_NAMESPACE'])
  end

  def self.non_empty_variable_value(variable)
    variable_value = ENV[variable]

    return if variable_value.nil? || variable_value.empty?

    variable_value
  end

  class Base
    # Can be overridden
    def self.access_token
      ENV['GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN']
    end

    def invoke!(post_comment: false, downstream_job_name: nil)
      pipeline_variables = variables

      puts "Triggering downstream pipeline on #{downstream_project_path}"
      puts "with variables #{pipeline_variables}"

      pipeline = gitlab_client(:downstream).run_trigger(
        downstream_project_path,
        trigger_token,
        ref,
        pipeline_variables)

      puts "Triggered downstream pipeline: #{pipeline.web_url}\n"
      puts "Waiting for downstream pipeline status"

      Trigger::CommitComment.post!(pipeline, gitlab_client(:upstream)) if post_comment
      downstream_job =
        if downstream_job_name
          gitlab_client(:downstream).pipeline_jobs(downstream_project_path, pipeline.id).auto_paginate.find do |potential_job|
            potential_job.name == downstream_job_name
          end
        end

      if downstream_job
        Trigger::Job.new(downstream_project_path, downstream_job.id, gitlab_client(:downstream))
      else
        Trigger::Pipeline.new(downstream_project_path, pipeline.id, gitlab_client(:downstream))
      end
    end

    private

    # Override to trigger and work with pipeline on different GitLab instance
    # type: :downstream -> downstream build and pipeline status
    # type: :upstream -> this project, e.g. for posting comments
    def gitlab_client(type)
      # By default, always use the same client
      @gitlab_client ||= Gitlab.client(
        endpoint: 'https://gitlab.com/api/v4',
        private_token: self.class.access_token
      )
    end

    # Must be overridden
    def downstream_project_path
      raise NotImplementedError
    end

    # Must be overridden
    def ref
      raise NotImplementedError
    end

    # Can be overridden
    def trigger_token
      ENV['CI_JOB_TOKEN']
    end

    # Can be overridden
    def extra_variables
      {}
    end

    # Can be overridden
    def version_param_value(version_file)
      ENV[version_file]&.strip || File.read(version_file).strip
    end

    def variables
      base_variables.merge(extra_variables).merge(version_file_variables)
    end

    def base_variables
      # Use CI_MERGE_REQUEST_SOURCE_BRANCH_SHA for omnibus checkouts due to pipeline for merged results,
      # and fallback to CI_COMMIT_SHA for the `detached` pipelines.
      {
        'GITLAB_REF_SLUG' => ENV['CI_COMMIT_TAG'] ? ENV['CI_COMMIT_REF_NAME'] : ENV['CI_COMMIT_REF_SLUG'],
        'TRIGGERED_USER' => ENV['TRIGGERED_USER'] || ENV['GITLAB_USER_NAME'],
        'TRIGGER_SOURCE' => ENV['CI_JOB_URL'],
        'TOP_UPSTREAM_SOURCE_PROJECT' => ENV['CI_PROJECT_PATH'],
        'TOP_UPSTREAM_SOURCE_JOB' => ENV['CI_JOB_URL'],
        'TOP_UPSTREAM_SOURCE_SHA' => Trigger.non_empty_variable_value('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA') || ENV['CI_COMMIT_SHA'],
        'TOP_UPSTREAM_SOURCE_REF' => ENV['CI_COMMIT_REF_NAME'],
        'TOP_UPSTREAM_MERGE_REQUEST_PROJECT_ID' => ENV['CI_MERGE_REQUEST_PROJECT_ID'],
        'TOP_UPSTREAM_MERGE_REQUEST_IID' => ENV['CI_MERGE_REQUEST_IID']
      }
    end

    # Read version files from all components
    def version_file_variables
      Dir.glob("*_VERSION").each_with_object({}) do |version_file, params|
        params[version_file] = version_param_value(version_file)
      end
    end
  end

  class Omnibus < Base
    private

    def downstream_project_path
      ENV['OMNIBUS_PROJECT_PATH'] || 'gitlab-org/build/omnibus-gitlab-mirror'
    end

    def ref
      ENV['OMNIBUS_BRANCH'] || 'master'
    end

    def extra_variables
      # Use CI_MERGE_REQUEST_SOURCE_BRANCH_SHA for omnibus checkouts due to pipeline for merged results
      # and fallback to CI_COMMIT_SHA for the `detached` pipelines.
      # We also set IMAGE_TAG so the GitLab and QA docker images are tagged with
      # that SHA.
      source_sha = Trigger.non_empty_variable_value('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA') || ENV['CI_COMMIT_SHA']
      {
        'GITLAB_VERSION' => source_sha,
        'IMAGE_TAG' => source_sha,
        'ALTERNATIVE_SOURCES' => 'true',
        'SECURITY_SOURCES' => Trigger.security? ? 'true' : 'false',
        'ee' => Trigger.ee? ? 'true' : 'false',
        'QA_BRANCH' => ENV['QA_BRANCH'] || 'master'
      }
    end
  end

  class CNG < Base
    private

    def downstream_project_path
      ENV['CNG_PROJECT_PATH'] || 'gitlab-org/build/CNG-mirror'
    end

    def ref
      default_ref =
        if ENV['CI_COMMIT_REF_NAME'] =~ /^[\d-]+-stable(-ee)?$/
          ENV['CI_COMMIT_REF_NAME']
        else
          'master'
        end

      ENV['CNG_BRANCH'] || default_ref
    end

    def trigger_token
      ENV['BUILD_TRIGGER_TOKEN']
    end

    def extra_variables
      edition = Trigger.ee? ? 'EE' : 'CE'

      {
        "ee" => Trigger.ee? ? "true" : "false",
        "GITLAB_VERSION" => ENV['CI_COMMIT_SHA'],
        "GITLAB_TAG" => ENV['CI_COMMIT_TAG'],
        "GITLAB_ASSETS_TAG" => ENV['CI_COMMIT_TAG'] ? ENV['CI_COMMIT_REF_NAME'] : ENV['CI_COMMIT_SHA'],
        "FORCE_RAILS_IMAGE_BUILDS" => 'true',
        "#{edition}_PIPELINE" => 'true'
      }
    end

    def version_param_value(_version_file)
      raw_version = super

      # if the version matches semver format, treat it as a tag and prepend `v`
      if raw_version =~ Regexp.compile(/^\d+\.\d+\.\d+(-rc\d+)?(-ee)?$/)
        "v#{raw_version}"
      else
        raw_version
      end
    end
  end

  class Docs < Base
    def self.access_token
      ENV['DOCS_PROJECT_API_TOKEN']
    end

    SUCCESS_MESSAGE = <<~MSG
    => You should now be able to preview your changes under the following URL:

    %<app_url>s

    => For more information, see the documentation
    => https://docs.gitlab.com/ee/development/documentation/index.html#previewing-the-changes-live

    => If something doesn't work, drop a line in the #docs chat channel.
    MSG

    # Create a remote branch in gitlab-docs and immediately cancel the pipeline
    # to avoid race conditions, since a triggered pipeline will also run right
    # after the branch creation. This only happens the very first time a branch
    # is created and will be skipped in subsequent runs. Read more in
    # https://gitlab.com/gitlab-org/gitlab-docs/issues/154.
    #
    def deploy!
      create_remote_branch!
      cancel_latest_pipeline!
      invoke!.wait!
      display_success_message
    end

    #
    # Remove a remote branch in gitlab-docs.
    #
    def cleanup!
      gitlab_client(:downstream).delete_branch(downstream_project_path, ref)
      puts "=> Remote branch '#{downstream_project_path}' deleted"
    end

    private

    def downstream_project_path
      ENV['DOCS_PROJECT_PATH'] || 'gitlab-org/gitlab-docs'
    end

    def ref
      if ENV['CI_MERGE_REQUEST_IID'].nil?
        "docs-preview-#{slug}-#{ENV['CI_COMMIT_REF_SLUG']}"
      else
        "docs-preview-#{slug}-#{ENV['CI_MERGE_REQUEST_IID']}"
      end
    end

    def extra_variables
      {
        "BRANCH_#{slug.upcase}" => ENV['CI_COMMIT_REF_NAME']
      }
    end

    def slug
      case ENV['CI_PROJECT_PATH']
      when 'gitlab-org/gitlab-foss'
        'ce'
      when 'gitlab-org/gitlab'
        'ee'
      when 'gitlab-org/gitlab-runner'
        'runner'
      when 'gitlab-org/omnibus-gitlab'
        'omnibus'
      when 'gitlab-org/charts/gitlab'
        'charts'
      end
    end

    def app_url
      "http://#{ref}.#{ENV['DOCS_REVIEW_APPS_DOMAIN']}/#{slug}"
    end

    def create_remote_branch!
      gitlab_client(:downstream).create_branch(downstream_project_path, ref, 'master')
      puts "=> Remote branch '#{ref}' created"
    rescue Gitlab::Error::BadRequest
      puts "=> Remote branch '#{ref}' already exists!"
    end

    def cancel_latest_pipeline!
      pipelines = nil

      # Wait until the pipeline is started
      loop do
        sleep 1
        puts "=> Waiting for pipeline to start..."
        pipelines = gitlab_client(:downstream).pipelines(downstream_project_path, { ref: ref })
        break if pipelines.any?
      end

      # Get the first pipeline ID which should be the only one for the branch
      pipeline_id = pipelines.first.id

      # Cancel the pipeline
      gitlab_client(:downstream).cancel_pipeline(downstream_project_path, pipeline_id)
    end

    def display_success_message
      format(SUCCESS_MESSAGE, app_url: app_url)
    end
  end

  class DatabaseTesting < Base
    def self.access_token
      ENV['GITLABCOM_DATABASE_TESTING_ACCESS_TOKEN']
    end

    private

    def gitlab_client(type)
      @gitlab_clients ||= {
        downstream: Gitlab.client(
          endpoint: 'https://ops.gitlab.net/api/v4',
          private_token: self.class.access_token
        ),
        upstream: Gitlab.client(
          endpoint: 'https://gitlab.com/api/v4',
          private_token: Base.access_token
        )
      }

      @gitlab_clients[type]
    end

    def trigger_token
      ENV['GITLABCOM_DATABASE_TESTING_TRIGGER_TOKEN']
    end

    def downstream_project_path
      ENV['GITLABCOM_DATABASE_TESTING_PROJECT_PATH'] || 'gitlab-com/database-team/gitlab-com-database-testing'
    end

    def extra_variables
      {
        # Use CI_MERGE_REQUEST_SOURCE_BRANCH_SHA for omnibus checkouts due to pipeline for merged results
        # and fallback to CI_COMMIT_SHA for the `detached` pipelines.
        'GITLAB_COMMIT_SHA' => Trigger.non_empty_variable_value('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA') || ENV['CI_COMMIT_SHA'],
        'TRIGGERED_USER_LOGIN' => ENV['GITLAB_USER_LOGIN']
      }
    end

    def ref
      ENV['GITLABCOM_DATABASE_TESTING_TRIGGER_REF'] || 'master'
    end
  end

  class CommitComment
    def self.post!(downstream_pipeline, gitlab_client)
      gitlab_client.create_commit_comment(
        ENV['CI_PROJECT_PATH'],
        Trigger.non_empty_variable_value('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA') || ENV['CI_COMMIT_SHA'],
        "The [`#{ENV['CI_JOB_NAME']}`](#{ENV['CI_JOB_URL']}) job from pipeline #{ENV['CI_PIPELINE_URL']} triggered #{downstream_pipeline.web_url} downstream.")

    rescue Gitlab::Error::Error => error
      puts "Ignoring the following error: #{error}"
    end
  end

  class Pipeline
    INTERVAL = 60 # seconds
    MAX_DURATION = 3600 * 3 # 3 hours

    def self.unscoped_class_name
      name.split('::').last
    end

    def self.gitlab_api_method_name
      unscoped_class_name.downcase
    end

    def initialize(project, id, gitlab_client)
      @project = project
      @id = id
      @gitlab_client = gitlab_client
      @start_time = Time.now.to_i
    end

    def wait!
      (MAX_DURATION / INTERVAL).times do
        case status
        when :created, :pending, :running
          print "."
          sleep INTERVAL
        when :success
          puts "#{self.class.unscoped_class_name} succeeded in #{duration} minutes!"
          return
        else
          raise "#{self.class.unscoped_class_name} did not succeed!"
        end

        STDOUT.flush
      end

      raise "#{self.class.unscoped_class_name} timed out after waiting for #{duration} minutes!"
    end

    def duration
      (Time.now.to_i - start_time) / 60
    end

    def status
      gitlab_client.public_send(self.class.gitlab_api_method_name, project, id).status.to_sym # rubocop:disable GitlabSecurity/PublicSend
    rescue Gitlab::Error::Error => error
      puts "Ignoring the following error: #{error}"
      # Ignore GitLab API hiccups. If GitLab is really down, we'll hit the job
      # timeout anyway.
      :running
    end

    private

    attr_reader :project, :id, :gitlab_client, :start_time
  end

  Job = Class.new(Pipeline)
end

case ARGV[0]
when 'omnibus'
  Trigger::Omnibus.new.invoke!(post_comment: true, downstream_job_name: 'Trigger:qa-test').wait!
when 'cng'
  Trigger::CNG.new.invoke!.wait!
when 'gitlab-com-database-testing'
  Trigger::DatabaseTesting.new.invoke!
when 'docs'
  docs_trigger = Trigger::Docs.new

  case ARGV[1]
  when 'deploy'
    docs_trigger.deploy!
  when 'cleanup'
    docs_trigger.cleanup!
  else
    puts 'usage: trigger-build docs <deploy|cleanup>'
    exit 1
  end
else
  puts "Please provide a valid option:
  omnibus - Triggers a pipeline that builds the omnibus-gitlab package
  cng - Triggers a pipeline that builds images used by the GitLab helm chart
  gitlab-com-database-testing - Triggers a pipeline that tests database changes on GitLab.com data"
end