debian-mirror-gitlab/app/models/ci/pipeline.rb

570 lines
15 KiB
Ruby
Raw Normal View History

2015-09-25 12:07:36 +05:30
module Ci
class Pipeline < ActiveRecord::Base
2018-03-17 18:26:18 +05:30
extend Gitlab::Ci::Model
2016-09-29 09:46:39 +05:30
include HasStatus
include Importable
2016-11-03 12:29:30 +05:30
include AfterCommitQueue
2017-08-17 22:00:37 +05:30
include Presentable
2018-03-17 18:26:18 +05:30
include Gitlab::OptimisticLocking
2018-05-09 12:01:36 +05:30
include Gitlab::Utils::StrongMemoize
2015-10-24 18:46:33 +05:30
2018-03-17 18:26:18 +05:30
belongs_to :project, inverse_of: :pipelines
2016-08-24 12:49:21 +05:30
belongs_to :user
2017-08-17 22:00:37 +05:30
belongs_to :auto_canceled_by, class_name: 'Ci::Pipeline'
belongs_to :pipeline_schedule, class_name: 'Ci::PipelineSchedule'
2017-09-10 17:25:29 +05:30
has_many :stages
2018-03-27 19:54:05 +05:30
has_many :statuses, class_name: 'CommitStatus', foreign_key: :commit_id, inverse_of: :pipeline
2018-05-09 12:01:36 +05:30
has_many :builds, foreign_key: :commit_id, inverse_of: :pipeline
2017-09-10 17:25:29 +05:30
has_many :trigger_requests, dependent: :destroy, foreign_key: :commit_id # rubocop:disable Cop/ActiveRecordDependent
has_many :variables, class_name: 'Ci::PipelineVariable'
# Merge requests for which the current pipeline is running against
# the merge request's latest commit.
has_many :merge_requests, foreign_key: "head_pipeline_id"
2017-08-17 22:00:37 +05:30
has_many :pending_builds, -> { pending }, foreign_key: :commit_id, class_name: 'Ci::Build'
2017-09-10 17:25:29 +05:30
has_many :retryable_builds, -> { latest.failed_or_canceled.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build'
2017-08-17 22:00:37 +05:30
has_many :cancelable_statuses, -> { cancelable }, foreign_key: :commit_id, class_name: 'CommitStatus'
2017-09-10 17:25:29 +05:30
has_many :manual_actions, -> { latest.manual_actions.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build'
has_many :artifacts, -> { latest.with_artifacts_not_expired.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build'
has_many :auto_canceled_pipelines, class_name: 'Ci::Pipeline', foreign_key: 'auto_canceled_by_id'
has_many :auto_canceled_jobs, class_name: 'CommitStatus', foreign_key: 'auto_canceled_by_id'
2017-08-17 22:00:37 +05:30
delegate :id, to: :project, prefix: true
2018-03-17 18:26:18 +05:30
delegate :full_path, to: :project, prefix: true
2015-09-25 12:07:36 +05:30
2017-09-10 17:25:29 +05:30
validates :source, exclusion: { in: %w(unknown), unless: :importing? }, on: :create
2017-08-17 22:00:37 +05:30
validates :sha, presence: { unless: :importing? }
validates :ref, presence: { unless: :importing? }
validates :status, presence: { unless: :importing? }
2016-09-29 09:46:39 +05:30
validate :valid_commit_sha, unless: :importing?
2015-09-25 12:07:36 +05:30
2016-11-03 12:29:30 +05:30
after_create :keep_around_commits, unless: :importing?
2016-08-24 12:49:21 +05:30
2017-09-10 17:25:29 +05:30
enum source: {
unknown: nil,
push: 1,
web: 2,
trigger: 3,
schedule: 4,
api: 5,
external: 6
}
2018-03-17 18:26:18 +05:30
enum config_source: {
unknown_source: nil,
repository_source: 1,
auto_devops_source: 2
}
enum failure_reason: {
unknown_failure: 0,
config_error: 1
}
2016-09-13 17:45:13 +05:30
state_machine :status, initial: :created do
event :enqueue do
2018-03-17 18:26:18 +05:30
transition [:created, :skipped] => :pending
transition [:success, :failed, :canceled] => :running
2016-09-13 17:45:13 +05:30
end
event :run do
2017-08-17 22:00:37 +05:30
transition any - [:running] => :running
2016-09-13 17:45:13 +05:30
end
event :skip do
2017-08-17 22:00:37 +05:30
transition any - [:skipped] => :skipped
2016-09-13 17:45:13 +05:30
end
event :drop do
2017-08-17 22:00:37 +05:30
transition any - [:failed] => :failed
2016-09-13 17:45:13 +05:30
end
event :succeed do
2017-08-17 22:00:37 +05:30
transition any - [:success] => :success
2016-09-13 17:45:13 +05:30
end
event :cancel do
2017-08-17 22:00:37 +05:30
transition any - [:canceled] => :canceled
end
event :block do
transition any - [:manual] => :manual
2016-09-13 17:45:13 +05:30
end
2016-11-03 12:29:30 +05:30
# IMPORTANT
# Do not add any operations to this state_machine
# Create a separate worker for each new operation
2016-09-13 17:45:13 +05:30
before_transition [:created, :pending] => :running do |pipeline|
pipeline.started_at = Time.now
end
before_transition any => [:success, :failed, :canceled] do |pipeline|
pipeline.finished_at = Time.now
2016-11-03 12:29:30 +05:30
pipeline.update_duration
2016-09-13 17:45:13 +05:30
end
2017-08-17 22:00:37 +05:30
before_transition any => [:manual] do |pipeline|
pipeline.update_duration
end
before_transition canceled: any - [:canceled] do |pipeline|
pipeline.auto_canceled_by = nil
end
2018-03-17 18:26:18 +05:30
before_transition any => :failed do |pipeline, transition|
transition.args.first.try do |reason|
pipeline.failure_reason = reason
end
end
2016-09-29 09:46:39 +05:30
after_transition [:created, :pending] => :running do |pipeline|
2017-08-17 22:00:37 +05:30
pipeline.run_after_commit { PipelineMetricsWorker.perform_async(pipeline.id) }
2016-09-29 09:46:39 +05:30
end
after_transition any => [:success] do |pipeline|
2017-08-17 22:00:37 +05:30
pipeline.run_after_commit { PipelineMetricsWorker.perform_async(pipeline.id) }
2016-09-29 09:46:39 +05:30
end
2016-11-03 12:29:30 +05:30
after_transition [:created, :pending, :running] => :success do |pipeline|
2017-08-17 22:00:37 +05:30
pipeline.run_after_commit { PipelineSuccessWorker.perform_async(pipeline.id) }
2016-09-13 17:45:13 +05:30
end
after_transition do |pipeline, transition|
2016-11-03 12:29:30 +05:30
next if transition.loopback?
pipeline.run_after_commit do
2017-08-17 22:00:37 +05:30
PipelineHooksWorker.perform_async(pipeline.id)
ExpirePipelineCacheWorker.perform_async(pipeline.id)
end
end
after_transition any => [:success, :failed] do |pipeline|
pipeline.run_after_commit do
PipelineNotificationWorker.perform_async(pipeline.id)
2016-11-03 12:29:30 +05:30
end
2016-09-13 17:45:13 +05:30
end
end
2018-03-17 18:26:18 +05:30
scope :internal, -> { where(source: internal_sources) }
2017-08-17 22:00:37 +05:30
2018-03-17 18:26:18 +05:30
# Returns the pipelines in descending order (= newest first), optionally
# limited to a number of references.
#
# ref - The name (or names) of the branch(es)/tag(s) to limit the list of
# pipelines to.
def self.newest_first(ref = nil)
relation = order(id: :desc)
ref ? relation.where(ref: ref) : relation
2017-08-17 22:00:37 +05:30
end
def self.latest_status(ref = nil)
2018-03-17 18:26:18 +05:30
newest_first(ref).pluck(:status).first
2017-08-17 22:00:37 +05:30
end
2016-09-29 09:46:39 +05:30
def self.latest_successful_for(ref)
2018-03-17 18:26:18 +05:30
newest_first(ref).success.take
2016-08-24 12:49:21 +05:30
end
2016-06-02 11:05:42 +05:30
2017-08-17 22:00:37 +05:30
def self.latest_successful_for_refs(refs)
2018-03-17 18:26:18 +05:30
relation = newest_first(refs).success
relation.each_with_object({}) do |pipeline, hash|
2017-08-17 22:00:37 +05:30
hash[pipeline.ref] ||= pipeline
end
2015-09-25 12:07:36 +05:30
end
2018-03-17 18:26:18 +05:30
# Returns a Hash containing the latest pipeline status for every given
# commit.
#
# The keys of this Hash are the commit SHAs, the values the statuses.
#
# commits - The list of commit SHAs to get the status for.
# ref - The ref to scope the data to (e.g. "master"). If the ref is not
# given we simply get the latest status for the commits, regardless
# of what refs their pipelines belong to.
def self.latest_status_per_commit(commits, ref = nil)
p1 = arel_table
p2 = arel_table.alias
# This LEFT JOIN will filter out all but the newest row for every
# combination of (project_id, sha) or (project_id, sha, ref) if a ref is
# given.
cond = p1[:sha].eq(p2[:sha])
.and(p1[:project_id].eq(p2[:project_id]))
.and(p1[:id].lt(p2[:id]))
cond = cond.and(p1[:ref].eq(p2[:ref])) if ref
join = p1.join(p2, Arel::Nodes::OuterJoin).on(cond)
relation = select(:sha, :status)
.where(sha: commits)
.where(p2[:id].eq(nil))
.joins(join.join_sources)
relation = relation.where(ref: ref) if ref
relation.each_with_object({}) do |row, hash|
hash[row[:sha]] = row[:status]
end
end
2017-08-17 22:00:37 +05:30
def self.truncate_sha(sha)
sha[0...8]
2015-09-25 12:07:36 +05:30
end
2016-09-13 17:45:13 +05:30
def self.total_duration
where.not(duration: nil).sum(:duration)
end
2017-09-10 17:25:29 +05:30
def self.internal_sources
sources.reject { |source| source == "external" }.values
2016-09-13 17:45:13 +05:30
end
2017-08-17 22:00:37 +05:30
def stages_count
statuses.select(:stage).distinct.count
end
2018-03-17 18:26:18 +05:30
def total_size
statuses.count(:id)
end
2017-09-10 17:25:29 +05:30
def stages_names
statuses.order(:stage_idx).distinct
.pluck(:stage, :stage_idx).map(&:first)
end
def legacy_stage(name)
stage = Ci::LegacyStage.new(self, name: name)
stage unless stage.statuses_count.zero?
2017-08-17 22:00:37 +05:30
end
2017-09-10 17:25:29 +05:30
def legacy_stages
2017-08-17 22:00:37 +05:30
# TODO, this needs refactoring, see gitlab-ce#26481.
stages_query = statuses
.group('stage').select(:stage).order('max(stage_idx)')
status_sql = statuses.latest.where('stage=sg.stage').status_sql
warnings_sql = statuses.latest.select('COUNT(*)')
.where('stage=sg.stage').failed_but_allowed.to_sql
stages_with_statuses = CommitStatus.from(stages_query, :sg)
.pluck('sg.stage', status_sql, "(#{warnings_sql})")
stages_with_statuses.map do |stage|
2017-09-10 17:25:29 +05:30
Ci::LegacyStage.new(self, Hash[%i[name status warnings].zip(stage)])
2017-08-17 22:00:37 +05:30
end
2015-10-24 18:46:33 +05:30
end
2015-09-25 12:07:36 +05:30
def valid_commit_sha
2015-12-23 02:04:40 +05:30
if self.sha == Gitlab::Git::BLANK_SHA
2015-09-25 12:07:36 +05:30
self.errors.add(:sha, " cant be 00000000 (branch removal)")
end
end
def git_author_name
2016-06-22 15:30:34 +05:30
commit.try(:author_name)
2015-09-25 12:07:36 +05:30
end
def git_author_email
2016-06-22 15:30:34 +05:30
commit.try(:author_email)
2015-09-25 12:07:36 +05:30
end
def git_commit_message
2016-06-22 15:30:34 +05:30
commit.try(:message)
2015-09-25 12:07:36 +05:30
end
2016-08-24 12:49:21 +05:30
def git_commit_title
commit.try(:title)
end
2015-09-25 12:07:36 +05:30
def short_sha
Ci::Pipeline.truncate_sha(sha)
2015-09-25 12:07:36 +05:30
end
2018-03-17 18:26:18 +05:30
# NOTE: This is loaded lazily and will never be nil, even if the commit
# cannot be found.
#
# Use constructs like: `pipeline.commit.present?`
2016-06-22 15:30:34 +05:30
def commit
2018-03-17 18:26:18 +05:30
@commit ||= Commit.lazy(project, sha)
2015-09-25 12:07:36 +05:30
end
2016-06-02 11:05:42 +05:30
def branch?
!tag?
end
2017-08-17 22:00:37 +05:30
def stuck?
pending_builds.any?(&:stuck?)
2016-08-24 12:49:21 +05:30
end
2016-06-02 11:05:42 +05:30
def retryable?
2017-08-17 22:00:37 +05:30
retryable_builds.any?
2016-06-02 11:05:42 +05:30
end
def cancelable?
2017-08-17 22:00:37 +05:30
cancelable_statuses.any?
end
def auto_canceled?
canceled? && auto_canceled_by_id?
end
2016-06-02 11:05:42 +05:30
def cancel_running
2018-03-17 18:26:18 +05:30
retry_optimistic_lock(cancelable_statuses) do |cancelable|
2017-08-17 22:00:37 +05:30
cancelable.find_each do |job|
yield(job) if block_given?
job.cancel
end
end
2016-06-02 11:05:42 +05:30
end
2017-08-17 22:00:37 +05:30
def auto_cancel_running(pipeline)
update(auto_canceled_by: pipeline)
cancel_running do |job|
job.auto_canceled_by = pipeline
end
2016-06-02 11:05:42 +05:30
end
2017-08-17 22:00:37 +05:30
def retry_failed(current_user)
Ci::RetryPipelineService.new(project, current_user)
.execute(self)
end
2016-09-13 17:45:13 +05:30
def mark_as_processable_after_stage(stage_idx)
2017-08-17 22:00:37 +05:30
builds.skipped.after_stage(stage_idx).find_each(&:process)
2016-09-13 17:45:13 +05:30
end
2016-06-02 11:05:42 +05:30
def latest?
2018-03-17 18:26:18 +05:30
return false unless ref && commit.present?
project.commit(ref) == commit
2015-09-25 12:07:36 +05:30
end
2015-10-24 18:46:33 +05:30
def retried
@retried ||= (statuses.order(id: :desc) - statuses.latest)
2015-09-25 12:07:36 +05:30
end
def coverage
2016-06-02 11:05:42 +05:30
coverage_array = statuses.latest.map(&:coverage).compact
2015-12-23 02:04:40 +05:30
if coverage_array.size >= 1
'%.2f' % (coverage_array.reduce(:+) / coverage_array.size)
2015-09-25 12:07:36 +05:30
end
end
2017-09-10 17:25:29 +05:30
def stage_seeds
2016-09-13 17:45:13 +05:30
return [] unless config_processor
2018-05-09 12:01:36 +05:30
strong_memoize(:stage_seeds) do
seeds = config_processor.stages_attributes.map do |attributes|
Gitlab::Ci::Pipeline::Seed::Stage.new(self, attributes)
end
seeds.select(&:included?)
end
2017-09-10 17:25:29 +05:30
end
2018-03-17 18:26:18 +05:30
def seeds_size
2018-05-09 12:01:36 +05:30
stage_seeds.sum(&:size)
2018-03-17 18:26:18 +05:30
end
def has_kubernetes_active?
project.deployment_platform&.active?
end
2016-08-24 12:49:21 +05:30
def has_warnings?
2016-11-03 12:29:30 +05:30
builds.latest.failed_but_allowed.any?
2016-08-24 12:49:21 +05:30
end
2018-03-17 18:26:18 +05:30
def set_config_source
if ci_yaml_from_repo
self.config_source = :repository_source
elsif implied_ci_yaml_file
self.config_source = :auto_devops_source
end
end
2018-05-09 12:01:36 +05:30
##
# TODO, setting yaml_errors should be moved to the pipeline creation chain.
#
2015-09-25 12:07:36 +05:30
def config_processor
2017-09-10 17:25:29 +05:30
return unless ci_yaml_file
2016-06-02 11:05:42 +05:30
return @config_processor if defined?(@config_processor)
@config_processor ||= begin
2018-03-17 18:26:18 +05:30
Gitlab::Ci::YamlProcessor.new(ci_yaml_file)
rescue Gitlab::Ci::YamlProcessor::ValidationError => e
2016-06-22 15:30:34 +05:30
self.yaml_errors = e.message
2016-06-02 11:05:42 +05:30
nil
rescue
2016-06-22 15:30:34 +05:30
self.yaml_errors = 'Undefined error'
2016-06-02 11:05:42 +05:30
nil
end
2015-09-25 12:07:36 +05:30
end
2017-09-10 17:25:29 +05:30
def ci_yaml_file_path
if project.ci_config_path.blank?
'.gitlab-ci.yml'
else
project.ci_config_path
end
end
2015-10-24 18:46:33 +05:30
def ci_yaml_file
2016-06-02 11:05:42 +05:30
return @ci_yaml_file if defined?(@ci_yaml_file)
2018-03-17 18:26:18 +05:30
@ci_yaml_file =
if auto_devops_source?
implied_ci_yaml_file
else
ci_yaml_from_repo
end
if @ci_yaml_file
@ci_yaml_file
else
self.yaml_errors = "Failed to load CI/CD config file for #{sha}"
2017-09-10 17:25:29 +05:30
nil
end
2017-08-17 22:00:37 +05:30
end
def has_yaml_errors?
yaml_errors.present?
2015-10-24 18:46:33 +05:30
end
def environments
builds.where.not(environment: nil).success.pluck(:environment).uniq
end
2016-08-24 12:49:21 +05:30
# Manually set the notes for a Ci::Pipeline
# There is no ActiveRecord relation between Ci::Pipeline and notes
# as they are related to a commit sha. This method helps importing
# them using the +Gitlab::ImportExport::RelationFactory+ class.
def notes=(notes)
notes.each do |note|
note[:id] = nil
note[:commit_id] = sha
note[:noteable_id] = self['id']
note.save!
end
end
2016-06-22 15:30:34 +05:30
def notes
2018-03-17 18:26:18 +05:30
project.notes.for_commit_id(sha)
2016-06-22 15:30:34 +05:30
end
2016-09-13 17:45:13 +05:30
def process!
Ci::ProcessPipelineService.new(project, user).execute(self)
end
2016-11-03 12:29:30 +05:30
def update_status
2018-03-17 18:26:18 +05:30
retry_optimistic_lock(self) do
2016-09-29 09:46:39 +05:30
case latest_builds_status
when 'pending' then enqueue
when 'running' then run
when 'success' then succeed
when 'failed' then drop
when 'canceled' then cancel
when 'skipped' then skip
2017-08-17 22:00:37 +05:30
when 'manual' then block
2016-09-29 09:46:39 +05:30
end
2016-09-13 17:45:13 +05:30
end
end
2018-05-09 12:01:36 +05:30
def protected_ref?
strong_memoize(:protected_ref) { project.protected_for?(ref) }
end
def legacy_trigger
strong_memoize(:legacy_trigger) { trigger_requests.first }
end
2016-08-24 12:49:21 +05:30
def predefined_variables
2018-05-09 12:01:36 +05:30
Gitlab::Ci::Variables::Collection.new
.append(key: 'CI_PIPELINE_ID', value: id.to_s)
.append(key: 'CI_CONFIG_PATH', value: ci_yaml_file_path)
.append(key: 'CI_PIPELINE_SOURCE', value: source.to_s)
2016-08-24 12:49:21 +05:30
end
2016-09-29 09:46:39 +05:30
def queued_duration
return unless started_at
seconds = (started_at - created_at).to_i
seconds unless seconds.zero?
end
2016-09-13 17:45:13 +05:30
def update_duration
2016-09-29 09:46:39 +05:30
return unless started_at
2018-03-17 18:26:18 +05:30
self.duration = Gitlab::Ci::Pipeline::Duration.from_pipeline(self)
2016-09-13 17:45:13 +05:30
end
def execute_hooks
data = pipeline_data
project.execute_hooks(data, :pipeline_hooks)
project.execute_services(data, :pipeline_hooks)
end
2017-08-17 22:00:37 +05:30
# All the merge requests for which the current pipeline runs/ran against
def all_merge_requests
@all_merge_requests ||= project.merge_requests.where(source_branch: ref)
end
def detailed_status(current_user)
Gitlab::Ci::Status::Pipeline::Factory
.new(self, current_user)
.fabricate!
2016-09-29 09:46:39 +05:30
end
2018-03-17 18:26:18 +05:30
def latest_builds_with_artifacts
# We purposely cast the builds to an Array here. Because we always use the
# rows if there are more than 0 this prevents us from having to run two
# queries: one to get the count and one to get the rows.
2018-03-27 19:54:05 +05:30
@latest_builds_with_artifacts ||= builds.latest.with_artifacts_archive.to_a
2018-03-17 18:26:18 +05:30
end
2015-09-25 12:07:36 +05:30
private
2018-03-17 18:26:18 +05:30
def ci_yaml_from_repo
return unless project
return unless sha
project.repository.gitlab_ci_yml_for(sha, ci_yaml_file_path)
rescue GRPC::NotFound, GRPC::Internal
nil
end
def implied_ci_yaml_file
return unless project
if project.auto_devops_enabled?
Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps').content
end
end
2016-09-13 17:45:13 +05:30
def pipeline_data
Gitlab::DataBuilder::Pipeline.build(self)
end
def latest_builds_status
return 'failed' unless yaml_errors.blank?
statuses.latest.status || 'skipped'
2016-06-02 11:05:42 +05:30
end
2016-08-24 12:49:21 +05:30
def keep_around_commits
return unless project
project.repository.keep_around(self.sha)
project.repository.keep_around(self.before_sha)
end
2015-09-25 12:07:36 +05:30
end
end