debian-mirror-gitlab/spec/services/ci/process_pipeline_service_spec.rb

require 'spec_helper'

describe Ci::ProcessPipelineService, '#execute' do
  let(:user) { create(:user) }
  let(:project) { create(:project) }

  let(:pipeline) do
    create(:ci_empty_pipeline, ref: 'master', project: project)
  end

  before do
    stub_ci_pipeline_to_return_yaml_file

    stub_not_protect_default_branch

    project.add_developer(user)
  end

  context 'when simple pipeline is defined' do
    before do
      create_build('linux', stage_idx: 0)
      create_build('mac', stage_idx: 0)
      create_build('rspec', stage_idx: 1)
      create_build('rubocop', stage_idx: 1)
      create_build('deploy', stage_idx: 2)
    end

    it 'processes a pipeline' do
      expect(process_pipeline).to be_truthy

      succeed_pending

      expect(builds.success.count).to eq(2)

      succeed_pending

      expect(builds.success.count).to eq(4)

      succeed_pending

      expect(builds.success.count).to eq(5)
    end

    it 'does not process pipeline if existing stage is running' do
      expect(process_pipeline).to be_truthy
      expect(builds.pending.count).to eq(2)

      expect(process_pipeline).to be_falsey
      expect(builds.pending.count).to eq(2)
    end
  end

  context 'custom stage with first job allowed to fail' do
    before do
      create_build('clean_job', stage_idx: 0, allow_failure: true)
      create_build('test_job', stage_idx: 1, allow_failure: true)
    end

    it 'automatically triggers a next stage when build finishes' do
      expect(process_pipeline).to be_truthy
      expect(builds_statuses).to eq ['pending']

      fail_running_or_pending

      expect(builds_statuses).to eq %w(failed pending)

      fail_running_or_pending

      expect(pipeline.reload).to be_success
    end
  end

  context 'when optional manual actions are defined' do
    before do
      create_build('build', stage_idx: 0)
      create_build('test', stage_idx: 1)
      create_build('test_failure', stage_idx: 2, when: 'on_failure')
      create_build('deploy', stage_idx: 3)
      create_build('production', stage_idx: 3, when: 'manual', allow_failure: true)
      create_build('cleanup', stage_idx: 4, when: 'always')
      create_build('clear:cache', stage_idx: 4, when: 'manual', allow_failure: true)
    end

    context 'when builds are successful' do
      it 'properly processes the pipeline' do
        expect(process_pipeline).to be_truthy
        expect(builds_names).to eq ['build']
        expect(builds_statuses).to eq ['pending']

        succeed_running_or_pending

        expect(builds_names).to eq %w(build test)
        expect(builds_statuses).to eq %w(success pending)

        succeed_running_or_pending

        expect(builds_names).to eq %w(build test deploy production)
        expect(builds_statuses).to eq %w(success success pending manual)

        succeed_running_or_pending

        expect(builds_names).to eq %w(build test deploy production cleanup clear:cache)
        expect(builds_statuses).to eq %w(success success success manual pending manual)

        succeed_running_or_pending

        expect(builds_statuses).to eq %w(success success success manual success manual)
        expect(pipeline.reload.status).to eq 'success'
      end
    end

    context 'when test job fails' do
      it 'properly processes the pipeline' do
        expect(process_pipeline).to be_truthy
        expect(builds_names).to eq ['build']
        expect(builds_statuses).to eq ['pending']

        succeed_running_or_pending

        expect(builds_names).to eq %w(build test)
        expect(builds_statuses).to eq %w(success pending)

        fail_running_or_pending

        expect(builds_names).to eq %w(build test test_failure)
        expect(builds_statuses).to eq %w(success failed pending)

        succeed_running_or_pending

        expect(builds_names).to eq %w(build test test_failure cleanup)
        expect(builds_statuses).to eq %w(success failed success pending)

        succeed_running_or_pending

        expect(builds_statuses).to eq %w(success failed success success)
        expect(pipeline.reload.status).to eq 'failed'
      end
    end

    context 'when test and test_failure jobs fail' do
      it 'properly processes the pipeline' do
        expect(process_pipeline).to be_truthy
        expect(builds_names).to eq ['build']
        expect(builds_statuses).to eq ['pending']

        succeed_running_or_pending

        expect(builds_names).to eq %w(build test)
        expect(builds_statuses).to eq %w(success pending)

        fail_running_or_pending

        expect(builds_names).to eq %w(build test test_failure)
        expect(builds_statuses).to eq %w(success failed pending)

        fail_running_or_pending

        expect(builds_names).to eq %w(build test test_failure cleanup)
        expect(builds_statuses).to eq %w(success failed failed pending)

        succeed_running_or_pending

        expect(builds_names).to eq %w(build test test_failure cleanup)
        expect(builds_statuses).to eq %w(success failed failed success)
        expect(pipeline.reload.status).to eq('failed')
      end
    end

    context 'when deploy job fails' do
      it 'properly processes the pipeline' do
        expect(process_pipeline).to be_truthy
        expect(builds_names).to eq ['build']
        expect(builds_statuses).to eq ['pending']

        succeed_running_or_pending

        expect(builds_names).to eq %w(build test)
        expect(builds_statuses).to eq %w(success pending)

        succeed_running_or_pending

        expect(builds_names).to eq %w(build test deploy production)
        expect(builds_statuses).to eq %w(success success pending manual)

        fail_running_or_pending

        expect(builds_names).to eq %w(build test deploy production cleanup)
        expect(builds_statuses).to eq %w(success success failed manual pending)

        succeed_running_or_pending

        expect(builds_statuses).to eq %w(success success failed manual success)
        expect(pipeline.reload).to be_failed
      end
    end

    context 'when build is canceled in the second stage' do
      it 'does not schedule builds after build has been canceled' do
        expect(process_pipeline).to be_truthy
        expect(builds_names).to eq ['build']
        expect(builds_statuses).to eq ['pending']

        succeed_running_or_pending

        expect(builds.running_or_pending).not_to be_empty
        expect(builds_names).to eq %w(build test)
        expect(builds_statuses).to eq %w(success pending)

        cancel_running_or_pending

        expect(builds.running_or_pending).to be_empty
        expect(builds_names).to eq %w[build test]
        expect(builds_statuses).to eq %w[success canceled]
        expect(pipeline.reload).to be_canceled
      end
    end

    context 'when listing optional manual actions' do
      it 'returns only for skipped builds' do
        # currently all builds are created
        expect(process_pipeline).to be_truthy
        expect(manual_actions).to be_empty

        # succeed stage build
        succeed_running_or_pending

        expect(manual_actions).to be_empty

        # succeed stage test
        succeed_running_or_pending

        expect(manual_actions).to be_one # production

        # succeed stage deploy
        succeed_running_or_pending

        expect(manual_actions).to be_many # production and clear cache
      end
    end
  end

  context 'when delayed jobs are defined' do
    context 'when the scene is timed incremental rollout' do
      before do
        create_build('build', stage_idx: 0)
        create_build('rollout10%', **delayed_options, stage_idx: 1)
        create_build('rollout100%', **delayed_options, stage_idx: 2)
        create_build('cleanup', stage_idx: 3)

        allow(Ci::BuildScheduleWorker).to receive(:perform_at)
      end

      context 'when builds are successful' do
        it 'properly processes the pipeline' do
          expect(process_pipeline).to be_truthy
          expect(builds_names_and_statuses).to eq({ 'build': 'pending' })

          succeed_pending

          expect(builds_names_and_statuses).to eq({ 'build': 'success', 'rollout10%': 'scheduled' })

          enqueue_scheduled('rollout10%')
          succeed_pending

          expect(builds_names_and_statuses).to eq({ 'build': 'success', 'rollout10%': 'success', 'rollout100%': 'scheduled' })

          enqueue_scheduled('rollout100%')
          succeed_pending

          expect(builds_names_and_statuses).to eq({ 'build': 'success', 'rollout10%': 'success', 'rollout100%': 'success', 'cleanup': 'pending' })

          succeed_pending

          expect(builds_names_and_statuses).to eq({ 'build': 'success', 'rollout10%': 'success', 'rollout100%': 'success', 'cleanup': 'success' })
          expect(pipeline.reload.status).to eq 'success'
        end
      end

      context 'when build job fails' do
        it 'properly processes the pipeline' do
          expect(process_pipeline).to be_truthy
          expect(builds_names_and_statuses).to eq({ 'build': 'pending' })

          fail_running_or_pending

          expect(builds_names_and_statuses).to eq({ 'build': 'failed' })
          expect(pipeline.reload.status).to eq 'failed'
        end
      end

      context 'when rollout 10% is unscheduled' do
        it 'properly processes the pipeline' do
          expect(process_pipeline).to be_truthy
          expect(builds_names_and_statuses).to eq({ 'build': 'pending' })

          succeed_pending

          expect(builds_names_and_statuses).to eq({ 'build': 'success', 'rollout10%': 'scheduled' })

          unschedule

          expect(builds_names_and_statuses).to eq({ 'build': 'success', 'rollout10%': 'manual' })
          expect(pipeline.reload.status).to eq 'manual'
        end

        context 'when user plays rollout 10%' do
          it 'schedules rollout100%' do
            process_pipeline
            succeed_pending
            unschedule
            play_manual_action('rollout10%')
            succeed_pending

            expect(builds_names_and_statuses).to eq({ 'build': 'success', 'rollout10%': 'success', 'rollout100%': 'scheduled' })
            expect(pipeline.reload.status).to eq 'scheduled'
          end
        end
      end

      context 'when rollout 10% fails' do
        it 'properly processes the pipeline' do
          expect(process_pipeline).to be_truthy
          expect(builds_names_and_statuses).to eq({ 'build': 'pending' })

          succeed_pending

          expect(builds_names_and_statuses).to eq({ 'build': 'success', 'rollout10%': 'scheduled' })

          enqueue_scheduled('rollout10%')
          fail_running_or_pending

          expect(builds_names_and_statuses).to eq({ 'build': 'success', 'rollout10%': 'failed' })
          expect(pipeline.reload.status).to eq 'failed'
        end

        context 'when user retries rollout 10%' do
          it 'does not schedule rollout10% again' do
            process_pipeline
            succeed_pending
            enqueue_scheduled('rollout10%')
            fail_running_or_pending
            retry_build('rollout10%')

            expect(builds_names_and_statuses).to eq({ 'build': 'success', 'rollout10%': 'pending' })
            expect(pipeline.reload.status).to eq 'running'
          end
        end
      end

      context 'when rollout 10% is played immidiately' do
        it 'properly processes the pipeline' do
          expect(process_pipeline).to be_truthy
          expect(builds_names_and_statuses).to eq({ 'build': 'pending' })

          succeed_pending

          expect(builds_names_and_statuses).to eq({ 'build': 'success', 'rollout10%': 'scheduled' })

          play_manual_action('rollout10%')

          expect(builds_names_and_statuses).to eq({ 'build': 'success', 'rollout10%': 'pending' })
          expect(pipeline.reload.status).to eq 'running'
        end
      end
    end

    context 'when only one scheduled job exists in a pipeline' do
      before do
        create_build('delayed', **delayed_options, stage_idx: 0)

        allow(Ci::BuildScheduleWorker).to receive(:perform_at)
      end

      it 'properly processes the pipeline' do
        expect(process_pipeline).to be_truthy
        expect(builds_names_and_statuses).to eq({ 'delayed': 'scheduled' })

        expect(pipeline.reload.status).to eq 'scheduled'
      end
    end

    context 'when there are two delayed jobs in a stage' do
      before do
        create_build('delayed1', **delayed_options, stage_idx: 0)
        create_build('delayed2', **delayed_options, stage_idx: 0)
        create_build('job', stage_idx: 1)

        allow(Ci::BuildScheduleWorker).to receive(:perform_at)
      end

      it 'blocks the stage until all scheduled jobs finished' do
        expect(process_pipeline).to be_truthy
        expect(builds_names_and_statuses).to eq({ 'delayed1': 'scheduled', 'delayed2': 'scheduled' })

        enqueue_scheduled('delayed1')

        expect(builds_names_and_statuses).to eq({ 'delayed1': 'pending', 'delayed2': 'scheduled' })
        expect(pipeline.reload.status).to eq 'running'
      end
    end

    context 'when a delayed job is allowed to fail' do
      before do
        create_build('delayed', **delayed_options, allow_failure: true, stage_idx: 0)
        create_build('job', stage_idx: 1)

        allow(Ci::BuildScheduleWorker).to receive(:perform_at)
      end

      it 'blocks the stage and continues after it failed' do
        expect(process_pipeline).to be_truthy
        expect(builds_names_and_statuses).to eq({ 'delayed': 'scheduled' })

        enqueue_scheduled('delayed')
        fail_running_or_pending

        expect(builds_names_and_statuses).to eq({ 'delayed': 'failed', 'job': 'pending' })
        expect(pipeline.reload.status).to eq 'pending'
      end
    end
  end

  context 'when there are manual action in earlier stages' do
    context 'when first stage has only optional manual actions' do
      before do
        create_build('build', stage_idx: 0, when: 'manual', allow_failure: true)
        create_build('check', stage_idx: 1)
        create_build('test', stage_idx: 2)

        process_pipeline
      end

      it 'starts from the second stage' do
        expect(all_builds_statuses).to eq %w[manual pending created]
      end
    end

    context 'when second stage has only optional manual actions' do
      before do
        create_build('check', stage_idx: 0)
        create_build('build', stage_idx: 1, when: 'manual', allow_failure: true)
        create_build('test', stage_idx: 2)

        process_pipeline
      end

      it 'skips second stage and continues on third stage' do
        expect(all_builds_statuses).to eq(%w[pending created created])

        builds.first.success

        expect(all_builds_statuses).to eq(%w[success manual pending])
      end
    end
  end

  context 'when there are only manual actions in stages' do
    before do
      create_build('image', stage_idx: 0, when: 'manual', allow_failure: true)
      create_build('build', stage_idx: 1, when: 'manual', allow_failure: true)
      create_build('deploy', stage_idx: 2, when: 'manual')
      create_build('check', stage_idx: 3)

      process_pipeline
    end

    it 'processes all jobs until blocking actions encountered' do
      expect(all_builds_statuses).to eq(%w[manual manual manual created])
      expect(all_builds_names).to eq(%w[image build deploy check])

      expect(pipeline.reload).to be_blocked
    end
  end

  context 'when there is only one manual action' do
    before do
      create_build('deploy', stage_idx: 0, when: 'manual', allow_failure: true)

      process_pipeline
    end

    it 'skips the pipeline' do
      expect(pipeline.reload).to be_skipped
    end

    context 'when the action was played' do
      before do
        play_manual_action('deploy')
      end

      it 'queues the action and pipeline' do
        expect(all_builds_statuses).to eq(%w[pending])

        expect(pipeline.reload).to be_pending
      end
    end
  end

  context 'when blocking manual actions are defined' do
    before do
      create_build('code:test', stage_idx: 0)
      create_build('staging:deploy', stage_idx: 1, when: 'manual')
      create_build('staging:test', stage_idx: 2, when: 'on_success')
      create_build('production:deploy', stage_idx: 3, when: 'manual')
      create_build('production:test', stage_idx: 4, when: 'always')
    end

    context 'when first stage succeeds' do
      it 'blocks pipeline on stage with first manual action' do
        process_pipeline

        expect(builds_names).to eq %w[code:test]
        expect(builds_statuses).to eq %w[pending]
        expect(pipeline.reload.status).to eq 'pending'

        succeed_running_or_pending

        expect(builds_names).to eq %w[code:test staging:deploy]
        expect(builds_statuses).to eq %w[success manual]
        expect(pipeline.reload).to be_manual
      end
    end

    context 'when first stage fails' do
      it 'does not take blocking action into account' do
        process_pipeline

        expect(builds_names).to eq %w[code:test]
        expect(builds_statuses).to eq %w[pending]
        expect(pipeline.reload.status).to eq 'pending'

        fail_running_or_pending

        expect(builds_names).to eq %w[code:test production:test]
        expect(builds_statuses).to eq %w[failed pending]

        succeed_running_or_pending

        expect(builds_statuses).to eq %w[failed success]
        expect(pipeline.reload).to be_failed
      end
    end

    context 'when pipeline is promoted sequentially up to the end' do
      before do
        # Users need ability to merge into a branch in order to trigger
        # protected manual actions.
        #
        create(:protected_branch, :developers_can_merge,
               name: 'master', project: project)
      end

      it 'properly processes entire pipeline' do
        process_pipeline

        expect(builds_names).to eq %w[code:test]
        expect(builds_statuses).to eq %w[pending]

        succeed_running_or_pending

        expect(builds_names).to eq %w[code:test staging:deploy]
        expect(builds_statuses).to eq %w[success manual]
        expect(pipeline.reload).to be_manual

        play_manual_action('staging:deploy')

        expect(builds_statuses).to eq %w[success pending]

        succeed_running_or_pending

        expect(builds_names).to eq %w[code:test staging:deploy staging:test]
        expect(builds_statuses).to eq %w[success success pending]

        succeed_running_or_pending

        expect(builds_names).to eq %w[code:test staging:deploy staging:test
                                      production:deploy]
        expect(builds_statuses).to eq %w[success success success manual]

        expect(pipeline.reload).to be_manual
        expect(pipeline.reload).to be_blocked
        expect(pipeline.reload).not_to be_active
        expect(pipeline.reload).not_to be_complete

        play_manual_action('production:deploy')

        expect(builds_statuses).to eq %w[success success success pending]
        expect(pipeline.reload).to be_running

        succeed_running_or_pending

        expect(builds_names).to eq %w[code:test staging:deploy staging:test
                                      production:deploy production:test]
        expect(builds_statuses).to eq %w[success success success success pending]
        expect(pipeline.reload).to be_running

        succeed_running_or_pending

        expect(builds_names).to eq %w[code:test staging:deploy staging:test
                                      production:deploy production:test]
        expect(builds_statuses).to eq %w[success success success success success]
        expect(pipeline.reload).to be_success
      end
    end
  end

  context 'when second stage has only on_failure jobs' do
    before do
      create_build('check', stage_idx: 0)
      create_build('build', stage_idx: 1, when: 'on_failure')
      create_build('test', stage_idx: 2)

      process_pipeline
    end

    it 'skips second stage and continues on third stage' do
      expect(all_builds_statuses).to eq(%w[pending created created])

      builds.first.success

      expect(all_builds_statuses).to eq(%w[success skipped pending])
    end
  end

  context 'when failed build in the middle stage is retried' do
    context 'when failed build is the only unsuccessful build in the stage' do
      before do
        create_build('build:1', stage_idx: 0)
        create_build('build:2', stage_idx: 0)
        create_build('test:1', stage_idx: 1)
        create_build('test:2', stage_idx: 1)
        create_build('deploy:1', stage_idx: 2)
        create_build('deploy:2', stage_idx: 2)
      end

      it 'does trigger builds in the next stage' do
        expect(process_pipeline).to be_truthy
        expect(builds_names).to eq ['build:1', 'build:2']

        succeed_running_or_pending

        expect(builds_names).to eq ['build:1', 'build:2', 'test:1', 'test:2']

        pipeline.builds.find_by(name: 'test:1').success
        pipeline.builds.find_by(name: 'test:2').drop

        expect(builds_names).to eq ['build:1', 'build:2', 'test:1', 'test:2']

        Ci::Build.retry(pipeline.builds.find_by(name: 'test:2'), user).success

        expect(builds_names).to eq ['build:1', 'build:2', 'test:1', 'test:2',
                                    'test:2', 'deploy:1', 'deploy:2']
      end
    end
  end

  context 'updates a list of retried builds' do
    subject { described_class.retried.order(:id) }

    let!(:build_retried) { create_build('build') }
    let!(:build) { create_build('build') }
    let!(:test) { create_build('test') }

    it 'returns unique statuses' do
      process_pipeline

      expect(all_builds.latest).to contain_exactly(build, test)
      expect(all_builds.retried).to contain_exactly(build_retried)
    end
  end

  context 'when builds with auto-retries are configured' do
    before do
      create_build('build:1', stage_idx: 0, user: user, options: { script: 'aa', retry: 2 })
      create_build('test:1', stage_idx: 1, user: user, when: :on_failure)
      create_build('test:2', stage_idx: 1, user: user, options: { script: 'aa', retry: 1 })
    end

    it 'automatically retries builds in a valid order' do
      expect(process_pipeline).to be_truthy

      fail_running_or_pending

      expect(builds_names).to eq %w[build:1 build:1]
      expect(builds_statuses).to eq %w[failed pending]

      succeed_running_or_pending

      expect(builds_names).to eq %w[build:1 build:1 test:2]
      expect(builds_statuses).to eq %w[failed success pending]

      succeed_running_or_pending

      expect(builds_names).to eq %w[build:1 build:1 test:2]
      expect(builds_statuses).to eq %w[failed success success]

      expect(pipeline.reload).to be_success
    end
  end

  def process_pipeline
    described_class.new(pipeline.project, user).execute(pipeline)
  end

  def all_builds
    pipeline.builds.order(:stage_idx, :id)
  end

  def builds
    all_builds.where.not(status: [:created, :skipped])
  end

  def builds_names
    builds.pluck(:name)
  end

  def builds_names_and_statuses
    builds.each_with_object({}) do |b, h|
      h[b.name.to_sym] = b.status
      h
    end
  end

  def all_builds_names
    all_builds.pluck(:name)
  end

  def builds_statuses
    builds.pluck(:status)
  end

  def all_builds_statuses
    all_builds.pluck(:status)
  end

  def succeed_pending
    builds.pending.map(&:success)
  end

  def succeed_running_or_pending
    pipeline.builds.running_or_pending.each(&:success)
  end

  def fail_running_or_pending
    pipeline.builds.running_or_pending.each(&:drop)
  end

  def cancel_running_or_pending
    pipeline.builds.running_or_pending.each(&:cancel)
  end

  def play_manual_action(name)
    builds.find_by(name: name).play(user)
  end

  def enqueue_scheduled(name)
    builds.scheduled.find_by(name: name).enqueue
  end

  def retry_build(name)
    Ci::Build.retry(builds.find_by(name: name), user)
  end

  def manual_actions
    pipeline.manual_actions(true)
  end

  def create_build(name, **opts)
    create(:ci_build, :created, pipeline: pipeline, name: name, **opts)
  end

  def delayed_options
    { when: 'delayed', options: { script: %w(echo), start_in: '1 minute' } }
  end

  def unschedule
    pipeline.builds.scheduled.map(&:unschedule)
  end
end