2017-08-17 22:00:37 +05:30
|
|
|
require 'spec_helper'
|
|
|
|
|
|
|
|
describe StuckCiJobsWorker do
|
2018-11-08 19:23:39 +05:30
|
|
|
include ExclusiveLeaseHelpers
|
|
|
|
|
2017-08-17 22:00:37 +05:30
|
|
|
let!(:runner) { create :ci_runner }
|
|
|
|
let!(:job) { create :ci_build, runner: runner }
|
2018-11-08 19:23:39 +05:30
|
|
|
let(:trace_lease_key) { "trace:archive:#{job.id}" }
|
|
|
|
let(:trace_lease_uuid) { SecureRandom.uuid }
|
|
|
|
let(:worker_lease_key) { StuckCiJobsWorker::EXCLUSIVE_LEASE_KEY }
|
|
|
|
let(:worker_lease_uuid) { SecureRandom.uuid }
|
|
|
|
|
|
|
|
subject(:worker) { described_class.new }
|
2017-08-17 22:00:37 +05:30
|
|
|
|
|
|
|
before do
|
2018-11-08 19:23:39 +05:30
|
|
|
stub_exclusive_lease(worker_lease_key, worker_lease_uuid)
|
|
|
|
stub_exclusive_lease(trace_lease_key, trace_lease_uuid)
|
2017-08-17 22:00:37 +05:30
|
|
|
job.update!(status: status, updated_at: updated_at)
|
|
|
|
end
|
|
|
|
|
|
|
|
shared_examples 'job is dropped' do
|
2018-03-17 18:26:18 +05:30
|
|
|
before do
|
2017-08-17 22:00:37 +05:30
|
|
|
worker.perform
|
2018-03-17 18:26:18 +05:30
|
|
|
job.reload
|
|
|
|
end
|
|
|
|
|
|
|
|
it "changes status" do
|
|
|
|
expect(job).to be_failed
|
|
|
|
expect(job).to be_stuck_or_timeout_failure
|
2017-08-17 22:00:37 +05:30
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
shared_examples 'job is unchanged' do
|
2018-03-17 18:26:18 +05:30
|
|
|
before do
|
2017-08-17 22:00:37 +05:30
|
|
|
worker.perform
|
2018-03-17 18:26:18 +05:30
|
|
|
job.reload
|
|
|
|
end
|
|
|
|
|
|
|
|
it "doesn't change status" do
|
|
|
|
expect(job.status).to eq(status)
|
2017-08-17 22:00:37 +05:30
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
context 'when job is pending' do
|
|
|
|
let(:status) { 'pending' }
|
|
|
|
|
|
|
|
context 'when job is not stuck' do
|
2017-09-10 17:25:29 +05:30
|
|
|
before do
|
|
|
|
allow_any_instance_of(Ci::Build).to receive(:stuck?).and_return(false)
|
|
|
|
end
|
2017-08-17 22:00:37 +05:30
|
|
|
|
|
|
|
context 'when job was not updated for more than 1 day ago' do
|
|
|
|
let(:updated_at) { 2.days.ago }
|
2018-11-08 19:23:39 +05:30
|
|
|
|
2017-08-17 22:00:37 +05:30
|
|
|
it_behaves_like 'job is dropped'
|
|
|
|
end
|
|
|
|
|
|
|
|
context 'when job was updated in less than 1 day ago' do
|
|
|
|
let(:updated_at) { 6.hours.ago }
|
2018-11-08 19:23:39 +05:30
|
|
|
|
2017-08-17 22:00:37 +05:30
|
|
|
it_behaves_like 'job is unchanged'
|
|
|
|
end
|
|
|
|
|
|
|
|
context 'when job was not updated for more than 1 hour ago' do
|
|
|
|
let(:updated_at) { 2.hours.ago }
|
2018-11-08 19:23:39 +05:30
|
|
|
|
2017-08-17 22:00:37 +05:30
|
|
|
it_behaves_like 'job is unchanged'
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
context 'when job is stuck' do
|
2017-09-10 17:25:29 +05:30
|
|
|
before do
|
|
|
|
allow_any_instance_of(Ci::Build).to receive(:stuck?).and_return(true)
|
|
|
|
end
|
2017-08-17 22:00:37 +05:30
|
|
|
|
|
|
|
context 'when job was not updated for more than 1 hour ago' do
|
|
|
|
let(:updated_at) { 2.hours.ago }
|
2018-11-08 19:23:39 +05:30
|
|
|
|
2017-08-17 22:00:37 +05:30
|
|
|
it_behaves_like 'job is dropped'
|
|
|
|
end
|
|
|
|
|
2018-11-08 19:23:39 +05:30
|
|
|
context 'when job was updated in less than 1
|
|
|
|
hour ago' do
|
2017-08-17 22:00:37 +05:30
|
|
|
let(:updated_at) { 30.minutes.ago }
|
2018-11-08 19:23:39 +05:30
|
|
|
|
2017-08-17 22:00:37 +05:30
|
|
|
it_behaves_like 'job is unchanged'
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
context 'when job is running' do
|
|
|
|
let(:status) { 'running' }
|
|
|
|
|
|
|
|
context 'when job was not updated for more than 1 hour ago' do
|
|
|
|
let(:updated_at) { 2.hours.ago }
|
2018-11-08 19:23:39 +05:30
|
|
|
|
2017-08-17 22:00:37 +05:30
|
|
|
it_behaves_like 'job is dropped'
|
|
|
|
end
|
|
|
|
|
|
|
|
context 'when job was updated in less than 1 hour ago' do
|
|
|
|
let(:updated_at) { 30.minutes.ago }
|
2018-11-08 19:23:39 +05:30
|
|
|
|
2017-08-17 22:00:37 +05:30
|
|
|
it_behaves_like 'job is unchanged'
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
%w(success skipped failed canceled).each do |status|
|
|
|
|
context "when job is #{status}" do
|
|
|
|
let(:status) { status }
|
|
|
|
let(:updated_at) { 2.days.ago }
|
2018-11-08 19:23:39 +05:30
|
|
|
|
2017-08-17 22:00:37 +05:30
|
|
|
it_behaves_like 'job is unchanged'
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
context 'for deleted project' do
|
|
|
|
let(:status) { 'running' }
|
|
|
|
let(:updated_at) { 2.days.ago }
|
|
|
|
|
2017-09-10 17:25:29 +05:30
|
|
|
before do
|
|
|
|
job.project.update(pending_delete: true)
|
|
|
|
end
|
2017-08-17 22:00:37 +05:30
|
|
|
|
2018-03-17 18:26:18 +05:30
|
|
|
it 'does drop job' do
|
|
|
|
expect_any_instance_of(Ci::Build).to receive(:drop).and_call_original
|
2017-08-17 22:00:37 +05:30
|
|
|
worker.perform
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2018-12-05 23:21:45 +05:30
|
|
|
describe 'drop stale scheduled builds' do
|
|
|
|
let(:status) { 'scheduled' }
|
|
|
|
let(:updated_at) { }
|
|
|
|
|
|
|
|
context 'when scheduled at 2 hours ago but it is not executed yet' do
|
|
|
|
let!(:job) { create(:ci_build, :scheduled, scheduled_at: 2.hours.ago) }
|
|
|
|
|
|
|
|
it 'drops the stale scheduled build' do
|
|
|
|
expect(Ci::Build.scheduled.count).to eq(1)
|
|
|
|
expect(job).to be_scheduled
|
|
|
|
|
|
|
|
worker.perform
|
|
|
|
job.reload
|
|
|
|
|
|
|
|
expect(Ci::Build.scheduled.count).to eq(0)
|
|
|
|
expect(job).to be_failed
|
|
|
|
expect(job).to be_stale_schedule
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
context 'when scheduled at 30 minutes ago but it is not executed yet' do
|
|
|
|
let!(:job) { create(:ci_build, :scheduled, scheduled_at: 30.minutes.ago) }
|
|
|
|
|
|
|
|
it 'does not drop the stale scheduled build yet' do
|
|
|
|
expect(Ci::Build.scheduled.count).to eq(1)
|
|
|
|
expect(job).to be_scheduled
|
|
|
|
|
|
|
|
worker.perform
|
|
|
|
|
|
|
|
expect(Ci::Build.scheduled.count).to eq(1)
|
|
|
|
expect(job).to be_scheduled
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
context 'when there are no stale scheduled builds' do
|
|
|
|
it 'does not drop the stale scheduled build yet' do
|
|
|
|
expect { worker.perform }.not_to raise_error
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2017-08-17 22:00:37 +05:30
|
|
|
describe 'exclusive lease' do
|
|
|
|
let(:status) { 'running' }
|
|
|
|
let(:updated_at) { 2.days.ago }
|
|
|
|
let(:worker2) { described_class.new }
|
|
|
|
|
|
|
|
it 'is guard by exclusive lease when executed concurrently' do
|
2018-03-17 18:26:18 +05:30
|
|
|
expect(worker).to receive(:drop).at_least(:once).and_call_original
|
2017-08-17 22:00:37 +05:30
|
|
|
expect(worker2).not_to receive(:drop)
|
2018-11-08 19:23:39 +05:30
|
|
|
|
2017-08-17 22:00:37 +05:30
|
|
|
worker.perform
|
2018-11-08 19:23:39 +05:30
|
|
|
|
|
|
|
stub_exclusive_lease_taken(worker_lease_key)
|
|
|
|
|
2017-08-17 22:00:37 +05:30
|
|
|
worker2.perform
|
|
|
|
end
|
|
|
|
|
|
|
|
it 'can be executed in sequence' do
|
2018-03-17 18:26:18 +05:30
|
|
|
expect(worker).to receive(:drop).at_least(:once).and_call_original
|
|
|
|
expect(worker2).to receive(:drop).at_least(:once).and_call_original
|
2018-11-08 19:23:39 +05:30
|
|
|
|
2017-08-17 22:00:37 +05:30
|
|
|
worker.perform
|
|
|
|
worker2.perform
|
|
|
|
end
|
|
|
|
|
2018-11-08 19:23:39 +05:30
|
|
|
it 'cancels exclusive leases after worker perform' do
|
|
|
|
expect_to_cancel_exclusive_lease(trace_lease_key, trace_lease_uuid)
|
|
|
|
expect_to_cancel_exclusive_lease(worker_lease_key, worker_lease_uuid)
|
|
|
|
|
2017-08-17 22:00:37 +05:30
|
|
|
worker.perform
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|