debian-mirror-gitlab/spec/services/projects/prometheus/alerts/notify_service_spec.rb

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

370 lines
12 KiB
Ruby
Raw Normal View History

2020-04-22 19:07:51 +05:30
# frozen_string_literal: true
require 'spec_helper'
2023-05-27 22:25:52 +05:30
RSpec.describe Projects::Prometheus::Alerts::NotifyService, feature_category: :metrics do
2020-06-23 00:09:42 +05:30
include PrometheusHelpers
2021-03-11 19:13:27 +05:30
using RSpec::Parameterized::TableSyntax
2020-06-23 00:09:42 +05:30
2021-06-08 01:23:25 +05:30
let_it_be_with_reload(:project) { create(:project) }
let_it_be_with_refind(:setting) do
create(:project_incident_management_setting, project: project, send_email: true, create_issue: true)
end
2020-04-22 19:07:51 +05:30
2021-02-22 17:27:13 +05:30
let(:service) { described_class.new(project, payload) }
2020-04-22 19:07:51 +05:30
let(:token_input) { 'token' }
2021-06-08 01:23:25 +05:30
subject { service.execute(token_input) }
2020-04-22 19:07:51 +05:30
context 'with valid payload' do
2020-06-23 00:09:42 +05:30
let_it_be(:alert_firing) { create(:prometheus_alert, project: project) }
let_it_be(:alert_resolved) { create(:prometheus_alert, project: project) }
2021-06-08 01:23:25 +05:30
let_it_be(:cluster, reload: true) { create(:cluster, :provided_by_user, projects: [project]) }
2020-06-23 00:09:42 +05:30
let(:payload_raw) { prometheus_alert_payload(firing: [alert_firing], resolved: [alert_resolved]) }
2020-04-22 19:07:51 +05:30
let(:payload) { ActionController::Parameters.new(payload_raw).permit! }
let(:payload_alert_firing) { payload_raw['alerts'].first }
let(:token) { 'token' }
2021-06-08 01:23:25 +05:30
let(:source) { 'Prometheus' }
2020-04-22 19:07:51 +05:30
2020-07-28 23:09:34 +05:30
context 'with environment specific clusters' do
let(:prd_cluster) do
cluster
end
let(:stg_cluster) do
create(:cluster, :provided_by_user, projects: [project], enabled: true, environment_scope: 'stg/*')
end
let(:stg_environment) do
create(:environment, project: project, name: 'stg/1')
end
let(:alert_firing) do
create(:prometheus_alert, project: project, environment: stg_environment)
end
before do
2023-06-20 00:43:36 +05:30
create(:clusters_integrations_prometheus, cluster: prd_cluster, alert_manager_token: token)
create(:clusters_integrations_prometheus, cluster: stg_cluster, alert_manager_token: nil)
2020-07-28 23:09:34 +05:30
end
context 'without token' do
let(:token_input) { nil }
2021-06-08 01:23:25 +05:30
include_examples 'processes one firing and one resolved prometheus alerts'
2020-07-28 23:09:34 +05:30
end
context 'with token' do
2021-06-08 01:23:25 +05:30
it_behaves_like 'alerts service responds with an error and takes no actions', :unauthorized
2020-07-28 23:09:34 +05:30
end
end
2021-06-08 01:23:25 +05:30
context 'with project specific cluster using prometheus integration' do
where(:cluster_enabled, :integration_enabled, :configured_token, :token_input, :result) do
true | true | token | token | :success
true | true | nil | nil | :success
true | true | token | 'x' | :failure
true | true | token | nil | :failure
true | false | token | token | :failure
false | true | token | token | :failure
false | nil | nil | token | :failure
end
with_them do
before do
cluster.update!(enabled: cluster_enabled)
unless integration_enabled.nil?
2023-06-20 00:43:36 +05:30
create(
:clusters_integrations_prometheus,
cluster: cluster,
enabled: integration_enabled,
alert_manager_token: configured_token
)
2021-06-08 01:23:25 +05:30
end
end
case result = params[:result]
when :success
include_examples 'processes one firing and one resolved prometheus alerts'
2020-04-22 19:07:51 +05:30
when :failure
2021-06-08 01:23:25 +05:30
it_behaves_like 'alerts service responds with an error and takes no actions', :unauthorized
2020-04-22 19:07:51 +05:30
else
raise "invalid result: #{result.inspect}"
end
end
end
context 'without project specific cluster' do
2021-06-08 01:23:25 +05:30
let_it_be(:cluster) { create(:cluster, enabled: true) }
2020-04-22 19:07:51 +05:30
2021-06-08 01:23:25 +05:30
it_behaves_like 'alerts service responds with an error and takes no actions', :unauthorized
2020-04-22 19:07:51 +05:30
end
context 'with manual prometheus installation' do
where(:alerting_setting, :configured_token, :token_input, :result) do
true | token | token | :success
true | token | 'x' | :failure
true | token | nil | :failure
false | nil | nil | :success
false | nil | token | :failure
end
with_them do
let(:alert_manager_token) { token_input }
before do
2021-09-30 23:02:18 +05:30
create(:prometheus_integration, project: project)
2020-04-22 19:07:51 +05:30
if alerting_setting
2023-06-20 00:43:36 +05:30
create(
:project_alerting_setting,
project: project,
token: configured_token
)
2020-04-22 19:07:51 +05:30
end
end
case result = params[:result]
when :success
2021-06-08 01:23:25 +05:30
it_behaves_like 'processes one firing and one resolved prometheus alerts'
2020-04-22 19:07:51 +05:30
when :failure
2021-06-08 01:23:25 +05:30
it_behaves_like 'alerts service responds with an error and takes no actions', :unauthorized
2020-04-22 19:07:51 +05:30
else
raise "invalid result: #{result.inspect}"
end
end
end
2021-02-22 17:27:13 +05:30
context 'with HTTP integration' do
where(:active, :token, :result) do
2020-06-23 00:09:42 +05:30
:active | :valid | :success
:active | :invalid | :failure
:active | nil | :failure
:inactive | :valid | :failure
nil | nil | :failure
end
with_them do
2021-02-22 17:27:13 +05:30
let(:valid) { integration.token }
2020-06-23 00:09:42 +05:30
let(:invalid) { 'invalid token' }
let(:token_input) { public_send(token) if token }
2021-02-22 17:27:13 +05:30
let(:integration) { create(:alert_management_http_integration, active, project: project) if active }
2020-06-23 00:09:42 +05:30
2021-06-08 01:23:25 +05:30
subject { service.execute(token_input, integration) }
2020-06-23 00:09:42 +05:30
case result = params[:result]
when :success
2021-06-08 01:23:25 +05:30
it_behaves_like 'processes one firing and one resolved prometheus alerts'
2020-06-23 00:09:42 +05:30
when :failure
2021-06-08 01:23:25 +05:30
it_behaves_like 'alerts service responds with an error and takes no actions', :unauthorized
2020-06-23 00:09:42 +05:30
else
raise "invalid result: #{result.inspect}"
end
end
end
2021-06-08 01:23:25 +05:30
context 'incident settings' do
2020-04-22 19:07:51 +05:30
before do
2021-09-30 23:02:18 +05:30
create(:prometheus_integration, project: project)
2020-04-22 19:07:51 +05:30
create(:project_alerting_setting, project: project, token: token)
end
2021-06-08 01:23:25 +05:30
it_behaves_like 'processes one firing and one resolved prometheus alerts'
2020-04-22 19:07:51 +05:30
2021-06-08 01:23:25 +05:30
context 'when incident_management_setting does not exist' do
before do
setting.destroy!
2020-04-22 19:07:51 +05:30
end
2021-06-08 01:23:25 +05:30
it { is_expected.to be_success }
2022-08-27 11:52:29 +05:30
2021-06-08 01:23:25 +05:30
include_examples 'does not send alert notification emails'
include_examples 'does not process incident issues'
2020-04-22 19:07:51 +05:30
end
context 'incident_management_setting.send_email is false' do
2021-06-08 01:23:25 +05:30
before do
setting.update!(send_email: false)
2020-04-22 19:07:51 +05:30
end
2021-06-08 01:23:25 +05:30
it { is_expected.to be_success }
2022-08-27 11:52:29 +05:30
2021-06-08 01:23:25 +05:30
include_examples 'does not send alert notification emails'
end
2020-04-22 19:07:51 +05:30
2021-06-08 01:23:25 +05:30
context 'incident_management_setting.create_issue is false' do
before do
setting.update!(create_issue: false)
2020-04-22 19:07:51 +05:30
end
2021-06-08 01:23:25 +05:30
it { is_expected.to be_success }
2022-08-27 11:52:29 +05:30
2021-06-08 01:23:25 +05:30
include_examples 'does not process incident issues'
2020-04-22 19:07:51 +05:30
end
end
2020-05-24 23:13:21 +05:30
context 'process Alert Management alerts' do
let(:process_service) { instance_double(AlertManagement::ProcessPrometheusAlertService) }
before do
2021-09-30 23:02:18 +05:30
create(:prometheus_integration, project: project)
2020-05-24 23:13:21 +05:30
create(:project_alerting_setting, project: project, token: token)
end
context 'with multiple firing alerts and resolving alerts' do
let(:payload_raw) do
2020-06-23 00:09:42 +05:30
prometheus_alert_payload(firing: [alert_firing, alert_firing], resolved: [alert_resolved])
2020-05-24 23:13:21 +05:30
end
it 'processes Prometheus alerts' do
expect(AlertManagement::ProcessPrometheusAlertService)
.to receive(:new)
2021-02-22 17:27:13 +05:30
.with(project, kind_of(Hash))
2020-05-24 23:13:21 +05:30
.exactly(3).times
2021-12-11 22:18:48 +05:30
.and_call_original
2020-05-24 23:13:21 +05:30
subject
end
end
end
2022-03-02 08:16:31 +05:30
context 'when payload exceeds max amount of processable alerts' do
# We are defining 2 alerts in payload_raw above
let(:max_alerts) { 1 }
2022-08-13 15:12:31 +05:30
let(:fingerprint) { prometheus_alert_payload_fingerprint(alert_resolved) }
2022-03-02 08:16:31 +05:30
before do
stub_const("#{described_class}::PROCESS_MAX_ALERTS", max_alerts)
create(:prometheus_integration, project: project)
create(:project_alerting_setting, project: project, token: token)
2022-08-13 15:12:31 +05:30
create(:alert_management_alert, project: project, fingerprint: fingerprint)
2022-03-02 08:16:31 +05:30
allow(Gitlab::AppLogger).to receive(:warn)
end
shared_examples 'process truncated alerts' do
2023-01-13 00:05:48 +05:30
it 'returns 201 but skips processing and logs a warning', :aggregate_failures do
2022-03-02 08:16:31 +05:30
expect(subject).to be_success
2023-01-13 00:05:48 +05:30
expect(subject.payload).to eq({})
expect(subject.http_status).to eq(:created)
2022-03-02 08:16:31 +05:30
expect(Gitlab::AppLogger)
.to have_received(:warn)
.with(
message: 'Prometheus payload exceeded maximum amount of alerts. Truncating alerts.',
project_id: project.id,
alerts: {
total: 2,
max: max_alerts
})
end
end
shared_examples 'process all alerts' do
2023-01-13 00:05:48 +05:30
it 'returns 201 and process alerts without warnings', :aggregate_failures do
2022-03-02 08:16:31 +05:30
expect(subject).to be_success
2023-01-13 00:05:48 +05:30
expect(subject.payload).to eq({})
expect(subject.http_status).to eq(:created)
2022-03-02 08:16:31 +05:30
expect(Gitlab::AppLogger).not_to have_received(:warn)
end
end
context 'with feature flag globally enabled' do
before do
stub_feature_flags(prometheus_notify_max_alerts: true)
end
include_examples 'process truncated alerts'
end
context 'with feature flag enabled on project' do
before do
stub_feature_flags(prometheus_notify_max_alerts: project)
end
include_examples 'process truncated alerts'
end
context 'with feature flag enabled on unrelated project' do
let(:another_project) { create(:project) }
before do
stub_feature_flags(prometheus_notify_max_alerts: another_project)
end
include_examples 'process all alerts'
end
context 'with feature flag disabled' do
before do
stub_feature_flags(prometheus_notify_max_alerts: false)
end
include_examples 'process all alerts'
end
end
2020-04-22 19:07:51 +05:30
end
context 'with invalid payload' do
2020-06-23 00:09:42 +05:30
context 'when payload is not processable' do
2020-04-22 19:07:51 +05:30
let(:payload) { {} }
2020-06-23 00:09:42 +05:30
before do
allow(described_class).to receive(:processable?).with(payload)
.and_return(false)
end
2020-04-22 19:07:51 +05:30
2021-06-08 01:23:25 +05:30
it_behaves_like 'alerts service responds with an error and takes no actions', :unprocessable_entity
2020-04-22 19:07:51 +05:30
end
context 'when the payload is too big' do
2022-08-27 11:52:29 +05:30
let(:payload_raw) { { 'the-payload-is-too-big' => true } }
let(:payload) { ActionController::Parameters.new(payload_raw).permit! }
2020-04-22 19:07:51 +05:30
before do
2022-08-27 11:52:29 +05:30
stub_const('::Gitlab::Utils::DeepSize::DEFAULT_MAX_DEPTH', 0)
2020-04-22 19:07:51 +05:30
end
2021-06-08 01:23:25 +05:30
it_behaves_like 'alerts service responds with an error and takes no actions', :bad_request
2020-04-22 19:07:51 +05:30
end
end
2020-06-23 00:09:42 +05:30
describe '.processable?' do
let(:valid_payload) { prometheus_alert_payload }
subject { described_class.processable?(payload) }
context 'with valid payload' do
let(:payload) { valid_payload }
it { is_expected.to eq(true) }
context 'containing unrelated keys' do
let(:payload) { valid_payload.merge('unrelated' => 'key') }
2020-04-22 19:07:51 +05:30
2020-06-23 00:09:42 +05:30
it { is_expected.to eq(true) }
end
end
context 'with invalid payload' do
where(:missing_key) do
described_class::REQUIRED_PAYLOAD_KEYS.to_a
end
with_them do
let(:payload) { valid_payload.except(missing_key) }
it { is_expected.to eq(false) }
end
end
context 'with unsupported version' do
let(:payload) { valid_payload.merge('version' => '5') }
it { is_expected.to eq(false) }
end
2020-04-22 19:07:51 +05:30
end
end