debian-mirror-gitlab/spec/experiments/application_experiment_spec.rb

455 lines
15 KiB
Ruby
Raw Normal View History

2021-02-22 17:27:13 +05:30
# frozen_string_literal: true
require 'spec_helper'
2021-03-11 19:13:27 +05:30
RSpec.describe ApplicationExperiment, :experiment do
2022-01-26 12:08:38 +05:30
subject(:application_experiment) { described_class.new('namespaced/stub', **context) }
2021-03-11 19:13:27 +05:30
2021-09-30 23:02:18 +05:30
let(:context) { {} }
let(:feature_definition) { { name: 'namespaced_stub', type: 'experiment', default_enabled: false } }
2021-03-11 19:13:27 +05:30
around do |example|
Feature::Definition.definitions[:namespaced_stub] = Feature::Definition.new('namespaced_stub.yml', feature_definition)
example.run
Feature::Definition.definitions.delete(:namespaced_stub)
end
before do
2022-01-26 12:08:38 +05:30
allow(application_experiment).to receive(:enabled?).and_return(true)
2021-03-11 19:13:27 +05:30
end
2021-06-08 01:23:25 +05:30
it "doesn't raise an exception without a defined control" do
# because we have a default behavior defined
expect { experiment('namespaced/stub') { } }.not_to raise_error
end
2021-09-30 23:02:18 +05:30
describe "#enabled?" do
2021-03-11 19:13:27 +05:30
before do
2022-01-26 12:08:38 +05:30
allow(application_experiment).to receive(:enabled?).and_call_original
2021-03-11 19:13:27 +05:30
allow(Feature::Definition).to receive(:get).and_return('_instance_')
allow(Gitlab).to receive(:dev_env_or_com?).and_return(true)
allow(Feature).to receive(:get).and_return(double(state: :on))
end
it "is enabled when all criteria are met" do
2022-01-26 12:08:38 +05:30
expect(application_experiment).to be_enabled
2021-03-11 19:13:27 +05:30
end
it "isn't enabled if the feature definition doesn't exist" do
expect(Feature::Definition).to receive(:get).with('namespaced_stub').and_return(nil)
2022-01-26 12:08:38 +05:30
expect(application_experiment).not_to be_enabled
2021-03-11 19:13:27 +05:30
end
it "isn't enabled if we're not in dev or dotcom environments" do
expect(Gitlab).to receive(:dev_env_or_com?).and_return(false)
2022-01-26 12:08:38 +05:30
expect(application_experiment).not_to be_enabled
2021-03-11 19:13:27 +05:30
end
it "isn't enabled if the feature flag state is :off" do
expect(Feature).to receive(:get).with('namespaced_stub').and_return(double(state: :off))
2022-01-26 12:08:38 +05:30
expect(application_experiment).not_to be_enabled
2021-03-11 19:13:27 +05:30
end
end
2021-02-22 17:27:13 +05:30
2021-09-30 23:02:18 +05:30
describe "#publish" do
2021-10-27 15:23:28 +05:30
let(:should_track) { true }
2021-04-17 20:07:23 +05:30
2021-10-27 15:23:28 +05:30
before do
2022-01-26 12:08:38 +05:30
allow(application_experiment).to receive(:should_track?).and_return(should_track)
2021-04-17 20:07:23 +05:30
end
2021-10-27 15:23:28 +05:30
it "tracks the assignment", :snowplow do
2022-01-26 12:08:38 +05:30
application_experiment.publish
2021-10-27 15:23:28 +05:30
expect_snowplow_event(
category: 'namespaced/stub',
action: 'assignment',
context: [{ schema: anything, data: anything }]
)
2021-02-22 17:27:13 +05:30
end
2021-10-27 15:23:28 +05:30
it "publishes to the client" do
2022-01-26 12:08:38 +05:30
expect(application_experiment).to receive(:publish_to_client)
2021-04-17 20:07:23 +05:30
2022-01-26 12:08:38 +05:30
application_experiment.publish
2021-02-22 17:27:13 +05:30
end
2021-10-27 15:23:28 +05:30
context 'when we should not track' do
let(:should_track) { false }
it 'does not track an event to Snowplow', :snowplow do
2022-01-26 12:08:38 +05:30
application_experiment.publish
2021-10-27 15:23:28 +05:30
expect_no_snowplow_event
end
end
2021-09-30 23:02:18 +05:30
describe "#publish_to_client" do
it "adds the data into Gon" do
signature = { key: '86208ac54ca798e11f127e8b23ec396a', variant: 'control' }
expect(Gon).to receive(:push).with({ experiment: { 'namespaced/stub' => hash_including(signature) } }, true)
2021-04-17 20:07:23 +05:30
2022-01-26 12:08:38 +05:30
application_experiment.publish_to_client
2021-09-30 23:02:18 +05:30
end
2021-09-04 01:27:46 +05:30
2021-09-30 23:02:18 +05:30
it "handles when Gon raises exceptions (like when it can't be pushed into)" do
expect(Gon).to receive(:push).and_raise(NoMethodError)
2021-09-04 01:27:46 +05:30
2022-01-26 12:08:38 +05:30
expect { application_experiment.publish_to_client }.not_to raise_error
2021-09-30 23:02:18 +05:30
end
2021-10-27 15:23:28 +05:30
context 'when we should not track' do
let(:should_track) { false }
it 'returns early' do
expect(Gon).not_to receive(:push)
2022-01-26 12:08:38 +05:30
application_experiment.publish_to_client
2021-10-27 15:23:28 +05:30
end
end
2021-09-04 01:27:46 +05:30
end
2021-10-27 15:23:28 +05:30
describe '#publish_to_database' do
2021-09-30 23:02:18 +05:30
using RSpec::Parameterized::TableSyntax
2022-01-26 12:08:38 +05:30
let(:publish_to_database) { application_experiment.publish_to_database }
2021-10-27 15:23:28 +05:30
shared_examples 'does not record to the database' do
it 'does not create an experiment record' do
2022-01-26 12:08:38 +05:30
expect { publish_to_database }.not_to change(Experiment, :count)
2021-10-27 15:23:28 +05:30
end
it 'does not create an experiment subject record' do
2022-01-26 12:08:38 +05:30
expect { publish_to_database }.not_to change(ExperimentSubject, :count)
2021-10-27 15:23:28 +05:30
end
2021-09-04 01:27:46 +05:30
end
2021-10-27 15:23:28 +05:30
context 'when there is a usable subject' do
let(:context) { { context_key => context_value } }
2021-09-30 23:02:18 +05:30
where(:context_key, :context_value, :object_type) do
2022-01-26 12:08:38 +05:30
:namespace | build(:namespace, id: non_existing_record_id) | :namespace
:group | build(:namespace, id: non_existing_record_id) | :namespace
:project | build(:project, id: non_existing_record_id) | :project
:user | build(:user, id: non_existing_record_id) | :user
:actor | build(:user, id: non_existing_record_id) | :user
2021-09-30 23:02:18 +05:30
end
2021-09-04 01:27:46 +05:30
2021-09-30 23:02:18 +05:30
with_them do
2021-10-27 15:23:28 +05:30
it 'creates an experiment and experiment subject record' do
2022-01-26 12:08:38 +05:30
expect { publish_to_database }.to change(Experiment, :count).by(1)
2021-09-04 01:27:46 +05:30
2021-09-30 23:02:18 +05:30
expect(Experiment.last.name).to eq('namespaced/stub')
expect(ExperimentSubject.last.send(object_type)).to eq(context[context_key])
end
2021-09-04 01:27:46 +05:30
end
end
2022-01-26 12:08:38 +05:30
context "when experiment hasn't ran" do
let(:context) { { user: create(:user) } }
it 'sets a variant on the experiment subject' do
publish_to_database
expect(ExperimentSubject.last.variant).to eq('control')
end
end
2021-10-27 15:23:28 +05:30
context 'when there is not a usable subject' do
let(:context) { { context_key => context_value } }
2021-09-30 23:02:18 +05:30
where(:context_key, :context_value) do
:namespace | nil
:foo | :bar
end
2021-09-04 01:27:46 +05:30
2021-09-30 23:02:18 +05:30
with_them do
2021-10-27 15:23:28 +05:30
include_examples 'does not record to the database'
2021-09-04 01:27:46 +05:30
end
end
2021-10-27 15:23:28 +05:30
context 'but we should not track' do
let(:should_track) { false }
include_examples 'does not record to the database'
end
2021-09-04 01:27:46 +05:30
end
end
2021-09-30 23:02:18 +05:30
describe "#track", :snowplow do
let(:fake_context) do
SnowplowTracker::SelfDescribingJson.new('iglu:com.gitlab/fake/jsonschema/0-0-0', { data: '_data_' })
end
2021-03-11 19:13:27 +05:30
it "doesn't track if we shouldn't track" do
2022-01-26 12:08:38 +05:30
allow(application_experiment).to receive(:should_track?).and_return(false)
2021-02-22 17:27:13 +05:30
2022-01-26 12:08:38 +05:30
application_experiment.track(:action)
2021-02-22 17:27:13 +05:30
expect_no_snowplow_event
end
it "tracks the event with the expected arguments and merged contexts" do
2022-01-26 12:08:38 +05:30
application_experiment.track(:action, property: '_property_', context: [fake_context])
2021-02-22 17:27:13 +05:30
expect_snowplow_event(
2021-03-11 19:13:27 +05:30
category: 'namespaced/stub',
2021-02-22 17:27:13 +05:30
action: 'action',
property: '_property_',
context: [
{
schema: 'iglu:com.gitlab/fake/jsonschema/0-0-0',
data: { data: '_data_' }
},
{
2021-04-29 21:17:54 +05:30
schema: 'iglu:com.gitlab/gitlab_experiment/jsonschema/1-0-0',
2021-03-11 19:13:27 +05:30
data: { experiment: 'namespaced/stub', key: '86208ac54ca798e11f127e8b23ec396a', variant: 'control' }
2021-02-22 17:27:13 +05:30
}
]
)
end
2021-09-30 23:02:18 +05:30
it "tracks the event correctly even when using the base class" do
subject = Gitlab::Experiment.new(:unnamed)
subject.track(:action, context: [fake_context])
expect_snowplow_event(
category: 'unnamed',
action: 'action',
context: [
{
schema: 'iglu:com.gitlab/fake/jsonschema/0-0-0',
data: { data: '_data_' }
},
{
schema: 'iglu:com.gitlab/gitlab_experiment/jsonschema/1-0-0',
data: { experiment: 'unnamed', key: subject.context.key, variant: 'control' }
}
]
)
end
2022-01-26 12:08:38 +05:30
context "when using known context resources" do
let(:user) { build(:user, id: non_existing_record_id) }
let(:project) { build(:project, id: non_existing_record_id) }
let(:namespace) { build(:namespace, id: non_existing_record_id) }
let(:group) { build(:group, id: non_existing_record_id) }
let(:actor) { user }
let(:context) { { user: user, project: project, namespace: namespace } }
it "includes those using the gitlab standard context" do
subject.track(:action)
expect_snowplow_event(
category: 'namespaced/stub',
action: 'action',
user: user,
project: project,
namespace: namespace,
context: an_instance_of(Array)
)
end
it "falls back to using the group key" do
subject.context(namespace: nil, group: group)
subject.track(:action)
expect_snowplow_event(
category: 'namespaced/stub',
action: 'action',
user: user,
project: project,
namespace: group,
context: an_instance_of(Array)
)
end
context "with the actor key" do
it "provides it to the tracking call as the user" do
subject.context(user: nil, actor: actor)
subject.track(:action)
expect_snowplow_event(
category: 'namespaced/stub',
action: 'action',
user: actor,
project: project,
namespace: namespace,
context: an_instance_of(Array)
)
end
it "handles when it's not a user record" do
subject.context(user: nil, actor: nil)
subject.track(:action)
expect_snowplow_event(
category: 'namespaced/stub',
action: 'action',
project: project,
namespace: namespace,
context: an_instance_of(Array)
)
end
end
end
2021-09-30 23:02:18 +05:30
end
describe "#key_for" do
it "generates MD5 hashes" do
2022-01-26 12:08:38 +05:30
expect(application_experiment.key_for(foo: :bar)).to eq('6f9ac12afdb9b58c2f19a136d09f9153')
2021-09-30 23:02:18 +05:30
end
2021-02-22 17:27:13 +05:30
end
2021-10-27 15:23:28 +05:30
describe "#process_redirect_url" do
using RSpec::Parameterized::TableSyntax
where(:url, :processed_url) do
'https://about.gitlab.com/' | 'https://about.gitlab.com/'
'https://gitlab.com/' | 'https://gitlab.com/'
'http://docs.gitlab.com' | 'http://docs.gitlab.com'
'https://docs.gitlab.com/some/path?foo=bar' | 'https://docs.gitlab.com/some/path?foo=bar'
'http://badgitlab.com' | nil
'https://gitlab.com.nefarious.net' | nil
'https://unknown.gitlab.com' | nil
"https://badplace.com\nhttps://gitlab.com" | nil
'https://gitlabbcom' | nil
'https://gitlabbcom/' | nil
2022-01-26 12:08:38 +05:30
'http://gdk.test/foo/bar' | 'http://gdk.test/foo/bar'
'http://localhost:3000/foo/bar' | 'http://localhost:3000/foo/bar'
2021-10-27 15:23:28 +05:30
end
with_them do
it "returns the url or nil if invalid" do
allow(Gitlab).to receive(:dev_env_or_com?).and_return(true)
2022-01-26 12:08:38 +05:30
expect(application_experiment.process_redirect_url(url)).to eq(processed_url)
2021-10-27 15:23:28 +05:30
end
it "considers all urls invalid when not on dev or com" do
allow(Gitlab).to receive(:dev_env_or_com?).and_return(false)
2022-01-26 12:08:38 +05:30
expect(application_experiment.process_redirect_url(url)).to be_nil
2021-10-27 15:23:28 +05:30
end
end
it "generates the correct urls based on where the engine was mounted" do
2022-01-26 12:08:38 +05:30
url = Rails.application.routes.url_helpers.experiment_redirect_url(application_experiment, url: 'https://docs.gitlab.com')
expect(url).to include("/-/experiment/namespaced%2Fstub:#{application_experiment.context.key}?https://docs.gitlab.com")
2021-10-27 15:23:28 +05:30
end
end
2021-09-30 23:02:18 +05:30
context "when resolving variants" do
2021-04-17 20:07:23 +05:30
it "uses the default value as specified in the yaml" do
2022-01-26 12:08:38 +05:30
expect(Feature).to receive(:enabled?).with('namespaced_stub', application_experiment, type: :experiment, default_enabled: :yaml)
2021-03-11 19:13:27 +05:30
2022-01-26 12:08:38 +05:30
expect(application_experiment.variant.name).to eq('control')
2021-02-22 17:27:13 +05:30
end
2021-04-17 20:07:23 +05:30
context "when rolled out to 100%" do
before do
stub_feature_flags(namespaced_stub: true)
2021-03-11 19:13:27 +05:30
end
2021-04-17 20:07:23 +05:30
it "returns the first variant name" do
2022-01-26 12:08:38 +05:30
application_experiment.try(:variant1) {}
application_experiment.try(:variant2) {}
2021-03-11 19:13:27 +05:30
2022-01-26 12:08:38 +05:30
expect(application_experiment.variant.name).to eq('variant1')
2021-02-22 17:27:13 +05:30
end
end
end
2022-01-26 12:08:38 +05:30
context "when nesting experiments" do
before do
stub_experiments(top: :control, nested: :control)
end
it "doesn't raise an exception" do
expect { experiment(:top) { |e| e.control { experiment(:nested) { } } } }.not_to raise_error
end
it "tracks an event", :snowplow do
experiment(:top) { |e| e.control { experiment(:nested) { } } }
expect(Gitlab::Tracking).to have_received(:event).with( # rubocop:disable RSpec/ExpectGitlabTracking
'top',
'nested',
hash_including(label: 'nested')
)
end
end
2021-02-22 17:27:13 +05:30
context "when caching" do
2021-04-17 20:07:23 +05:30
let(:cache) { Gitlab::Experiment::Configuration.cache }
2021-02-22 17:27:13 +05:30
before do
2021-04-17 20:07:23 +05:30
allow(Gitlab::Experiment::Configuration).to receive(:cache).and_call_original
2022-01-26 12:08:38 +05:30
cache.clear(key: application_experiment.name)
2021-02-22 17:27:13 +05:30
2022-01-26 12:08:38 +05:30
application_experiment.use { } # setup the control
application_experiment.try { } # setup the candidate
2021-02-22 17:27:13 +05:30
end
it "caches the variant determined by the variant resolver" do
2022-01-26 12:08:38 +05:30
expect(application_experiment.variant.name).to eq('candidate') # we should be in the experiment
2021-02-22 17:27:13 +05:30
2022-01-26 12:08:38 +05:30
application_experiment.run
2021-02-22 17:27:13 +05:30
2022-01-26 12:08:38 +05:30
expect(application_experiment.cache.read).to eq('candidate')
2021-02-22 17:27:13 +05:30
end
it "doesn't cache a variant if we don't explicitly provide one" do
# by not caching "empty" variants, we effectively create a mostly
# optimal combination of caching and rollout flexibility. If we cached
# every control variant assigned, we'd inflate the cache size and
# wouldn't be able to roll out to subjects that we'd already assigned to
# the control.
2021-03-11 19:13:27 +05:30
stub_feature_flags(namespaced_stub: false) # simulate being not rolled out
2021-02-22 17:27:13 +05:30
2022-01-26 12:08:38 +05:30
expect(application_experiment.variant.name).to eq('control') # if we ask, it should be control
2021-02-22 17:27:13 +05:30
2022-01-26 12:08:38 +05:30
application_experiment.run
2021-02-22 17:27:13 +05:30
2022-01-26 12:08:38 +05:30
expect(application_experiment.cache.read).to be_nil
2021-02-22 17:27:13 +05:30
end
it "caches a control variant if we assign it specifically" do
# by specifically assigning the control variant here, we're guaranteeing
# that this context will always get the control variant unless we delete
# the field from the cache (or clear the entire experiment cache) -- or
# write code that would specify a different variant.
2022-01-26 12:08:38 +05:30
application_experiment.run(:control)
2021-02-22 17:27:13 +05:30
2022-01-26 12:08:38 +05:30
expect(application_experiment.cache.read).to eq('control')
2021-04-17 20:07:23 +05:30
end
context "arbitrary attributes" do
before do
2022-01-26 12:08:38 +05:30
application_experiment.cache.store.clear(key: application_experiment.name + '_attrs')
2021-04-17 20:07:23 +05:30
end
it "sets and gets attributes about an experiment" do
2022-01-26 12:08:38 +05:30
application_experiment.cache.attr_set(:foo, :bar)
2021-04-17 20:07:23 +05:30
2022-01-26 12:08:38 +05:30
expect(application_experiment.cache.attr_get(:foo)).to eq('bar')
2021-04-17 20:07:23 +05:30
end
it "increments a value for an experiment" do
2022-01-26 12:08:38 +05:30
expect(application_experiment.cache.attr_get(:foo)).to be_nil
2021-04-17 20:07:23 +05:30
2022-01-26 12:08:38 +05:30
expect(application_experiment.cache.attr_inc(:foo)).to eq(1)
expect(application_experiment.cache.attr_inc(:foo)).to eq(2)
2021-04-17 20:07:23 +05:30
end
2021-02-22 17:27:13 +05:30
end
end
end