2019-12-21 20:55:43 +05:30
|
|
|
# frozen_string_literal: true
|
|
|
|
|
|
|
|
# == Experimentation
|
|
|
|
#
|
2020-05-24 23:13:21 +05:30
|
|
|
# Utility module for A/B testing experimental features. Define your experiments in the `EXPERIMENTS` constant.
|
|
|
|
# Experiment options:
|
|
|
|
# - environment (optional, defaults to enabled for development and GitLab.com)
|
|
|
|
# - tracking_category (optional, used to set the category when tracking an experiment event)
|
|
|
|
#
|
|
|
|
# The experiment is controlled by a Feature Flag (https://docs.gitlab.com/ee/development/feature_flags/controls.html),
|
|
|
|
# which is named "#{experiment_key}_experiment_percentage" and *must* be set with a percentage and not be used for other purposes.
|
|
|
|
#
|
|
|
|
# To enable the experiment for 10% of the users:
|
|
|
|
#
|
|
|
|
# chatops: `/chatops run feature set experiment_key_experiment_percentage 10`
|
2020-06-23 00:09:42 +05:30
|
|
|
# console: `Feature.enable_percentage_of_time(:experiment_key_experiment_percentage, 10)`
|
2020-05-24 23:13:21 +05:30
|
|
|
#
|
|
|
|
# To disable the experiment:
|
|
|
|
#
|
|
|
|
# chatops: `/chatops run feature delete experiment_key_experiment_percentage`
|
2020-06-23 00:09:42 +05:30
|
|
|
# console: `Feature.remove(:experiment_key_experiment_percentage)`
|
2020-05-24 23:13:21 +05:30
|
|
|
#
|
|
|
|
# To check the current rollout percentage:
|
|
|
|
#
|
|
|
|
# chatops: `/chatops run feature get experiment_key_experiment_percentage`
|
|
|
|
# console: `Feature.get(:experiment_key_experiment_percentage).percentage_of_time_value`
|
2019-12-21 20:55:43 +05:30
|
|
|
#
|
2020-06-23 00:09:42 +05:30
|
|
|
|
|
|
|
# TODO: see https://gitlab.com/gitlab-org/gitlab/-/issues/217490
|
2019-12-21 20:55:43 +05:30
|
|
|
module Gitlab
|
|
|
|
module Experimentation
|
|
|
|
EXPERIMENTS = {
|
|
|
|
signup_flow: {
|
2019-12-26 22:10:19 +05:30
|
|
|
tracking_category: 'Growth::Acquisition::Experiment::SignUpFlow'
|
2020-03-13 15:44:24 +05:30
|
|
|
},
|
2020-05-24 23:13:21 +05:30
|
|
|
onboarding_issues: {
|
|
|
|
tracking_category: 'Growth::Conversion::Experiment::OnboardingIssues'
|
2020-03-13 15:44:24 +05:30
|
|
|
},
|
|
|
|
suggest_pipeline: {
|
|
|
|
tracking_category: 'Growth::Expansion::Experiment::SuggestPipeline'
|
2020-04-22 19:07:51 +05:30
|
|
|
},
|
|
|
|
ci_notification_dot: {
|
|
|
|
tracking_category: 'Growth::Expansion::Experiment::CiNotificationDot'
|
|
|
|
},
|
2020-05-24 23:13:21 +05:30
|
|
|
upgrade_link_in_user_menu_a: {
|
|
|
|
tracking_category: 'Growth::Expansion::Experiment::UpgradeLinkInUserMenuA'
|
2020-06-23 00:09:42 +05:30
|
|
|
},
|
|
|
|
invite_members_version_a: {
|
|
|
|
tracking_category: 'Growth::Expansion::Experiment::InviteMembersVersionA'
|
|
|
|
},
|
2021-01-03 14:25:43 +05:30
|
|
|
invite_members_version_b: {
|
|
|
|
tracking_category: 'Growth::Expansion::Experiment::InviteMembersVersionB'
|
|
|
|
},
|
2020-06-23 00:09:42 +05:30
|
|
|
new_create_project_ui: {
|
|
|
|
tracking_category: 'Manage::Import::Experiment::NewCreateProjectUi'
|
2020-10-24 23:57:45 +05:30
|
|
|
},
|
|
|
|
contact_sales_btn_in_app: {
|
|
|
|
tracking_category: 'Growth::Conversion::Experiment::ContactSalesInApp'
|
|
|
|
},
|
|
|
|
customize_homepage: {
|
|
|
|
tracking_category: 'Growth::Expansion::Experiment::CustomizeHomepage'
|
2020-11-24 15:15:51 +05:30
|
|
|
},
|
|
|
|
invite_email: {
|
|
|
|
tracking_category: 'Growth::Acquisition::Experiment::InviteEmail'
|
2021-01-03 14:25:43 +05:30
|
|
|
},
|
|
|
|
invitation_reminders: {
|
|
|
|
tracking_category: 'Growth::Acquisition::Experiment::InvitationReminders'
|
|
|
|
},
|
|
|
|
group_only_trials: {
|
|
|
|
tracking_category: 'Growth::Conversion::Experiment::GroupOnlyTrials'
|
|
|
|
},
|
|
|
|
default_to_issues_board: {
|
|
|
|
tracking_category: 'Growth::Conversion::Experiment::DefaultToIssuesBoard'
|
2019-12-21 20:55:43 +05:30
|
|
|
}
|
|
|
|
}.freeze
|
|
|
|
|
2020-10-24 23:57:45 +05:30
|
|
|
GROUP_CONTROL = :control
|
|
|
|
GROUP_EXPERIMENTAL = :experimental
|
|
|
|
|
2020-05-24 23:13:21 +05:30
|
|
|
# Controller concern that checks if an `experimentation_subject_id cookie` is present and sets it if absent.
|
2019-12-21 20:55:43 +05:30
|
|
|
# Used for A/B testing of experimental features. Exposes the `experiment_enabled?(experiment_name)` method
|
2019-12-26 22:10:19 +05:30
|
|
|
# to controllers and views. It returns true when the experiment is enabled and the user is selected as part
|
|
|
|
# of the experimental group.
|
2019-12-21 20:55:43 +05:30
|
|
|
#
|
|
|
|
module ControllerConcern
|
|
|
|
extend ActiveSupport::Concern
|
|
|
|
|
|
|
|
included do
|
2020-04-08 14:13:33 +05:30
|
|
|
before_action :set_experimentation_subject_id_cookie, unless: :dnt_enabled?
|
2020-11-24 15:15:51 +05:30
|
|
|
helper_method :experiment_enabled?, :experiment_tracking_category_and_group
|
2019-12-21 20:55:43 +05:30
|
|
|
end
|
|
|
|
|
|
|
|
def set_experimentation_subject_id_cookie
|
|
|
|
return if cookies[:experimentation_subject_id].present?
|
|
|
|
|
|
|
|
cookies.permanent.signed[:experimentation_subject_id] = {
|
|
|
|
value: SecureRandom.uuid,
|
2019-12-26 22:10:19 +05:30
|
|
|
secure: ::Gitlab.config.gitlab.https,
|
|
|
|
httponly: true
|
2019-12-21 20:55:43 +05:30
|
|
|
}
|
|
|
|
end
|
|
|
|
|
2021-01-03 14:25:43 +05:30
|
|
|
def push_frontend_experiment(experiment_key)
|
|
|
|
var_name = experiment_key.to_s.camelize(:lower)
|
|
|
|
enabled = experiment_enabled?(experiment_key)
|
|
|
|
|
|
|
|
gon.push({ experiments: { var_name => enabled } }, true)
|
|
|
|
end
|
|
|
|
|
2019-12-21 20:55:43 +05:30
|
|
|
def experiment_enabled?(experiment_key)
|
2020-04-08 14:13:33 +05:30
|
|
|
return false if dnt_enabled?
|
|
|
|
|
2021-01-03 14:25:43 +05:30
|
|
|
return true if Experimentation.enabled_for_value?(experiment_key, experimentation_subject_index)
|
2020-04-08 14:13:33 +05:30
|
|
|
return true if forced_enabled?(experiment_key)
|
|
|
|
|
|
|
|
false
|
2019-12-26 22:10:19 +05:30
|
|
|
end
|
|
|
|
|
2020-03-13 15:44:24 +05:30
|
|
|
def track_experiment_event(experiment_key, action, value = nil)
|
2021-01-03 14:25:43 +05:30
|
|
|
return if dnt_enabled?
|
|
|
|
|
2020-03-13 15:44:24 +05:30
|
|
|
track_experiment_event_for(experiment_key, action, value) do |tracking_data|
|
2021-01-03 14:25:43 +05:30
|
|
|
::Gitlab::Tracking.event(tracking_data.delete(:category), tracking_data.delete(:action), **tracking_data)
|
2019-12-26 22:10:19 +05:30
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2020-03-13 15:44:24 +05:30
|
|
|
def frontend_experimentation_tracking_data(experiment_key, action, value = nil)
|
2021-01-03 14:25:43 +05:30
|
|
|
return if dnt_enabled?
|
|
|
|
|
2020-03-13 15:44:24 +05:30
|
|
|
track_experiment_event_for(experiment_key, action, value) do |tracking_data|
|
2019-12-26 22:10:19 +05:30
|
|
|
gon.push(tracking_data: tracking_data)
|
|
|
|
end
|
2019-12-21 20:55:43 +05:30
|
|
|
end
|
|
|
|
|
2020-10-24 23:57:45 +05:30
|
|
|
def record_experiment_user(experiment_key)
|
2021-01-03 14:25:43 +05:30
|
|
|
return if dnt_enabled?
|
2020-10-24 23:57:45 +05:30
|
|
|
return unless Experimentation.enabled?(experiment_key) && current_user
|
|
|
|
|
|
|
|
::Experiment.add_user(experiment_key, tracking_group(experiment_key), current_user)
|
|
|
|
end
|
|
|
|
|
2020-11-24 15:15:51 +05:30
|
|
|
def experiment_tracking_category_and_group(experiment_key)
|
|
|
|
"#{tracking_category(experiment_key)}:#{tracking_group(experiment_key, '_group')}"
|
|
|
|
end
|
|
|
|
|
2019-12-21 20:55:43 +05:30
|
|
|
private
|
|
|
|
|
2020-04-08 14:13:33 +05:30
|
|
|
def dnt_enabled?
|
|
|
|
Gitlab::Utils.to_boolean(request.headers['DNT'])
|
|
|
|
end
|
|
|
|
|
2019-12-26 22:10:19 +05:30
|
|
|
def experimentation_subject_id
|
|
|
|
cookies.signed[:experimentation_subject_id]
|
|
|
|
end
|
|
|
|
|
2019-12-21 20:55:43 +05:30
|
|
|
def experimentation_subject_index
|
|
|
|
return if experimentation_subject_id.blank?
|
|
|
|
|
|
|
|
experimentation_subject_id.delete('-').hex % 100
|
|
|
|
end
|
2019-12-26 22:10:19 +05:30
|
|
|
|
2020-03-13 15:44:24 +05:30
|
|
|
def track_experiment_event_for(experiment_key, action, value)
|
2019-12-26 22:10:19 +05:30
|
|
|
return unless Experimentation.enabled?(experiment_key)
|
|
|
|
|
2020-03-13 15:44:24 +05:30
|
|
|
yield experimentation_tracking_data(experiment_key, action, value)
|
2019-12-26 22:10:19 +05:30
|
|
|
end
|
|
|
|
|
2020-03-13 15:44:24 +05:30
|
|
|
def experimentation_tracking_data(experiment_key, action, value)
|
2019-12-26 22:10:19 +05:30
|
|
|
{
|
|
|
|
category: tracking_category(experiment_key),
|
|
|
|
action: action,
|
2020-11-24 15:15:51 +05:30
|
|
|
property: tracking_group(experiment_key, "_group"),
|
2020-03-13 15:44:24 +05:30
|
|
|
label: experimentation_subject_id,
|
|
|
|
value: value
|
|
|
|
}.compact
|
2019-12-26 22:10:19 +05:30
|
|
|
end
|
|
|
|
|
|
|
|
def tracking_category(experiment_key)
|
|
|
|
Experimentation.experiment(experiment_key).tracking_category
|
|
|
|
end
|
|
|
|
|
2020-11-24 15:15:51 +05:30
|
|
|
def tracking_group(experiment_key, suffix = nil)
|
2019-12-26 22:10:19 +05:30
|
|
|
return unless Experimentation.enabled?(experiment_key)
|
|
|
|
|
2020-11-24 15:15:51 +05:30
|
|
|
group = experiment_enabled?(experiment_key) ? GROUP_EXPERIMENTAL : GROUP_CONTROL
|
|
|
|
|
|
|
|
suffix ? "#{group}#{suffix}" : group
|
2019-12-26 22:10:19 +05:30
|
|
|
end
|
|
|
|
|
|
|
|
def forced_enabled?(experiment_key)
|
|
|
|
params.has_key?(:force_experiment) && params[:force_experiment] == experiment_key.to_s
|
|
|
|
end
|
2019-12-21 20:55:43 +05:30
|
|
|
end
|
|
|
|
|
|
|
|
class << self
|
|
|
|
def experiment(key)
|
|
|
|
Experiment.new(EXPERIMENTS[key].merge(key: key))
|
|
|
|
end
|
|
|
|
|
2019-12-26 22:10:19 +05:30
|
|
|
def enabled?(experiment_key)
|
2019-12-21 20:55:43 +05:30
|
|
|
return false unless EXPERIMENTS.key?(experiment_key)
|
|
|
|
|
|
|
|
experiment = experiment(experiment_key)
|
2020-05-24 23:13:21 +05:30
|
|
|
experiment.enabled? && experiment.enabled_for_environment?
|
2019-12-26 22:10:19 +05:30
|
|
|
end
|
2019-12-21 20:55:43 +05:30
|
|
|
|
2021-01-03 14:25:43 +05:30
|
|
|
def enabled_for_attribute?(experiment_key, attribute)
|
|
|
|
index = Digest::SHA1.hexdigest(attribute).hex % 100
|
|
|
|
enabled_for_value?(experiment_key, index)
|
|
|
|
end
|
|
|
|
|
|
|
|
def enabled_for_value?(experiment_key, experimentation_subject_index)
|
2019-12-26 22:10:19 +05:30
|
|
|
enabled?(experiment_key) &&
|
2021-01-03 14:25:43 +05:30
|
|
|
experiment(experiment_key).enabled_for_index?(experimentation_subject_index)
|
2019-12-21 20:55:43 +05:30
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2020-05-24 23:13:21 +05:30
|
|
|
Experiment = Struct.new(:key, :environment, :tracking_category, keyword_init: true) do
|
|
|
|
def enabled?
|
2020-10-24 23:57:45 +05:30
|
|
|
experiment_percentage > 0
|
2019-12-21 20:55:43 +05:30
|
|
|
end
|
|
|
|
|
|
|
|
def enabled_for_environment?
|
2020-05-24 23:13:21 +05:30
|
|
|
return ::Gitlab.dev_env_or_com? if environment.nil?
|
2019-12-21 20:55:43 +05:30
|
|
|
|
|
|
|
environment
|
|
|
|
end
|
|
|
|
|
2021-01-03 14:25:43 +05:30
|
|
|
def enabled_for_index?(index)
|
|
|
|
return false if index.blank?
|
2020-05-24 23:13:21 +05:30
|
|
|
|
2021-01-03 14:25:43 +05:30
|
|
|
index <= experiment_percentage
|
2020-05-24 23:13:21 +05:30
|
|
|
end
|
|
|
|
|
|
|
|
private
|
2019-12-21 20:55:43 +05:30
|
|
|
|
2020-05-24 23:13:21 +05:30
|
|
|
# When a feature does not exist, the `percentage_of_time_value` method will return 0
|
|
|
|
def experiment_percentage
|
2020-06-23 00:09:42 +05:30
|
|
|
@experiment_percentage ||= Feature.get(:"#{key}_experiment_percentage").percentage_of_time_value # rubocop:disable Gitlab/AvoidFeatureGet
|
2019-12-21 20:55:43 +05:30
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|