# E2E tests pipeline loaded dynamically by script: scripts/generate-e2e-pipeline # For adding new tests, refer to: doc/development/testing_guide/end_to_end/package_and_test_pipeline.md default: interruptible: true include: - local: .gitlab/ci/package-and-test/rules.gitlab-ci.yml - local: .gitlab/ci/package-and-test/variables.gitlab-ci.yml - project: gitlab-org/quality/pipeline-common ref: 2.0.0 file: - /ci/base.gitlab-ci.yml - /ci/allure-report.yml - /ci/knapsack-report.yml stages: - test - report - notify # ========================================== # Templates # ========================================== .parallel: parallel: 5 variables: QA_KNAPSACK_REPORT_PATH: $CI_PROJECT_DIR/qa/knapsack .ruby-image: # Because this pipeline template can be included directly in other projects, # image path and registry needs to be defined explicitly image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-bullseye-ruby-${RUBY_VERSION}:bundler-2.3 .qa-install: variables: BUNDLE_SUPPRESS_INSTALL_USING_MESSAGES: "true" BUNDLE_SILENCE_ROOT_WARNING: "true" extends: - .gitlab-qa-install .update-script: script: - export QA_COMMAND="bundle exec gitlab-qa Test::Omnibus::UpdateFromPrevious $RELEASE $GITLAB_VERSION $UPDATE_TYPE -- $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS" - echo "Running - '$QA_COMMAND'" - eval "$QA_COMMAND" .qa: extends: - .qa-base - .qa-install - .gitlab-qa-report stage: test tags: - e2e needs: - job: trigger-omnibus optional: true - job: download-knapsack-report artifacts: true optional: true variables: QA_GENERATE_ALLURE_REPORT: "true" QA_CAN_TEST_PRAEFECT: "false" QA_INTERCEPT_REQUESTS: "true" QA_RUN_TYPE: e2e-package-and-test EE_LICENSE: $QA_EE_LICENSE GITHUB_ACCESS_TOKEN: $QA_GITHUB_ACCESS_TOKEN GITLAB_QA_ADMIN_ACCESS_TOKEN: $QA_ADMIN_ACCESS_TOKEN GITLAB_LICENSE_MODE: test # ========================================== # Prepare stage # ========================================== check-release-set: extends: .rules:prepare stage: .pre script: - | if [ -z "$RELEASE" ]; then echo "E2E test pipeline requires omnibus installation docker image to be set via $RELEASE environment variable" exit 1 else echo "Omnibus installation image is set to '$RELEASE'" fi dont-interrupt-me: extends: .rules:dont-interrupt stage: .pre interruptible: false script: - echo "This jobs makes sure this pipeline won't be interrupted! See https://docs.gitlab.com/ee/ci/yaml/#interruptible." trigger-omnibus-env: extends: - .rules:omnibus-build stage: .pre needs: # We need this job because we need its `cached-assets-hash.txt` artifact, so that we can pass the assets image tag to the downstream omnibus-gitlab pipeline. - pipeline: $PARENT_PIPELINE_ID job: build-assets-image variables: BUILD_ENV: build.env before_script: - | # This is duplicating the function from `scripts/utils.sh` since `.gitlab/ci/package-and-test/main.gitlab-ci.yml` can be included in other projects. function assets_image_tag() { local cache_assets_hash_file="cached-assets-hash.txt" if [[ -n "${CI_COMMIT_TAG}" ]]; then echo -n "${CI_COMMIT_REF_NAME}" elif [[ -f "${cache_assets_hash_file}" ]]; then echo -n "assets-hash-$(cat ${cache_assets_hash_file} | cut -c1-10)" else echo -n "${CI_COMMIT_SHA}" fi } script: - | SECURITY_SOURCES=$([[ ! "$CI_PROJECT_NAMESPACE" =~ ^gitlab-org\/security ]] || echo "true") echo "SECURITY_SOURCES=${SECURITY_SOURCES:-false}" > $BUILD_ENV echo "OMNIBUS_GITLAB_CACHE_UPDATE=${OMNIBUS_GITLAB_CACHE_UPDATE:-false}" >> $BUILD_ENV for version_file in *_VERSION; do echo "$version_file=$(cat $version_file)" >> $BUILD_ENV; done echo "OMNIBUS_GITLAB_RUBY3_BUILD=${OMNIBUS_GITLAB_RUBY3_BUILD:-false}" >> $BUILD_ENV echo "OMNIBUS_GITLAB_CACHE_EDITION=${OMNIBUS_GITLAB_CACHE_EDITION:-GITLAB}" >> $BUILD_ENV echo "GITLAB_ASSETS_TAG=$(assets_image_tag)" >> $BUILD_ENV echo "Built environment file for omnibus build:" cat $BUILD_ENV artifacts: reports: dotenv: $BUILD_ENV trigger-omnibus: extends: .rules:omnibus-build stage: .pre needs: - trigger-omnibus-env inherit: variables: false variables: GITALY_SERVER_VERSION: $GITALY_SERVER_VERSION GITLAB_ELASTICSEARCH_INDEXER_VERSION: $GITLAB_ELASTICSEARCH_INDEXER_VERSION GITLAB_KAS_VERSION: $GITLAB_KAS_VERSION GITLAB_METRICS_EXPORTER_VERSION: $GITLAB_METRICS_EXPORTER_VERSION GITLAB_PAGES_VERSION: $GITLAB_PAGES_VERSION GITLAB_SHELL_VERSION: $GITLAB_SHELL_VERSION GITLAB_WORKHORSE_VERSION: $GITLAB_WORKHORSE_VERSION GITLAB_VERSION: $CI_COMMIT_SHA GITLAB_ASSETS_TAG: $GITLAB_ASSETS_TAG IMAGE_TAG: $CI_COMMIT_SHA TOP_UPSTREAM_SOURCE_PROJECT: $CI_PROJECT_PATH SECURITY_SOURCES: $SECURITY_SOURCES CACHE_UPDATE: $OMNIBUS_GITLAB_CACHE_UPDATE RUBY3_BUILD: $OMNIBUS_GITLAB_RUBY3_BUILD CACHE_EDITION: $OMNIBUS_GITLAB_CACHE_EDITION SKIP_QA_DOCKER: "true" SKIP_QA_TEST: "true" ee: "true" trigger: project: gitlab-org/build/omnibus-gitlab-mirror strategy: depend download-knapsack-report: extends: - .gitlab-qa-image - .rules:download-knapsack stage: .pre variables: KNAPSACK_DIR: ${CI_PROJECT_DIR}/qa/knapsack GIT_STRATEGY: none script: # when using qa-image, code runs in /home/gitlab/qa folder - bundle exec rake "knapsack:download[test]" - mkdir -p "$KNAPSACK_DIR" && cp knapsack/*.json "${KNAPSACK_DIR}/" allow_failure: true artifacts: paths: - qa/knapsack/*.json expire_in: 1 day cache-gems: extends: - .qa-install - .ruby-image - .rules:update-cache stage: .pre tags: - e2e script: - echo "Populated qa cache" cache: policy: pull-push # ========================================== # Test stage # ========================================== # ------------------------------------------ # Manual jobs # ------------------------------------------ # Run manual quarantine job # this job requires passing QA_SCENARIO variable # and optionally QA_TESTS to run specific quarantined tests _ee:quarantine: extends: - .qa - .rules:test:manual needs: - job: trigger-omnibus optional: true stage: test variables: QA_RSPEC_TAGS: --tag quarantine # ------------------------------------------ # FF changes # ------------------------------------------ # Run specs with feature flags set to the opposite of the default state ee:instance-ff-inverse: extends: - .qa - .parallel variables: QA_SCENARIO: Test::Instance::Image QA_KNAPSACK_REPORT_NAME: ee-instance GITLAB_QA_OPTS: --set-feature-flags $QA_FEATURE_FLAGS rules: - !reference [.rules:test:feature-flags-set, rules] # ------------------------------------------ # Jobs with parallel variant # ------------------------------------------ ee:instance-selective: extends: .qa variables: QA_SCENARIO: Test::Instance::Image rules: - !reference [.rules:test:qa-selective, rules] - if: $QA_SUITES =~ /Test::Instance::All/ ee:instance: extends: - .parallel - ee:instance-selective rules: - !reference [.rules:test:feature-flags-set, rules] # always run ee:instance to validate ff change - !reference [.rules:test:qa-parallel, rules] - if: $QA_SUITES =~ /Test::Instance::All/ ee:praefect-selective: extends: .qa variables: QA_SCENARIO: Test::Integration::Praefect QA_CAN_TEST_PRAEFECT: "true" rules: - !reference [.rules:test:qa-selective, rules] - if: $QA_SUITES =~ /Test::Instance::All/ ee:praefect: extends: - .parallel - ee:praefect-selective rules: - !reference [.rules:test:qa-parallel, rules] - if: $QA_SUITES =~ /Test::Instance::All/ ee:relative-url-selective: extends: .qa variables: QA_SCENARIO: Test::Instance::RelativeUrl rules: - !reference [.rules:test:qa-selective, rules] - if: $QA_SUITES =~ /Test::Instance::All/ ee:relative-url: extends: - .parallel - ee:relative-url-selective rules: - !reference [.rules:test:qa-parallel, rules] - if: $QA_SUITES =~ /Test::Instance::All/ ee:decomposition-single-db-selective: extends: .qa variables: QA_SCENARIO: Test::Instance::Image GITLAB_QA_OPTS: --omnibus-config decomposition_single_db rules: - !reference [.rules:test:qa-selective, rules] - if: $QA_SUITES =~ /Test::Instance::All/ ee:decomposition-single-db: extends: - .parallel - ee:decomposition-single-db-selective rules: - !reference [.rules:test:qa-parallel, rules] - if: $QA_SUITES =~ /Test::Instance::All/ ee:decomposition-multiple-db-selective: extends: .qa variables: QA_SCENARIO: Test::Instance::Image GITLAB_ALLOW_SEPARATE_CI_DATABASE: "true" GITLAB_QA_OPTS: --omnibus-config decomposition_multiple_db rules: - !reference [.rules:test:qa-selective, rules] - if: $QA_SUITES =~ /Test::Instance::All/ ee:decomposition-multiple-db: extends: - .parallel - ee:decomposition-multiple-db-selective rules: - !reference [.rules:test:qa-parallel, rules] - if: $QA_SUITES =~ /Test::Instance::All/ ee:object-storage-selective: extends: .qa variables: QA_SCENARIO: Test::Instance::Image QA_RSPEC_TAGS: --tag object_storage GITLAB_QA_OPTS: --omnibus-config object_storage rules: - !reference [.rules:test:qa-selective, rules] - if: $QA_SUITES =~ /Test::Instance::ObjectStorage/ ee:object-storage: extends: ee:object-storage-selective parallel: 2 rules: - !reference [.rules:test:qa-parallel, rules] - if: $QA_SUITES =~ /Test::Instance::ObjectStorage/ ee:object-storage-aws-selective: extends: ee:object-storage-selective variables: AWS_S3_ACCESS_KEY: $QA_AWS_S3_ACCESS_KEY AWS_S3_BUCKET_NAME: $QA_AWS_S3_BUCKET_NAME AWS_S3_KEY_ID: $QA_AWS_S3_KEY_ID AWS_S3_REGION: $QA_AWS_S3_REGION GITLAB_QA_OPTS: --omnibus-config object_storage_aws ee:object-storage-aws: extends: ee:object-storage-aws-selective parallel: 2 rules: - !reference [ee:object-storage, rules] ee:object-storage-gcs-selective: extends: ee:object-storage-selective variables: GCS_BUCKET_NAME: $QA_GCS_BUCKET_NAME GOOGLE_PROJECT: $QA_GOOGLE_PROJECT GOOGLE_JSON_KEY: $QA_GOOGLE_JSON_KEY GOOGLE_CLIENT_EMAIL: $QA_GOOGLE_CLIENT_EMAIL GITLAB_QA_OPTS: --omnibus-config object_storage_gcs ee:object-storage-gcs: extends: ee:object-storage-gcs-selective parallel: 2 rules: - !reference [ee:object-storage, rules] ee:packages-selective: extends: .qa variables: QA_SCENARIO: Test::Instance::Image QA_RSPEC_TAGS: --tag packages GITLAB_QA_OPTS: --omnibus-config packages rules: - !reference [.rules:test:qa-selective, rules] - if: $QA_SUITES =~ /Test::Instance::Packages/ ee:packages: extends: ee:packages-selective parallel: 2 rules: - !reference [.rules:test:qa-parallel, rules] - if: $QA_SUITES =~ /Test::Instance::Packages/ # ------------------------------------------ # Non parallel jobs # ------------------------------------------ ee:update-minor: extends: - .qa - .update-script variables: UPDATE_TYPE: minor QA_RSPEC_TAGS: --tag smoke rules: - !reference [.rules:test:update, rules] - if: $QA_SUITES =~ /Test::Instance::Smoke/ - !reference [.rules:test:manual, rules] ee:update-major: extends: - .qa - .update-script variables: UPDATE_TYPE: major QA_RSPEC_TAGS: --tag smoke rules: - !reference [.rules:test:update, rules] - if: $QA_SUITES =~ /Test::Instance::Smoke/ - !reference [.rules:test:manual, rules] ee:gitlab-pages: extends: .qa variables: QA_SCENARIO: Test::Integration::GitlabPages rules: - !reference [.rules:test:qa, rules] - if: $QA_SUITES =~ /Test::Instance::GitlabPages/ - !reference [.rules:test:manual, rules] ee:gitaly-cluster: extends: .qa variables: QA_SCENARIO: Test::Integration::GitalyCluster rules: - !reference [.rules:test:qa, rules] - if: $QA_SUITES =~ /Test::Integration::GitalyCluster/ - !reference [.rules:test:manual, rules] ee:group-saml: extends: .qa variables: QA_SCENARIO: Test::Integration::GroupSAML rules: - !reference [.rules:test:qa, rules] - if: $QA_SUITES =~ /Test::Integration::GroupSAML/ - !reference [.rules:test:manual, rules] ee:instance-saml: extends: .qa variables: QA_SCENARIO: Test::Integration::InstanceSAML rules: - !reference [.rules:test:qa, rules] - if: $QA_SUITES =~ /Test::Integration::InstanceSAML/ - !reference [.rules:test:manual, rules] ee:jira: extends: .qa variables: QA_SCENARIO: Test::Integration::Jira JIRA_ADMIN_USERNAME: $QA_JIRA_ADMIN_USERNAME JIRA_ADMIN_PASSWORD: $QA_JIRA_ADMIN_PASSWORD rules: - !reference [.rules:test:qa, rules] - if: $QA_SUITES =~ /Test::Integration::Jira/ - !reference [.rules:test:manual, rules] ee:integrations: extends: .qa variables: QA_SCENARIO: Test::Integration::Integrations rules: - !reference [.rules:test:qa, rules] - if: $QA_SUITES =~ /Test::Integration::Integrations/ - !reference [.rules:test:manual, rules] ee:ldap-no-server: extends: .qa variables: QA_SCENARIO: Test::Integration::LDAPNoServer rules: - !reference [.rules:test:qa, rules] - if: $QA_SUITES =~ /Test::Integration::LDAPNoServer/ - !reference [.rules:test:manual, rules] ee:ldap-tls: extends: .qa variables: QA_SCENARIO: Test::Integration::LDAPTLS rules: - !reference [.rules:test:qa, rules] - if: $QA_SUITES =~ /Test::Integration::LDAPTLS/ - !reference [.rules:test:manual, rules] ee:ldap-no-tls: extends: .qa variables: QA_SCENARIO: Test::Integration::LDAPNoTLS rules: - !reference [.rules:test:qa, rules] - if: $QA_SUITES =~ /Test::Integration::LDAPNoTLS/ - !reference [.rules:test:manual, rules] ee:mtls: extends: .qa variables: QA_SCENARIO: Test::Integration::MTLS rules: - !reference [.rules:test:qa, rules] - if: $QA_SUITES =~ /Test::Integration::Mtls/ - !reference [.rules:test:manual, rules] ee:mattermost: extends: .qa variables: QA_SCENARIO: Test::Integration::Mattermost rules: - !reference [.rules:test:qa, rules] - if: $QA_SUITES =~ /Test::Integration::Mattermost/ - !reference [.rules:test:manual, rules] ee:registry: extends: .qa variables: QA_SCENARIO: Test::Integration::Registry rules: - !reference [.rules:test:qa, rules] - if: $QA_SUITES =~ /Test::Integration::Registry/ - !reference [.rules:test:manual, rules] ee:registry-with-cdn: extends: .qa variables: QA_SCENARIO: Test::Integration::RegistryWithCDN GCS_CDN_BUCKET_NAME: $QA_GCS_CDN_BUCKET_NAME GOOGLE_CDN_LB: $QA_GOOGLE_CDN_LB GOOGLE_CDN_JSON_KEY: $QA_GOOGLE_CDN_JSON_KEY GOOGLE_CDN_SIGNURL_KEY: $QA_GOOGLE_CDN_SIGNURL_KEY GOOGLE_CDN_SIGNURL_KEY_NAME: $QA_GOOGLE_CDN_SIGNURL_KEY_NAME before_script: - unset GITLAB_QA_ADMIN_ACCESS_TOKEN - !reference [.qa, before_script] rules: - !reference [.rules:test:qa, rules] - if: $QA_SUITES =~ /Test::Integration::RegistryWithCDN/ - !reference [.rules:test:manual, rules] ee:repository-storage: extends: .qa variables: QA_SCENARIO: Test::Instance::RepositoryStorage rules: - !reference [.rules:test:qa, rules] - if: $QA_SUITES =~ /Test::Instance::RepositoryStorage/ - !reference [.rules:test:manual, rules] ee:service-ping-disabled: extends: .qa variables: QA_SCENARIO: Test::Integration::ServicePingDisabled rules: - !reference [.rules:test:qa, rules] - if: $QA_SUITES =~ /Test::Integration::ServicePingDisabled/ - !reference [.rules:test:manual, rules] ee:smtp: extends: .qa variables: QA_SCENARIO: Test::Integration::SMTP rules: - !reference [.rules:test:qa, rules] - if: $QA_SUITES =~ /Test::Integration::SMTP/ - !reference [.rules:test:manual, rules] ee:cloud-activation: extends: .qa variables: QA_SCENARIO: Test::Instance::Image QA_RSPEC_TAGS: --tag cloud_activation rules: - !reference [.rules:test:qa, rules] - if: $QA_SUITES =~ /Test::Instance::CloudActivation/ - !reference [.rules:test:manual, rules] ee:large-setup: extends: .qa variables: QA_SCENARIO: Test::Instance::Image QA_RSPEC_TAGS: --tag can_use_large_setup rules: - !reference [.rules:test:qa, rules] - if: $QA_SUITES =~ /Test::Instance::LargeSetup/ - !reference [.rules:test:manual, rules] ee:metrics: extends: .qa variables: QA_SCENARIO: Test::Integration::Metrics rules: - !reference [.rules:test:qa, rules] - if: $QA_SUITES =~ /Test::Instance::Metrics/ - !reference [.rules:test:manual, rules] ee:elasticsearch: extends: .qa variables: QA_SCENARIO: "Test::Integration::Elasticsearch" before_script: - !reference [.qa, before_script] rules: - !reference [.rules:test:qa, rules] - if: $QA_SUITES =~ /Test::Integration::Elasticsearch/ - !reference [.rules:test:manual, rules] ee:registry-object-storage-tls: extends: ee:object-storage-aws-selective variables: QA_SCENARIO: Test::Integration::RegistryTLS QA_RSPEC_TAGS: "" GITLAB_TLS_CERTIFICATE: $QA_GITLAB_TLS_CERTIFICATE GITLAB_QA_OPTS: --omnibus-config registry_object_storage ee:importers: extends: .qa variables: QA_SCENARIO: Test::Integration::Import QA_MOCK_GITHUB: "true" GITLAB_QA_OPTS: --set-feature-flags bulk_import_projects=enabled rules: - !reference [.rules:test:qa, rules] - if: $QA_SUITES =~ /Test::Integration::Import/ - !reference [.rules:test:manual, rules] # ========================================== # Post test stage # ========================================== e2e-test-report: extends: - .generate-allure-report-base - .rules:report:allure-report stage: report variables: GITLAB_AUTH_TOKEN: $PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE ALLURE_PROJECT_PATH: $CI_PROJECT_PATH ALLURE_MERGE_REQUEST_IID: $CI_MERGE_REQUEST_IID ALLURE_JOB_NAME: e2e-package-and-test GIT_STRATEGY: none upload-knapsack-report: extends: - .generate-knapsack-report-base - .qa-install - .ruby-image - .rules:report:process-results stage: report when: always export-test-metrics: extends: - .qa-install - .ruby-image - .rules:report:process-results stage: report when: always script: - bundle exec rake "ci:export_test_metrics[$CI_PROJECT_DIR/gitlab-qa-run-*/**/test-metrics-*.json]" relate-test-failures: extends: - .qa-install - .ruby-image - .rules:report:process-results stage: report variables: QA_FAILURES_REPORTING_PROJECT: gitlab-org/gitlab QA_FAILURES_MAX_DIFF_RATIO: "0.15" GITLAB_QA_ACCESS_TOKEN: $QA_GITLAB_CI_TOKEN when: on_failure script: - | bundle exec gitlab-qa-report \ --relate-failure-issue "$CI_PROJECT_DIR/gitlab-qa-run-*/**/rspec-*.json" \ --project "$QA_FAILURES_REPORTING_PROJECT" \ --max-diff-ratio "$QA_FAILURES_MAX_DIFF_RATIO" generate-test-session: extends: - .qa-install - .ruby-image - .rules:report:process-results stage: report variables: QA_TESTCASE_SESSIONS_PROJECT: gitlab-org/quality/testcase-sessions GITLAB_QA_ACCESS_TOKEN: $QA_TEST_SESSION_TOKEN GITLAB_CI_API_TOKEN: $QA_GITLAB_CI_TOKEN when: always script: - | bundle exec gitlab-qa-report \ --generate-test-session "$CI_PROJECT_DIR/gitlab-qa-run-*/**/rspec-*.json" \ --project "$QA_TESTCASE_SESSIONS_PROJECT" artifacts: when: always expire_in: 1d paths: - qa/REPORT_ISSUE_URL notify-slack: extends: - .notify-slack-qa - .qa-install - .ruby-image - .rules:report:process-results stage: notify variables: ALLURE_JOB_NAME: e2e-package-and-test SLACK_ICON_EMOJI: ci_failing STATUS_SYM: ☠️ STATUS: failed TYPE: "(package-and-test) " when: on_failure script: - bundle exec gitlab-qa-report --prepare-stage-reports "$CI_PROJECT_DIR/gitlab-qa-run-*/**/rspec-*.xml" # generate summary - !reference [.notify-slack-qa, script]