include: - local: .gitlab/ci/rails/shared.gitlab-ci.yml ############################################################### # EE/FOSS: default refs (MRs, default branch, schedules) jobs # setup-test-env: extends: - .rails-job-base - .setup-test-env-cache - .rails:rules:setup-test-env stage: prepare variables: SETUP_DB: "false" script: - echo $CI_MERGE_REQUEST_APPROVED - source scripts/gitlab_component_helpers.sh - run_timed_command "download_and_extract_gitlab_workhorse_package" || true - run_timed_command "scripts/setup-test-env" - run_timed_command "select_gitlab_workhorse_essentials" - echo -e "\e[0Ksection_start:`date +%s`:gitaly-test-build[collapsed=true]\r\e[0KCompiling Gitaly binaries" - run_timed_command "scripts/gitaly-test-build" # Do not use 'bundle exec' here - echo -e "\e[0Ksection_end:`date +%s`:gitaly-test-build\r\e[0K" artifacts: expire_in: 7d paths: - ${TMP_TEST_FOLDER}/gitaly/_build/bin/ - ${TMP_TEST_FOLDER}/gitaly/config.toml - ${TMP_TEST_FOLDER}/gitaly/gitaly2.config.toml - ${TMP_TEST_FOLDER}/gitaly/internal/ - ${TMP_TEST_FOLDER}/gitaly/Makefile - ${TMP_TEST_FOLDER}/gitaly/praefect.config.toml - ${TMP_TEST_FOLDER}/gitaly/praefect-db.config.toml - ${TMP_TEST_FOLDER}/gitaly/ruby/ - ${TMP_TEST_FOLDER}/gitlab-elasticsearch-indexer/bin/gitlab-elasticsearch-indexer - ${TMP_TEST_FOLDER}/gitlab-shell/ - ${TMP_TEST_FOLDER}/gitlab-test-fork/ - ${TMP_TEST_FOLDER}/gitlab-test-fork.bundle - ${TMP_TEST_FOLDER}/gitlab-test/ - ${TMP_TEST_FOLDER}/gitlab-test.bundle - ${TMP_TEST_FOLDER}/repositories/ - ${TMP_TEST_FOLDER}/second_storage/ - ${TMP_TEST_GITLAB_WORKHORSE_PATH}/ when: always update-setup-test-env-cache: extends: - setup-test-env - .setup-test-env-cache-push - .shared:rules:update-cache artifacts: paths: [] # This job's purpose is only to update the cache. update-gitaly-binaries-cache: extends: - setup-test-env - .gitaly-binaries-cache-push - .shared:rules:update-gitaly-binaries-cache artifacts: paths: [] # This job's purpose is only to update the cache. .coverage-base: extends: - .default-retry - .coverage-cache before_script: - source scripts/utils.sh - export BUNDLE_WITHOUT="${BUNDLE_WITHOUT}:default:test:puma:kerberos:metrics:omnibus:ed25519" - bundle_install_script rspec migration pg12: extends: - .rspec-base-pg12 - .rspec-base-migration - .rspec-migration-parallel rspec migration pg12 minimal: extends: - rspec migration pg12 - .minimal-rspec-tests - .rails:rules:ee-and-foss-migration:minimal rspec migration pg12 single-db: extends: - rspec migration pg12 - .single-db-rspec - .rails:rules:single-db rspec migration pg12 praefect: extends: - rspec migration pg12 - .praefect-with-db - .rails:rules:praefect-with-db rspec unit pg12: extends: - .rspec-base-pg12 - .rails:rules:ee-and-foss-unit - .rspec-unit-parallel rspec unit pg12 minimal: extends: - rspec unit pg12 - .minimal-rspec-tests - .rails:rules:ee-and-foss-unit:minimal rspec unit pg12 single-db: extends: - rspec unit pg12 - .single-db-rspec - .rails:rules:single-db rspec unit pg12 praefect: extends: - rspec unit pg12 - .praefect-with-db - .rails:rules:praefect-with-db rspec integration pg12: extends: - .rspec-base-pg12 - .rails:rules:ee-and-foss-integration - .rspec-integration-parallel rspec integration pg12 minimal: extends: - rspec integration pg12 - .minimal-rspec-tests - .rails:rules:ee-and-foss-integration:minimal rspec integration pg12 single-db: extends: - rspec integration pg12 - .single-db-rspec - .rails:rules:single-db rspec integration pg12 praefect: extends: - rspec integration pg12 - .praefect-with-db - .rails:rules:praefect-with-db rspec system pg12: extends: - .rspec-base-pg12 - .rails:rules:ee-and-foss-system - .rspec-system-parallel variables: DEBUG_GITLAB_TRANSACTION_STACK: "true" rspec system pg12 minimal: extends: - rspec system pg12 - .minimal-rspec-tests - .rails:rules:ee-and-foss-system:minimal rspec system pg12 single-db: extends: - rspec system pg12 - .single-db-rspec - .rails:rules:single-db rspec system pg12 praefect: extends: - rspec system pg12 - .praefect-with-db - .rails:rules:praefect-with-db # Dedicated job to test DB library code against PG11. # Note that these are already tested against PG12 in the `rspec unit pg12` / `rspec-ee unit pg12` jobs. rspec db-library-code pg11: extends: - .rspec-base-pg11 - .rails:rules:ee-and-foss-db-library-code script: - !reference [.base-script, script] - rspec_db_library_code rspec fast_spec_helper: extends: - .rspec-base-pg12 - .rails:rules:ee-and-foss-fast_spec_helper script: - fast_spec_helper_specs=$(git grep -l -E '^require.*fast_spec_helper') # Load fast_spec_helper as well just in case there are no specs available. - bin/rspec --dry-run spec/fast_spec_helper.rb $fast_spec_helper_specs db:rollback: extends: .db-job-base script: - scripts/db_tasks db:migrate VERSION=20210301200959 - scripts/db_tasks db:migrate SKIP_SCHEMA_VERSION_CHECK=true db:rollback single-db: extends: - db:rollback - .single-db - .rails:rules:single-db db:migrate:reset: extends: .db-job-base script: - bundle exec rake db:migrate:reset db:migrate:reset single-db: extends: - db:migrate:reset - .single-db - .rails:rules:single-db db:check-schema: extends: - .db-job-base - .rails:rules:ee-mr-and-default-branch-only script: - run_timed_command "bundle exec rake db:drop db:create" - run_timed_command "scripts/db_tasks db:migrate" db:check-schema-single-db: extends: - db:check-schema - .single-db - .rails:rules:single-db db:check-migrations: extends: - .db-job-base - .rails:rules:ee-and-foss-mr-with-migration script: - git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:$CI_MERGE_REQUEST_TARGET_BRANCH_NAME --depth 20 - scripts/validate_migration_schema allow_failure: true db:check-migrations-single-db: extends: - db:check-migrations - .single-db - .rails:rules:single-db db:post_deployment_migrations_validator: extends: - .db-job-base - .rails:rules:ee-and-foss-mr-with-migration script: - git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:$CI_MERGE_REQUEST_TARGET_BRANCH_NAME --depth 20 - scripts/post_deployment_migrations_validator allow_failure: true db:post_deployment_migrations_validator-single-db: extends: - db:post_deployment_migrations_validator - .single-db - .rails:rules:single-db db:migrate-non-superuser: extends: - .db-job-base - .rails:rules:ee-and-foss-mr-with-migration script: - bundle exec rake gitlab:db:reset_as_non_superuser db:gitlabcom-database-testing: extends: .rails:rules:db:gitlabcom-database-testing stage: test image: ruby:${RUBY_VERSION}-alpine needs: [] allow_failure: true script: - source scripts/utils.sh - install_gitlab_gem - ./scripts/trigger-build.rb gitlab-com-database-testing gitlab:setup: extends: .db-job-base variables: SETUP_DB: "false" script: # Manually clone gitlab-test and only seed this project in # db/fixtures/development/04_project.rb thanks to SIZE=1 below - git clone https://gitlab.com/gitlab-org/gitlab-test.git /home/git/repositories/gitlab-org/gitlab-test.git - !reference [.base-script, script] - force=yes SIZE=1 FIXTURE_PATH="db/fixtures/development" bundle exec rake gitlab:setup artifacts: when: on_failure expire_in: 1d paths: - log/*.log db:backup_and_restore: extends: .db-job-base variables: SETUP_DB: "false" GITLAB_ASSUME_YES: "1" script: - . scripts/prepare_build.sh - bundle exec rake db:drop db:create db:structure:load db:seed_fu - mkdir -p tmp/tests/public/uploads tmp/tests/{artifacts,pages,lfs-objects,terraform_state,registry,packages} - bundle exec rake gitlab:backup:create - date - bundle exec rake gitlab:backup:restore rules: - changes: - "lib/backup/**/*" - "lib/tasks/gitlab/backup.rake" db:backup_and_restore single-db: extends: - db:backup_and_restore - .single-db - .rails:rules:single-db rspec:deprecations: extends: - .default-retry - .default-before_script - .static-analysis-cache - .rails:rules:deprecations stage: post-test allow_failure: true # We cannot use needs since it would mean needing 84 jobs (since most are parallelized) # so we use `dependencies` here. dependencies: - rspec migration pg12 - rspec unit pg12 - rspec integration pg12 - rspec system pg12 - rspec-ee migration pg12 - rspec-ee unit pg12 - rspec-ee integration pg12 - rspec-ee system pg12 variables: SETUP_DB: "false" script: - grep -h -R "keyword" deprecations/ | awk '{$1=$1};1' | sort | uniq -c | sort - grep -R "keyword" deprecations/ | wc - run_timed_command "bundle exec rubocop --only Lint/LastKeywordArgument --parallel" artifacts: expire_in: 31d when: always paths: - deprecations/ rspec:coverage: extends: - .coverage-base - .rails:rules:rspec-coverage stage: post-test # We cannot use needs since it would mean needing 84 jobs (since most are parallelized) # so we use `dependencies` here. dependencies: - setup-test-env # FOSS/EE jobs - rspec migration pg12 - rspec unit pg12 - rspec integration pg12 - rspec system pg12 # FOSS/EE minimal jobs - rspec migration pg12 minimal - rspec unit pg12 minimal - rspec integration pg12 minimal - rspec system pg12 minimal # FOSS/EE single-db jobs - rspec migration pg12 single-db - rspec unit pg12 single-db - rspec integration pg12 single-db - rspec system pg12 single-db # EE jobs - rspec-ee migration pg12 - rspec-ee unit pg12 - rspec-ee integration pg12 - rspec-ee system pg12 # EE minimal jobs - rspec-ee migration pg12 minimal - rspec-ee unit pg12 minimal - rspec-ee integration pg12 minimal - rspec-ee system pg12 minimal # EE single-db jobs - rspec-ee migration pg12 single-db - rspec-ee unit pg12 single-db - rspec-ee integration pg12 single-db - rspec-ee system pg12 single-db # Memory jobs - memory-on-boot # As-if-FOSS jobs - rspec migration pg12-as-if-foss - rspec migration pg12-as-if-foss minimal - rspec migration pg12-as-if-foss single-db - rspec unit pg12-as-if-foss - rspec unit pg12-as-if-foss minimal - rspec unit pg12-as-if-foss single-db - rspec integration pg12-as-if-foss - rspec integration pg12-as-if-foss minimal - rspec integration pg12-as-if-foss single-db - rspec system pg12-as-if-foss - rspec system pg12-as-if-foss minimal - rspec system pg12-as-if-foss single-db script: - run_timed_command "bundle exec scripts/merge-simplecov" coverage: '/LOC \((\d+\.\d+%)\) covered.$/' artifacts: name: coverage expire_in: 31d paths: - coverage/index.html - coverage/assets/ - coverage/lcov/ reports: coverage_report: coverage_format: cobertura path: coverage/coverage.xml rspec:undercoverage: extends: - .coverage-base - .rails:rules:rspec-undercoverage stage: post-test needs: ["rspec:coverage"] script: - if [ -n "$CI_MERGE_REQUEST_TARGET_BRANCH_SHA" ]; then echo "HEAD is $(git rev-parse HEAD). \$CI_MERGE_REQUEST_TARGET_BRANCH_SHA is ${CI_MERGE_REQUEST_TARGET_BRANCH_SHA}"; else echo "HEAD is $(git rev-parse HEAD). \$CI_MERGE_REQUEST_DIFF_BASE_SHA is ${CI_MERGE_REQUEST_DIFF_BASE_SHA}"; fi; - UNDERCOVERAGE_COMPARE="${CI_MERGE_REQUEST_TARGET_BRANCH_SHA:-$CI_MERGE_REQUEST_DIFF_BASE_SHA}" - git diff ${UNDERCOVERAGE_COMPARE} --stat - echo "Undercoverage comparing with ${UNDERCOVERAGE_COMPARE}." - if [ -f scripts/undercoverage ]; then run_timed_command "bundle exec scripts/undercoverage ${UNDERCOVERAGE_COMPARE}"; fi; rspec:feature-flags: extends: - .coverage-base - .rails:rules:rspec-feature-flags stage: post-test needs: - job: "feature-flags-usage" - job: "haml-lint" - job: "haml-lint ee" optional: true script: - if [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then run_timed_command "bundle exec scripts/used-feature-flags" || (scripts/slack master-broken "☠️ \`${CI_JOB_NAME}\` failed! ☠️ See ${CI_JOB_URL}" ci_failing "GitLab Bot" && exit 1); else run_timed_command "bundle exec scripts/used-feature-flags"; fi rspec:flaky-tests-report: extends: - .default-retry - .rails:rules:flaky-tests-report stage: post-test # We cannot use needs since it would mean needing 84 jobs (since most are parallelized) # so we use `dependencies` here. dependencies: !reference ["rspec:coverage", "dependencies"] variables: SKIPPED_FLAKY_TESTS_REPORT_PATH: rspec/flaky/skipped_flaky_tests_report.txt RETRIED_TESTS_REPORT_PATH: rspec/flaky/retried_tests_report.txt before_script: - source scripts/utils.sh - source scripts/rspec_helpers.sh script: - generate_flaky_tests_reports artifacts: expire_in: 31d paths: - rspec/ # EE/FOSS: default refs (MRs, default branch, schedules) jobs # ####################################################### ################################################## # EE: default refs (MRs, default branch, schedules) jobs # rspec migration pg12-as-if-foss: extends: - .rspec-base-pg12-as-if-foss - .rspec-base-migration - .rails:rules:as-if-foss-migration - .rspec-migration-parallel rspec migration pg12-as-if-foss minimal: extends: - rspec migration pg12-as-if-foss - .minimal-rspec-tests - .rails:rules:as-if-foss-migration:minimal rspec migration pg12-as-if-foss single-db: extends: - rspec migration pg12-as-if-foss - .single-db-rspec - .rails:rules:single-db rspec unit pg12-as-if-foss: extends: - .rspec-base-pg12-as-if-foss - .rails:rules:as-if-foss-unit - .rspec-unit-parallel rspec unit pg12-as-if-foss minimal: extends: - rspec unit pg12-as-if-foss - .minimal-rspec-tests - .rails:rules:as-if-foss-unit:minimal rspec unit pg12-as-if-foss single-db: extends: - rspec unit pg12-as-if-foss - .single-db-rspec - .rails:rules:single-db rspec integration pg12-as-if-foss: extends: - .rspec-base-pg12-as-if-foss - .rails:rules:as-if-foss-integration - .rspec-integration-parallel rspec integration pg12-as-if-foss minimal: extends: - rspec integration pg12-as-if-foss - .minimal-rspec-tests - .rails:rules:as-if-foss-integration:minimal rspec integration pg12-as-if-foss single-db: extends: - rspec integration pg12-as-if-foss - .single-db-rspec - .rails:rules:single-db rspec system pg12-as-if-foss: extends: - .rspec-base-pg12-as-if-foss - .rails:rules:as-if-foss-system - .rspec-system-parallel rspec system pg12-as-if-foss minimal: extends: - rspec system pg12-as-if-foss - .minimal-rspec-tests - .rails:rules:as-if-foss-system:minimal rspec system pg12-as-if-foss single-db: extends: - rspec system pg12-as-if-foss - .single-db-rspec - .rails:rules:single-db rspec-ee migration pg12: extends: - .rspec-ee-base-pg12 - .rspec-base-migration - .rails:rules:ee-only-migration - .rspec-ee-migration-parallel rspec-ee migration pg12 minimal: extends: - rspec-ee migration pg12 - .minimal-rspec-tests - .rails:rules:ee-only-migration:minimal rspec-ee migration pg12 single-db: extends: - rspec-ee migration pg12 - .single-db-rspec - .rails:rules:single-db rspec-ee unit pg12: extends: - .rspec-ee-base-pg12 - .rails:rules:ee-only-unit - .rspec-ee-unit-parallel rspec-ee unit pg12 es8: extends: - .rspec-ee-base-pg12-es8 - .rspec-ee-unit-parallel rspec-ee unit pg12 opensearch1: extends: - .rspec-ee-base-pg12-opensearch1 - .rspec-ee-unit-parallel rspec-ee unit pg12 minimal: extends: - rspec-ee unit pg12 - .minimal-rspec-tests - .rails:rules:ee-only-unit:minimal rspec-ee unit pg12 single-db: extends: - rspec-ee unit pg12 - .single-db-rspec - .rails:rules:single-db rspec-ee integration pg12: extends: - .rspec-ee-base-pg12 - .rails:rules:ee-only-integration - .rspec-ee-integration-parallel rspec-ee integration pg12 es8: extends: - .rspec-ee-base-pg12-es8 - .rspec-ee-integration-parallel rspec-ee integration pg12 opensearch1: extends: - .rspec-ee-base-pg12-opensearch1 - .rspec-ee-integration-parallel rspec-ee integration pg12 minimal: extends: - rspec-ee integration pg12 - .minimal-rspec-tests - .rails:rules:ee-only-integration:minimal rspec-ee integration pg12 single-db: extends: - rspec-ee integration pg12 - .single-db-rspec - .rails:rules:single-db rspec-ee system pg12: extends: - .rspec-ee-base-pg12 - .rails:rules:ee-only-system - .rspec-ee-system-parallel rspec-ee system pg12 es8: extends: - .rspec-ee-base-pg12-es8 - .rspec-ee-system-parallel rspec-ee system pg12 opensearch1: extends: - .rspec-ee-base-pg12-opensearch1 - .rspec-ee-system-parallel rspec-ee system pg12 minimal: extends: - rspec-ee system pg12 - .minimal-rspec-tests - .rails:rules:ee-only-system:minimal rspec-ee system pg12 single-db: extends: - rspec-ee system pg12 - .single-db-rspec - .rails:rules:single-db db:rollback geo: extends: - db:rollback - .rails:rules:ee-only-migration script: - bundle exec rake db:migrate:geo VERSION=20170627195211 - bundle exec rake db:migrate:geo # EE: default refs (MRs, default branch, schedules) jobs # ################################################## ########################################## # EE/FOSS: default branch nightly scheduled jobs # # PG11 rspec migration pg11: extends: - .rspec-base-pg11 - .rspec-base-migration - .rails:rules:default-branch-schedule-nightly--code-backstage - .rspec-migration-parallel rspec unit pg11: extends: - .rspec-base-pg11 - .rails:rules:default-branch-schedule-nightly--code-backstage - .rspec-unit-parallel rspec integration pg11: extends: - .rspec-base-pg11 - .rails:rules:default-branch-schedule-nightly--code-backstage - .rspec-integration-parallel rspec system pg11: extends: - .rspec-base-pg11 - .rails:rules:default-branch-schedule-nightly--code-backstage - .rspec-system-parallel # PG13 rspec migration pg13: extends: - .rspec-base-pg13 - .rspec-base-migration - .rails:rules:default-branch-schedule-nightly--code-backstage - .rspec-migration-parallel rspec unit pg13: extends: - .rspec-base-pg13 - .rails:rules:default-branch-schedule-nightly--code-backstage - .rspec-unit-parallel rspec integration pg13: extends: - .rspec-base-pg13 - .rails:rules:default-branch-schedule-nightly--code-backstage - .rspec-integration-parallel rspec system pg13: extends: - .rspec-base-pg13 - .rails:rules:default-branch-schedule-nightly--code-backstage - .rspec-system-parallel # EE/FOSS: default branch nightly scheduled jobs # ########################################## ##################################### # EE: default branch nightly scheduled jobs # # PG11 rspec-ee migration pg11: extends: - .rspec-ee-base-pg11 - .rspec-base-migration - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only - .rspec-ee-migration-parallel rspec-ee unit pg11: extends: - .rspec-ee-base-pg11 - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only - .rspec-ee-unit-parallel rspec-ee integration pg11: extends: - .rspec-ee-base-pg11 - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only - .rspec-ee-integration-parallel rspec-ee system pg11: extends: - .rspec-ee-base-pg11 - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only - .rspec-ee-system-parallel # PG13 rspec-ee migration pg13: extends: - .rspec-ee-base-pg13 - .rspec-base-migration - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only - .rspec-ee-migration-parallel rspec-ee unit pg13: extends: - .rspec-ee-base-pg13 - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only - .rspec-ee-unit-parallel rspec-ee integration pg13: extends: - .rspec-ee-base-pg13 - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only - .rspec-ee-integration-parallel rspec-ee system pg13: extends: - .rspec-ee-base-pg13 - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only - .rspec-ee-system-parallel # EE: default branch nightly scheduled jobs # ##################################### ################################################## # EE: Canonical MR pipelines rspec fail-fast: extends: - .rspec-ee-base-pg12 # This job also runs EE spec which needs elasticsearch - .rails:rules:rspec fail-fast stage: test needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets", "detect-tests"] script: - !reference [.base-script, script] - rspec_fail_fast "${RSPEC_MATCHING_TESTS_PATH}" "--tag ~quarantine" artifacts: expire_in: 7d paths: - tmp/capybara/ rspec-foss-impact:pipeline-generate: extends: - .rails:rules:rspec-foss-impact stage: prepare needs: ["detect-tests"] script: - scripts/generate-rspec-foss-impact-pipeline "${RSPEC_MATCHING_TESTS_FOSS_PATH}" "${RSPEC_FOSS_IMPACT_PIPELINE_YML}" artifacts: expire_in: 1 day paths: - $RSPEC_FOSS_IMPACT_PIPELINE_YML rspec-foss-impact:trigger: extends: - .rails:rules:rspec-foss-impact stage: test needs: - job: "setup-test-env" artifacts: false - job: "retrieve-tests-metadata" artifacts: false - job: "compile-test-assets as-if-foss" artifacts: false - job: "rspec-foss-impact:pipeline-generate" artifacts: true variables: PARENT_PIPELINE_ID: $CI_PIPELINE_ID trigger: strategy: depend forward: yaml_variables: true pipeline_variables: true include: - artifact: $RSPEC_FOSS_IMPACT_PIPELINE_YML job: rspec-foss-impact:pipeline-generate fail-pipeline-early: extends: - .rails:rules:fail-pipeline-early stage: post-test needs: - job: rspec fail-fast artifacts: false variables: GIT_DEPTH: 1 before_script: - source scripts/utils.sh - install_gitlab_gem script: - fail_pipeline_early rspec rspec-pg12-rerun-previous-failed-tests: extends: - .rspec-base-pg12 - .rails:rules:rerun-previous-failed-tests stage: test needs: ["setup-test-env", "compile-test-assets", "detect-previous-failed-tests"] script: - !reference [.base-script, script] - rspec_rerun_previous_failed_tests tmp/previous_failed_tests/rspec_failed_files.txt rspec rspec-ee-pg12-rerun-previous-failed-tests: extends: - "rspec rspec-pg12-rerun-previous-failed-tests" - .rspec-ee-base-pg12 script: - !reference [.base-script, script] - rspec_rerun_previous_failed_tests tmp/previous_failed_tests/rspec_ee_failed_files.txt # EE: Canonical MR pipelines ##################################################