debian-mirror-gitlab/.gitlab/ci/rails.gitlab-ci.yml
2021-06-08 01:23:25 +05:30

775 lines
21 KiB
YAML

#######################
# rspec job base specs
.rails-job-base:
extends:
- .default-retry
- .default-before_script
- .rails-cache
.minimal-bundle-install:
script:
- export BUNDLE_WITHOUT="${BUNDLE_WITHOUT}:default:test:puma:unicorn:kerberos:metrics:omnibus:ed25519"
- bundle_install_script
.base-script:
script:
# Only install knapsack after bundle install! Otherwise oddly some native
# gems could not be found under some circumstance. No idea why, hours wasted.
- run_timed_command "gem install knapsack --no-document"
- run_timed_command "scripts/gitaly-test-spawn"
- source ./scripts/rspec_helpers.sh
.minimal-rspec-tests:
variables:
RSPEC_TESTS_MAPPING_ENABLED: "true"
.rspec-base:
extends: .rails-job-base
stage: test
variables:
RUBY_GC_MALLOC_LIMIT: 67108864
RUBY_GC_MALLOC_LIMIT_MAX: 134217728
CRYSTALBALL: "true"
RECORD_DEPRECATIONS: "true"
needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets", "detect-tests"]
script:
- !reference [.base-script, script]
- rspec_paralellized_job "--tag ~quarantine --tag ~geo --tag ~level:migration"
artifacts:
expire_in: 31d
when: always
paths:
- coverage/
- crystalball/
- deprecations/
- knapsack/
- rspec_flaky/
- rspec_profiling/
- tmp/capybara/
- tmp/memory_test/
- tmp/feature_flags/
- log/*.log
reports:
junit: junit_rspec.xml
.rspec-base-migration:
extends: .rails:rules:ee-and-foss-migration
script:
- !reference [.base-script, script]
- rspec_paralellized_job "--tag ~quarantine --tag ~geo --tag level:migration"
.rspec-base-pg11:
extends:
- .rspec-base
- .use-pg11
.rspec-base-pg12:
extends:
- .rspec-base
- .use-pg12
.rspec-base-pg12-as-if-foss:
extends:
- .rspec-base
- .as-if-foss
- .use-pg12
needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets as-if-foss", "detect-tests"]
.rspec-ee-base-pg11:
extends:
- .rspec-base
- .use-pg11-ee
.rspec-ee-base-pg12:
extends:
- .rspec-base
- .use-pg12-ee
.rspec-ee-base-geo:
extends: .rspec-base
script:
- !reference [.base-script, script]
- rspec_paralellized_job "--tag ~quarantine --tag geo"
.rspec-ee-base-geo-pg11:
extends:
- .rspec-ee-base-geo
- .use-pg11-ee
.rspec-ee-base-geo-pg12:
extends:
- .rspec-ee-base-geo
- .use-pg12-ee
.db-job-base:
extends:
- .rails-job-base
- .rails:rules:ee-and-foss-migration
- .use-pg12
stage: test
needs: ["setup-test-env"]
# rspec job base specs
######################
############################
# rspec job parallel configs
.rspec-migration-parallel:
parallel: 7
.rspec-ee-migration-parallel:
parallel: 3
.rspec-unit-parallel:
parallel: 20
.rspec-ee-unit-parallel:
parallel: 10
.rspec-ee-unit-geo-parallel:
parallel: 2
.rspec-integration-parallel:
parallel: 8
.rspec-ee-integration-parallel:
parallel: 4
.rspec-system-parallel:
parallel: 24
.rspec-ee-system-parallel:
parallel: 6
# rspec job parallel configs
############################
#######################################################
# EE/FOSS: default refs (MRs, default branch, schedules) jobs #
setup-test-env:
extends:
- .rails-job-base
- .setup-test-env-cache
- .rails:rules:code-backstage-qa
- .use-pg12
stage: prepare
variables:
GITLAB_TEST_EAGER_LOAD: "0"
script:
- run_timed_command "bundle exec ruby -I. -e 'require \"config/environment\"; TestEnv.init'"
- run_timed_command "scripts/gitaly-test-build" # Do not use 'bundle exec' here
artifacts:
expire_in: 7d
paths:
- config/secrets.yml
- tmp/tests/gitaly/_build/bin/
- tmp/tests/gitaly/config.toml
- tmp/tests/gitaly/gitaly2.config.toml
- tmp/tests/gitaly/internal/
- tmp/tests/gitaly/internal_gitaly2/
- tmp/tests/gitaly/internal_sockets/
- tmp/tests/gitaly/Makefile
- tmp/tests/gitaly/praefect.config.toml
- tmp/tests/gitaly/ruby/
- tmp/tests/gitlab-elasticsearch-indexer/bin/gitlab-elasticsearch-indexer
- tmp/tests/gitlab-shell/
- tmp/tests/gitlab-test-fork/
- tmp/tests/gitlab-test-fork_bare/
- tmp/tests/gitlab-test/
- tmp/tests/gitlab-workhorse/gitlab-zip-metadata
- tmp/tests/gitlab-workhorse/gitlab-zip-cat
- tmp/tests/gitlab-workhorse/gitlab-workhorse
- tmp/tests/gitlab-workhorse/gitlab-resize-image
- tmp/tests/gitlab-workhorse/config.toml
- tmp/tests/gitlab-workhorse/WORKHORSE_TREE
- tmp/tests/repositories/
- tmp/tests/second_storage/
when: always
update-setup-test-env-cache:
extends:
- setup-test-env
- .setup-test-env-cache-push
- .shared:rules:update-cache
artifacts:
paths: [] # This job's purpose is only to update the cache.
.coverage-base:
extends:
- .default-retry
- .default-before_script
- .coverage-cache
variables:
SETUP_DB: "false"
USE_BUNDLE_INSTALL: "false"
.static-analysis-base:
extends:
- .default-retry
- .default-before_script
- .static-analysis-cache
needs: []
variables:
SETUP_DB: "false"
ENABLE_SPRING: "1"
update-static-analysis-cache:
extends:
- .static-analysis-base
- .static-analysis-cache-push
- .shared:rules:update-cache
stage: prepare
script:
- run_timed_command "bundle exec rubocop --parallel" # For the moment we only cache `tmp/rubocop_cache` so we don't need to run all the tasks.
static-analysis:
extends:
- .static-analysis-base
- .static-analysis:rules:ee-and-foss
stage: test
parallel: 4
script:
- run_timed_command "retry yarn install --frozen-lockfile"
- scripts/static-analysis
static-analysis as-if-foss:
extends:
- static-analysis
- .static-analysis:rules:as-if-foss
- .as-if-foss
rspec migration pg12:
extends:
- .rspec-base-pg12
- .rspec-base-migration
- .rspec-migration-parallel
rspec migration pg12 minimal:
extends:
- rspec migration pg12
- .minimal-rspec-tests
- .rails:rules:ee-and-foss-migration:minimal
rspec unit pg12:
extends:
- .rspec-base-pg12
- .rails:rules:ee-and-foss-unit
- .rspec-unit-parallel
rspec unit pg12 minimal:
extends:
- rspec unit pg12
- .minimal-rspec-tests
- .rails:rules:ee-and-foss-unit:minimal
rspec integration pg12:
extends:
- .rspec-base-pg12
- .rails:rules:ee-and-foss-integration
- .rspec-integration-parallel
rspec integration pg12 minimal:
extends:
- rspec integration pg12
- .minimal-rspec-tests
- .rails:rules:ee-and-foss-integration:minimal
rspec system pg12:
extends:
- .rspec-base-pg12
- .rails:rules:ee-and-foss-system
- .rspec-system-parallel
rspec system pg12 minimal:
extends:
- rspec system pg12
- .minimal-rspec-tests
- .rails:rules:ee-and-foss-system:minimal
# Dedicated job to test DB library code against PG11.
# Note that these are already tested against PG12 in the `rspec unit pg12` / `rspec-ee unit pg12` jobs.
rspec db-library-code pg11:
extends:
- .rspec-base-pg11
- .rails:rules:ee-and-foss-db-library-code
script:
- !reference [.base-script, script]
- rspec_db_library_code
rspec fast_spec_helper:
extends:
- .rspec-base-pg12
- .rails:rules:ee-and-foss-fast_spec_helper
script:
- bin/rspec spec/fast_spec_helper.rb
rspec fast_spec_helper minimal:
extends:
- rspec fast_spec_helper
- .minimal-rspec-tests
- .rails:rules:ee-and-foss-fast_spec_helper:minimal
db:migrate:reset:
extends: .db-job-base
script:
- bundle exec rake db:migrate:reset
db:check-schema:
extends:
- .db-job-base
- .rails:rules:ee-mr-and-default-branch-only
script:
- source scripts/schema_changed.sh
- scripts/validate_migration_timestamps
db:check-migrations:
extends:
- .db-job-base
- .rails:rules:ee-and-foss-mr-with-migration
script:
- scripts/validate_migration_schema
allow_failure: true
db:migrate-from-previous-major-version:
extends: .db-job-base
variables:
USE_BUNDLE_INSTALL: "false"
SETUP_DB: "false"
PROJECT_TO_CHECKOUT: "gitlab-foss"
TAG_TO_CHECKOUT: "v12.10.14"
script:
- '[[ -d "ee/" ]] || export PROJECT_TO_CHECKOUT="gitlab"'
- '[[ -d "ee/" ]] || export TAG_TO_CHECKOUT="${TAG_TO_CHECKOUT}-ee"'
- retry 'git fetch https://gitlab.com/gitlab-org/$PROJECT_TO_CHECKOUT.git $TAG_TO_CHECKOUT'
- git checkout -f FETCH_HEAD
# Patch Gemfile of the previous major version for compatibility.
- sed -i -e "s/gem 'grpc', '~> 1.24.0'/gem 'grpc', '~> 1.30.2'/" Gemfile # Update gRPC for Ruby 2.7
- sed -i -e "s/gem 'google-protobuf', '~> 3.8.0'/gem 'google-protobuf', '~> 3.12'/" Gemfile
- sed -i -e "s/gem 'nokogiri', '~> 1.10.5'/gem 'nokogiri', '~> 1.11.0'/" Gemfile
- sed -i -e "s/gem 'mimemagic', '~> 0.3.2'/gem 'ruby-magic', '~> 0.4.0'/" Gemfile
- run_timed_command "gem install bundler:1.17.3"
- run_timed_command "bundle update google-protobuf nokogiri grpc mimemagic bootsnap"
- run_timed_command "bundle install ${BUNDLE_INSTALL_FLAGS}"
- cp config/gitlab.yml.example config/gitlab.yml
- run_timed_command "bundle exec rake db:drop db:create db:structure:load db:migrate db:seed_fu"
- git checkout -f $CI_COMMIT_SHA
- run_timed_command "bundle install ${BUNDLE_INSTALL_FLAGS}"
- run_timed_command "bundle exec rake db:migrate"
db:rollback:
extends: .db-job-base
script:
- bundle exec rake db:migrate VERSION=20181228175414
- bundle exec rake db:migrate SKIP_SCHEMA_VERSION_CHECK=true
db:gitlabcom-database-testing:
extends: .rails:rules:db:gitlabcom-database-testing
stage: test
image: ruby:2.7-alpine
needs: []
allow_failure: true
script:
- source scripts/utils.sh
- install_gitlab_gem
- ./scripts/trigger-build gitlab-com-database-testing
gitlab:setup:
extends: .db-job-base
variables:
SETUP_DB: "false"
script:
# Manually clone gitlab-test and only seed this project in
# db/fixtures/development/04_project.rb thanks to SIZE=1 below
- git clone https://gitlab.com/gitlab-org/gitlab-test.git
/home/git/repositories/gitlab-org/gitlab-test.git
- !reference [.base-script, script]
- force=yes SIZE=1 FIXTURE_PATH="db/fixtures/development" bundle exec rake gitlab:setup
artifacts:
when: on_failure
expire_in: 1d
paths:
- log/*.log
db:backup_and_restore:
extends: .db-job-base
variables:
SETUP_DB: "false"
GITLAB_ASSUME_YES: "1"
script:
- . scripts/prepare_build.sh
- bundle exec rake db:drop db:create db:structure:load db:seed_fu
- mkdir -p tmp/tests/public/uploads tmp/tests/{artifacts,pages,lfs-objects,registry}
- bundle exec rake gitlab:backup:create
- date
- bundle exec rake gitlab:backup:restore
rules:
- changes: ["lib/backup/**/*"]
rspec:deprecations:
extends:
- .default-retry
- .default-before_script
- .static-analysis-cache
- .rails:rules:deprecations
stage: post-test
allow_failure: true
# We cannot use needs since it would mean needing 84 jobs (since most are parallelized)
# so we use `dependencies` here.
dependencies:
- rspec migration pg12
- rspec unit pg12
- rspec integration pg12
- rspec system pg12
- rspec-ee migration pg12
- rspec-ee unit pg12
- rspec-ee integration pg12
- rspec-ee system pg12
- rspec-ee unit pg12 geo
- rspec-ee integration pg12 geo
- rspec-ee system pg12 geo
variables:
SETUP_DB: "false"
script:
- grep -h -R "keyword" deprecations/ | awk '{$1=$1};1' | sort | uniq -c | sort
- grep -R "keyword" deprecations/ | wc
- run_timed_command "bundle exec rubocop --only Lint/LastKeywordArgument --parallel"
artifacts:
expire_in: 31d
when: always
paths:
- deprecations/
rspec:coverage:
extends:
- .coverage-base
- .rails:rules:rspec-coverage
stage: post-test
# We cannot use needs since it would mean needing 84 jobs (since most are parallelized)
# so we use `dependencies` here.
dependencies:
- setup-test-env
- rspec migration pg12
- rspec unit pg12
- rspec integration pg12
- rspec system pg12
- rspec-ee migration pg12
- rspec-ee unit pg12
- rspec-ee integration pg12
- rspec-ee system pg12
- rspec-ee unit pg12 geo
- rspec-ee integration pg12 geo
- rspec-ee system pg12 geo
- memory-static
- memory-on-boot
script:
- !reference [.minimal-bundle-install, script]
- run_timed_command "bundle exec scripts/merge-simplecov"
- run_timed_command "bundle exec scripts/gather-test-memory-data"
coverage: '/LOC \((\d+\.\d+%)\) covered.$/'
artifacts:
name: coverage
expire_in: 31d
paths:
- coverage/index.html
- coverage/assets/
- tmp/memory_test/
reports:
cobertura: coverage/coverage.xml
rspec:feature-flags:
extends:
- .coverage-base
- .rails:rules:rspec-feature-flags
stage: post-test
# We cannot use needs since it would mean needing 84 jobs (since most are parallelized)
# so we use `dependencies` here.
dependencies:
- setup-test-env
- rspec migration pg12
- rspec unit pg12
- rspec integration pg12
- rspec system pg12
- rspec-ee migration pg12
- rspec-ee unit pg12
- rspec-ee integration pg12
- rspec-ee system pg12
- rspec-ee unit pg12 geo
- rspec-ee integration pg12 geo
- rspec-ee system pg12 geo
- memory-static
- memory-on-boot
script:
- !reference [.minimal-bundle-install, script]
- if [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then
run_timed_command "bundle exec scripts/used-feature-flags" || (scripts/slack master-broken "☠️ \`${CI_JOB_NAME}\` failed! ☠️ See ${CI_JOB_URL}" ci_failing "GitLab Bot" && exit 1);
else
run_timed_command "bundle exec scripts/used-feature-flags";
fi
# EE/FOSS: default refs (MRs, default branch, schedules) jobs #
#######################################################
##################################################
# EE: default refs (MRs, default branch, schedules) jobs #
rspec migration pg12-as-if-foss:
extends:
- .rspec-base-pg12-as-if-foss
- .rspec-base-migration
- .rails:rules:as-if-foss-migration
- .rspec-migration-parallel
rspec migration pg12-as-if-foss minimal:
extends:
- rspec migration pg12-as-if-foss
- .minimal-rspec-tests
- .rails:rules:as-if-foss-migration:minimal
rspec unit pg12-as-if-foss:
extends:
- .rspec-base-pg12-as-if-foss
- .rails:rules:as-if-foss-unit
- .rspec-unit-parallel
rspec unit pg12-as-if-foss minimal:
extends:
- rspec unit pg12-as-if-foss
- .minimal-rspec-tests
- .rails:rules:as-if-foss-unit:minimal
rspec integration pg12-as-if-foss:
extends:
- .rspec-base-pg12-as-if-foss
- .rails:rules:as-if-foss-integration
- .rspec-integration-parallel
rspec integration pg12-as-if-foss minimal:
extends:
- rspec integration pg12-as-if-foss
- .minimal-rspec-tests
- .rails:rules:as-if-foss-integration:minimal
rspec system pg12-as-if-foss:
extends:
- .rspec-base-pg12-as-if-foss
- .rails:rules:as-if-foss-system
- .rspec-system-parallel
rspec system pg12-as-if-foss minimal:
extends:
- rspec system pg12-as-if-foss
- .minimal-rspec-tests
- .rails:rules:as-if-foss-system:minimal
rspec-ee migration pg12:
extends:
- .rspec-ee-base-pg12
- .rspec-base-migration
- .rails:rules:ee-only-migration
- .rspec-ee-migration-parallel
rspec-ee migration pg12 minimal:
extends:
- rspec-ee migration pg12
- .minimal-rspec-tests
- .rails:rules:ee-only-migration:minimal
rspec-ee unit pg12:
extends:
- .rspec-ee-base-pg12
- .rails:rules:ee-only-unit
- .rspec-ee-unit-parallel
rspec-ee unit pg12 minimal:
extends:
- rspec-ee unit pg12
- .minimal-rspec-tests
- .rails:rules:ee-only-unit:minimal
rspec-ee integration pg12:
extends:
- .rspec-ee-base-pg12
- .rails:rules:ee-only-integration
- .rspec-ee-integration-parallel
rspec-ee integration pg12 minimal:
extends:
- rspec-ee integration pg12
- .minimal-rspec-tests
- .rails:rules:ee-only-integration:minimal
rspec-ee system pg12:
extends:
- .rspec-ee-base-pg12
- .rails:rules:ee-only-system
- .rspec-ee-system-parallel
rspec-ee system pg12 minimal:
extends:
- rspec-ee system pg12
- .minimal-rspec-tests
- .rails:rules:ee-only-system:minimal
rspec-ee unit pg12 geo:
extends:
- .rspec-ee-base-geo-pg12
- .rails:rules:ee-only-unit
- .rspec-ee-unit-geo-parallel
# FIXME: Temporarily disable geo minimal rspec jobs https://gitlab.com/gitlab-org/gitlab/-/issues/294212
#rspec-ee unit pg12 geo minimal:
# extends:
# - rspec-ee unit pg12 geo
# - .minimal-rspec-tests
# - .rails:rules:ee-only-unit:minimal
rspec-ee integration pg12 geo:
extends:
- .rspec-ee-base-geo-pg12
- .rails:rules:ee-only-integration
# FIXME: Temporarily disable geo minimal rspec jobs https://gitlab.com/gitlab-org/gitlab/-/issues/294212
#rspec-ee integration pg12 geo minimal:
# extends:
# - rspec-ee integration pg12 geo
# - .minimal-rspec-tests
# - .rails:rules:ee-only-integration:minimal
rspec-ee system pg12 geo:
extends:
- .rspec-ee-base-geo-pg12
- .rails:rules:ee-only-system
# FIXME: Temporarily disable geo minimal rspec jobs https://gitlab.com/gitlab-org/gitlab/-/issues/294212
#rspec-ee system pg12 geo minimal:
# extends:
# - rspec-ee system pg12 geo
# - .minimal-rspec-tests
# - .rails:rules:ee-only-system:minimal
db:rollback geo:
extends:
- db:rollback
- .rails:rules:ee-only-migration
script:
- bundle exec rake geo:db:migrate VERSION=20170627195211
- bundle exec rake geo:db:migrate
# EE: default refs (MRs, default branch, schedules) jobs #
##################################################
##########################################
# EE/FOSS: default branch nightly scheduled jobs #
rspec migration pg11:
extends:
- .rspec-base-pg11
- .rspec-base-migration
- .rails:rules:default-branch-schedule-nightly--code-backstage
- .rspec-migration-parallel
rspec unit pg11:
extends:
- .rspec-base-pg11
- .rails:rules:default-branch-schedule-nightly--code-backstage
- .rspec-unit-parallel
rspec integration pg11:
extends:
- .rspec-base-pg11
- .rails:rules:default-branch-schedule-nightly--code-backstage
- .rspec-integration-parallel
rspec system pg11:
extends:
- .rspec-base-pg11
- .rails:rules:default-branch-schedule-nightly--code-backstage
- .rspec-system-parallel
# EE/FOSS: default branch nightly scheduled jobs #
##########################################
#####################################
# EE: default branch nightly scheduled jobs #
rspec-ee migration pg11:
extends:
- .rspec-ee-base-pg11
- .rspec-base-migration
- .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only
- .rspec-ee-migration-parallel
rspec-ee unit pg11:
extends:
- .rspec-ee-base-pg11
- .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only
- .rspec-ee-unit-parallel
rspec-ee integration pg11:
extends:
- .rspec-ee-base-pg11
- .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only
- .rspec-ee-integration-parallel
rspec-ee system pg11:
extends:
- .rspec-ee-base-pg11
- .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only
- .rspec-ee-system-parallel
rspec-ee unit pg11 geo:
extends:
- .rspec-ee-base-geo-pg11
- .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only
- .rspec-ee-unit-geo-parallel
rspec-ee integration pg11 geo:
extends:
- .rspec-ee-base-geo-pg11
- .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only
rspec-ee system pg11 geo:
extends:
- .rspec-ee-base-geo-pg11
- .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only
# EE: default branch nightly scheduled jobs #
#####################################
##################################################
# EE: Canonical MR pipelines
rspec fail-fast:
extends:
- .rspec-ee-base-pg12 # This job also runs EE spec which needs elasticsearch
- .rails:rules:rspec fail-fast
stage: test
needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets", "detect-tests"]
script:
- !reference [.base-script, script]
- rspec_fail_fast tmp/matching_tests.txt "--tag ~quarantine"
artifacts:
expire_in: 7d
paths:
- tmp/capybara/
rspec foss-impact:
extends:
- .rspec-base-pg12-as-if-foss
- .rails:rules:rspec-foss-impact
needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets as-if-foss", "detect-tests as-if-foss"]
script:
- !reference [.base-script, script]
- rspec_matched_foss_tests tmp/matching_foss_tests.txt "--tag ~quarantine"
artifacts:
expire_in: 7d
paths:
- tmp/capybara/
fail-pipeline-early:
extends:
- .rails:rules:fail-pipeline-early
stage: post-test
needs:
- job: rspec fail-fast
artifacts: false
variables:
GIT_DEPTH: 1
before_script:
- source scripts/utils.sh
- install_gitlab_gem
script:
- fail_pipeline_early
# EE: Canonical MR pipelines
##################################################