diff --git a/.codeclimate.yml b/.codeclimate.yml index c479454285..fe3c605689 100644 --- a/.codeclimate.yml +++ b/.codeclimate.yml @@ -1,5 +1,6 @@ --- -engines: +version: "2" +plugins: bundler-audit: enabled: true duplication: @@ -8,33 +9,22 @@ engines: languages: - ruby - javascript -ratings: - paths: - - Gemfile.lock - - "**.erb" - - "**.haml" - - "**.rb" - - "**.rhtml" - - "**.slim" - - "**.inc" - - "**.js" - - "**.jsx" - - "**.module" -exclude_paths: - - config/ - - db/ - - features/ - - node_modules/ - - spec/ - - vendor/ + rubocop: + enabled: false +exclude_patterns: + - "{ee/,jh/,}config/" + - "{ee/,jh/,}db/" + - "**/log/" + - "**/node_modules/" + - "**/spec/" + - "**/tmp/" + - "**/vendor/" - .yarn-cache/ - - tmp/ + - backups/ - builds/ - coverage/ + - file_hooks/ + - plugins/ - public/ - shared/ - webpack-report/ - - log/ - - backups/ - - plugins/ - - file_hooks/ diff --git a/.eslintignore b/.eslintignore index d5f89284b5..1d069e1938 100644 --- a/.eslintignore +++ b/.eslintignore @@ -7,3 +7,5 @@ /tmp/ /vendor/ /sitespeed-result/ +/fixtures/**/*.graphql +spec/fixtures/**/*.graphql diff --git a/.eslintrc.yml b/.eslintrc.yml index cd3cd82d4e..623c1a6630 100644 --- a/.eslintrc.yml +++ b/.eslintrc.yml @@ -113,3 +113,24 @@ overrides: - '*.stories.js' rules: filenames/match-regex: off + - files: + - '*.graphql' + plugins: + - '@graphql-eslint' + parserOptions: + parser: '@graphql-eslint/eslint-plugin' + operations: + - '{,ee/,jh/}app/**/*.graphql' + # You can run `bundle exec rake gitlab:graphql:schema:dump` and then uncomment this line + # schema: './tmp/tests/graphql/gitlab_schema.graphql' + rules: + filenames/match-regex: off + spaced-comment: off + # TODO: We need a way to include this rule + support ee_else_ce fragments + #'@graphql-eslint/unique-fragment-name': error + # TODO: Uncomment these rules when then `schema` is available + #'@graphql-eslint/fragments-on-composite-type': error + #'@graphql-eslint/known-argument-names': error + #'@graphql-eslint/known-type-names': error + '@graphql-eslint/no-anonymous-operations': error + '@graphql-eslint/unique-operation-name': error diff --git a/.gitignore b/.gitignore index 5152ef2057..bff82967fc 100644 --- a/.gitignore +++ b/.gitignore @@ -74,6 +74,7 @@ eslint-report.html /.gitlab_kas_secret /webpack-report/ /crystalball/ +/test_results/ /deprecations/ /knapsack/ /rspec_flaky/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b72ad35953..e810edd813 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,6 +3,7 @@ stages: - prepare - build-images - fixtures + - lint - test - post-test - review @@ -15,7 +16,7 @@ stages: # in cases where jobs require Docker-in-Docker, the job # definition must be extended with `.use-docker-in-docker` default: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36" + image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36" tags: - gitlab-org # All jobs are interruptible by default diff --git a/.gitlab/CODEOWNERS b/.gitlab/CODEOWNERS index 64e74dd12e..c6a89ddfa9 100644 --- a/.gitlab/CODEOWNERS +++ b/.gitlab/CODEOWNERS @@ -26,7 +26,7 @@ /doc/administration/troubleshooting @marcel.amirault @marcia @eread /doc/api/graphql/ @msedlakjakubowski @kpaizee /doc/api/graphql/reference/ @kpaizee -/doc/api/group_activity_analytics.md @msedlakjakubowski +/doc/api/group_activity_analytics.md @fneill /doc/ci/ @marcel.amirault @sselhorn /doc/ci/environments/ @rdickenson /doc/ci/services/ @sselhorn @@ -34,7 +34,7 @@ /doc/development/ @marcia /doc/development/documentation/ @cnorris @dianalogan /doc/development/i18n/ @ngaskill -/doc/development/value_stream_analytics.md @msedlakjakubowski +/doc/development/value_stream_analytics.md @fneill /doc/gitlab-basics/ @aqualls /doc/install/ @marcel.amirault /doc/operations/ @ngaskill @rdickenson @@ -45,19 +45,19 @@ /doc/topics/autodevops/ @marcia /doc/topics/git/ @aqualls /doc/update/ @marcel.amirault @marcia -/doc/user/analytics/ @msedlakjakubowski @ngaskill +/doc/user/analytics/ @fneill @ngaskill /doc/user/application_security/ @rdickenson /doc/user/application_security/container_scanning/ @ngaskill /doc/user/application_security/cluster_image_scanning/ @ngaskill /doc/user/clusters/ @marcia /doc/user/compliance/ @rdickenson @eread /doc/user/group/ @msedlakjakubowski -/doc/user/group/devops_adoption/ @msedlakjakubowski +/doc/user/group/devops_adoption/ @fneill /doc/user/group/epics/ @msedlakjakubowski -/doc/user/group/insights/ @msedlakjakubowski +/doc/user/group/insights/ @fneill /doc/user/group/iterations/ @msedlakjakubowski /doc/user/group/roadmap/ @msedlakjakubowski -/doc/user/group/value_stream_analytics/ @msedlakjakubowski +/doc/user/group/value_stream_analytics/ @fneill /doc/user/infrastructure/ @marcia /doc/user/packages/ @ngaskill /doc/user/packages/infrastructure_registry/ @marcia @@ -76,6 +76,7 @@ /doc/user/project/web_ide/index.md @aqualls /doc/user/project/wiki/index.md @aqualls /doc/user/search/ @marcia @aqualls +/doc/user/workspace/ @fneill [Docs Create] /doc/administration/file_hooks.md @aqualls diff --git a/.gitlab/ci/cache-repo.gitlab-ci.yml b/.gitlab/ci/cache-repo.gitlab-ci.yml deleted file mode 100644 index 98c8c72ae3..0000000000 --- a/.gitlab/ci/cache-repo.gitlab-ci.yml +++ /dev/null @@ -1,63 +0,0 @@ -# Builds a cached .tar.gz of the $CI_DEFAULT_BRANCH branch with full history and -# uploads it to Google Cloud Storage. This archive is downloaded by a -# script defined by a CI/CD variable named CI_PRE_CLONE_SCRIPT. This has -# two benefits: -# -# 1. It speeds up builds. A 800 MB download only takes seconds. -# 2. It significantly reduces load on the file server. Smaller deltas -# means less time spent in git pack-objects. -# -# Since the destination directory of the archive depends on the project -# ID, this is only run on GitLab.com. -# -# CI_REPO_CACHE_CREDENTIALS contains the Google Cloud service account -# JSON for uploading to the gitlab-ci-git-repo-cache bucket. These -# credentials are stored in the Production vault. -# -# Note that this bucket should be located in the same continent as the -# runner, or network egress charges will apply: -# https://cloud.google.com/storage/pricing -cache-repo: - extends: .cache-repo:rules - image: gcr.io/google.com/cloudsdktool/cloud-sdk:alpine - stage: sync - variables: - GIT_STRATEGY: none - SHALLOW_CLONE_TAR_FILENAME: gitlab-master-shallow.tar - FULL_CLONE_TAR_FILENAME: gitlab-master.tar - before_script: - - '[ -z "$CI_REPO_CACHE_CREDENTIALS" ] || gcloud auth activate-service-account --key-file=$CI_REPO_CACHE_CREDENTIALS' - script: - # Enable shallow repo caching unless the $DISABLE_SHALLOW_REPO_CACHING variable exists (in the case the shallow clone caching isn't working well) - # The `git repack` call works around a Git bug with shallow clones: https://gitlab.com/gitlab-org/git/-/issues/86 - - if [ -z "$DISABLE_SHALLOW_REPO_CACHING" ]; then - cd .. && rm -rf $CI_PROJECT_NAME; - today=$(date +%Y-%m-%d); - year=$(date +%Y); - last_year=`expr $year - 1`; - one_year_ago=$(echo $today | sed "s/$year/$last_year/"); - echo "Cloning $CI_REPOSITORY_URL into $CI_PROJECT_NAME with commits from $one_year_ago."; - time git clone --progress --no-checkout --shallow-since=$one_year_ago $CI_REPOSITORY_URL $CI_PROJECT_NAME; - cd $CI_PROJECT_NAME; - time git repack -d; - echo "Archiving $CI_PROJECT_NAME into /tmp/$SHALLOW_CLONE_TAR_FILENAME."; - time git remote rm origin; - time tar cf /tmp/$SHALLOW_CLONE_TAR_FILENAME .; - echo "GZipping /tmp/$SHALLOW_CLONE_TAR_FILENAME."; - time gzip /tmp/$SHALLOW_CLONE_TAR_FILENAME; - [ -z "$CI_REPO_CACHE_CREDENTIALS" ] || (echo "Uploading /tmp/$SHALLOW_CLONE_TAR_FILENAME.gz to GCloud." && time gsutil cp /tmp/$SHALLOW_CLONE_TAR_FILENAME.gz gs://gitlab-ci-git-repo-cache/project-$CI_PROJECT_ID/$SHALLOW_CLONE_TAR_FILENAME.gz); - fi - # Disable the full repo caching unless the $DISABLE_SHALLOW_REPO_CACHING variable exists (in the case the shallow clone caching isn't working well) - - if [ -n "$DISABLE_SHALLOW_REPO_CACHING" ]; then - cd .. && rm -rf $CI_PROJECT_NAME; - echo "Cloning $CI_REPOSITORY_URL into $CI_PROJECT_NAME."; - time git clone --progress $CI_REPOSITORY_URL $CI_PROJECT_NAME; - cd $CI_PROJECT_NAME; - time git repack -d; - echo "Archiving $CI_PROJECT_NAME into /tmp/$FULL_CLONE_TAR_FILENAME."; - time git remote rm origin; - time tar cf /tmp/$FULL_CLONE_TAR_FILENAME .; - echo "GZipping /tmp/$FULL_CLONE_TAR_FILENAME."; - time gzip /tmp/$FULL_CLONE_TAR_FILENAME; - [ -z "$CI_REPO_CACHE_CREDENTIALS" ] || (echo "Uploading /tmp/$FULL_CLONE_TAR_FILENAME.gz to GCloud." && time gsutil cp /tmp/$FULL_CLONE_TAR_FILENAME.gz gs://gitlab-ci-git-repo-cache/project-$CI_PROJECT_ID/$FULL_CLONE_TAR_FILENAME.gz); - fi diff --git a/.gitlab/ci/docs.gitlab-ci.yml b/.gitlab/ci/docs.gitlab-ci.yml index f4d8698f22..6aa3d53331 100644 --- a/.gitlab/ci/docs.gitlab-ci.yml +++ b/.gitlab/ci/docs.gitlab-ci.yml @@ -44,8 +44,8 @@ docs-lint markdown: - .default-retry - .docs:rules:docs-lint # When updating the image version here, update it in /scripts/lint-doc.sh too. - image: registry.gitlab.com/gitlab-org/gitlab-docs/lint-markdown:alpine-3.14-vale-2.10.4-markdownlint-0.28.1 - stage: test + image: registry.gitlab.com/gitlab-org/gitlab-docs/lint-markdown:alpine-3.14-vale-2.12.0-markdownlint-0.29.0 + stage: lint needs: [] script: - scripts/lint-doc.sh @@ -53,8 +53,8 @@ docs-lint markdown: docs-lint links: extends: - .docs:rules:docs-lint - image: registry.gitlab.com/gitlab-org/gitlab-docs/lint-html:alpine-3.14-ruby-2.7.4-db71f027 - stage: test + image: registry.gitlab.com/gitlab-org/gitlab-docs/lint-html:alpine-3.14-ruby-2.7.4-0fae0f62 + stage: lint needs: [] script: # Prepare docs for build @@ -71,7 +71,21 @@ ui-docs-links lint: extends: - .docs:rules:docs-lint - .static-analysis-base - stage: test + - .ruby-cache + stage: lint needs: [] script: - bundle exec haml-lint -i DocumentationLinks + +docs-lint deprecations: + variables: + SETUP_DB: "false" + extends: + - .default-retry + - .rails-cache + - .default-before_script + - .docs:rules:deprecations + stage: lint + needs: [] + script: + - bundle exec rake gitlab:docs:check_deprecations diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml index 6974d63a49..d3844d0121 100644 --- a/.gitlab/ci/frontend.gitlab-ci.yml +++ b/.gitlab/ci/frontend.gitlab-ci.yml @@ -11,7 +11,7 @@ - .default-retry - .default-before_script - .assets-compile-cache - image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7-git-2.31-lfs-2.9-node-14.15-yarn-1.22-graphicsmagick-1.3.36 + image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7-git-2.33-lfs-2.9-node-14.15-yarn-1.22-graphicsmagick-1.3.36 variables: SETUP_DB: "false" WEBPACK_VENDOR_DLL: "true" @@ -133,6 +133,7 @@ update-storybook-yarn-cache: paths: - tmp/tests/frontend/ - knapsack/ + - crystalball/ # Builds FOSS, and EE fixtures in the EE project. # Builds FOSS fixtures in the FOSS project. @@ -193,21 +194,10 @@ graphql-schema-dump: # Disable warnings in browserslist which can break on backports # https://github.com/browserslist/browserslist/blob/a287ec6/node.js#L367-L384 BROWSERSLIST_IGNORE_OLD_DATA: "true" - SETUP_DB: "false" before_script: - - !reference [.default-before_script, before_script] - *yarn-install stage: test -eslint-as-if-foss: - extends: - - .frontend-test-base - - .frontend:rules:eslint-as-if-foss - - .as-if-foss - needs: [] - script: - - run_timed_command "yarn run lint:eslint:all" - .jest-base: extends: .frontend-test-base script: diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml index d0c26d6006..22bd3b1e2e 100644 --- a/.gitlab/ci/global.gitlab-ci.yml +++ b/.gitlab/ci/global.gitlab-ci.yml @@ -44,6 +44,7 @@ prefix: "gitaly-binaries" paths: - tmp/tests/gitaly/_build/bin/ + - tmp/tests/gitaly/_build/deps/git/install/ - tmp/tests/gitaly/config.toml - tmp/tests/gitaly/gitaly2.config.toml - tmp/tests/gitaly/internal/ @@ -139,6 +140,10 @@ - <<: *gitaly-binaries-cache policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up. +.ruby-cache: + cache: + - *ruby-gems-cache + .rails-cache: cache: - *ruby-gems-cache @@ -150,7 +155,12 @@ - *node-modules-cache - *rubocop-cache -.static-analysis-cache-push: +.rubocop-job-cache: + cache: + - *ruby-gems-cache + - *rubocop-cache + +.rubocop-job-cache-push: cache: - *ruby-gems-cache # We don't push this cache as it's already rebuilt by `update-setup-test-env-cache` - *rubocop-cache-push @@ -203,7 +213,7 @@ - *storybook-node-modules-cache-push .use-pg11: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36" + image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36" services: - name: postgres:11.6 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] @@ -212,7 +222,7 @@ POSTGRES_HOST_AUTH_METHOD: trust .use-pg12: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36" + image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36" services: - name: postgres:12 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] @@ -221,7 +231,7 @@ POSTGRES_HOST_AUTH_METHOD: trust .use-pg13: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-13-graphicsmagick-1.3.36" + image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-13-graphicsmagick-1.3.36" services: - name: postgres:13 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] @@ -230,34 +240,34 @@ POSTGRES_HOST_AUTH_METHOD: trust .use-pg11-ee: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36" + image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36" services: - name: postgres:11.6 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - name: redis:5.0-alpine - - name: elasticsearch:7.11.1 + - name: elasticsearch:7.14.2 command: ["elasticsearch", "-E", "discovery.type=single-node"] variables: POSTGRES_HOST_AUTH_METHOD: trust .use-pg12-ee: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36" + image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36" services: - name: postgres:12 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - name: redis:5.0-alpine - - name: elasticsearch:7.11.1 + - name: elasticsearch:7.14.2 command: ["elasticsearch", "-E", "discovery.type=single-node"] variables: POSTGRES_HOST_AUTH_METHOD: trust .use-pg13-ee: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-13-graphicsmagick-1.3.36" + image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-13-graphicsmagick-1.3.36" services: - name: postgres:13 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - name: redis:5.0-alpine - - name: elasticsearch:7.11.1 + - name: elasticsearch:7.14.2 command: ["elasticsearch", "-E", "discovery.type=single-node"] variables: POSTGRES_HOST_AUTH_METHOD: trust diff --git a/.gitlab/ci/qa-report.gitlab-ci.yml b/.gitlab/ci/qa-report.gitlab-ci.yml new file mode 100644 index 0000000000..61cbcfd58d --- /dev/null +++ b/.gitlab/ci/qa-report.gitlab-ci.yml @@ -0,0 +1,15 @@ +test-reliability-report: + extends: + - .qa:rules:reliable-reports:schedule + image: + name: ${CI_REGISTRY_IMAGE}/gitlab-ee-qa:${CI_DEFAULT_BRANCH} + entrypoint: [""] + before_script: + - cd /home/gitlab/qa + script: + - echo "Generate report for 'staging-full' runs" + - bundle exec rake "reliable_spec_report[staging-full,30,true]" + - bundle exec rake "unreliable_spec_report[staging-full,30,true]" + - echo "Generate report for 'package-and-qa' runs" + - bundle exec rake "reliable_spec_report[package-and-qa,30,true]" + - bundle exec rake "unreliable_spec_report[package-and-qa,30,true]" diff --git a/.gitlab/ci/qa.gitlab-ci.yml b/.gitlab/ci/qa.gitlab-ci.yml index 88e732c2e7..74fc5f2cdc 100644 --- a/.gitlab/ci/qa.gitlab-ci.yml +++ b/.gitlab/ci/qa.gitlab-ci.yml @@ -55,14 +55,18 @@ update-qa-cache: before_script: - source scripts/utils.sh - install_gitlab_gem + - tooling/bin/find_change_diffs ${CHANGES_DIFFS_DIR} script: - - ./scripts/trigger-build omnibus - -package-and-qa: - extends: - - .package-and-qa-base - - .qa:rules:package-and-qa - # This job often times out, so temporarily use private runners and a long timeout: https://gitlab.com/gitlab-org/gitlab/-/issues/238563 + - | + tooling/bin/qa/package_and_qa_check ${CHANGES_DIFFS_DIR} && exit_code=$? + if [ $exit_code -eq 0 ]; then + ./scripts/trigger-build omnibus + elif [ $exit_code -eq 1 ]; then + exit 1 + else + echo "Downstream jobs will not be triggered because package_and_qa_check exited with code: $exit_code" + fi + # These jobs often time out, so temporarily use private runners and a long timeout: https://gitlab.com/gitlab-org/gitlab/-/issues/238563 tags: - prm timeout: 4h @@ -71,3 +75,39 @@ package-and-qa: artifacts: false - job: build-assets-image artifacts: false + - detect-tests + artifacts: + expire_in: 7d + paths: + - ${CHANGES_FILE} + - ${CHANGES_DIFFS_DIR}/* + variables: + CHANGES_FILE: tmp/changed_files.txt + CHANGES_DIFFS_DIR: tmp/diffs + +.package-and-qa-ff-base: + script: + - export GITLAB_QA_OPTIONS="--set-feature-flags $(scripts/changed-feature-flags --files $(cat $CHANGES_FILE | tr ' ' ',') --state $QA_FF_STATE)" + - echo $GITLAB_QA_OPTIONS + - ./scripts/trigger-build omnibus + +package-and-qa: + extends: + - .package-and-qa-base + - .qa:rules:package-and-qa + +package-and-qa-ff-enabled: + extends: + - .package-and-qa-base + - .package-and-qa-ff-base + - .qa:rules:package-and-qa:feature-flags + variables: + QA_FF_STATE: "enable" + +package-and-qa-ff-disabled: + extends: + - .package-and-qa-base + - .package-and-qa-ff-base + - .qa:rules:package-and-qa:feature-flags + variables: + QA_FF_STATE: "disable" diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index 00f65ab7ca..1fb1c887e5 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -6,11 +6,6 @@ - .default-before_script - .rails-cache -.minimal-bundle-install: - script: - - export BUNDLE_WITHOUT="${BUNDLE_WITHOUT}:default:test:puma:kerberos:metrics:omnibus:ed25519" - - bundle_install_script - .base-script: script: # Only install knapsack after bundle install! Otherwise oddly some native @@ -26,6 +21,8 @@ .decomposed-database-rspec: variables: DECOMPOSED_DB: "true" + GITLAB_LOAD_BALANCING_REUSE_PRIMARY_ci: "main" + GITLAB_USE_MODEL_LOAD_BALANCING: "true" .rspec-base: extends: .rails-job-base @@ -77,6 +74,12 @@ - .use-pg12 needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets as-if-foss", "detect-tests"] +.rspec-base-pg12-as-if-jh: + extends: + - .rspec-base + - .use-pg12 + needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets as-if-jh", "detect-tests", "add-jh-folder"] + .rspec-base-pg13: extends: - .rspec-base @@ -92,6 +95,11 @@ - .rspec-base - .use-pg12-ee +.rspec-jh-base-pg12: + extends: + - .rspec-base-pg12-as-if-jh + - .use-pg12-ee + .rspec-ee-base-pg13: extends: - .rspec-base @@ -113,6 +121,12 @@ - .rspec-ee-base-geo - .use-pg12-ee +.rspec-jh-base-geo-pg12: + extends: + - .rspec-jh-base-pg12 + script: + - !reference [.rspec-ee-base-geo, script] + .rspec-ee-base-geo-pg13: extends: - .rspec-ee-base-geo @@ -165,10 +179,9 @@ setup-test-env: extends: - .rails-job-base - .setup-test-env-cache - - .rails:rules:code-backstage-qa + - .rails:rules:setup-test-env stage: prepare variables: - GITLAB_TEST_EAGER_LOAD: "0" SETUP_DB: "false" script: - run_timed_command "scripts/setup-test-env" @@ -181,6 +194,7 @@ setup-test-env: paths: - config/secrets.yml - tmp/tests/gitaly/_build/bin/ + - tmp/tests/gitaly/_build/deps/git/install - tmp/tests/gitaly/config.toml - tmp/tests/gitaly/gitaly2.config.toml - tmp/tests/gitaly/internal/ @@ -223,11 +237,11 @@ update-gitaly-binaries-cache: .coverage-base: extends: - .default-retry - - .default-before_script - .coverage-cache - variables: - SETUP_DB: "false" - USE_BUNDLE_INSTALL: "false" + before_script: + - source scripts/utils.sh + - export BUNDLE_WITHOUT="${BUNDLE_WITHOUT}:default:test:puma:kerberos:metrics:omnibus:ed25519" + - bundle_install_script rspec migration pg12: extends: @@ -324,27 +338,17 @@ rspec fast_spec_helper minimal: - .minimal-rspec-tests - .rails:rules:ee-and-foss-fast_spec_helper:minimal +db:rollback: + extends: .db-job-base + script: + - bundle exec rake db:migrate VERSION=20181228175414 + - bundle exec rake db:migrate SKIP_SCHEMA_VERSION_CHECK=true + db:migrate:reset: extends: .db-job-base script: - bundle exec rake db:migrate:reset -db:check-schema: - extends: - - .db-job-base - - .rails:rules:ee-mr-and-default-branch-only - script: - - source scripts/schema_changed.sh - - scripts/validate_migration_timestamps - -db:check-migrations: - extends: - - .db-job-base - - .rails:rules:ee-and-foss-mr-with-migration - script: - - scripts/validate_migration_schema - allow_failure: true - db:migrate-from-previous-major-version: extends: .db-job-base variables: @@ -352,7 +356,8 @@ db:migrate-from-previous-major-version: SETUP_DB: "false" PROJECT_TO_CHECKOUT: "gitlab-foss" TAG_TO_CHECKOUT: "v13.12.9" - script: + before_script: + - !reference [.default-before_script, before_script] - '[[ -d "ee/" ]] || export PROJECT_TO_CHECKOUT="gitlab"' - '[[ -d "ee/" ]] || export TAG_TO_CHECKOUT="${TAG_TO_CHECKOUT}-ee"' - retry 'git fetch https://gitlab.com/gitlab-org/$PROJECT_TO_CHECKOUT.git $TAG_TO_CHECKOUT' @@ -361,13 +366,28 @@ db:migrate-from-previous-major-version: - run_timed_command "bundle exec rake db:drop db:create db:structure:load db:migrate db:seed_fu" - git checkout -f $CI_COMMIT_SHA - SETUP_DB=false USE_BUNDLE_INSTALL=true bash scripts/prepare_build.sh + script: - run_timed_command "bundle exec rake db:migrate" -db:rollback: - extends: .db-job-base +db:check-schema: + extends: + - db:migrate-from-previous-major-version + - .rails:rules:ee-mr-and-default-branch-only + variables: + TAG_TO_CHECKOUT: "v14.4.0" script: - - bundle exec rake db:migrate VERSION=20181228175414 - - bundle exec rake db:migrate SKIP_SCHEMA_VERSION_CHECK=true + - run_timed_command "bundle exec rake db:migrate" + - scripts/schema_changed.sh + - scripts/validate_migration_timestamps + +db:check-migrations: + extends: + - .db-job-base + - .rails:rules:ee-and-foss-mr-with-migration + script: + - git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:$CI_MERGE_REQUEST_TARGET_BRANCH_NAME --depth 20 + - scripts/validate_migration_schema + allow_failure: true db:gitlabcom-database-testing: extends: .rails:rules:db:gitlabcom-database-testing @@ -457,21 +477,38 @@ rspec:coverage: # so we use `dependencies` here. dependencies: - setup-test-env + # FOSS/EE jobs - rspec migration pg12 - rspec unit pg12 - rspec integration pg12 - rspec system pg12 + # FOSS/EE minimal jobs + - rspec migration pg12 minimal + - rspec unit pg12 minimal + - rspec integration pg12 minimal + - rspec system pg12 minimal + # EE jobs - rspec-ee migration pg12 - rspec-ee unit pg12 - rspec-ee integration pg12 - rspec-ee system pg12 + # EE minimal jobs + - rspec-ee migration pg12 minimal + - rspec-ee unit pg12 minimal + - rspec-ee integration pg12 minimal + - rspec-ee system pg12 minimal + # Geo jobs - rspec-ee unit pg12 geo - rspec-ee integration pg12 geo - rspec-ee system pg12 geo + # Geo minimal jobs + - rspec-ee unit pg12 geo minimal + - rspec-ee integration pg12 geo minimal + - rspec-ee system pg12 geo minimal + # Memory jobs - memory-static - memory-on-boot script: - - !reference [.minimal-bundle-install, script] - run_timed_command "bundle exec scripts/merge-simplecov" - run_timed_command "bundle exec scripts/gather-test-memory-data" coverage: '/LOC \((\d+\.\d+%)\) covered.$/' @@ -490,15 +527,67 @@ rspec:feature-flags: - .coverage-base - .rails:rules:rspec-feature-flags stage: post-test - needs: ["static-analysis"] + needs: + - job: "feature-flags-usage" + - job: "haml-lint foss" + - job: "haml-lint ee" + optional: true script: - - !reference [.minimal-bundle-install, script] - if [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then run_timed_command "bundle exec scripts/used-feature-flags" || (scripts/slack master-broken "☠️ \`${CI_JOB_NAME}\` failed! ☠️ See ${CI_JOB_URL}" ci_failing "GitLab Bot" && exit 1); else run_timed_command "bundle exec scripts/used-feature-flags"; fi +rspec:skipped-flaky-tests-report: + extends: + - .default-retry + - .rails:rules:skipped-flaky-tests-report + image: ruby:2.7-alpine + stage: post-test + # We cannot use needs since it would mean needing 84 jobs (since most are parallelized) + # so we use `dependencies` here. + dependencies: + # FOSS/EE jobs + - rspec migration pg12 + - rspec unit pg12 + - rspec integration pg12 + - rspec system pg12 + # FOSS/EE minimal jobs + - rspec migration pg12 minimal + - rspec unit pg12 minimal + - rspec integration pg12 minimal + - rspec system pg12 minimal + # EE jobs + - rspec-ee migration pg12 + - rspec-ee unit pg12 + - rspec-ee integration pg12 + - rspec-ee system pg12 + # EE minimal jobs + - rspec-ee migration pg12 minimal + - rspec-ee unit pg12 minimal + - rspec-ee integration pg12 minimal + - rspec-ee system pg12 minimal + # Geo jobs + - rspec-ee unit pg12 geo + - rspec-ee integration pg12 geo + - rspec-ee system pg12 geo + # Geo minimal jobs + - rspec-ee unit pg12 geo minimal + - rspec-ee integration pg12 geo minimal + - rspec-ee system pg12 geo minimal + variables: + SKIPPED_FLAKY_TESTS_REPORT: skipped_flaky_tests_report.txt + before_script: + - 'echo "SKIP_FLAKY_TESTS_AUTOMATICALLY: $SKIP_FLAKY_TESTS_AUTOMATICALLY"' + - mkdir -p rspec_flaky + script: + - find rspec_flaky/ -type f -name 'skipped_flaky_tests_*_report.txt' -exec cat {} + >> "${SKIPPED_FLAKY_TESTS_REPORT}" + artifacts: + expire_in: 31d + paths: + - ${SKIPPED_FLAKY_TESTS_REPORT} + # EE/FOSS: default refs (MRs, default branch, schedules) jobs # ####################################################### @@ -577,6 +666,31 @@ rspec system pg12-as-if-foss decomposed: - .decomposed-database-rspec - .rails:rules:decomposed-databases +rspec migration pg12-as-if-jh: + extends: + - .rspec-base-pg12-as-if-jh + - .rspec-base-migration + - .rails:rules:as-if-jh-rspec + - .rspec-migration-parallel + +rspec unit pg12-as-if-jh: + extends: + - .rspec-base-pg12-as-if-jh + - .rails:rules:as-if-jh-rspec + - .rspec-unit-parallel + +rspec integration pg12-as-if-jh: + extends: + - .rspec-base-pg12-as-if-jh + - .rails:rules:as-if-jh-rspec + - .rspec-integration-parallel + +rspec system pg12-as-if-jh: + extends: + - .rspec-base-pg12-as-if-jh + - .rails:rules:as-if-jh-rspec + - .rspec-system-parallel + rspec-ee migration pg12: extends: - .rspec-ee-base-pg12 @@ -684,6 +798,83 @@ rspec-ee system pg12 geo minimal: - .minimal-rspec-tests - .rails:rules:ee-only-system:minimal +rspec-ee migration pg12-as-if-jh: + extends: + - .rspec-jh-base-pg12 + - .rspec-base-migration + - .rails:rules:as-if-jh-rspec + - .rspec-ee-migration-parallel + +rspec-ee unit pg12-as-if-jh: + extends: + - .rspec-jh-base-pg12 + - .rails:rules:as-if-jh-rspec + - .rspec-ee-unit-parallel + +rspec-ee integration pg12-as-if-jh: + extends: + - .rspec-jh-base-pg12 + - .rails:rules:as-if-jh-rspec + - .rspec-ee-integration-parallel + +rspec-ee system pg12-as-if-jh: + extends: + - .rspec-jh-base-pg12 + - .rails:rules:as-if-jh-rspec + - .rspec-ee-system-parallel + +rspec-ee unit pg12-as-if-jh geo: + extends: + - .rspec-jh-base-geo-pg12 + - .rails:rules:as-if-jh-rspec + - .rspec-ee-unit-geo-parallel + +rspec-ee integration pg12-as-if-jh geo: + extends: + - .rspec-jh-base-geo-pg12 + - .rails:rules:as-if-jh-rspec + +rspec-ee system pg12-as-if-jh geo: + extends: + - .rspec-jh-base-geo-pg12 + - .rails:rules:as-if-jh-rspec + +rspec-jh migration pg12-as-if-jh: + extends: + - .rspec-jh-base-pg12 + - .rspec-base-migration + - .rails:rules:as-if-jh-rspec + +rspec-jh unit pg12-as-if-jh: + extends: + - .rspec-jh-base-pg12 + - .rails:rules:as-if-jh-rspec + +rspec-jh integration pg12-as-if-jh: + extends: + - .rspec-jh-base-pg12 + - .rails:rules:as-if-jh-rspec + +rspec-jh system pg12-as-if-jh: + extends: + - .rspec-jh-base-pg12 + - .rails:rules:as-if-jh-rspec + +rspec-jh unit pg12-as-if-jh geo: + extends: + - .rspec-jh-base-geo-pg12 + - .rails:rules:as-if-jh-rspec + +rspec-jh integration pg12-as-if-jh geo: + extends: + - .rspec-jh-base-geo-pg12 + - .rails:rules:as-if-jh-rspec + +rspec-jh system pg12-as-if-jh geo: + extends: + - .rspec-jh-base-geo-pg12 + - .rails:rules:as-if-jh-rspec + db:rollback geo: extends: - db:rollback @@ -884,5 +1075,24 @@ fail-pipeline-early: - install_gitlab_gem script: - fail_pipeline_early + +rspec rspec-pg12-rerun-previous-failed-tests: + extends: + - .rspec-base-pg12 + - .rails:rules:rerun-previous-failed-tests + stage: test + needs: ["setup-test-env", "compile-test-assets", "detect-previous-failed-tests"] + script: + - !reference [.base-script, script] + - rspec_rerun_previous_failed_tests tmp/previous_failed_tests/rspec_failed_files.txt + +rspec rspec-ee-pg12-rerun-previous-failed-tests: + extends: + - "rspec rspec-pg12-rerun-previous-failed-tests" + - .rspec-ee-base-pg12 + script: + - !reference [.base-script, script] + - rspec_rerun_previous_failed_tests tmp/previous_failed_tests/rspec_ee_failed_files.txt + # EE: Canonical MR pipelines ################################################## diff --git a/.gitlab/ci/reports.gitlab-ci.yml b/.gitlab/ci/reports.gitlab-ci.yml index b581cf83d5..33efc3c478 100644 --- a/.gitlab/ci/reports.gitlab-ci.yml +++ b/.gitlab/ci/reports.gitlab-ci.yml @@ -9,6 +9,7 @@ code_quality: extends: - .default-retry - .use-docker-in-docker + stage: lint artifacts: paths: - gl-code-quality-report.json # GitLab-specific @@ -20,6 +21,7 @@ code_quality: extends: - .default-retry - sast + stage: lint needs: [] artifacts: paths: @@ -54,6 +56,7 @@ gosec-sast: .secret-analyzer: extends: .default-retry + stage: lint needs: [] artifacts: paths: @@ -68,6 +71,7 @@ secret_detection: extends: - .default-retry - dependency_scanning + stage: lint needs: [] variables: DS_EXCLUDED_PATHS: "qa/qa/ee/fixtures/secure_premade_reports, spec, ee/spec, tmp" # GitLab-specific @@ -92,6 +96,13 @@ retire-js-dependency_scanning: gemnasium-python-dependency_scanning: rules: !reference [".reports:rules:gemnasium-python-dependency_scanning", rules] +yarn-audit-dependency_scanning: + extends: .ds-analyzer + image: "registry.gitlab.com/gitlab-org/security-products/analyzers/npm-audit:1.4.0" + variables: + TOOL: yarn + rules: !reference [".reports:rules:yarn-audit-dependency_scanning", rules] + # Analyze dependencies for malicious behavior # See https://gitlab.com/gitlab-com/gl-security/security-research/package-hunter .package_hunter-base: @@ -134,6 +145,7 @@ package_hunter-bundler: license_scanning: extends: .default-retry + stage: lint needs: [] artifacts: expire_in: 1 week # GitLab-specific diff --git a/.gitlab/ci/review-apps/main.gitlab-ci.yml b/.gitlab/ci/review-apps/main.gitlab-ci.yml index 6fe9e39cb8..670175a6f1 100644 --- a/.gitlab/ci/review-apps/main.gitlab-ci.yml +++ b/.gitlab/ci/review-apps/main.gitlab-ci.yml @@ -35,10 +35,11 @@ review-build-cng: extends: - .default-retry image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-helm3.5-kubectl1.17 + resource_group: "review/${CI_COMMIT_REF_NAME}" variables: HOST_SUFFIX: "${CI_ENVIRONMENT_SLUG}" DOMAIN: "-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN}" - GITLAB_HELM_CHART_REF: "v5.2.1" + GITLAB_HELM_CHART_REF: "v5.4.1" environment: name: review/${CI_COMMIT_REF_SLUG}${FREQUENCY} url: https://gitlab-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN} @@ -51,7 +52,6 @@ review-deploy: - .review:rules:review-deploy stage: deploy needs: ["review-build-cng"] - resource_group: "review/${CI_COMMIT_REF_NAME}" before_script: - export GITLAB_SHELL_VERSION=$( /label ~"Category:Audit Events" -/label ~"feature" +/label ~"type::feature" /label ~"group::compliance" diff --git a/.gitlab/issue_templates/Bug.md b/.gitlab/issue_templates/Bug.md index 41b694fdf2..b9fed3745d 100644 --- a/.gitlab/issue_templates/Bug.md +++ b/.gitlab/issue_templates/Bug.md @@ -2,10 +2,10 @@ Please read this! Before opening a new issue, make sure to search for keywords in the issues -filtered by the "regression" or "bug" label: +filtered by the "regression" or "type::bug" label: - https://gitlab.com/gitlab-org/gitlab/issues?label_name%5B%5D=regression -- https://gitlab.com/gitlab-org/gitlab/issues?label_name%5B%5D=bug +- https://gitlab.com/gitlab-org/gitlab/issues?label_name%5B%5D=type::bug and verify the issue you're about to submit isn't a duplicate. ---> @@ -82,4 +82,4 @@ will also determine whether the bug is fixed in a more recent version. --> -/label ~bug +/label ~"type::bug" diff --git a/.gitlab/issue_templates/Deprecations.md b/.gitlab/issue_templates/Deprecations.md index ff51699c6b..caef5c6433 100644 --- a/.gitlab/issue_templates/Deprecations.md +++ b/.gitlab/issue_templates/Deprecations.md @@ -41,7 +41,11 @@ Which tier is this feature available in? ### Deprecation Milestone - + + +### Planned Removal Milestone + + ### Links diff --git a/.gitlab/issue_templates/Empty state.md b/.gitlab/issue_templates/Empty state.md new file mode 100644 index 0000000000..d92ea8522e --- /dev/null +++ b/.gitlab/issue_templates/Empty state.md @@ -0,0 +1,80 @@ + + +## Description + + + +## Location + + + +## Use case + + + +- [ ] Blank content +- [ ] Empty search results +- [ ] Configuration required +- [ ] Higher tier + +## Checklist + + + +### Blank content + +- [ ] The solution follows the `Blank content` specifications [in Pajamas](https://design.gitlab.com/regions/empty-states#blank-content). +- [ ] Follow the instructions from the [`After merge` section](#after-merge) below to add Snowplow tracking. + +### Empty search results + +- [ ] The solution follows the `Empty search results` specifications [in Pajamas](https://design.gitlab.com/regions/empty-states#empty-search-results). +- [ ] Follow the instructions from the [`After merge` section](#after-merge) below to add Snowplow tracking. + +### Configuration required + +- [ ] The solution follows the `Configuration required` specifications [in Pajamas](https://design.gitlab.com/regions/empty-states#configuration-required). +- [ ] Ask a [Growth product manager or Designer](https://about.gitlab.com/handbook/engineering/development/growth/#stable-counterparts) to review your solution. +- [ ] Is your solution introducing a new empty states or modifying an existing one? + - [ ] Introducing a new empty state: Follow the instructions from the [`After merge` section](#after-merge) below to add Snowplow tracking. + - [ ] Modifying an existing empty state: Follow the [`Experimentation` process](#experimentation) below. _Note_: If the empty state you want to replace hasn't been updated in a long time, doesn't pitch the value of the feature, or does not contain a next step action CTA, then we recommend you skip the experimentation process to implement and add tracking to your new empty state. + + +#### Experimentation + +- [ ] Collaborate with a [Growth product manager](https://about.gitlab.com/handbook/engineering/development/growth/#stable-counterparts) to help you determine if you can validate your solution through an experiment on SaaS. +- [ ] If an experiment is possible, create an issue using the [experiment idea template](https://gitlab.com/gitlab-org/gitlab/-/issues/new?issuable_template=Experiment%20Idea) and follow the template intructions. Otherwise, follow the instructions from the [`After merge` section](#after-merge) below to add Snowplow tracking. +- [ ] Ask a [Growth product manager or Designer](https://about.gitlab.com/handbook/engineering/development/growth/#stable-counterparts) to review your experiment set-up. +- [ ] Implement and monitor the experiment following the [implementation guide](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/development/experiment_guide/gitlab_experiment.md#implement-an-experiment). +- [ ] Review and discuss the findings. +- [ ] Add the findings to the [Growth experimentation knowledge](https://about.gitlab.com/direction/growth/#growth-experiments-knowledge-base---concluded-experiments). + +### Higher tier + +- [ ] The solution follows the `Higher tier` specifications [in Pajamas](https://design.gitlab.com/regions/empty-states#higher-tier). +- [ ] Ask a Product Manager or Designer from the [Conversion group](https://about.gitlab.com/handbook/engineering/development/growth/conversion/#group-members) to review your solution. +- [ ] Is your solution introducing a new empty states or modifying an existing one? + - [ ] Introducing a new empty state: follow the instructions from the [`After merge` section](#after-merge) below to add Snowplow tracking. + - [ ] Modifying an existing empty state, follow the [`Experimentation` process](#experimentation) below. + + +#### Experimentation + +- [ ] Collaborate with a [Growth product manager](https://about.gitlab.com/handbook/engineering/development/growth/#stable-counterparts) to help you determine if you can validate your solution through an experiment on SaaS. +- [ ] If an experiment is possible, create an issue using the [experiment idea template](https://gitlab.com/gitlab-org/gitlab/-/issues/new?issuable_template=Experiment%20Idea) and follow the template intructions. Otherwise, follow the instructions from the [`After merge` section](#after-merge) below to add Snowplow tracking. +- [ ] Add a ~"Category:Conversion Experiment" label to the experiment idea issue. +- [ ] Ask a Product Manager or Designer from the [Conversion group](https://about.gitlab.com/handbook/engineering/development/growth/conversion/#group-members) to review your experiment set-up. +- [ ] Implement and monitor the experiment following the [implementation guide](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/development/experiment_guide/gitlab_experiment.md#implement-an-experiment) . +- [ ] Review and discuss the findings. +- [ ] Add the findings to the [Growth experimentation knowledge](https://about.gitlab.com/direction/growth/#growth-experiments-knowledge-base---concluded-experiments). + + +## After merge + +- [ ] Use the `Snowplow event tracking` [issue template](https://gitlab.com/gitlab-org/gitlab/-/issues/new?issuable_template=Snowplow%20event%20tracking) and open an issue to add Snowplow event tracking to your new empty state solution. + - [ ] Add your ~devops:: and ~group:: labels to the new issue. diff --git a/.gitlab/issue_templates/Experiment Implementation.md b/.gitlab/issue_templates/Experiment Implementation.md new file mode 100644 index 0000000000..b3883f08c2 --- /dev/null +++ b/.gitlab/issue_templates/Experiment Implementation.md @@ -0,0 +1,25 @@ + + +# Experiment Summary + + +# Design + + +# Rollout strategy + + +# Inclusions and exclusions + + +# Segmentation + + +# Tracking Details + +- [json schema](https://gitlab.com/gitlab-org/iglu/-/blob/master/public/schemas/com.gitlab/gitlab_experiment/jsonschema/0-3-0) used in `gitlab-experiment` tracking. +- see [taxonomy](https://docs.gitlab.com/ee/development/snowplow/index.html#structured-event-taxonomy) for a guide. + +| activity | category | action | label | context | property | value | +| -------- | -------- | ------ | ----- | ------- | -------- | ----- | +| | | | | json schema | | | diff --git a/.gitlab/issue_templates/Experiment Rollout.md b/.gitlab/issue_templates/Experiment Rollout.md index c5fdc73994..9209423ba3 100644 --- a/.gitlab/issue_templates/Experiment Rollout.md +++ b/.gitlab/issue_templates/Experiment Rollout.md @@ -105,5 +105,16 @@ In this rollout issue, ensure the scoped `experiment::` label is kept accurate. /chatops run feature set false ``` +## Experiment Successful Cleanup Concerns + +_Items to be considered if candidate experience is to become a permanent part of GitLab_ + + + /label ~"feature flag" ~"devops::growth" ~"growth experiment" ~"experiment-rollout" ~Engineering ~"workflow::scheduling" ~"experiment::pending" /milestone %"Next 1-3 releases" diff --git a/.gitlab/issue_templates/Experiment Successful Cleanup.md b/.gitlab/issue_templates/Experiment Successful Cleanup.md index 42f2634234..1dd57332b8 100644 --- a/.gitlab/issue_templates/Experiment Successful Cleanup.md +++ b/.gitlab/issue_templates/Experiment Successful Cleanup.md @@ -12,9 +12,10 @@ The changes need to become an official part of the product. - [ ] Determine if tracking should be kept as is, removed, or modified. - [ ] Ensure any relevant documentation has been updated. - [ ] Consider changes to any `feature_category:` introduced by the experiment if ownership is changing (PM for Growth and PM for the new category as DRIs) +- [ ] Check to see if the experiment introduced new design assets. Add them to the appropriate repos and document them if needed. - [ ] Optional: Migrate experiment to a default enabled [feature flag](https://docs.gitlab.com/ee/development/feature_flags) for one milestone and add a changelog. Converting to a feature flag can be skipped at the ICs discretion if risk is deemed low with consideration to both SaaS and (if applicable) self managed - [ ] In the next milestone, [remove the feature flag](https://docs.gitlab.com/ee/development/feature_flags/controls.html#cleaning-up) if applicable - [ ] After the flag removal is deployed, [clean up the feature/experiment feature flags](https://docs.gitlab.com/ee/development/feature_flags/controls.html#cleaning-up) by running chatops command in `#production` channel -- [ ] Ensure the corresponding [Experiment Tracking](https://gitlab.com/groups/gitlab-org/-/boards/1352542?label_name[]=devops%3A%3Agrowth&label_name[]=growth%20experiment&label_name[]=experiment%20tracking) issue is updated +- [ ] Ensure the corresponding [Experiment Rollout](https://gitlab.com/groups/gitlab-org/-/boards/1352542?label_name[]=devops%3A%3Agrowth&label_name[]=growth%20experiment&label_name[]=experiment-rollout) issue is updated -/label ~"feature" ~"feature::maintenance" ~"workflow::scheduling" ~"growth experiment" ~"feature flag" +/label ~"type::maintenance" ~"workflow::scheduling" ~"growth experiment" ~"feature flag" diff --git a/.gitlab/issue_templates/Experimentation.md b/.gitlab/issue_templates/Experimentation.md deleted file mode 100644 index ba7839fb94..0000000000 --- a/.gitlab/issue_templates/Experimentation.md +++ /dev/null @@ -1,25 +0,0 @@ - - -# Experiment Summary - - -# Design - - -# Rollout strategy - - -# Inclusions and exclusions - - -# Segmentation - - -# Tracking Details - -- [json schema](https://gitlab.com/gitlab-org/iglu/-/blob/master/public/schemas/com.gitlab/gitlab_experiment/jsonschema/0-3-0) used in `gitlab-experiment` tracking. -- see [taxonomy](https://docs.gitlab.com/ee/development/snowplow/index.html#structured-event-taxonomy) for a guide. - -| activity | category | action | label | context | property | value | -| -------- | -------- | ------ | ----- | ------- | -------- | ----- | -| | | | | json schema | | | diff --git a/.gitlab/issue_templates/Feature Flag Roll Out.md b/.gitlab/issue_templates/Feature Flag Roll Out.md index 00b396bac4..bc1a23729e 100644 --- a/.gitlab/issue_templates/Feature Flag Roll Out.md +++ b/.gitlab/issue_templates/Feature Flag Roll Out.md @@ -30,6 +30,17 @@ Are there any other stages or teams involved that need to be kept in the loop? +### When is the feature viable? + + + + + ### What might happen if this goes wrong? @@ -37,6 +48,12 @@ Are there any other stages or teams involved that need to be kept in the loop? ### What can we monitor to detect problems with this? +_Consider mentioning checks for 5xx errors or other anomalies like an increase in redirects +(302 HTTP response status)_ + +### What can we check for monitoring production after rollouts? + +_Consider adding links to check for Sentry errors, Production logs for 5xx, 302s, etc._ ## Rollout Steps @@ -73,11 +90,14 @@ Are there any other stages or teams involved that need to be kept in the loop? If a different developer will be covering, or an exception is needed, please inform the oncall SRE by using the `@sre-oncall` Slack alias. - [ ] Ensure that documentation has been updated ([More info](https://docs.gitlab.com/ee/development/documentation/feature_flags.html#features-that-became-enabled-by-default)). - [ ] Announce on [the feature issue](ISSUE LINK) an estimated time this will be enabled on GitLab.com. -- [ ] If the feature might impact the user experience, notify `#support_gitlab-com` and your team channel ([more guidance when this is necessary in the dev docs](https://docs.gitlab.com/ee/development/feature_flags/controls.html#communicate-the-change)). +- [ ] Notify `#support_gitlab-com` and your team channel ([more guidance when this is necessary in the dev docs](https://docs.gitlab.com/ee/development/feature_flags/controls.html#communicate-the-change)). ### Global rollout on production -All `/chatops` commands that target production should be done in the `#production` slack channel for visibility. +For visibility, all `/chatops` commands that target production should be: + +- Executed in the `#production` slack channel. +- Cross-posted (with the command results) to the responsible team's slack channel (`#g_TEAM_NAME`). - [ ] [Incrementally roll out](https://docs.gitlab.com/ee/development/feature_flags/controls.html#process) the feature. - If the feature flag in code has [an actor](https://docs.gitlab.com/ee/development/feature_flags/#feature-actors), perform **actor-based** rollout. @@ -148,5 +168,5 @@ codebase. /chatops run feature set false ``` -/label ~"feature flag" +/label ~"feature flag" ~"type::feature" ~"feature::addition" /assign DRI diff --git a/.gitlab/issue_templates/Feature Proposal - basic.md b/.gitlab/issue_templates/Feature Proposal - basic.md index 0c05b7a016..980751621f 100644 --- a/.gitlab/issue_templates/Feature Proposal - basic.md +++ b/.gitlab/issue_templates/Feature Proposal - basic.md @@ -1,10 +1,16 @@ -### Proposal +### Proposal -/label ~feature::addition ~"group::" ~"section::" ~"Category:" ~"GitLab Core"/~"GitLab Premium"/~"GitLab Ultimate" + + +/label ~"type::feature" ~feature::addition ~"group::" ~"section::" ~"Category:" ~"GitLab Core"/~"GitLab Premium"/~"GitLab Ultimate" diff --git a/.gitlab/issue_templates/Feature Proposal - lean.md b/.gitlab/issue_templates/Feature Proposal - lean.md index 9dd4bdc6b2..504bfbb03d 100644 --- a/.gitlab/issue_templates/Feature Proposal - lean.md +++ b/.gitlab/issue_templates/Feature Proposal - lean.md @@ -1,33 +1,22 @@ - + + +### Release notes -### Problem to solve +### Problem to solve -### Proposal +### Proposal - - -/label ~"feature" ~"group::" ~"section::" ~"Category::" ~"GitLab Free"/~"GitLab Premium"/~"GitLab Ultimate" - - - -### User experience goal +### Metrics -What is the single user experience workflow this problem addresses? -For example, "The user should be able to use the UI/API/.gitlab-ci.yml with GitLab to " -https://about.gitlab.com/handbook/engineering/ux/ux-research-training/user-story-mapping/ + -Include use cases, benefits, goals, or any other details that will help us understand the problem better. + -### Permissions and Security - - - -/label ~documentation -/label ~direction +/label ~"type::feature" ~"group::" ~"section::" ~"Category::" ~"GitLab Free"/~"GitLab Premium"/~"GitLab Ultimate" ~documentation ~direction diff --git a/.gitlab/issue_templates/Feature proposal - detailed.md b/.gitlab/issue_templates/Feature proposal - detailed.md index 9759bb7e2d..c787fc9933 100644 --- a/.gitlab/issue_templates/Feature proposal - detailed.md +++ b/.gitlab/issue_templates/Feature proposal - detailed.md @@ -31,6 +31,14 @@ Personas are described at https://about.gitlab.com/handbook/marketing/product-ma * [Eddie (Content Editor)](https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/#eddie-content-editor) --> +### Metrics + + + ### User experience goal /label ~devops:: ~group: ~Category: /label ~"GitLab Free"/~"GitLab Premium"/~"GitLab Ultimate" -/label ~feature -/label ~documentation -/label ~direction +/label ~"type::feature" ~documentation ~direction diff --git a/.gitlab/issue_templates/Geo Replicate a new Git repository type.md b/.gitlab/issue_templates/Geo Replicate a new Git repository type.md index ee4cb27053..71a962d178 100644 --- a/.gitlab/issue_templates/Geo Replicate a new Git repository type.md +++ b/.gitlab/issue_templates/Geo Replicate a new Git repository type.md @@ -111,115 +111,9 @@ Geo secondary sites have a [Geo tracking database](https://gitlab.com/gitlab-org - [ ] Be sure to commit the relevant changes in `ee/db/geo/structure.sql` -### Add verification state fields on the Geo primary site +### Add verification state to the Model -The Geo primary site needs to checksum every replicable in order for secondaries to verify their own checksums. To do this, Geo requires fields on the Model. There are two ways to add the necessary verification state fields. If the table is large and wide, then it may be a good idea to add verification state fields to a separate table (Option 2). Consult a database expert if needed. - -#### Add verification state fields to the model table (Option 1) - -- [ ] Create the migration file in `db/migrate`: - - ```shell - bin/rails generate migration AddVerificationStateToCoolWidgets - ``` - -- [ ] Replace the contents of the migration file with: - - ```ruby - # frozen_string_literal: true - - class AddVerificationStateToCoolWidgets < ActiveRecord::Migration[6.0] - def change - change_table(:cool_widgets) do |t| - t.integer :verification_state, default: 0, limit: 2, null: false - t.column :verification_started_at, :datetime_with_timezone - t.integer :verification_retry_count, limit: 2, null: false - t.column :verification_retry_at, :datetime_with_timezone - t.column :verified_at, :datetime_with_timezone - t.binary :verification_checksum, using: 'verification_checksum::bytea' - - t.text :verification_failure # rubocop:disable Migration/AddLimitToTextColumns - end - end - end - ``` - -- [ ] If deviating from the above example, then be sure to order columns according to [our guidelines](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/development/ordering_table_columns.md). -- [ ] If `cool_widgets` is a high-traffic table, follow [the database documentation to use `with_lock_retries`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/development/migration_style_guide.md#when-to-use-the-helper-method) -- [ ] Adding a `text` column also [requires](../database/strings_and_the_text_data_type.md#add-a-text-column-to-an-existing-table) setting a limit. Create the migration file in `db/migrate`: - - ```shell - bin/rails generate migration AddVerificationFailureLimitToCoolWidgets - ``` - -- [ ] Replace the contents of the migration file with: - - ```ruby - # frozen_string_literal: true - - class AddVerificationFailureLimitToCoolWidgets < ActiveRecord::Migration[6.0] - include Gitlab::Database::MigrationHelpers - - disable_ddl_transaction! - - CONSTRAINT_NAME = 'cool_widget_verification_failure_text_limit' - - def up - add_text_limit :cool_widget, :verification_failure, 255, constraint_name: CONSTRAINT_NAME - end - - def down - remove_check_constraint(:cool_widget, CONSTRAINT_NAME) - end - end - ``` - -- [ ] Add indexes on verification fields to ensure verification can be performed efficiently. Some or all of these indexes can be omitted if the table is guaranteed to be small. Ask a database expert if you are considering omitting indexes. Create the migration file in `db/migrate`: - - ```shell - bin/rails generate migration AddVerificationIndexesToCoolWidgets - ``` - -- [ ] Replace the contents of the migration file with: - - ```ruby - # frozen_string_literal: true - - class AddVerificationIndexesToCoolWidgets < ActiveRecord::Migration[6.0] - include Gitlab::Database::MigrationHelpers - - VERIFICATION_STATE_INDEX_NAME = "index_cool_widgets_on_verification_state" - PENDING_VERIFICATION_INDEX_NAME = "index_cool_widgets_pending_verification" - FAILED_VERIFICATION_INDEX_NAME = "index_cool_widgets_failed_verification" - NEEDS_VERIFICATION_INDEX_NAME = "index_cool_widgets_needs_verification" - - disable_ddl_transaction! - - def up - add_concurrent_index :cool_widgets, :verification_state, name: VERIFICATION_STATE_INDEX_NAME - add_concurrent_index :cool_widgets, :verified_at, where: "(verification_state = 0)", order: { verified_at: 'ASC NULLS FIRST' }, name: PENDING_VERIFICATION_INDEX_NAME - add_concurrent_index :cool_widgets, :verification_retry_at, where: "(verification_state = 3)", order: { verification_retry_at: 'ASC NULLS FIRST' }, name: FAILED_VERIFICATION_INDEX_NAME - add_concurrent_index :cool_widgets, :verification_state, where: "(verification_state = 0 OR verification_state = 3)", name: NEEDS_VERIFICATION_INDEX_NAME - end - - def down - remove_concurrent_index_by_name :cool_widgets, VERIFICATION_STATE_INDEX_NAME - remove_concurrent_index_by_name :cool_widgets, PENDING_VERIFICATION_INDEX_NAME - remove_concurrent_index_by_name :cool_widgets, FAILED_VERIFICATION_INDEX_NAME - remove_concurrent_index_by_name :cool_widgets, NEEDS_VERIFICATION_INDEX_NAME - end - end - ``` - -- [ ] Run database migrations: - - ```shell - bin/rake db:migrate - ``` - -- [ ] Be sure to commit the relevant changes in `db/structure.sql` - -#### Add verification state fields to a separate table (Option 2) +The Geo primary site needs to checksum every replicable so secondaries can verify their own checksums. To do this, Geo requires the Model to have an associated table to track verification state. - [ ] Create the migration file in `db/migrate`: @@ -273,6 +167,7 @@ The Geo primary site needs to checksum every replicable in order for secondaries ``` - [ ] If deviating from the above example, then be sure to order columns according to [our guidelines](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/development/ordering_table_columns.md). + - [ ] Run database migrations: ```shell @@ -287,7 +182,14 @@ That's all of the required database changes. #### Step 1. Implement replication and verification -- [ ] Include `Gitlab::Geo::ReplicableModel` in the `CoolWidget` class, and specify the Replicator class `with_replicator Geo::CoolWidgetReplicator`. +- [ ] Add the following lines to the `cool_widget` model to accomplish some important tasks: + - Include `Gitlab::Geo::ReplicableModel` in the `CoolWidget` class, and specify the Replicator class `with_replicator Geo::CoolWidgetReplicator`. + - Include the `::Gitlab::Geo::VerificationState` concern. + - Delegate verification related methods to the `cool_widget_state` model. + - For verification, override some scopes to use the `cool_widget_states` table instead of the model table. + - Implement the `verification_state_object` method to return the object that holds + the verification details + - Override some methods to use the `cool_widget_states` table in verification-related queries. Pay some attention to method `pool_repository`. Not every repository type uses repository pooling. As Geo prefers to use repository snapshotting, it can lead to data loss. Make sure to overwrite `pool_repository` so it returns nil for repositories that do not have pools. @@ -297,6 +199,7 @@ That's all of the required database changes. # frozen_string_literal: true class CoolWidget < ApplicationRecord + ... include ::Gitlab::Geo::ReplicableModel include ::Gitlab::Geo::VerificationState @@ -304,31 +207,62 @@ That's all of the required database changes. mount_uploader :file, CoolWidgetUploader + has_one :cool_widget_state, autosave: false, inverse_of: :cool_widget, class_name: 'Geo::CoolWidgetState' + + delegate :verification_retry_at, :verification_retry_at=, + :verified_at, :verified_at=, + :verification_checksum, :verification_checksum=, + :verification_failure, :verification_failure=, + :verification_retry_count, :verification_retry_count=, + :verification_state=, :verification_state, + :verification_started_at=, :verification_started_at, + to: :cool_widget_state + ... + + scope :with_verification_state, ->(state) { joins(:cool_widget_state).where(cool_widget_states: { verification_state: verification_state_value(state) }) } + scope :checksummed, -> { joins(:cool_widget_state).where.not(cool_widget_states: { verification_checksum: nil } ) } + scope :not_checksummed, -> { joins(:cool_widget_state).where(cool_widget_states: { verification_checksum: nil } ) } + # Override the `all` default if not all records can be replicated. For an # example of an existing Model that needs to do this, see # `EE::MergeRequestDiff`. # scope :available_replicables, -> { all } - # @param primary_key_in [Range, CoolWidget] arg to pass to primary_key_in scope - # @return [ActiveRecord::Relation] everything that should be synced to this node, restricted by primary key - def self.replicables_for_current_secondary(primary_key_in) - # This issue template does not help you write this method. - # - # This method is called only on Geo secondary sites. It is called when - # we want to know which records to replicate. This is not easy to automate - # because for example: - # - # * The "selective sync" feature allows admins to choose which namespaces # to replicate, per secondary site. Most Models are scoped to a - # namespace, but the nature of the relationship to a namespace varies - # between Models. - # * The "selective sync" feature allows admins to choose which shards to - # replicate, per secondary site. Repositories are associated with - # shards. Most blob types are not, but Project Uploads are. - # * Remote stored replicables are not replicated, by default. But the - # setting `sync_object_storage` enables replication of remote stored - # replicables. - # - # Search the codebase for examples, and consult a Geo expert if needed. + def verification_state_object + cool_widget_state + end + ... + + class_methods do + extend ::Gitlab::Utils::Override + ... + + # @param primary_key_in [Range, CoolWidget] arg to pass to primary_key_in scope + # @return [ActiveRecord::Relation] everything that should be synced to this node, restricted by primary key + def replicables_for_current_secondary(primary_key_in) + # This issue template does not help you write this method. + # + # This method is called only on Geo secondary sites. It is called when + # we want to know which records to replicate. This is not easy to automate + # because for example: + # + # * The "selective sync" feature allows admins to choose which namespaces # to replicate, per secondary site. Most Models are scoped to a + # namespace, but the nature of the relationship to a namespace varies + # between Models. + # * The "selective sync" feature allows admins to choose which shards to + # replicate, per secondary site. Repositories are associated with + # shards. Most blob types are not, but Project Uploads are. + # * Remote stored replicables are not replicated, by default. But the + # setting `sync_object_storage` enables replication of remote stored + # replicables. + # + # Search the codebase for examples, and consult a Geo expert if needed. + end + + override :verification_state_table_class + def verification_state_table_class + CoolWidgetState + end end # Geo checks this method in FrameworkRepositorySyncService to avoid @@ -336,6 +270,11 @@ That's all of the required database changes. def pool_repository nil end + ... + + def cool_widget_state + super || build_cool_widget_state + end ... end @@ -343,6 +282,15 @@ That's all of the required database changes. - [ ] Implement `CoolWidget.replicables_for_current_secondary` above. - [ ] Ensure `CoolWidget.replicables_for_current_secondary` is well-tested. Search the codebase for `replicables_for_current_secondary` to find examples of parameterized table specs. You may need to add more `FactoryBot` traits. +- [ ] Add the following shared examples to `ee/spec/models/ee/cool_widget_spec.rb`: + + ```ruby + include_examples 'a replicable model with a separate table for verification state' do + let(:verifiable_model_record) { build(:cool_widget) } # add extra params if needed to make sure the record is included in `available_verifiables` + let(:unverifiable_model_record) { build(:cool_widget) } # add extra params if needed to make sure the record is NOT included in `available_verifiables` + end + ``` + - [ ] Create `ee/app/replicators/geo/cool_widget_replicator.rb`. Implement the `#repository` method which should return a `` instance, and implement the class method `.model` to return the `CoolWidget` class: ```ruby @@ -536,13 +484,7 @@ That's all of the required database changes. - [ ] Make sure the factory also allows setting a `project` attribute. If the model does not have a direct relation to a project, you can use a `transient` attribute. Check out `spec/factories/merge_request_diffs.rb` for an example. -##### If you added verification state fields to a separate table (option 2 above), then you need to make additional model and factory changes - -If you did not add verification state fields to a separate table, `cool_widget_states`, then skip to [Step 2. Implement metrics gathering](#step-2-implement-metrics-gathering). - -Otherwise, you can follow [the example of Merge Request Diffs](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/63309). - -- [ ] Add a `Geo::CoolWidgetState` model in `ee/app/models/geo/cool_widget_state.rb`: +- [ ] Following [the example of Merge Request Diffs](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/63309) add a `Geo::CoolWidgetState` model in `ee/app/models/ee/geo/cool_widget_state.rb`: ``` ruby # frozen_string_literal: true @@ -576,63 +518,6 @@ Otherwise, you can follow [the example of Merge Request Diffs](https://gitlab.co end ``` -- [ ] Add the following lines to the `cool_widget` model to accomplish some important tasks: - - Include the `::Gitlab::Geo::VerificationState` concern. - - Delegate verification related methods to the `cool_widget_state` model. - - Override some scopes to use the `cool_widget_states` table instead of the model table, for verification. - - Override some methods to use the `cool_widget_states` table in verification related queries. - - ```ruby - class CoolWidget < ApplicationRecord - ... - include ::Gitlab::Geo::VerificationState - - has_one :cool_widget_state, autosave: true, inverse_of: :cool_widget, class_name: 'Geo::CoolWidgetState' - - delegate :verification_retry_at, :verification_retry_at=, - :verified_at, :verified_at=, - :verification_checksum, :verification_checksum=, - :verification_failure, :verification_failure=, - :verification_retry_count, :verification_retry_count=, - :verification_state=, :verification_state, - :verification_started_at=, :verification_started_at, - to: :cool_widget_state - ... - - scope :with_verification_state, ->(state) { joins(:cool_widget_state).where(cool_widget_states: { verification_state: verification_state_value(state) }) } - scope :checksummed, -> { joins(:cool_widget_state).where.not(cool_widget_states: { verification_checksum: nil } ) } - scope :not_checksummed, -> { joins(:cool_widget_state).where(cool_widget_states: { verification_checksum: nil } ) } - - ... - - class_methods do - extend ::Gitlab::Utils::Override - ... - override :verification_state_table_name - def verification_state_table_name - 'cool_widget_states' - end - - override :verification_state_model_key - def verification_state_model_key - 'cool_widget_id' - end - - override :verification_arel_table - def verification_arel_table - CoolWidgetState.arel_table - end - end - ... - - def cool_widget_state - super || build_cool_widget_state - end - - ... - end - ``` - #### Step 2. Implement metrics gathering Metrics are gathered by `Geo::MetricsUpdateWorker`, persisted in `GeoNodeStatus` for display in the UI, and sent to Prometheus: diff --git a/.gitlab/issue_templates/Geo Replicate a new blob type.md b/.gitlab/issue_templates/Geo Replicate a new blob type.md index 00a71fa406..7c927e79e9 100644 --- a/.gitlab/issue_templates/Geo Replicate a new blob type.md +++ b/.gitlab/issue_templates/Geo Replicate a new blob type.md @@ -37,6 +37,7 @@ It is also a good idea to first open a proof-of-concept merge request. It can be You can look into the following examples of MRs for implementing replication/verification for a new blob type: - [Add db changes](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/60935) and [add verification for MR diffs using SSF](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/63309) - [Verify Terraform state versions](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/58800) +- [Verify LFS objects](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/63981) ### Modify database schemas to prepare to add Geo support for Cool Widgets @@ -114,113 +115,9 @@ Geo secondary sites have a [Geo tracking database](https://gitlab.com/gitlab-org ### Add verification state fields on the Geo primary site -The Geo primary site needs to checksum every replicable in order for secondaries to verify their own checksums. To do this, Geo requires fields on the Model. There are two ways to add the necessary verification state fields. If the table is large and wide, then it may be a good idea to add verification state fields to a separate table (Option 2). Consult a database expert if needed. +The Geo primary site needs to checksum every replicable so secondaries can verify their own checksums. To do this, Geo requires fields on the Model. Add verification state fields to a separate table. Consult a database expert if needed. -#### Add verification state fields to the model table (Option 1) - -- [ ] Create the migration file in `db/migrate`: - - ```shell - bin/rails generate migration AddVerificationStateToCoolWidgets - ``` - -- [ ] Replace the contents of the migration file with: - - ```ruby - # frozen_string_literal: true - - class AddVerificationStateToCoolWidgets < ActiveRecord::Migration[6.0] - def change - change_table(:cool_widgets) do |t| - t.integer :verification_state, default: 0, limit: 2, null: false - t.column :verification_started_at, :datetime_with_timezone - t.integer :verification_retry_count, limit: 2, null: false - t.column :verification_retry_at, :datetime_with_timezone - t.column :verified_at, :datetime_with_timezone - t.binary :verification_checksum, using: 'verification_checksum::bytea' - - t.text :verification_failure # rubocop:disable Migration/AddLimitToTextColumns - end - end - end - ``` - -- [ ] If deviating from the above example, then be sure to order columns according to [our guidelines](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/development/ordering_table_columns.md). -- [ ] If `cool_widgets` is a high-traffic table, follow [the database documentation to use `with_lock_retries`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/development/migration_style_guide.md#when-to-use-the-helper-method) -- [ ] Adding a `text` column also [requires](../database/strings_and_the_text_data_type.md#add-a-text-column-to-an-existing-table) setting a limit. Create the migration file in `db/migrate`: - - ```shell - bin/rails generate migration AddVerificationFailureLimitToCoolWidgets - ``` - -- [ ] Replace the contents of the migration file with: - - ```ruby - # frozen_string_literal: true - - class AddVerificationFailureLimitToCoolWidgets < ActiveRecord::Migration[6.0] - include Gitlab::Database::MigrationHelpers - - disable_ddl_transaction! - - CONSTRAINT_NAME = 'cool_widget_verification_failure_text_limit' - - def up - add_text_limit :cool_widget, :verification_failure, 255, constraint_name: CONSTRAINT_NAME - end - - def down - remove_check_constraint(:cool_widget, CONSTRAINT_NAME) - end - end - ``` - -- [ ] Add indexes on verification fields to ensure verification can be performed efficiently. Some or all of these indexes can be omitted if the table is guaranteed to be small. Ask a database expert if you are considering omitting indexes. Create the migration file in `db/migrate`: - - ```shell - bin/rails generate migration AddVerificationIndexesToCoolWidgets - ``` - -- [ ] Replace the contents of the migration file with: - - ```ruby - # frozen_string_literal: true - - class AddVerificationIndexesToCoolWidgets < ActiveRecord::Migration[6.0] - include Gitlab::Database::MigrationHelpers - - VERIFICATION_STATE_INDEX_NAME = "index_cool_widgets_on_verification_state" - PENDING_VERIFICATION_INDEX_NAME = "index_cool_widgets_pending_verification" - FAILED_VERIFICATION_INDEX_NAME = "index_cool_widgets_failed_verification" - NEEDS_VERIFICATION_INDEX_NAME = "index_cool_widgets_needs_verification" - - disable_ddl_transaction! - - def up - add_concurrent_index :cool_widgets, :verification_state, name: VERIFICATION_STATE_INDEX_NAME - add_concurrent_index :cool_widgets, :verified_at, where: "(verification_state = 0)", order: { verified_at: 'ASC NULLS FIRST' }, name: PENDING_VERIFICATION_INDEX_NAME - add_concurrent_index :cool_widgets, :verification_retry_at, where: "(verification_state = 3)", order: { verification_retry_at: 'ASC NULLS FIRST' }, name: FAILED_VERIFICATION_INDEX_NAME - add_concurrent_index :cool_widgets, :verification_state, where: "(verification_state = 0 OR verification_state = 3)", name: NEEDS_VERIFICATION_INDEX_NAME - end - - def down - remove_concurrent_index_by_name :cool_widgets, VERIFICATION_STATE_INDEX_NAME - remove_concurrent_index_by_name :cool_widgets, PENDING_VERIFICATION_INDEX_NAME - remove_concurrent_index_by_name :cool_widgets, FAILED_VERIFICATION_INDEX_NAME - remove_concurrent_index_by_name :cool_widgets, NEEDS_VERIFICATION_INDEX_NAME - end - end - ``` - -- [ ] Run database migrations: - - ```shell - bin/rake db:migrate - ``` - -- [ ] Be sure to commit the relevant changes in `db/structure.sql` - -#### Add verification state fields to a separate table (Option 2) +#### Add verification state fields to a new table - [ ] Create the migration file in `db/migrate`: @@ -274,12 +171,15 @@ The Geo primary site needs to checksum every replicable in order for secondaries ``` - [ ] If deviating from the above example, then be sure to order columns according to [our guidelines](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/development/ordering_table_columns.md). + - [ ] Run database migrations: ```shell bin/rake db:migrate ``` +- [ ] If `cool_widgets` is a high-traffic table, follow [the database documentation to use `with_lock_retries`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/development/migration_style_guide.md#when-to-use-the-helper-method) + - [ ] Be sure to commit the relevant changes in `db/structure.sql` That's all of the required database changes. @@ -288,14 +188,22 @@ That's all of the required database changes. #### Step 1. Implement replication and verification -- [ ] Include `Gitlab::Geo::ReplicableModel` in the `CoolWidget` class, and specify the Replicator class `with_replicator Geo::CoolWidgetReplicator`. +- [ ] Add the following lines to the `cool_widget` model to accomplish some important tasks: + - Include `Gitlab::Geo::ReplicableModel` in the `CoolWidget` class, and specify the Replicator class `with_replicator Geo::CoolWidgetReplicator`. + - Include the `::Gitlab::Geo::VerificationState` concern. + - Delegate verification related methods to the `cool_widget_state` model. + - For verification, override some scopes to use the `cool_widget_states` table instead of the model table. + - Implement the `verification_state_object` method to return the object that holds + the verification details + - Override some methods to use the `cool_widget_states` table in verification-related queries. At this point the `CoolWidget` class should look like this: ```ruby # frozen_string_literal: true - + class CoolWidget < ApplicationRecord + ... include ::Gitlab::Geo::ReplicableModel include ::Gitlab::Geo::VerificationState @@ -303,38 +211,84 @@ That's all of the required database changes. mount_uploader :file, CoolWidgetUploader + has_one :cool_widget_state, autosave: false, inverse_of: :cool_widget, class_name: 'Geo::CoolWidgetState' + + delegate :verification_retry_at, :verification_retry_at=, + :verified_at, :verified_at=, + :verification_checksum, :verification_checksum=, + :verification_failure, :verification_failure=, + :verification_retry_count, :verification_retry_count=, + :verification_state=, :verification_state, + :verification_started_at=, :verification_started_at, + to: :cool_widget_state + ... + + scope :with_verification_state, ->(state) { joins(:cool_widget_state).where(cool_widget_states: { verification_state: verification_state_value(state) }) } + scope :checksummed, -> { joins(:cool_widget_state).where.not(cool_widget_states: { verification_checksum: nil } ) } + scope :not_checksummed, -> { joins(:cool_widget_state).where(cool_widget_states: { verification_checksum: nil } ) } + # Override the `all` default if not all records can be replicated. For an # example of an existing Model that needs to do this, see # `EE::MergeRequestDiff`. # scope :available_replicables, -> { all } - # @param primary_key_in [Range, CoolWidget] arg to pass to primary_key_in scope - # @return [ActiveRecord::Relation] everything that should be synced to this node, restricted by primary key - def self.replicables_for_current_secondary(primary_key_in) - # This issue template does not help you write this method. - # - # This method is called only on Geo secondary sites. It is called when - # we want to know which records to replicate. This is not easy to automate - # because for example: - # - # * The "selective sync" feature allows admins to choose which namespaces # to replicate, per secondary site. Most Models are scoped to a - # namespace, but the nature of the relationship to a namespace varies - # between Models. - # * The "selective sync" feature allows admins to choose which shards to - # replicate, per secondary site. Repositories are associated with - # shards. Most blob types are not, but Project Uploads are. - # * Remote stored replicables are not replicated, by default. But the - # setting `sync_object_storage` enables replication of remote stored - # replicables. - # - # Search the codebase for examples, and consult a Geo expert if needed. + def verification_state_object + cool_widget_state end ... + + class_methods do + extend ::Gitlab::Utils::Override + ... + + # @param primary_key_in [Range, CoolWidget] arg to pass to primary_key_in scope + # @return [ActiveRecord::Relation] everything that should be synced to this node, restricted by primary key + def self.replicables_for_current_secondary(primary_key_in) + # This issue template does not help you write this method. + # + # This method is called only on Geo secondary sites. It is called when + # we want to know which records to replicate. This is not easy to automate + # because for example: + # + # * The "selective sync" feature allows admins to choose which namespaces # to replicate, per secondary site. Most Models are scoped to a + # namespace, but the nature of the relationship to a namespace varies + # between Models. + # * The "selective sync" feature allows admins to choose which shards to + # replicate, per secondary site. Repositories are associated with + # shards. Most blob types are not, but Project Uploads are. + # * Remote stored replicables are not replicated, by default. But the + # setting `sync_object_storage` enables replication of remote stored + # replicables. + # + # Search the codebase for examples, and consult a Geo expert if needed. + end + + override :verification_state_table_class + def verification_state_table_class + CoolWidgetState + end + end + ... + + def cool_widget_state + super || build_cool_widget_state + end + + ... end ``` - [ ] Implement `CoolWidget.replicables_for_current_secondary` above. - [ ] Ensure `CoolWidget.replicables_for_current_secondary` is well-tested. Search the codebase for `replicables_for_current_secondary` to find examples of parameterized table specs. You may need to add more `FactoryBot` traits. +- [ ] Add the following shared examples to `ee/spec/models/ee/cool_widget_spec.rb`: + + ```ruby + include_examples 'a replicable model with a separate table for verification state' do + let(:verifiable_model_record) { build(:cool_widget) } # add extra params if needed to make sure the record is included in `available_verifiables` + let(:unverifiable_model_record) { build(:cool_widget) } # add extra params if needed to make sure the record is NOT included in `available_verifiables` + end + ``` + - [ ] Create `ee/app/replicators/geo/cool_widget_replicator.rb`. Implement the `#carrierwave_uploader` method which should return a `CarrierWave::Uploader`, and implement the class method `.model` to return the `CoolWidget` class: ```ruby @@ -498,13 +452,7 @@ That's all of the required database changes. - [ ] Make sure the factory also allows setting a `project` attribute. If the model does not have a direct relation to a project, you can use a `transient` attribute. Check out `spec/factories/merge_request_diffs.rb` for an example. -##### If you added verification state fields to a separate table (option 2 above), then you need to make additional model and factory changes - -If you did not add verification state fields to a separate table, `cool_widget_states`, then skip to [Step 2. Implement metrics gathering](#step-2-implement-metrics-gathering). - -Otherwise, you can follow [the example of Merge Request Diffs](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/63309). - -- [ ] Add a `Geo::CoolWidgetState` model in `ee/app/models/ee/geo/cool_widget_state.rb`: +- [ ] Following [the example of Merge Request Diffs](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/63309) add a `Geo::CoolWidgetState` model in `ee/app/models/ee/geo/cool_widget_state.rb`: ``` ruby module Geo @@ -536,63 +484,6 @@ Otherwise, you can follow [the example of Merge Request Diffs](https://gitlab.co end ``` -- [ ] Add the following lines to the `cool_widget` model to accomplish some important tasks: - - Include the `::Gitlab::Geo::VerificationState` concern. - - Delegate verification related methods to the `cool_widget_state` model. - - Override some scopes to use the `cool_widget_states` table instead of the model table, for verification. - - Override some methods to use the `cool_widget_states` table in verification related queries. - - ```ruby - class CoolWidget < ApplicationRecord - ... - include ::Gitlab::Geo::VerificationState - - has_one :cool_widget_state, autosave: true, inverse_of: :cool_widget, class_name: 'Geo::CoolWidgetState' - - delegate :verification_retry_at, :verification_retry_at=, - :verified_at, :verified_at=, - :verification_checksum, :verification_checksum=, - :verification_failure, :verification_failure=, - :verification_retry_count, :verification_retry_count=, - :verification_state=, :verification_state, - :verification_started_at=, :verification_started_at, - to: :cool_widget_state - ... - - scope :with_verification_state, ->(state) { joins(:cool_widget_state).where(cool_widget_states: { verification_state: verification_state_value(state) }) } - scope :checksummed, -> { joins(:cool_widget_state).where.not(cool_widget_states: { verification_checksum: nil } ) } - scope :not_checksummed, -> { joins(:cool_widget_state).where(cool_widget_states: { verification_checksum: nil } ) } - - ... - - class_methods do - extend ::Gitlab::Utils::Override - ... - override :verification_state_table_name - def verification_state_table_name - 'cool_widget_states' - end - - override :verification_state_model_key - def verification_state_model_key - 'cool_widget_id' - end - - override :verification_arel_table - def verification_arel_table - CoolWidgetState.arel_table - end - end - ... - - def cool_widget_state - super || build_cool_widget_state - end - - ... - end - ``` - #### Step 2. Implement metrics gathering Metrics are gathered by `Geo::MetricsUpdateWorker`, persisted in `GeoNodeStatus` for display in the UI, and sent to Prometheus: diff --git a/.gitlab/issue_templates/InfraDev.md b/.gitlab/issue_templates/InfraDev.md index bc0e65c3c2..d337e75289 100644 --- a/.gitlab/issue_templates/InfraDev.md +++ b/.gitlab/issue_templates/InfraDev.md @@ -53,4 +53,4 @@ See also: --> /label ~"infradev" -/label ~"bug" +/label ~"type::bug" diff --git a/.gitlab/issue_templates/Problem Validation.md b/.gitlab/issue_templates/Problem Validation.md index 5d417c5a26..3f92510b6a 100644 --- a/.gitlab/issue_templates/Problem Validation.md +++ b/.gitlab/issue_templates/Problem Validation.md @@ -1,3 +1,7 @@ + + ## Problem Statement @@ -45,4 +49,8 @@ For example, if the solution will take a product manager, designer, and engineer - [ ] The problem is well described and detailed with necessary requirements for product design to understand the problem - [ ] The problem is well described and detailed with necessary requirements for engineering to understand the problem +## Research Issue + + + /label ~"workflow::validation backlog" ~devops:: ~category: ~group:: diff --git a/.gitlab/issue_templates/Productivity Improvement.md b/.gitlab/issue_templates/Productivity Improvement.md index 06692d3ede..040d8ea2f8 100644 --- a/.gitlab/issue_templates/Productivity Improvement.md +++ b/.gitlab/issue_templates/Productivity Improvement.md @@ -2,7 +2,7 @@ ### Problem identification checklist diff --git a/.gitlab/issue_templates/Refactoring.md b/.gitlab/issue_templates/Refactoring.md index d9466185ff..df18dcf765 100644 --- a/.gitlab/issue_templates/Refactoring.md +++ b/.gitlab/issue_templates/Refactoring.md @@ -41,9 +41,9 @@ please list them here. -/label ~"feature::maintenance" +/label ~"type::maintenance" diff --git a/.gitlab/issue_templates/Security developer workflow.md b/.gitlab/issue_templates/Security developer workflow.md index 7f2c54f4f4..6bf9e6971d 100644 --- a/.gitlab/issue_templates/Security developer workflow.md +++ b/.gitlab/issue_templates/Security developer workflow.md @@ -12,7 +12,7 @@ Set the title to: `Description of the original issue` - [ ] Make sure the issue really needs to follow the security release workflow. - Verify if the issue you're working on `gitlab-org/gitlab` is confidential, if it's public fix should be placed on GitLab canonical and no backports are required. - If the issue you're fixing doesn't appear to be something that can be exploited by a malicious person and is instead simply a security enhancement do not hesitate to ping `@gitlab-com/gl-security/appsec` to discuss if the issue can be fixed in the canonical repository. -- [ ] **IMPORTANT**: Mark this [issue as linked] to the Security Release Tracking Issue. You can find it on the topic of the `#releases` Slack channel. This issue +- [ ] **IMPORTANT**: Mark this [issue as linked] to the Security Release Tracking Issue. You can find it [here](https://gitlab.com/gitlab-org/gitlab/-/issues?sort=created_date&state=opened&label_name[]=upcoming+security+release). This issue MUST be linked for the release bot to know that the associated merge requests should be merged for this security release. - Fill out the [Links section](#links): - [ ] Next to **Issue on GitLab**, add a link to the `gitlab-org/gitlab` issue that describes the security vulnerability. diff --git a/.gitlab/issue_templates/Technical Evaluation.md b/.gitlab/issue_templates/Technical Evaluation.md index cf939725a7..680ecb7d9a 100644 --- a/.gitlab/issue_templates/Technical Evaluation.md +++ b/.gitlab/issue_templates/Technical Evaluation.md @@ -29,5 +29,5 @@ ### Team -- [ ] Add ~"workflow::planning breakdown" ~feature and the corresponding `~devops::` and `~group::` labels. +- [ ] Add ~"workflow::planning breakdown" ~"type::feature" and the corresponding `~devops::` and `~group::` labels. - [ ] Ping the PM and EM. diff --git a/.gitlab/merge_request_templates/Deprecations.md b/.gitlab/merge_request_templates/Deprecations.md index 8431e9ca39..1449246b9b 100644 --- a/.gitlab/merge_request_templates/Deprecations.md +++ b/.gitlab/merge_request_templates/Deprecations.md @@ -6,16 +6,19 @@ **Be sure to link this MR to the relevant deprecation issue(s).** +Deprecation announcements can and should be created and merged into Docs at any time, to optimize user awareness and planning. We encourage confirmed deprecations to be merged as soon as the required reviews are complete, even if weeks ahead of the target milestone's release post. For the announcement to be included in a specific release post and that release's documentation packages, this MR must be reviewed/merged per the due dates below: + **By the 10th**: Assign this MR to these team members as Reviewer and for Approval (optional unless noted as required): - Product Marketing: `@PMM` - Product Designer(s): `@ProductDesigners` - Group Manager or Director: `@manager` - Engineering Manager: `@EM` - Required +- Technical writer: `@TW` - Required -**By 8:00 AM PDT 15th**: PM will assign this MR to the TW reviewer: `@PM` +**By 11:59 AM PDT 15th**: PM assigns this MR to the TW reviewer for final review and merge: `@PM` -**By 11:59 PM PDT 15th**: TW Reviewer will perform final review and merge this MR to Master: `@TW` +**By 11:59 PM PDT 17th**: TW Reviewer updates Docs by merging this MR to `master`: `@TW` --- @@ -31,8 +34,9 @@ They are frequently updated, and everyone should make sure they are aware of the ## PM release post item checklist - [ ] Set yourself as the Assignee. +- [ ] If the deprecation is a [breaking change](https://about.gitlab.com/handbook/product/gitlab-the-product/#breaking-change), add label `breaking change`. - [ ] Follow the process to [create a deprecation YAML file](https://about.gitlab.com/handbook/marketing/blog/release-posts/#creating-a-deprecation-entry). -- [ ] Add reviewers by the 10th +- [ ] Add reviewers by the 10th. - [ ] When ready to be merged and not later than the 15th, add the ~ready label and @ message the TW for final review and merge. ## Reviewers @@ -78,5 +82,18 @@ yourself as a reviewer if it's not ready for merge yet. -When the PM indicates it is ready for merge, all issues have been addressed merge this MR. - - You must merge this MR by the 15th so the Release Post TW lead can run the [deprecations in Docs rake task](https://about.gitlab.com/handbook/marketing/blog/release-posts/#update-the-deprecations-doc) on the 16th +When the PM indicates it is ready for merge and all issues have been addressed, start the merge process. + +#### Technical writer merge process + +The [deprecations doc's `.md` file](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/update/deprecations.md) +must be updated before this MR is merged: + +1. Check out the MR's branch (in the [`gitlab-org/gitlab`](https://gitlab.com/gitlab-org/gitlab) project). +1. From the command line (in the branch), run `bin/rake gitlab:docs:compile_deprecations`. + If you want to double check that it worked, you can run `bin/rake gitlab:docs:check_deprecations` + to verify that the doc is up to date. +1. Commit the updated file and push the changes. +1. Set the MR to merge when the pipeline succeeds (or merge if the pipeline is already complete). + +If you have trouble running the rake task, check the [troubleshooting steps](https://about.gitlab.com/handbook/marketing/blog/release-posts/#deprecation-rake-task-troubleshooting). diff --git a/.gitlab/merge_request_templates/Documentation.md b/.gitlab/merge_request_templates/Documentation.md index 66c1eff412..893ae7b93b 100644 --- a/.gitlab/merge_request_templates/Documentation.md +++ b/.gitlab/merge_request_templates/Documentation.md @@ -20,10 +20,10 @@ If you are only adding documentation, do not add any of the following labels: -- `~"feature"` +- `~"type::feature"` - `~"frontend"` - `~"backend"` -- `~"bug"` +- `~"type::bug"` - `~"database"` These labels cause the MR to be added to code verification QA issues. diff --git a/.gitlab/merge_request_templates/New End To End Test.md b/.gitlab/merge_request_templates/New End To End Test.md index f9664c6315..9ecf8999f6 100644 --- a/.gitlab/merge_request_templates/New End To End Test.md +++ b/.gitlab/merge_request_templates/New End To End Test.md @@ -24,5 +24,5 @@ Please link to the respective test case in the testcases project /label ~"Quality:test-gap" ~"Quality:EE test gaps" - -/label ~"feature::addition" ~"feature::maintenance" + +/label ~"feature::addition" ~"type::maintenance" diff --git a/.gitlab/merge_request_templates/Pipeline Configuration.md b/.gitlab/merge_request_templates/Pipeline Configuration.md index 920abf086c..62210028c1 100644 --- a/.gitlab/merge_request_templates/Pipeline Configuration.md +++ b/.gitlab/merge_request_templates/Pipeline Configuration.md @@ -33,6 +33,6 @@ This will help keep track of expected cost increases to the [GitLab project aver ### Post-merge -- [ ] Consider communicating these changes to the broader team following the [communication guideline for pipeline changes](https://about.gitlab.com/handbook/engineering/quality/engineering-productivity-team/#pipeline-changes) +- [ ] Consider communicating these changes to the broader team following the [communication guideline for pipeline changes](https://about.gitlab.com/handbook/engineering/quality/engineering-productivity/#pipeline-changes) -/label ~tooling ~"tooling::pipelines" ~"Engineering Productivity" +/label ~"type::tooling" ~"tooling::pipelines" ~"Engineering Productivity" diff --git a/.gitlab/merge_request_templates/Quarantine End to End Test.md b/.gitlab/merge_request_templates/Quarantine End to End Test.md index 4edfd2a8c8..4caebb7f1b 100644 --- a/.gitlab/merge_request_templates/Quarantine End to End Test.md +++ b/.gitlab/merge_request_templates/Quarantine End to End Test.md @@ -30,7 +30,7 @@ the noise (due to constantly failing tests, flaky tests, and so on) so that new - [ ] To ensure a faster turnaround, ask in the `#quality` Slack channel for someone to review and merge the merge request, rather than assigning it directly. -/label ~"Quality" ~"QA" ~"feature" ~"feature::maintenance" +/label ~"Quality" ~"QA" ~"type::maintenance" - + + + + +
@@ -159,7 +159,7 @@ export default { ) }}