Update upstream source from tag 'upstream/13.1.4'

Update to upstream version '13.1.4'
with Debian dir cdde1738b8

CHANGELOG.md

@@ -2,6 +2,16 @@
 documentation](doc/development/changelog.md) for instructions on adding your own
 entry.
 
+## 13.1.4 (2020-07-09)
+
+### Fixed (4 changes)
+
+- Fix path conflict for Ghost on UpdateRoutesForLostAndFoundGroupAndOrphanedProjects. !35425
+- Fix existing repository_storages_weighted migrations. !35814
+- Fix error 500s creating new projects due to empty weights. !35829
+- Fix gitlab:*:check Rake tasks. !35944
+
+
 ## 13.1.3 (2020-07-06)
 
 - No changes.

GITALY_SERVER_VERSION

@@ -1 +1 @@
-13.1.3
+13.1.4

VERSION

@@ -1 +1 @@
-13.1.3
+13.1.4

app/controllers/projects/blob_controller.rb

@@ -30,7 +30,7 @@ class Projects::BlobController < Projects::ApplicationController
   before_action :set_last_commit_sha, only: [:edit, :update]
 
   before_action only: :show do
-    push_frontend_feature_flag(:code_navigation, @project)
+    push_frontend_feature_flag(:code_navigation, @project, default_enabled: true)
     push_frontend_feature_flag(:suggest_pipeline) if experiment_enabled?(:suggest_pipeline)
   end
 

app/controllers/projects/merge_requests_controller.rb

@@ -29,7 +29,7 @@ class Projects::MergeRequestsController < Projects::MergeRequests::ApplicationCo
     push_frontend_feature_flag(:deploy_from_footer, @project, default_enabled: true)
     push_frontend_feature_flag(:single_mr_diff_view, @project, default_enabled: true)
     push_frontend_feature_flag(:suggest_pipeline) if experiment_enabled?(:suggest_pipeline)
-    push_frontend_feature_flag(:code_navigation, @project)
+    push_frontend_feature_flag(:code_navigation, @project, default_enabled: true)
     push_frontend_feature_flag(:widget_visibility_polling, @project, default_enabled: true)
     push_frontend_feature_flag(:merge_ref_head_comments, @project)
     push_frontend_feature_flag(:mr_commit_neighbor_nav, @project, default_enabled: true)

db/migrate/20200508203901_add_repository_storages_weighted_to_application_settings.rb

@@ -3,11 +3,37 @@
 class AddRepositoryStoragesWeightedToApplicationSettings < ActiveRecord::Migration[6.0]
   DOWNTIME = false
 
+  class ApplicationSetting < ActiveRecord::Base
+    serialize :repository_storages
+    self.table_name = 'application_settings'
+  end
+
   def up
     add_column :application_settings, :repository_storages_weighted, :jsonb, default: {}, null: false
+
+    seed_repository_storages_weighted
   end
 
   def down
     remove_column :application_settings, :repository_storages_weighted
   end
+
+  private
+
+  def seed_repository_storages_weighted
+    # We need to flush the cache to ensure the newly-added column is loaded
+    ApplicationSetting.reset_column_information
+
+    # There should only be one row here due to
+    # 20200420162730_remove_additional_application_settings_rows.rb
+    ApplicationSetting.all.each do |settings|
+      storages = Gitlab.config.repositories.storages.keys.collect do |storage|
+        weight = settings.repository_storages.include?(storage) ? 100 : 0
+        [storage.to_sym, weight]
+      end
+
+      settings.repository_storages_weighted = Hash[storages]
+      settings.save!
+    end
+  end
 end

db/migrate/20200509203901_reseed_repository_storages_weighted.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+class ReseedRepositoryStoragesWeighted < ActiveRecord::Migration[6.0]
+  DOWNTIME = false
+
+  class ApplicationSetting < ActiveRecord::Base
+    serialize :repository_storages
+    self.table_name = 'application_settings'
+  end
+
+  def up
+    reseed_repository_storages_weighted
+  end
+
+  private
+
+  def reseed_repository_storages_weighted
+    # We need to flush the cache to ensure the newly-added column is loaded
+    ApplicationSetting.reset_column_information
+
+    # There should only be one row here due to
+    # 20200420162730_remove_additional_application_settings_rows.rb
+    ApplicationSetting.all.each do |settings|
+      # Admins may have already tweaked these values, so don't do anything
+      # if there is data already.
+      next if settings.repository_storages_weighted.present?
+
+      storages = Gitlab.config.repositories.storages.keys.collect do |storage|
+        weight = settings.repository_storages.include?(storage) ? 100 : 0
+        [storage.to_sym, weight]
+      end
+
+      settings.repository_storages_weighted = Hash[storages]
+      settings.save!
+    end
+  end
+end

db/post_migrate/20200526000407_seed_repository_storages_weighted.rb

@@ -1,16 +1,23 @@
 # frozen_string_literal: true
 
 class SeedRepositoryStoragesWeighted < ActiveRecord::Migration[6.0]
+  DOWNTIME = false
+
   class ApplicationSetting < ActiveRecord::Base
     serialize :repository_storages
     self.table_name = 'application_settings'
   end
 
   def up
+    # We need to flush the cache to ensure the newly-added column is loaded
+    ApplicationSetting.reset_column_information
+
+    # There should only be one row here due to
+    # 20200420162730_remove_additional_application_settings_rows.rb
     ApplicationSetting.all.each do |settings|
       storages = Gitlab.config.repositories.storages.keys.collect do |storage|
         weight = settings.repository_storages.include?(storage) ? 100 : 0
-        [storage, weight]
+        [storage.to_sym, weight]
       end
 
       settings.repository_storages_weighted = Hash[storages]

db/post_migrate/20200602143020_update_routes_for_lost_and_found_group_and_orphaned_projects.rb

@@ -136,6 +136,7 @@ class UpdateRoutesForLostAndFoundGroupAndOrphanedProjects < ActiveRecord::Migrat
     # to ensure the Active Record's knowledge of the table structure is current
     Namespace.reset_column_information
     Route.reset_column_information
+    User.reset_column_information
 
     # Find the ghost user, its namespace and the "lost and found" group
     ghost_user = User.ghost
@@ -158,6 +159,15 @@ class UpdateRoutesForLostAndFoundGroupAndOrphanedProjects < ActiveRecord::Migrat
       'More info: gitlab.com/gitlab-org/gitlab/-/issues/198603'
     lost_and_found_group.save!
 
+    # make sure that the ghost namespace has a unique path
+    ghost_namespace.generate_unique_path
+
+    if ghost_namespace.path_changed?
+      ghost_namespace.save!
+      # If the path changed, also update the Ghost User's username to match the new path.
+      ghost_user.update!(username: ghost_namespace.path)
+    end
+
     # Update the routes for the Ghost user, the "lost and found" group
     # and all the orphaned projects
     ghost_namespace.ensure_route!

db/structure.sql

@@ -13903,6 +13903,7 @@ COPY "schema_migrations" (version) FROM STDIN;
 20200508091106
 20200508140959
 20200508203901
+20200509203901
 20200511080113
 20200511083541
 20200511092246

doc/administration/raketasks/doctor.md

@@ -0,0 +1,84 @@
+# Doctor Rake tasks **(CORE ONLY)**
+
+This is a collection of tasks to help investigate and repair
+problems caused by data integrity issues.
+
+## Verify database values can be decrypted using the current secrets
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/20069) in GitLab 13.1.
+
+This task runs through all possible encrypted values in the
+database, verifying that they are decryptable using the current
+secrets file (`gitlab-secrets.json`).
+
+Automatic resolution is not yet implemented. If you have values that
+cannot be decrypted, you can follow steps to reset them, see our
+docs on what to do [when the secrets file is lost](../../raketasks/backup_restore.md#when-the-secrets-file-is-lost).
+
+NOTE: **Note:**
+This can take a very long time, depending on the size of your
+database, as it checks all rows in all tables.
+
+**Omnibus Installation**
+
+```shell
+sudo gitlab-rake gitlab:doctor:secrets
+```
+
+**Source Installation**
+
+```shell
+bundle exec rake gitlab:doctor:secrets RAILS_ENV=production
+```
+
+**Example output**
+
+<!-- vale gitlab.SentenceSpacing = NO -->
+```plaintext
+I, [2020-06-11T17:17:54.951815 #27148]  INFO -- : Checking encrypted values in the database
+I, [2020-06-11T17:18:12.677708 #27148]  INFO -- : - ApplicationSetting failures: 0
+I, [2020-06-11T17:18:12.823692 #27148]  INFO -- : - User failures: 0
+[...] other models possibly containing encrypted data
+I, [2020-06-11T17:18:14.938335 #27148]  INFO -- : - Group failures: 1
+I, [2020-06-11T17:18:15.559162 #27148]  INFO -- : - Operations::FeatureFlagsClient failures: 0
+I, [2020-06-11T17:18:15.575533 #27148]  INFO -- : - ScimOauthAccessToken failures: 0
+I, [2020-06-11T17:18:15.575678 #27148]  INFO -- : Total: 1 row(s) affected
+I, [2020-06-11T17:18:15.575711 #27148]  INFO -- : Done!
+```
+<!-- vale gitlab.SentenceSpacing = YES -->
+
+### Verbose mode
+
+In order to get more detailed information about which
+rows and columns cannot be decrypted, you can pass a VERBOSE
+environment variable:
+
+**Omnibus Installation**
+
+```shell
+sudo gitlab-rake gitlab:doctor:secrets VERBOSE=1
+```
+
+**Source Installation**
+
+```shell
+bundle exec rake gitlab:doctor:secrets RAILS_ENV=production VERBOSE=1
+```
+
+**Example verbose output**
+
+<!-- vale gitlab.SentenceSpacing = NO -->
+```plaintext
+I, [2020-06-11T17:17:54.951815 #27148]  INFO -- : Checking encrypted values in the database
+I, [2020-06-11T17:18:12.677708 #27148]  INFO -- : - ApplicationSetting failures: 0
+I, [2020-06-11T17:18:12.823692 #27148]  INFO -- : - User failures: 0
+[...] other models possibly containing encrypted data
+D, [2020-06-11T17:19:53.224344 #27351] DEBUG -- : > Something went wrong for Group[10].runners_token: Validation failed: Route can't be blank
+I, [2020-06-11T17:19:53.225178 #27351]  INFO -- : - Group failures: 1
+D, [2020-06-11T17:19:53.225267 #27351] DEBUG -- :   - Group[10]: runners_token
+I, [2020-06-11T17:18:15.559162 #27148]  INFO -- : - Operations::FeatureFlagsClient failures: 0
+I, [2020-06-11T17:18:15.575533 #27148]  INFO -- : - ScimOauthAccessToken failures: 0
+I, [2020-06-11T17:18:15.575678 #27148]  INFO -- : Total: 1 row(s) affected
+I, [2020-06-11T17:18:15.575711 #27148]  INFO -- : Done!
+```
+<!-- vale gitlab.SentenceSpacing = YES -->

doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md

@@ -320,23 +320,7 @@ end
 
 ### Find mirrors with "bad decrypt" errors
 
-```ruby
+This content has been converted to a Rake task, see the [Doctor Rake tasks docs](../raketasks/doctor.md).
-total = 0
-bad = []
-ProjectImportData.find_each do |data|
-  begin
-    total += 1
-    data.credentials
-  rescue => e
-    bad << data
-  end
-end
-
-puts "Bad count: #{bad.count} / #{total}"
-bad.each do |repo|
-  puts Project.find(repo.project_id).full_path
-end; bad.count
-```
 
 ### Transfer mirror users and tokens to a single service account
 
@@ -755,18 +739,9 @@ area on disk. It remains to be seen exactly how or whether the deletion is usefu
 
 ### Bad Decrypt Script (for encrypted variables)
 
-See <https://gitlab.com/snippets/1730735/raw>.
+This content has been converted to a Rake task, see the [Doctor Rake tasks docs](../raketasks/doctor.md).
 
-This script will go through all the encrypted variables and count how many are not able
+As an example of repairing, if `ProjectImportData Bad count:` is detected and the decision is made to delete the
-to be decrypted. Might be helpful to run on multiple nodes to see which `gitlab-secrets.json`
-file is most up to date:
-
-```shell
-wget -O /tmp/bad-decrypt.rb https://gitlab.com/snippets/1730735/raw
-gitlab-rails runner /tmp/bad-decrypt.rb
-```
-
-If `ProjectImportData Bad count:` is detected and the decision is made to delete the
 encrypted credentials to allow manual reentry:
 
 ```ruby
@@ -797,16 +772,18 @@ encrypted credentials to allow manual reentry:
 If `User OTP Secret Bad count:` is detected. For each user listed disable/enable
 two-factor authentication.
 
-### Decrypt Script for encrypted tokens
+The following script will search in some of the tables for encrypted tokens that are
-
+causing decryption errors, and update or reset as needed:
-This script will search for all encrypted tokens that are causing decryption errors,
-and update or reset as needed:
 
 ```shell
 wget -O /tmp/encrypted-tokens.rb https://gitlab.com/snippets/1876342/raw
 gitlab-rails runner /tmp/encrypted-tokens.rb
 ```
 
+### Decrypt Script for encrypted tokens
+
+This content has been converted to a Rake task, see the [Doctor Rake tasks docs](../raketasks/doctor.md).
+
 ## Geo
 
 ### Artifacts

doc/raketasks/README.md

@@ -20,6 +20,7 @@ The following are available Rake tasks:
 | [Back up and restore](backup_restore.md)                                                            | Back up, restore, and migrate GitLab instances between servers.                           |
 | [Clean up](cleanup.md)                                                                              | Clean up unneeded items from GitLab instances.                                            |
 | [Development](../development/rake_tasks.md)                                                         | Tasks for GitLab contributors.                                                            |
+| [Doctor tasks](../administration/raketasks/doctor.md)                                               | Checks for data integrity issues.                                                         |
 | [Elasticsearch](../integration/elasticsearch.md#gitlab-elasticsearch-rake-tasks) **(STARTER ONLY)** | Maintain Elasticsearch in a GitLab instance.                                              |
 | [Enable namespaces](features.md)                                                                    | Enable usernames and namespaces for user projects.                                        |
 | [General maintenance](../administration/raketasks/maintenance.md)                                   | General maintenance and self-check tasks.                                                 |

doc/raketasks/backup_restore.md

@@ -941,6 +941,9 @@ experience some unexpected behavior such as:
 - Stuck jobs.
 - 500 errors.
 
+You can check whether you have undecryptable values in the database using
+the [Secrets Doctor Rake task](../administration/raketasks/doctor.md).
+
 In this case, you are required to reset all the tokens for CI/CD variables
 and Runner Authentication, which is described in more detail below. After
 resetting the tokens, you should be able to visit your project and the jobs

lib/gitlab/verify/batch_verifier.rb

@@ -71,9 +71,8 @@ module Gitlab
       # It's already set to Logger::INFO, but acts as if it is set to
       # Logger::DEBUG, and this fixes it...
       def fix_google_api_logger
-        if Object.const_defined?('Google::Apis')
+        require 'google/apis'
-          Google::Apis.logger.level = Logger::INFO
+        Google::Apis.logger.level = Logger::INFO
-        end
       end
 
       # This should return an ActiveRecord::Relation suitable for calling #in_batches on

spec/features/merge_request/maintainer_edits_fork_spec.rb

@@ -20,7 +20,7 @@ RSpec.describe 'a maintainer edits files on a source-branch of an MR from a fork
   end
 
   before do
-    stub_feature_flags(web_ide_default: false, single_mr_diff_view: false, code_navigation: false)
+    stub_feature_flags(web_ide_default: false, single_mr_diff_view: false)
 
     target_project.add_maintainer(user)
     sign_in(user)

spec/features/projects/blobs/blob_show_spec.rb

@@ -13,10 +13,6 @@ RSpec.describe 'File blob', :js do
     wait_for_requests
   end
 
-  before do
-    stub_feature_flags(code_navigation: false)
-  end
-
   context 'Ruby file' do
     before do
       visit_blob('files/ruby/popen.rb')

spec/features/projects/blobs/edit_spec.rb

@@ -69,8 +69,6 @@ RSpec.describe 'Editing file blob', :js do
 
     context 'from blob file path' do
       before do
-        stub_feature_flags(code_navigation: false)
-
         visit project_blob_path(project, tree_join(branch, file_path))
       end
 

spec/features/projects/blobs/user_creates_new_blob_in_new_project_spec.rb

@@ -8,7 +8,6 @@ RSpec.describe 'User creates blob in new project', :js do
 
   shared_examples 'creating a file' do
     before do
-      stub_feature_flags(code_navigation: false)
       sign_in(user)
       visit project_path(project)
     end

spec/features/projects/files/user_creates_files_spec.rb

@@ -14,7 +14,7 @@ RSpec.describe 'Projects > Files > User creates files', :js do
   let(:user) { create(:user) }
 
   before do
-    stub_feature_flags(web_ide_default: false, code_navigation: false)
+    stub_feature_flags(web_ide_default: false)
 
     project.add_maintainer(user)
     sign_in(user)

spec/features/projects/files/user_deletes_files_spec.rb

@@ -14,8 +14,6 @@ RSpec.describe 'Projects > Files > User deletes files', :js do
   let(:user) { create(:user) }
 
   before do
-    stub_feature_flags(code_navigation: false)
-
     sign_in(user)
   end
 

spec/features/projects/files/user_replaces_files_spec.rb

@@ -16,8 +16,6 @@ RSpec.describe 'Projects > Files > User replaces files', :js do
   let(:user) { create(:user) }
 
   before do
-    stub_feature_flags(code_navigation: false)
-
     sign_in(user)
   end
 

spec/migrations/add_repository_storages_weighted_to_application_settings_spec.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require Rails.root.join('db', 'migrate', '20200508203901_add_repository_storages_weighted_to_application_settings.rb')
+
+RSpec.describe AddRepositoryStoragesWeightedToApplicationSettings, :migration do
+  let(:storages) { { "foo" => {}, "baz" => {} } }
+  let(:application_settings) do
+    table(:application_settings).tap do |klass|
+      klass.class_eval do
+        serialize :repository_storages
+      end
+    end
+  end
+
+  before do
+    allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
+  end
+
+  let(:application_setting) { application_settings.create! }
+  let(:repository_storages) { ["foo"] }
+
+  it 'populates repository_storages_weighted properly' do
+    application_setting.repository_storages = repository_storages
+    application_setting.save!
+
+    migrate!
+
+    expect(application_settings.find(application_setting.id).repository_storages_weighted).to eq({ "foo" => 100, "baz" => 0 })
+  end
+end

spec/migrations/reseed_repository_storages_weighted_spec.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require Rails.root.join('db', 'migrate', '20200509203901_reseed_repository_storages_weighted.rb')
+
+RSpec.describe ReseedRepositoryStoragesWeighted do
+  let(:storages) { { "foo" => {}, "baz" => {} } }
+  let(:application_settings) do
+    table(:application_settings).tap do |klass|
+      klass.class_eval do
+        serialize :repository_storages
+      end
+    end
+  end
+
+  before do
+    allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
+  end
+
+  let(:repository_storages) { ["foo"] }
+  let!(:application_setting) { application_settings.create!(repository_storages: repository_storages) }
+
+  context 'with empty repository_storages_weighted column' do
+    it 'populates repository_storages_weighted properly' do
+      migrate!
+
+      expect(application_settings.find(application_setting.id).repository_storages_weighted).to eq({ "foo" => 100, "baz" => 0 })
+    end
+  end
+
+  context 'with already-populated repository_storages_weighted column' do
+    let(:existing_weights) { { "foo" => 100, "baz" => 50 } }
+
+    it 'does not change repository_storages_weighted properly' do
+      application_setting.repository_storages_weighted = existing_weights
+      application_setting.save!
+
+      migrate!
+
+      expect(application_settings.find(application_setting.id).repository_storages_weighted).to eq(existing_weights)
+    end
+  end
+end

spec/migrations/update_routes_for_lost_and_found_group_and_orphaned_projects_spec.rb

@@ -62,8 +62,8 @@ describe UpdateRoutesForLostAndFoundGroupAndOrphanedProjects, :migration do
       source_type: 'Project',
       path: 'orphaned_project',
       name: 'orphaned_project',
-      created_at: Time.zone.now,
+      created_at: Time.current,
-      updated_at: Time.zone.now
+      updated_at: Time.current
     )
 
     # Create another user named ghost which is not the Ghost User
@@ -90,10 +90,10 @@ describe UpdateRoutesForLostAndFoundGroupAndOrphanedProjects, :migration do
     routes.create!(
       source_id: fake_ghost_user_namespace.id,
       source_type: 'Namespace',
-      path: 'Ghost User',
+      path: 'ghost1',
-      name: 'ghost1',
+      name: 'Ghost User',
-      created_at: Time.zone.now,
+      created_at: Time.current,
-      updated_at: Time.zone.now
+      updated_at: Time.current
     )
 
     fake_lost_and_found_group = namespaces.create!(
@@ -109,8 +109,8 @@ describe UpdateRoutesForLostAndFoundGroupAndOrphanedProjects, :migration do
       source_type: 'Namespace',
       path: described_class::User::LOST_AND_FOUND_GROUP, # same path as the lost-and-found group
       name: 'Lost and Found',
-      created_at: Time.zone.now,
+      created_at: Time.current,
-      updated_at: Time.zone.now
+      updated_at: Time.current
     )
 
     members.create!(
@@ -135,17 +135,58 @@ describe UpdateRoutesForLostAndFoundGroupAndOrphanedProjects, :migration do
       source_type: 'Project',
       path: "#{described_class::User::LOST_AND_FOUND_GROUP}/normal_project",
       name: 'Lost and Found / normal_project',
-      created_at: Time.zone.now,
+      created_at: Time.current,
-      updated_at: Time.zone.now
+      updated_at: Time.current
+    )
+
+    # Add a project whose route conflicts with the ghost username
+    # and should force the data migration to pick a new Ghost username and path
+    ghost_project = projects.create!(
+      name: 'Ghost Project',
+      path: 'ghost',
+      visibility_level: 20,
+      archived: false,
+      namespace_id: fake_lost_and_found_group.id
+    )
+
+    routes.create!(
+      source_id: ghost_project.id,
+      source_type: 'Project',
+      path: 'ghost',
+      name: 'Ghost Project',
+      created_at: Time.current,
+      updated_at: Time.current
     )
   end
 
+  it 'fixes the ghost user username and namespace path' do
+    ghost_user = users.find_by(user_type: described_class::User::USER_TYPE_GHOST)
+    ghost_namespace = namespaces.find_by(owner_id: ghost_user.id)
+
+    expect(ghost_user.username).to eq('ghost')
+    expect(ghost_namespace.path).to eq('ghost')
+
+    disable_migrations_output { migrate! }
+
+    ghost_user = users.find_by(user_type: described_class::User::USER_TYPE_GHOST)
+    ghost_namespace = namespaces.find_by(owner_id: ghost_user.id)
+    ghost_namespace_route = routes.find_by(source_id: ghost_namespace.id, source_type: 'Namespace')
+
+    expect(ghost_user.username).to eq('ghost2')
+    expect(ghost_namespace.path).to eq('ghost2')
+    expect(ghost_namespace_route.path).to eq('ghost2')
+  end
+
   it 'creates the route for the ghost user namespace' do
-    expect(routes.where(path: 'ghost').count).to eq(0)
+    expect(routes.where(path: 'ghost').count).to eq(1)
+    expect(routes.where(path: 'ghost1').count).to eq(1)
+    expect(routes.where(path: 'ghost2').count).to eq(0)
 
     disable_migrations_output { migrate! }
 
     expect(routes.where(path: 'ghost').count).to eq(1)
+    expect(routes.where(path: 'ghost1').count).to eq(1)
+    expect(routes.where(path: 'ghost2').count).to eq(1)
   end
 
   it 'fixes the path for the lost-and-found group by generating a unique one' do