realaravinth/debian-mirror-gitlab
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Update upstream source from tag 'upstream/13.1.4'

Browse source 
Update to upstream version '13.1.4'
with Debian dir cdde1738b8
This commit is contained in:
Pirate Praveen 2020-07-10 23:48:49 +05:30
commit f63ed07031
25 changed files with 323 additions and 64 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
CHANGELOG.md
View file

@ -2,6 +2,16 @@
documentation](doc/development/changelog.md) for instructions on adding your own
entry.
## 13.1.4 (2020-07-09)
### Fixed (4 changes)
- Fix path conflict for Ghost on UpdateRoutesForLostAndFoundGroupAndOrphanedProjects. !35425
- Fix existing repository_storages_weighted migrations. !35814
- Fix error 500s creating new projects due to empty weights. !35829
- Fix gitlab:*:check Rake tasks. !35944
## 13.1.3 (2020-07-06)
- No changes.

2
GITALY_SERVER_VERSION
View file

@ -1 +1 @@
13.1.3
13.1.4

2
VERSION
View file

@ -1 +1 @@
13.1.3
13.1.4

2
app/controllers/projects/blob_controller.rb
View file

@ -30,7 +30,7 @@ class Projects::BlobController < Projects::ApplicationController
before_action :set_last_commit_sha, only: [:edit, :update]
before_action only: :show do
push_frontend_feature_flag(:code_navigation, @project)
push_frontend_feature_flag(:code_navigation, @project, default_enabled: true)
push_frontend_feature_flag(:suggest_pipeline) if experiment_enabled?(:suggest_pipeline)
end

2
app/controllers/projects/merge_requests_controller.rb
View file

@ -29,7 +29,7 @@ class Projects::MergeRequestsController < Projects::MergeRequests::ApplicationCo
push_frontend_feature_flag(:deploy_from_footer, @project, default_enabled: true)
push_frontend_feature_flag(:single_mr_diff_view, @project, default_enabled: true)
push_frontend_feature_flag(:suggest_pipeline) if experiment_enabled?(:suggest_pipeline)
push_frontend_feature_flag(:code_navigation, @project)
push_frontend_feature_flag(:code_navigation, @project, default_enabled: true)
push_frontend_feature_flag(:widget_visibility_polling, @project, default_enabled: true)
push_frontend_feature_flag(:merge_ref_head_comments, @project)
push_frontend_feature_flag(:mr_commit_neighbor_nav, @project, default_enabled: true)

26
db/migrate/20200508203901_add_repository_storages_weighted_to_application_settings.rb
View file

@ -3,11 +3,37 @@
class AddRepositoryStoragesWeightedToApplicationSettings < ActiveRecord::Migration[6.0]
DOWNTIME = false
class ApplicationSetting < ActiveRecord::Base
serialize :repository_storages
self.table_name = 'application_settings'
end
def up
add_column :application_settings, :repository_storages_weighted, :jsonb, default: {}, null: false
seed_repository_storages_weighted
end
def down
remove_column :application_settings, :repository_storages_weighted
end
private
def seed_repository_storages_weighted
# We need to flush the cache to ensure the newly-added column is loaded
ApplicationSetting.reset_column_information
# There should only be one row here due to
# 20200420162730_remove_additional_application_settings_rows.rb
ApplicationSetting.all.each do |settings|
storages = Gitlab.config.repositories.storages.keys.collect do |storage|
weight = settings.repository_storages.include?(storage) ? 100 : 0
[storage.to_sym, weight]
end
settings.repository_storages_weighted = Hash[storages]
settings.save!
end
end
end

37
db/migrate/20200509203901_reseed_repository_storages_weighted.rb Normal file
View file

@ -0,0 +1,37 @@
# frozen_string_literal: true
class ReseedRepositoryStoragesWeighted < ActiveRecord::Migration[6.0]
DOWNTIME = false
class ApplicationSetting < ActiveRecord::Base
serialize :repository_storages
self.table_name = 'application_settings'
end
def up
reseed_repository_storages_weighted
end
private
def reseed_repository_storages_weighted
# We need to flush the cache to ensure the newly-added column is loaded
ApplicationSetting.reset_column_information
# There should only be one row here due to
# 20200420162730_remove_additional_application_settings_rows.rb
ApplicationSetting.all.each do |settings|
# Admins may have already tweaked these values, so don't do anything
# if there is data already.
next if settings.repository_storages_weighted.present?
storages = Gitlab.config.repositories.storages.keys.collect do |storage|
weight = settings.repository_storages.include?(storage) ? 100 : 0
[storage.to_sym, weight]
end
settings.repository_storages_weighted = Hash[storages]
settings.save!
end
end
end

9
db/post_migrate/20200526000407_seed_repository_storages_weighted.rb
View file

@ -1,16 +1,23 @@
# frozen_string_literal: true
class SeedRepositoryStoragesWeighted < ActiveRecord::Migration[6.0]
DOWNTIME = false
class ApplicationSetting < ActiveRecord::Base
serialize :repository_storages
self.table_name = 'application_settings'
end
def up
# We need to flush the cache to ensure the newly-added column is loaded
ApplicationSetting.reset_column_information
# There should only be one row here due to
# 20200420162730_remove_additional_application_settings_rows.rb
ApplicationSetting.all.each do |settings|
storages = Gitlab.config.repositories.storages.keys.collect do |storage|
weight = settings.repository_storages.include?(storage) ? 100 : 0
[storage, weight]
[storage.to_sym, weight]
end
settings.repository_storages_weighted = Hash[storages]

10
db/post_migrate/20200602143020_update_routes_for_lost_and_found_group_and_orphaned_projects.rb
View file

@ -136,6 +136,7 @@ class UpdateRoutesForLostAndFoundGroupAndOrphanedProjects < ActiveRecord::Migrat
# to ensure the Active Record's knowledge of the table structure is current
Namespace.reset_column_information
Route.reset_column_information
User.reset_column_information
# Find the ghost user, its namespace and the "lost and found" group
ghost_user = User.ghost
@ -158,6 +159,15 @@ class UpdateRoutesForLostAndFoundGroupAndOrphanedProjects < ActiveRecord::Migrat
'More info: gitlab.com/gitlab-org/gitlab/-/issues/198603'
lost_and_found_group.save!
# make sure that the ghost namespace has a unique path
ghost_namespace.generate_unique_path
if ghost_namespace.path_changed?
ghost_namespace.save!
# If the path changed, also update the Ghost User's username to match the new path.
ghost_user.update!(username: ghost_namespace.path)
end
# Update the routes for the Ghost user, the "lost and found" group
# and all the orphaned projects
ghost_namespace.ensure_route!

1
db/structure.sql
View file

@ -13903,6 +13903,7 @@ COPY "schema_migrations" (version) FROM STDIN;
20200508091106
20200508140959
20200508203901
20200509203901
20200511080113
20200511083541
20200511092246

84
doc/administration/raketasks/doctor.md Normal file
View file

@ -0,0 +1,84 @@
# Doctor Rake tasks **(CORE ONLY)**
This is a collection of tasks to help investigate and repair
problems caused by data integrity issues.
## Verify database values can be decrypted using the current secrets
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/20069) in GitLab 13.1.
This task runs through all possible encrypted values in the
database, verifying that they are decryptable using the current
secrets file (`gitlab-secrets.json`).
Automatic resolution is not yet implemented. If you have values that
cannot be decrypted, you can follow steps to reset them, see our
docs on what to do [when the secrets file is lost](../../raketasks/backup_restore.md#when-the-secrets-file-is-lost).
NOTE: **Note:**
This can take a very long time, depending on the size of your
database, as it checks all rows in all tables.
**Omnibus Installation**
```shell
sudo gitlab-rake gitlab:doctor:secrets
```
**Source Installation**
```shell
bundle exec rake gitlab:doctor:secrets RAILS_ENV=production
```
**Example output**
<!-- vale gitlab.SentenceSpacing = NO -->
```plaintext
I, [2020-06-11T17:17:54.951815 #27148] INFO -- : Checking encrypted values in the database
I, [2020-06-11T17:18:12.677708 #27148] INFO -- : - ApplicationSetting failures: 0
I, [2020-06-11T17:18:12.823692 #27148] INFO -- : - User failures: 0
[...] other models possibly containing encrypted data
I, [2020-06-11T17:18:14.938335 #27148] INFO -- : - Group failures: 1
I, [2020-06-11T17:18:15.559162 #27148] INFO -- : - Operations::FeatureFlagsClient failures: 0
I, [2020-06-11T17:18:15.575533 #27148] INFO -- : - ScimOauthAccessToken failures: 0
I, [2020-06-11T17:18:15.575678 #27148] INFO -- : Total: 1 row(s) affected
I, [2020-06-11T17:18:15.575711 #27148] INFO -- : Done!
```
<!-- vale gitlab.SentenceSpacing = YES -->
### Verbose mode
In order to get more detailed information about which
rows and columns cannot be decrypted, you can pass a VERBOSE
environment variable:
**Omnibus Installation**
```shell
sudo gitlab-rake gitlab:doctor:secrets VERBOSE=1
```
**Source Installation**
```shell
bundle exec rake gitlab:doctor:secrets RAILS_ENV=production VERBOSE=1
```
**Example verbose output**
<!-- vale gitlab.SentenceSpacing = NO -->
```plaintext
I, [2020-06-11T17:17:54.951815 #27148] INFO -- : Checking encrypted values in the database
I, [2020-06-11T17:18:12.677708 #27148] INFO -- : - ApplicationSetting failures: 0
I, [2020-06-11T17:18:12.823692 #27148] INFO -- : - User failures: 0
[...] other models possibly containing encrypted data
D, [2020-06-11T17:19:53.224344 #27351] DEBUG -- : > Something went wrong for Group[10].runners_token: Validation failed: Route can't be blank
I, [2020-06-11T17:19:53.225178 #27351] INFO -- : - Group failures: 1
D, [2020-06-11T17:19:53.225267 #27351] DEBUG -- : - Group[10]: runners_token
I, [2020-06-11T17:18:15.559162 #27148] INFO -- : - Operations::FeatureFlagsClient failures: 0
I, [2020-06-11T17:18:15.575533 #27148] INFO -- : - ScimOauthAccessToken failures: 0
I, [2020-06-11T17:18:15.575678 #27148] INFO -- : Total: 1 row(s) affected
I, [2020-06-11T17:18:15.575711 #27148] INFO -- : Done!
```
<!-- vale gitlab.SentenceSpacing = YES -->

41
doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md
View file

@ -320,23 +320,7 @@ end
### Find mirrors with "bad decrypt" errors
```ruby
total = 0
bad = []
ProjectImportData.find_each do |data|
begin
total += 1
data.credentials
rescue => e
bad << data
end
end
puts "Bad count: #{bad.count} / #{total}"
bad.each do |repo|
puts Project.find(repo.project_id).full_path
end; bad.count
```
This content has been converted to a Rake task, see the [Doctor Rake tasks docs](../raketasks/doctor.md).
### Transfer mirror users and tokens to a single service account
@ -755,18 +739,9 @@ area on disk. It remains to be seen exactly how or whether the deletion is usefu
### Bad Decrypt Script (for encrypted variables)
See <https://gitlab.com/snippets/1730735/raw>.
This content has been converted to a Rake task, see the [Doctor Rake tasks docs](../raketasks/doctor.md).
This script will go through all the encrypted variables and count how many are not able
to be decrypted. Might be helpful to run on multiple nodes to see which `gitlab-secrets.json`
file is most up to date:
```shell
wget -O /tmp/bad-decrypt.rb https://gitlab.com/snippets/1730735/raw
gitlab-rails runner /tmp/bad-decrypt.rb
```
If `ProjectImportData Bad count:` is detected and the decision is made to delete the
As an example of repairing, if `ProjectImportData Bad count:` is detected and the decision is made to delete the
encrypted credentials to allow manual reentry:
```ruby
@ -797,16 +772,18 @@ encrypted credentials to allow manual reentry:
If `User OTP Secret Bad count:` is detected. For each user listed disable/enable
two-factor authentication.
### Decrypt Script for encrypted tokens
This script will search for all encrypted tokens that are causing decryption errors,
and update or reset as needed:
The following script will search in some of the tables for encrypted tokens that are
causing decryption errors, and update or reset as needed:
```shell
wget -O /tmp/encrypted-tokens.rb https://gitlab.com/snippets/1876342/raw
gitlab-rails runner /tmp/encrypted-tokens.rb
```
### Decrypt Script for encrypted tokens
This content has been converted to a Rake task, see the [Doctor Rake tasks docs](../raketasks/doctor.md).
## Geo
### Artifacts

1
doc/raketasks/README.md
View file

@ -20,6 +20,7 @@ The following are available Rake tasks:
| [Back up and restore](backup_restore.md) | Back up, restore, and migrate GitLab instances between servers. |
| [Clean up](cleanup.md) | Clean up unneeded items from GitLab instances. |
| [Development](../development/rake_tasks.md) | Tasks for GitLab contributors. |
| [Doctor tasks](../administration/raketasks/doctor.md) | Checks for data integrity issues. |
| [Elasticsearch](../integration/elasticsearch.md#gitlab-elasticsearch-rake-tasks) **(STARTER ONLY)** | Maintain Elasticsearch in a GitLab instance. |
| [Enable namespaces](features.md) | Enable usernames and namespaces for user projects. |
| [General maintenance](../administration/raketasks/maintenance.md) | General maintenance and self-check tasks. |

3
doc/raketasks/backup_restore.md
View file

@ -941,6 +941,9 @@ experience some unexpected behavior such as:
- Stuck jobs.
- 500 errors.
You can check whether you have undecryptable values in the database using
the [Secrets Doctor Rake task](../administration/raketasks/doctor.md).
In this case, you are required to reset all the tokens for CI/CD variables
and Runner Authentication, which is described in more detail below. After
resetting the tokens, you should be able to visit your project and the jobs

3
lib/gitlab/verify/batch_verifier.rb
View file

@ -71,10 +71,9 @@ module Gitlab
# It's already set to Logger::INFO, but acts as if it is set to
# Logger::DEBUG, and this fixes it...
def fix_google_api_logger
if Object.const_defined?('Google::Apis')
require 'google/apis'
Google::Apis.logger.level = Logger::INFO
end
end
# This should return an ActiveRecord::Relation suitable for calling #in_batches on
def all_relation

2
spec/features/merge_request/maintainer_edits_fork_spec.rb
View file

@ -20,7 +20,7 @@ RSpec.describe 'a maintainer edits files on a source-branch of an MR from a fork
end
before do
stub_feature_flags(web_ide_default: false, single_mr_diff_view: false, code_navigation: false)
stub_feature_flags(web_ide_default: false, single_mr_diff_view: false)
target_project.add_maintainer(user)
sign_in(user)

4
spec/features/projects/blobs/blob_show_spec.rb
View file

@ -13,10 +13,6 @@ RSpec.describe 'File blob', :js do
wait_for_requests
end
before do
stub_feature_flags(code_navigation: false)
end
context 'Ruby file' do
before do
visit_blob('files/ruby/popen.rb')

2
spec/features/projects/blobs/edit_spec.rb
View file

@ -69,8 +69,6 @@ RSpec.describe 'Editing file blob', :js do
context 'from blob file path' do
before do
stub_feature_flags(code_navigation: false)
visit project_blob_path(project, tree_join(branch, file_path))
end

1
spec/features/projects/blobs/user_creates_new_blob_in_new_project_spec.rb
View file

@ -8,7 +8,6 @@ RSpec.describe 'User creates blob in new project', :js do
shared_examples 'creating a file' do
before do
stub_feature_flags(code_navigation: false)
sign_in(user)
visit project_path(project)
end

2
spec/features/projects/files/user_creates_files_spec.rb
View file

@ -14,7 +14,7 @@ RSpec.describe 'Projects > Files > User creates files', :js do
let(:user) { create(:user) }
before do
stub_feature_flags(web_ide_default: false, code_navigation: false)
stub_feature_flags(web_ide_default: false)
project.add_maintainer(user)
sign_in(user)

2
spec/features/projects/files/user_deletes_files_spec.rb
View file

@ -14,8 +14,6 @@ RSpec.describe 'Projects > Files > User deletes files', :js do
let(:user) { create(:user) }
before do
stub_feature_flags(code_navigation: false)
sign_in(user)
end

2
spec/features/projects/files/user_replaces_files_spec.rb
View file

@ -16,8 +16,6 @@ RSpec.describe 'Projects > Files > User replaces files', :js do
let(:user) { create(:user) }
before do
stub_feature_flags(code_navigation: false)
sign_in(user)
end

31
spec/migrations/add_repository_storages_weighted_to_application_settings_spec.rb Normal file
View file

@ -0,0 +1,31 @@
# frozen_string_literal: true
require 'spec_helper'
require Rails.root.join('db', 'migrate', '20200508203901_add_repository_storages_weighted_to_application_settings.rb')
RSpec.describe AddRepositoryStoragesWeightedToApplicationSettings, :migration do
let(:storages) { { "foo" => {}, "baz" => {} } }
let(:application_settings) do
table(:application_settings).tap do |klass|
klass.class_eval do
serialize :repository_storages
end
end
end
before do
allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
end
let(:application_setting) { application_settings.create! }
let(:repository_storages) { ["foo"] }
it 'populates repository_storages_weighted properly' do
application_setting.repository_storages = repository_storages
application_setting.save!
migrate!
expect(application_settings.find(application_setting.id).repository_storages_weighted).to eq({ "foo" => 100, "baz" => 0 })
end
end

43
spec/migrations/reseed_repository_storages_weighted_spec.rb Normal file
View file

@ -0,0 +1,43 @@
# frozen_string_literal: true
require 'spec_helper'
require Rails.root.join('db', 'migrate', '20200509203901_reseed_repository_storages_weighted.rb')
RSpec.describe ReseedRepositoryStoragesWeighted do
let(:storages) { { "foo" => {}, "baz" => {} } }
let(:application_settings) do
table(:application_settings).tap do |klass|
klass.class_eval do
serialize :repository_storages
end
end
end
before do
allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
end
let(:repository_storages) { ["foo"] }
let!(:application_setting) { application_settings.create!(repository_storages: repository_storages) }
context 'with empty repository_storages_weighted column' do
it 'populates repository_storages_weighted properly' do
migrate!
expect(application_settings.find(application_setting.id).repository_storages_weighted).to eq({ "foo" => 100, "baz" => 0 })
end
end
context 'with already-populated repository_storages_weighted column' do
let(:existing_weights) { { "foo" => 100, "baz" => 50 } }
it 'does not change repository_storages_weighted properly' do
application_setting.repository_storages_weighted = existing_weights
application_setting.save!
migrate!
expect(application_settings.find(application_setting.id).repository_storages_weighted).to eq(existing_weights)
end
end
end

63
spec/migrations/update_routes_for_lost_and_found_group_and_orphaned_projects_spec.rb
View file

@ -62,8 +62,8 @@ describe UpdateRoutesForLostAndFoundGroupAndOrphanedProjects, :migration do
source_type: 'Project',
path: 'orphaned_project',
name: 'orphaned_project',
created_at: Time.zone.now,
updated_at: Time.zone.now
created_at: Time.current,
updated_at: Time.current
)
# Create another user named ghost which is not the Ghost User
@ -90,10 +90,10 @@ describe UpdateRoutesForLostAndFoundGroupAndOrphanedProjects, :migration do
routes.create!(
source_id: fake_ghost_user_namespace.id,
source_type: 'Namespace',
path: 'Ghost User',
name: 'ghost1',
created_at: Time.zone.now,
updated_at: Time.zone.now
path: 'ghost1',
name: 'Ghost User',
created_at: Time.current,
updated_at: Time.current
)
fake_lost_and_found_group = namespaces.create!(
@ -109,8 +109,8 @@ describe UpdateRoutesForLostAndFoundGroupAndOrphanedProjects, :migration do
source_type: 'Namespace',
path: described_class::User::LOST_AND_FOUND_GROUP, # same path as the lost-and-found group
name: 'Lost and Found',
created_at: Time.zone.now,
updated_at: Time.zone.now
created_at: Time.current,
updated_at: Time.current
)
members.create!(
@ -135,17 +135,58 @@ describe UpdateRoutesForLostAndFoundGroupAndOrphanedProjects, :migration do
source_type: 'Project',
path: "#{described_class::User::LOST_AND_FOUND_GROUP}/normal_project",
name: 'Lost and Found / normal_project',
created_at: Time.zone.now,
updated_at: Time.zone.now
created_at: Time.current,
updated_at: Time.current
)
# Add a project whose route conflicts with the ghost username
# and should force the data migration to pick a new Ghost username and path
ghost_project = projects.create!(
name: 'Ghost Project',
path: 'ghost',
visibility_level: 20,
archived: false,
namespace_id: fake_lost_and_found_group.id
)
routes.create!(
source_id: ghost_project.id,
source_type: 'Project',
path: 'ghost',
name: 'Ghost Project',
created_at: Time.current,
updated_at: Time.current
)
end
it 'fixes the ghost user username and namespace path' do
ghost_user = users.find_by(user_type: described_class::User::USER_TYPE_GHOST)
ghost_namespace = namespaces.find_by(owner_id: ghost_user.id)
expect(ghost_user.username).to eq('ghost')
expect(ghost_namespace.path).to eq('ghost')
disable_migrations_output { migrate! }
ghost_user = users.find_by(user_type: described_class::User::USER_TYPE_GHOST)
ghost_namespace = namespaces.find_by(owner_id: ghost_user.id)
ghost_namespace_route = routes.find_by(source_id: ghost_namespace.id, source_type: 'Namespace')
expect(ghost_user.username).to eq('ghost2')
expect(ghost_namespace.path).to eq('ghost2')
expect(ghost_namespace_route.path).to eq('ghost2')
end
it 'creates the route for the ghost user namespace' do
expect(routes.where(path: 'ghost').count).to eq(0)
expect(routes.where(path: 'ghost').count).to eq(1)
expect(routes.where(path: 'ghost1').count).to eq(1)
expect(routes.where(path: 'ghost2').count).to eq(0)
disable_migrations_output { migrate! }
expect(routes.where(path: 'ghost').count).to eq(1)
expect(routes.where(path: 'ghost1').count).to eq(1)
expect(routes.where(path: 'ghost2').count).to eq(1)
end
it 'fixes the path for the lost-and-found group by generating a unique one' do