realaravinth/debian-mirror-gitlab
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

New upstream version 13.1.4

Browse source
This commit is contained in:
Pirate Praveen 2020-07-10 23:44:40 +05:30
parent ff5ea591ca
commit b8c1f2cd92
25 changed files with 323 additions and 64 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
CHANGELOG.md
View file

@ -2,6 +2,16 @@
documentation](doc/development/changelog.md) for instructions on adding your own
entry.
## 13.1.4 (2020-07-09)
### Fixed (4 changes)
- Fix path conflict for Ghost on UpdateRoutesForLostAndFoundGroupAndOrphanedProjects. !35425
- Fix existing repository_storages_weighted migrations. !35814
- Fix error 500s creating new projects due to empty weights. !35829
- Fix gitlab:*:check Rake tasks. !35944
## 13.1.3 (2020-07-06)
- No changes.

2
GITALY_SERVER_VERSION
View file

@ -1 +1 @@
13.1.3
13.1.4

2
VERSION
View file

@ -1 +1 @@
13.1.3
13.1.4

2
app/controllers/projects/blob_controller.rb
View file

@ -30,7 +30,7 @@ class Projects::BlobController < Projects::ApplicationController
before_action :set_last_commit_sha, only: [:edit, :update]
before_action only: :show do
push_frontend_feature_flag(:code_navigation, @project)
push_frontend_feature_flag(:code_navigation, @project, default_enabled: true)
push_frontend_feature_flag(:suggest_pipeline) if experiment_enabled?(:suggest_pipeline)
end

2
app/controllers/projects/merge_requests_controller.rb
View file

@ -29,7 +29,7 @@ class Projects::MergeRequestsController < Projects::MergeRequests::ApplicationCo
push_frontend_feature_flag(:deploy_from_footer, @project, default_enabled: true)
push_frontend_feature_flag(:single_mr_diff_view, @project, default_enabled: true)
push_frontend_feature_flag(:suggest_pipeline) if experiment_enabled?(:suggest_pipeline)
push_frontend_feature_flag(:code_navigation, @project)
push_frontend_feature_flag(:code_navigation, @project, default_enabled: true)
push_frontend_feature_flag(:widget_visibility_polling, @project, default_enabled: true)
push_frontend_feature_flag(:merge_ref_head_comments, @project)
push_frontend_feature_flag(:mr_commit_neighbor_nav, @project, default_enabled: true)

26
db/migrate/20200508203901_add_repository_storages_weighted_to_application_settings.rb
View file

@ -3,11 +3,37 @@
class AddRepositoryStoragesWeightedToApplicationSettings < ActiveRecord::Migration[6.0]
DOWNTIME = false
class ApplicationSetting < ActiveRecord::Base
serialize :repository_storages
self.table_name = 'application_settings'
end
def up
add_column :application_settings, :repository_storages_weighted, :jsonb, default: {}, null: false
seed_repository_storages_weighted
end
def down
remove_column :application_settings, :repository_storages_weighted
end
private
def seed_repository_storages_weighted
# We need to flush the cache to ensure the newly-added column is loaded
ApplicationSetting.reset_column_information
# There should only be one row here due to
# 20200420162730_remove_additional_application_settings_rows.rb
ApplicationSetting.all.each do |settings|
storages = Gitlab.config.repositories.storages.keys.collect do |storage|
weight = settings.repository_storages.include?(storage) ? 100 : 0
[storage.to_sym, weight]
end
settings.repository_storages_weighted = Hash[storages]
settings.save!
end
end
end

37
db/migrate/20200509203901_reseed_repository_storages_weighted.rb Normal file
View file

@ -0,0 +1,37 @@
# frozen_string_literal: true
class ReseedRepositoryStoragesWeighted < ActiveRecord::Migration[6.0]
DOWNTIME = false
class ApplicationSetting < ActiveRecord::Base
serialize :repository_storages
self.table_name = 'application_settings'
end
def up
reseed_repository_storages_weighted
end
private
def reseed_repository_storages_weighted
# We need to flush the cache to ensure the newly-added column is loaded
ApplicationSetting.reset_column_information
# There should only be one row here due to
# 20200420162730_remove_additional_application_settings_rows.rb
ApplicationSetting.all.each do |settings|
# Admins may have already tweaked these values, so don't do anything
# if there is data already.
next if settings.repository_storages_weighted.present?
storages = Gitlab.config.repositories.storages.keys.collect do |storage|
weight = settings.repository_storages.include?(storage) ? 100 : 0
[storage.to_sym, weight]
end
settings.repository_storages_weighted = Hash[storages]
settings.save!
end
end
end

9
db/post_migrate/20200526000407_seed_repository_storages_weighted.rb
View file

@ -1,16 +1,23 @@
# frozen_string_literal: true
class SeedRepositoryStoragesWeighted < ActiveRecord::Migration[6.0]
DOWNTIME = false
class ApplicationSetting < ActiveRecord::Base
serialize :repository_storages
self.table_name = 'application_settings'
end
def up
# We need to flush the cache to ensure the newly-added column is loaded
ApplicationSetting.reset_column_information
# There should only be one row here due to
# 20200420162730_remove_additional_application_settings_rows.rb
ApplicationSetting.all.each do |settings|
storages = Gitlab.config.repositories.storages.keys.collect do |storage|
weight = settings.repository_storages.include?(storage) ? 100 : 0
[storage, weight]
[storage.to_sym, weight]
end
settings.repository_storages_weighted = Hash[storages]

10
db/post_migrate/20200602143020_update_routes_for_lost_and_found_group_and_orphaned_projects.rb
View file

@ -136,6 +136,7 @@ class UpdateRoutesForLostAndFoundGroupAndOrphanedProjects < ActiveRecord::Migrat
# to ensure the Active Record's knowledge of the table structure is current
Namespace.reset_column_information
Route.reset_column_information
User.reset_column_information
# Find the ghost user, its namespace and the "lost and found" group
ghost_user = User.ghost
@ -158,6 +159,15 @@ class UpdateRoutesForLostAndFoundGroupAndOrphanedProjects < ActiveRecord::Migrat
'More info: gitlab.com/gitlab-org/gitlab/-/issues/198603'
lost_and_found_group.save!
# make sure that the ghost namespace has a unique path
ghost_namespace.generate_unique_path
if ghost_namespace.path_changed?
ghost_namespace.save!
# If the path changed, also update the Ghost User's username to match the new path.
ghost_user.update!(username: ghost_namespace.path)
end
# Update the routes for the Ghost user, the "lost and found" group
# and all the orphaned projects
ghost_namespace.ensure_route!

1
db/structure.sql
View file

@ -13903,6 +13903,7 @@ COPY "schema_migrations" (version) FROM STDIN;
20200508091106
20200508140959
20200508203901
20200509203901
20200511080113
20200511083541
20200511092246

84
doc/administration/raketasks/doctor.md Normal file
View file

@ -0,0 +1,84 @@
# Doctor Rake tasks **(CORE ONLY)**
This is a collection of tasks to help investigate and repair
problems caused by data integrity issues.
## Verify database values can be decrypted using the current secrets
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/20069) in GitLab 13.1.
This task runs through all possible encrypted values in the
database, verifying that they are decryptable using the current
secrets file (`gitlab-secrets.json`).
Automatic resolution is not yet implemented. If you have values that
cannot be decrypted, you can follow steps to reset them, see our
docs on what to do [when the secrets file is lost](../../raketasks/backup_restore.md#when-the-secrets-file-is-lost).
NOTE: **Note:**
This can take a very long time, depending on the size of your
database, as it checks all rows in all tables.
**Omnibus Installation**
```shell
sudo gitlab-rake gitlab:doctor:secrets
```
**Source Installation**
```shell
bundle exec rake gitlab:doctor:secrets RAILS_ENV=production
```
**Example output**
<!-- vale gitlab.SentenceSpacing = NO -->
```plaintext
I, [2020-06-11T17:17:54.951815 #27148] INFO -- : Checking encrypted values in the database
I, [2020-06-11T17:18:12.677708 #27148] INFO -- : - ApplicationSetting failures: 0
I, [2020-06-11T17:18:12.823692 #27148] INFO -- : - User failures: 0
[...] other models possibly containing encrypted data
I, [2020-06-11T17:18:14.938335 #27148] INFO -- : - Group failures: 1
I, [2020-06-11T17:18:15.559162 #27148] INFO -- : - Operations::FeatureFlagsClient failures: 0
I, [2020-06-11T17:18:15.575533 #27148] INFO -- : - ScimOauthAccessToken failures: 0
I, [2020-06-11T17:18:15.575678 #27148] INFO -- : Total: 1 row(s) affected
I, [2020-06-11T17:18:15.575711 #27148] INFO -- : Done!
```
<!-- vale gitlab.SentenceSpacing = YES -->
### Verbose mode
In order to get more detailed information about which
rows and columns cannot be decrypted, you can pass a VERBOSE
environment variable:
**Omnibus Installation**
```shell
sudo gitlab-rake gitlab:doctor:secrets VERBOSE=1
```
**Source Installation**
```shell
bundle exec rake gitlab:doctor:secrets RAILS_ENV=production VERBOSE=1
```
**Example verbose output**
<!-- vale gitlab.SentenceSpacing = NO -->
```plaintext
I, [2020-06-11T17:17:54.951815 #27148] INFO -- : Checking encrypted values in the database
I, [2020-06-11T17:18:12.677708 #27148] INFO -- : - ApplicationSetting failures: 0
I, [2020-06-11T17:18:12.823692 #27148] INFO -- : - User failures: 0
[...] other models possibly containing encrypted data
D, [2020-06-11T17:19:53.224344 #27351] DEBUG -- : > Something went wrong for Group[10].runners_token: Validation failed: Route can't be blank
I, [2020-06-11T17:19:53.225178 #27351] INFO -- : - Group failures: 1
D, [2020-06-11T17:19:53.225267 #27351] DEBUG -- : - Group[10]: runners_token
I, [2020-06-11T17:18:15.559162 #27148] INFO -- : - Operations::FeatureFlagsClient failures: 0
I, [2020-06-11T17:18:15.575533 #27148] INFO -- : - ScimOauthAccessToken failures: 0
I, [2020-06-11T17:18:15.575678 #27148] INFO -- : Total: 1 row(s) affected
I, [2020-06-11T17:18:15.575711 #27148] INFO -- : Done!
```
<!-- vale gitlab.SentenceSpacing = YES -->

41
doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md
View file

@ -320,23 +320,7 @@ end
### Find mirrors with "bad decrypt" errors
```ruby
total = 0
bad = []
ProjectImportData.find_each do |data|
begin
total += 1
data.credentials
rescue => e
bad << data
end
end
puts "Bad count: #{bad.count} / #{total}"
bad.each do |repo|
puts Project.find(repo.project_id).full_path
end; bad.count
```
This content has been converted to a Rake task, see the [Doctor Rake tasks docs](../raketasks/doctor.md).
### Transfer mirror users and tokens to a single service account
@ -755,18 +739,9 @@ area on disk. It remains to be seen exactly how or whether the deletion is usefu
### Bad Decrypt Script (for encrypted variables)
See <https://gitlab.com/snippets/1730735/raw>.
This content has been converted to a Rake task, see the [Doctor Rake tasks docs](../raketasks/doctor.md).
This script will go through all the encrypted variables and count how many are not able
to be decrypted. Might be helpful to run on multiple nodes to see which `gitlab-secrets.json`
file is most up to date:
```shell
wget -O /tmp/bad-decrypt.rb https://gitlab.com/snippets/1730735/raw
gitlab-rails runner /tmp/bad-decrypt.rb
```
If `ProjectImportData Bad count:` is detected and the decision is made to delete the
As an example of repairing, if `ProjectImportData Bad count:` is detected and the decision is made to delete the
encrypted credentials to allow manual reentry:
```ruby
@ -797,16 +772,18 @@ encrypted credentials to allow manual reentry:
If `User OTP Secret Bad count:` is detected. For each user listed disable/enable
two-factor authentication.
### Decrypt Script for encrypted tokens
This script will search for all encrypted tokens that are causing decryption errors,
and update or reset as needed:
The following script will search in some of the tables for encrypted tokens that are
causing decryption errors, and update or reset as needed:
```shell
wget -O /tmp/encrypted-tokens.rb https://gitlab.com/snippets/1876342/raw
gitlab-rails runner /tmp/encrypted-tokens.rb
```
### Decrypt Script for encrypted tokens
This content has been converted to a Rake task, see the [Doctor Rake tasks docs](../raketasks/doctor.md).
## Geo
### Artifacts

1
doc/raketasks/README.md
View file

@ -20,6 +20,7 @@ The following are available Rake tasks:
| [Back up and restore](backup_restore.md) | Back up, restore, and migrate GitLab instances between servers. |
| [Clean up](cleanup.md) | Clean up unneeded items from GitLab instances. |
| [Development](../development/rake_tasks.md) | Tasks for GitLab contributors. |
| [Doctor tasks](../administration/raketasks/doctor.md) | Checks for data integrity issues. |
| [Elasticsearch](../integration/elasticsearch.md#gitlab-elasticsearch-rake-tasks) **(STARTER ONLY)** | Maintain Elasticsearch in a GitLab instance. |
| [Enable namespaces](features.md) | Enable usernames and namespaces for user projects. |
| [General maintenance](../administration/raketasks/maintenance.md) | General maintenance and self-check tasks. |

3
doc/raketasks/backup_restore.md
View file

@ -941,6 +941,9 @@ experience some unexpected behavior such as:
- Stuck jobs.
- 500 errors.
You can check whether you have undecryptable values in the database using
the [Secrets Doctor Rake task](../administration/raketasks/doctor.md).
In this case, you are required to reset all the tokens for CI/CD variables
and Runner Authentication, which is described in more detail below. After
resetting the tokens, you should be able to visit your project and the jobs

3
lib/gitlab/verify/batch_verifier.rb
View file

@ -71,10 +71,9 @@ module Gitlab
# It's already set to Logger::INFO, but acts as if it is set to
# Logger::DEBUG, and this fixes it...
def fix_google_api_logger
if Object.const_defined?('Google::Apis')
require 'google/apis'
Google::Apis.logger.level = Logger::INFO
end
end
# This should return an ActiveRecord::Relation suitable for calling #in_batches on
def all_relation

2
spec/features/merge_request/maintainer_edits_fork_spec.rb
View file

@ -20,7 +20,7 @@ RSpec.describe 'a maintainer edits files on a source-branch of an MR from a fork
end
before do
stub_feature_flags(web_ide_default: false, single_mr_diff_view: false, code_navigation: false)
stub_feature_flags(web_ide_default: false, single_mr_diff_view: false)
target_project.add_maintainer(user)
sign_in(user)

4
spec/features/projects/blobs/blob_show_spec.rb
View file

@ -13,10 +13,6 @@ RSpec.describe 'File blob', :js do
wait_for_requests
end
before do
stub_feature_flags(code_navigation: false)
end
context 'Ruby file' do
before do
visit_blob('files/ruby/popen.rb')

2
spec/features/projects/blobs/edit_spec.rb
View file

@ -69,8 +69,6 @@ RSpec.describe 'Editing file blob', :js do
context 'from blob file path' do
before do
stub_feature_flags(code_navigation: false)
visit project_blob_path(project, tree_join(branch, file_path))
end

1
spec/features/projects/blobs/user_creates_new_blob_in_new_project_spec.rb
View file

@ -8,7 +8,6 @@ RSpec.describe 'User creates blob in new project', :js do
shared_examples 'creating a file' do
before do
stub_feature_flags(code_navigation: false)
sign_in(user)
visit project_path(project)
end

2
spec/features/projects/files/user_creates_files_spec.rb
View file

@ -14,7 +14,7 @@ RSpec.describe 'Projects > Files > User creates files', :js do
let(:user) { create(:user) }
before do
stub_feature_flags(web_ide_default: false, code_navigation: false)
stub_feature_flags(web_ide_default: false)
project.add_maintainer(user)
sign_in(user)

2
spec/features/projects/files/user_deletes_files_spec.rb
View file

@ -14,8 +14,6 @@ RSpec.describe 'Projects > Files > User deletes files', :js do
let(:user) { create(:user) }
before do
stub_feature_flags(code_navigation: false)
sign_in(user)
end

2
spec/features/projects/files/user_replaces_files_spec.rb
View file

@ -16,8 +16,6 @@ RSpec.describe 'Projects > Files > User replaces files', :js do
let(:user) { create(:user) }
before do
stub_feature_flags(code_navigation: false)
sign_in(user)
end

31
spec/migrations/add_repository_storages_weighted_to_application_settings_spec.rb Normal file
View file

@ -0,0 +1,31 @@
# frozen_string_literal: true
require 'spec_helper'
require Rails.root.join('db', 'migrate', '20200508203901_add_repository_storages_weighted_to_application_settings.rb')
RSpec.describe AddRepositoryStoragesWeightedToApplicationSettings, :migration do
let(:storages) { { "foo" => {}, "baz" => {} } }
let(:application_settings) do
table(:application_settings).tap do |klass|
klass.class_eval do
serialize :repository_storages
end
end
end
before do
allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
end
let(:application_setting) { application_settings.create! }
let(:repository_storages) { ["foo"] }
it 'populates repository_storages_weighted properly' do
application_setting.repository_storages = repository_storages
application_setting.save!
migrate!
expect(application_settings.find(application_setting.id).repository_storages_weighted).to eq({ "foo" => 100, "baz" => 0 })
end
end

43
spec/migrations/reseed_repository_storages_weighted_spec.rb Normal file
View file

@ -0,0 +1,43 @@
# frozen_string_literal: true
require 'spec_helper'
require Rails.root.join('db', 'migrate', '20200509203901_reseed_repository_storages_weighted.rb')
RSpec.describe ReseedRepositoryStoragesWeighted do
let(:storages) { { "foo" => {}, "baz" => {} } }
let(:application_settings) do
table(:application_settings).tap do |klass|
klass.class_eval do
serialize :repository_storages
end
end
end
before do
allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
end
let(:repository_storages) { ["foo"] }
let!(:application_setting) { application_settings.create!(repository_storages: repository_storages) }
context 'with empty repository_storages_weighted column' do
it 'populates repository_storages_weighted properly' do
migrate!
expect(application_settings.find(application_setting.id).repository_storages_weighted).to eq({ "foo" => 100, "baz" => 0 })
end
end
context 'with already-populated repository_storages_weighted column' do
let(:existing_weights) { { "foo" => 100, "baz" => 50 } }
it 'does not change repository_storages_weighted properly' do
application_setting.repository_storages_weighted = existing_weights
application_setting.save!
migrate!
expect(application_settings.find(application_setting.id).repository_storages_weighted).to eq(existing_weights)
end
end
end

63
spec/migrations/update_routes_for_lost_and_found_group_and_orphaned_projects_spec.rb
View file

@ -62,8 +62,8 @@ describe UpdateRoutesForLostAndFoundGroupAndOrphanedProjects, :migration do
source_type: 'Project',
path: 'orphaned_project',
name: 'orphaned_project',
created_at: Time.zone.now,
updated_at: Time.zone.now
created_at: Time.current,
updated_at: Time.current
)
# Create another user named ghost which is not the Ghost User
@ -90,10 +90,10 @@ describe UpdateRoutesForLostAndFoundGroupAndOrphanedProjects, :migration do
routes.create!(
source_id: fake_ghost_user_namespace.id,
source_type: 'Namespace',
path: 'Ghost User',
name: 'ghost1',
created_at: Time.zone.now,
updated_at: Time.zone.now
path: 'ghost1',
name: 'Ghost User',
created_at: Time.current,
updated_at: Time.current
)
fake_lost_and_found_group = namespaces.create!(
@ -109,8 +109,8 @@ describe UpdateRoutesForLostAndFoundGroupAndOrphanedProjects, :migration do
source_type: 'Namespace',
path: described_class::User::LOST_AND_FOUND_GROUP, # same path as the lost-and-found group
name: 'Lost and Found',
created_at: Time.zone.now,
updated_at: Time.zone.now
created_at: Time.current,
updated_at: Time.current
)
members.create!(
@ -135,17 +135,58 @@ describe UpdateRoutesForLostAndFoundGroupAndOrphanedProjects, :migration do
source_type: 'Project',
path: "#{described_class::User::LOST_AND_FOUND_GROUP}/normal_project",
name: 'Lost and Found / normal_project',
created_at: Time.zone.now,
updated_at: Time.zone.now
created_at: Time.current,
updated_at: Time.current
)
# Add a project whose route conflicts with the ghost username
# and should force the data migration to pick a new Ghost username and path
ghost_project = projects.create!(
name: 'Ghost Project',
path: 'ghost',
visibility_level: 20,
archived: false,
namespace_id: fake_lost_and_found_group.id
)
routes.create!(
source_id: ghost_project.id,
source_type: 'Project',
path: 'ghost',
name: 'Ghost Project',
created_at: Time.current,
updated_at: Time.current
)
end
it 'fixes the ghost user username and namespace path' do
ghost_user = users.find_by(user_type: described_class::User::USER_TYPE_GHOST)
ghost_namespace = namespaces.find_by(owner_id: ghost_user.id)
expect(ghost_user.username).to eq('ghost')
expect(ghost_namespace.path).to eq('ghost')
disable_migrations_output { migrate! }
ghost_user = users.find_by(user_type: described_class::User::USER_TYPE_GHOST)
ghost_namespace = namespaces.find_by(owner_id: ghost_user.id)
ghost_namespace_route = routes.find_by(source_id: ghost_namespace.id, source_type: 'Namespace')
expect(ghost_user.username).to eq('ghost2')
expect(ghost_namespace.path).to eq('ghost2')
expect(ghost_namespace_route.path).to eq('ghost2')
end
it 'creates the route for the ghost user namespace' do
expect(routes.where(path: 'ghost').count).to eq(0)
expect(routes.where(path: 'ghost').count).to eq(1)
expect(routes.where(path: 'ghost1').count).to eq(1)
expect(routes.where(path: 'ghost2').count).to eq(0)
disable_migrations_output { migrate! }
expect(routes.where(path: 'ghost').count).to eq(1)
expect(routes.where(path: 'ghost1').count).to eq(1)
expect(routes.where(path: 'ghost2').count).to eq(1)
end
it 'fixes the path for the lost-and-found group by generating a unique one' do