Update upstream source from tag 'upstream/11.8.9'

Update to upstream version '11.8.9'
with Debian dir bff64168d3

.gitlab-ci.yml

@@ -315,7 +315,7 @@ cloud-native-image:
   variables:
     GIT_DEPTH: "1"
   cache: {}
-  when: always
+  when: manual
   script:
     - gem install gitlab --no-document
     - CNG_PROJECT_PATH="gitlab-org/build/CNG" BUILD_TRIGGER_TOKEN=$CI_JOB_TOKEN ./scripts/trigger-build cng

CHANGELOG.md

@@ -2,6 +2,64 @@
 documentation](doc/development/changelog.md) for instructions on adding your own
 entry.
 
+## 11.8.9 (2019-04-25)
+
+### Security (5 changes)
+
+- Improve credentials sanitization on repository mirror integration. !3078
+- Stop sending emails to users who can't read commit.
+- Escape path in new merge request mail.
+- Only allow modification of content when note is edited.
+- Upgrade Rails to 5.0.7.2.
+
+
+## 11.8.8 (2019-04-23)
+
+### Fixed (5 changes)
+
+- Bring back Rugged implementation of find_commit. !25477
+- Fix bug in BitBucket imports with SHA shorter than 40 chars. !26050
+- Fix health checks not working behind load balancers. !26055
+- Fix error creating a merge request when diff includes a null byte. !26190
+- Avoid excessive recursive calls with Rugged TreeEntries. !26813
+
+### Performance (1 change)
+
+- Bring back Rugged implementation of ListCommitsByOid. !27441
+
+### Other (4 changes)
+
+- Bring back Rugged implementation of GetTreeEntries. !25674
+- Bring back Rugged implementation of CommitIsAncestor. !25702
+- Bring back Rugged implementation of TreeEntry. !25706
+- Bring back Rugged implementation of commit_tree_entry. !25896
+
+
+## 11.8.7 (2019-04-09)
+
+- No changes.
+
+## 11.8.6 (2019-03-28)
+
+### Security (7 changes)
+
+- Disallow guest users from accessing Releases.
+- Fix PDF.js vulnerability.
+- Hide "related branches" when user does not have permission.
+- Fix XSS in resolve conflicts form.
+- Added rake task for removing EXIF data from existing uploads.
+- Disallow updating namespace when updating a project.
+- Use UntrustedRegexp for matching refs policy.
+
+
+## 11.8.5 (2019-03-27)
+
+- Unreleased due to QA failure.
+
+## 11.8.4 (2019-03-26)
+
+- Unreleased due to QA failure.
+
 ## 11.8.3 (2019-03-19)
 
 ### Security (1 change)

GITALY_SERVER_VERSION

@@ -1 +1 @@
-1.20.0
+1.20.1

GITLAB_WORKHORSE_VERSION

@@ -1 +1 @@
-8.3.1
+8.3.3

Gemfile

@@ -1,6 +1,6 @@
 source 'https://rubygems.org'
 
-gem 'rails', '5.0.7.1'
+gem 'rails', '5.0.7.2'
 gem 'rails-deprecated_sanitizer', '~> 1.0.3'
 
 # Improves copy-on-write performance for MRI

Gemfile.lock

@@ -4,41 +4,41 @@ GEM
     RedCloth (4.3.2)
     abstract_type (0.0.7)
     ace-rails-ap (4.1.2)
-    actioncable (5.0.7.1)
+    actioncable (5.0.7.2)
-      actionpack (= 5.0.7.1)
+      actionpack (= 5.0.7.2)
       nio4r (>= 1.2, < 3.0)
       websocket-driver (~> 0.6.1)
-    actionmailer (5.0.7.1)
+    actionmailer (5.0.7.2)
-      actionpack (= 5.0.7.1)
+      actionpack (= 5.0.7.2)
-      actionview (= 5.0.7.1)
+      actionview (= 5.0.7.2)
-      activejob (= 5.0.7.1)
+      activejob (= 5.0.7.2)
       mail (~> 2.5, >= 2.5.4)
       rails-dom-testing (~> 2.0)
-    actionpack (5.0.7.1)
+    actionpack (5.0.7.2)
-      actionview (= 5.0.7.1)
+      actionview (= 5.0.7.2)
-      activesupport (= 5.0.7.1)
+      activesupport (= 5.0.7.2)
       rack (~> 2.0)
       rack-test (~> 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.0.2)
-    actionview (5.0.7.1)
+    actionview (5.0.7.2)
-      activesupport (= 5.0.7.1)
+      activesupport (= 5.0.7.2)
       builder (~> 3.1)
       erubis (~> 2.7.0)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.0.3)
-    activejob (5.0.7.1)
+    activejob (5.0.7.2)
-      activesupport (= 5.0.7.1)
+      activesupport (= 5.0.7.2)
       globalid (>= 0.3.6)
-    activemodel (5.0.7.1)
+    activemodel (5.0.7.2)
-      activesupport (= 5.0.7.1)
+      activesupport (= 5.0.7.2)
-    activerecord (5.0.7.1)
+    activerecord (5.0.7.2)
-      activemodel (= 5.0.7.1)
+      activemodel (= 5.0.7.2)
-      activesupport (= 5.0.7.1)
+      activesupport (= 5.0.7.2)
       arel (~> 7.0)
     activerecord_sane_schema_dumper (1.0)
       rails (>= 5, < 6)
-    activesupport (5.0.7.1)
+    activesupport (5.0.7.2)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
@@ -295,7 +295,7 @@ GEM
       omniauth (~> 1.3)
       pyu-ruby-sasl (>= 0.0.3.3, < 0.1)
       rubyntlm (~> 0.5)
-    globalid (0.4.1)
+    globalid (0.4.2)
       activesupport (>= 4.2.0)
     gon (6.2.0)
       actionpack (>= 3.0)
@@ -385,7 +385,7 @@ GEM
       json (~> 1.8)
       multi_xml (>= 0.5.2)
     httpclient (2.8.3)
-    i18n (1.2.0)
+    i18n (1.6.0)
       concurrent-ruby (~> 1.0)
     icalendar (2.4.1)
     ice_nine (0.11.2)
@@ -636,17 +636,17 @@ GEM
       rack
     rack-test (0.6.3)
       rack (>= 1.0)
-    rails (5.0.7.1)
+    rails (5.0.7.2)
-      actioncable (= 5.0.7.1)
+      actioncable (= 5.0.7.2)
-      actionmailer (= 5.0.7.1)
+      actionmailer (= 5.0.7.2)
-      actionpack (= 5.0.7.1)
+      actionpack (= 5.0.7.2)
-      actionview (= 5.0.7.1)
+      actionview (= 5.0.7.2)
-      activejob (= 5.0.7.1)
+      activejob (= 5.0.7.2)
-      activemodel (= 5.0.7.1)
+      activemodel (= 5.0.7.2)
-      activerecord (= 5.0.7.1)
+      activerecord (= 5.0.7.2)
-      activesupport (= 5.0.7.1)
+      activesupport (= 5.0.7.2)
       bundler (>= 1.3.0)
-      railties (= 5.0.7.1)
+      railties (= 5.0.7.2)
       sprockets-rails (>= 2.0.0)
     rails-controller-testing (1.0.2)
       actionpack (~> 5.x, >= 5.0.1)
@@ -662,9 +662,9 @@ GEM
     rails-i18n (5.1.1)
       i18n (>= 0.7, < 2)
       railties (>= 5.0, < 6)
-    railties (5.0.7.1)
+    railties (5.0.7.2)
-      actionpack (= 5.0.7.1)
+      actionpack (= 5.0.7.2)
-      activesupport (= 5.0.7.1)
+      activesupport (= 5.0.7.2)
       method_source
       rake (>= 0.8.7)
       thor (>= 0.18.1, < 2.0)
@@ -1110,7 +1110,7 @@ DEPENDENCIES
   rack-cors (~> 1.0.0)
   rack-oauth2 (~> 1.2.1)
   rack-proxy (~> 0.6.0)
-  rails (= 5.0.7.1)
+  rails (= 5.0.7.2)
   rails-controller-testing
   rails-deprecated_sanitizer (~> 1.0.3)
   rails-i18n (~> 5.1)

VERSION

@@ -1 +1 @@
-11.8.3
+11.8.9

app/assets/javascripts/issue.js

@@ -16,7 +16,9 @@ export default class Issue {
     Issue.createMrDropdownWrap = document.querySelector('.create-mr-dropdown-wrap');
 
     Issue.initMergeRequests();
-    Issue.initRelatedBranches();
+    if (document.querySelector('#related-branches')) {
+      Issue.initRelatedBranches();
+    }
 
     this.closeButtons = $('a.btn-close');
     this.reopenButtons = $('a.btn-reopen');

app/assets/javascripts/pdf/index.vue

@@ -28,7 +28,7 @@ export default {
   },
   watch: { pdf: 'load' },
   mounted() {
-    pdfjsLib.PDFJS.workerSrc = workerSrc;
+    pdfjsLib.GlobalWorkerOptions.workerSrc = workerSrc;
     if (this.hasPDF) this.load();
   },
   methods: {

app/assets/javascripts/pipelines/pipeline_details_mediator.js

@@ -19,6 +19,7 @@ export default class pipelinesMediator {
     this.poll = new Poll({
       resource: this.service,
       method: 'getPipeline',
+      data: this.store.state.expandedPipelines ? this.getExpandedParameters() : undefined,
       successCallback: this.successCallback.bind(this),
       errorCallback: this.errorCallback.bind(this),
     });
@@ -56,6 +57,19 @@ export default class pipelinesMediator {
       .getPipeline()
       .then(response => this.successCallback(response))
       .catch(() => this.errorCallback())
-      .finally(() => this.poll.restart());
+      .finally(() =>
+        this.poll.restart(
+          this.store.state.expandedPipelines ? this.getExpandedParameters() : undefined,
+        ),
+      );
+  }
+
+  /**
+   * Backend expects paramets in the following format: `expanded[]=id&expanded[]=id`
+   */
+  getExpandedParameters() {
+    return {
+      expanded: this.store.state.expandedPipelines,
+    };
   }
 }

app/assets/javascripts/pipelines/services/pipeline_service.js

@@ -5,8 +5,8 @@ export default class PipelineService {
     this.pipeline = endpoint;
   }
 
-  getPipeline() {
+  getPipeline(params) {
-    return axios.get(this.pipeline);
+    return axios.get(this.pipeline, { params });
   }
 
   // eslint-disable-next-line class-methods-use-this

app/controllers/concerns/notes_actions.rb

@@ -78,7 +78,7 @@ module NotesActions
 
   # rubocop:disable Gitlab/ModuleWithInstanceVariables
   def update
-    @note = Notes::UpdateService.new(project, current_user, note_params).execute(note)
+    @note = Notes::UpdateService.new(project, current_user, update_note_params).execute(note)
     prepare_notes_for_rendering([@note])
 
     respond_to do |format|
@@ -216,6 +216,10 @@ module NotesActions
     )
   end
 
+  def update_note_params
+    params.require(:note).permit(:note)
+  end
+
   def set_polling_interval_header
     Gitlab::PollingInterval.set_header(response, interval: 6_000)
   end

app/controllers/projects/issues_controller.rb

@@ -39,6 +39,7 @@ class Projects::IssuesController < Projects::ApplicationController
   before_action :authorize_create_merge_request_from!, only: [:create_merge_request]
 
   before_action :authorize_import_issues!, only: [:import_csv]
+  before_action :authorize_download_code!, only: [:related_branches]
 
   before_action :set_suggested_issues_feature_flags, only: [:new]
 

app/controllers/projects_controller.rb

@@ -47,7 +47,7 @@ class ProjectsController < Projects::ApplicationController
   end
 
   def create
-    @project = ::Projects::CreateService.new(current_user, project_params).execute
+    @project = ::Projects::CreateService.new(current_user, project_params(attributes: project_params_create_attributes)).execute
 
     if @project.saved?
       cookies[:issue_board_welcome_hidden] = { path: project_path(@project), value: nil, expires: Time.at(0) }
@@ -328,9 +328,9 @@ class ProjectsController < Projects::ApplicationController
   end
   # rubocop: enable CodeReuse/ActiveRecord
 
-  def project_params
+  def project_params(attributes: [])
     params.require(:project)
-      .permit(project_params_attributes)
+      .permit(project_params_attributes + attributes)
   end
 
   def project_params_attributes
@@ -349,11 +349,10 @@ class ProjectsController < Projects::ApplicationController
       :last_activity_at,
       :lfs_enabled,
       :name,
-      :namespace_id,
       :only_allow_merge_if_all_discussions_are_resolved,
       :only_allow_merge_if_pipeline_succeeds,
-      :printing_merge_request_link_enabled,
       :path,
+      :printing_merge_request_link_enabled,
       :public_builds,
       :request_access_enabled,
       :runners_token,
@@ -375,6 +374,10 @@ class ProjectsController < Projects::ApplicationController
     ]
   end
 
+  def project_params_create_attributes
+    [:namespace_id]
+  end
+
   def custom_import_params
     {}
   end

app/helpers/tree_helper.rb

@@ -136,18 +136,9 @@ module TreeHelper
   end
 
   # returns the relative path of the first subdir that doesn't have only one directory descendant
-  # rubocop: disable CodeReuse/ActiveRecord
   def flatten_tree(root_path, tree)
-    return tree.flat_path.sub(%r{\A#{Regexp.escape(root_path)}/}, '') if tree.flat_path.present?
+    tree.flat_path.sub(%r{\A#{Regexp.escape(root_path)}/}, '')
-
-    subtree = Gitlab::Git::Tree.where(@repository, @commit.id, tree.path)
-    if subtree.count == 1 && subtree.first.dir?
-      return tree_join(tree.name, flatten_tree(root_path, subtree.first))
-    else
-      return tree.name
-    end
   end
-  # rubocop: enable CodeReuse/ActiveRecord
 
   def selected_branch
     @branch_name || tree_edit_branch

app/models/application_record.rb

@@ -7,6 +7,19 @@ class ApplicationRecord < ActiveRecord::Base
     where(id: ids)
   end
 
+  def self.safe_ensure_unique(retries: 0)
+    transaction(requires_new: true) do
+      yield
+    end
+  rescue ActiveRecord::RecordNotUnique
+    if retries > 0
+      retries -= 1
+      retry
+    end
+
+    false
+  end
+
   def self.safe_find_or_create_by!(*args)
     safe_find_or_create_by(*args).tap do |record|
       record.validate! unless record.persisted?
@@ -14,10 +27,8 @@ class ApplicationRecord < ActiveRecord::Base
   end
 
   def self.safe_find_or_create_by(*args)
-    transaction(requires_new: true) do
+    safe_ensure_unique(retries: 1) do
       find_or_create_by(*args)
     end
-  rescue ActiveRecord::RecordNotUnique
-    retry
   end
 end

app/models/label.rb

@@ -126,6 +126,10 @@ class Label < ActiveRecord::Base
     fuzzy_search(query, [:title, :description])
   end
 
+  def self.by_ids(ids)
+    where(id: ids)
+  end
+
   def open_issues_count(user = nil)
     issues_count(user, state: 'opened')
   end

app/models/merge_request_diff.rb

@@ -301,6 +301,11 @@ class MergeRequestDiff < ActiveRecord::Base
 
   private
 
+  def encode_in_base64?(diff_text)
+    (diff_text.encoding == Encoding::BINARY && !diff_text.ascii_only?) ||
+      diff_text.include?("\0")
+  end
+
   def create_merge_request_diff_files(diffs)
     rows =
       if has_attribute?(:external_diff) && Gitlab.config.external_diffs.enabled
@@ -353,7 +358,7 @@ class MergeRequestDiff < ActiveRecord::Base
       diff_hash.tap do |hash|
         diff_text = hash[:diff]
 
-        if diff_text.encoding == Encoding::BINARY && !diff_text.ascii_only?
+        if encode_in_base64?(diff_text)
           hash[:binary] = true
           hash[:diff] = [diff_text].pack('m0')
         end

app/models/notification_recipient.rb

@@ -123,15 +123,19 @@ class NotificationRecipient
     return @read_ability if instance_variable_defined?(:@read_ability)
 
     @read_ability =
-      case @target
+      if @target.is_a?(Ci::Pipeline)
-      when Issuable
-        :"read_#{@target.to_ability_name}"
-      when Ci::Pipeline
         :read_build # We have build trace in pipeline emails
-      when ActiveRecord::Base
+      elsif default_ability_for_target
-        :"read_#{@target.class.model_name.name.underscore}"
+        :"read_#{default_ability_for_target}"
-      else
+      end
-        nil
+  end
+
+  def default_ability_for_target
+    @default_ability_for_target ||=
+      if @target.respond_to?(:to_ability_name)
+        @target.to_ability_name
+      elsif @target.class.respond_to?(:model_name)
+        @target.class.model_name.name.underscore
       end
   end
 

app/policies/project_policy.rb

@@ -178,7 +178,6 @@ class ProjectPolicy < BasePolicy
     enable :read_cycle_analytics
     enable :award_emoji
     enable :read_pages_content
-    enable :read_release
   end
 
   # These abilities are not allowed to admins that are not members of the project,
@@ -204,6 +203,7 @@ class ProjectPolicy < BasePolicy
     enable :read_deployment
     enable :read_merge_request
     enable :read_sentry_issue
+    enable :read_release
   end
 
   # We define `:public_user_access` separately because there are cases in gitlab-ee

app/services/issuable_base_service.rb

@@ -70,10 +70,14 @@ class IssuableBaseService < BaseService
   end
 
   def filter_labels
-    filter_labels_in_param(:add_label_ids)
+    params[:add_label_ids] = labels_service.filter_labels_ids_in_param(:add_label_ids) if params[:add_label_ids]
-    filter_labels_in_param(:remove_label_ids)
+    params[:remove_label_ids] = labels_service.filter_labels_ids_in_param(:remove_label_ids) if params[:remove_label_ids]
-    filter_labels_in_param(:label_ids)
+
-    find_or_create_label_ids
+    if params[:label_ids]
+      params[:label_ids] = labels_service.filter_labels_ids_in_param(:label_ids)
+    elsif params[:labels]
+      params[:label_ids] = labels_service.find_or_create_by_titles.map(&:id)
+    end
   end
 
   # rubocop: disable CodeReuse/ActiveRecord
@@ -101,6 +105,10 @@ class IssuableBaseService < BaseService
     end.compact
   end
 
+  def labels_service
+    @labels_service ||= ::Labels::AvailableLabelsService.new(current_user, parent, params)
+  end
+
   def process_label_ids(attributes, existing_label_ids: nil)
     label_ids = attributes.delete(:label_ids)
     add_label_ids = attributes.delete(:add_label_ids)
@@ -118,10 +126,6 @@ class IssuableBaseService < BaseService
     new_label_ids
   end
 
-  def available_labels
-    @available_labels ||= LabelsFinder.new(current_user, project_id: @project.id, include_ancestor_groups: true).execute
-  end
-
   def handle_quick_actions_on_create(issuable)
     merge_quick_actions_into_params!(issuable)
   end

app/services/labels/available_labels_service.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+module Labels
+  class AvailableLabelsService
+    attr_reader :current_user, :parent, :params
+
+    def initialize(current_user, parent, params)
+      @current_user = current_user
+      @parent = parent
+      @params = params
+    end
+
+    def find_or_create_by_titles
+      labels = params.delete(:labels)
+
+      return [] unless labels
+
+      labels = labels.split(',') if labels.is_a?(String)
+
+      labels.map do |label_name|
+        label = Labels::FindOrCreateService.new(
+          current_user,
+          parent,
+          include_ancestor_groups: true,
+          title: label_name.strip,
+          available_labels: available_labels
+        ).execute
+
+        label
+      end.compact
+    end
+
+    def filter_labels_ids_in_param(key)
+      return [] if params[key].to_a.empty?
+
+      # rubocop:disable CodeReuse/ActiveRecord
+      available_labels.by_ids(params[key]).pluck(:id)
+      # rubocop:enable CodeReuse/ActiveRecord
+    end
+
+    private
+
+    def available_labels
+      @available_labels ||= LabelsFinder.new(current_user, finder_params).execute
+    end
+
+    def finder_params
+      params = { include_ancestor_groups: true }
+
+      case parent
+      when Group
+        params[:group_id] = parent.id
+        params[:only_group_labels] = true
+      when Project
+        params[:project_id] = parent.id
+      end
+
+      params
+    end
+  end
+end

app/validators/sha_validator.rb

@@ -2,7 +2,7 @@
 
 class ShaValidator < ActiveModel::EachValidator
   def validate_each(record, attribute, value)
-    return if value.blank? || value.match(/\A\h{40}\z/)
+    return if value.blank? || Commit.valid_hash?(value)
 
     record.errors.add(attribute, 'is not a valid SHA')
   end

app/views/notify/new_merge_request_email.html.haml

@@ -3,7 +3,7 @@
     #{link_to @merge_request.author_name, user_url(@merge_request.author)} created a merge request:
 
 %p.details
-  != merge_path_description(@merge_request, '→')
+  = merge_path_description(@merge_request, '→')
 
 - if @merge_request.assignee_id.present?
   %p

app/views/projects/buttons/_clone.html.haml

@@ -7,7 +7,7 @@
     = sprite_icon("arrow-down", css_class: "icon")
   %ul.p-3.dropdown-menu.dropdown-menu-right.dropdown-menu-large.dropdown-menu-selectable.clone-options-dropdown.qa-clone-options
     - if ssh_enabled?
-      %li.pb-2
+      %li
         %label.label-bold
           = _('Clone with SSH')
         .input-group
@@ -16,7 +16,7 @@
             = clipboard_button(target: '#ssh_project_clone', title: _("Copy URL to clipboard"), class: "input-group-text btn-default btn-clipboard")
             = render_if_exists 'projects/buttons/geo'
     - if http_enabled?
-      %li
+      %li.pt-2
         %label.label-bold
           = _('Clone with %{http_label}') % { http_label: gitlab_config.protocol.upcase }
         .input-group
@@ -24,5 +24,6 @@
           .input-group-append
             = clipboard_button(target: '#http_project_clone', title: _("Copy URL to clipboard"), class: "input-group-text btn-default btn-clipboard")
             = render_if_exists 'projects/buttons/geo'
+    = render_if_exists 'projects/buttons/kerberos_clone_field'
 
 = render_if_exists 'shared/geo_info_modal', project: project

app/views/projects/issues/show.html.haml

@@ -77,8 +77,9 @@
     #merge-requests{ data: { url: referenced_merge_requests_project_issue_path(@project, @issue) } }
       // This element is filled in using JavaScript.
 
-    #related-branches{ data: { url: related_branches_project_issue_path(@project, @issue) } }
+    - if can?(current_user, :download_code, @project)
-      // This element is filled in using JavaScript.
+      #related-branches{ data: { url: related_branches_project_issue_path(@project, @issue) } }
+        // This element is filled in using JavaScript.
 
   .content-block.emoji-block.emoji-block-sticky
     .row

app/views/projects/merge_requests/conflicts/_submit_form.html.haml

@@ -6,7 +6,7 @@
   .form-group.row
     .col-md-4
       %h4= _('Resolve conflicts on source branch')
-      .resolve-info
+      .resolve-info{ "v-pre": true }
         = translation.html_safe
     .col-md-8
       %label.label-bold{ "for" => "commit-message" }

app/views/shared/_mobile_clone_panel.html.haml

@@ -13,3 +13,4 @@
     - if http_enabled?
       %li
         = dropdown_item_with_description(http_copy_label, project.http_url_to_repo, href: project.http_url_to_repo, data: { clone_type: 'http' })
+    = render_if_exists 'shared/mobile_kerberos_clone'

doc/administration/high_availability/nfs.md

@@ -37,6 +37,28 @@ options:
   circumstances it could lead to data loss if a failure occurs before data has
   synced.
 
+### Improving NFS performance with GitLab
+
+If you are using NFS to share Git data, we recommend that you enable a
+number of feature flags that will allow GitLab application processes to
+access Git data directly instead of going through the [Gitaly
+service](../gitaly/index.md). Depending on your workload and disk
+performance, these flags may help improve performance. See [the
+issue](https://gitlab.com/gitlab-org/gitlab-ce/issues/57317) for more
+details.
+
+To do this, run the Rake task:
+
+```sh
+gitlab-rake gitlab:features:enable_rugged
+```
+
+If you need to undo this setting for some reason, run:
+
+```sh
+gitlab-rake gitlab:features:disable_rugged
+```
+
 ### Known issues
 
 On some customer systems, we have seen NFS clients slow precipitously due to

doc/administration/raketasks/uploads/sanitize.md

@@ -0,0 +1,62 @@
+# Uploads Sanitize tasks
+
+## Requirements
+
+You need `exiftool` installed on your system. If you installed GitLab:
+
+-   Using the Omnibus package, you're all set.
+-   From source, make sure `exiftool` is installed:
+
+    ```sh
+    # Debian/Ubuntu
+    sudo apt-get install libimage-exiftool-perl
+
+    # RHEL/CentOS
+    sudo yum install perl-Image-ExifTool
+    ```
+
+## Remove EXIF data from existing uploads
+
+Since 11.9 EXIF data are automatically stripped from JPG or TIFF image uploads.
+Because EXIF data may contain sensitive information (e.g. GPS location), you
+can remove EXIF data also from existing images which were uploaded before
+with the following command:
+
+```bash
+sudo RAILS_ENV=production -u git -H bundle exec rake gitlab:uploads:sanitize:remove_exif
+```
+
+This command by default runs in dry mode and it doesn't remove EXIF data. It can be used for
+checking if (and how many) images should be sanitized.
+
+The rake task accepts following parameters.
+
+Parameter | Type | Description
+--------- | ---- | -----------
+`start_id` | integer | Only uploads with equal or greater ID will be processed
+`stop_id` | integer | Only uploads with equal or smaller ID will be processed
+`dry_run` | boolean | Do not remove EXIF data, only check if EXIF data are present or not, default: true
+`sleep_time` | float | Pause for number of seconds after processing each image, default: 0.3 seconds
+
+If you have too many uploads, you can speed up sanitization by setting
+`sleep_time` to a lower value or by running multiple rake tasks in parallel,
+each with a separate range of upload IDs (by setting `start_id` and `stop_id`).
+
+To run the command without dry mode and remove EXIF data from all uploads, you can use:
+
+```bash
+sudo RAILS_ENV=production -u git -H bundle exec rake gitlab:uploads:sanitize:remove_exif[,,false,] 2>&1 | tee exif.log
+```
+
+To run the command without dry mode on uploads with ID between 100 and 5000 and pause for 0.1 second, you can use:
+
+```bash
+sudo RAILS_ENV=production -u git -H bundle exec rake gitlab:uploads:sanitize:remove_exif[100,5000,false,0.1] 2>&1 | tee exif.log
+```
+
+Because the output of commands will be probably long, the output is written also into exif.log file.
+
+If sanitization fails for an upload, an error message should be in the output of the rake task (typical reasons may
+be that the file is missing in the storage or it's not a valid image). Please
+[report](https://gitlab.com/gitlab-org/gitlab-ce/issues/new) any issues at `gitlab.com` and use
+prefix 'EXIF' in issue title with the error output and (if possible) the image.

doc/ci/yaml/README.md

@@ -255,6 +255,19 @@ job:
     - branches
 ```
 
+Pattern matching is case-sensitive by default. Use `i` flag modifier, like
+`/pattern/i` to make a pattern case-insensitive:
+
+```yaml
+job:
+  # use regexp
+  only:
+    - /^issue-.*$/i
+  # use special keyword
+  except:
+    - branches
+```
+
 In this example, `job` will run only for refs that are tagged, or if a build is
 explicitly requested via an API trigger or a [Pipeline Schedule][schedules]:
 

doc/development/gitaly.md

@@ -56,6 +56,46 @@ If your test-suite is failing with Gitaly issues, as a first step, try running:
 rm -rf tmp/tests/gitaly
 ```
 
+## Legacy Rugged code
+
+While Gitaly can handle all Git access, many of GitLab customers still
+run Gitaly atop NFS. The legacy Rugged implementation for Git calls may
+be faster than the Gitaly RPC due to N+1 Gitaly calls and other
+reasons. See [the
+issue](https://gitlab.com/gitlab-org/gitlab-ce/issues/57317) for more
+details.
+
+Until GitLab has eliminated most of these inefficiencies or the use of
+NFS is discontinued for Git data, Rugged implementations of some of the
+most commonly-used RPCs can be enabled via feature flags:
+
+* `rugged_find_commit`
+* `rugged_get_tree_entries`
+* `rugged_tree_entry`
+* `rugged_commit_is_ancestor`
+* `rugged_commit_tree_entry`
+* `rugged_list_commits_by_oid`
+
+A convenience Rake task can be used to enable or disable these flags
+all together. To enable:
+
+```sh
+bundle exec rake gitlab:features:enable_rugged
+```
+
+To disable:
+
+```sh
+bundle exec rake gitlab:features:disable_rugged
+```
+
+Most of this code exists in the `lib/gitlab/git/rugged_impl` directory.
+
+NOTE: **Note:** You should NOT need to add or modify code related to
+Rugged unless explicitly discussed with the [Gitaly
+Team](https://gitlab.com/groups/gl-gitaly/group_members). This code will
+NOT work on GitLab.com or other GitLab instances that do not use NFS.
+
 ## `TooManyInvocationsError` errors
 
 During development and testing, you may experience `Gitlab::GitalyClient::TooManyInvocationsError` failures.

doc/raketasks/README.md

@@ -15,3 +15,4 @@ comments: false
 - [Import](import.md) of git repositories in bulk
 - [Rebuild authorized_keys file](http://docs.gitlab.com/ce/raketasks/maintenance.html#rebuild-authorized_keys-file) task for administrators
 - [Migrate Uploads](../administration/raketasks/uploads/migrate.md)
+- [Sanitize Uploads](../administration/raketasks/uploads/sanitize.md)

doc/update/11.7-to-11.8.md

@@ -30,8 +30,8 @@ sudo -u git -H bundle exec rake gitlab:backup:create RAILS_ENV=production
 
 ### 3. Update Ruby
 
-NOTE: Beginning in GitLab 11.0, we only support Ruby 2.4 or higher, and dropped
+NOTE: Beginning in GitLab 11.6, we only support Ruby 2.5 or higher, and dropped
-support for Ruby 2.3. Be sure to upgrade if necessary.
+support for Ruby 2.4. Be sure to upgrade if necessary.
 
 You can check which version you are running with `ruby -v`.
 

lib/gitlab/ci/build/policy/refs.rb

@@ -35,8 +35,8 @@ module Gitlab
             # patterns can be matched only when branch or tag is used
             # the pattern matching does not work for merge requests pipelines
             if pipeline.branch? || pipeline.tag?
-              if pattern.first == "/" && pattern.last == "/"
+              if regexp = Gitlab::UntrustedRegexp::RubySyntax.fabricate(pattern)
-                Regexp.new(pattern[1...-1]) =~ pipeline.ref
+                regexp.match?(pipeline.ref)
               else
                 pattern == pipeline.ref
               end

lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb

@@ -13,13 +13,13 @@ module Gitlab
             def initialize(regexp)
               @value = regexp
 
-              unless Gitlab::UntrustedRegexp.valid?(@value)
+              unless Gitlab::UntrustedRegexp::RubySyntax.valid?(@value)
                 raise Lexer::SyntaxError, 'Invalid regular expression!'
               end
             end
 
             def evaluate(variables = {})
-              Gitlab::UntrustedRegexp.fabricate(@value)
+              Gitlab::UntrustedRegexp::RubySyntax.fabricate!(@value)
             rescue RegexpError
               raise Expression::RuntimeError, 'Invalid regular expression!'
             end

lib/gitlab/config/entry/legacy_validation_helpers.rb

@@ -45,17 +45,15 @@ module Gitlab
         end
 
         def validate_regexp(value)
-          !value.nil? && Regexp.new(value.to_s) && true
+          Gitlab::UntrustedRegexp::RubySyntax.valid?(value)
-        rescue RegexpError, TypeError
-          false
         end
 
         def validate_string_or_regexp(value)
           return true if value.is_a?(Symbol)
           return false unless value.is_a?(String)
 
-          if value.first == '/' && value.last == '/'
+          if Gitlab::UntrustedRegexp::RubySyntax.matches_syntax?(value)
-            validate_regexp(value[1...-1])
+            validate_regexp(value)
           else
             true
           end

lib/gitlab/config/entry/validators.rb

@@ -120,17 +120,13 @@ module Gitlab
 
           private
 
-          def look_like_regexp?(value)
+          def matches_syntax?(value)
-            value.is_a?(String) && value.start_with?('/') &&
+            Gitlab::UntrustedRegexp::RubySyntax.matches_syntax?(value)
-              value.end_with?('/')
           end
 
           def validate_regexp(value)
-            look_like_regexp?(value) &&
+            matches_syntax?(value) &&
-              Regexp.new(value.to_s[1...-1]) &&
+              Gitlab::UntrustedRegexp::RubySyntax.valid?(value)
-              true
-          rescue RegexpError
-            false
           end
         end
 
@@ -149,7 +145,7 @@ module Gitlab
 
           def validate_string_or_regexp(value)
             return false unless value.is_a?(String)
-            return validate_regexp(value) if look_like_regexp?(value)
+            return validate_regexp(value) if matches_syntax?(value)
 
             true
           end

lib/gitlab/git/blob.rb

@@ -23,6 +23,10 @@ module Gitlab
 
       class << self
         def find(repository, sha, path, limit: MAX_DATA_DISPLAY_SIZE)
+          tree_entry(repository, sha, path, limit)
+        end
+
+        def tree_entry(repository, sha, path, limit)
           return unless path
 
           path = path.sub(%r{\A/*}, '')
@@ -179,3 +183,5 @@ module Gitlab
     end
   end
 end
+
+Gitlab::Git::Blob.singleton_class.prepend Gitlab::Git::RuggedImpl::Blob::ClassMethods

lib/gitlab/git/commit.rb

@@ -5,6 +5,7 @@ module Gitlab
   module Git
     class Commit
       include Gitlab::EncodingHelper
+      prepend Gitlab::Git::RuggedImpl::Commit
       extend Gitlab::Git::WrapsGitalyErrors
 
       attr_accessor :raw_commit, :head
@@ -62,15 +63,19 @@ module Gitlab
           # This saves us an RPC round trip.
           return nil if commit_id.include?(':')
 
-          commit = wrapped_gitaly_errors do
+          commit = find_commit(repo, commit_id)
-            repo.gitaly_commit_client.find_commit(commit_id)
-          end
 
           decorate(repo, commit) if commit
         rescue Gitlab::Git::CommandError, Gitlab::Git::Repository::NoRepository, ArgumentError
           nil
         end
 
+        def find_commit(repo, commit_id)
+          wrapped_gitaly_errors do
+            repo.gitaly_commit_client.find_commit(commit_id)
+          end
+        end
+
         # Get last commit for HEAD
         #
         # Ex.
@@ -185,6 +190,10 @@ module Gitlab
         @repository = repository
         @head = head
 
+        init_commit(raw_commit)
+      end
+
+      def init_commit(raw_commit)
         case raw_commit
         when Hash
           init_from_hash(raw_commit)
@@ -305,11 +314,16 @@ module Gitlab
       def tree_entry(path)
         return unless path.present?
 
+        commit_tree_entry(path)
+      end
+
+      def commit_tree_entry(path)
         # We're only interested in metadata, so limit actual data to 1 byte
         # since Gitaly doesn't support "send no data" option.
         entry = @repository.gitaly_commit_client.tree_entry(id, path, 1)
         return unless entry
 
+        # To be compatible with the rugged format
         entry = entry.to_h
         entry.delete(:data)
         entry[:name] = File.basename(path)
@@ -400,3 +414,5 @@ module Gitlab
     end
   end
 end
+
+Gitlab::Git::Commit.singleton_class.prepend Gitlab::Git::RuggedImpl::Commit::ClassMethods

lib/gitlab/git/ref.rb

@@ -4,6 +4,7 @@ module Gitlab
   module Git
     class Ref
       include Gitlab::EncodingHelper
+      include Gitlab::Git::RuggedImpl::Ref
 
       # Branch or tag name
       # without "refs/tags|heads" prefix

lib/gitlab/git/repository.rb

@@ -11,6 +11,7 @@ module Gitlab
       include Gitlab::Git::WrapsGitalyErrors
       include Gitlab::EncodingHelper
       include Gitlab::Utils::StrongMemoize
+      prepend Gitlab::Git::RuggedImpl::Repository
 
       SEARCH_CONTEXT_LINES = 3
       REV_LIST_COMMIT_LIMIT = 2_000

lib/gitlab/git/rugged_impl/blob.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+# NOTE: This code is legacy. Do not add/modify code here unless you have
+# discussed with the Gitaly team.  See
+# https://docs.gitlab.com/ee/development/gitaly.html#legacy-rugged-code
+# for more details.
+
+module Gitlab
+  module Git
+    module RuggedImpl
+      module Blob
+        module ClassMethods
+          extend ::Gitlab::Utils::Override
+
+          override :tree_entry
+          def tree_entry(repository, sha, path, limit)
+            if Feature.enabled?(:rugged_tree_entry)
+              rugged_tree_entry(repository, sha, path, limit)
+            else
+              super
+            end
+          end
+
+          private
+
+          def rugged_tree_entry(repository, sha, path, limit)
+            return unless path
+
+            # Strip any leading / characters from the path
+            path = path.sub(%r{\A/*}, '')
+
+            rugged_commit = repository.lookup(sha)
+            root_tree = rugged_commit.tree
+
+            blob_entry = find_entry_by_path(repository, root_tree.oid, *path.split('/'))
+
+            return unless blob_entry
+
+            if blob_entry[:type] == :commit
+              submodule_blob(blob_entry, path, sha)
+            else
+              blob = repository.lookup(blob_entry[:oid])
+
+              if blob
+                new(
+                  id: blob.oid,
+                  name: blob_entry[:name],
+                  size: blob.size,
+                  # Rugged::Blob#content is expensive; don't call it if we don't have to.
+                  data: limit.zero? ? '' : blob.content(limit),
+                  mode: blob_entry[:filemode].to_s(8),
+                  path: path,
+                  commit_id: sha,
+                  binary: blob.binary?
+                )
+              end
+            end
+          rescue Rugged::ReferenceError
+            nil
+          end
+
+          # Recursive search of blob id by path
+          #
+          # Ex.
+          #   blog/            # oid: 1a
+          #     app/           # oid: 2a
+          #       models/      # oid: 3a
+          #       file.rb      # oid: 4a
+          #
+          #
+          # Blob.find_entry_by_path(repo, '1a', 'blog', 'app', 'file.rb') # => '4a'
+          #
+          def find_entry_by_path(repository, root_id, *path_parts)
+            root_tree = repository.lookup(root_id)
+
+            entry = root_tree.find do |entry|
+              entry[:name] == path_parts[0]
+            end
+
+            return unless entry
+
+            if path_parts.size > 1
+              return unless entry[:type] == :tree
+
+              path_parts.shift
+              find_entry_by_path(repository, entry[:oid], *path_parts)
+            else
+              [:blob, :commit].include?(entry[:type]) ? entry : nil
+            end
+          end
+
+          def submodule_blob(blob_entry, path, sha)
+            new(
+              id: blob_entry[:oid],
+              name: blob_entry[:name],
+              size: 0,
+              data: '',
+              path: path,
+              commit_id: sha
+            )
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/git/rugged_impl/commit.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+# NOTE: This code is legacy. Do not add/modify code here unless you have
+# discussed with the Gitaly team.  See
+# https://docs.gitlab.com/ee/development/gitaly.html#legacy-rugged-code
+# for more details.
+
+# rubocop:disable Gitlab/ModuleWithInstanceVariables
+module Gitlab
+  module Git
+    module RuggedImpl
+      module Commit
+        module ClassMethods
+          extend ::Gitlab::Utils::Override
+
+          def rugged_find(repo, commit_id)
+            obj = repo.rev_parse_target(commit_id)
+
+            obj.is_a?(::Rugged::Commit) ? obj : nil
+          rescue ::Rugged::Error
+            nil
+          end
+
+          # This needs to return an array of Gitlab::Git:Commit objects
+          # instead of Rugged::Commit objects to ensure upstream models
+          # operate on a consistent interface. Unlike
+          # Gitlab::Git::Commit.find, Gitlab::Git::Commit.batch_by_oid
+          # doesn't attempt to decorate the result.
+          def rugged_batch_by_oid(repo, oids)
+            oids.map { |oid| rugged_find(repo, oid) }
+              .compact
+              .map { |commit| decorate(repo, commit) }
+          end
+
+          override :find_commit
+          def find_commit(repo, commit_id)
+            if Feature.enabled?(:rugged_find_commit)
+              rugged_find(repo, commit_id)
+            else
+              super
+            end
+          end
+
+          override :batch_by_oid
+          def batch_by_oid(repo, oids)
+            if Feature.enabled?(:rugged_list_commits_by_oid)
+              rugged_batch_by_oid(repo, oids)
+            else
+              super
+            end
+          end
+        end
+
+        extend ::Gitlab::Utils::Override
+
+        override :init_commit
+        def init_commit(raw_commit)
+          case raw_commit
+          when ::Rugged::Commit
+            init_from_rugged(raw_commit)
+          else
+            super
+          end
+        end
+
+        override :commit_tree_entry
+        def commit_tree_entry(path)
+          if Feature.enabled?(:rugged_commit_tree_entry)
+            rugged_tree_entry(path)
+          else
+            super
+          end
+        end
+
+        # Is this the same as Blob.find_entry_by_path ?
+        def rugged_tree_entry(path)
+          rugged_commit.tree.path(path)
+        rescue Rugged::TreeError
+          nil
+        end
+
+        def rugged_commit
+          @rugged_commit ||= if raw_commit.is_a?(Rugged::Commit)
+                               raw_commit
+                             else
+                               @repository.rev_parse_target(id)
+                             end
+        end
+
+        def init_from_rugged(commit)
+          author = commit.author
+          committer = commit.committer
+
+          @raw_commit = commit
+          @id = commit.oid
+          @message = commit.message
+          @authored_date = author[:time]
+          @committed_date = committer[:time]
+          @author_name = author[:name]
+          @author_email = author[:email]
+          @committer_name = committer[:name]
+          @committer_email = committer[:email]
+          @parent_ids = commit.parents.map(&:oid)
+        end
+      end
+    end
+  end
+end
+# rubocop:enable Gitlab/ModuleWithInstanceVariables

lib/gitlab/git/rugged_impl/ref.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# NOTE: This code is legacy. Do not add/modify code here unless you have
+# discussed with the Gitaly team.  See
+# https://docs.gitlab.com/ee/development/gitaly.html#legacy-rugged-code
+# for more details.
+
+module Gitlab
+  module Git
+    module RuggedImpl
+      module Ref
+        def self.dereference_object(object)
+          object = object.target while object.is_a?(::Rugged::Tag::Annotation)
+
+          object
+        end
+      end
+    end
+  end
+end

lib/gitlab/git/rugged_impl/repository.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+# NOTE: This code is legacy. Do not add/modify code here unless you have
+# discussed with the Gitaly team.  See
+# https://docs.gitlab.com/ee/development/gitaly.html#legacy-rugged-code
+# for more details.
+
+# rubocop:disable Gitlab/ModuleWithInstanceVariables
+module Gitlab
+  module Git
+    module RuggedImpl
+      module Repository
+        extend ::Gitlab::Utils::Override
+
+        FEATURE_FLAGS = %i(rugged_find_commit rugged_tree_entries rugged_tree_entry rugged_commit_is_ancestor rugged_commit_tree_entry rugged_list_commits_by_oid).freeze
+
+        def alternate_object_directories
+          relative_object_directories.map { |d| File.join(path, d) }
+        end
+
+        ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES = %w[
+          GIT_OBJECT_DIRECTORY_RELATIVE
+          GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE
+        ].freeze
+
+        def relative_object_directories
+          Gitlab::Git::HookEnv.all(gl_repository).values_at(*ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES).flatten.compact
+        end
+
+        def rugged
+          @rugged ||= ::Rugged::Repository.new(path, alternates: alternate_object_directories)
+        rescue ::Rugged::RepositoryError, ::Rugged::OSError
+          raise ::Gitlab::Git::Repository::NoRepository.new('no repository for such path')
+        end
+
+        def cleanup
+          @rugged&.close
+        end
+
+        # Return the object that +revspec+ points to.  If +revspec+ is an
+        # annotated tag, then return the tag's target instead.
+        def rev_parse_target(revspec)
+          obj = rugged.rev_parse(revspec)
+          Ref.dereference_object(obj)
+        end
+
+        override :ancestor?
+        def ancestor?(from, to)
+          if Feature.enabled?(:rugged_commit_is_ancestor)
+            rugged_is_ancestor?(from, to)
+          else
+            super
+          end
+        end
+
+        def rugged_is_ancestor?(ancestor_id, descendant_id)
+          return false if ancestor_id.nil? || descendant_id.nil?
+
+          rugged_merge_base(ancestor_id, descendant_id) == ancestor_id
+        rescue Rugged::OdbError
+          false
+        end
+
+        def rugged_merge_base(from, to)
+          rugged.merge_base(from, to)
+        rescue Rugged::ReferenceError
+          nil
+        end
+
+        # Lookup for rugged object by oid or ref name
+        def lookup(oid_or_ref_name)
+          rugged.rev_parse(oid_or_ref_name)
+        end
+      end
+    end
+  end
+end
+# rubocop:enable Gitlab/ModuleWithInstanceVariables

lib/gitlab/git/rugged_impl/tree.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+# NOTE: This code is legacy. Do not add/modify code here unless you have
+# discussed with the Gitaly team.  See
+# https://docs.gitlab.com/ee/development/gitaly.html#legacy-rugged-code
+# for more details.
+
+module Gitlab
+  module Git
+    module RuggedImpl
+      module Tree
+        module ClassMethods
+          extend ::Gitlab::Utils::Override
+
+          override :tree_entries
+          def tree_entries(repository, sha, path, recursive)
+            if Feature.enabled?(:rugged_tree_entries)
+              tree_entries_with_flat_path_from_rugged(repository, sha, path, recursive)
+            else
+              super
+            end
+          end
+
+          def tree_entries_with_flat_path_from_rugged(repository, sha, path, recursive)
+            tree_entries_from_rugged(repository, sha, path, recursive).tap do |entries|
+              # This was an optimization to reduce N+1 queries for Gitaly
+              # (https://gitlab.com/gitlab-org/gitaly/issues/530).  It
+              # used to be done lazily in the view via
+              # TreeHelper#flatten_tree, so it's possible there's a
+              # performance impact by loading this eagerly.
+              rugged_populate_flat_path(repository, sha, path, entries)
+            end
+          end
+
+          def tree_entries_from_rugged(repository, sha, path, recursive)
+            current_path_entries = get_tree_entries_from_rugged(repository, sha, path)
+            ordered_entries = []
+
+            current_path_entries.each do |entry|
+              ordered_entries << entry
+
+              if recursive && entry.dir?
+                ordered_entries.concat(tree_entries_from_rugged(repository, sha, entry.path, true))
+              end
+            end
+          end
+
+          def rugged_populate_flat_path(repository, sha, path, entries)
+            entries.each do |entry|
+              entry.flat_path = entry.path
+
+              next unless entry.dir?
+
+              entry.flat_path =
+                if path
+                  File.join(path, rugged_flatten_tree(repository, sha, entry, path))
+                else
+                  rugged_flatten_tree(repository, sha, entry, path)
+                end
+            end
+          end
+
+          # Returns the relative path of the first subdir that doesn't have only one directory descendant
+          def rugged_flatten_tree(repository, sha, tree, root_path)
+            subtree = tree_entries_from_rugged(repository, sha, tree.path, false)
+
+            if subtree.count == 1 && subtree.first.dir?
+              File.join(tree.name, rugged_flatten_tree(repository, sha, subtree.first, root_path))
+            else
+              tree.name
+            end
+          end
+
+          def get_tree_entries_from_rugged(repository, sha, path)
+            commit = repository.lookup(sha)
+            root_tree = commit.tree
+
+            tree = if path
+                     id = find_id_by_path(repository, root_tree.oid, path)
+                     if id
+                       repository.lookup(id)
+                     else
+                       []
+                     end
+                   else
+                     root_tree
+                   end
+
+            tree.map do |entry|
+              current_path = path ? File.join(path, entry[:name]) : entry[:name]
+
+              new(
+                id: entry[:oid],
+                root_id: root_tree.oid,
+                name: entry[:name],
+                type: entry[:type],
+                mode: entry[:filemode].to_s(8),
+                path: current_path,
+                commit_id: sha
+              )
+            end
+          rescue Rugged::ReferenceError
+            []
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/git/tree.rb

@@ -18,6 +18,10 @@ module Gitlab
         def where(repository, sha, path = nil, recursive = false)
           path = nil if path == '' || path == '/'
 
+          tree_entries(repository, sha, path, recursive)
+        end
+
+        def tree_entries(repository, sha, path, recursive)
           wrapped_gitaly_errors do
             repository.gitaly_commit_client.tree_entries(repository, sha, path, recursive)
           end
@@ -95,3 +99,5 @@ module Gitlab
     end
   end
 end
+
+Gitlab::Git::Tree.singleton_class.prepend Gitlab::Git::RuggedImpl::Tree::ClassMethods

lib/gitlab/gitaly_client/storage_settings.rb

@@ -32,11 +32,19 @@ module Gitlab
       end
 
       def self.disk_access_denied?
+        return false if rugged_enabled?
+
         !temporarily_allowed?(ALLOW_KEY) && GitalyClient.feature_enabled?(DISK_ACCESS_DENIED_FLAG)
       rescue
         false # Err on the side of caution, don't break gitlab for people
       end
 
+      def self.rugged_enabled?
+        Gitlab::Git::RuggedImpl::Repository::FEATURE_FLAGS.any? do |flag|
+          Feature.enabled?(flag)
+        end
+      end
+
       def initialize(storage)
         raise InvalidConfigurationError, "expected a Hash, got a #{storage.class.name}" unless storage.is_a?(Hash)
         raise InvalidConfigurationError, INVALID_STORAGE_MESSAGE unless storage.has_key?('path')

lib/gitlab/middleware/basic_health_check.rb

@@ -24,7 +24,13 @@ module Gitlab
       def call(env)
         return @app.call(env) unless env['PATH_INFO'] == HEALTH_PATH
 
-        request = ActionDispatch::Request.new(env)
+        # We should be using ActionDispatch::Request instead of
+        # Rack::Request to be consistent with Rails, but due to a Rails
+        # bug described in
+        # https://gitlab.com/gitlab-org/gitlab-ce/issues/58573#note_149799010
+        # hosts behind a load balancer will only see 127.0.0.1 for the
+        # load balancer's IP.
+        request = Rack::Request.new(env)
 
         return OK_RESPONSE if client_ip_whitelisted?(request)
 

lib/gitlab/request_context.rb

@@ -13,7 +13,13 @@ module Gitlab
     end
 
     def call(env)
-      req = ActionDispatch::Request.new(env)
+      # We should be using ActionDispatch::Request instead of
+      # Rack::Request to be consistent with Rails, but due to a Rails
+      # bug described in
+      # https://gitlab.com/gitlab-org/gitlab-ce/issues/58573#note_149799010
+      # hosts behind a load balancer will only see 127.0.0.1 for the
+      # load balancer's IP.
+      req = Rack::Request.new(env)
 
       Gitlab::SafeRequestStore[:client_ip] = req.ip
 

lib/gitlab/sanitizers/exif.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Sanitizers
+    class Exif
+      # these tags are not removed from the image
+      WHITELISTED_TAGS = %w(
+        ResolutionUnit
+        XResolution
+        YResolution
+        YCbCrSubSampling
+        YCbCrPositioning
+        BitsPerSample
+        ImageHeight
+        ImageWidth
+        ImageSize
+        Copyright
+        CopyrightNotice
+        Orientation
+      ).freeze
+
+      # these tags are common in exiftool output, these
+      # do not contain any sensitive information, but
+      # we don't need to preserve them when removing
+      # exif tags
+      IGNORED_TAGS = %w(
+        ColorComponents
+        EncodingProcess
+        ExifByteOrder
+        ExifToolVersion
+        JFIFVersion
+        Directory
+        FileAccessDate
+        FileInodeChangeDate
+        FileModifyDate
+        FileName
+        FilePermissions
+        FileSize
+        SourceFile
+        Megapixels
+        FileType
+        FileTypeExtension
+        MIMEType
+      ).freeze
+
+      ALLOWED_TAGS = WHITELISTED_TAGS + IGNORED_TAGS
+      EXCLUDE_PARAMS = WHITELISTED_TAGS.map { |tag| "-#{tag}" }
+
+      attr_reader :logger
+
+      def initialize(logger: Rails.logger)
+        @logger = logger
+      end
+
+      # rubocop: disable CodeReuse/ActiveRecord
+      def batch_clean(start_id: nil, stop_id: nil, dry_run: true, sleep_time: nil)
+        relation = Upload.where('lower(path) like ? or lower(path) like ? or lower(path) like ?',
+                                '%.jpg', '%.jpeg', '%.tiff')
+
+        logger.info "running in dry run mode, no images will be rewritten" if dry_run
+
+        find_params = {
+          start: start_id.present? ? start_id.to_i : nil,
+          finish: stop_id.present? ? stop_id.to_i : Upload.last&.id
+        }
+
+        relation.find_each(find_params) do |upload|
+          begin
+            clean(upload.build_uploader, dry_run: dry_run)
+            sleep sleep_time if sleep_time
+          rescue => err
+            logger.error "failed to sanitize #{upload_ref(upload)}: #{err.message}"
+            logger.debug err.backtrace.join("\n ")
+          end
+        end
+      end
+      # rubocop: enable CodeReuse/ActiveRecord
+
+      def clean(uploader, dry_run: true)
+        Dir.mktmpdir('gitlab-exif') do |tmpdir|
+          src_path = fetch_upload_to_file(uploader, tmpdir)
+
+          to_remove = extra_tags(src_path)
+
+          if to_remove.empty?
+            logger.info "#{upload_ref(uploader.upload)}: only whitelisted tags present, skipping"
+            break
+          end
+
+          logger.info "#{upload_ref(uploader.upload)}: found exif tags to remove: #{to_remove}"
+
+          break if dry_run
+
+          remove_and_store(tmpdir, src_path, uploader)
+        end
+      end
+
+      def extra_tags(path)
+        exif_tags(path).keys - ALLOWED_TAGS
+      end
+
+      private
+
+      def remove_and_store(tmpdir, src_path, uploader)
+        exec_remove_exif!(src_path)
+        logger.info "#{upload_ref(uploader.upload)}: exif removed, storing"
+        File.open(src_path, 'r') { |f| uploader.store!(f) }
+      end
+
+      def exec_remove_exif!(path)
+        # IPTC and XMP-iptcExt groups may keep copyright information so
+        # we always preserve them
+        cmd = ["exiftool", "-all=", "-tagsFromFile", "@", *EXCLUDE_PARAMS, "--IPTC:all", "--XMP-iptcExt:all", path]
+        output, status = Gitlab::Popen.popen(cmd)
+
+        if status != 0
+          raise "exiftool return code is #{status}: #{output}"
+        end
+
+        if File.size(path) == 0
+          raise "size of file is 0"
+        end
+
+        # exiftool creates backup of the original file in filename_original
+        old_path = "#{path}_original"
+        if File.size(path) == File.size(old_path)
+          raise "size of sanitized file is same as original size"
+        end
+      end
+
+      def fetch_upload_to_file(uploader, dir)
+        # upload is stored into the file with the original name - this filename
+        # is used by carrierwave when storing the file back to the storage
+        filename = File.join(dir, uploader.filename)
+
+        File.open(filename, 'w') do |file|
+          file.binmode
+          file.write uploader.read
+        end
+
+        filename
+      end
+
+      def upload_ref(upload)
+        "#{upload.id}:#{upload.path}"
+      end
+
+      def exif_tags(path)
+        cmd = ["exiftool", "-all", "-j", "-sort", "--IPTC:all", "--XMP-iptcExt:all", path]
+        output, status = Gitlab::Popen.popen(cmd)
+
+        raise "failed to get exif tags: #{output}" if status != 0
+
+        JSON.parse(output).first
+      end
+    end
+  end
+end

lib/gitlab/untrusted_regexp.rb

@@ -35,6 +35,10 @@ module Gitlab
       matches
     end
 
+    def match?(text)
+      text.present? && scan(text).present?
+    end
+
     def replace(text, rewrite)
       RE2.Replace(text, regexp, rewrite)
     end
@@ -43,37 +47,6 @@ module Gitlab
       self.source == other.source
     end
 
-    # Handles regular expressions with the preferred RE2 library where possible
-    # via UntustedRegex. Falls back to Ruby's built-in regular expression library
-    # when the syntax would be invalid in RE2.
-    #
-    # One difference between these is `(?m)` multi-line mode. Ruby regex enables
-    # this by default, but also handles `^` and `$` differently.
-    # See: https://www.regular-expressions.info/modifiers.html
-    def self.with_fallback(pattern, multiline: false)
-      UntrustedRegexp.new(pattern, multiline: multiline)
-    rescue RegexpError
-      Regexp.new(pattern)
-    end
-
-    def self.valid?(pattern)
-      !!self.fabricate(pattern)
-    rescue RegexpError
-      false
-    end
-
-    def self.fabricate(pattern)
-      matches = pattern.match(%r{^/(?<regexp>.+)/(?<flags>[ismU]*)$})
-
-      raise RegexpError, 'Invalid regular expression!' if matches.nil?
-
-      expression = matches[:regexp]
-      flags = matches[:flags]
-      expression.prepend("(?#{flags})") if flags.present?
-
-      self.new(expression, multiline: false)
-    end
-
     private
 
     attr_reader :regexp

lib/gitlab/untrusted_regexp/ruby_syntax.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Gitlab
+  class UntrustedRegexp
+    # This class implements support for Ruby syntax of regexps
+    # and converts that to RE2 representation:
+    # /<regexp>/<flags>
+    class RubySyntax
+      PATTERN = %r{^/(?<regexp>.+)/(?<flags>[ismU]*)$}.freeze
+
+      # Checks if pattern matches a regexp pattern
+      # but does not enforce it's validity
+      def self.matches_syntax?(pattern)
+        pattern.is_a?(String) && pattern.match(PATTERN).present?
+      end
+
+      # The regexp can match the pattern `/.../`, but may not be fabricatable:
+      # it can be invalid or incomplete: `/match ( string/`
+      def self.valid?(pattern)
+        !!self.fabricate(pattern)
+      end
+
+      def self.fabricate(pattern)
+        self.fabricate!(pattern)
+      rescue RegexpError
+        nil
+      end
+
+      def self.fabricate!(pattern)
+        raise RegexpError, 'Pattern is not string!' unless pattern.is_a?(String)
+
+        matches = pattern.match(PATTERN)
+        raise RegexpError, 'Invalid regular expression!' if matches.nil?
+
+        expression = matches[:regexp]
+        flags = matches[:flags]
+        expression.prepend("(?#{flags})") if flags.present?
+
+        UntrustedRegexp.new(expression, multiline: false)
+      end
+    end
+  end
+end

lib/tasks/gitlab/features.rake

@@ -0,0 +1,24 @@
+namespace :gitlab do
+  namespace :features do
+    desc 'GitLab | Features | Enable direct Git access via Rugged for NFS'
+    task enable_rugged: :environment do
+      set_rugged_feature_flags(true)
+      puts 'All Rugged feature flags were enabled.'
+    end
+
+    task disable_rugged: :environment do
+      set_rugged_feature_flags(false)
+      puts 'All Rugged feature flags were disabled.'
+    end
+  end
+
+  def set_rugged_feature_flags(status)
+    Gitlab::Git::RuggedImpl::Repository::FEATURE_FLAGS.each do |flag|
+      if status
+        Feature.enable(flag)
+      else
+        Feature.disable(flag)
+      end
+    end
+  end
+end

lib/tasks/gitlab/uploads/sanitize.rake

@@ -0,0 +1,18 @@
+namespace :gitlab do
+  namespace :uploads do
+    namespace :sanitize do
+      desc 'GitLab | Uploads | Remove EXIF from images.'
+      task :remove_exif, [:start_id, :stop_id, :dry_run, :sleep_time] => :environment do |task, args|
+        args.with_defaults(dry_run: 'true')
+        args.with_defaults(sleep_time: 0.3)
+
+        logger = Logger.new(STDOUT)
+
+        sanitizer = Gitlab::Sanitizers::Exif.new(logger: logger)
+        sanitizer.batch_clean(start_id: args.start_id, stop_id: args.stop_id,
+                              dry_run: args.dry_run != 'false',
+                              sleep_time: args.sleep_time.to_f)
+      end
+    end
+  end
+end

scripts/lint-rugged

@@ -5,12 +5,18 @@ ALLOWED = [
   'lib/gitlab/bare_repository_import/repository.rb',
 
   # Needed to avoid using the git binary to validate a branch name
-  'lib/gitlab/git_ref_validator.rb'
+  'lib/gitlab/git_ref_validator.rb',
+
+  # Reverted Rugged calls due to Gitaly atop NFS performance
+  # See https://docs.gitlab.com/ee/development/gitaly.html#legacy-rugged-code.
+  'lib/gitlab/git/rugged_impl/',
+  'lib/gitlab/gitaly_client/storage_settings.rb'
 ].freeze
 
 rugged_lines = IO.popen(%w[git grep -i -n rugged -- app config lib], &:read).lines
 rugged_lines = rugged_lines.select { |l| /^[^:]*\.rb:/ =~ l }
 rugged_lines = rugged_lines.reject { |l| l.start_with?(*ALLOWED) }
+rugged_lines = rugged_lines.reject { |l| /(include|prepend) Gitlab::Git::RuggedImpl/ =~ l}
 rugged_lines = rugged_lines.reject do |line|
   code, _comment = line.split('# ', 2)
   code !~ /rugged/i

spec/controllers/projects/notes_controller_spec.rb

@@ -431,28 +431,77 @@ describe Projects::NotesController do
   end
 
   describe 'PUT update' do
-    context "should update the note with a valid issue" do
+    context "updates the note" do
-      let(:request_params) do
+      context 'with a valid issue' do
-        {
+        let(:request_params) do
-          namespace_id: project.namespace,
+          {
-          project_id: project,
+            namespace_id: project.namespace,
-          id: note,
+            project_id: project,
-          format: :json,
+            id: note,
-          note: {
+            format: :json,
-            note: "New comment"
+            note: {
+              note: "New comment"
+            }
           }
-        }
+        end
+
+        before do
+          sign_in(note.author)
+          project.add_developer(note.author)
+        end
+
+        it "updates the note content" do
+          expect { put :update, params: request_params }.to change { note.reload.note }
+        end
       end
 
-      before do
+      context "when the note is edited and a different issue is targeted" do
-        sign_in(note.author)
+        ##
-        project.add_developer(note.author)
+        # We are editing a note originally in a public issue of a public project,
-      end
+        # but the edit can be intercepted to change the target to a different, even confidential, issue
+        # see https://gitlab.com/gitlab-org/gitlab-ce/issues/57153
+        ##
 
-      it "updates the note" do
+        let!(:confidential_issue) { create(:issue, :confidential, project: project) }
-        expect { put :update, params: request_params }.to change { note.reload.note }
+        let(:new_content) { "splendiferous new content" }
+        let(:request_params) do
+          {
+            namespace_id: project.namespace,
+            project_id: project,
+            id: note,
+            format: :json,
+            note: {
+              note: new_content,
+              noteable_id: confidential_issue.id
+            }
+          }
+        end
+
+        before do
+          sign_in(note.author)
+          project.add_developer(note.author)
+
+          put :update, params: request_params
+        end
+
+        it 'returns success' do
+          expect(response.status).to eq 200
+        end
+
+        it 'edits the note content' do
+          expect(note.reload.note).to eq new_content
+        end
+
+        it 'does not create a note in the confidential issue' do
+          expect(confidential_issue.reload.notes).to be_empty
+        end
+
+        it "does not modify the note's issue" do
+          expect(note.noteable_id).to match note.reload.noteable_id
+        end
       end
     end
+
     context "doesnt update the note" do
       let(:issue)   { create(:issue, :confidential, project: project) }
       let(:note)    { create(:note, noteable: issue, project: project) }

spec/controllers/projects_controller_spec.rb

@@ -369,6 +369,23 @@ describe ProjectsController do
         end
       end
 
+      it 'does not update namespace' do
+        controller.instance_variable_set(:@project, project)
+
+        params = {
+          namespace_id: 'test'
+        }
+
+        expect do
+          put :update,
+            params: {
+              namespace_id: project.namespace,
+              id: project.id,
+              project: params
+            }
+        end.not_to change { project.namespace.reload }
+      end
+
       def update_project(**parameters)
         put :update,
             params: {

spec/factories/ci/pipelines.rb

@@ -82,6 +82,12 @@ FactoryBot.define do
         end
       end
 
+      trait :with_job do
+        after(:build) do |pipeline, evaluator|
+          pipeline.builds << build(:ci_build, pipeline: pipeline, project: pipeline.project)
+        end
+      end
+
       trait :auto_devops_source do
         config_source { Ci::Pipeline.config_sources[:auto_devops_source] }
       end

spec/features/issues/user_creates_branch_and_merge_request_spec.rb

@@ -1,6 +1,7 @@
 require 'rails_helper'
 
 describe 'User creates branch and merge request on issue page', :js do
+  let(:membership_level) { :developer }
   let(:user) { create(:user) }
   let!(:project) { create(:project, :repository) }
   let(:issue) { create(:issue, project: project, title: 'Cherry-Coloured Funk') }
@@ -17,7 +18,7 @@ describe 'User creates branch and merge request on issue page', :js do
 
   context 'when signed in' do
     before do
-      project.add_developer(user)
+      project.add_user(user, membership_level)
 
       sign_in(user)
     end
@@ -167,6 +168,39 @@ describe 'User creates branch and merge request on issue page', :js do
         expect(page).not_to have_css('.create-mr-dropdown-wrap')
       end
     end
+
+    context 'when related branch exists' do
+      let!(:project) { create(:project, :repository, :private) }
+      let(:branch_name) { "#{issue.iid}-foo" }
+
+      before do
+        project.repository.create_branch(branch_name, 'master')
+
+        visit project_issue_path(project, issue)
+      end
+
+      context 'when user is developer' do
+        it 'shows related branches' do
+          expect(page).to have_css('#related-branches')
+
+          wait_for_requests
+
+          expect(page).to have_content(branch_name)
+        end
+      end
+
+      context 'when user is guest' do
+        let(:membership_level) { :guest }
+
+        it 'does not show related branches' do
+          expect(page).not_to have_css('#related-branches')
+
+          wait_for_requests
+
+          expect(page).not_to have_content(branch_name)
+        end
+      end
+    end
   end
 
   private

spec/features/merge_request/user_resolves_conflicts_spec.rb

@@ -164,6 +164,21 @@ describe 'Merge request > User resolves conflicts', :js do
         expect(page).to have_content('Gregor Samsa woke from troubled dreams')
       end
     end
+
+    context "with malicious branch name" do
+      let(:bad_branch_name) { "malicious-branch-{{toString.constructor('alert(/xss/)')()}}" }
+      let(:branch) { project.repository.create_branch(bad_branch_name, 'conflict-resolvable') }
+      let(:merge_request) { create_merge_request(branch.name) }
+
+      before do
+        visit project_merge_request_path(project, merge_request)
+        click_link('conflicts', href: %r{/conflicts\Z})
+      end
+
+      it "renders bad name without xss issues" do
+        expect(find('.resolve-conflicts-form .resolve-info')).to have_content(bad_branch_name)
+      end
+    end
   end
 
   UNRESOLVABLE_CONFLICTS = {

spec/javascripts/pdf/index_spec.js

@@ -1,11 +1,11 @@
 import Vue from 'vue';
-import { PDFJS } from 'vendor/pdf';
+import { GlobalWorkerOptions } from 'vendor/pdf';
 import workerSrc from 'vendor/pdf.worker.min';
 
 import PDFLab from '~/pdf/index.vue';
 import pdf from '../fixtures/blob/pdf/test.pdf';
 
-PDFJS.workerSrc = workerSrc;
+GlobalWorkerOptions.workerSrc = workerSrc;
 const Component = Vue.extend(PDFLab);
 
 describe('PDF component', () => {

spec/javascripts/pdf/page_spec.js

@@ -12,7 +12,7 @@ describe('Page component', () => {
   let testPage;
 
   beforeEach(done => {
-    pdfjsLib.PDFJS.workerSrc = workerSrc;
+    pdfjsLib.GlobalWorkerOptions.workerSrc = workerSrc;
     pdfjsLib
       .getDocument(testPDF)
       .then(pdf => pdf.getPage(1))

spec/lib/gitlab/bitbucket_import/importer_spec.rb

@@ -95,6 +95,9 @@ describe Gitlab::BitbucketImport::Importer do
   subject { described_class.new(project) }
 
   describe '#import_pull_requests' do
+    let(:source_branch_sha) { sample.commits.last }
+    let(:target_branch_sha) { sample.commits.first }
+
     before do
       allow(subject).to receive(:import_wiki)
       allow(subject).to receive(:import_issues)
@@ -102,9 +105,9 @@ describe Gitlab::BitbucketImport::Importer do
       pull_request = instance_double(
         Bitbucket::Representation::PullRequest,
         iid: 10,
-        source_branch_sha: sample.commits.last,
+        source_branch_sha: source_branch_sha,
         source_branch_name: Gitlab::Git::BRANCH_REF_PREFIX + sample.source_branch,
-        target_branch_sha: sample.commits.first,
+        target_branch_sha: target_branch_sha,
         target_branch_name: Gitlab::Git::BRANCH_REF_PREFIX + sample.target_branch,
         title: 'This is a title',
         description: 'This is a test pull request',
@@ -162,6 +165,19 @@ describe Gitlab::BitbucketImport::Importer do
       expect(reply_note).to be_a(DiffNote)
       expect(reply_note.note).to eq(@reply.note)
     end
+
+    context "when branches' sha is not found in the repository" do
+      let(:source_branch_sha) { 'a' * Commit::MIN_SHA_LENGTH }
+      let(:target_branch_sha) { 'b' * Commit::MIN_SHA_LENGTH }
+
+      it 'uses the pull request sha references' do
+        expect { subject.execute }.to change { MergeRequest.count }.by(1)
+
+        merge_request_diff = MergeRequest.first.merge_request_diff
+        expect(merge_request_diff.head_commit_sha).to eq source_branch_sha
+        expect(merge_request_diff.start_commit_sha).to eq target_branch_sha
+      end
+    end
   end
 
   context 'issues statuses' do

spec/lib/gitlab/ci/build/policy/refs_spec.rb

@@ -78,10 +78,23 @@ describe Gitlab::Ci::Build::Policy::Refs do
           .to be_satisfied_by(pipeline)
       end
 
+      it 'is satisfied when case-insensitive regexp matches pipeline ref' do
+        expect(described_class.new(['/DOCS-.*/i']))
+          .to be_satisfied_by(pipeline)
+      end
+
       it 'is not satisfied when regexp does not match pipeline ref' do
         expect(described_class.new(['/fix-.*/']))
           .not_to be_satisfied_by(pipeline)
       end
     end
+
+    context 'malicious regexp' do
+      let(:pipeline) { build_stubbed(:ci_pipeline, ref: malicious_text) }
+
+      subject { described_class.new([malicious_regexp_ruby]) }
+
+      include_examples 'malicious regexp'
+    end
   end
 end

spec/lib/gitlab/ci/pipeline/expression/lexeme/pattern_spec.rb

@@ -85,7 +85,7 @@ describe Gitlab::Ci::Pipeline::Expression::Lexeme::Pattern do
     end
 
     it 'raises error if evaluated regexp is not valid' do
-      allow(Gitlab::UntrustedRegexp).to receive(:valid?).and_return(true)
+      allow(Gitlab::UntrustedRegexp::RubySyntax).to receive(:valid?).and_return(true)
 
       regexp = described_class.new('/invalid ( .*/')
 

spec/lib/gitlab/ci/trace/stream_spec.rb

@@ -414,7 +414,7 @@ describe Gitlab::Ci::Trace::Stream, :clean_gitlab_redis_cache do
 
       context 'malicious regexp' do
         let(:data) { malicious_text }
-        let(:regex) { malicious_regexp }
+        let(:regex) { malicious_regexp_re2 }
 
         include_examples 'malicious regexp'
       end

spec/lib/gitlab/git/blob_spec.rb

@@ -18,7 +18,7 @@ describe Gitlab::Git::Blob, :seed_helper do
     end
   end
 
-  describe '.find' do
+  shared_examples '.find' do
     context 'nil path' do
       let(:blob) { Gitlab::Git::Blob.find(repository, SeedRepo::Commit::ID, nil) }
 
@@ -128,6 +128,20 @@ describe Gitlab::Git::Blob, :seed_helper do
     end
   end
 
+  describe '.find with Gitaly enabled' do
+    it_behaves_like '.find'
+  end
+
+  describe '.find with Rugged enabled', :enable_rugged do
+    it 'calls out to the Rugged implementation' do
+      allow_any_instance_of(Rugged).to receive(:rev_parse).with(SeedRepo::Commit::ID).and_call_original
+
+      described_class.find(repository, SeedRepo::Commit::ID, 'files/images/6049019_460s.jpg')
+    end
+
+    it_behaves_like '.find'
+  end
+
   describe '.raw' do
     let(:raw_blob) { Gitlab::Git::Blob.raw(repository, SeedRepo::RubyBlob::ID) }
     let(:bad_blob) { Gitlab::Git::Blob.raw(repository, SeedRepo::BigCommit::ID) }

spec/lib/gitlab/git/commit_spec.rb

@@ -112,7 +112,7 @@ describe Gitlab::Git::Commit, :seed_helper do
   end
 
   context 'Class methods' do
-    describe '.find' do
+    shared_examples '.find' do
       it "should return first head commit if without params" do
         expect(described_class.last(repository).id).to eq(
           rugged_repo.head.target.oid
@@ -154,6 +154,20 @@ describe Gitlab::Git::Commit, :seed_helper do
       end
     end
 
+    describe '.find with Gitaly enabled' do
+      it_should_behave_like '.find'
+    end
+
+    describe '.find with Rugged enabled', :enable_rugged do
+      it 'calls out to the Rugged implementation' do
+        allow_any_instance_of(Rugged).to receive(:rev_parse).with(SeedRepo::Commit::ID).and_call_original
+
+        described_class.find(repository, SeedRepo::Commit::ID)
+      end
+
+      it_should_behave_like '.find'
+    end
+
     describe '.last_for_path' do
       context 'no path' do
         subject { described_class.last_for_path(repository, 'master') }
@@ -366,7 +380,32 @@ describe Gitlab::Git::Commit, :seed_helper do
       end
     end
 
-    describe '#batch_by_oid' do
+    shared_examples '.batch_by_oid' do
+      context 'with multiple OIDs' do
+        let(:oids) { [SeedRepo::Commit::ID, SeedRepo::FirstCommit::ID] }
+
+        it 'returns multiple commits' do
+          commits = described_class.batch_by_oid(repository, oids)
+
+          expect(commits.count).to eq(2)
+          expect(commits).to all( be_a(Gitlab::Git::Commit) )
+          expect(commits.first.sha).to eq(SeedRepo::Commit::ID)
+          expect(commits.second.sha).to eq(SeedRepo::FirstCommit::ID)
+        end
+      end
+
+      context 'when oids is empty' do
+        it 'returns empty commits' do
+          commits = described_class.batch_by_oid(repository, [])
+
+          expect(commits.count).to eq(0)
+        end
+      end
+    end
+
+    describe '.batch_by_oid with Gitaly enabled' do
+      it_should_behave_like '.batch_by_oid'
+
       context 'when oids is empty' do
         it 'makes no Gitaly request' do
           expect(Gitlab::GitalyClient).not_to receive(:call)
@@ -376,6 +415,16 @@ describe Gitlab::Git::Commit, :seed_helper do
       end
     end
 
+    describe '.batch_by_oid with Rugged enabled', :enable_rugged do
+      it_should_behave_like '.batch_by_oid'
+
+      it 'calls out to the Rugged implementation' do
+        allow_any_instance_of(Rugged).to receive(:rev_parse).with(SeedRepo::Commit::ID).and_call_original
+
+        described_class.batch_by_oid(repository, [SeedRepo::Commit::ID])
+      end
+    end
+
     shared_examples 'extracting commit signature' do
       context 'when the commit is signed' do
         let(:commit_id) { '0b4bc9a49b562e85de7cc9e834518ea6828729b9' }

spec/lib/gitlab/git/tree_spec.rb

@@ -3,7 +3,7 @@ require "spec_helper"
 describe Gitlab::Git::Tree, :seed_helper do
   let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH, '', 'group/project') }
 
-  context :repo do
+  shared_examples :repo do
     let(:tree) { Gitlab::Git::Tree.where(repository, SeedRepo::Commit::ID) }
 
     it { expect(tree).to be_kind_of Array }
@@ -12,6 +12,17 @@ describe Gitlab::Git::Tree, :seed_helper do
     it { expect(tree.select(&:file?).size).to eq(10) }
     it { expect(tree.select(&:submodule?).size).to eq(2) }
 
+    it 'returns an empty array when called with an invalid ref' do
+      expect(described_class.where(repository, 'foobar-does-not-exist')).to eq([])
+    end
+
+    it 'returns a list of tree objects' do
+      entries = described_class.where(repository, SeedRepo::Commit::ID, 'files', true)
+
+      expect(entries.count).to be >= 5
+      expect(entries).to all(be_a(Gitlab::Git::Tree))
+    end
+
     describe '#dir?' do
       let(:dir) { tree.select(&:dir?).first }
 
@@ -20,8 +31,8 @@ describe Gitlab::Git::Tree, :seed_helper do
       it { expect(dir.commit_id).to eq(SeedRepo::Commit::ID) }
       it { expect(dir.name).to eq('encoding') }
       it { expect(dir.path).to eq('encoding') }
-      it { expect(dir.flat_path).to eq('encoding') }
       it { expect(dir.mode).to eq('40000') }
+      it { expect(dir.flat_path).to eq('encoding') }
 
       context :subdir do
         let(:subdir) { Gitlab::Git::Tree.where(repository, SeedRepo::Commit::ID, 'files').first }
@@ -44,6 +55,51 @@ describe Gitlab::Git::Tree, :seed_helper do
         it { expect(subdir_file.path).to eq('files/ruby/popen.rb') }
         it { expect(subdir_file.flat_path).to eq('files/ruby/popen.rb') }
       end
+
+      context :flat_path do
+        let(:filename) { 'files/flat/path/correct/content.txt' }
+        let(:oid) { create_file(filename) }
+        let(:subdir_file) { Gitlab::Git::Tree.where(repository, oid, 'files/flat').first }
+        let(:repository_rugged) { Rugged::Repository.new(File.join(SEED_STORAGE_PATH, TEST_REPO_PATH)) }
+
+        it { expect(subdir_file.flat_path).to eq('files/flat/path/correct') }
+      end
+
+      def create_file(path)
+        oid = repository_rugged.write('test', :blob)
+        index = repository_rugged.index
+        index.add(path: filename, oid: oid, mode: 0100644)
+
+        options = commit_options(
+          repository_rugged,
+          index,
+          repository_rugged.head.target,
+          'HEAD',
+          'Add new file')
+
+        Rugged::Commit.create(repository_rugged, options)
+      end
+
+      # Build the options hash that's passed to Rugged::Commit#create
+      def commit_options(repo, index, target, ref, message)
+        options = {}
+        options[:tree] = index.write_tree(repo)
+        options[:author] = {
+          email: "test@example.com",
+          name: "Test Author",
+          time: Time.gm(2014, "mar", 3, 20, 15, 1)
+        }
+        options[:committer] = {
+          email: "test@example.com",
+          name: "Test Author",
+          time: Time.gm(2014, "mar", 3, 20, 15, 1)
+        }
+        options[:message] ||= message
+        options[:parents] = repo.empty? ? [] : [target].compact
+        options[:update_ref] = ref
+
+        options
+      end
     end
 
     describe '#file?' do
@@ -79,9 +135,17 @@ describe Gitlab::Git::Tree, :seed_helper do
     end
   end
 
-  describe '#where' do
+  describe '.where with Gitaly enabled' do
-    it 'returns an empty array when called with an invalid ref' do
+    it_behaves_like :repo
-      expect(described_class.where(repository, 'foobar-does-not-exist')).to eq([])
+  end
+
+  describe '.where with Rugged enabled', :enable_rugged do
+    it 'calls out to the Rugged implementation' do
+      allow_any_instance_of(Rugged).to receive(:lookup).with(SeedRepo::Commit::ID)
+
+      described_class.where(repository, SeedRepo::Commit::ID, 'files', false)
     end
+
+    it_behaves_like :repo
   end
 end

spec/lib/gitlab/gitaly_client/storage_settings_spec.rb

@@ -26,4 +26,14 @@ describe Gitlab::GitalyClient::StorageSettings do
       end
     end
   end
+
+  describe '.disk_access_denied?' do
+    it 'return false when Rugged is enabled', :enable_rugged do
+      expect(described_class.disk_access_denied?).to be_falsey
+    end
+
+    it 'returns true' do
+      expect(described_class.disk_access_denied?).to be_truthy
+    end
+  end
 end

spec/lib/gitlab/middleware/basic_health_check_spec.rb

@@ -28,6 +28,35 @@ describe Gitlab::Middleware::BasicHealthCheck do
       end
     end
 
+    context 'with X-Forwarded-For headers' do
+      let(:load_balancer_ip) { '1.2.3.4' }
+
+      before do
+        env['HTTP_X_FORWARDED_FOR'] = "#{load_balancer_ip}, 127.0.0.1"
+        env['REMOTE_ADDR'] = '127.0.0.1'
+        env['PATH_INFO'] = described_class::HEALTH_PATH
+      end
+
+      it 'returns 200 response when endpoint is allowed' do
+        allow(Settings.monitoring).to receive(:ip_whitelist).and_return([load_balancer_ip])
+        expect(app).not_to receive(:call)
+
+        response = middleware.call(env)
+
+        expect(response[0]).to eq(200)
+        expect(response[1]).to eq({ 'Content-Type' => 'text/plain' })
+        expect(response[2]).to eq(['GitLab OK'])
+      end
+
+      it 'returns 404 when whitelist is not configured' do
+        allow(Settings.monitoring).to receive(:ip_whitelist).and_return([])
+
+        response = middleware.call(env)
+
+        expect(response[0]).to eq(404)
+      end
+    end
+
     context 'whitelisted IP' do
       before do
         env['REMOTE_ADDR'] = '127.0.0.1'

spec/lib/gitlab/request_context_spec.rb

@@ -6,6 +6,31 @@ describe Gitlab::RequestContext do
     let(:app) { -> (env) {} }
     let(:env) { Hash.new }
 
+    context 'with X-Forwarded-For headers', :request_store do
+      let(:load_balancer_ip) { '1.2.3.4' }
+      let(:headers) do
+        {
+          'HTTP_X_FORWARDED_FOR' => "#{load_balancer_ip}, 127.0.0.1",
+          'REMOTE_ADDR' => '127.0.0.1'
+        }
+      end
+
+      let(:env) { Rack::MockRequest.env_for("/").merge(headers) }
+
+      it 'returns the load balancer IP' do
+        client_ip = nil
+
+        endpoint = proc do
+          client_ip = Gitlab::SafeRequestStore[:client_ip]
+          [200, {}, ["Hello"]]
+        end
+
+        Rails.application.middleware.build(endpoint).call(env)
+
+        expect(client_ip).to eq(load_balancer_ip)
+      end
+    end
+
     context 'when RequestStore::Middleware is used' do
       around do |example|
         RequestStore::Middleware.new(-> (env) { example.run }).call({})
@@ -15,7 +40,7 @@ describe Gitlab::RequestContext do
         let(:ip) { '192.168.1.11' }
 
         before do
-          allow_any_instance_of(ActionDispatch::Request).to receive(:ip).and_return(ip)
+          allow_any_instance_of(Rack::Request).to receive(:ip).and_return(ip)
           described_class.new(app).call(env)
         end
 

spec/lib/gitlab/route_map_spec.rb

@@ -60,7 +60,7 @@ describe Gitlab::RouteMap do
 
       subject do
         map = described_class.new(<<-"MAP".strip_heredoc)
-        - source: '#{malicious_regexp}'
+        - source: '#{malicious_regexp_re2}'
           public: '/'
         MAP
 

spec/lib/gitlab/sanitizers/exif_spec.rb

@@ -0,0 +1,120 @@
+require 'spec_helper'
+
+describe Gitlab::Sanitizers::Exif do
+  let(:sanitizer) { described_class.new }
+
+  describe '#batch_clean' do
+    context 'with image uploads' do
+      let!(:uploads) { create_list(:upload, 3, :with_file, :issuable_upload) }
+
+      it 'processes all uploads if range ID is not set' do
+        expect(sanitizer).to receive(:clean).exactly(3).times
+
+        sanitizer.batch_clean
+      end
+
+      it 'processes only uploads in the selected range' do
+        expect(sanitizer).to receive(:clean).once
+
+        sanitizer.batch_clean(start_id: uploads[1].id, stop_id: uploads[1].id)
+      end
+
+      it 'pauses if sleep_time is set' do
+        expect(sanitizer).to receive(:sleep).exactly(3).times.with(1.second)
+        expect(sanitizer).to receive(:clean).exactly(3).times
+
+        sanitizer.batch_clean(sleep_time: 1)
+      end
+    end
+
+    it 'filters only jpg/tiff images' do
+      create(:upload, path: 'filename.jpg')
+      create(:upload, path: 'filename.jpeg')
+      create(:upload, path: 'filename.JPG')
+      create(:upload, path: 'filename.tiff')
+      create(:upload, path: 'filename.TIFF')
+      create(:upload, path: 'filename.png')
+      create(:upload, path: 'filename.txt')
+
+      expect(sanitizer).to receive(:clean).exactly(5).times
+      sanitizer.batch_clean
+    end
+  end
+
+  describe '#clean' do
+    let(:uploader) { create(:upload, :with_file, :issuable_upload).build_uploader }
+
+    context "no dry run" do
+      it "removes exif from the image" do
+        uploader.store!(fixture_file_upload('spec/fixtures/rails_sample.jpg'))
+
+        original_upload = uploader.upload
+        expected_args = ["exiftool", "-all=", "-tagsFromFile", "@", *Gitlab::Sanitizers::Exif::EXCLUDE_PARAMS, "--IPTC:all", "--XMP-iptcExt:all", kind_of(String)]
+
+        expect(sanitizer).to receive(:extra_tags).and_return(["", 0])
+        expect(sanitizer).to receive(:exec_remove_exif!).once.and_call_original
+        expect(uploader).to receive(:store!).and_call_original
+        expect(Gitlab::Popen).to receive(:popen).with(expected_args) do |args|
+          File.write("#{args.last}_original", "foo") if args.last.start_with?(Dir.tmpdir)
+
+          [expected_args, 0]
+        end
+
+        sanitizer.clean(uploader, dry_run: false)
+
+        expect(uploader.upload.id).not_to eq(original_upload.id)
+        expect(uploader.upload.path).to eq(original_upload.path)
+      end
+
+      it "ignores image without exif" do
+        expected_args = ["exiftool", "-all", "-j", "-sort", "--IPTC:all", "--XMP-iptcExt:all", kind_of(String)]
+
+        expect(Gitlab::Popen).to receive(:popen).with(expected_args).and_return(["[{}]", 0])
+        expect(sanitizer).not_to receive(:exec_remove_exif!)
+        expect(uploader).not_to receive(:store!)
+
+        sanitizer.clean(uploader, dry_run: false)
+      end
+
+      it "raises an error if the exiftool fails with an error" do
+        expect(Gitlab::Popen).to receive(:popen).and_return(["error", 1])
+
+        expect { sanitizer.clean(uploader, dry_run: false) }.to raise_exception(RuntimeError, "failed to get exif tags: error")
+      end
+    end
+
+    context "dry run" do
+      it "doesn't change the image" do
+        expect(sanitizer).to receive(:extra_tags).and_return({ 'foo' => 'bar' })
+        expect(sanitizer).not_to receive(:exec_remove_exif!)
+        expect(uploader).not_to receive(:store!)
+
+        sanitizer.clean(uploader, dry_run: true)
+      end
+    end
+  end
+
+  describe "#extra_tags" do
+    it "returns a list of keys for exif file" do
+      tags = '[{
+                "DigitalSourceType": "some source",
+                "ImageHeight": 654
+              }]'
+
+      expect(Gitlab::Popen).to receive(:popen).and_return([tags, 0])
+
+      expect(sanitizer.extra_tags('filename')).not_to be_empty
+    end
+
+    it "returns an empty list for file with only whitelisted and ignored tags" do
+      tags = '[{
+                "ImageHeight": 654,
+                "Megapixels": 0.641
+              }]'
+
+      expect(Gitlab::Popen).to receive(:popen).and_return([tags, 0])
+
+      expect(sanitizer.extra_tags('some file')).to be_empty
+    end
+  end
+end

spec/lib/gitlab/untrusted_regexp/ruby_syntax_spec.rb

@@ -0,0 +1,72 @@
+require 'fast_spec_helper'
+require 'support/shared_examples/malicious_regexp_shared_examples'
+
+describe Gitlab::UntrustedRegexp::RubySyntax do
+  describe '.matches_syntax?' do
+    it 'returns true if regexp is valid' do
+      expect(described_class.matches_syntax?('/some .* thing/'))
+        .to be true
+    end
+
+    it 'returns true if regexp is invalid, but resembles regexp' do
+      expect(described_class.matches_syntax?('/some ( thing/'))
+        .to be true
+    end
+  end
+
+  describe '.valid?' do
+    it 'returns true if regexp is valid' do
+      expect(described_class.valid?('/some .* thing/'))
+        .to be true
+    end
+
+    it 'returns false if regexp is invalid' do
+      expect(described_class.valid?('/some ( thing/'))
+        .to be false
+    end
+  end
+
+  describe '.fabricate' do
+    context 'when regexp is valid' do
+      it 'fabricates regexp without flags' do
+        expect(described_class.fabricate('/some .* thing/')).not_to be_nil
+      end
+    end
+
+    context 'when regexp is a raw pattern' do
+      it 'returns error' do
+        expect(described_class.fabricate('some .* thing')).to be_nil
+      end
+    end
+  end
+
+  describe '.fabricate!' do
+    context 'when regexp is using /regexp/ scheme with flags' do
+      it 'fabricates regexp with a single flag' do
+        regexp = described_class.fabricate!('/something/i')
+
+        expect(regexp).to eq Gitlab::UntrustedRegexp.new('(?i)something')
+        expect(regexp.scan('SOMETHING')).to be_one
+      end
+
+      it 'fabricates regexp with multiple flags' do
+        regexp = described_class.fabricate!('/something/im')
+
+        expect(regexp).to eq Gitlab::UntrustedRegexp.new('(?im)something')
+      end
+
+      it 'fabricates regexp without flags' do
+        regexp = described_class.fabricate!('/something/')
+
+        expect(regexp).to eq Gitlab::UntrustedRegexp.new('something')
+      end
+    end
+
+    context 'when regexp is a raw pattern' do
+      it 'raises an error' do
+        expect { described_class.fabricate!('some .* thing') }
+          .to raise_error(RegexpError)
+      end
+    end
+  end
+end

spec/lib/gitlab/untrusted_regexp_spec.rb

@@ -2,48 +2,6 @@ require 'fast_spec_helper'
 require 'support/shared_examples/malicious_regexp_shared_examples'
 
 describe Gitlab::UntrustedRegexp do
-  describe '.valid?' do
-    it 'returns true if regexp is valid' do
-      expect(described_class.valid?('/some ( thing/'))
-        .to be false
-    end
-
-    it 'returns true if regexp is invalid' do
-      expect(described_class.valid?('/some .* thing/'))
-        .to be true
-    end
-  end
-
-  describe '.fabricate' do
-    context 'when regexp is using /regexp/ scheme with flags' do
-      it 'fabricates regexp with a single flag' do
-        regexp = described_class.fabricate('/something/i')
-
-        expect(regexp).to eq described_class.new('(?i)something')
-        expect(regexp.scan('SOMETHING')).to be_one
-      end
-
-      it 'fabricates regexp with multiple flags' do
-        regexp = described_class.fabricate('/something/im')
-
-        expect(regexp).to eq described_class.new('(?im)something')
-      end
-
-      it 'fabricates regexp without flags' do
-        regexp = described_class.fabricate('/something/')
-
-        expect(regexp).to eq described_class.new('something')
-      end
-    end
-
-    context 'when regexp is a raw pattern' do
-      it 'raises an error' do
-        expect { described_class.fabricate('some .* thing') }
-          .to raise_error(RegexpError)
-      end
-    end
-  end
-
   describe '#initialize' do
     subject { described_class.new(pattern) }
 
@@ -92,11 +50,41 @@ describe Gitlab::UntrustedRegexp do
     end
   end
 
-  describe '#scan' do
+  describe '#match?' do
-    subject { described_class.new(regexp).scan(text) }
+    subject { described_class.new(regexp).match?(text) }
+
     context 'malicious regexp' do
       let(:text) { malicious_text }
-      let(:regexp) { malicious_regexp }
+      let(:regexp) { malicious_regexp_re2 }
+
+      include_examples 'malicious regexp'
+    end
+
+    context 'matching regexp' do
+      let(:regexp) { 'foo' }
+      let(:text) { 'foo' }
+
+      it 'returns an array of nil matches' do
+        is_expected.to eq(true)
+      end
+    end
+
+    context 'non-matching regexp' do
+      let(:regexp) { 'boo' }
+      let(:text) { 'foo' }
+
+      it 'returns an array of nil matches' do
+        is_expected.to eq(false)
+      end
+    end
+  end
+
+  describe '#scan' do
+    subject { described_class.new(regexp).scan(text) }
+
+    context 'malicious regexp' do
+      let(:text) { malicious_text }
+      let(:regexp) { malicious_regexp_re2 }
 
       include_examples 'malicious regexp'
     end

spec/models/application_record_spec.rb

@@ -11,6 +11,25 @@ describe ApplicationRecord do
     end
   end
 
+  describe '.safe_ensure_unique' do
+    let(:model) { build(:suggestion) }
+    let(:klass) { model.class }
+
+    before do
+      allow(model).to receive(:save).and_raise(ActiveRecord::RecordNotUnique)
+    end
+
+    it 'returns false when ActiveRecord::RecordNotUnique is raised' do
+      expect(model).to receive(:save).once
+      expect(klass.safe_ensure_unique { model.save }).to be_falsey
+    end
+
+    it 'retries based on retry count specified' do
+      expect(model).to receive(:save).exactly(3).times
+      expect(klass.safe_ensure_unique(retries: 2) { model.save }).to be_falsey
+    end
+  end
+
   describe '.safe_find_or_create_by' do
     it 'creates the user avoiding race conditions' do
       expect(Suggestion).to receive(:find_or_create_by).and_raise(ActiveRecord::RecordNotUnique)

spec/models/commit_spec.rb

@@ -542,7 +542,7 @@ eos
     end
   end
 
-  describe '#uri_type' do
+  shared_examples '#uri_type' do
     it 'returns the URI type at the given path' do
       expect(commit.uri_type('files/html')).to be(:tree)
       expect(commit.uri_type('files/images/logo-black.png')).to be(:raw)
@@ -561,6 +561,20 @@ eos
     end
   end
 
+  describe '#uri_type with Gitaly enabled' do
+    it_behaves_like "#uri_type"
+  end
+
+  describe '#uri_type with Rugged enabled', :enable_rugged do
+    it 'calls out to the Rugged implementation' do
+      allow_any_instance_of(Rugged::Tree).to receive(:path).with('files/html').and_call_original
+
+      commit.uri_type('files/html')
+    end
+
+    it_behaves_like '#uri_type'
+  end
+
   describe '.from_hash' do
     let(:new_commit) { described_class.from_hash(commit.to_hash, project) }
 

spec/models/merge_request_diff_spec.rb

@@ -1,6 +1,8 @@
 require 'spec_helper'
 
 describe MergeRequestDiff do
+  include RepoHelpers
+
   let(:diff_with_commits) { create(:merge_request).merge_request_diff }
 
   describe 'validations' do
@@ -194,6 +196,25 @@ describe MergeRequestDiff do
         expect(diff_file).to be_binary
         expect(diff_file.diff).to eq(mr_diff.compare.diffs(paths: [path]).to_a.first.diff)
       end
+
+      context 'with diffs that contain a null byte' do
+        let(:filename) { 'test-null.txt' }
+        let(:content) { "a" * 10000 + "\x00" }
+        let(:project) { create(:project, :repository) }
+        let(:branch) { 'null-data' }
+        let(:target_branch) { 'master' }
+
+        it 'saves diffs correctly' do
+          create_file_in_repo(project, target_branch, branch, filename, content)
+
+          mr_diff = create(:merge_request, target_project: project, source_project: project, source_branch: branch, target_branch: target_branch).merge_request_diff
+          diff_file = mr_diff.merge_request_diff_files.find_by(new_path: filename)
+
+          expect(diff_file).to be_binary
+          expect(diff_file.diff).to eq(mr_diff.compare.diffs(paths: [filename]).to_a.first.diff)
+          expect(diff_file.diff).to include(content)
+        end
+      end
     end
   end
 

spec/models/notification_recipient_spec.rb

@@ -7,11 +7,43 @@ describe NotificationRecipient do
 
   subject(:recipient) { described_class.new(user, :watch, target: target, project: project) }
 
-  it 'denies access to a target when cross project access is denied' do
+  describe '#has_access?' do
-    allow(Ability).to receive(:allowed?).and_call_original
+    before do
-    expect(Ability).to receive(:allowed?).with(user, :read_cross_project, :global).and_return(false)
+      allow(user).to receive(:can?).and_call_original
+    end
 
-    expect(recipient.has_access?).to be_falsy
+    context 'user cannot read cross project' do
+      it 'returns false' do
+        expect(user).to receive(:can?).with(:read_cross_project).and_return(false)
+        expect(recipient.has_access?).to eq false
+      end
+    end
+
+    context 'user cannot read build' do
+      let(:target) { build(:ci_pipeline) }
+
+      it 'returns false' do
+        expect(user).to receive(:can?).with(:read_build, target).and_return(false)
+        expect(recipient.has_access?).to eq false
+      end
+    end
+
+    context 'user cannot read commit' do
+      let(:target) { build(:commit) }
+
+      it 'returns false' do
+        expect(user).to receive(:can?).with(:read_commit, target).and_return(false)
+        expect(recipient.has_access?).to eq false
+      end
+    end
+
+    context 'target has no policy' do
+      let(:target) { double.as_null_object }
+
+      it 'returns true' do
+        expect(recipient.has_access?).to eq true
+      end
+    end
   end
 
   context '#notification_setting' do

spec/models/repository_spec.rb

@@ -2214,7 +2214,7 @@ describe Repository do
     rugged.references.create("refs/remotes/#{remote_name}/#{branch_name}", target.id)
   end
 
-  describe '#ancestor?' do
+  shared_examples '#ancestor?' do
     let(:commit) { repository.commit }
     let(:ancestor) { commit.parents.first }
 
@@ -2238,6 +2238,20 @@ describe Repository do
     end
   end
 
+  describe '#ancestor? with Gitaly enabled' do
+    it_behaves_like "#ancestor?"
+  end
+
+  describe '#ancestor? with Rugged enabled', :enable_rugged do
+    it 'calls out to the Rugged implementation' do
+      allow_any_instance_of(Rugged).to receive(:merge_base).with(repository.commit.id, Gitlab::Git::BLANK_SHA).and_call_original
+
+      repository.ancestor?(repository.commit.id, Gitlab::Git::BLANK_SHA)
+    end
+
+    it_behaves_like '#ancestor?'
+  end
+
   describe '#archive_metadata' do
     let(:ref) { 'master' }
     let(:storage_path) { '/tmp' }

spec/policies/project_policy_spec.rb

@@ -15,7 +15,7 @@ describe ProjectPolicy do
       read_project_for_iids read_issue_iid read_label
       read_milestone read_project_snippet read_project_member read_note
       create_project create_issue create_note upload_file create_merge_request_in
-      award_emoji read_release
+      award_emoji
     ]
   end
 
@@ -24,7 +24,7 @@ describe ProjectPolicy do
       download_code fork_project create_project_snippet update_issue
       admin_issue admin_label admin_list read_commit_status read_build
       read_container_image read_pipeline read_environment read_deployment
-      read_merge_request download_wiki_code read_sentry_issue
+      read_merge_request download_wiki_code read_sentry_issue read_release
     ]
   end
 

spec/requests/api/releases_spec.rb

@@ -4,12 +4,14 @@ describe API::Releases do
   let(:project) { create(:project, :repository, :private) }
   let(:maintainer) { create(:user) }
   let(:reporter) { create(:user) }
+  let(:guest) { create(:user) }
   let(:non_project_member) { create(:user) }
   let(:commit) { create(:commit, project: project) }
 
   before do
     project.add_maintainer(maintainer)
     project.add_reporter(reporter)
+    project.add_guest(guest)
 
     project.repository.add_tag(maintainer, 'v0.1', commit.id)
     project.repository.add_tag(maintainer, 'v0.2', commit.id)
@@ -66,6 +68,24 @@ describe API::Releases do
       end
     end
 
+    context 'when user is a guest' do
+      it 'responds 403 Forbidden' do
+        get api("/projects/#{project.id}/releases", guest)
+
+        expect(response).to have_gitlab_http_status(:forbidden)
+      end
+
+      context 'when project is public' do
+        let(:project) { create(:project, :repository, :public) }
+
+        it 'responds 200 OK' do
+          get api("/projects/#{project.id}/releases", guest)
+
+          expect(response).to have_gitlab_http_status(:ok)
+        end
+      end
+    end
+
     context 'when user is not a project member' do
       it 'cannot find the project' do
         get api("/projects/#{project.id}/releases", non_project_member)
@@ -189,6 +209,24 @@ describe API::Releases do
           end
         end
       end
+
+      context 'when user is a guest' do
+        it 'responds 403 Forbidden' do
+          get api("/projects/#{project.id}/releases/v0.1", guest)
+
+          expect(response).to have_gitlab_http_status(:forbidden)
+        end
+
+        context 'when project is public' do
+          let(:project) { create(:project, :repository, :public) }
+
+          it 'responds 200 OK' do
+            get api("/projects/#{project.id}/releases/v0.1", guest)
+
+            expect(response).to have_gitlab_http_status(:ok)
+          end
+        end
+      end
     end
 
     context 'when specified tag is not found in the project' do

spec/services/labels/available_labels_service_spec.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+require 'spec_helper'
+
+describe Labels::AvailableLabelsService do
+  let(:user) { create(:user) }
+  let(:project) { create(:project, :public, group: group) }
+  let(:group) { create(:group) }
+
+  let(:project_label) { create(:label, project: project) }
+  let(:other_project_label) { create(:label) }
+  let(:group_label) { create(:group_label, group: group) }
+  let(:other_group_label) { create(:group_label) }
+  let(:labels) { [project_label, other_project_label, group_label, other_group_label] }
+
+  context '#find_or_create_by_titles' do
+    let(:label_titles) { labels.map(&:title).push('non existing title') }
+
+    context 'when parent is a project' do
+      context 'when a user is not a project member' do
+        it 'returns only relevant label ids' do
+          result = described_class.new(user, project, labels: label_titles).find_or_create_by_titles
+
+          expect(result).to match_array([project_label, group_label])
+        end
+      end
+
+      context 'when a user is a project member' do
+        before do
+          project.add_developer(user)
+        end
+
+        it 'creates new labels for not found titles' do
+          result = described_class.new(user, project, labels: label_titles).find_or_create_by_titles
+
+          expect(result.count).to eq(5)
+          expect(result).to include(project_label, group_label)
+          expect(result).not_to include(other_project_label, other_group_label)
+        end
+      end
+    end
+
+    context 'when parent is a group' do
+      context 'when a user is not a group member' do
+        it 'returns only relevant label ids' do
+          result = described_class.new(user, group, labels: label_titles).find_or_create_by_titles
+
+          expect(result).to match_array([group_label])
+        end
+      end
+
+      context 'when a user is a group member' do
+        before do
+          group.add_developer(user)
+        end
+
+        it 'creates new labels for not found titles' do
+          result = described_class.new(user, group, labels: label_titles).find_or_create_by_titles
+
+          expect(result.count).to eq(5)
+          expect(result).to include(group_label)
+          expect(result).not_to include(project_label, other_project_label, other_group_label)
+        end
+      end
+    end
+  end
+
+  context '#filter_labels_ids_in_param' do
+    let(:label_ids) { labels.map(&:id).push(99999) }
+
+    context 'when parent is a project' do
+      it 'returns only relevant label ids' do
+        result = described_class.new(user, project, ids: label_ids).filter_labels_ids_in_param(:ids)
+
+        expect(result).to match_array([project_label.id, group_label.id])
+      end
+    end
+
+    context 'when parent is a group' do
+      it 'returns only relevant label ids' do
+        result = described_class.new(user, group, ids: label_ids).filter_labels_ids_in_param(:ids)
+
+        expect(result).to match_array([group_label.id])
+      end
+    end
+  end
+end

spec/spec_helper.rb

@@ -115,10 +115,17 @@ RSpec.configure do |config|
     TestEnv.clean_test_path
   end
 
-  config.before do
+  config.before do |example|
     # Enable all features by default for testing
     allow(Feature).to receive(:enabled?) { true }
 
+    enabled = example.metadata[:enable_rugged].present?
+
+    # Disable Rugged features by default
+    Gitlab::Git::RuggedImpl::Repository::FEATURE_FLAGS.each do |flag|
+      allow(Feature).to receive(:enabled?).with(flag).and_return(enabled)
+    end
+
     # The following can be removed when we remove the staged rollout strategy
     # and we can just enable it using instance wide settings
     # (ie. ApplicationSetting#auto_devops_enabled)

spec/support/helpers/repo_helpers.rb

@@ -115,4 +115,18 @@ eos
       commits: commits
     )
   end
+
+  def create_file_in_repo(
+        project, start_branch, branch_name, filename, content,
+        commit_message: 'Add new content')
+    Files::CreateService.new(
+      project,
+      project.owner,
+      commit_message: commit_message,
+      start_branch: start_branch,
+      branch_name: branch_name,
+      file_path: filename,
+      file_content: content
+    ).execute
+  end
 end

spec/support/shared_examples/malicious_regexp_shared_examples.rb

@@ -2,7 +2,8 @@ require 'timeout'
 
 shared_examples 'malicious regexp' do
   let(:malicious_text) { 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa!' }
-  let(:malicious_regexp) { '(?i)^(([a-z])+.)+[A-Z]([a-z])+$' }
+  let(:malicious_regexp_re2) { '(?i)^(([a-z])+.)+[A-Z]([a-z])+$' }
+  let(:malicious_regexp_ruby) { '/^(([a-z])+.)+[A-Z]([a-z])+$/i' }
 
   it 'takes under a second' do
     expect { Timeout.timeout(1) { subject } }.not_to raise_error

spec/validators/sha_validator_spec.rb

@@ -2,7 +2,7 @@ require 'spec_helper'
 
 describe ShaValidator do
   let(:validator) { described_class.new(attributes: [:base_commit_sha]) }
-  let(:merge_diff) { build(:merge_request_diff) }
+  let!(:merge_diff) { build(:merge_request_diff) }
 
   subject { validator.validate_each(merge_diff, :base_commit_sha, value) }
 
@@ -10,6 +10,8 @@ describe ShaValidator do
     let(:value) { nil }
 
     it 'does not add any error if value is empty' do
+      expect(Commit).not_to receive(:valid_hash?)
+
       subject
 
       expect(merge_diff.errors).to be_empty
@@ -19,7 +21,9 @@ describe ShaValidator do
   context 'with valid sha' do
     let(:value) { Digest::SHA1.hexdigest(SecureRandom.hex) }
 
-    it 'does not add any error if value is empty' do
+    it 'does not add any error' do
+      expect(Commit).to receive(:valid_hash?).and_call_original
+
       subject
 
       expect(merge_diff.errors).to be_empty
@@ -30,6 +34,7 @@ describe ShaValidator do
     let(:value) { 'foo' }
 
     it 'adds error to the record' do
+      expect(Commit).to receive(:valid_hash?).and_call_original
       expect(merge_diff.errors).to be_empty
 
       subject