From daf83fa62c940b0da7dc4e0893586b6a9a2dbbf9 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 19 Dec 2016 09:37:16 +0000 Subject: [PATCH 1/3] [8.13 Backport] Merge branch '25301-git-2.11-force-push-bug' into 'master' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Accept environment variables from the `pre-receive` script 1. Starting version 2.11, git changed the way the pre-receive flow works. - Previously, the new potential objects would be added to the main repo. If the pre-receive passes, the new objects stay in the repo but are linked up. If the pre-receive fails, the new objects stay orphaned in the repo, and are cleaned up during the next `git gc`. - In 2.11, the new potential objects are added to a temporary "alternate object directory", that git creates for this purpose. If the pre-receive passes, the objects from the alternate object directory are migrated to the main repo. If the pre-receive fails the alternate object directory is simply deleted. 2. In our workflow, the pre-recieve script (in `gitlab-shell`) calls the `/allowed` endpoint, which calls out directly to git to perform various checks. These direct calls to git do _not_ have the necessary environment variables set which allow access to the "alternate object directory" (explained above). Therefore these calls to git are not able to access any of the new potential objects to be added during this push. 3. We fix this by accepting the relevant environment variables (`GIT_ALTERNATE_OBJECT_DIRECTORIES`, `GIT_OBJECT_DIRECTORY`, and `GIT_QUARANTINE_PATH`) on the `/allowed` endpoint, and then include these environment variables while calling out to git. 4. This commit includes these environment variables while making the "force push" check. See https://gitlab.com/gitlab-org/gitlab-shell/merge_requests/120 Signed-off-by: Rémy Coutable --- .../unreleased/25301-git-2-11-force-push-bug.yml | 4 ++ lib/api/internal.rb | 14 +++++- lib/gitlab/checks/change_access.rb | 5 +- lib/gitlab/checks/force_push.rb | 11 +++-- lib/gitlab/git/rev_list.rb | 42 +++++++++++++++++ lib/gitlab/git_access.rb | 5 +- lib/gitlab/popen.rb | 4 +- spec/lib/gitlab/checks/force_push_spec.rb | 19 ++++++++ spec/lib/gitlab/git/rev_list_spec.rb | 53 ++++++++++++++++++++++ 9 files changed, 147 insertions(+), 10 deletions(-) create mode 100644 changelogs/unreleased/25301-git-2-11-force-push-bug.yml create mode 100644 lib/gitlab/git/rev_list.rb create mode 100644 spec/lib/gitlab/checks/force_push_spec.rb create mode 100644 spec/lib/gitlab/git/rev_list_spec.rb diff --git a/changelogs/unreleased/25301-git-2-11-force-push-bug.yml b/changelogs/unreleased/25301-git-2-11-force-push-bug.yml new file mode 100644 index 0000000..afe5772 --- /dev/null +++ b/changelogs/unreleased/25301-git-2-11-force-push-bug.yml @@ -0,0 +1,4 @@ +--- +title: Accept environment variables from the `pre-receive` script +merge_request: 7967 +author: diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 9a5d1ec..89e47a7 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -43,6 +43,14 @@ module API :push_code ] end + + def parse_allowed_environment_variables + return if params[:env].blank? + + JSON.parse(params[:env]) + + rescue JSON::ParserError + end end post "/allowed" do @@ -61,7 +69,11 @@ module API if wiki? Gitlab::GitAccessWiki.new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities) else - Gitlab::GitAccess.new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities) + Gitlab::GitAccess.new(actor, + project, + protocol, + authentication_abilities: ssh_authentication_abilities, + env: parse_allowed_environment_variables) end access_status = access.check(params[:action], params[:changes]) diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb index cb10652..3d20301 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/change_access.rb @@ -3,11 +3,12 @@ module Gitlab class ChangeAccess attr_reader :user_access, :project - def initialize(change, user_access:, project:) + def initialize(change, user_access:, project:, env: {}) @oldrev, @newrev, @ref = change.values_at(:oldrev, :newrev, :ref) @branch_name = Gitlab::Git.branch_name(@ref) @user_access = user_access @project = project + @env = env end def exec @@ -68,7 +69,7 @@ module Gitlab end def forced_push? - Gitlab::Checks::ForcePush.force_push?(@project, @oldrev, @newrev) + Gitlab::Checks::ForcePush.force_push?(@project, @oldrev, @newrev, env: @env) end def matching_merge_request? diff --git a/lib/gitlab/checks/force_push.rb b/lib/gitlab/checks/force_push.rb index 5fe8655..de0c904 100644 --- a/lib/gitlab/checks/force_push.rb +++ b/lib/gitlab/checks/force_push.rb @@ -1,15 +1,20 @@ module Gitlab module Checks class ForcePush - def self.force_push?(project, oldrev, newrev) + def self.force_push?(project, oldrev, newrev, env: {}) return false if project.empty_repo? # Created or deleted branch if Gitlab::Git.blank_ref?(oldrev) || Gitlab::Git.blank_ref?(newrev) false else - missed_ref, _ = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} --git-dir=#{project.repository.path_to_repo} rev-list --max-count=1 #{oldrev} ^#{newrev})) - missed_ref.present? + missed_ref, exit_status = Gitlab::Git::RevList.new(oldrev, newrev, project: project, env: env).execute + + if exit_status == 0 + missed_ref.present? + else + raise "Got a non-zero exit code while calling out to `git rev-list` in the force-push check." + end end end end diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb new file mode 100644 index 0000000..25e9d61 --- /dev/null +++ b/lib/gitlab/git/rev_list.rb @@ -0,0 +1,42 @@ +module Gitlab + module Git + class RevList + attr_reader :project, :env + + ALLOWED_VARIABLES = %w[GIT_OBJECT_DIRECTORY GIT_ALTERNATE_OBJECT_DIRECTORIES].freeze + + def initialize(oldrev, newrev, project:, env: nil) + @project = project + @env = env.presence || {} + @args = [Gitlab.config.git.bin_path, + "--git-dir=#{project.repository.path_to_repo}", + "rev-list", + "--max-count=1", + oldrev, + "^#{newrev}"] + end + + def execute + Gitlab::Popen.popen(@args, nil, parse_environment_variables) + end + + def valid? + environment_variables.all? do |(name, value)| + value.start_with?(project.repository.path_to_repo) + end + end + + private + + def parse_environment_variables + return {} unless valid? + + environment_variables + end + + def environment_variables + @environment_variables ||= env.slice(*ALLOWED_VARIABLES) + end + end + end +end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index bcbf645..74e8713 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -17,12 +17,13 @@ module Gitlab attr_reader :actor, :project, :protocol, :user_access, :authentication_abilities - def initialize(actor, project, protocol, authentication_abilities:) + def initialize(actor, project, protocol, authentication_abilities:, env: {}) @actor = actor @project = project @protocol = protocol @authentication_abilities = authentication_abilities @user_access = UserAccess.new(user, project: project) + @env = env end def check(cmd, changes) @@ -99,7 +100,7 @@ module Gitlab end def change_access_check(change) - Checks::ChangeAccess.new(change, user_access: user_access, project: project).exec + Checks::ChangeAccess.new(change, user_access: user_access, project: project, env: @env).exec end def protocol_allowed? diff --git a/lib/gitlab/popen.rb b/lib/gitlab/popen.rb index cc74bb2..4bc5cda 100644 --- a/lib/gitlab/popen.rb +++ b/lib/gitlab/popen.rb @@ -5,13 +5,13 @@ module Gitlab module Popen extend self - def popen(cmd, path = nil) + def popen(cmd, path = nil, vars = {}) unless cmd.is_a?(Array) raise "System commands must be given as an array of strings" end path ||= Dir.pwd - vars = { "PWD" => path } + vars['PWD'] = path options = { chdir: path } unless File.directory?(path) diff --git a/spec/lib/gitlab/checks/force_push_spec.rb b/spec/lib/gitlab/checks/force_push_spec.rb new file mode 100644 index 0000000..f628801 --- /dev/null +++ b/spec/lib/gitlab/checks/force_push_spec.rb @@ -0,0 +1,19 @@ +require 'spec_helper' + +describe Gitlab::Checks::ChangeAccess, lib: true do + let(:project) { create(:project) } + + context "exit code checking" do + it "does not raise a runtime error if the `popen` call to git returns a zero exit code" do + allow(Gitlab::Popen).to receive(:popen).and_return(['normal output', 0]) + + expect { Gitlab::Checks::ForcePush.force_push?(project, 'oldrev', 'newrev') }.not_to raise_error + end + + it "raises a runtime error if the `popen` call to git returns a non-zero exit code" do + allow(Gitlab::Popen).to receive(:popen).and_return(['error', 1]) + + expect { Gitlab::Checks::ForcePush.force_push?(project, 'oldrev', 'newrev') }.to raise_error(RuntimeError) + end + end +end diff --git a/spec/lib/gitlab/git/rev_list_spec.rb b/spec/lib/gitlab/git/rev_list_spec.rb new file mode 100644 index 0000000..444639a --- /dev/null +++ b/spec/lib/gitlab/git/rev_list_spec.rb @@ -0,0 +1,53 @@ +require 'spec_helper' + +describe Gitlab::Git::RevList, lib: true do + let(:project) { create(:project) } + + context "validations" do + described_class::ALLOWED_VARIABLES.each do |var| + context var do + it "accepts values starting with the project repo path" do + env = { var => "#{project.repository.path_to_repo}/objects" } + rev_list = described_class.new('oldrev', 'newrev', project: project, env: env) + + expect(rev_list).to be_valid + end + + it "rejects values starting not with the project repo path" do + env = { var => "/some/other/path" } + rev_list = described_class.new('oldrev', 'newrev', project: project, env: env) + + expect(rev_list).not_to be_valid + end + + it "rejects values containing the project repo path but not starting with it" do + env = { var => "/some/other/path/#{project.repository.path_to_repo}" } + rev_list = described_class.new('oldrev', 'newrev', project: project, env: env) + + expect(rev_list).not_to be_valid + end + end + end + end + + context "#execute" do + let(:env) { { "GIT_OBJECT_DIRECTORY" => project.repository.path_to_repo } } + let(:rev_list) { Gitlab::Git::RevList.new('oldrev', 'newrev', project: project, env: env) } + + it "calls out to `popen` without environment variables if the record is invalid" do + allow(rev_list).to receive(:valid?).and_return(false) + + expect(Open3).to receive(:popen3).with(hash_excluding(env), any_args) + + rev_list.execute + end + + it "calls out to `popen` with environment variables if the record is valid" do + allow(rev_list).to receive(:valid?).and_return(true) + + expect(Open3).to receive(:popen3).with(hash_including(env), any_args) + + rev_list.execute + end + end +end -- 2.10.2 From 0ce20138298eaebfb9e8225d21e7b0088716e5ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Tue, 20 Dec 2016 09:45:37 +0100 Subject: [PATCH 2/3] Reject blank environment vcariables in Gitlab::Git::RevList MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- lib/gitlab/git/rev_list.rb | 4 ++-- spec/lib/gitlab/git/rev_list_spec.rb | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb index 25e9d61..79dd0cf 100644 --- a/lib/gitlab/git/rev_list.rb +++ b/lib/gitlab/git/rev_list.rb @@ -22,7 +22,7 @@ module Gitlab def valid? environment_variables.all? do |(name, value)| - value.start_with?(project.repository.path_to_repo) + value.to_s.start_with?(project.repository.path_to_repo) end end @@ -35,7 +35,7 @@ module Gitlab end def environment_variables - @environment_variables ||= env.slice(*ALLOWED_VARIABLES) + @environment_variables ||= env.slice(*ALLOWED_VARIABLES).compact end end end diff --git a/spec/lib/gitlab/git/rev_list_spec.rb b/spec/lib/gitlab/git/rev_list_spec.rb index 444639a..1f9c987 100644 --- a/spec/lib/gitlab/git/rev_list_spec.rb +++ b/spec/lib/gitlab/git/rev_list_spec.rb @@ -26,6 +26,13 @@ describe Gitlab::Git::RevList, lib: true do expect(rev_list).not_to be_valid end + + it "ignores nil values" do + env = { var => nil } + rev_list = described_class.new('oldrev', 'newrev', project: project, env: env) + + expect(rev_list).to be_valid + end end end end -- 2.10.2 From b54b031638e7a98c1e51b369cff53602db40e4b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Mon, 6 Feb 2017 10:04:21 +0100 Subject: [PATCH 3/3] Update gitlab-shell to 3.6.7 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- changelogs/unreleased/use-gitlab-shell-3-6-7.yml | 4 ++++ doc/update/8.12-to-8.13.md | 4 ++-- 3 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 changelogs/unreleased/use-gitlab-shell-3-6-7.yml diff --git a/changelogs/unreleased/use-gitlab-shell-3-6-7.yml b/changelogs/unreleased/use-gitlab-shell-3-6-7.yml new file mode 100644 index 0000000..c6600ce --- /dev/null +++ b/changelogs/unreleased/use-gitlab-shell-3-6-7.yml @@ -0,0 +1,4 @@ +--- +title: Use gitlab-shell v3.6.7 +merge_request: +author: diff --git a/doc/update/8.12-to-8.13.md b/doc/update/8.12-to-8.13.md index c0084d9..6457ec9 100644 --- a/doc/update/8.12-to-8.13.md +++ b/doc/update/8.12-to-8.13.md @@ -72,7 +72,7 @@ sudo -u git -H git checkout 8-13-stable-ee ```bash cd /home/git/gitlab-shell sudo -u git -H git fetch --all --tags -sudo -u git -H git checkout v3.6.6 +sudo -u git -H git checkout v3.6.7 ``` ### 6. Update gitlab-workhorse @@ -166,7 +166,7 @@ See [smtp_settings.rb.sample] as an example. Ensure you're still up-to-date with the latest init script changes: sudo cp lib/support/init.d/gitlab /etc/init.d/gitlab - + For Ubuntu 16.04.1 LTS: sudo systemctl daemon-reload -- 2.10.2