Build gitlab-workhorse binary from workhorse directory

debian/clean

@@ -10,3 +10,23 @@ debian/upstream-config-file-list.new
 
 # We have rails > 6.0.2
 config/initializers/active_record_fix_insert_all.rb
+
+# For workhorse
+## Failing tests that require test data from the Internet:
+workhorse/main_test.go
+workhorse/authorization_test.go
+workhorse/channel_test.go
+workhorse/gitaly_integration_test.go
+workhorse/gitaly_test.go
+workhorse/jobs_test.go
+workhorse/sendfile_test.go
+workhorse/upload_test.go
+workhorse/internal/upload/*_test.go
+workhorse/internal/objectstore/*_test.go
+workhorse/internal/redis/*_test.go
+## Need definitions from excluded test files
+workhorse/cable_test.go
+## Need github.com/johannesboyne/gofakes3/backend/s3mem
+workhorse/internal/objectstore/s3_object_test.go
+workhorse/internal/objectstore/test/s3_stub.go
+workhorse/internal/filestore/file_handler_test.go

debian/control

@@ -7,12 +7,57 @@ Uploaders: Cédric Boutillier <boutil@debian.org>,
  Balasankar C <balasankarc@autistici.org>,
  Sruthi Chandran <srud@debian.org>,
  Utkarsh Gupta <utkarsh@debian.org>
-Build-Depends: debhelper (>= 10~), gem2deb, bc
+Build-Depends: debhelper (>= 10~),
+               gem2deb,
+               dh-golang,
+               golang-any,
+               bc,
+               golang-github-alecthomas-chroma-dev,
+               golang-github-aws-aws-sdk-go-dev,
+               golang-github-azure-azure-storage-blob-go-dev (>= 0.10~),
+               golang-github-beorn7-perks-dev,
+               golang-github-client9-reopen-dev,
+               golang-github-davecgh-go-spew-dev,
+               golang-github-dgrijalva-jwt-go-dev (>= 3.2.0~),
+               golang-github-disintegration-imaging-dev,
+               golang-github-fzambia-sentinel-dev,
+               golang-github-garyburd-redigo-dev,
+               golang-github-google-uuid-dev,
+               golang-github-gomodule-redigo-dev,
+               golang-github-grpc-ecosystem-go-grpc-prometheus-dev,
+               golang-github-jfbus-httprs-dev,
+                 golang-github-jtolds-gls-dev (>= 4.20~),
+               golang-github-jpillora-backoff-dev,
+               golang-github-mitchellh-copystructure-dev,
+               golang-github-mitchellh-reflectwalk-dev,
+               golang-github-pmezard-go-difflib-dev,
+               golang-github-prometheus-client-golang-dev,
+               golang-github-prometheus-client-model-dev,
+               golang-github-prometheus-common-dev,
+               golang-github-ryszard-goskiplist-dev,
+               golang-github-sebest-xff-dev,
+               golang-github-sirupsen-logrus-dev,
+               golang-github-stretchr-testify-dev (>= 1.4~),
+               golang-gitlab-gitlab-org-labkit-dev (>= 1.3.0-4~),
+               golang-gocloud-dev,
+               golang-golang-x-crypto-dev,
+               golang-golang-x-net-dev,
+               golang-golang-x-sys-dev (>= 0.0~git20180510.7dfd129~),
+               golang-google-genproto-dev,
+               golang-goprotobuf-dev (>= 1.4.3~),
+               golang-honnef-go-tools-dev,
+               golang-procfs-dev,
+               golang-protobuf-extensions-dev,
+               golang-toml-dev,
+               golang-websocket-dev,
+               golang-google-grpc-dev (>= 1.22~),
+               libimage-exiftool-perl
 Standards-Version: 4.5.1
 Vcs-Git: https://salsa.debian.org/ruby-team/gitlab.git
 Vcs-Browser: https://salsa.debian.org/ruby-team/gitlab
 Homepage: https://about.gitlab.com/
 XS-Ruby-Versions: all
+XS-Go-Import-Path: gitlab.com/gitlab-org/gitlab-workhorse
 
 Package: gitlab
 Section: contrib/net
@@ -462,3 +507,17 @@ Description: git powered software platform to collaborate on code (non-omnibus)
  .
  Note: Currently this package is in contrib because it uses yarn to install
  some of its front end dependencies.
+
+Package: gitlab-workhorse
+Architecture: any
+Depends: ${misc:Depends},
+         ${shlibs:Depends}
+Built-Using: ${misc:Built-Using}
+Description: unloads Git HTTP traffic from the GitLab Rails app (Unicorn)
+ gitlab-workhorse was designed to unload Git HTTP traffic from the GitLab Rails
+ app (Unicorn) to a separate daemon. It also serves 'git archive' downloads for
+ GitLab. All authentication and authorization logic is still handled by the
+ GitLab Rails app.
+ .
+ Architecture: Git client -> NGINX -> gitlab-workhorse (makes auth request to
+ GitLab Rails app) -> git-upload-pack

debian/gbp.conf

@@ -1,2 +1,2 @@
 [DEFAULT]
-component = [ 'elasticsearch-rails', 'snowplow-javascript-tracker' ]
+component = [ 'elasticsearch-rails', 'snowplow-javascript-tracker', 'workhorse-vendor' ]

debian/gitlab-workhorse.install

@@ -0,0 +1 @@
+_build/bin usr

debian/patches/0800-search-binaries-in-path.patch

@@ -0,0 +1,27 @@
+Description: Search for gitlab-workhorse binaries also in the environment in PATH variable
+Author: Nilesh Patra <npatra974@gmail.com>
+Last-Update: 2021-01-26
+--- a/workhorse/internal/testhelper/testhelper.go
++++ b/workhorse/internal/testhelper/testhelper.go
+@@ -13,6 +13,7 @@
+ 	"runtime"
+ 	"testing"
+ 	"time"
++	"os/exec"
+ 
+ 	"github.com/dgrijalva/jwt-go"
+ 	"github.com/stretchr/testify/require"
+@@ -80,8 +81,11 @@
+ 	rootDir := RootDir()
+ 
+ 	for _, exe := range workhorseExecutables {
+-		if _, err := os.Stat(path.Join(rootDir, exe)); os.IsNotExist(err) {
+-			return fmt.Errorf("cannot find executable %s. Please run 'make prepare-tests'", exe)
++		_, err := exec.LookPath(exe)
++		if err != nil {
++			if _, rootdirerr := os.Stat(path.Join(rootDir, exe)); os.IsNotExist(rootdirerr) {
++				return fmt.Errorf("cannot find executable %s. Please run 'make prepare-tests'", exe)
++			}
+ 		}
+ 	}
+ 

debian/patches/series

@@ -32,3 +32,4 @@
 0740-use-packaged-modules.patch
 0750-fix-relative-paths.patch
 0770-remove-capybara-screenshot-rspec.patch
+0800-search-binaries-in-path.patch

debian/rules

@@ -3,10 +3,23 @@
 include /usr/share/dpkg/pkg-info.mk
 
 %:
-	dh $@
+	dh $@ --package=gitlab
+	dh $@ --buildsystem=golang --with=golang --builddirectory=_build --package=gitlab-workhorse --sourcedirectory=workhorse
+
+override_dh_auto_configure:
+	dh_auto_configure -O--package=gitlab
+	dh_auto_configure -O--package=gitlab-workhorse
+	mkdir -p _build/src/gitlab.com/gitlab-org
+	cp -r workhorse _build/src/gitlab.com/gitlab-org/gitlab-workhorse
+	if [ -d workhorse-vendor ]; then mv workhorse-vendor _build/src/gitlab.com/gitlab-org/gitlab-workhorse/vendor; fi
+
+override_dh_auto_test:
+	dh_auto_test -O--package=gitlab
+	PATH="$(CURDIR)/_build/bin:$$PATH" dh_auto_test -O--package=gitlab-workhorse
 
 override_dh_install:
-	dh_install -XLICENSE
+	dh_install -XLICENSE -O--package=gitlab
+	dh_installexamples -O--package=gitlab
 	# Make sure we are installing all required files in debian/install
 	sh debian/upstream-file-count-check.sh
 	sh debian/upstream-config-file-check.sh
@@ -14,15 +27,17 @@ override_dh_install:
 	find debian/gitlab/usr/share/gitlab/ -name .eslintrc.yml -delete
 	mv debian/gitlab/usr/share/gitlab/app/assets/javascripts/locale \
 	debian/gitlab/usr/share/gitlab/app/assets/javascripts/locale.static
-	mv debian/gitlab/var/lib/gitlab/db/structure.sql debian/gitlab/var/lib/gitlab/db/structure.sql.template
+	if [ -f debian/gitlab/var/lib/gitlab/db/structure.sql ]; then \
+	  mv debian/gitlab/var/lib/gitlab/db/structure.sql debian/gitlab/var/lib/gitlab/db/structure.sql.template; fi
 	sed -i 's/__NEW_VERSION__/${DEB_VERSION}/g' debian/gitlab/usr/lib/gitlab/templates/gitlab-debian.conf.example
+	dh_install -XLICENSE -O--package=gitlab-workhorse
 
 override_dh_installinit:
-	dh_installinit --no-start -p gitlab --name=gitlab-sidekiq
+	dh_installinit --no-start -p gitlab --name=gitlab-sidekiq -O--package=gitlab
-	dh_installinit --no-start -p gitlab --name=gitlab-puma
+	dh_installinit --no-start -p gitlab --name=gitlab-puma -O--package=gitlab
-	dh_installinit --no-start -p gitlab --name=gitlab-mailroom
+	dh_installinit --no-start -p gitlab --name=gitlab-mailroom -O--package=gitlab
-	dh_installinit --no-start -p gitlab --name=gitlab-workhorse
+	dh_installinit --no-start -p gitlab --name=gitlab-workhorse -O--package=gitlab
-	dh_installinit
+	dh_installinit -O--package=gitlab
 
 override_dh_systemd_start:
-	dh_systemd_start --no-start
+	dh_systemd_start --no-start -O--package=gitlab

debian/upstream-file-list

@@ -57,6 +57,7 @@ Pipfile.lock
 README.md
 Rakefile
 VERSION
+_build
 app
 babel.config.js
 bin