From 0a98dbda3b2d418e33c7d185a7abe5654b4a7c32 Mon Sep 17 00:00:00 2001 From: Pirate Praveen Date: Sat, 17 Apr 2021 16:43:17 +0530 Subject: [PATCH] Build gitlab-workhorse binary from workhorse directory --- debian/clean | 20 ++++++ debian/control | 61 ++++++++++++++++++- debian/gbp.conf | 2 +- debian/gitlab-workhorse.install | 1 + .../0800-search-binaries-in-path.patch | 27 ++++++++ debian/patches/series | 1 + debian/rules | 33 +++++++--- debian/upstream-file-list | 1 + 8 files changed, 135 insertions(+), 11 deletions(-) create mode 100644 debian/gitlab-workhorse.install create mode 100644 debian/patches/0800-search-binaries-in-path.patch diff --git a/debian/clean b/debian/clean index cd7341846d..379d5dc0ec 100644 --- a/debian/clean +++ b/debian/clean @@ -10,3 +10,23 @@ debian/upstream-config-file-list.new # We have rails > 6.0.2 config/initializers/active_record_fix_insert_all.rb + +# For workhorse +## Failing tests that require test data from the Internet: +workhorse/main_test.go +workhorse/authorization_test.go +workhorse/channel_test.go +workhorse/gitaly_integration_test.go +workhorse/gitaly_test.go +workhorse/jobs_test.go +workhorse/sendfile_test.go +workhorse/upload_test.go +workhorse/internal/upload/*_test.go +workhorse/internal/objectstore/*_test.go +workhorse/internal/redis/*_test.go +## Need definitions from excluded test files +workhorse/cable_test.go +## Need github.com/johannesboyne/gofakes3/backend/s3mem +workhorse/internal/objectstore/s3_object_test.go +workhorse/internal/objectstore/test/s3_stub.go +workhorse/internal/filestore/file_handler_test.go diff --git a/debian/control b/debian/control index 8aed9fd40b..e9920fc4ae 100644 --- a/debian/control +++ b/debian/control @@ -7,12 +7,57 @@ Uploaders: Cédric Boutillier , Balasankar C , Sruthi Chandran , Utkarsh Gupta -Build-Depends: debhelper (>= 10~), gem2deb, bc +Build-Depends: debhelper (>= 10~), + gem2deb, + dh-golang, + golang-any, + bc, + golang-github-alecthomas-chroma-dev, + golang-github-aws-aws-sdk-go-dev, + golang-github-azure-azure-storage-blob-go-dev (>= 0.10~), + golang-github-beorn7-perks-dev, + golang-github-client9-reopen-dev, + golang-github-davecgh-go-spew-dev, + golang-github-dgrijalva-jwt-go-dev (>= 3.2.0~), + golang-github-disintegration-imaging-dev, + golang-github-fzambia-sentinel-dev, + golang-github-garyburd-redigo-dev, + golang-github-google-uuid-dev, + golang-github-gomodule-redigo-dev, + golang-github-grpc-ecosystem-go-grpc-prometheus-dev, + golang-github-jfbus-httprs-dev, + golang-github-jtolds-gls-dev (>= 4.20~), + golang-github-jpillora-backoff-dev, + golang-github-mitchellh-copystructure-dev, + golang-github-mitchellh-reflectwalk-dev, + golang-github-pmezard-go-difflib-dev, + golang-github-prometheus-client-golang-dev, + golang-github-prometheus-client-model-dev, + golang-github-prometheus-common-dev, + golang-github-ryszard-goskiplist-dev, + golang-github-sebest-xff-dev, + golang-github-sirupsen-logrus-dev, + golang-github-stretchr-testify-dev (>= 1.4~), + golang-gitlab-gitlab-org-labkit-dev (>= 1.3.0-4~), + golang-gocloud-dev, + golang-golang-x-crypto-dev, + golang-golang-x-net-dev, + golang-golang-x-sys-dev (>= 0.0~git20180510.7dfd129~), + golang-google-genproto-dev, + golang-goprotobuf-dev (>= 1.4.3~), + golang-honnef-go-tools-dev, + golang-procfs-dev, + golang-protobuf-extensions-dev, + golang-toml-dev, + golang-websocket-dev, + golang-google-grpc-dev (>= 1.22~), + libimage-exiftool-perl Standards-Version: 4.5.1 Vcs-Git: https://salsa.debian.org/ruby-team/gitlab.git Vcs-Browser: https://salsa.debian.org/ruby-team/gitlab Homepage: https://about.gitlab.com/ XS-Ruby-Versions: all +XS-Go-Import-Path: gitlab.com/gitlab-org/gitlab-workhorse Package: gitlab Section: contrib/net @@ -462,3 +507,17 @@ Description: git powered software platform to collaborate on code (non-omnibus) . Note: Currently this package is in contrib because it uses yarn to install some of its front end dependencies. + +Package: gitlab-workhorse +Architecture: any +Depends: ${misc:Depends}, + ${shlibs:Depends} +Built-Using: ${misc:Built-Using} +Description: unloads Git HTTP traffic from the GitLab Rails app (Unicorn) + gitlab-workhorse was designed to unload Git HTTP traffic from the GitLab Rails + app (Unicorn) to a separate daemon. It also serves 'git archive' downloads for + GitLab. All authentication and authorization logic is still handled by the + GitLab Rails app. + . + Architecture: Git client -> NGINX -> gitlab-workhorse (makes auth request to + GitLab Rails app) -> git-upload-pack diff --git a/debian/gbp.conf b/debian/gbp.conf index e000355ad8..9331c87730 100644 --- a/debian/gbp.conf +++ b/debian/gbp.conf @@ -1,2 +1,2 @@ [DEFAULT] -component = [ 'elasticsearch-rails', 'snowplow-javascript-tracker' ] +component = [ 'elasticsearch-rails', 'snowplow-javascript-tracker', 'workhorse-vendor' ] diff --git a/debian/gitlab-workhorse.install b/debian/gitlab-workhorse.install new file mode 100644 index 0000000000..1a1cc70fd5 --- /dev/null +++ b/debian/gitlab-workhorse.install @@ -0,0 +1 @@ +_build/bin usr diff --git a/debian/patches/0800-search-binaries-in-path.patch b/debian/patches/0800-search-binaries-in-path.patch new file mode 100644 index 0000000000..1015981b52 --- /dev/null +++ b/debian/patches/0800-search-binaries-in-path.patch @@ -0,0 +1,27 @@ +Description: Search for gitlab-workhorse binaries also in the environment in PATH variable +Author: Nilesh Patra +Last-Update: 2021-01-26 +--- a/workhorse/internal/testhelper/testhelper.go ++++ b/workhorse/internal/testhelper/testhelper.go +@@ -13,6 +13,7 @@ + "runtime" + "testing" + "time" ++ "os/exec" + + "github.com/dgrijalva/jwt-go" + "github.com/stretchr/testify/require" +@@ -80,8 +81,11 @@ + rootDir := RootDir() + + for _, exe := range workhorseExecutables { +- if _, err := os.Stat(path.Join(rootDir, exe)); os.IsNotExist(err) { +- return fmt.Errorf("cannot find executable %s. Please run 'make prepare-tests'", exe) ++ _, err := exec.LookPath(exe) ++ if err != nil { ++ if _, rootdirerr := os.Stat(path.Join(rootDir, exe)); os.IsNotExist(rootdirerr) { ++ return fmt.Errorf("cannot find executable %s. Please run 'make prepare-tests'", exe) ++ } + } + } + diff --git a/debian/patches/series b/debian/patches/series index 677e46ae4a..a405d65187 100644 --- a/debian/patches/series +++ b/debian/patches/series @@ -32,3 +32,4 @@ 0740-use-packaged-modules.patch 0750-fix-relative-paths.patch 0770-remove-capybara-screenshot-rspec.patch +0800-search-binaries-in-path.patch diff --git a/debian/rules b/debian/rules index 33953b84df..2d942e4200 100755 --- a/debian/rules +++ b/debian/rules @@ -3,10 +3,23 @@ include /usr/share/dpkg/pkg-info.mk %: - dh $@ + dh $@ --package=gitlab + dh $@ --buildsystem=golang --with=golang --builddirectory=_build --package=gitlab-workhorse --sourcedirectory=workhorse + +override_dh_auto_configure: + dh_auto_configure -O--package=gitlab + dh_auto_configure -O--package=gitlab-workhorse + mkdir -p _build/src/gitlab.com/gitlab-org + cp -r workhorse _build/src/gitlab.com/gitlab-org/gitlab-workhorse + if [ -d workhorse-vendor ]; then mv workhorse-vendor _build/src/gitlab.com/gitlab-org/gitlab-workhorse/vendor; fi + +override_dh_auto_test: + dh_auto_test -O--package=gitlab + PATH="$(CURDIR)/_build/bin:$$PATH" dh_auto_test -O--package=gitlab-workhorse override_dh_install: - dh_install -XLICENSE + dh_install -XLICENSE -O--package=gitlab + dh_installexamples -O--package=gitlab # Make sure we are installing all required files in debian/install sh debian/upstream-file-count-check.sh sh debian/upstream-config-file-check.sh @@ -14,15 +27,17 @@ override_dh_install: find debian/gitlab/usr/share/gitlab/ -name .eslintrc.yml -delete mv debian/gitlab/usr/share/gitlab/app/assets/javascripts/locale \ debian/gitlab/usr/share/gitlab/app/assets/javascripts/locale.static - mv debian/gitlab/var/lib/gitlab/db/structure.sql debian/gitlab/var/lib/gitlab/db/structure.sql.template + if [ -f debian/gitlab/var/lib/gitlab/db/structure.sql ]; then \ + mv debian/gitlab/var/lib/gitlab/db/structure.sql debian/gitlab/var/lib/gitlab/db/structure.sql.template; fi sed -i 's/__NEW_VERSION__/${DEB_VERSION}/g' debian/gitlab/usr/lib/gitlab/templates/gitlab-debian.conf.example + dh_install -XLICENSE -O--package=gitlab-workhorse override_dh_installinit: - dh_installinit --no-start -p gitlab --name=gitlab-sidekiq - dh_installinit --no-start -p gitlab --name=gitlab-puma - dh_installinit --no-start -p gitlab --name=gitlab-mailroom - dh_installinit --no-start -p gitlab --name=gitlab-workhorse - dh_installinit + dh_installinit --no-start -p gitlab --name=gitlab-sidekiq -O--package=gitlab + dh_installinit --no-start -p gitlab --name=gitlab-puma -O--package=gitlab + dh_installinit --no-start -p gitlab --name=gitlab-mailroom -O--package=gitlab + dh_installinit --no-start -p gitlab --name=gitlab-workhorse -O--package=gitlab + dh_installinit -O--package=gitlab override_dh_systemd_start: - dh_systemd_start --no-start + dh_systemd_start --no-start -O--package=gitlab diff --git a/debian/upstream-file-list b/debian/upstream-file-list index 1a03fe2649..b7682e2c42 100644 --- a/debian/upstream-file-list +++ b/debian/upstream-file-list @@ -57,6 +57,7 @@ Pipfile.lock README.md Rakefile VERSION +_build app babel.config.js bin