debian-mirror-gitlab/workhorse/gitaly_integration_test.go
2022-10-11 01:57:18 +05:30

370 lines
11 KiB
Go

// Tests in this file need access to a real Gitaly server to run. The address
// is supplied via the GITALY_ADDRESS environment variable
package main
import (
"archive/tar"
"bufio"
"bytes"
"context"
"fmt"
"os"
"os/exec"
"path"
"regexp"
"strconv"
"strings"
"testing"
"github.com/stretchr/testify/require"
"gitlab.com/gitlab-org/gitaly/v15/proto/go/gitalypb"
"gitlab.com/gitlab-org/gitaly/v15/streamio"
"gitlab.com/gitlab-org/gitlab/workhorse/internal/api"
"gitlab.com/gitlab-org/gitlab/workhorse/internal/gitaly"
"gitlab.com/gitlab-org/gitlab/workhorse/internal/testhelper"
)
var (
gitalyAddress string
jsonGitalyServer string
)
func init() {
gitalyAddress = os.Getenv("GITALY_ADDRESS")
jsonGitalyServer = fmt.Sprintf(`"GitalyServer":{"Address":"%s", "Token": ""}`, gitalyAddress)
}
func skipUnlessRealGitaly(t *testing.T) {
t.Log(gitalyAddress)
if gitalyAddress != "" {
return
}
t.Skip(`Please set GITALY_ADDRESS="..." to run Gitaly integration tests`)
}
func realGitalyAuthResponse(apiResponse *api.Response) *api.Response {
apiResponse.GitalyServer.Address = gitalyAddress
return apiResponse
}
func realGitalyOkBody(t *testing.T) *api.Response {
return realGitalyAuthResponse(gitOkBody(t))
}
func ensureGitalyRepository(t *testing.T, apiResponse *api.Response) error {
ctx, namespace, err := gitaly.NewNamespaceClient(
context.Background(),
apiResponse.GitalyServer,
gitaly.WithFeatures(apiResponse.GitalyServer.Features),
)
if err != nil {
return err
}
ctx, repository, err := gitaly.NewRepositoryClient(ctx, apiResponse.GitalyServer)
if err != nil {
return err
}
// Remove the repository if it already exists, for consistency
rmNsReq := &gitalypb.RemoveNamespaceRequest{
StorageName: apiResponse.Repository.StorageName,
Name: apiResponse.Repository.RelativePath,
}
_, err = namespace.RemoveNamespace(ctx, rmNsReq)
if err != nil {
return err
}
stream, err := repository.CreateRepositoryFromBundle(ctx)
if err != nil {
return fmt.Errorf("initiate stream: %w", err)
}
if err := stream.Send(&gitalypb.CreateRepositoryFromBundleRequest{Repository: &apiResponse.Repository}); err != nil {
return err
}
gitBundle := exec.Command("git", "-C", path.Join(testRepoRoot, testRepo), "bundle", "create", "-", "--all")
gitBundle.Stdout = streamio.NewWriter(func(p []byte) error {
return stream.Send(&gitalypb.CreateRepositoryFromBundleRequest{Data: p})
})
if err := gitBundle.Run(); err != nil {
return fmt.Errorf("run git bundle --create: %w", err)
}
if _, err := stream.CloseAndRecv(); err != nil {
return fmt.Errorf("finish CreateRepositoryFromBundle: %w", err)
}
return nil
}
func TestAllowedClone(t *testing.T) {
skipUnlessRealGitaly(t)
// Create the repository in the Gitaly server
apiResponse := realGitalyOkBody(t)
require.NoError(t, ensureGitalyRepository(t, apiResponse))
// Prepare test server and backend
ts := testAuthServer(t, nil, nil, 200, apiResponse)
defer ts.Close()
ws := startWorkhorseServer(ts.URL)
defer ws.Close()
// Do the git clone
require.NoError(t, os.RemoveAll(scratchDir))
cloneCmd := exec.Command("git", "clone", fmt.Sprintf("%s/%s", ws.URL, testRepo), checkoutDir)
runOrFail(t, cloneCmd)
// We may have cloned an 'empty' repository, 'git log' will fail in it
logCmd := exec.Command("git", "log", "-1", "--oneline")
logCmd.Dir = checkoutDir
runOrFail(t, logCmd)
}
func TestAllowedShallowClone(t *testing.T) {
skipUnlessRealGitaly(t)
// Create the repository in the Gitaly server
apiResponse := realGitalyOkBody(t)
require.NoError(t, ensureGitalyRepository(t, apiResponse))
// Prepare test server and backend
ts := testAuthServer(t, nil, nil, 200, apiResponse)
defer ts.Close()
ws := startWorkhorseServer(ts.URL)
defer ws.Close()
// Shallow git clone (depth 1)
require.NoError(t, os.RemoveAll(scratchDir))
cloneCmd := exec.Command("git", "clone", "--depth", "1", fmt.Sprintf("%s/%s", ws.URL, testRepo), checkoutDir)
runOrFail(t, cloneCmd)
// We may have cloned an 'empty' repository, 'git log' will fail in it
logCmd := exec.Command("git", "log", "-1", "--oneline")
logCmd.Dir = checkoutDir
runOrFail(t, logCmd)
}
func TestAllowedPush(t *testing.T) {
skipUnlessRealGitaly(t)
// Create the repository in the Gitaly server
apiResponse := realGitalyOkBody(t)
require.NoError(t, ensureGitalyRepository(t, apiResponse))
// Prepare the test server and backend
ts := testAuthServer(t, nil, nil, 200, apiResponse)
defer ts.Close()
ws := startWorkhorseServer(ts.URL)
defer ws.Close()
// Perform the git push
pushCmd := exec.Command("git", "push", fmt.Sprintf("%s/%s", ws.URL, testRepo), fmt.Sprintf("master:%s", newBranch()))
pushCmd.Dir = checkoutDir
runOrFail(t, pushCmd)
}
func TestAllowedGetGitBlob(t *testing.T) {
skipUnlessRealGitaly(t)
// Create the repository in the Gitaly server
apiResponse := realGitalyOkBody(t)
require.NoError(t, ensureGitalyRepository(t, apiResponse))
// the LICENSE file in the test repository
oid := "50b27c6518be44c42c4d87966ae2481ce895624c"
expectedBody := "The MIT License (MIT)"
bodyLen := 1075
jsonParams := fmt.Sprintf(
`{
%s,
"GetBlobRequest":{
"repository":{"storage_name":"%s", "relative_path":"%s"},
"oid":"%s",
"limit":-1
}
}`,
jsonGitalyServer, apiResponse.Repository.StorageName, apiResponse.Repository.RelativePath, oid,
)
resp, body, err := doSendDataRequest("/something", "git-blob", jsonParams)
require.NoError(t, err)
shortBody := string(body[:len(expectedBody)])
require.Equal(t, 200, resp.StatusCode, "GET %q: status code", resp.Request.URL)
require.Equal(t, expectedBody, shortBody, "GET %q: response body", resp.Request.URL)
testhelper.RequireResponseHeader(t, resp, "Content-Length", strconv.Itoa(bodyLen))
requireNginxResponseBuffering(t, "no", resp, "GET %q: nginx response buffering", resp.Request.URL)
}
func TestAllowedGetGitArchive(t *testing.T) {
skipUnlessRealGitaly(t)
// Create the repository in the Gitaly server
apiResponse := realGitalyOkBody(t)
require.NoError(t, ensureGitalyRepository(t, apiResponse))
archivePath := path.Join(scratchDir, "my/path")
archivePrefix := "repo-1"
msg := serializedProtoMessage("GetArchiveRequest", &gitalypb.GetArchiveRequest{
Repository: &apiResponse.Repository,
CommitId: "HEAD",
Prefix: archivePrefix,
Format: gitalypb.GetArchiveRequest_TAR,
Path: []byte("files"),
})
jsonParams := buildGitalyRPCParams(gitalyAddress, rpcArg{"ArchivePath", archivePath}, msg)
resp, body, err := doSendDataRequest("/archive.tar", "git-archive", jsonParams)
require.NoError(t, err)
require.Equal(t, 200, resp.StatusCode, "GET %q: status code", resp.Request.URL)
requireNginxResponseBuffering(t, "no", resp, "GET %q: nginx response buffering", resp.Request.URL)
// Ensure the tar file is readable
foundEntry := false
tr := tar.NewReader(bytes.NewReader(body))
for {
hdr, err := tr.Next()
if err != nil {
break
}
if hdr.Name == archivePrefix+"/" {
foundEntry = true
break
}
}
require.True(t, foundEntry, "Couldn't find %v directory entry", archivePrefix)
}
func TestAllowedGetGitArchiveOldPayload(t *testing.T) {
skipUnlessRealGitaly(t)
// Create the repository in the Gitaly server
apiResponse := realGitalyOkBody(t)
repo := &apiResponse.Repository
require.NoError(t, ensureGitalyRepository(t, apiResponse))
archivePath := path.Join(scratchDir, "my/path")
archivePrefix := "repo-1"
jsonParams := fmt.Sprintf(
`{
%s,
"GitalyRepository":{"storage_name":"%s","relative_path":"%s"},
"ArchivePath":"%s",
"ArchivePrefix":"%s",
"CommitId":"%s"
}`,
jsonGitalyServer, repo.StorageName, repo.RelativePath, archivePath, archivePrefix, "HEAD",
)
resp, body, err := doSendDataRequest("/archive.tar", "git-archive", jsonParams)
require.NoError(t, err)
require.Equal(t, 200, resp.StatusCode, "GET %q: status code", resp.Request.URL)
requireNginxResponseBuffering(t, "no", resp, "GET %q: nginx response buffering", resp.Request.URL)
// Ensure the tar file is readable
foundEntry := false
tr := tar.NewReader(bytes.NewReader(body))
for {
hdr, err := tr.Next()
if err != nil {
break
}
if hdr.Name == archivePrefix+"/" {
foundEntry = true
break
}
}
require.True(t, foundEntry, "Couldn't find %v directory entry", archivePrefix)
}
func TestAllowedGetGitDiff(t *testing.T) {
skipUnlessRealGitaly(t)
// Create the repository in the Gitaly server
apiResponse := realGitalyOkBody(t)
require.NoError(t, ensureGitalyRepository(t, apiResponse))
msg := serializedMessage("RawDiffRequest", &gitalypb.RawDiffRequest{
Repository: &apiResponse.Repository,
LeftCommitId: "b0e52af38d7ea43cf41d8a6f2471351ac036d6c9",
RightCommitId: "732401c65e924df81435deb12891ef570167d2e2",
})
jsonParams := buildGitalyRPCParams(gitalyAddress, msg)
resp, body, err := doSendDataRequest("/something", "git-diff", jsonParams)
require.NoError(t, err)
require.Equal(t, 200, resp.StatusCode, "GET %q: status code", resp.Request.URL)
requireNginxResponseBuffering(t, "no", resp, "GET %q: nginx response buffering", resp.Request.URL)
expectedBody := "diff --git a/LICENSE b/LICENSE\n"
require.Equal(t, expectedBody, string(body[:len(expectedBody)]),
"GET %q: response body", resp.Request.URL)
}
func TestAllowedGetGitFormatPatch(t *testing.T) {
skipUnlessRealGitaly(t)
// Create the repository in the Gitaly server
apiResponse := realGitalyOkBody(t)
require.NoError(t, ensureGitalyRepository(t, apiResponse))
msg := serializedMessage("RawPatchRequest", &gitalypb.RawPatchRequest{
Repository: &apiResponse.Repository,
LeftCommitId: "b0e52af38d7ea43cf41d8a6f2471351ac036d6c9",
RightCommitId: "0e1b353b348f8477bdbec1ef47087171c5032cd9",
})
jsonParams := buildGitalyRPCParams(gitalyAddress, msg)
resp, body, err := doSendDataRequest("/something", "git-format-patch", jsonParams)
require.NoError(t, err)
require.Equal(t, 200, resp.StatusCode, "GET %q: status code", resp.Request.URL)
requireNginxResponseBuffering(t, "no", resp, "GET %q: nginx response buffering", resp.Request.URL)
requirePatchSeries(t, body,
"732401c65e924df81435deb12891ef570167d2e2",
"33bcff41c232a11727ac6d660bd4b0c2ba86d63d",
"0e1b353b348f8477bdbec1ef47087171c5032cd9",
)
}
var extractPatchSeriesMatcher = regexp.MustCompile(`^From (\w+)`)
// RequirePatchSeries takes a `git format-patch` blob, extracts the From xxxxx
// lines and compares the SHAs to expected list.
func requirePatchSeries(t *testing.T, blob []byte, expected ...string) {
t.Helper()
var actual []string
footer := make([]string, 3)
scanner := bufio.NewScanner(bytes.NewReader(blob))
for scanner.Scan() {
line := scanner.Text()
if matches := extractPatchSeriesMatcher.FindStringSubmatch(line); len(matches) == 2 {
actual = append(actual, matches[1])
}
footer = []string{footer[1], footer[2], line}
}
require.Equal(t, strings.Join(expected, "\n"), strings.Join(actual, "\n"), "patch series")
// Check the last returned patch is complete
// Don't assert on the final line, it is a git version
require.Equal(t, "-- ", footer[0], "end of patch marker")
}