debian-mirror-gitlab/workhorse/internal/zipartifacts/open_archive.go
2021-02-22 17:27:13 +05:30

138 lines
3.9 KiB
Go

package zipartifacts
import (
"archive/zip"
"context"
"fmt"
"io"
"net"
"net/http"
"os"
"strings"
"time"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/httprs"
"gitlab.com/gitlab-org/labkit/correlation"
"gitlab.com/gitlab-org/labkit/mask"
"gitlab.com/gitlab-org/labkit/tracing"
)
var httpClient = &http.Client{
Transport: tracing.NewRoundTripper(correlation.NewInstrumentedRoundTripper(&http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 10 * time.Second,
}).DialContext,
IdleConnTimeout: 30 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 10 * time.Second,
ResponseHeaderTimeout: 30 * time.Second,
DisableCompression: true,
})),
}
type archive struct {
reader io.ReaderAt
size int64
}
// OpenArchive will open a zip.Reader from a local path or a remote object store URL
// in case of remote url it will make use of ranged requestes to support seeking.
// If the path do not exists error will be ErrArchiveNotFound,
// if the file isn't a zip archive error will be ErrNotAZip
func OpenArchive(ctx context.Context, archivePath string) (*zip.Reader, error) {
archive, err := openArchiveLocation(ctx, archivePath)
if err != nil {
return nil, err
}
return openZipReader(archive.reader, archive.size)
}
// OpenArchiveWithReaderFunc opens a zip.Reader from either local path or a
// remote object, similarly to OpenArchive function. The difference is that it
// allows passing a readerFunc that takes a io.ReaderAt that is either going to
// be os.File or a custom reader we use to read from object storage. The
// readerFunc can augment the archive reader and return a type that satisfies
// io.ReaderAt.
func OpenArchiveWithReaderFunc(ctx context.Context, location string, readerFunc func(io.ReaderAt, int64) io.ReaderAt) (*zip.Reader, error) {
archive, err := openArchiveLocation(ctx, location)
if err != nil {
return nil, err
}
return openZipReader(readerFunc(archive.reader, archive.size), archive.size)
}
func openArchiveLocation(ctx context.Context, location string) (*archive, error) {
if isURL(location) {
return openHTTPArchive(ctx, location)
}
return openFileArchive(ctx, location)
}
func isURL(path string) bool {
return strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://")
}
func openHTTPArchive(ctx context.Context, archivePath string) (*archive, error) {
scrubbedArchivePath := mask.URL(archivePath)
req, err := http.NewRequest(http.MethodGet, archivePath, nil)
if err != nil {
return nil, fmt.Errorf("can't create HTTP GET %q: %v", scrubbedArchivePath, err)
}
req = req.WithContext(ctx)
resp, err := httpClient.Do(req.WithContext(ctx))
if err != nil {
return nil, fmt.Errorf("HTTP GET %q: %v", scrubbedArchivePath, err)
} else if resp.StatusCode == http.StatusNotFound {
return nil, ErrorCode[CodeArchiveNotFound]
} else if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("HTTP GET %q: %d: %v", scrubbedArchivePath, resp.StatusCode, resp.Status)
}
rs := httprs.NewHttpReadSeeker(resp, httpClient)
go func() {
<-ctx.Done()
resp.Body.Close()
rs.Close()
}()
return &archive{reader: rs, size: resp.ContentLength}, nil
}
func openFileArchive(ctx context.Context, archivePath string) (*archive, error) {
file, err := os.Open(archivePath)
if err != nil {
if os.IsNotExist(err) {
return nil, ErrorCode[CodeArchiveNotFound]
}
}
go func() {
<-ctx.Done()
// We close the archive from this goroutine so that we can safely return a *zip.Reader instead of a *zip.ReadCloser
file.Close()
}()
stat, err := file.Stat()
if err != nil {
return nil, err
}
return &archive{reader: file, size: stat.Size()}, nil
}
func openZipReader(archive io.ReaderAt, size int64) (*zip.Reader, error) {
reader, err := zip.NewReader(archive, size)
if err != nil {
return nil, ErrorCode[CodeNotZip]
}
return reader, nil
}