138 lines
3.9 KiB
Go
138 lines
3.9 KiB
Go
package zipartifacts
|
|
|
|
import (
|
|
"archive/zip"
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"net"
|
|
"net/http"
|
|
"os"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitlab.com/gitlab-org/gitlab-workhorse/internal/httprs"
|
|
|
|
"gitlab.com/gitlab-org/labkit/correlation"
|
|
"gitlab.com/gitlab-org/labkit/mask"
|
|
"gitlab.com/gitlab-org/labkit/tracing"
|
|
)
|
|
|
|
var httpClient = &http.Client{
|
|
Transport: tracing.NewRoundTripper(correlation.NewInstrumentedRoundTripper(&http.Transport{
|
|
Proxy: http.ProxyFromEnvironment,
|
|
DialContext: (&net.Dialer{
|
|
Timeout: 30 * time.Second,
|
|
KeepAlive: 10 * time.Second,
|
|
}).DialContext,
|
|
IdleConnTimeout: 30 * time.Second,
|
|
TLSHandshakeTimeout: 10 * time.Second,
|
|
ExpectContinueTimeout: 10 * time.Second,
|
|
ResponseHeaderTimeout: 30 * time.Second,
|
|
DisableCompression: true,
|
|
})),
|
|
}
|
|
|
|
type archive struct {
|
|
reader io.ReaderAt
|
|
size int64
|
|
}
|
|
|
|
// OpenArchive will open a zip.Reader from a local path or a remote object store URL
|
|
// in case of remote url it will make use of ranged requestes to support seeking.
|
|
// If the path do not exists error will be ErrArchiveNotFound,
|
|
// if the file isn't a zip archive error will be ErrNotAZip
|
|
func OpenArchive(ctx context.Context, archivePath string) (*zip.Reader, error) {
|
|
archive, err := openArchiveLocation(ctx, archivePath)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return openZipReader(archive.reader, archive.size)
|
|
}
|
|
|
|
// OpenArchiveWithReaderFunc opens a zip.Reader from either local path or a
|
|
// remote object, similarly to OpenArchive function. The difference is that it
|
|
// allows passing a readerFunc that takes a io.ReaderAt that is either going to
|
|
// be os.File or a custom reader we use to read from object storage. The
|
|
// readerFunc can augment the archive reader and return a type that satisfies
|
|
// io.ReaderAt.
|
|
func OpenArchiveWithReaderFunc(ctx context.Context, location string, readerFunc func(io.ReaderAt, int64) io.ReaderAt) (*zip.Reader, error) {
|
|
archive, err := openArchiveLocation(ctx, location)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return openZipReader(readerFunc(archive.reader, archive.size), archive.size)
|
|
}
|
|
|
|
func openArchiveLocation(ctx context.Context, location string) (*archive, error) {
|
|
if isURL(location) {
|
|
return openHTTPArchive(ctx, location)
|
|
}
|
|
|
|
return openFileArchive(ctx, location)
|
|
}
|
|
|
|
func isURL(path string) bool {
|
|
return strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://")
|
|
}
|
|
|
|
func openHTTPArchive(ctx context.Context, archivePath string) (*archive, error) {
|
|
scrubbedArchivePath := mask.URL(archivePath)
|
|
req, err := http.NewRequest(http.MethodGet, archivePath, nil)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("can't create HTTP GET %q: %v", scrubbedArchivePath, err)
|
|
}
|
|
req = req.WithContext(ctx)
|
|
|
|
resp, err := httpClient.Do(req.WithContext(ctx))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("HTTP GET %q: %v", scrubbedArchivePath, err)
|
|
} else if resp.StatusCode == http.StatusNotFound {
|
|
return nil, ErrorCode[CodeArchiveNotFound]
|
|
} else if resp.StatusCode != http.StatusOK {
|
|
return nil, fmt.Errorf("HTTP GET %q: %d: %v", scrubbedArchivePath, resp.StatusCode, resp.Status)
|
|
}
|
|
|
|
rs := httprs.NewHttpReadSeeker(resp, httpClient)
|
|
|
|
go func() {
|
|
<-ctx.Done()
|
|
resp.Body.Close()
|
|
rs.Close()
|
|
}()
|
|
|
|
return &archive{reader: rs, size: resp.ContentLength}, nil
|
|
}
|
|
|
|
func openFileArchive(ctx context.Context, archivePath string) (*archive, error) {
|
|
file, err := os.Open(archivePath)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return nil, ErrorCode[CodeArchiveNotFound]
|
|
}
|
|
}
|
|
|
|
go func() {
|
|
<-ctx.Done()
|
|
// We close the archive from this goroutine so that we can safely return a *zip.Reader instead of a *zip.ReadCloser
|
|
file.Close()
|
|
}()
|
|
|
|
stat, err := file.Stat()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return &archive{reader: file, size: stat.Size()}, nil
|
|
}
|
|
|
|
func openZipReader(archive io.ReaderAt, size int64) (*zip.Reader, error) {
|
|
reader, err := zip.NewReader(archive, size)
|
|
if err != nil {
|
|
return nil, ErrorCode[CodeNotZip]
|
|
}
|
|
|
|
return reader, nil
|
|
}
|