Improve sync performance for pull-mirrors (#19125)
This addresses https://github.com/go-gitea/gitea/issues/18352 It aims to improve performance (and resource use) of the `SyncReleasesWithTags` operation for pull-mirrors. For large repositories with many tags, `SyncReleasesWithTags` can be a costly operation (taking several minutes to complete). The reason is two-fold: 1. on sync, every upstream repo tag is compared (for changes) against existing local entries in the release table to ensure that they are up-to-date. 2. the procedure for getting _each tag_ involves a series of git operations ```bash git show-ref --tags -- v8.2.4477 git cat-file -t 29ab6ce9f36660cffaad3c8789e71162e5db5d2f git cat-file -p 29ab6ce9f36660cffaad3c8789e71162e5db5d2f git rev-list --count 29ab6ce9f36660cffaad3c8789e71162e5db5d2f ``` of which the `git rev-list --count` can be particularly heavy. This PR optimizes performance for pull-mirrors. We utilize the fact that a pull-mirror is always identical to its upstream and rebuild the entire release table on every sync and use a batch `git for-each-ref .. refs/tags` call to retrieve all tags in one go. For large mirror repos, with hundreds of annotated tags, this brings down the duration of the sync operation from several minutes to a few seconds. A few unscientific examples run on my local machine: - https://github.com/spring-projects/spring-boot (223 tags) - before: `0m28,673s` - after: `0m2,244s` - https://github.com/kubernetes/kubernetes (890 tags) - before: `8m00s` - after: `0m8,520s` - https://github.com/vim/vim (13954 tags) - before: `14m20,383s` - after: `0m35,467s` I added a `foreachref` package which contains a flexible way of specifying which reference fields are of interest (`git-for-each-ref(1)`) and to produce a parser for the expected output. These could be reused in other places where `for-each-ref` is used. I'll add unit tests for those if the overall PR looks promising.
This commit is contained in:
Peter Gardfjäll
GitHub
parent
b877504b03
commit
e28cc79c92
7 changed files with 836 additions and 21 deletions
84
modules/git/foreachref/format.go
Normal file
84
modules/git/foreachref/format.go
Normal file
|
|
@ -0,0 +1,84 @@
|
||||||
|
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a MIT-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package foreachref
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/hex"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
nullChar = []byte("\x00")
|
||||||
|
dualNullChar = []byte("\x00\x00")
|
||||||
|
)
|
||||||
|
|
||||||
|
// Format supports specifying and parsing an output format for 'git
|
||||||
|
// for-each-ref'. See See git-for-each-ref(1) for available fields.
|
||||||
|
type Format struct {
|
||||||
|
// fieldNames hold %(fieldname)s to be passed to the '--format' flag of
|
||||||
|
// for-each-ref. See git-for-each-ref(1) for available fields.
|
||||||
|
fieldNames []string
|
||||||
|
|
||||||
|
// fieldDelim is the character sequence that is used to separate fields
|
||||||
|
// for each reference. fieldDelim and refDelim should be selected to not
|
||||||
|
// interfere with each other and to not be present in field values.
|
||||||
|
fieldDelim []byte
|
||||||
|
// fieldDelimStr is a string representation of fieldDelim. Used to save
|
||||||
|
// us from repetitive reallocation whenever we need the delimiter as a
|
||||||
|
// string.
|
||||||
|
fieldDelimStr string
|
||||||
|
// refDelim is the character sequence used to separate reference from
|
||||||
|
// each other in the output. fieldDelim and refDelim should be selected
|
||||||
|
// to not interfere with each other and to not be present in field
|
||||||
|
// values.
|
||||||
|
refDelim []byte
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewFormat creates a forEachRefFormat using the specified fieldNames. See
|
||||||
|
// git-for-each-ref(1) for available fields.
|
||||||
|
func NewFormat(fieldNames ...string) Format {
|
||||||
|
return Format{
|
||||||
|
fieldNames: fieldNames,
|
||||||
|
fieldDelim: nullChar,
|
||||||
|
fieldDelimStr: string(nullChar),
|
||||||
|
refDelim: dualNullChar,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Flag returns a for-each-ref --format flag value that captures the fieldNames.
|
||||||
|
func (f Format) Flag() string {
|
||||||
|
var formatFlag strings.Builder
|
||||||
|
for i, field := range f.fieldNames {
|
||||||
|
// field key and field value
|
||||||
|
formatFlag.WriteString(fmt.Sprintf("%s %%(%s)", field, field))
|
||||||
|
|
||||||
|
if i < len(f.fieldNames)-1 {
|
||||||
|
// note: escape delimiters to allow control characters as
|
||||||
|
// delimiters. For example, '%00' for null character or '%0a'
|
||||||
|
// for newline.
|
||||||
|
formatFlag.WriteString(f.hexEscaped(f.fieldDelim))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
formatFlag.WriteString(f.hexEscaped(f.refDelim))
|
||||||
|
return formatFlag.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parser returns a Parser capable of parsing 'git for-each-ref' output produced
|
||||||
|
// with this Format.
|
||||||
|
func (f Format) Parser(r io.Reader) *Parser {
|
||||||
|
return NewParser(r, f)
|
||||||
|
}
|
||||||
|
|
||||||
|
// hexEscaped produces hex-escpaed characters from a string. For example, "\n\0"
|
||||||
|
// would turn into "%0a%00".
|
||||||
|
func (f Format) hexEscaped(delim []byte) string {
|
||||||
|
escaped := ""
|
||||||
|
for i := 0; i < len(delim); i++ {
|
||||||
|
escaped += "%" + hex.EncodeToString([]byte{delim[i]})
|
||||||
|
}
|
||||||
|
return escaped
|
||||||
|
}
|
||||||
67
modules/git/foreachref/format_test.go
Normal file
67
modules/git/foreachref/format_test.go
Normal file
|
|
@ -0,0 +1,67 @@
|
||||||
|
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a MIT-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package foreachref_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"code.gitea.io/gitea/modules/git/foreachref"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestFormat_Flag(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
|
||||||
|
givenFormat foreachref.Format
|
||||||
|
|
||||||
|
wantFlag string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "references are delimited by dual null chars",
|
||||||
|
|
||||||
|
// no reference fields requested
|
||||||
|
givenFormat: foreachref.NewFormat(),
|
||||||
|
|
||||||
|
// only a reference delimiter field in --format
|
||||||
|
wantFlag: "%00%00",
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
name: "a field is a space-separated key-value pair",
|
||||||
|
|
||||||
|
givenFormat: foreachref.NewFormat("refname:short"),
|
||||||
|
|
||||||
|
// only a reference delimiter field
|
||||||
|
wantFlag: "refname:short %(refname:short)%00%00",
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
name: "fields are separated by a null char field-delimiter",
|
||||||
|
|
||||||
|
givenFormat: foreachref.NewFormat("refname:short", "author"),
|
||||||
|
|
||||||
|
wantFlag: "refname:short %(refname:short)%00author %(author)%00%00",
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
name: "multiple fields",
|
||||||
|
|
||||||
|
givenFormat: foreachref.NewFormat("refname:short", "objecttype", "objectname"),
|
||||||
|
|
||||||
|
wantFlag: "refname:short %(refname:short)%00objecttype %(objecttype)%00objectname %(objectname)%00%00",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
tc := test // don't close over loop variable
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
gotFlag := tc.givenFormat.Flag()
|
||||||
|
|
||||||
|
require.Equal(t, tc.wantFlag, gotFlag, "unexpected for-each-ref --format string. wanted: '%s', got: '%s'", tc.wantFlag, gotFlag)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
131
modules/git/foreachref/parser.go
Normal file
131
modules/git/foreachref/parser.go
Normal file
|
|
@ -0,0 +1,131 @@
|
||||||
|
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a MIT-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package foreachref
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"bytes"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Parser parses 'git for-each-ref' output according to a given output Format.
|
||||||
|
type Parser struct {
|
||||||
|
// tokenizes 'git for-each-ref' output into "reference paragraphs".
|
||||||
|
scanner *bufio.Scanner
|
||||||
|
|
||||||
|
// format represents the '--format' string that describes the expected
|
||||||
|
// 'git for-each-ref' output structure.
|
||||||
|
format Format
|
||||||
|
|
||||||
|
// err holds the last encountered error during parsing.
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewParser creates a 'git for-each-ref' output parser that will parse all
|
||||||
|
// references in the provided Reader. The references in the output are assumed
|
||||||
|
// to follow the specified Format.
|
||||||
|
func NewParser(r io.Reader, format Format) *Parser {
|
||||||
|
scanner := bufio.NewScanner(r)
|
||||||
|
|
||||||
|
// in addition to the reference delimiter we specified in the --format,
|
||||||
|
// `git for-each-ref` will always add a newline after every reference.
|
||||||
|
refDelim := make([]byte, 0, len(format.refDelim)+1)
|
||||||
|
refDelim = append(refDelim, format.refDelim...)
|
||||||
|
refDelim = append(refDelim, '\n')
|
||||||
|
|
||||||
|
// Split input into delimiter-separated "reference blocks".
|
||||||
|
scanner.Split(
|
||||||
|
func(data []byte, atEOF bool) (advance int, token []byte, err error) {
|
||||||
|
// Scan until delimiter, marking end of reference.
|
||||||
|
delimIdx := bytes.Index(data, refDelim)
|
||||||
|
if delimIdx >= 0 {
|
||||||
|
token := data[:delimIdx]
|
||||||
|
advance := delimIdx + len(refDelim)
|
||||||
|
return advance, token, nil
|
||||||
|
}
|
||||||
|
// If we're at EOF, we have a final, non-terminated reference. Return it.
|
||||||
|
if atEOF {
|
||||||
|
return len(data), data, nil
|
||||||
|
}
|
||||||
|
// Not yet a full field. Request more data.
|
||||||
|
return 0, nil, nil
|
||||||
|
})
|
||||||
|
|
||||||
|
return &Parser{
|
||||||
|
scanner: scanner,
|
||||||
|
format: format,
|
||||||
|
err: nil,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Next returns the next reference as a collection of key-value pairs. nil
|
||||||
|
// denotes EOF but is also returned on errors. The Err method should always be
|
||||||
|
// consulted after Next returning nil.
|
||||||
|
//
|
||||||
|
// It could, for example return something like:
|
||||||
|
//
|
||||||
|
// { "objecttype": "tag", "refname:short": "v1.16.4", "object": "f460b7543ed500e49c133c2cd85c8c55ee9dbe27" }
|
||||||
|
//
|
||||||
|
func (p *Parser) Next() map[string]string {
|
||||||
|
if !p.scanner.Scan() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
fields, err := p.parseRef(p.scanner.Text())
|
||||||
|
if err != nil {
|
||||||
|
p.err = err
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return fields
|
||||||
|
}
|
||||||
|
|
||||||
|
// Err returns the latest encountered parsing error.
|
||||||
|
func (p *Parser) Err() error {
|
||||||
|
return p.err
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseRef parses out all key-value pairs from a single reference block, such as
|
||||||
|
//
|
||||||
|
// "objecttype tag\0refname:short v1.16.4\0object f460b7543ed500e49c133c2cd85c8c55ee9dbe27"
|
||||||
|
//
|
||||||
|
func (p *Parser) parseRef(refBlock string) (map[string]string, error) {
|
||||||
|
if refBlock == "" {
|
||||||
|
// must be at EOF
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
fieldValues := make(map[string]string)
|
||||||
|
|
||||||
|
fields := strings.Split(refBlock, p.format.fieldDelimStr)
|
||||||
|
if len(fields) != len(p.format.fieldNames) {
|
||||||
|
return nil, fmt.Errorf("unexpected number of reference fields: wanted %d, was %d",
|
||||||
|
len(fields), len(p.format.fieldNames))
|
||||||
|
}
|
||||||
|
for i, field := range fields {
|
||||||
|
field = strings.TrimSpace(field)
|
||||||
|
|
||||||
|
var fieldKey string
|
||||||
|
var fieldVal string
|
||||||
|
firstSpace := strings.Index(field, " ")
|
||||||
|
if firstSpace > 0 {
|
||||||
|
fieldKey = field[:firstSpace]
|
||||||
|
fieldVal = field[firstSpace+1:]
|
||||||
|
} else {
|
||||||
|
// could be the case if the requested field had no value
|
||||||
|
fieldKey = field
|
||||||
|
}
|
||||||
|
|
||||||
|
// enforce the format order of fields
|
||||||
|
if p.format.fieldNames[i] != fieldKey {
|
||||||
|
return nil, fmt.Errorf("unexpected field name at position %d: wanted: '%s', was: '%s'",
|
||||||
|
i, p.format.fieldNames[i], fieldKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
fieldValues[fieldKey] = fieldVal
|
||||||
|
}
|
||||||
|
|
||||||
|
return fieldValues, nil
|
||||||
|
}
|
||||||
228
modules/git/foreachref/parser_test.go
Normal file
228
modules/git/foreachref/parser_test.go
Normal file
|
|
@ -0,0 +1,228 @@
|
||||||
|
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a MIT-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package foreachref_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"code.gitea.io/gitea/modules/git/foreachref"
|
||||||
|
"code.gitea.io/gitea/modules/json"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
type refSlice = []map[string]string
|
||||||
|
|
||||||
|
func TestParser(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
|
||||||
|
givenFormat foreachref.Format
|
||||||
|
givenInput io.Reader
|
||||||
|
|
||||||
|
wantRefs refSlice
|
||||||
|
wantErr bool
|
||||||
|
expectedErr error
|
||||||
|
}{
|
||||||
|
// this would, for example, be the result when running `git
|
||||||
|
// for-each-ref refs/tags` on a repo without tags.
|
||||||
|
{
|
||||||
|
name: "no references on empty input",
|
||||||
|
|
||||||
|
givenFormat: foreachref.NewFormat("refname:short"),
|
||||||
|
givenInput: strings.NewReader(``),
|
||||||
|
|
||||||
|
wantRefs: []map[string]string{},
|
||||||
|
},
|
||||||
|
|
||||||
|
// note: `git for-each-ref` will add a newline between every
|
||||||
|
// reference (in addition to the ref-delimiter we've chosen)
|
||||||
|
{
|
||||||
|
name: "single field requested, single reference in output",
|
||||||
|
|
||||||
|
givenFormat: foreachref.NewFormat("refname:short"),
|
||||||
|
givenInput: strings.NewReader("refname:short v0.0.1\x00\x00" + "\n"),
|
||||||
|
|
||||||
|
wantRefs: []map[string]string{
|
||||||
|
{"refname:short": "v0.0.1"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "single field requested, multiple references in output",
|
||||||
|
|
||||||
|
givenFormat: foreachref.NewFormat("refname:short"),
|
||||||
|
givenInput: strings.NewReader(
|
||||||
|
"refname:short v0.0.1\x00\x00" + "\n" +
|
||||||
|
"refname:short v0.0.2\x00\x00" + "\n" +
|
||||||
|
"refname:short v0.0.3\x00\x00" + "\n"),
|
||||||
|
|
||||||
|
wantRefs: []map[string]string{
|
||||||
|
{"refname:short": "v0.0.1"},
|
||||||
|
{"refname:short": "v0.0.2"},
|
||||||
|
{"refname:short": "v0.0.3"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
name: "multiple fields requested for each reference",
|
||||||
|
|
||||||
|
givenFormat: foreachref.NewFormat("refname:short", "objecttype", "objectname"),
|
||||||
|
givenInput: strings.NewReader(
|
||||||
|
|
||||||
|
"refname:short v0.0.1\x00objecttype commit\x00objectname 7b2c5ac9fc04fc5efafb60700713d4fa609b777b\x00\x00" + "\n" +
|
||||||
|
"refname:short v0.0.2\x00objecttype commit\x00objectname a1f051bc3eba734da4772d60e2d677f47cf93ef4\x00\x00" + "\n" +
|
||||||
|
"refname:short v0.0.3\x00objecttype commit\x00objectname ef82de70bb3f60c65fb8eebacbb2d122ef517385\x00\x00" + "\n",
|
||||||
|
),
|
||||||
|
|
||||||
|
wantRefs: []map[string]string{
|
||||||
|
{
|
||||||
|
"refname:short": "v0.0.1",
|
||||||
|
"objecttype": "commit",
|
||||||
|
"objectname": "7b2c5ac9fc04fc5efafb60700713d4fa609b777b",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"refname:short": "v0.0.2",
|
||||||
|
"objecttype": "commit",
|
||||||
|
"objectname": "a1f051bc3eba734da4772d60e2d677f47cf93ef4",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"refname:short": "v0.0.3",
|
||||||
|
"objecttype": "commit",
|
||||||
|
"objectname": "ef82de70bb3f60c65fb8eebacbb2d122ef517385",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
name: "must handle multi-line fields such as 'content'",
|
||||||
|
|
||||||
|
givenFormat: foreachref.NewFormat("refname:short", "contents", "author"),
|
||||||
|
givenInput: strings.NewReader(
|
||||||
|
"refname:short v0.0.1\x00contents Create new buffer if not present yet (#549)\n\nFixes a nil dereference when ProcessFoo is used\nwith multiple commands.\x00author Foo Bar <foo@bar.com> 1507832733 +0200\x00\x00" + "\n" +
|
||||||
|
"refname:short v0.0.2\x00contents Update CI config (#651)\n\n\x00author John Doe <john.doe@foo.com> 1521643174 +0000\x00\x00" + "\n" +
|
||||||
|
"refname:short v0.0.3\x00contents Fixed code sample for bash completion (#687)\n\n\x00author Foo Baz <foo@baz.com> 1524836750 +0200\x00\x00" + "\n",
|
||||||
|
),
|
||||||
|
|
||||||
|
wantRefs: []map[string]string{
|
||||||
|
{
|
||||||
|
"refname:short": "v0.0.1",
|
||||||
|
"contents": "Create new buffer if not present yet (#549)\n\nFixes a nil dereference when ProcessFoo is used\nwith multiple commands.",
|
||||||
|
"author": "Foo Bar <foo@bar.com> 1507832733 +0200",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"refname:short": "v0.0.2",
|
||||||
|
"contents": "Update CI config (#651)",
|
||||||
|
"author": "John Doe <john.doe@foo.com> 1521643174 +0000",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"refname:short": "v0.0.3",
|
||||||
|
"contents": "Fixed code sample for bash completion (#687)",
|
||||||
|
"author": "Foo Baz <foo@baz.com> 1524836750 +0200",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
name: "must handle fields without values",
|
||||||
|
|
||||||
|
givenFormat: foreachref.NewFormat("refname:short", "object", "objecttype"),
|
||||||
|
givenInput: strings.NewReader(
|
||||||
|
"refname:short v0.0.1\x00object \x00objecttype commit\x00\x00" + "\n" +
|
||||||
|
"refname:short v0.0.2\x00object \x00objecttype commit\x00\x00" + "\n" +
|
||||||
|
"refname:short v0.0.3\x00object \x00objecttype commit\x00\x00" + "\n",
|
||||||
|
),
|
||||||
|
|
||||||
|
wantRefs: []map[string]string{
|
||||||
|
{
|
||||||
|
"refname:short": "v0.0.1",
|
||||||
|
"object": "",
|
||||||
|
"objecttype": "commit",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"refname:short": "v0.0.2",
|
||||||
|
"object": "",
|
||||||
|
"objecttype": "commit",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"refname:short": "v0.0.3",
|
||||||
|
"object": "",
|
||||||
|
"objecttype": "commit",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
name: "must fail when the number of fields in the input doesn't match expected format",
|
||||||
|
|
||||||
|
givenFormat: foreachref.NewFormat("refname:short", "objecttype", "objectname"),
|
||||||
|
givenInput: strings.NewReader(
|
||||||
|
"refname:short v0.0.1\x00objecttype commit\x00\x00" + "\n" +
|
||||||
|
"refname:short v0.0.2\x00objecttype commit\x00\x00" + "\n" +
|
||||||
|
"refname:short v0.0.3\x00objecttype commit\x00\x00" + "\n",
|
||||||
|
),
|
||||||
|
|
||||||
|
wantErr: true,
|
||||||
|
expectedErr: errors.New("unexpected number of reference fields: wanted 2, was 3"),
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
name: "must fail input fields don't match expected format",
|
||||||
|
|
||||||
|
givenFormat: foreachref.NewFormat("refname:short", "objectname"),
|
||||||
|
givenInput: strings.NewReader(
|
||||||
|
"refname:short v0.0.1\x00objecttype commit\x00\x00" + "\n" +
|
||||||
|
"refname:short v0.0.2\x00objecttype commit\x00\x00" + "\n" +
|
||||||
|
"refname:short v0.0.3\x00objecttype commit\x00\x00" + "\n",
|
||||||
|
),
|
||||||
|
|
||||||
|
wantErr: true,
|
||||||
|
expectedErr: errors.New("unexpected field name at position 1: wanted: 'objectname', was: 'objecttype'"),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
tc := test // don't close over loop variable
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
parser := tc.givenFormat.Parser(tc.givenInput)
|
||||||
|
|
||||||
|
//
|
||||||
|
// parse references from input
|
||||||
|
//
|
||||||
|
gotRefs := make([]map[string]string, 0)
|
||||||
|
for {
|
||||||
|
ref := parser.Next()
|
||||||
|
if ref == nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
gotRefs = append(gotRefs, ref)
|
||||||
|
}
|
||||||
|
err := parser.Err()
|
||||||
|
|
||||||
|
//
|
||||||
|
// verify expectations
|
||||||
|
//
|
||||||
|
if tc.wantErr {
|
||||||
|
require.Error(t, err)
|
||||||
|
require.EqualError(t, err, tc.expectedErr.Error())
|
||||||
|
} else {
|
||||||
|
require.NoError(t, err, "for-each-ref parser unexpectedly failed with: %v", err)
|
||||||
|
require.Equal(t, tc.wantRefs, gotRefs, "for-each-ref parser produced unexpected reference set. wanted: %v, got: %v", pretty(tc.wantRefs), pretty(gotRefs))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func pretty(v interface{}) string {
|
||||||
|
data, err := json.MarshalIndent(v, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
// shouldn't happen
|
||||||
|
panic(fmt.Sprintf("json-marshalling failed: %v", err))
|
||||||
|
}
|
||||||
|
return string(data)
|
||||||
|
}
|
||||||
|
|
@ -8,8 +8,10 @@ package git
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"code.gitea.io/gitea/modules/git/foreachref"
|
||||||
"code.gitea.io/gitea/modules/util"
|
"code.gitea.io/gitea/modules/util"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -111,37 +113,98 @@ func (repo *Repository) GetTagWithID(idStr, name string) (*Tag, error) {
|
||||||
|
|
||||||
// GetTagInfos returns all tag infos of the repository.
|
// GetTagInfos returns all tag infos of the repository.
|
||||||
func (repo *Repository) GetTagInfos(page, pageSize int) ([]*Tag, int, error) {
|
func (repo *Repository) GetTagInfos(page, pageSize int) ([]*Tag, int, error) {
|
||||||
// TODO this a slow implementation, makes one git command per tag
|
forEachRefFmt := foreachref.NewFormat("objecttype", "refname:short", "object", "objectname", "creator", "contents", "contents:signature")
|
||||||
stdout, err := NewCommand(repo.Ctx, "tag").RunInDir(repo.Path)
|
|
||||||
|
stdoutReader, stdoutWriter := io.Pipe()
|
||||||
|
defer stdoutReader.Close()
|
||||||
|
defer stdoutWriter.Close()
|
||||||
|
stderr := strings.Builder{}
|
||||||
|
rc := &RunContext{Dir: repo.Path, Stdout: stdoutWriter, Stderr: &stderr, Timeout: -1}
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
err := NewCommand(repo.Ctx, "for-each-ref", "--format", forEachRefFmt.Flag(), "--sort", "-*creatordate", "refs/tags").RunWithContext(rc)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, 0, err
|
_ = stdoutWriter.CloseWithError(ConcatenateError(err, stderr.String()))
|
||||||
|
} else {
|
||||||
|
_ = stdoutWriter.Close()
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
var tags []*Tag
|
||||||
|
parser := forEachRefFmt.Parser(stdoutReader)
|
||||||
|
for {
|
||||||
|
ref := parser.Next()
|
||||||
|
if ref == nil {
|
||||||
|
break
|
||||||
}
|
}
|
||||||
|
|
||||||
tagNames := strings.Split(strings.TrimRight(stdout, "\n"), "\n")
|
tag, err := parseTagRef(ref)
|
||||||
tagsTotal := len(tagNames)
|
|
||||||
|
|
||||||
if page != 0 {
|
|
||||||
tagNames = util.PaginateSlice(tagNames, page, pageSize).([]string)
|
|
||||||
}
|
|
||||||
|
|
||||||
tags := make([]*Tag, 0, len(tagNames))
|
|
||||||
for _, tagName := range tagNames {
|
|
||||||
tagName = strings.TrimSpace(tagName)
|
|
||||||
if len(tagName) == 0 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
tag, err := repo.GetTag(tagName)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, tagsTotal, err
|
return nil, 0, fmt.Errorf("GetTagInfos: parse tag: %w", err)
|
||||||
}
|
}
|
||||||
tag.Name = tagName
|
|
||||||
tags = append(tags, tag)
|
tags = append(tags, tag)
|
||||||
}
|
}
|
||||||
|
if err := parser.Err(); err != nil {
|
||||||
|
return nil, 0, fmt.Errorf("GetTagInfos: parse output: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
sortTagsByTime(tags)
|
sortTagsByTime(tags)
|
||||||
|
tagsTotal := len(tags)
|
||||||
|
if page != 0 {
|
||||||
|
tags = util.PaginateSlice(tags, page, pageSize).([]*Tag)
|
||||||
|
}
|
||||||
|
|
||||||
return tags, tagsTotal, nil
|
return tags, tagsTotal, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// parseTagRef parses a tag from a 'git for-each-ref'-produced reference.
|
||||||
|
func parseTagRef(ref map[string]string) (tag *Tag, err error) {
|
||||||
|
tag = &Tag{
|
||||||
|
Type: ref["objecttype"],
|
||||||
|
Name: ref["refname:short"],
|
||||||
|
}
|
||||||
|
|
||||||
|
tag.ID, err = NewIDFromString(ref["objectname"])
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("parse objectname '%s': %v", ref["objectname"], err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if tag.Type == "commit" {
|
||||||
|
// lightweight tag
|
||||||
|
tag.Object = tag.ID
|
||||||
|
} else {
|
||||||
|
// annotated tag
|
||||||
|
tag.Object, err = NewIDFromString(ref["object"])
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("parse object '%s': %v", ref["object"], err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
tag.Tagger, err = newSignatureFromCommitline([]byte(ref["creator"]))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("parse tagger: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
tag.Message = ref["contents"]
|
||||||
|
// strip PGP signature if present in contents field
|
||||||
|
pgpStart := strings.Index(tag.Message, beginpgp)
|
||||||
|
if pgpStart >= 0 {
|
||||||
|
tag.Message = tag.Message[0:pgpStart]
|
||||||
|
}
|
||||||
|
|
||||||
|
// annotated tag with GPG signature
|
||||||
|
if tag.Type == "tag" && ref["contents:signature"] != "" {
|
||||||
|
payload := fmt.Sprintf("object %s\ntype commit\ntag %s\ntagger %s\n\n%s\n",
|
||||||
|
tag.Object, tag.Name, ref["creator"], strings.TrimSpace(tag.Message))
|
||||||
|
tag.Signature = &CommitGPGSignature{
|
||||||
|
Signature: ref["contents:signature"],
|
||||||
|
Payload: payload,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return tag, nil
|
||||||
|
}
|
||||||
|
|
||||||
// GetAnnotatedTag returns a Git tag by its SHA, must be an annotated tag
|
// GetAnnotatedTag returns a Git tag by its SHA, must be an annotated tag
|
||||||
func (repo *Repository) GetAnnotatedTag(sha string) (*Tag, error) {
|
func (repo *Repository) GetAnnotatedTag(sha string) (*Tag, error) {
|
||||||
id, err := NewIDFromString(sha)
|
id, err := NewIDFromString(sha)
|
||||||
|
|
|
||||||
|
|
@ -11,6 +11,7 @@ import (
|
||||||
"code.gitea.io/gitea/modules/util"
|
"code.gitea.io/gitea/modules/util"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestRepository_GetTags(t *testing.T) {
|
func TestRepository_GetTags(t *testing.T) {
|
||||||
|
|
@ -195,3 +196,184 @@ func TestRepository_GetAnnotatedTag(t *testing.T) {
|
||||||
assert.True(t, IsErrNotExist(err))
|
assert.True(t, IsErrNotExist(err))
|
||||||
assert.Nil(t, tag4)
|
assert.Nil(t, tag4)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRepository_parseTagRef(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
|
||||||
|
givenRef map[string]string
|
||||||
|
|
||||||
|
want *Tag
|
||||||
|
wantErr bool
|
||||||
|
expectedErr error
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "lightweight tag",
|
||||||
|
|
||||||
|
givenRef: map[string]string{
|
||||||
|
"objecttype": "commit",
|
||||||
|
"refname:short": "v1.9.1",
|
||||||
|
// object will be empty for lightweight tags
|
||||||
|
"object": "",
|
||||||
|
"objectname": "ab23e4b7f4cd0caafe0174c0e7ef6d651ba72889",
|
||||||
|
"creator": "Foo Bar <foo@bar.com> 1565789218 +0300",
|
||||||
|
"contents": `Add changelog of v1.9.1 (#7859)
|
||||||
|
|
||||||
|
* add changelog of v1.9.1
|
||||||
|
* Update CHANGELOG.md
|
||||||
|
`,
|
||||||
|
"contents:signature": "",
|
||||||
|
},
|
||||||
|
|
||||||
|
want: &Tag{
|
||||||
|
Name: "v1.9.1",
|
||||||
|
ID: MustIDFromString("ab23e4b7f4cd0caafe0174c0e7ef6d651ba72889"),
|
||||||
|
Object: MustIDFromString("ab23e4b7f4cd0caafe0174c0e7ef6d651ba72889"),
|
||||||
|
Type: "commit",
|
||||||
|
Tagger: parseAuthorLine(t, "Foo Bar <foo@bar.com> 1565789218 +0300"),
|
||||||
|
Message: "Add changelog of v1.9.1 (#7859)\n\n* add changelog of v1.9.1\n* Update CHANGELOG.md\n",
|
||||||
|
Signature: nil,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
name: "annotated tag",
|
||||||
|
|
||||||
|
givenRef: map[string]string{
|
||||||
|
"objecttype": "tag",
|
||||||
|
"refname:short": "v0.0.1",
|
||||||
|
// object will refer to commit hash for annotated tag
|
||||||
|
"object": "3325fd8a973321fd59455492976c042dde3fd1ca",
|
||||||
|
"objectname": "8c68a1f06fc59c655b7e3905b159d761e91c53c9",
|
||||||
|
"creator": "Foo Bar <foo@bar.com> 1565789218 +0300",
|
||||||
|
"contents": `Add changelog of v1.9.1 (#7859)
|
||||||
|
|
||||||
|
* add changelog of v1.9.1
|
||||||
|
* Update CHANGELOG.md
|
||||||
|
`,
|
||||||
|
"contents:signature": "",
|
||||||
|
},
|
||||||
|
|
||||||
|
want: &Tag{
|
||||||
|
Name: "v0.0.1",
|
||||||
|
ID: MustIDFromString("8c68a1f06fc59c655b7e3905b159d761e91c53c9"),
|
||||||
|
Object: MustIDFromString("3325fd8a973321fd59455492976c042dde3fd1ca"),
|
||||||
|
Type: "tag",
|
||||||
|
Tagger: parseAuthorLine(t, "Foo Bar <foo@bar.com> 1565789218 +0300"),
|
||||||
|
Message: "Add changelog of v1.9.1 (#7859)\n\n* add changelog of v1.9.1\n* Update CHANGELOG.md\n",
|
||||||
|
Signature: nil,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
name: "annotated tag with signature",
|
||||||
|
|
||||||
|
givenRef: map[string]string{
|
||||||
|
"objecttype": "tag",
|
||||||
|
"refname:short": "v0.0.1",
|
||||||
|
"object": "3325fd8a973321fd59455492976c042dde3fd1ca",
|
||||||
|
"objectname": "8c68a1f06fc59c655b7e3905b159d761e91c53c9",
|
||||||
|
"creator": "Foo Bar <foo@bar.com> 1565789218 +0300",
|
||||||
|
"contents": `Add changelog of v1.9.1 (#7859)
|
||||||
|
|
||||||
|
* add changelog of v1.9.1
|
||||||
|
* Update CHANGELOG.md
|
||||||
|
-----BEGIN PGP SIGNATURE-----
|
||||||
|
|
||||||
|
aBCGzBAABCgAdFiEEyWRwv/q1Q6IjSv+D4IPOwzt33PoFAmI8jbIACgkQ4IPOwzt3
|
||||||
|
3PoRuAv9FVSbPBXvzECubls9KQd7urwEvcfG20Uf79iBwifQJUv+egNQojrs6APT
|
||||||
|
T4CdIXeGRpwJZaGTUX9RWnoDO1SLXAWnc82CypWraNwrHq8Go2YeoVu0Iy3vb0EU
|
||||||
|
REdob/tXYZecMuP8AjhUR0XfdYaERYAvJ2dYsH/UkFrqDjM3V4kPXWG+R5DCaZiE
|
||||||
|
slB5U01i4Dwb/zm/ckzhUGEcOgcnpOKX8SnY5kYRVDY47dl/yJZ1u2XWir3mu60G
|
||||||
|
1geIitH7StBddHi/8rz+sJwTfcVaLjn2p59p/Dr9aGbk17GIaKq1j0pZA2lKT0Xt
|
||||||
|
f9jDqU+9vCxnKgjSDhrwN69LF2jT47ZFjEMGV/wFPOa1EBxVWpgQ/CfEolBlbUqx
|
||||||
|
yVpbxi/6AOK2lmG130e9jEZJcu+WeZUeq851WgKSEkf2d5f/JpwtSTEOlOedu6V6
|
||||||
|
kl845zu5oE2nKM4zMQ7XrYQn538I31ps+VGQ0H8R07WrZP8WKUWugL2cU8KmXFwg
|
||||||
|
qbHDASXl
|
||||||
|
=2yGi
|
||||||
|
-----END PGP SIGNATURE-----
|
||||||
|
|
||||||
|
`,
|
||||||
|
"contents:signature": `-----BEGIN PGP SIGNATURE-----
|
||||||
|
|
||||||
|
aBCGzBAABCgAdFiEEyWRwv/q1Q6IjSv+D4IPOwzt33PoFAmI8jbIACgkQ4IPOwzt3
|
||||||
|
3PoRuAv9FVSbPBXvzECubls9KQd7urwEvcfG20Uf79iBwifQJUv+egNQojrs6APT
|
||||||
|
T4CdIXeGRpwJZaGTUX9RWnoDO1SLXAWnc82CypWraNwrHq8Go2YeoVu0Iy3vb0EU
|
||||||
|
REdob/tXYZecMuP8AjhUR0XfdYaERYAvJ2dYsH/UkFrqDjM3V4kPXWG+R5DCaZiE
|
||||||
|
slB5U01i4Dwb/zm/ckzhUGEcOgcnpOKX8SnY5kYRVDY47dl/yJZ1u2XWir3mu60G
|
||||||
|
1geIitH7StBddHi/8rz+sJwTfcVaLjn2p59p/Dr9aGbk17GIaKq1j0pZA2lKT0Xt
|
||||||
|
f9jDqU+9vCxnKgjSDhrwN69LF2jT47ZFjEMGV/wFPOa1EBxVWpgQ/CfEolBlbUqx
|
||||||
|
yVpbxi/6AOK2lmG130e9jEZJcu+WeZUeq851WgKSEkf2d5f/JpwtSTEOlOedu6V6
|
||||||
|
kl845zu5oE2nKM4zMQ7XrYQn538I31ps+VGQ0H8R07WrZP8WKUWugL2cU8KmXFwg
|
||||||
|
qbHDASXl
|
||||||
|
=2yGi
|
||||||
|
-----END PGP SIGNATURE-----
|
||||||
|
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
|
||||||
|
want: &Tag{
|
||||||
|
Name: "v0.0.1",
|
||||||
|
ID: MustIDFromString("8c68a1f06fc59c655b7e3905b159d761e91c53c9"),
|
||||||
|
Object: MustIDFromString("3325fd8a973321fd59455492976c042dde3fd1ca"),
|
||||||
|
Type: "tag",
|
||||||
|
Tagger: parseAuthorLine(t, "Foo Bar <foo@bar.com> 1565789218 +0300"),
|
||||||
|
Message: "Add changelog of v1.9.1 (#7859)\n\n* add changelog of v1.9.1\n* Update CHANGELOG.md",
|
||||||
|
Signature: &CommitGPGSignature{
|
||||||
|
Signature: `-----BEGIN PGP SIGNATURE-----
|
||||||
|
|
||||||
|
aBCGzBAABCgAdFiEEyWRwv/q1Q6IjSv+D4IPOwzt33PoFAmI8jbIACgkQ4IPOwzt3
|
||||||
|
3PoRuAv9FVSbPBXvzECubls9KQd7urwEvcfG20Uf79iBwifQJUv+egNQojrs6APT
|
||||||
|
T4CdIXeGRpwJZaGTUX9RWnoDO1SLXAWnc82CypWraNwrHq8Go2YeoVu0Iy3vb0EU
|
||||||
|
REdob/tXYZecMuP8AjhUR0XfdYaERYAvJ2dYsH/UkFrqDjM3V4kPXWG+R5DCaZiE
|
||||||
|
slB5U01i4Dwb/zm/ckzhUGEcOgcnpOKX8SnY5kYRVDY47dl/yJZ1u2XWir3mu60G
|
||||||
|
1geIitH7StBddHi/8rz+sJwTfcVaLjn2p59p/Dr9aGbk17GIaKq1j0pZA2lKT0Xt
|
||||||
|
f9jDqU+9vCxnKgjSDhrwN69LF2jT47ZFjEMGV/wFPOa1EBxVWpgQ/CfEolBlbUqx
|
||||||
|
yVpbxi/6AOK2lmG130e9jEZJcu+WeZUeq851WgKSEkf2d5f/JpwtSTEOlOedu6V6
|
||||||
|
kl845zu5oE2nKM4zMQ7XrYQn538I31ps+VGQ0H8R07WrZP8WKUWugL2cU8KmXFwg
|
||||||
|
qbHDASXl
|
||||||
|
=2yGi
|
||||||
|
-----END PGP SIGNATURE-----
|
||||||
|
|
||||||
|
`,
|
||||||
|
Payload: `object 3325fd8a973321fd59455492976c042dde3fd1ca
|
||||||
|
type commit
|
||||||
|
tag v0.0.1
|
||||||
|
tagger Foo Bar <foo@bar.com> 1565789218 +0300
|
||||||
|
|
||||||
|
Add changelog of v1.9.1 (#7859)
|
||||||
|
|
||||||
|
* add changelog of v1.9.1
|
||||||
|
* Update CHANGELOG.md
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
tc := test // don't close over loop variable
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
got, err := parseTagRef(tc.givenRef)
|
||||||
|
|
||||||
|
if tc.wantErr {
|
||||||
|
require.Error(t, err)
|
||||||
|
require.ErrorIs(t, err, tc.expectedErr)
|
||||||
|
} else {
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, tc.want, got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseAuthorLine(t *testing.T, committer string) *Signature {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
sig, err := newSignatureFromCommitline([]byte(committer))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse author line '%s': %v", committer, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return sig
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -150,6 +150,9 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User,
|
||||||
}
|
}
|
||||||
|
|
||||||
if !opts.Releases {
|
if !opts.Releases {
|
||||||
|
// note: this will greatly improve release (tag) sync
|
||||||
|
// for pull-mirrors with many tags
|
||||||
|
repo.IsMirror = opts.Mirror
|
||||||
if err = SyncReleasesWithTags(repo, gitRepo); err != nil {
|
if err = SyncReleasesWithTags(repo, gitRepo); err != nil {
|
||||||
log.Error("Failed to synchronize tags to releases for repository: %v", err)
|
log.Error("Failed to synchronize tags to releases for repository: %v", err)
|
||||||
}
|
}
|
||||||
|
|
@ -254,6 +257,14 @@ func CleanUpMigrateInfo(ctx context.Context, repo *repo_model.Repository) (*repo
|
||||||
|
|
||||||
// SyncReleasesWithTags synchronizes release table with repository tags
|
// SyncReleasesWithTags synchronizes release table with repository tags
|
||||||
func SyncReleasesWithTags(repo *repo_model.Repository, gitRepo *git.Repository) error {
|
func SyncReleasesWithTags(repo *repo_model.Repository, gitRepo *git.Repository) error {
|
||||||
|
log.Debug("SyncReleasesWithTags: in Repo[%d:%s/%s]", repo.ID, repo.OwnerName, repo.Name)
|
||||||
|
|
||||||
|
// optimized procedure for pull-mirrors which saves a lot of time (in
|
||||||
|
// particular for repos with many tags).
|
||||||
|
if repo.IsMirror {
|
||||||
|
return pullMirrorReleaseSync(repo, gitRepo)
|
||||||
|
}
|
||||||
|
|
||||||
existingRelTags := make(map[string]struct{})
|
existingRelTags := make(map[string]struct{})
|
||||||
opts := models.FindReleasesOptions{
|
opts := models.FindReleasesOptions{
|
||||||
IncludeDrafts: true,
|
IncludeDrafts: true,
|
||||||
|
|
@ -450,3 +461,52 @@ func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *repo_model.Re
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// pullMirrorReleaseSync is a pull-mirror specific tag<->release table
|
||||||
|
// synchronization which overwrites all Releases from the repository tags. This
|
||||||
|
// can be relied on since a pull-mirror is always identical to its
|
||||||
|
// upstream. Hence, after each sync we want the pull-mirror release set to be
|
||||||
|
// identical to the upstream tag set. This is much more efficient for
|
||||||
|
// repositories like https://github.com/vim/vim (with over 13000 tags).
|
||||||
|
func pullMirrorReleaseSync(repo *repo_model.Repository, gitRepo *git.Repository) error {
|
||||||
|
log.Trace("pullMirrorReleaseSync: rebuilding releases for pull-mirror Repo[%d:%s/%s]", repo.ID, repo.OwnerName, repo.Name)
|
||||||
|
tags, numTags, err := gitRepo.GetTagInfos(0, 0)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("unable to GetTagInfos in pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err)
|
||||||
|
}
|
||||||
|
err = db.WithTx(func(ctx context.Context) error {
|
||||||
|
//
|
||||||
|
// clear out existing releases
|
||||||
|
//
|
||||||
|
if _, err := db.DeleteByBean(ctx, &models.Release{RepoID: repo.ID}); err != nil {
|
||||||
|
return fmt.Errorf("unable to clear releases for pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err)
|
||||||
|
}
|
||||||
|
//
|
||||||
|
// make release set identical to upstream tags
|
||||||
|
//
|
||||||
|
for _, tag := range tags {
|
||||||
|
release := models.Release{
|
||||||
|
RepoID: repo.ID,
|
||||||
|
TagName: tag.Name,
|
||||||
|
LowerTagName: strings.ToLower(tag.Name),
|
||||||
|
Sha1: tag.Object.String(),
|
||||||
|
// NOTE: ignored, since NumCommits are unused
|
||||||
|
// for pull-mirrors (only relevant when
|
||||||
|
// displaying releases, IsTag: false)
|
||||||
|
NumCommits: -1,
|
||||||
|
CreatedUnix: timeutil.TimeStamp(tag.Tagger.When.Unix()),
|
||||||
|
IsTag: true,
|
||||||
|
}
|
||||||
|
if err := db.Insert(ctx, release); err != nil {
|
||||||
|
return fmt.Errorf("unable insert tag %s for pull-mirror Repo[%d:%s/%s]: %w", tag.Name, repo.ID, repo.OwnerName, repo.Name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("unable to rebuild release table for pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Trace("pullMirrorReleaseSync: done rebuilding %d releases", numTags)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue