From a2cfcdb91a9098381bb1d71ebdf56baed4c0981d Mon Sep 17 00:00:00 2001 From: zeripath Date: Mon, 25 Jul 2022 16:39:42 +0100 Subject: [PATCH] Slightly simplify LastCommitCache (#20444) The LastCommitCache code is a little complex and there is unnecessary duplication between the gogit and nogogit variants. This PR adds the LastCommitCache as a field to the git.Repository and pre-creates it in the ReferencesGit helpers etc. There has been some simplification and unification of the variant code. Signed-off-by: Andrew Thornton --- modules/context/repo.go | 2 + modules/git/commit.go | 3 + modules/git/commit_info_gogit.go | 31 ++++----- modules/git/commit_info_nogogit.go | 21 +++--- modules/git/commit_info_test.go | 4 +- modules/git/last_commit_cache.go | 87 +++++++++++++++++++++++- modules/git/last_commit_cache_gogit.go | 62 +++-------------- modules/git/last_commit_cache_nogogit.go | 63 ++--------------- modules/git/log_name_status.go | 6 +- modules/git/notes_gogit.go | 2 +- modules/git/notes_nogogit.go | 2 +- modules/git/repo_base_gogit.go | 5 +- modules/git/repo_base_nogogit.go | 5 +- routers/api/v1/repo/file.go | 8 +-- routers/api/v1/utils/git.go | 24 ++++++- routers/web/repo/download.go | 8 +-- routers/web/repo/view.go | 8 +-- routers/web/web.go | 2 + services/repository/cache.go | 16 ++--- 19 files changed, 177 insertions(+), 182 deletions(-) diff --git a/modules/context/repo.go b/modules/context/repo.go index 183637391..1d9f98158 100644 --- a/modules/context/repo.go +++ b/modules/context/repo.go @@ -1001,6 +1001,8 @@ func RepoRefByType(refType RepoRefType, ignoreNotExistErr ...bool) func(*Context return } ctx.Data["CommitsCount"] = ctx.Repo.CommitsCount + ctx.Repo.GitRepo.LastCommitCache = git.NewLastCommitCache(ctx.Repo.CommitsCount, ctx.Repo.Repository.FullName(), ctx.Repo.GitRepo, cache.GetCache()) + return cancel } } diff --git a/modules/git/commit.go b/modules/git/commit.go index 82712dd1e..32589f534 100644 --- a/modules/git/commit.go +++ b/modules/git/commit.go @@ -80,6 +80,9 @@ func (c *Commit) ParentCount() int { // GetCommitByPath return the commit of relative path object. func (c *Commit) GetCommitByPath(relpath string) (*Commit, error) { + if c.repo.LastCommitCache != nil { + return c.repo.LastCommitCache.GetCommitByPath(c.ID.String(), relpath) + } return c.repo.getCommitByPathWithID(c.ID, relpath) } diff --git a/modules/git/commit_info_gogit.go b/modules/git/commit_info_gogit.go index 91a1804db..341698ab3 100644 --- a/modules/git/commit_info_gogit.go +++ b/modules/git/commit_info_gogit.go @@ -17,7 +17,7 @@ import ( ) // GetCommitsInfo gets information of all commits that are corresponding to these entries -func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath string, cache *LastCommitCache) ([]CommitInfo, *Commit, error) { +func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath string) ([]CommitInfo, *Commit, error) { entryPaths := make([]string, len(tes)+1) // Get the commit for the treePath itself entryPaths[0] = "" @@ -35,15 +35,15 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath return nil, nil, err } - var revs map[string]*object.Commit - if cache != nil { + var revs map[string]*Commit + if commit.repo.LastCommitCache != nil { var unHitPaths []string - revs, unHitPaths, err = getLastCommitForPathsByCache(commit.ID.String(), treePath, entryPaths, cache) + revs, unHitPaths, err = getLastCommitForPathsByCache(commit.ID.String(), treePath, entryPaths, commit.repo.LastCommitCache) if err != nil { return nil, nil, err } if len(unHitPaths) > 0 { - revs2, err := GetLastCommitForPaths(ctx, cache, c, treePath, unHitPaths) + revs2, err := GetLastCommitForPaths(ctx, commit.repo.LastCommitCache, c, treePath, unHitPaths) if err != nil { return nil, nil, err } @@ -68,8 +68,7 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath } // Check if we have found a commit for this entry in time - if rev, ok := revs[entry.Name()]; ok { - entryCommit := convertCommit(rev) + if entryCommit, ok := revs[entry.Name()]; ok { commitsInfo[i].Commit = entryCommit } @@ -96,10 +95,10 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath // get it for free during the tree traversal and it's used for listing // pages to display information about newest commit for a given path. var treeCommit *Commit + var ok bool if treePath == "" { treeCommit = commit - } else if rev, ok := revs[""]; ok { - treeCommit = convertCommit(rev) + } else if treeCommit, ok = revs[""]; ok { treeCommit.repo = commit.repo } return commitsInfo, treeCommit, nil @@ -155,16 +154,16 @@ func getFileHashes(c cgobject.CommitNode, treePath string, paths []string) (map[ return hashes, nil } -func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cache *LastCommitCache) (map[string]*object.Commit, []string, error) { +func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cache *LastCommitCache) (map[string]*Commit, []string, error) { var unHitEntryPaths []string - results := make(map[string]*object.Commit) + results := make(map[string]*Commit) for _, p := range paths { lastCommit, err := cache.Get(commitID, path.Join(treePath, p)) if err != nil { return nil, nil, err } if lastCommit != nil { - results[p] = lastCommit.(*object.Commit) + results[p] = lastCommit continue } @@ -175,7 +174,7 @@ func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cac } // GetLastCommitForPaths returns last commit information -func GetLastCommitForPaths(ctx context.Context, cache *LastCommitCache, c cgobject.CommitNode, treePath string, paths []string) (map[string]*object.Commit, error) { +func GetLastCommitForPaths(ctx context.Context, cache *LastCommitCache, c cgobject.CommitNode, treePath string, paths []string) (map[string]*Commit, error) { refSha := c.ID().String() // We do a tree traversal with nodes sorted by commit time @@ -293,13 +292,13 @@ heaploop: } // Post-processing - result := make(map[string]*object.Commit) + result := make(map[string]*Commit) for path, commitNode := range resultNodes { - var err error - result[path], err = commitNode.Commit() + commit, err := commitNode.Commit() if err != nil { return nil, err } + result[path] = convertCommit(commit) } return result, nil diff --git a/modules/git/commit_info_nogogit.go b/modules/git/commit_info_nogogit.go index ceab11adb..d7bca3b94 100644 --- a/modules/git/commit_info_nogogit.go +++ b/modules/git/commit_info_nogogit.go @@ -17,7 +17,7 @@ import ( ) // GetCommitsInfo gets information of all commits that are corresponding to these entries -func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath string, cache *LastCommitCache) ([]CommitInfo, *Commit, error) { +func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath string) ([]CommitInfo, *Commit, error) { entryPaths := make([]string, len(tes)+1) // Get the commit for the treePath itself entryPaths[0] = "" @@ -28,15 +28,15 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath var err error var revs map[string]*Commit - if cache != nil { + if commit.repo.LastCommitCache != nil { var unHitPaths []string - revs, unHitPaths, err = getLastCommitForPathsByCache(ctx, commit.ID.String(), treePath, entryPaths, cache) + revs, unHitPaths, err = getLastCommitForPathsByCache(ctx, commit.ID.String(), treePath, entryPaths, commit.repo.LastCommitCache) if err != nil { return nil, nil, err } if len(unHitPaths) > 0 { sort.Strings(unHitPaths) - commits, err := GetLastCommitForPaths(ctx, cache, commit, treePath, unHitPaths) + commits, err := GetLastCommitForPaths(ctx, commit, treePath, unHitPaths) if err != nil { return nil, nil, err } @@ -47,7 +47,7 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath } } else { sort.Strings(entryPaths) - revs, err = GetLastCommitForPaths(ctx, nil, commit, treePath, entryPaths) + revs, err = GetLastCommitForPaths(ctx, commit, treePath, entryPaths) } if err != nil { return nil, nil, err @@ -99,18 +99,15 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath } func getLastCommitForPathsByCache(ctx context.Context, commitID, treePath string, paths []string, cache *LastCommitCache) (map[string]*Commit, []string, error) { - wr, rd, cancel := cache.repo.CatFileBatch(ctx) - defer cancel() - var unHitEntryPaths []string results := make(map[string]*Commit) for _, p := range paths { - lastCommit, err := cache.Get(commitID, path.Join(treePath, p), wr, rd) + lastCommit, err := cache.Get(commitID, path.Join(treePath, p)) if err != nil { return nil, nil, err } if lastCommit != nil { - results[p] = lastCommit.(*Commit) + results[p] = lastCommit continue } @@ -121,9 +118,9 @@ func getLastCommitForPathsByCache(ctx context.Context, commitID, treePath string } // GetLastCommitForPaths returns last commit information -func GetLastCommitForPaths(ctx context.Context, cache *LastCommitCache, commit *Commit, treePath string, paths []string) (map[string]*Commit, error) { +func GetLastCommitForPaths(ctx context.Context, commit *Commit, treePath string, paths []string) (map[string]*Commit, error) { // We read backwards from the commit to obtain all of the commits - revs, err := WalkGitLog(ctx, cache, commit.repo, commit, treePath, paths...) + revs, err := WalkGitLog(ctx, commit.repo, commit, treePath, paths...) if err != nil { return nil, err } diff --git a/modules/git/commit_info_test.go b/modules/git/commit_info_test.go index 49845522a..a12452c40 100644 --- a/modules/git/commit_info_test.go +++ b/modules/git/commit_info_test.go @@ -91,7 +91,7 @@ func testGetCommitsInfo(t *testing.T, repo1 *Repository) { } // FIXME: Context.TODO() - if graceful has started we should use its Shutdown context otherwise use install signals in TestMain. - commitsInfo, treeCommit, err := entries.GetCommitsInfo(context.TODO(), commit, testCase.Path, nil) + commitsInfo, treeCommit, err := entries.GetCommitsInfo(context.TODO(), commit, testCase.Path) assert.NoError(t, err, "Unable to get commit information for entries of subtree: %s in commit: %s from testcase due to error: %v", testCase.Path, testCase.CommitID, err) if err != nil { t.FailNow() @@ -170,7 +170,7 @@ func BenchmarkEntries_GetCommitsInfo(b *testing.B) { b.ResetTimer() b.Run(benchmark.name, func(b *testing.B) { for i := 0; i < b.N; i++ { - _, _, err := entries.GetCommitsInfo(context.Background(), commit, "", nil) + _, _, err := entries.GetCommitsInfo(context.Background(), commit, "") if err != nil { b.Fatal(err) } diff --git a/modules/git/last_commit_cache.go b/modules/git/last_commit_cache.go index d4ec517b5..2b51d5972 100644 --- a/modules/git/last_commit_cache.go +++ b/modules/git/last_commit_cache.go @@ -9,6 +9,7 @@ import ( "fmt" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" ) // Cache represents a caching interface @@ -19,16 +20,96 @@ type Cache interface { Get(key string) interface{} } -func (c *LastCommitCache) getCacheKey(repoPath, ref, entryPath string) string { - hashBytes := sha256.Sum256([]byte(fmt.Sprintf("%s:%s:%s", repoPath, ref, entryPath))) +func getCacheKey(repoPath, commitID, entryPath string) string { + hashBytes := sha256.Sum256([]byte(fmt.Sprintf("%s:%s:%s", repoPath, commitID, entryPath))) return fmt.Sprintf("last_commit:%x", hashBytes) } +// LastCommitCache represents a cache to store last commit +type LastCommitCache struct { + repoPath string + ttl func() int64 + repo *Repository + commitCache map[string]*Commit + cache Cache +} + +// NewLastCommitCache creates a new last commit cache for repo +func NewLastCommitCache(count int64, repoPath string, gitRepo *Repository, cache Cache) *LastCommitCache { + if cache == nil { + return nil + } + if !setting.CacheService.LastCommit.Enabled || count < setting.CacheService.LastCommit.CommitsCount { + return nil + } + + return &LastCommitCache{ + repoPath: repoPath, + repo: gitRepo, + ttl: setting.LastCommitCacheTTLSeconds, + cache: cache, + } +} + // Put put the last commit id with commit and entry path func (c *LastCommitCache) Put(ref, entryPath, commitID string) error { if c == nil || c.cache == nil { return nil } log.Debug("LastCommitCache save: [%s:%s:%s]", ref, entryPath, commitID) - return c.cache.Put(c.getCacheKey(c.repoPath, ref, entryPath), commitID, c.ttl()) + return c.cache.Put(getCacheKey(c.repoPath, ref, entryPath), commitID, c.ttl()) +} + +// Get gets the last commit information by commit id and entry path +func (c *LastCommitCache) Get(ref, entryPath string) (*Commit, error) { + if c == nil || c.cache == nil { + return nil, nil + } + + commitID, ok := c.cache.Get(getCacheKey(c.repoPath, ref, entryPath)).(string) + if !ok || commitID == "" { + return nil, nil + } + + log.Debug("LastCommitCache hit level 1: [%s:%s:%s]", ref, entryPath, commitID) + if c.commitCache != nil { + if commit, ok := c.commitCache[commitID]; ok { + log.Debug("LastCommitCache hit level 2: [%s:%s:%s]", ref, entryPath, commitID) + return commit, nil + } + } + + commit, err := c.repo.GetCommit(commitID) + if err != nil { + return nil, err + } + if c.commitCache == nil { + c.commitCache = make(map[string]*Commit) + } + c.commitCache[commitID] = commit + return commit, nil +} + +// GetCommitByPath gets the last commit for the entry in the provided commit +func (c *LastCommitCache) GetCommitByPath(commitID, entryPath string) (*Commit, error) { + sha1, err := NewIDFromString(commitID) + if err != nil { + return nil, err + } + + lastCommit, err := c.Get(sha1.String(), entryPath) + if err != nil || lastCommit != nil { + return lastCommit, err + } + + lastCommit, err = c.repo.getCommitByPathWithID(sha1, entryPath) + if err != nil { + return nil, err + } + + if err := c.Put(commitID, entryPath, lastCommit.ID.String()); err != nil { + log.Error("Unable to cache %s as the last commit for %q in %s %s. Error %v", lastCommit.ID.String(), entryPath, commitID, c.repoPath, err) + } + + return lastCommit, nil } diff --git a/modules/git/last_commit_cache_gogit.go b/modules/git/last_commit_cache_gogit.go index 889700035..82c76bad2 100644 --- a/modules/git/last_commit_cache_gogit.go +++ b/modules/git/last_commit_cache_gogit.go @@ -9,71 +9,25 @@ package git import ( "context" - "code.gitea.io/gitea/modules/log" - - "github.com/go-git/go-git/v5/plumbing/object" cgobject "github.com/go-git/go-git/v5/plumbing/object/commitgraph" ) -// LastCommitCache represents a cache to store last commit -type LastCommitCache struct { - repoPath string - ttl func() int64 - repo *Repository - commitCache map[string]*object.Commit - cache Cache -} - -// NewLastCommitCache creates a new last commit cache for repo -func NewLastCommitCache(repoPath string, gitRepo *Repository, ttl func() int64, cache Cache) *LastCommitCache { - if cache == nil { +// CacheCommit will cache the commit from the gitRepository +func (c *Commit) CacheCommit(ctx context.Context) error { + if c.repo.LastCommitCache == nil { return nil } - return &LastCommitCache{ - repoPath: repoPath, - repo: gitRepo, - commitCache: make(map[string]*object.Commit), - ttl: ttl, - cache: cache, - } -} + commitNodeIndex, _ := c.repo.CommitNodeIndex() -// Get get the last commit information by commit id and entry path -func (c *LastCommitCache) Get(ref, entryPath string) (interface{}, error) { - v := c.cache.Get(c.getCacheKey(c.repoPath, ref, entryPath)) - if vs, ok := v.(string); ok { - log.Debug("LastCommitCache hit level 1: [%s:%s:%s]", ref, entryPath, vs) - if commit, ok := c.commitCache[vs]; ok { - log.Debug("LastCommitCache hit level 2: [%s:%s:%s]", ref, entryPath, vs) - return commit, nil - } - id, err := c.repo.ConvertToSHA1(vs) - if err != nil { - return nil, err - } - commit, err := c.repo.GoGitRepo().CommitObject(id) - if err != nil { - return nil, err - } - c.commitCache[vs] = commit - return commit, nil - } - return nil, nil -} - -// CacheCommit will cache the commit from the gitRepository -func (c *LastCommitCache) CacheCommit(ctx context.Context, commit *Commit) error { - commitNodeIndex, _ := commit.repo.CommitNodeIndex() - - index, err := commitNodeIndex.Get(commit.ID) + index, err := commitNodeIndex.Get(c.ID) if err != nil { return err } - return c.recursiveCache(ctx, index, &commit.Tree, "", 1) + return c.recursiveCache(ctx, index, &c.Tree, "", 1) } -func (c *LastCommitCache) recursiveCache(ctx context.Context, index cgobject.CommitNode, tree *Tree, treePath string, level int) error { +func (c *Commit) recursiveCache(ctx context.Context, index cgobject.CommitNode, tree *Tree, treePath string, level int) error { if level == 0 { return nil } @@ -90,7 +44,7 @@ func (c *LastCommitCache) recursiveCache(ctx context.Context, index cgobject.Com entryMap[entry.Name()] = entry } - commits, err := GetLastCommitForPaths(ctx, c, index, treePath, entryPaths) + commits, err := GetLastCommitForPaths(ctx, c.repo.LastCommitCache, index, treePath, entryPaths) if err != nil { return err } diff --git a/modules/git/last_commit_cache_nogogit.go b/modules/git/last_commit_cache_nogogit.go index 030d5486b..1f4d693a2 100644 --- a/modules/git/last_commit_cache_nogogit.go +++ b/modules/git/last_commit_cache_nogogit.go @@ -7,67 +7,18 @@ package git import ( - "bufio" "context" - - "code.gitea.io/gitea/modules/log" ) -// LastCommitCache represents a cache to store last commit -type LastCommitCache struct { - repoPath string - ttl func() int64 - repo *Repository - commitCache map[string]*Commit - cache Cache -} - -// NewLastCommitCache creates a new last commit cache for repo -func NewLastCommitCache(repoPath string, gitRepo *Repository, ttl func() int64, cache Cache) *LastCommitCache { - if cache == nil { +// CacheCommit will cache the commit from the gitRepository +func (c *Commit) CacheCommit(ctx context.Context) error { + if c.repo.LastCommitCache == nil { return nil } - return &LastCommitCache{ - repoPath: repoPath, - repo: gitRepo, - commitCache: make(map[string]*Commit), - ttl: ttl, - cache: cache, - } + return c.recursiveCache(ctx, &c.Tree, "", 1) } -// Get get the last commit information by commit id and entry path -func (c *LastCommitCache) Get(ref, entryPath string, wr WriteCloserError, rd *bufio.Reader) (interface{}, error) { - v := c.cache.Get(c.getCacheKey(c.repoPath, ref, entryPath)) - if vs, ok := v.(string); ok { - log.Debug("LastCommitCache hit level 1: [%s:%s:%s]", ref, entryPath, vs) - if commit, ok := c.commitCache[vs]; ok { - log.Debug("LastCommitCache hit level 2: [%s:%s:%s]", ref, entryPath, vs) - return commit, nil - } - id, err := c.repo.ConvertToSHA1(vs) - if err != nil { - return nil, err - } - if _, err := wr.Write([]byte(vs + "\n")); err != nil { - return nil, err - } - commit, err := c.repo.getCommitFromBatchReader(rd, id) - if err != nil { - return nil, err - } - c.commitCache[vs] = commit - return commit, nil - } - return nil, nil -} - -// CacheCommit will cache the commit from the gitRepository -func (c *LastCommitCache) CacheCommit(ctx context.Context, commit *Commit) error { - return c.recursiveCache(ctx, commit, &commit.Tree, "", 1) -} - -func (c *LastCommitCache) recursiveCache(ctx context.Context, commit *Commit, tree *Tree, treePath string, level int) error { +func (c *Commit) recursiveCache(ctx context.Context, tree *Tree, treePath string, level int) error { if level == 0 { return nil } @@ -82,7 +33,7 @@ func (c *LastCommitCache) recursiveCache(ctx context.Context, commit *Commit, tr entryPaths[i] = entry.Name() } - _, err = WalkGitLog(ctx, c, commit.repo, commit, treePath, entryPaths...) + _, err = WalkGitLog(ctx, c.repo, c, treePath, entryPaths...) if err != nil { return err } @@ -94,7 +45,7 @@ func (c *LastCommitCache) recursiveCache(ctx context.Context, commit *Commit, tr if err != nil { return err } - if err := c.recursiveCache(ctx, commit, subTree, treeEntry.Name(), level-1); err != nil { + if err := c.recursiveCache(ctx, subTree, treeEntry.Name(), level-1); err != nil { return err } } diff --git a/modules/git/log_name_status.go b/modules/git/log_name_status.go index e1e117ff4..80f160270 100644 --- a/modules/git/log_name_status.go +++ b/modules/git/log_name_status.go @@ -281,7 +281,7 @@ func (g *LogNameStatusRepoParser) Close() { } // WalkGitLog walks the git log --name-status for the head commit in the provided treepath and files -func WalkGitLog(ctx context.Context, cache *LastCommitCache, repo *Repository, head *Commit, treepath string, paths ...string) (map[string]string, error) { +func WalkGitLog(ctx context.Context, repo *Repository, head *Commit, treepath string, paths ...string) (map[string]string, error) { headRef := head.ID.String() tree, err := head.SubTree(treepath) @@ -374,14 +374,14 @@ heaploop: changed[i] = false if results[i] == "" { results[i] = current.CommitID - if err := cache.Put(headRef, path.Join(treepath, paths[i]), current.CommitID); err != nil { + if err := repo.LastCommitCache.Put(headRef, path.Join(treepath, paths[i]), current.CommitID); err != nil { return nil, err } delete(path2idx, paths[i]) remaining-- if results[0] == "" { results[0] = current.CommitID - if err := cache.Put(headRef, treepath, current.CommitID); err != nil { + if err := repo.LastCommitCache.Put(headRef, treepath, current.CommitID); err != nil { return nil, err } delete(path2idx, "") diff --git a/modules/git/notes_gogit.go b/modules/git/notes_gogit.go index 76bc82895..fe6d1f1e5 100644 --- a/modules/git/notes_gogit.go +++ b/modules/git/notes_gogit.go @@ -83,7 +83,7 @@ func GetNote(ctx context.Context, repo *Repository, commitID string, note *Note) log.Error("Unable to get the commit for the path %q. Error: %v", path, err) return err } - note.Commit = convertCommit(lastCommits[path]) + note.Commit = lastCommits[path] return nil } diff --git a/modules/git/notes_nogogit.go b/modules/git/notes_nogogit.go index 1476805dc..ba216ce3e 100644 --- a/modules/git/notes_nogogit.go +++ b/modules/git/notes_nogogit.go @@ -81,7 +81,7 @@ func GetNote(ctx context.Context, repo *Repository, commitID string, note *Note) path = path[idx+1:] } - lastCommits, err := GetLastCommitForPaths(ctx, nil, notes, treePath, []string{path}) + lastCommits, err := GetLastCommitForPaths(ctx, notes, treePath, []string{path}) if err != nil { log.Error("Unable to get the commit for the path %q. Error: %v", treePath, err) return err diff --git a/modules/git/repo_base_gogit.go b/modules/git/repo_base_gogit.go index cd2ca25df..8fe9c404c 100644 --- a/modules/git/repo_base_gogit.go +++ b/modules/git/repo_base_gogit.go @@ -31,7 +31,8 @@ type Repository struct { gogitStorage *filesystem.Storage gpgSettings *GPGSettings - Ctx context.Context + Ctx context.Context + LastCommitCache *LastCommitCache } // openRepositoryWithDefaultContext opens the repository at the given path with DefaultContext. @@ -79,6 +80,8 @@ func (repo *Repository) Close() (err error) { if err := repo.gogitStorage.Close(); err != nil { gitealog.Error("Error closing storage: %v", err) } + repo.LastCommitCache = nil + repo.tagCache = nil return } diff --git a/modules/git/repo_base_nogogit.go b/modules/git/repo_base_nogogit.go index 63c278c26..56af2c640 100644 --- a/modules/git/repo_base_nogogit.go +++ b/modules/git/repo_base_nogogit.go @@ -32,7 +32,8 @@ type Repository struct { checkReader *bufio.Reader checkWriter WriteCloserError - Ctx context.Context + Ctx context.Context + LastCommitCache *LastCommitCache } // openRepositoryWithDefaultContext opens the repository at the given path with DefaultContext. @@ -101,5 +102,7 @@ func (repo *Repository) Close() (err error) { repo.checkReader = nil repo.checkWriter = nil } + repo.LastCommitCache = nil + repo.tagCache = nil return err } diff --git a/routers/api/v1/repo/file.go b/routers/api/v1/repo/file.go index ba8a938b8..8353a4e50 100644 --- a/routers/api/v1/repo/file.go +++ b/routers/api/v1/repo/file.go @@ -18,7 +18,6 @@ import ( git_model "code.gitea.io/gitea/models/git" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unit" - "code.gitea.io/gitea/modules/cache" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/httpcache" @@ -240,12 +239,7 @@ func getBlobForEntry(ctx *context.APIContext) (blob *git.Blob, entry *git.TreeEn return } - var c *git.LastCommitCache - if setting.CacheService.LastCommit.Enabled && ctx.Repo.CommitsCount >= setting.CacheService.LastCommit.CommitsCount { - c = git.NewLastCommitCache(ctx.Repo.Repository.FullName(), ctx.Repo.GitRepo, setting.LastCommitCacheTTLSeconds, cache.GetCache()) - } - - info, _, err := git.Entries([]*git.TreeEntry{entry}).GetCommitsInfo(ctx, ctx.Repo.Commit, path.Dir("/" + ctx.Repo.TreePath)[1:], c) + info, _, err := git.Entries([]*git.TreeEntry{entry}).GetCommitsInfo(ctx, ctx.Repo.Commit, path.Dir("/" + ctx.Repo.TreePath)[1:]) if err != nil { ctx.Error(http.StatusInternalServerError, "GetCommitsInfo", err) return diff --git a/routers/api/v1/utils/git.go b/routers/api/v1/utils/git.go index ac64d5b87..f18442d04 100644 --- a/routers/api/v1/utils/git.go +++ b/routers/api/v1/utils/git.go @@ -8,8 +8,10 @@ import ( "fmt" "net/http" + "code.gitea.io/gitea/modules/cache" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" ) // ResolveRefOrSha resolve ref to sha if exist @@ -19,6 +21,7 @@ func ResolveRefOrSha(ctx *context.APIContext, ref string) string { return "" } + sha := ref // Search branches and tags for _, refType := range []string{"heads", "tags"} { refSHA, lastMethodName, err := searchRefCommitByType(ctx, refType, ref) @@ -27,10 +30,27 @@ func ResolveRefOrSha(ctx *context.APIContext, ref string) string { return "" } if refSHA != "" { - return refSHA + sha = refSHA + break } } - return ref + + if ctx.Repo.GitRepo != nil && ctx.Repo.GitRepo.LastCommitCache == nil { + commitsCount, err := cache.GetInt64(ctx.Repo.Repository.GetCommitsCountCacheKey(ref, true), func() (int64, error) { + commit, err := ctx.Repo.GitRepo.GetCommit(sha) + if err != nil { + return 0, err + } + return commit.CommitsCount() + }) + if err != nil { + log.Error("Unable to get commits count for %s in %s. Error: %v", sha, ctx.Repo.Repository.FullName(), err) + return sha + } + ctx.Repo.GitRepo.LastCommitCache = git.NewLastCommitCache(commitsCount, ctx.Repo.Repository.FullName(), ctx.Repo.GitRepo, cache.GetCache()) + } + + return sha } // GetGitRefs return git references based on filter diff --git a/routers/web/repo/download.go b/routers/web/repo/download.go index 6755cda87..cd2d305de 100644 --- a/routers/web/repo/download.go +++ b/routers/web/repo/download.go @@ -10,7 +10,6 @@ import ( "time" git_model "code.gitea.io/gitea/models/git" - "code.gitea.io/gitea/modules/cache" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/httpcache" @@ -99,12 +98,7 @@ func getBlobForEntry(ctx *context.Context) (blob *git.Blob, lastModified time.Ti return } - var c *git.LastCommitCache - if setting.CacheService.LastCommit.Enabled && ctx.Repo.CommitsCount >= setting.CacheService.LastCommit.CommitsCount { - c = git.NewLastCommitCache(ctx.Repo.Repository.FullName(), ctx.Repo.GitRepo, setting.LastCommitCacheTTLSeconds, cache.GetCache()) - } - - info, _, err := git.Entries([]*git.TreeEntry{entry}).GetCommitsInfo(ctx, ctx.Repo.Commit, path.Dir("/" + ctx.Repo.TreePath)[1:], c) + info, _, err := git.Entries([]*git.TreeEntry{entry}).GetCommitsInfo(ctx, ctx.Repo.Commit, path.Dir("/" + ctx.Repo.TreePath)[1:]) if err != nil { ctx.ServerError("GetCommitsInfo", err) return diff --git a/routers/web/repo/view.go b/routers/web/repo/view.go index 6b6660f77..a396be8ae 100644 --- a/routers/web/repo/view.go +++ b/routers/web/repo/view.go @@ -27,7 +27,6 @@ import ( unit_model "code.gitea.io/gitea/models/unit" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/base" - "code.gitea.io/gitea/modules/cache" "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" @@ -812,11 +811,6 @@ func renderDirectoryFiles(ctx *context.Context, timeout time.Duration) git.Entri defer cancel() } - var c *git.LastCommitCache - if setting.CacheService.LastCommit.Enabled && ctx.Repo.CommitsCount >= setting.CacheService.LastCommit.CommitsCount { - c = git.NewLastCommitCache(ctx.Repo.Repository.FullName(), ctx.Repo.GitRepo, setting.LastCommitCacheTTLSeconds, cache.GetCache()) - } - selected := map[string]bool{} for _, pth := range ctx.FormStrings("f[]") { selected[pth] = true @@ -833,7 +827,7 @@ func renderDirectoryFiles(ctx *context.Context, timeout time.Duration) git.Entri } var latestCommit *git.Commit - ctx.Data["Files"], latestCommit, err = entries.GetCommitsInfo(commitInfoCtx, ctx.Repo.Commit, ctx.Repo.TreePath, c) + ctx.Data["Files"], latestCommit, err = entries.GetCommitsInfo(commitInfoCtx, ctx.Repo.Commit, ctx.Repo.TreePath) if err != nil { ctx.ServerError("GetCommitsInfo", err) return nil diff --git a/routers/web/web.go b/routers/web/web.go index fbece620b..b4e8666c4 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -12,6 +12,7 @@ import ( "code.gitea.io/gitea/models/perm" "code.gitea.io/gitea/models/unit" + "code.gitea.io/gitea/modules/cache" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/httpcache" @@ -1011,6 +1012,7 @@ func RegisterRoutes(m *web.Route) { return } ctx.Data["CommitsCount"] = ctx.Repo.CommitsCount + ctx.Repo.GitRepo.LastCommitCache = git.NewLastCommitCache(ctx.Repo.CommitsCount, ctx.Repo.Repository.FullName(), ctx.Repo.GitRepo, cache.GetCache()) }) }, ignSignIn, context.RepoAssignment, context.UnitTypes(), reqRepoReleaseReader) diff --git a/services/repository/cache.go b/services/repository/cache.go index 5b0c929be..855fe7f4a 100644 --- a/services/repository/cache.go +++ b/services/repository/cache.go @@ -34,15 +34,13 @@ func CacheRef(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Rep return err } - commitsCount, err := cache.GetInt64(repo.GetCommitsCountCacheKey(getRefName(fullRefName), true), commit.CommitsCount) - if err != nil { - return err - } - if commitsCount < setting.CacheService.LastCommit.CommitsCount { - return nil + if gitRepo.LastCommitCache == nil { + commitsCount, err := cache.GetInt64(repo.GetCommitsCountCacheKey(getRefName(fullRefName), true), commit.CommitsCount) + if err != nil { + return err + } + gitRepo.LastCommitCache = git.NewLastCommitCache(commitsCount, repo.FullName(), gitRepo, cache.GetCache()) } - commitCache := git.NewLastCommitCache(repo.FullName(), gitRepo, setting.LastCommitCacheTTLSeconds, cache.GetCache()) - - return commitCache.CacheCommit(ctx, commit) + return commit.CacheCommit(ctx) }