From be82b8f99af753c571e88f2f5310b1f656d34d84 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 22 Oct 2020 23:45:54 +0800 Subject: [PATCH] Keep database transactions not too big (#13254) * Keep database transactions not too big * Fix #13255 Signed-off-by: Andrew Thornton * Only cache the last repo Signed-off-by: Andrew Thornton Co-authored-by: Andrew Thornton --- models/migrations/v156.go | 81 +++++++++++++++++++-------------------- 1 file changed, 40 insertions(+), 41 deletions(-) diff --git a/models/migrations/v156.go b/models/migrations/v156.go index dc9c20188..6092a37d5 100644 --- a/models/migrations/v156.go +++ b/models/migrations/v156.go @@ -10,6 +10,7 @@ import ( "strings" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "xorm.io/xorm" @@ -35,9 +36,10 @@ func fixPublisherIDforTagReleases(x *xorm.Engine) error { } type Repository struct { - ID int64 - OwnerID int64 - Name string + ID int64 + OwnerID int64 + OwnerName string + Name string } type User struct { @@ -50,27 +52,23 @@ func fixPublisherIDforTagReleases(x *xorm.Engine) error { sess := x.NewSession() defer sess.Close() - if err := sess.Begin(); err != nil { - return err - } - var ( - gitRepoCache = make(map[int64]*git.Repository) - gitRepo *git.Repository - repoCache = make(map[int64]*Repository) - userCache = make(map[int64]*User) - ok bool - err error + repo *Repository + gitRepo *git.Repository ) defer func() { - for i := range gitRepoCache { - gitRepoCache[i].Close() + if gitRepo != nil { + gitRepo.Close() } }() for start := 0; ; start += batchSize { releases := make([]*Release, 0, batchSize) - if err := sess.Limit(batchSize, start).Asc("id").Where("is_tag=?", true).Find(&releases); err != nil { + if err := sess.Begin(); err != nil { + return err + } + + if err := sess.Limit(batchSize, start).Asc("repo_id", "id").Where("is_tag=?", true).Find(&releases); err != nil { return err } @@ -79,39 +77,36 @@ func fixPublisherIDforTagReleases(x *xorm.Engine) error { } for _, release := range releases { - gitRepo, ok = gitRepoCache[release.RepoID] - if !ok { - repo, ok := repoCache[release.RepoID] - if !ok { - repo = new(Repository) - has, err := sess.ID(release.RepoID).Get(repo) - if err != nil { - return err - } else if !has { - return fmt.Errorf("Repository %d is not exist", release.RepoID) - } - - repoCache[release.RepoID] = repo + if repo == nil || repo.ID != release.RepoID { + if gitRepo != nil { + gitRepo.Close() + gitRepo = nil + } + repo = new(Repository) + has, err := sess.ID(release.RepoID).Get(repo) + if err != nil { + return err + } else if !has { + log.Warn("Release[%d] is orphaned and refers to non-existing repository %d", release.ID, release.RepoID) + log.Warn("This release should be deleted") + continue } - user, ok := userCache[repo.OwnerID] - if !ok { - user = new(User) - has, err := sess.ID(repo.OwnerID).Get(user) - if err != nil { + if repo.OwnerName == "" { + // v120.go migration may not have been run correctly - we'll just replicate it here + // because this appears to be a common-ish problem. + if _, err := sess.Exec("UPDATE repository SET owner_name = (SELECT name FROM `user` WHERE `user`.id = repository.owner_id)"); err != nil { return err - } else if !has { - return fmt.Errorf("User %d is not exist", repo.OwnerID) } - userCache[repo.OwnerID] = user + if _, err := sess.ID(release.RepoID).Get(repo); err != nil { + return err + } } - - gitRepo, err = git.OpenRepository(repoPath(user.Name, repo.Name)) + gitRepo, err = git.OpenRepository(repoPath(repo.OwnerName, repo.Name)) if err != nil { return err } - gitRepoCache[release.RepoID] = gitRepo } commit, err := gitRepo.GetTagCommit(release.TagName) @@ -134,7 +129,11 @@ func fixPublisherIDforTagReleases(x *xorm.Engine) error { return err } } + + if err := sess.Commit(); err != nil { + return err + } } - return sess.Commit() + return nil }