bench-forgejo/routers/repo/lfs.go
KN4CK3R c03e488e14
Add LFS Migration and Mirror (#14726)
* Implemented LFS client.

* Implemented scanning for pointer files.

* Implemented downloading of lfs files.

* Moved model-dependent code into services.

* Removed models dependency. Added TryReadPointerFromBuffer.

* Migrated code from service to module.

* Centralised storage creation.

* Removed dependency from models.

* Moved ContentStore into modules.

* Share structs between server and client.

* Moved method to services.

* Implemented lfs download on clone.

* Implemented LFS sync on clone and mirror update.

* Added form fields.

* Updated templates.

* Fixed condition.

* Use alternate endpoint.

* Added missing methods.

* Fixed typo and make linter happy.

* Detached pointer parser from gogit dependency.

* Fixed TestGetLFSRange test.

* Added context to support cancellation.

* Use ReadFull to probably read more data.

* Removed duplicated code from models.

* Moved scan implementation into pointer_scanner_nogogit.

* Changed method name.

* Added comments.

* Added more/specific log/error messages.

* Embedded lfs.Pointer into models.LFSMetaObject.

* Moved code from models to module.

* Moved code from models to module.

* Moved code from models to module.

* Reduced pointer usage.

* Embedded type.

* Use promoted fields.

* Fixed unexpected eof.

* Added unit tests.

* Implemented migration of local file paths.

* Show an error on invalid LFS endpoints.

* Hide settings if not used.

* Added LFS info to mirror struct.

* Fixed comment.

* Check LFS endpoint.

* Manage LFS settings from mirror page.

* Fixed selector.

* Adjusted selector.

* Added more tests.

* Added local filesystem migration test.

* Fixed typo.

* Reset settings.

* Added special windows path handling.

* Added unit test for HTTPClient.

* Added unit test for BasicTransferAdapter.

* Moved into util package.

* Test if LFS endpoint is allowed.

* Added support for git://

* Just use a static placeholder as the displayed url may be invalid.

* Reverted to original code.

* Added "Advanced Settings".

* Updated wording.

* Added discovery info link.

* Implemented suggestion.

* Fixed missing format parameter.

* Added Pointer.IsValid().

* Always remove model on error.

* Added suggestions.

* Use channel instead of array.

* Update routers/repo/migrate.go

* fmt

Signed-off-by: Andrew Thornton <art27@cantab.net>

Co-authored-by: zeripath <art27@cantab.net>
2021-04-08 18:25:57 -04:00

543 lines
15 KiB
Go

// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package repo
import (
"bytes"
"fmt"
gotemplate "html/template"
"io"
"io/ioutil"
"net/http"
"path"
"strconv"
"strings"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/charset"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/git/pipeline"
"code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
)
const (
tplSettingsLFS base.TplName = "repo/settings/lfs"
tplSettingsLFSLocks base.TplName = "repo/settings/lfs_locks"
tplSettingsLFSFile base.TplName = "repo/settings/lfs_file"
tplSettingsLFSFileFind base.TplName = "repo/settings/lfs_file_find"
tplSettingsLFSPointers base.TplName = "repo/settings/lfs_pointers"
)
// LFSFiles shows a repository's LFS files
func LFSFiles(ctx *context.Context) {
if !setting.LFS.StartServer {
ctx.NotFound("LFSFiles", nil)
return
}
page := ctx.QueryInt("page")
if page <= 1 {
page = 1
}
total, err := ctx.Repo.Repository.CountLFSMetaObjects()
if err != nil {
ctx.ServerError("LFSFiles", err)
return
}
ctx.Data["Total"] = total
pager := context.NewPagination(int(total), setting.UI.ExplorePagingNum, page, 5)
ctx.Data["Title"] = ctx.Tr("repo.settings.lfs")
ctx.Data["PageIsSettingsLFS"] = true
lfsMetaObjects, err := ctx.Repo.Repository.GetLFSMetaObjects(pager.Paginater.Current(), setting.UI.ExplorePagingNum)
if err != nil {
ctx.ServerError("LFSFiles", err)
return
}
ctx.Data["LFSFiles"] = lfsMetaObjects
ctx.Data["Page"] = pager
ctx.HTML(http.StatusOK, tplSettingsLFS)
}
// LFSLocks shows a repository's LFS locks
func LFSLocks(ctx *context.Context) {
if !setting.LFS.StartServer {
ctx.NotFound("LFSLocks", nil)
return
}
ctx.Data["LFSFilesLink"] = ctx.Repo.RepoLink + "/settings/lfs"
page := ctx.QueryInt("page")
if page <= 1 {
page = 1
}
total, err := models.CountLFSLockByRepoID(ctx.Repo.Repository.ID)
if err != nil {
ctx.ServerError("LFSLocks", err)
return
}
ctx.Data["Total"] = total
pager := context.NewPagination(int(total), setting.UI.ExplorePagingNum, page, 5)
ctx.Data["Title"] = ctx.Tr("repo.settings.lfs_locks")
ctx.Data["PageIsSettingsLFS"] = true
lfsLocks, err := models.GetLFSLockByRepoID(ctx.Repo.Repository.ID, pager.Paginater.Current(), setting.UI.ExplorePagingNum)
if err != nil {
ctx.ServerError("LFSLocks", err)
return
}
ctx.Data["LFSLocks"] = lfsLocks
if len(lfsLocks) == 0 {
ctx.Data["Page"] = pager
ctx.HTML(http.StatusOK, tplSettingsLFSLocks)
return
}
// Clone base repo.
tmpBasePath, err := models.CreateTemporaryPath("locks")
if err != nil {
log.Error("Failed to create temporary path: %v", err)
ctx.ServerError("LFSLocks", err)
return
}
defer func() {
if err := models.RemoveTemporaryPath(tmpBasePath); err != nil {
log.Error("LFSLocks: RemoveTemporaryPath: %v", err)
}
}()
if err := git.Clone(ctx.Repo.Repository.RepoPath(), tmpBasePath, git.CloneRepoOptions{
Bare: true,
Shared: true,
}); err != nil {
log.Error("Failed to clone repository: %s (%v)", ctx.Repo.Repository.FullName(), err)
ctx.ServerError("LFSLocks", fmt.Errorf("Failed to clone repository: %s (%v)", ctx.Repo.Repository.FullName(), err))
return
}
gitRepo, err := git.OpenRepository(tmpBasePath)
if err != nil {
log.Error("Unable to open temporary repository: %s (%v)", tmpBasePath, err)
ctx.ServerError("LFSLocks", fmt.Errorf("Failed to open new temporary repository in: %s %v", tmpBasePath, err))
return
}
defer gitRepo.Close()
filenames := make([]string, len(lfsLocks))
for i, lock := range lfsLocks {
filenames[i] = lock.Path
}
if err := gitRepo.ReadTreeToIndex(ctx.Repo.Repository.DefaultBranch); err != nil {
log.Error("Unable to read the default branch to the index: %s (%v)", ctx.Repo.Repository.DefaultBranch, err)
ctx.ServerError("LFSLocks", fmt.Errorf("Unable to read the default branch to the index: %s (%v)", ctx.Repo.Repository.DefaultBranch, err))
return
}
name2attribute2info, err := gitRepo.CheckAttribute(git.CheckAttributeOpts{
Attributes: []string{"lockable"},
Filenames: filenames,
CachedOnly: true,
})
if err != nil {
log.Error("Unable to check attributes in %s (%v)", tmpBasePath, err)
ctx.ServerError("LFSLocks", err)
return
}
lockables := make([]bool, len(lfsLocks))
for i, lock := range lfsLocks {
attribute2info, has := name2attribute2info[lock.Path]
if !has {
continue
}
if attribute2info["lockable"] != "set" {
continue
}
lockables[i] = true
}
ctx.Data["Lockables"] = lockables
filelist, err := gitRepo.LsFiles(filenames...)
if err != nil {
log.Error("Unable to lsfiles in %s (%v)", tmpBasePath, err)
ctx.ServerError("LFSLocks", err)
return
}
filemap := make(map[string]bool, len(filelist))
for _, name := range filelist {
filemap[name] = true
}
linkable := make([]bool, len(lfsLocks))
for i, lock := range lfsLocks {
linkable[i] = filemap[lock.Path]
}
ctx.Data["Linkable"] = linkable
ctx.Data["Page"] = pager
ctx.HTML(http.StatusOK, tplSettingsLFSLocks)
}
// LFSLockFile locks a file
func LFSLockFile(ctx *context.Context) {
if !setting.LFS.StartServer {
ctx.NotFound("LFSLocks", nil)
return
}
originalPath := ctx.Query("path")
lockPath := originalPath
if len(lockPath) == 0 {
ctx.Flash.Error(ctx.Tr("repo.settings.lfs_invalid_locking_path", originalPath))
ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs/locks")
return
}
if lockPath[len(lockPath)-1] == '/' {
ctx.Flash.Error(ctx.Tr("repo.settings.lfs_invalid_lock_directory", originalPath))
ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs/locks")
return
}
lockPath = path.Clean("/" + lockPath)[1:]
if len(lockPath) == 0 {
ctx.Flash.Error(ctx.Tr("repo.settings.lfs_invalid_locking_path", originalPath))
ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs/locks")
return
}
_, err := models.CreateLFSLock(&models.LFSLock{
Repo: ctx.Repo.Repository,
Path: lockPath,
Owner: ctx.User,
})
if err != nil {
if models.IsErrLFSLockAlreadyExist(err) {
ctx.Flash.Error(ctx.Tr("repo.settings.lfs_lock_already_exists", originalPath))
ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs/locks")
return
}
ctx.ServerError("LFSLockFile", err)
return
}
ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs/locks")
}
// LFSUnlock forcibly unlocks an LFS lock
func LFSUnlock(ctx *context.Context) {
if !setting.LFS.StartServer {
ctx.NotFound("LFSUnlock", nil)
return
}
_, err := models.DeleteLFSLockByID(ctx.ParamsInt64("lid"), ctx.User, true)
if err != nil {
ctx.ServerError("LFSUnlock", err)
return
}
ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs/locks")
}
// LFSFileGet serves a single LFS file
func LFSFileGet(ctx *context.Context) {
if !setting.LFS.StartServer {
ctx.NotFound("LFSFileGet", nil)
return
}
ctx.Data["LFSFilesLink"] = ctx.Repo.RepoLink + "/settings/lfs"
oid := ctx.Params("oid")
ctx.Data["Title"] = oid
ctx.Data["PageIsSettingsLFS"] = true
meta, err := ctx.Repo.Repository.GetLFSMetaObjectByOid(oid)
if err != nil {
if err == models.ErrLFSObjectNotExist {
ctx.NotFound("LFSFileGet", nil)
return
}
ctx.ServerError("LFSFileGet", err)
return
}
ctx.Data["LFSFile"] = meta
dataRc, err := lfs.ReadMetaObject(meta.Pointer)
if err != nil {
ctx.ServerError("LFSFileGet", err)
return
}
defer dataRc.Close()
buf := make([]byte, 1024)
n, err := dataRc.Read(buf)
if err != nil {
ctx.ServerError("Data", err)
return
}
buf = buf[:n]
ctx.Data["IsTextFile"] = base.IsTextFile(buf)
isRepresentableAsText := base.IsRepresentableAsText(buf)
fileSize := meta.Size
ctx.Data["FileSize"] = meta.Size
ctx.Data["RawFileLink"] = fmt.Sprintf("%s%s.git/info/lfs/objects/%s/%s", setting.AppURL, ctx.Repo.Repository.FullName(), meta.Oid, "direct")
switch {
case isRepresentableAsText:
// This will be true for SVGs.
if base.IsImageFile(buf) {
ctx.Data["IsImageFile"] = true
}
if fileSize >= setting.UI.MaxDisplayFileSize {
ctx.Data["IsFileTooLarge"] = true
break
}
d, _ := ioutil.ReadAll(dataRc)
buf = charset.ToUTF8WithFallback(append(buf, d...))
// Building code view blocks with line number on server side.
var fileContent string
if content, err := charset.ToUTF8WithErr(buf); err != nil {
log.Error("ToUTF8WithErr: %v", err)
fileContent = string(buf)
} else {
fileContent = content
}
var output bytes.Buffer
lines := strings.Split(fileContent, "\n")
//Remove blank line at the end of file
if len(lines) > 0 && lines[len(lines)-1] == "" {
lines = lines[:len(lines)-1]
}
for index, line := range lines {
line = gotemplate.HTMLEscapeString(line)
if index != len(lines)-1 {
line += "\n"
}
output.WriteString(fmt.Sprintf(`<li class="L%d" rel="L%d">%s</li>`, index+1, index+1, line))
}
ctx.Data["FileContent"] = gotemplate.HTML(output.String())
output.Reset()
for i := 0; i < len(lines); i++ {
output.WriteString(fmt.Sprintf(`<span id="L%d">%d</span>`, i+1, i+1))
}
ctx.Data["LineNums"] = gotemplate.HTML(output.String())
case base.IsPDFFile(buf):
ctx.Data["IsPDFFile"] = true
case base.IsVideoFile(buf):
ctx.Data["IsVideoFile"] = true
case base.IsAudioFile(buf):
ctx.Data["IsAudioFile"] = true
case base.IsImageFile(buf):
ctx.Data["IsImageFile"] = true
}
ctx.HTML(http.StatusOK, tplSettingsLFSFile)
}
// LFSDelete disassociates the provided oid from the repository and if the lfs file is no longer associated with any repositories - deletes it
func LFSDelete(ctx *context.Context) {
if !setting.LFS.StartServer {
ctx.NotFound("LFSDelete", nil)
return
}
oid := ctx.Params("oid")
count, err := ctx.Repo.Repository.RemoveLFSMetaObjectByOid(oid)
if err != nil {
ctx.ServerError("LFSDelete", err)
return
}
// FIXME: Warning: the LFS store is not locked - and can't be locked - there could be a race condition here
// Please note a similar condition happens in models/repo.go DeleteRepository
if count == 0 {
oidPath := path.Join(oid[0:2], oid[2:4], oid[4:])
err = storage.LFS.Delete(oidPath)
if err != nil {
ctx.ServerError("LFSDelete", err)
return
}
}
ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs")
}
// LFSFileFind guesses a sha for the provided oid (or uses the provided sha) and then finds the commits that contain this sha
func LFSFileFind(ctx *context.Context) {
if !setting.LFS.StartServer {
ctx.NotFound("LFSFind", nil)
return
}
oid := ctx.Query("oid")
size := ctx.QueryInt64("size")
if len(oid) == 0 || size == 0 {
ctx.NotFound("LFSFind", nil)
return
}
sha := ctx.Query("sha")
ctx.Data["Title"] = oid
ctx.Data["PageIsSettingsLFS"] = true
var hash git.SHA1
if len(sha) == 0 {
pointer := lfs.Pointer{Oid: oid, Size: size}
hash = git.ComputeBlobHash([]byte(pointer.StringContent()))
sha = hash.String()
} else {
hash = git.MustIDFromString(sha)
}
ctx.Data["LFSFilesLink"] = ctx.Repo.RepoLink + "/settings/lfs"
ctx.Data["Oid"] = oid
ctx.Data["Size"] = size
ctx.Data["SHA"] = sha
results, err := pipeline.FindLFSFile(ctx.Repo.GitRepo, hash)
if err != nil && err != io.EOF {
log.Error("Failure in FindLFSFile: %v", err)
ctx.ServerError("LFSFind: FindLFSFile.", err)
return
}
ctx.Data["Results"] = results
ctx.HTML(http.StatusOK, tplSettingsLFSFileFind)
}
// LFSPointerFiles will search the repository for pointer files and report which are missing LFS files in the content store
func LFSPointerFiles(ctx *context.Context) {
if !setting.LFS.StartServer {
ctx.NotFound("LFSFileGet", nil)
return
}
ctx.Data["PageIsSettingsLFS"] = true
err := git.LoadGitVersion()
if err != nil {
log.Fatal("Error retrieving git version: %v", err)
}
ctx.Data["LFSFilesLink"] = ctx.Repo.RepoLink + "/settings/lfs"
err = func() error {
pointerChan := make(chan lfs.PointerBlob)
errChan := make(chan error, 1)
go lfs.SearchPointerBlobs(ctx.Req.Context(), ctx.Repo.GitRepo, pointerChan, errChan)
numPointers := 0
var numAssociated, numNoExist, numAssociatable int
type pointerResult struct {
SHA string
Oid string
Size int64
InRepo bool
Exists bool
Accessible bool
}
results := []pointerResult{}
contentStore := lfs.NewContentStore()
repo := ctx.Repo.Repository
for pointerBlob := range pointerChan {
numPointers++
result := pointerResult{
SHA: pointerBlob.Hash,
Oid: pointerBlob.Oid,
Size: pointerBlob.Size,
}
if _, err := repo.GetLFSMetaObjectByOid(pointerBlob.Oid); err != nil {
if err != models.ErrLFSObjectNotExist {
return err
}
} else {
result.InRepo = true
}
result.Exists, err = contentStore.Exists(pointerBlob.Pointer)
if err != nil {
return err
}
if result.Exists {
if !result.InRepo {
// Can we fix?
// OK well that's "simple"
// - we need to check whether current user has access to a repo that has access to the file
result.Accessible, err = models.LFSObjectAccessible(ctx.User, pointerBlob.Oid)
if err != nil {
return err
}
} else {
result.Accessible = true
}
}
if result.InRepo {
numAssociated++
}
if !result.Exists {
numNoExist++
}
if !result.InRepo && result.Accessible {
numAssociatable++
}
results = append(results, result)
}
err, has := <-errChan
if has {
return err
}
ctx.Data["Pointers"] = results
ctx.Data["NumPointers"] = numPointers
ctx.Data["NumAssociated"] = numAssociated
ctx.Data["NumAssociatable"] = numAssociatable
ctx.Data["NumNoExist"] = numNoExist
ctx.Data["NumNotAssociated"] = numPointers - numAssociated
return nil
}()
if err != nil {
ctx.ServerError("LFSPointerFiles", err)
return
}
ctx.HTML(http.StatusOK, tplSettingsLFSPointers)
}
// LFSAutoAssociate auto associates accessible lfs files
func LFSAutoAssociate(ctx *context.Context) {
if !setting.LFS.StartServer {
ctx.NotFound("LFSAutoAssociate", nil)
return
}
oids := ctx.QueryStrings("oid")
metas := make([]*models.LFSMetaObject, len(oids))
for i, oid := range oids {
idx := strings.IndexRune(oid, ' ')
if idx < 0 || idx+1 > len(oid) {
ctx.ServerError("LFSAutoAssociate", fmt.Errorf("Illegal oid input: %s", oid))
return
}
var err error
metas[i] = &models.LFSMetaObject{}
metas[i].Size, err = strconv.ParseInt(oid[idx+1:], 10, 64)
if err != nil {
ctx.ServerError("LFSAutoAssociate", fmt.Errorf("Illegal oid input: %s %v", oid, err))
return
}
metas[i].Oid = oid[:idx]
//metas[i].RepositoryID = ctx.Repo.Repository.ID
}
if err := models.LFSAutoAssociate(metas, ctx.User, ctx.Repo.Repository.ID); err != nil {
ctx.ServerError("LFSAutoAssociate", err)
return
}
ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs")
}