Refactor editor upload, update and delete to use git plumbing and add LFS support (#5702)

* Use git plumbing for upload: #5621 repo_editor.go: UploadRepoFile

* Use git plumbing for upload: #5621 repo_editor.go: GetDiffPreview

* Use git plumbing for upload: #5621 repo_editor.go: DeleteRepoFile

* Use git plumbing for upload: #5621 repo_editor.go: UploadRepoFiles

* Move branch checkout functions out of repo_editor.go as they are no longer used there

* BUGFIX: The default permissions should be 100644

    This is a change from the previous code but is more in keeping
    with the default behaviour of git.

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Standardise cleanUploadFilename to more closely match git

See verify_path in: 7f4e641693/read-cache.c (L951)

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Redirect on bad paths

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Refactor to move the uploading functions out to a module

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Add LFS support

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Update upload.go attribution header

Upload.go is essentially the remnants of repo_editor.go. The remaining code is essentially unchanged from the Gogs code, hence the Gogs attribution.

* Delete upload files after session committed

* Ensure that GIT_AUTHOR_NAME etc. are valid for git

see #5774

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Add in test cases per @lafriks comment

* Add space between gitea and github imports

Signed-off-by: Andrew Thornton <art27@cantab.net>

* more examples in TestCleanUploadName

Signed-off-by: Andrew Thornton <art27@cantab.net>

* fix formatting

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Set the SSH_ORIGINAL_COMMAND to ensure hooks are run

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Switch off SSH_ORIGINAL_COMMAND

Signed-off-by: Andrew Thornton <art27@cantab.net>

models/lfs.go

@@ -1,7 +1,11 @@
 package models
 
 import (
+	"crypto/sha256"
+	"encoding/hex"
 	"errors"
+	"fmt"
+	"io"
 
 	"code.gitea.io/gitea/modules/util"
 )
@@ -16,6 +20,11 @@ type LFSMetaObject struct {
 	CreatedUnix  util.TimeStamp `xorm:"created"`
 }
 
+// Pointer returns the string representation of an LFS pointer file
+func (m *LFSMetaObject) Pointer() string {
+	return fmt.Sprintf("%s\n%s%s\nsize %d\n", LFSMetaFileIdentifier, LFSMetaFileOidPrefix, m.Oid, m.Size)
+}
+
 // LFSTokenResponse defines the JSON structure in which the JWT token is stored.
 // This structure is fetched via SSH and passed by the Git LFS client to the server
 // endpoint for authorization.
@@ -67,6 +76,16 @@ func NewLFSMetaObject(m *LFSMetaObject) (*LFSMetaObject, error) {
 	return m, sess.Commit()
 }
 
+// GenerateLFSOid generates a Sha256Sum to represent an oid for arbitrary content
+func GenerateLFSOid(content io.Reader) (string, error) {
+	h := sha256.New()
+	if _, err := io.Copy(h, content); err != nil {
+		return "", err
+	}
+	sum := h.Sum(nil)
+	return hex.EncodeToString(sum), nil
+}
+
 // GetLFSMetaObjectByOid selects a LFSMetaObject entry from database by its OID.
 // It may return ErrLFSObjectNotExist or a database error. If the error is nil,
 // the returned pointer is a valid LFSMetaObject.

models/repo_branch.go

@@ -14,6 +14,46 @@ import (
 	"github.com/Unknwon/com"
 )
 
+// discardLocalRepoBranchChanges discards local commits/changes of
+// given branch to make sure it is even to remote branch.
+func discardLocalRepoBranchChanges(localPath, branch string) error {
+	if !com.IsExist(localPath) {
+		return nil
+	}
+	// No need to check if nothing in the repository.
+	if !git.IsBranchExist(localPath, branch) {
+		return nil
+	}
+
+	refName := "origin/" + branch
+	if err := git.ResetHEAD(localPath, true, refName); err != nil {
+		return fmt.Errorf("git reset --hard %s: %v", refName, err)
+	}
+	return nil
+}
+
+// DiscardLocalRepoBranchChanges discards the local repository branch changes
+func (repo *Repository) DiscardLocalRepoBranchChanges(branch string) error {
+	return discardLocalRepoBranchChanges(repo.LocalCopyPath(), branch)
+}
+
+// checkoutNewBranch checks out to a new branch from the a branch name.
+func checkoutNewBranch(repoPath, localPath, oldBranch, newBranch string) error {
+	if err := git.Checkout(localPath, git.CheckoutOptions{
+		Timeout:   time.Duration(setting.Git.Timeout.Pull) * time.Second,
+		Branch:    newBranch,
+		OldBranch: oldBranch,
+	}); err != nil {
+		return fmt.Errorf("git checkout -b %s %s: %v", newBranch, oldBranch, err)
+	}
+	return nil
+}
+
+// CheckoutNewBranch checks out a new branch
+func (repo *Repository) CheckoutNewBranch(oldBranch, newBranch string) error {
+	return checkoutNewBranch(repo.RepoPath(), repo.LocalCopyPath(), oldBranch, newBranch)
+}
+
 // Branch holds the branch information
 type Branch struct {
 	Path string

models/repo_editor.go

@@ -1,572 +0,0 @@
-// Copyright 2016 The Gogs Authors. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package models
-
-import (
-	"fmt"
-	"io"
-	"io/ioutil"
-	"mime/multipart"
-	"os"
-	"os/exec"
-	"path"
-	"path/filepath"
-	"time"
-
-	"github.com/Unknwon/com"
-	gouuid "github.com/satori/go.uuid"
-
-	"code.gitea.io/git"
-
-	"code.gitea.io/gitea/modules/log"
-	"code.gitea.io/gitea/modules/process"
-	"code.gitea.io/gitea/modules/setting"
-)
-
-// ___________    .___.__  __    ___________.__.__
-// \_   _____/  __| _/|__|/  |_  \_   _____/|__|  |   ____
-//  |    __)_  / __ | |  \   __\  |    __)  |  |  | _/ __ \
-//  |        \/ /_/ | |  ||  |    |     \   |  |  |_\  ___/
-// /_______  /\____ | |__||__|    \___  /   |__|____/\___  >
-//         \/      \/                 \/                 \/
-
-// discardLocalRepoBranchChanges discards local commits/changes of
-// given branch to make sure it is even to remote branch.
-func discardLocalRepoBranchChanges(localPath, branch string) error {
-	if !com.IsExist(localPath) {
-		return nil
-	}
-	// No need to check if nothing in the repository.
-	if !git.IsBranchExist(localPath, branch) {
-		return nil
-	}
-
-	refName := "origin/" + branch
-	if err := git.ResetHEAD(localPath, true, refName); err != nil {
-		return fmt.Errorf("git reset --hard %s: %v", refName, err)
-	}
-	return nil
-}
-
-// DiscardLocalRepoBranchChanges discards the local repository branch changes
-func (repo *Repository) DiscardLocalRepoBranchChanges(branch string) error {
-	return discardLocalRepoBranchChanges(repo.LocalCopyPath(), branch)
-}
-
-// checkoutNewBranch checks out to a new branch from the a branch name.
-func checkoutNewBranch(repoPath, localPath, oldBranch, newBranch string) error {
-	if err := git.Checkout(localPath, git.CheckoutOptions{
-		Timeout:   time.Duration(setting.Git.Timeout.Pull) * time.Second,
-		Branch:    newBranch,
-		OldBranch: oldBranch,
-	}); err != nil {
-		return fmt.Errorf("git checkout -b %s %s: %v", newBranch, oldBranch, err)
-	}
-	return nil
-}
-
-// CheckoutNewBranch checks out a new branch
-func (repo *Repository) CheckoutNewBranch(oldBranch, newBranch string) error {
-	return checkoutNewBranch(repo.RepoPath(), repo.LocalCopyPath(), oldBranch, newBranch)
-}
-
-// UpdateRepoFileOptions holds the repository file update options
-type UpdateRepoFileOptions struct {
-	LastCommitID string
-	OldBranch    string
-	NewBranch    string
-	OldTreeName  string
-	NewTreeName  string
-	Message      string
-	Content      string
-	IsNewFile    bool
-}
-
-// UpdateRepoFile adds or updates a file in repository.
-func (repo *Repository) UpdateRepoFile(doer *User, opts UpdateRepoFileOptions) (err error) {
-	repoWorkingPool.CheckIn(com.ToStr(repo.ID))
-	defer repoWorkingPool.CheckOut(com.ToStr(repo.ID))
-
-	if err = repo.DiscardLocalRepoBranchChanges(opts.OldBranch); err != nil {
-		return fmt.Errorf("DiscardLocalRepoBranchChanges [branch: %s]: %v", opts.OldBranch, err)
-	} else if err = repo.UpdateLocalCopyBranch(opts.OldBranch); err != nil {
-		return fmt.Errorf("UpdateLocalCopyBranch [branch: %s]: %v", opts.OldBranch, err)
-	}
-
-	if opts.OldBranch != opts.NewBranch {
-		if err := repo.CheckoutNewBranch(opts.OldBranch, opts.NewBranch); err != nil {
-			return fmt.Errorf("CheckoutNewBranch [old_branch: %s, new_branch: %s]: %v", opts.OldBranch, opts.NewBranch, err)
-		}
-	}
-
-	localPath := repo.LocalCopyPath()
-	oldFilePath := path.Join(localPath, opts.OldTreeName)
-	filePath := path.Join(localPath, opts.NewTreeName)
-	dir := path.Dir(filePath)
-
-	if err := os.MkdirAll(dir, os.ModePerm); err != nil {
-		return fmt.Errorf("Failed to create dir %s: %v", dir, err)
-	}
-
-	// If it's meant to be a new file, make sure it doesn't exist.
-	if opts.IsNewFile {
-		if com.IsExist(filePath) {
-			return ErrRepoFileAlreadyExist{filePath}
-		}
-	}
-
-	// Ignore move step if it's a new file under a directory.
-	// Otherwise, move the file when name changed.
-	if com.IsFile(oldFilePath) && opts.OldTreeName != opts.NewTreeName {
-		if err = git.MoveFile(localPath, opts.OldTreeName, opts.NewTreeName); err != nil {
-			return fmt.Errorf("git mv %s %s: %v", opts.OldTreeName, opts.NewTreeName, err)
-		}
-	}
-
-	if err = ioutil.WriteFile(filePath, []byte(opts.Content), 0666); err != nil {
-		return fmt.Errorf("WriteFile: %v", err)
-	}
-
-	if err = git.AddChanges(localPath, true); err != nil {
-		return fmt.Errorf("git add --all: %v", err)
-	} else if err = git.CommitChanges(localPath, git.CommitChangesOptions{
-		Committer: doer.NewGitSig(),
-		Message:   opts.Message,
-	}); err != nil {
-		return fmt.Errorf("CommitChanges: %v", err)
-	} else if err = git.Push(localPath, git.PushOptions{
-		Remote: "origin",
-		Branch: opts.NewBranch,
-	}); err != nil {
-		return fmt.Errorf("git push origin %s: %v", opts.NewBranch, err)
-	}
-
-	gitRepo, err := git.OpenRepository(repo.RepoPath())
-	if err != nil {
-		log.Error(4, "OpenRepository: %v", err)
-		return nil
-	}
-	commit, err := gitRepo.GetBranchCommit(opts.NewBranch)
-	if err != nil {
-		log.Error(4, "GetBranchCommit [branch: %s]: %v", opts.NewBranch, err)
-		return nil
-	}
-
-	// Simulate push event.
-	oldCommitID := opts.LastCommitID
-	if opts.NewBranch != opts.OldBranch {
-		oldCommitID = git.EmptySHA
-	}
-
-	if err = repo.GetOwner(); err != nil {
-		return fmt.Errorf("GetOwner: %v", err)
-	}
-	err = PushUpdate(
-		opts.NewBranch,
-		PushUpdateOptions{
-			PusherID:     doer.ID,
-			PusherName:   doer.Name,
-			RepoUserName: repo.Owner.Name,
-			RepoName:     repo.Name,
-			RefFullName:  git.BranchPrefix + opts.NewBranch,
-			OldCommitID:  oldCommitID,
-			NewCommitID:  commit.ID.String(),
-		},
-	)
-	if err != nil {
-		return fmt.Errorf("PushUpdate: %v", err)
-	}
-	UpdateRepoIndexer(repo)
-
-	return nil
-}
-
-// GetDiffPreview produces and returns diff result of a file which is not yet committed.
-func (repo *Repository) GetDiffPreview(branch, treePath, content string) (diff *Diff, err error) {
-	repoWorkingPool.CheckIn(com.ToStr(repo.ID))
-	defer repoWorkingPool.CheckOut(com.ToStr(repo.ID))
-
-	if err = repo.DiscardLocalRepoBranchChanges(branch); err != nil {
-		return nil, fmt.Errorf("DiscardLocalRepoBranchChanges [branch: %s]: %v", branch, err)
-	} else if err = repo.UpdateLocalCopyBranch(branch); err != nil {
-		return nil, fmt.Errorf("UpdateLocalCopyBranch [branch: %s]: %v", branch, err)
-	}
-
-	localPath := repo.LocalCopyPath()
-	filePath := path.Join(localPath, treePath)
-	dir := filepath.Dir(filePath)
-
-	if err := os.MkdirAll(dir, os.ModePerm); err != nil {
-		return nil, fmt.Errorf("Failed to create dir %s: %v", dir, err)
-	}
-
-	if err = ioutil.WriteFile(filePath, []byte(content), 0666); err != nil {
-		return nil, fmt.Errorf("WriteFile: %v", err)
-	}
-
-	cmd := exec.Command("git", "diff", treePath)
-	cmd.Dir = localPath
-	cmd.Stderr = os.Stderr
-
-	stdout, err := cmd.StdoutPipe()
-	if err != nil {
-		return nil, fmt.Errorf("StdoutPipe: %v", err)
-	}
-
-	if err = cmd.Start(); err != nil {
-		return nil, fmt.Errorf("Start: %v", err)
-	}
-
-	pid := process.GetManager().Add(fmt.Sprintf("GetDiffPreview [repo_path: %s]", repo.RepoPath()), cmd)
-	defer process.GetManager().Remove(pid)
-
-	diff, err = ParsePatch(setting.Git.MaxGitDiffLines, setting.Git.MaxGitDiffLineCharacters, setting.Git.MaxGitDiffFiles, stdout)
-	if err != nil {
-		return nil, fmt.Errorf("ParsePatch: %v", err)
-	}
-
-	if err = cmd.Wait(); err != nil {
-		return nil, fmt.Errorf("Wait: %v", err)
-	}
-
-	return diff, nil
-}
-
-// ________         .__          __           ___________.__.__
-// \______ \   ____ |  |   _____/  |_  ____   \_   _____/|__|  |   ____
-//  |    |  \_/ __ \|  | _/ __ \   __\/ __ \   |    __)  |  |  | _/ __ \
-//  |    `   \  ___/|  |_\  ___/|  | \  ___/   |     \   |  |  |_\  ___/
-// /_______  /\___  >____/\___  >__|  \___  >  \___  /   |__|____/\___  >
-//         \/     \/          \/          \/       \/                 \/
-//
-
-// DeleteRepoFileOptions holds the repository delete file options
-type DeleteRepoFileOptions struct {
-	LastCommitID string
-	OldBranch    string
-	NewBranch    string
-	TreePath     string
-	Message      string
-}
-
-// DeleteRepoFile deletes a repository file
-func (repo *Repository) DeleteRepoFile(doer *User, opts DeleteRepoFileOptions) (err error) {
-	repoWorkingPool.CheckIn(com.ToStr(repo.ID))
-	defer repoWorkingPool.CheckOut(com.ToStr(repo.ID))
-
-	if err = repo.DiscardLocalRepoBranchChanges(opts.OldBranch); err != nil {
-		return fmt.Errorf("DiscardLocalRepoBranchChanges [branch: %s]: %v", opts.OldBranch, err)
-	} else if err = repo.UpdateLocalCopyBranch(opts.OldBranch); err != nil {
-		return fmt.Errorf("UpdateLocalCopyBranch [branch: %s]: %v", opts.OldBranch, err)
-	}
-
-	if opts.OldBranch != opts.NewBranch {
-		if err := repo.CheckoutNewBranch(opts.OldBranch, opts.NewBranch); err != nil {
-			return fmt.Errorf("CheckoutNewBranch [old_branch: %s, new_branch: %s]: %v", opts.OldBranch, opts.NewBranch, err)
-		}
-	}
-
-	localPath := repo.LocalCopyPath()
-	if err = os.Remove(path.Join(localPath, opts.TreePath)); err != nil {
-		return fmt.Errorf("Remove: %v", err)
-	}
-
-	if err = git.AddChanges(localPath, true); err != nil {
-		return fmt.Errorf("git add --all: %v", err)
-	} else if err = git.CommitChanges(localPath, git.CommitChangesOptions{
-		Committer: doer.NewGitSig(),
-		Message:   opts.Message,
-	}); err != nil {
-		return fmt.Errorf("CommitChanges: %v", err)
-	} else if err = git.Push(localPath, git.PushOptions{
-		Remote: "origin",
-		Branch: opts.NewBranch,
-	}); err != nil {
-		return fmt.Errorf("git push origin %s: %v", opts.NewBranch, err)
-	}
-
-	gitRepo, err := git.OpenRepository(repo.RepoPath())
-	if err != nil {
-		log.Error(4, "OpenRepository: %v", err)
-		return nil
-	}
-	commit, err := gitRepo.GetBranchCommit(opts.NewBranch)
-	if err != nil {
-		log.Error(4, "GetBranchCommit [branch: %s]: %v", opts.NewBranch, err)
-		return nil
-	}
-
-	// Simulate push event.
-	oldCommitID := opts.LastCommitID
-	if opts.NewBranch != opts.OldBranch {
-		oldCommitID = git.EmptySHA
-	}
-
-	if err = repo.GetOwner(); err != nil {
-		return fmt.Errorf("GetOwner: %v", err)
-	}
-	err = PushUpdate(
-		opts.NewBranch,
-		PushUpdateOptions{
-			PusherID:     doer.ID,
-			PusherName:   doer.Name,
-			RepoUserName: repo.Owner.Name,
-			RepoName:     repo.Name,
-			RefFullName:  git.BranchPrefix + opts.NewBranch,
-			OldCommitID:  oldCommitID,
-			NewCommitID:  commit.ID.String(),
-		},
-	)
-	if err != nil {
-		return fmt.Errorf("PushUpdate: %v", err)
-	}
-	return nil
-}
-
-//  ____ ___        .__                    .___ ___________.___.__
-// |    |   \______ |  |   _________     __| _/ \_   _____/|   |  |   ____   ______
-// |    |   /\____ \|  |  /  _ \__  \   / __ |   |    __)  |   |  | _/ __ \ /  ___/
-// |    |  / |  |_> >  |_(  <_> ) __ \_/ /_/ |   |     \   |   |  |_\  ___/ \___ \
-// |______/  |   __/|____/\____(____  /\____ |   \___  /   |___|____/\___  >____  >
-//           |__|                   \/      \/       \/                  \/     \/
-//
-
-// Upload represent a uploaded file to a repo to be deleted when moved
-type Upload struct {
-	ID   int64  `xorm:"pk autoincr"`
-	UUID string `xorm:"uuid UNIQUE"`
-	Name string
-}
-
-// UploadLocalPath returns where uploads is stored in local file system based on given UUID.
-func UploadLocalPath(uuid string) string {
-	return path.Join(setting.Repository.Upload.TempPath, uuid[0:1], uuid[1:2], uuid)
-}
-
-// LocalPath returns where uploads are temporarily stored in local file system.
-func (upload *Upload) LocalPath() string {
-	return UploadLocalPath(upload.UUID)
-}
-
-// NewUpload creates a new upload object.
-func NewUpload(name string, buf []byte, file multipart.File) (_ *Upload, err error) {
-	upload := &Upload{
-		UUID: gouuid.NewV4().String(),
-		Name: name,
-	}
-
-	localPath := upload.LocalPath()
-	if err = os.MkdirAll(path.Dir(localPath), os.ModePerm); err != nil {
-		return nil, fmt.Errorf("MkdirAll: %v", err)
-	}
-
-	fw, err := os.Create(localPath)
-	if err != nil {
-		return nil, fmt.Errorf("Create: %v", err)
-	}
-	defer fw.Close()
-
-	if _, err = fw.Write(buf); err != nil {
-		return nil, fmt.Errorf("Write: %v", err)
-	} else if _, err = io.Copy(fw, file); err != nil {
-		return nil, fmt.Errorf("Copy: %v", err)
-	}
-
-	if _, err := x.Insert(upload); err != nil {
-		return nil, err
-	}
-
-	return upload, nil
-}
-
-// GetUploadByUUID returns the Upload by UUID
-func GetUploadByUUID(uuid string) (*Upload, error) {
-	upload := &Upload{UUID: uuid}
-	has, err := x.Get(upload)
-	if err != nil {
-		return nil, err
-	} else if !has {
-		return nil, ErrUploadNotExist{0, uuid}
-	}
-	return upload, nil
-}
-
-// GetUploadsByUUIDs returns multiple uploads by UUIDS
-func GetUploadsByUUIDs(uuids []string) ([]*Upload, error) {
-	if len(uuids) == 0 {
-		return []*Upload{}, nil
-	}
-
-	// Silently drop invalid uuids.
-	uploads := make([]*Upload, 0, len(uuids))
-	return uploads, x.In("uuid", uuids).Find(&uploads)
-}
-
-// DeleteUploads deletes multiple uploads
-func DeleteUploads(uploads ...*Upload) (err error) {
-	if len(uploads) == 0 {
-		return nil
-	}
-
-	sess := x.NewSession()
-	defer sess.Close()
-	if err = sess.Begin(); err != nil {
-		return err
-	}
-
-	ids := make([]int64, len(uploads))
-	for i := 0; i < len(uploads); i++ {
-		ids[i] = uploads[i].ID
-	}
-	if _, err = sess.
-		In("id", ids).
-		Delete(new(Upload)); err != nil {
-		return fmt.Errorf("delete uploads: %v", err)
-	}
-
-	for _, upload := range uploads {
-		localPath := upload.LocalPath()
-		if !com.IsFile(localPath) {
-			continue
-		}
-
-		if err := os.Remove(localPath); err != nil {
-			return fmt.Errorf("remove upload: %v", err)
-		}
-	}
-
-	return sess.Commit()
-}
-
-// DeleteUpload delete a upload
-func DeleteUpload(u *Upload) error {
-	return DeleteUploads(u)
-}
-
-// DeleteUploadByUUID deletes a upload by UUID
-func DeleteUploadByUUID(uuid string) error {
-	upload, err := GetUploadByUUID(uuid)
-	if err != nil {
-		if IsErrUploadNotExist(err) {
-			return nil
-		}
-		return fmt.Errorf("GetUploadByUUID: %v", err)
-	}
-
-	if err := DeleteUpload(upload); err != nil {
-		return fmt.Errorf("DeleteUpload: %v", err)
-	}
-
-	return nil
-}
-
-// UploadRepoFileOptions contains the uploaded repository file options
-type UploadRepoFileOptions struct {
-	LastCommitID string
-	OldBranch    string
-	NewBranch    string
-	TreePath     string
-	Message      string
-	Files        []string // In UUID format.
-}
-
-// UploadRepoFiles uploads files to a repository
-func (repo *Repository) UploadRepoFiles(doer *User, opts UploadRepoFileOptions) (err error) {
-	if len(opts.Files) == 0 {
-		return nil
-	}
-
-	uploads, err := GetUploadsByUUIDs(opts.Files)
-	if err != nil {
-		return fmt.Errorf("GetUploadsByUUIDs [uuids: %v]: %v", opts.Files, err)
-	}
-
-	repoWorkingPool.CheckIn(com.ToStr(repo.ID))
-	defer repoWorkingPool.CheckOut(com.ToStr(repo.ID))
-
-	if err = repo.DiscardLocalRepoBranchChanges(opts.OldBranch); err != nil {
-		return fmt.Errorf("DiscardLocalRepoBranchChanges [branch: %s]: %v", opts.OldBranch, err)
-	} else if err = repo.UpdateLocalCopyBranch(opts.OldBranch); err != nil {
-		return fmt.Errorf("UpdateLocalCopyBranch [branch: %s]: %v", opts.OldBranch, err)
-	}
-
-	if opts.OldBranch != opts.NewBranch {
-		if err = repo.CheckoutNewBranch(opts.OldBranch, opts.NewBranch); err != nil {
-			return fmt.Errorf("CheckoutNewBranch [old_branch: %s, new_branch: %s]: %v", opts.OldBranch, opts.NewBranch, err)
-		}
-	}
-
-	localPath := repo.LocalCopyPath()
-	dirPath := path.Join(localPath, opts.TreePath)
-
-	if err := os.MkdirAll(dirPath, os.ModePerm); err != nil {
-		return fmt.Errorf("Failed to create dir %s: %v", dirPath, err)
-	}
-
-	// Copy uploaded files into repository.
-	for _, upload := range uploads {
-		tmpPath := upload.LocalPath()
-		targetPath := path.Join(dirPath, upload.Name)
-		if !com.IsFile(tmpPath) {
-			continue
-		}
-
-		if err = com.Copy(tmpPath, targetPath); err != nil {
-			return fmt.Errorf("Copy: %v", err)
-		}
-	}
-
-	if err = git.AddChanges(localPath, true); err != nil {
-		return fmt.Errorf("git add --all: %v", err)
-	} else if err = git.CommitChanges(localPath, git.CommitChangesOptions{
-		Committer: doer.NewGitSig(),
-		Message:   opts.Message,
-	}); err != nil {
-		return fmt.Errorf("CommitChanges: %v", err)
-	} else if err = git.Push(localPath, git.PushOptions{
-		Remote: "origin",
-		Branch: opts.NewBranch,
-	}); err != nil {
-		return fmt.Errorf("git push origin %s: %v", opts.NewBranch, err)
-	}
-
-	gitRepo, err := git.OpenRepository(repo.RepoPath())
-	if err != nil {
-		log.Error(4, "OpenRepository: %v", err)
-		return nil
-	}
-	commit, err := gitRepo.GetBranchCommit(opts.NewBranch)
-	if err != nil {
-		log.Error(4, "GetBranchCommit [branch: %s]: %v", opts.NewBranch, err)
-		return nil
-	}
-
-	// Simulate push event.
-	oldCommitID := opts.LastCommitID
-	if opts.NewBranch != opts.OldBranch {
-		oldCommitID = git.EmptySHA
-	}
-
-	if err = repo.GetOwner(); err != nil {
-		return fmt.Errorf("GetOwner: %v", err)
-	}
-	err = PushUpdate(
-		opts.NewBranch,
-		PushUpdateOptions{
-			PusherID:     doer.ID,
-			PusherName:   doer.Name,
-			RepoUserName: repo.Owner.Name,
-			RepoName:     repo.Name,
-			RefFullName:  git.BranchPrefix + opts.NewBranch,
-			OldCommitID:  oldCommitID,
-			NewCommitID:  commit.ID.String(),
-		},
-	)
-	if err != nil {
-		return fmt.Errorf("PushUpdate: %v", err)
-	}
-
-	return DeleteUploads(uploads...)
-}

models/upload.go

@@ -0,0 +1,155 @@
+// Copyright 2016 The Gogs Authors. All rights reserved.
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package models
+
+import (
+	"fmt"
+	"io"
+	"mime/multipart"
+	"os"
+	"path"
+
+	"github.com/Unknwon/com"
+	gouuid "github.com/satori/go.uuid"
+
+	"code.gitea.io/gitea/modules/setting"
+)
+
+//  ____ ___        .__                    .___ ___________.___.__
+// |    |   \______ |  |   _________     __| _/ \_   _____/|   |  |   ____   ______
+// |    |   /\____ \|  |  /  _ \__  \   / __ |   |    __)  |   |  | _/ __ \ /  ___/
+// |    |  / |  |_> >  |_(  <_> ) __ \_/ /_/ |   |     \   |   |  |_\  ___/ \___ \
+// |______/  |   __/|____/\____(____  /\____ |   \___  /   |___|____/\___  >____  >
+//           |__|                   \/      \/       \/                  \/     \/
+//
+
+// Upload represent a uploaded file to a repo to be deleted when moved
+type Upload struct {
+	ID   int64  `xorm:"pk autoincr"`
+	UUID string `xorm:"uuid UNIQUE"`
+	Name string
+}
+
+// UploadLocalPath returns where uploads is stored in local file system based on given UUID.
+func UploadLocalPath(uuid string) string {
+	return path.Join(setting.Repository.Upload.TempPath, uuid[0:1], uuid[1:2], uuid)
+}
+
+// LocalPath returns where uploads are temporarily stored in local file system.
+func (upload *Upload) LocalPath() string {
+	return UploadLocalPath(upload.UUID)
+}
+
+// NewUpload creates a new upload object.
+func NewUpload(name string, buf []byte, file multipart.File) (_ *Upload, err error) {
+	upload := &Upload{
+		UUID: gouuid.NewV4().String(),
+		Name: name,
+	}
+
+	localPath := upload.LocalPath()
+	if err = os.MkdirAll(path.Dir(localPath), os.ModePerm); err != nil {
+		return nil, fmt.Errorf("MkdirAll: %v", err)
+	}
+
+	fw, err := os.Create(localPath)
+	if err != nil {
+		return nil, fmt.Errorf("Create: %v", err)
+	}
+	defer fw.Close()
+
+	if _, err = fw.Write(buf); err != nil {
+		return nil, fmt.Errorf("Write: %v", err)
+	} else if _, err = io.Copy(fw, file); err != nil {
+		return nil, fmt.Errorf("Copy: %v", err)
+	}
+
+	if _, err := x.Insert(upload); err != nil {
+		return nil, err
+	}
+
+	return upload, nil
+}
+
+// GetUploadByUUID returns the Upload by UUID
+func GetUploadByUUID(uuid string) (*Upload, error) {
+	upload := &Upload{UUID: uuid}
+	has, err := x.Get(upload)
+	if err != nil {
+		return nil, err
+	} else if !has {
+		return nil, ErrUploadNotExist{0, uuid}
+	}
+	return upload, nil
+}
+
+// GetUploadsByUUIDs returns multiple uploads by UUIDS
+func GetUploadsByUUIDs(uuids []string) ([]*Upload, error) {
+	if len(uuids) == 0 {
+		return []*Upload{}, nil
+	}
+
+	// Silently drop invalid uuids.
+	uploads := make([]*Upload, 0, len(uuids))
+	return uploads, x.In("uuid", uuids).Find(&uploads)
+}
+
+// DeleteUploads deletes multiple uploads
+func DeleteUploads(uploads ...*Upload) (err error) {
+	if len(uploads) == 0 {
+		return nil
+	}
+
+	sess := x.NewSession()
+	defer sess.Close()
+	if err = sess.Begin(); err != nil {
+		return err
+	}
+
+	ids := make([]int64, len(uploads))
+	for i := 0; i < len(uploads); i++ {
+		ids[i] = uploads[i].ID
+	}
+	if _, err = sess.
+		In("id", ids).
+		Delete(new(Upload)); err != nil {
+		return fmt.Errorf("delete uploads: %v", err)
+	}
+
+	if err = sess.Commit(); err != nil {
+		return err
+	}
+
+	for _, upload := range uploads {
+		localPath := upload.LocalPath()
+		if !com.IsFile(localPath) {
+			continue
+		}
+
+		if err := os.Remove(localPath); err != nil {
+			return fmt.Errorf("remove upload: %v", err)
+		}
+	}
+
+	return nil
+}
+
+// DeleteUploadByUUID deletes a upload by UUID
+func DeleteUploadByUUID(uuid string) error {
+	upload, err := GetUploadByUUID(uuid)
+	if err != nil {
+		if IsErrUploadNotExist(err) {
+			return nil
+		}
+		return fmt.Errorf("GetUploadByUUID: %v", err)
+	}
+
+	if err := DeleteUploads(upload); err != nil {
+		return fmt.Errorf("DeleteUpload: %v", err)
+	}
+
+	return nil
+}

modules/uploader/delete.go

@@ -0,0 +1,100 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package uploader
+
+import (
+	"fmt"
+
+	"code.gitea.io/git"
+	"code.gitea.io/gitea/models"
+)
+
+// DeleteRepoFileOptions holds the repository delete file options
+type DeleteRepoFileOptions struct {
+	LastCommitID string
+	OldBranch    string
+	NewBranch    string
+	TreePath     string
+	Message      string
+}
+
+// DeleteRepoFile deletes a file in the given repository
+func DeleteRepoFile(repo *models.Repository, doer *models.User, opts *DeleteRepoFileOptions) error {
+	t, err := NewTemporaryUploadRepository(repo)
+	defer t.Close()
+	if err != nil {
+		return err
+	}
+	if err := t.Clone(opts.OldBranch); err != nil {
+		return err
+	}
+	if err := t.SetDefaultIndex(); err != nil {
+		return err
+	}
+
+	filesInIndex, err := t.LsFiles(opts.TreePath)
+	if err != nil {
+		return fmt.Errorf("UpdateRepoFile: %v", err)
+	}
+
+	inFilelist := false
+	for _, file := range filesInIndex {
+		if file == opts.TreePath {
+			inFilelist = true
+		}
+	}
+	if !inFilelist {
+		return git.ErrNotExist{RelPath: opts.TreePath}
+	}
+
+	if err := t.RemoveFilesFromIndex(opts.TreePath); err != nil {
+		return err
+	}
+
+	// Now write the tree
+	treeHash, err := t.WriteTree()
+	if err != nil {
+		return err
+	}
+
+	// Now commit the tree
+	commitHash, err := t.CommitTree(doer, treeHash, opts.Message)
+	if err != nil {
+		return err
+	}
+
+	// Then push this tree to NewBranch
+	if err := t.Push(doer, commitHash, opts.NewBranch); err != nil {
+		return err
+	}
+
+	// Simulate push event.
+	oldCommitID := opts.LastCommitID
+	if opts.NewBranch != opts.OldBranch {
+		oldCommitID = git.EmptySHA
+	}
+
+	if err = repo.GetOwner(); err != nil {
+		return fmt.Errorf("GetOwner: %v", err)
+	}
+	err = models.PushUpdate(
+		opts.NewBranch,
+		models.PushUpdateOptions{
+			PusherID:     doer.ID,
+			PusherName:   doer.Name,
+			RepoUserName: repo.Owner.Name,
+			RepoName:     repo.Name,
+			RefFullName:  git.BranchPrefix + opts.NewBranch,
+			OldCommitID:  oldCommitID,
+			NewCommitID:  commitHash,
+		},
+	)
+	if err != nil {
+		return fmt.Errorf("PushUpdate: %v", err)
+	}
+
+	// FIXME: Should we UpdateRepoIndexer(repo) here?
+	return nil
+}

modules/uploader/diff.go

@@ -0,0 +1,38 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package uploader
+
+import (
+	"strings"
+
+	"code.gitea.io/gitea/models"
+)
+
+// GetDiffPreview produces and returns diff result of a file which is not yet committed.
+func GetDiffPreview(repo *models.Repository, branch, treePath, content string) (*models.Diff, error) {
+	t, err := NewTemporaryUploadRepository(repo)
+	defer t.Close()
+	if err != nil {
+		return nil, err
+	}
+	if err := t.Clone(branch); err != nil {
+		return nil, err
+	}
+	if err := t.SetDefaultIndex(); err != nil {
+		return nil, err
+	}
+
+	// Add the object to the database
+	objectHash, err := t.HashObject(strings.NewReader(content))
+	if err != nil {
+		return nil, err
+	}
+
+	// Add the object to the index
+	if err := t.AddObjectToIndex("100644", objectHash, treePath); err != nil {
+		return nil, err
+	}
+	return t.DiffIndex()
+}

modules/uploader/repo.go

@@ -0,0 +1,359 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package uploader
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path"
+	"strings"
+	"time"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/process"
+	"code.gitea.io/gitea/modules/setting"
+
+	"github.com/Unknwon/com"
+)
+
+// TemporaryUploadRepository is a type to wrap our upload repositories
+type TemporaryUploadRepository struct {
+	repo     *models.Repository
+	basePath string
+}
+
+// NewTemporaryUploadRepository creates a new temporary upload repository
+func NewTemporaryUploadRepository(repo *models.Repository) (*TemporaryUploadRepository, error) {
+	timeStr := com.ToStr(time.Now().Nanosecond()) // SHOULD USE SOMETHING UNIQUE
+	basePath := path.Join(models.LocalCopyPath(), "upload-"+timeStr+".git")
+	if err := os.MkdirAll(path.Dir(basePath), os.ModePerm); err != nil {
+		return nil, fmt.Errorf("Failed to create dir %s: %v", basePath, err)
+	}
+	t := &TemporaryUploadRepository{repo: repo, basePath: basePath}
+	return t, nil
+}
+
+// Close the repository cleaning up all files
+func (t *TemporaryUploadRepository) Close() {
+	if _, err := os.Stat(t.basePath); !os.IsNotExist(err) {
+		os.RemoveAll(t.basePath)
+	}
+}
+
+// Clone the base repository to our path and set branch as the HEAD
+func (t *TemporaryUploadRepository) Clone(branch string) error {
+	if _, stderr, err := process.GetManager().ExecTimeout(5*time.Minute,
+		fmt.Sprintf("Clone (git clone -s --bare): %s", t.basePath),
+		"git", "clone", "-s", "--bare", "-b", branch, t.repo.RepoPath(), t.basePath); err != nil {
+		return fmt.Errorf("Clone: %v %s", err, stderr)
+	}
+	return nil
+}
+
+// SetDefaultIndex sets the git index to our HEAD
+func (t *TemporaryUploadRepository) SetDefaultIndex() error {
+	if _, stderr, err := process.GetManager().ExecDir(5*time.Minute,
+		t.basePath,
+		fmt.Sprintf("SetDefaultIndex (git read-tree HEAD): %s", t.basePath),
+		"git", "read-tree", "HEAD"); err != nil {
+		return fmt.Errorf("SetDefaultIndex: %v %s", err, stderr)
+	}
+	return nil
+}
+
+// LsFiles checks if the given filename arguments are in the index
+func (t *TemporaryUploadRepository) LsFiles(filenames ...string) ([]string, error) {
+	stdOut := new(bytes.Buffer)
+	stdErr := new(bytes.Buffer)
+
+	timeout := 5 * time.Minute
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	cmdArgs := []string{"ls-files", "-z", "--"}
+	for _, arg := range filenames {
+		if arg != "" {
+			cmdArgs = append(cmdArgs, arg)
+		}
+	}
+
+	cmd := exec.CommandContext(ctx, "git", cmdArgs...)
+	desc := fmt.Sprintf("lsFiles: (git ls-files) %v", cmdArgs)
+	cmd.Dir = t.basePath
+	cmd.Stdout = stdOut
+	cmd.Stderr = stdErr
+
+	if err := cmd.Start(); err != nil {
+		return nil, fmt.Errorf("exec(%s) failed: %v(%v)", desc, err, ctx.Err())
+	}
+
+	pid := process.GetManager().Add(desc, cmd)
+	err := cmd.Wait()
+	process.GetManager().Remove(pid)
+
+	if err != nil {
+		err = fmt.Errorf("exec(%d:%s) failed: %v(%v) stdout: %v stderr: %v", pid, desc, err, ctx.Err(), stdOut, stdErr)
+		return nil, err
+	}
+
+	filelist := make([]string, len(filenames))
+	for _, line := range bytes.Split(stdOut.Bytes(), []byte{'\000'}) {
+		filelist = append(filelist, string(line))
+	}
+
+	return filelist, err
+}
+
+// RemoveFilesFromIndex removes the given files from the index
+func (t *TemporaryUploadRepository) RemoveFilesFromIndex(filenames ...string) error {
+	stdOut := new(bytes.Buffer)
+	stdErr := new(bytes.Buffer)
+	stdIn := new(bytes.Buffer)
+	for _, file := range filenames {
+		if file != "" {
+			stdIn.WriteString("0 0000000000000000000000000000000000000000\t")
+			stdIn.WriteString(file)
+			stdIn.WriteByte('\000')
+		}
+	}
+
+	timeout := 5 * time.Minute
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	cmdArgs := []string{"update-index", "--remove", "-z", "--index-info"}
+	cmd := exec.CommandContext(ctx, "git", cmdArgs...)
+	desc := fmt.Sprintf("removeFilesFromIndex: (git update-index) %v", filenames)
+	cmd.Dir = t.basePath
+	cmd.Stdout = stdOut
+	cmd.Stderr = stdErr
+	cmd.Stdin = bytes.NewReader(stdIn.Bytes())
+
+	if err := cmd.Start(); err != nil {
+		return fmt.Errorf("exec(%s) failed: %v(%v)", desc, err, ctx.Err())
+	}
+
+	pid := process.GetManager().Add(desc, cmd)
+	err := cmd.Wait()
+	process.GetManager().Remove(pid)
+
+	if err != nil {
+		err = fmt.Errorf("exec(%d:%s) failed: %v(%v) stdout: %v stderr: %v", pid, desc, err, ctx.Err(), stdOut, stdErr)
+	}
+
+	return err
+}
+
+// HashObject writes the provided content to the object db and returns its hash
+func (t *TemporaryUploadRepository) HashObject(content io.Reader) (string, error) {
+	timeout := 5 * time.Minute
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	hashCmd := exec.CommandContext(ctx, "git", "hash-object", "-w", "--stdin")
+	hashCmd.Dir = t.basePath
+	hashCmd.Stdin = content
+	stdOutBuffer := new(bytes.Buffer)
+	stdErrBuffer := new(bytes.Buffer)
+	hashCmd.Stdout = stdOutBuffer
+	hashCmd.Stderr = stdErrBuffer
+	desc := fmt.Sprintf("hashObject: (git hash-object)")
+	if err := hashCmd.Start(); err != nil {
+		return "", fmt.Errorf("git hash-object: %s", err)
+	}
+
+	pid := process.GetManager().Add(desc, hashCmd)
+	err := hashCmd.Wait()
+	process.GetManager().Remove(pid)
+
+	if err != nil {
+		err = fmt.Errorf("exec(%d:%s) failed: %v(%v) stdout: %v stderr: %v", pid, desc, err, ctx.Err(), stdOutBuffer, stdErrBuffer)
+		return "", err
+	}
+
+	return strings.TrimSpace(stdOutBuffer.String()), nil
+}
+
+// AddObjectToIndex adds the provided object hash to the index with the provided mode and path
+func (t *TemporaryUploadRepository) AddObjectToIndex(mode, objectHash, objectPath string) error {
+	if _, stderr, err := process.GetManager().ExecDir(5*time.Minute,
+		t.basePath,
+		fmt.Sprintf("addObjectToIndex (git update-index): %s", t.basePath),
+		"git", "update-index", "--add", "--replace", "--cacheinfo", mode, objectHash, objectPath); err != nil {
+		return fmt.Errorf("git update-index: %s", stderr)
+	}
+	return nil
+}
+
+// WriteTree writes the current index as a tree to the object db and returns its hash
+func (t *TemporaryUploadRepository) WriteTree() (string, error) {
+	treeHash, stderr, err := process.GetManager().ExecDir(5*time.Minute,
+		t.basePath,
+		fmt.Sprintf("WriteTree (git write-tree): %s", t.basePath),
+		"git", "write-tree")
+	if err != nil {
+		return "", fmt.Errorf("git write-tree: %s", stderr)
+	}
+	return strings.TrimSpace(treeHash), nil
+
+}
+
+// CommitTree creates a commit from a given tree for the user with provided message
+func (t *TemporaryUploadRepository) CommitTree(doer *models.User, treeHash string, message string) (string, error) {
+	commitTimeStr := time.Now().Format(time.UnixDate)
+	sig := doer.NewGitSig()
+
+	// FIXME: Should we add SSH_ORIGINAL_COMMAND to this
+	// Because this may call hooks we should pass in the environment
+	env := append(os.Environ(),
+		"GIT_AUTHOR_NAME="+sig.Name,
+		"GIT_AUTHOR_EMAIL="+sig.Email,
+		"GIT_AUTHOR_DATE="+commitTimeStr,
+		"GIT_COMMITTER_NAME="+sig.Name,
+		"GIT_COMMITTER_EMAIL="+sig.Email,
+		"GIT_COMMITTER_DATE="+commitTimeStr,
+	)
+	commitHash, stderr, err := process.GetManager().ExecDirEnv(5*time.Minute,
+		t.basePath,
+		fmt.Sprintf("commitTree (git commit-tree): %s", t.basePath),
+		env,
+		"git", "commit-tree", treeHash, "-p", "HEAD", "-m", message)
+	if err != nil {
+		return "", fmt.Errorf("git commit-tree: %s", stderr)
+	}
+	return strings.TrimSpace(commitHash), nil
+}
+
+// Push the provided commitHash to the repository branch by the provided user
+func (t *TemporaryUploadRepository) Push(doer *models.User, commitHash string, branch string) error {
+	isWiki := "false"
+	if strings.HasSuffix(t.repo.Name, ".wiki") {
+		isWiki = "true"
+	}
+
+	sig := doer.NewGitSig()
+
+	// FIXME: Should we add SSH_ORIGINAL_COMMAND to this
+	// Because calls hooks we need to pass in the environment
+	env := append(os.Environ(),
+		"GIT_AUTHOR_NAME="+sig.Name,
+		"GIT_AUTHOR_EMAIL="+sig.Email,
+		"GIT_COMMITTER_NAME="+sig.Name,
+		"GIT_COMMITTER_EMAIL="+sig.Email,
+		models.EnvRepoName+"="+t.repo.Name,
+		models.EnvRepoUsername+"="+t.repo.OwnerName,
+		models.EnvRepoIsWiki+"="+isWiki,
+		models.EnvPusherName+"="+doer.Name,
+		models.EnvPusherID+"="+fmt.Sprintf("%d", doer.ID),
+		models.ProtectedBranchRepoID+"="+fmt.Sprintf("%d", t.repo.ID),
+	)
+
+	if _, stderr, err := process.GetManager().ExecDirEnv(5*time.Minute,
+		t.basePath,
+		fmt.Sprintf("actuallyPush (git push): %s", t.basePath),
+		env,
+		"git", "push", t.repo.RepoPath(), strings.TrimSpace(commitHash)+":refs/heads/"+strings.TrimSpace(branch)); err != nil {
+		return fmt.Errorf("git push: %s", stderr)
+	}
+	return nil
+}
+
+// DiffIndex returns a Diff of the current index to the head
+func (t *TemporaryUploadRepository) DiffIndex() (diff *models.Diff, err error) {
+	timeout := 5 * time.Minute
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	stdErr := new(bytes.Buffer)
+
+	cmd := exec.CommandContext(ctx, "git", "diff-index", "--cached", "-p", "HEAD")
+	cmd.Dir = t.basePath
+	cmd.Stderr = stdErr
+
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return nil, fmt.Errorf("StdoutPipe: %v stderr %s", err, stdErr.String())
+	}
+
+	if err = cmd.Start(); err != nil {
+		return nil, fmt.Errorf("Start: %v stderr %s", err, stdErr.String())
+	}
+
+	pid := process.GetManager().Add(fmt.Sprintf("diffIndex [repo_path: %s]", t.repo.RepoPath()), cmd)
+	defer process.GetManager().Remove(pid)
+
+	diff, err = models.ParsePatch(setting.Git.MaxGitDiffLines, setting.Git.MaxGitDiffLineCharacters, setting.Git.MaxGitDiffFiles, stdout)
+	if err != nil {
+		return nil, fmt.Errorf("ParsePatch: %v", err)
+	}
+
+	if err = cmd.Wait(); err != nil {
+		return nil, fmt.Errorf("Wait: %v", err)
+	}
+
+	return diff, nil
+}
+
+// CheckAttribute checks the given attribute of the provided files
+func (t *TemporaryUploadRepository) CheckAttribute(attribute string, args ...string) (map[string]map[string]string, error) {
+	stdOut := new(bytes.Buffer)
+	stdErr := new(bytes.Buffer)
+
+	timeout := 5 * time.Minute
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	cmdArgs := []string{"check-attr", "-z", attribute, "--cached", "--"}
+	for _, arg := range args {
+		if arg != "" {
+			cmdArgs = append(cmdArgs, arg)
+		}
+	}
+
+	cmd := exec.CommandContext(ctx, "git", cmdArgs...)
+	desc := fmt.Sprintf("checkAttr: (git check-attr) %s %v", attribute, cmdArgs)
+	cmd.Dir = t.basePath
+	cmd.Stdout = stdOut
+	cmd.Stderr = stdErr
+
+	if err := cmd.Start(); err != nil {
+		return nil, fmt.Errorf("exec(%s) failed: %v(%v)", desc, err, ctx.Err())
+	}
+
+	pid := process.GetManager().Add(desc, cmd)
+	err := cmd.Wait()
+	process.GetManager().Remove(pid)
+
+	if err != nil {
+		err = fmt.Errorf("exec(%d:%s) failed: %v(%v) stdout: %v stderr: %v", pid, desc, err, ctx.Err(), stdOut, stdErr)
+		return nil, err
+	}
+
+	fields := bytes.Split(stdOut.Bytes(), []byte{'\000'})
+
+	if len(fields)%3 != 1 {
+		return nil, fmt.Errorf("Wrong number of fields in return from check-attr")
+	}
+
+	var name2attribute2info = make(map[string]map[string]string)
+
+	for i := 0; i < (len(fields) / 3); i++ {
+		filename := string(fields[3*i])
+		attribute := string(fields[3*i+1])
+		info := string(fields[3*i+2])
+		attribute2info := name2attribute2info[filename]
+		if attribute2info == nil {
+			attribute2info = make(map[string]string)
+		}
+		attribute2info[attribute] = info
+		name2attribute2info[filename] = attribute2info
+	}
+
+	return name2attribute2info, err
+}

modules/uploader/update.go

@@ -0,0 +1,159 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package uploader
+
+import (
+	"fmt"
+	"strings"
+
+	"code.gitea.io/git"
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/lfs"
+	"code.gitea.io/gitea/modules/setting"
+)
+
+// UpdateRepoFileOptions holds the repository file update options
+type UpdateRepoFileOptions struct {
+	LastCommitID string
+	OldBranch    string
+	NewBranch    string
+	OldTreeName  string
+	NewTreeName  string
+	Message      string
+	Content      string
+	IsNewFile    bool
+}
+
+// UpdateRepoFile adds or updates a file in the given repository
+func UpdateRepoFile(repo *models.Repository, doer *models.User, opts *UpdateRepoFileOptions) error {
+	t, err := NewTemporaryUploadRepository(repo)
+	defer t.Close()
+	if err != nil {
+		return err
+	}
+	if err := t.Clone(opts.OldBranch); err != nil {
+		return err
+	}
+	if err := t.SetDefaultIndex(); err != nil {
+		return err
+	}
+
+	filesInIndex, err := t.LsFiles(opts.NewTreeName, opts.OldTreeName)
+	if err != nil {
+		return fmt.Errorf("UpdateRepoFile: %v", err)
+	}
+
+	if opts.IsNewFile {
+		for _, file := range filesInIndex {
+			if file == opts.NewTreeName {
+				return models.ErrRepoFileAlreadyExist{FileName: opts.NewTreeName}
+			}
+		}
+	}
+
+	//var stdout string
+	if opts.OldTreeName != opts.NewTreeName && len(filesInIndex) > 0 {
+		for _, file := range filesInIndex {
+			if file == opts.OldTreeName {
+				if err := t.RemoveFilesFromIndex(opts.OldTreeName); err != nil {
+					return err
+				}
+			}
+		}
+
+	}
+
+	// Check there is no way this can return multiple infos
+	filename2attribute2info, err := t.CheckAttribute("filter", opts.NewTreeName)
+	if err != nil {
+		return err
+	}
+
+	content := opts.Content
+	var lfsMetaObject *models.LFSMetaObject
+
+	if filename2attribute2info[opts.NewTreeName] != nil && filename2attribute2info[opts.NewTreeName]["filter"] == "lfs" {
+		// OK so we are supposed to LFS this data!
+		oid, err := models.GenerateLFSOid(strings.NewReader(opts.Content))
+		if err != nil {
+			return err
+		}
+		lfsMetaObject = &models.LFSMetaObject{Oid: oid, Size: int64(len(opts.Content)), RepositoryID: repo.ID}
+		content = lfsMetaObject.Pointer()
+	}
+
+	// Add the object to the database
+	objectHash, err := t.HashObject(strings.NewReader(content))
+	if err != nil {
+		return err
+	}
+
+	// Add the object to the index
+	if err := t.AddObjectToIndex("100644", objectHash, opts.NewTreeName); err != nil {
+		return err
+	}
+
+	// Now write the tree
+	treeHash, err := t.WriteTree()
+	if err != nil {
+		return err
+	}
+
+	// Now commit the tree
+	commitHash, err := t.CommitTree(doer, treeHash, opts.Message)
+	if err != nil {
+		return err
+	}
+
+	if lfsMetaObject != nil {
+		// We have an LFS object - create it
+		lfsMetaObject, err = models.NewLFSMetaObject(lfsMetaObject)
+		if err != nil {
+			return err
+		}
+		contentStore := &lfs.ContentStore{BasePath: setting.LFS.ContentPath}
+		if !contentStore.Exists(lfsMetaObject) {
+			if err := contentStore.Put(lfsMetaObject, strings.NewReader(opts.Content)); err != nil {
+				if err2 := repo.RemoveLFSMetaObjectByOid(lfsMetaObject.Oid); err2 != nil {
+					return fmt.Errorf("Error whilst removing failed inserted LFS object %s: %v (Prev Error: %v)", lfsMetaObject.Oid, err2, err)
+				}
+				return err
+			}
+		}
+	}
+
+	// Then push this tree to NewBranch
+	if err := t.Push(doer, commitHash, opts.NewBranch); err != nil {
+		return err
+	}
+
+	// Simulate push event.
+	oldCommitID := opts.LastCommitID
+	if opts.NewBranch != opts.OldBranch {
+		oldCommitID = git.EmptySHA
+	}
+
+	if err = repo.GetOwner(); err != nil {
+		return fmt.Errorf("GetOwner: %v", err)
+	}
+	err = models.PushUpdate(
+		opts.NewBranch,
+		models.PushUpdateOptions{
+			PusherID:     doer.ID,
+			PusherName:   doer.Name,
+			RepoUserName: repo.Owner.Name,
+			RepoName:     repo.Name,
+			RefFullName:  git.BranchPrefix + opts.NewBranch,
+			OldCommitID:  oldCommitID,
+			NewCommitID:  commitHash,
+		},
+	)
+	if err != nil {
+		return fmt.Errorf("PushUpdate: %v", err)
+	}
+	models.UpdateRepoIndexer(repo)
+
+	return nil
+}

modules/uploader/upload.go

@@ -0,0 +1,206 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package uploader
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"strings"
+
+	"code.gitea.io/gitea/modules/lfs"
+	"code.gitea.io/gitea/modules/setting"
+
+	"code.gitea.io/git"
+	"code.gitea.io/gitea/models"
+)
+
+// UploadRepoFileOptions contains the uploaded repository file options
+type UploadRepoFileOptions struct {
+	LastCommitID string
+	OldBranch    string
+	NewBranch    string
+	TreePath     string
+	Message      string
+	Files        []string // In UUID format.
+}
+
+type uploadInfo struct {
+	upload        *models.Upload
+	lfsMetaObject *models.LFSMetaObject
+}
+
+func cleanUpAfterFailure(infos *[]uploadInfo, t *TemporaryUploadRepository, original error) error {
+	for _, info := range *infos {
+		if info.lfsMetaObject == nil {
+			continue
+		}
+		if !info.lfsMetaObject.Existing {
+			if err := t.repo.RemoveLFSMetaObjectByOid(info.lfsMetaObject.Oid); err != nil {
+				original = fmt.Errorf("%v, %v", original, err)
+			}
+		}
+	}
+	return original
+}
+
+// UploadRepoFiles uploads files to the given repository
+func UploadRepoFiles(repo *models.Repository, doer *models.User, opts *UploadRepoFileOptions) error {
+	if len(opts.Files) == 0 {
+		return nil
+	}
+
+	uploads, err := models.GetUploadsByUUIDs(opts.Files)
+	if err != nil {
+		return fmt.Errorf("GetUploadsByUUIDs [uuids: %v]: %v", opts.Files, err)
+	}
+
+	t, err := NewTemporaryUploadRepository(repo)
+	defer t.Close()
+	if err != nil {
+		return err
+	}
+	if err := t.Clone(opts.OldBranch); err != nil {
+		return err
+	}
+	if err := t.SetDefaultIndex(); err != nil {
+		return err
+	}
+
+	names := make([]string, len(uploads))
+	infos := make([]uploadInfo, len(uploads))
+	for i, upload := range uploads {
+		names[i] = upload.Name
+		infos[i] = uploadInfo{upload: upload}
+	}
+
+	filename2attribute2info, err := t.CheckAttribute("filter", names...)
+	if err != nil {
+		return err
+	}
+
+	// Copy uploaded files into repository.
+	for i, uploadInfo := range infos {
+		file, err := os.Open(uploadInfo.upload.LocalPath())
+		if err != nil {
+			return err
+		}
+		defer file.Close()
+
+		var objectHash string
+		if filename2attribute2info[uploadInfo.upload.Name] != nil && filename2attribute2info[uploadInfo.upload.Name]["filter"] == "lfs" {
+			// Handle LFS
+			// FIXME: Inefficient! this should probably happen in models.Upload
+			oid, err := models.GenerateLFSOid(file)
+			if err != nil {
+				return err
+			}
+			fileInfo, err := file.Stat()
+			if err != nil {
+				return err
+			}
+
+			uploadInfo.lfsMetaObject = &models.LFSMetaObject{Oid: oid, Size: fileInfo.Size(), RepositoryID: t.repo.ID}
+
+			if objectHash, err = t.HashObject(strings.NewReader(uploadInfo.lfsMetaObject.Pointer())); err != nil {
+				return err
+			}
+			infos[i] = uploadInfo
+
+		} else {
+			if objectHash, err = t.HashObject(file); err != nil {
+				return err
+			}
+		}
+
+		// Add the object to the index
+		if err := t.AddObjectToIndex("100644", objectHash, path.Join(opts.TreePath, uploadInfo.upload.Name)); err != nil {
+			return err
+
+		}
+	}
+
+	// Now write the tree
+	treeHash, err := t.WriteTree()
+	if err != nil {
+		return err
+	}
+
+	// Now commit the tree
+	commitHash, err := t.CommitTree(doer, treeHash, opts.Message)
+	if err != nil {
+		return err
+	}
+
+	// Now deal with LFS objects
+	for _, uploadInfo := range infos {
+		if uploadInfo.lfsMetaObject == nil {
+			continue
+		}
+		uploadInfo.lfsMetaObject, err = models.NewLFSMetaObject(uploadInfo.lfsMetaObject)
+		if err != nil {
+			// OK Now we need to cleanup
+			return cleanUpAfterFailure(&infos, t, err)
+		}
+		// Don't move the files yet - we need to ensure that
+		// everything can be inserted first
+	}
+
+	// OK now we can insert the data into the store - there's no way to clean up the store
+	// once it's in there, it's in there.
+	contentStore := &lfs.ContentStore{BasePath: setting.LFS.ContentPath}
+	for _, uploadInfo := range infos {
+		if uploadInfo.lfsMetaObject == nil {
+			continue
+		}
+		if !contentStore.Exists(uploadInfo.lfsMetaObject) {
+			file, err := os.Open(uploadInfo.upload.LocalPath())
+			if err != nil {
+				return cleanUpAfterFailure(&infos, t, err)
+			}
+			defer file.Close()
+			// FIXME: Put regenerates the hash and copies the file over.
+			// I guess this strictly ensures the soundness of the store but this is inefficient.
+			if err := contentStore.Put(uploadInfo.lfsMetaObject, file); err != nil {
+				// OK Now we need to cleanup
+				// Can't clean up the store, once uploaded there they're there.
+				return cleanUpAfterFailure(&infos, t, err)
+			}
+		}
+	}
+
+	// Then push this tree to NewBranch
+	if err := t.Push(doer, commitHash, opts.NewBranch); err != nil {
+		return err
+	}
+
+	// Simulate push event.
+	oldCommitID := opts.LastCommitID
+	if opts.NewBranch != opts.OldBranch {
+		oldCommitID = git.EmptySHA
+	}
+
+	if err = repo.GetOwner(); err != nil {
+		return fmt.Errorf("GetOwner: %v", err)
+	}
+	err = models.PushUpdate(
+		opts.NewBranch,
+		models.PushUpdateOptions{
+			PusherID:     doer.ID,
+			PusherName:   doer.Name,
+			RepoUserName: repo.Owner.Name,
+			RepoName:     repo.Name,
+			RefFullName:  git.BranchPrefix + opts.NewBranch,
+			OldCommitID:  oldCommitID,
+			NewCommitID:  commitHash,
+		},
+	)
+	if err != nil {
+		return fmt.Errorf("PushUpdate: %v", err)
+	}
+	// FIXME: Should we models.UpdateRepoIndexer(repo) here?
+
+	return models.DeleteUploads(uploads...)
+}

routers/repo/editor.go

@@ -19,6 +19,7 @@ import (
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/templates"
+	"code.gitea.io/gitea/modules/uploader"
 )
 
 const (
@@ -62,6 +63,16 @@ func editFile(ctx *context.Context, isNewFile bool) {
 	ctx.Data["RequireSimpleMDE"] = true
 	canCommit := renderCommitRights(ctx)
 
+	treePath := cleanUploadFileName(ctx.Repo.TreePath)
+	if treePath != ctx.Repo.TreePath {
+		if isNewFile {
+			ctx.Redirect(path.Join(ctx.Repo.RepoLink, "_new", ctx.Repo.BranchName, treePath))
+		} else {
+			ctx.Redirect(path.Join(ctx.Repo.RepoLink, "_edit", ctx.Repo.BranchName, treePath))
+		}
+		return
+	}
+
 	treeNames, treePaths := getParentTreeFields(ctx.Repo.TreePath)
 
 	if !isNewFile {
@@ -155,7 +166,7 @@ func editFilePost(ctx *context.Context, form auth.EditRepoFileForm, isNewFile bo
 
 	oldBranchName := ctx.Repo.BranchName
 	branchName := oldBranchName
-	oldTreePath := ctx.Repo.TreePath
+	oldTreePath := cleanUploadFileName(ctx.Repo.TreePath)
 	lastCommit := form.LastCommit
 	form.LastCommit = ctx.Repo.Commit.ID.String()
 
@@ -298,7 +309,7 @@ func editFilePost(ctx *context.Context, form auth.EditRepoFileForm, isNewFile bo
 		message += "\n\n" + form.CommitMessage
 	}
 
-	if err := ctx.Repo.Repository.UpdateRepoFile(ctx.User, models.UpdateRepoFileOptions{
+	if err := uploader.UpdateRepoFile(ctx.Repo.Repository, ctx.User, &uploader.UpdateRepoFileOptions{
 		LastCommitID: lastCommit,
 		OldBranch:    oldBranchName,
 		NewBranch:    branchName,
@@ -328,7 +339,11 @@ func NewFilePost(ctx *context.Context, form auth.EditRepoFileForm) {
 
 // DiffPreviewPost render preview diff page
 func DiffPreviewPost(ctx *context.Context, form auth.EditPreviewDiffForm) {
-	treePath := ctx.Repo.TreePath
+	treePath := cleanUploadFileName(ctx.Repo.TreePath)
+	if len(treePath) == 0 {
+		ctx.Error(500, "file name to diff is invalid")
+		return
+	}
 
 	entry, err := ctx.Repo.Commit.GetTreeEntryByPath(treePath)
 	if err != nil {
@@ -339,7 +354,7 @@ func DiffPreviewPost(ctx *context.Context, form auth.EditPreviewDiffForm) {
 		return
 	}
 
-	diff, err := ctx.Repo.Repository.GetDiffPreview(ctx.Repo.BranchName, treePath, form.Content)
+	diff, err := uploader.GetDiffPreview(ctx.Repo.Repository, ctx.Repo.BranchName, treePath, form.Content)
 	if err != nil {
 		ctx.Error(500, "GetDiffPreview: "+err.Error())
 		return
@@ -358,7 +373,14 @@ func DiffPreviewPost(ctx *context.Context, form auth.EditPreviewDiffForm) {
 func DeleteFile(ctx *context.Context) {
 	ctx.Data["PageIsDelete"] = true
 	ctx.Data["BranchLink"] = ctx.Repo.RepoLink + "/src/" + ctx.Repo.BranchNameSubURL()
-	ctx.Data["TreePath"] = ctx.Repo.TreePath
+	treePath := cleanUploadFileName(ctx.Repo.TreePath)
+
+	if treePath != ctx.Repo.TreePath {
+		ctx.Redirect(path.Join(ctx.Repo.RepoLink, "_delete", ctx.Repo.BranchName, treePath))
+		return
+	}
+
+	ctx.Data["TreePath"] = treePath
 	canCommit := renderCommitRights(ctx)
 
 	ctx.Data["commit_summary"] = ""
@@ -426,7 +448,7 @@ func DeleteFilePost(ctx *context.Context, form auth.DeleteRepoFileForm) {
 		message += "\n\n" + form.CommitMessage
 	}
 
-	if err := ctx.Repo.Repository.DeleteRepoFile(ctx.User, models.DeleteRepoFileOptions{
+	if err := uploader.DeleteRepoFile(ctx.Repo.Repository, ctx.User, &uploader.DeleteRepoFileOptions{
 		LastCommitID: ctx.Repo.CommitID,
 		OldBranch:    oldBranchName,
 		NewBranch:    branchName,
@@ -453,6 +475,12 @@ func UploadFile(ctx *context.Context) {
 	ctx.Data["PageIsUpload"] = true
 	renderUploadSettings(ctx)
 	canCommit := renderCommitRights(ctx)
+	treePath := cleanUploadFileName(ctx.Repo.TreePath)
+	if treePath != ctx.Repo.TreePath {
+		ctx.Redirect(path.Join(ctx.Repo.RepoLink, "_upload", ctx.Repo.BranchName, treePath))
+		return
+	}
+	ctx.Repo.TreePath = treePath
 
 	treeNames, treePaths := getParentTreeFields(ctx.Repo.TreePath)
 	if len(treeNames) == 0 {
@@ -489,10 +517,6 @@ func UploadFilePost(ctx *context.Context, form auth.UploadRepoFileForm) {
 	}
 
 	form.TreePath = cleanUploadFileName(form.TreePath)
-	if len(form.TreePath) == 0 {
-		ctx.Error(500, "Upload file name is invalid")
-		return
-	}
 
 	treeNames, treePaths := getParentTreeFields(form.TreePath)
 	if len(treeNames) == 0 {
@@ -559,7 +583,7 @@ func UploadFilePost(ctx *context.Context, form auth.UploadRepoFileForm) {
 		message += "\n\n" + form.CommitMessage
 	}
 
-	if err := ctx.Repo.Repository.UploadRepoFiles(ctx.User, models.UploadRepoFileOptions{
+	if err := uploader.UploadRepoFiles(ctx.Repo.Repository, ctx.User, &uploader.UploadRepoFileOptions{
 		LastCommitID: ctx.Repo.CommitID,
 		OldBranch:    oldBranchName,
 		NewBranch:    branchName,
@@ -576,12 +600,13 @@ func UploadFilePost(ctx *context.Context, form auth.UploadRepoFileForm) {
 }
 
 func cleanUploadFileName(name string) string {
-	name = strings.TrimLeft(name, "./\\")
+	// Rebase the filename
-	name = strings.Replace(name, "../", "", -1)
+	name = strings.Trim(path.Clean("/"+name), " /")
-	name = strings.Replace(name, "..\\", "", -1)
+	// Git disallows any filenames to have a .git directory in them.
-	name = strings.TrimPrefix(path.Clean(name), ".git/")
+	for _, part := range strings.Split(name, "/") {
-	if name == ".git" {
+		if strings.ToLower(part) == ".git" {
-		return ""
+			return ""
+		}
 	}
 	return name
 }

routers/repo/editor_test.go

@@ -15,16 +15,24 @@ func TestCleanUploadName(t *testing.T) {
 	models.PrepareTestEnv(t)
 
 	var kases = map[string]string{
-		".git/refs/master": "git/refs/master",
+		".git/refs/master":               "",
-		"/root/abc":        "root/abc",
+		"/root/abc":                      "root/abc",
-		"./../../abc":      "abc",
+		"./../../abc":                    "abc",
-		"a/../.git":        "a/.git",
+		"a/../.git":                      "",
-		"a/../../../abc":   "a/abc",
+		"a/../../../abc":                 "abc",
-		"../../../acd":     "acd",
+		"../../../acd":                   "acd",
-		"../../.git/abc":   "git/abc",
+		"../../.git/abc":                 "",
-		"..\\..\\.git/abc": "git/abc",
+		"..\\..\\.git/abc":               "..\\..\\.git/abc",
+		"..\\../.git/abc":                "",
+		"..\\../.git":                    "",
+		"abc/../def":                     "def",
+		".drone.yml":                     ".drone.yml",
+		".abc/def/.drone.yml":            ".abc/def/.drone.yml",
+		"..drone.yml.":                   "..drone.yml.",
+		"..a.dotty...name...":            "..a.dotty...name...",
+		"..a.dotty../.folder../.name...": "..a.dotty../.folder../.name...",
 	}
 	for k, v := range kases {
-		assert.EqualValues(t, v, cleanUploadFileName(k))
+		assert.EqualValues(t, cleanUploadFileName(k), v)
 	}
 }