From 80a6b0f5bce15a641fc75f5f1ef6e42ef54424bc Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 14 Oct 2020 21:07:51 +0800 Subject: [PATCH] Avatars and Repo avatars support storing in minio (#12516) * Avatar support minio * Support repo avatar minio storage * Add missing migration * Fix bug * Fix test * Add test for minio store type on avatars and repo avatars; Add documents * Fix bug * Fix bug * Add back missed avatar link method * refactor codes * Simplify the codes * Code improvements * Fix lint * Fix test mysql * Fix test mysql * Fix test mysql * Fix settings * Fix test * fix test * Fix bug --- cmd/migrate_storage.go | 25 +- .../doc/advanced/config-cheat-sheet.en-us.md | 13 +- .../doc/advanced/config-cheat-sheet.zh-cn.md | 14 ++ integrations/mysql.ini.tmpl | 3 +- models/migrations/v115.go | 8 +- models/org.go | 10 +- models/repo.go | 213 ++---------------- models/repo_avatar.go | 190 ++++++++++++++++ models/repo_generate.go | 4 +- models/unit_tests.go | 5 + models/user.go | 192 +++------------- models/user_avatar.go | 169 ++++++++++++++ modules/avatar/avatar.go | 9 +- modules/avatar/avatar_test.go | 16 +- modules/setting/database.go | 3 +- modules/setting/picture.go | 114 ++++++++++ modules/setting/setting.go | 69 +----- modules/storage/storage.go | 38 ++++ routers/repo/setting.go | 5 +- routers/routes/routes.go | 77 +++++-- routers/user/setting/profile.go | 5 +- 21 files changed, 705 insertions(+), 477 deletions(-) create mode 100644 models/repo_avatar.go create mode 100644 models/user_avatar.go create mode 100644 modules/setting/picture.go diff --git a/cmd/migrate_storage.go b/cmd/migrate_storage.go index 5f19556d8..871baed92 100644 --- a/cmd/migrate_storage.go +++ b/cmd/migrate_storage.go @@ -91,6 +91,20 @@ func migrateLFS(dstStorage storage.ObjectStorage) error { }) } +func migrateAvatars(dstStorage storage.ObjectStorage) error { + return models.IterateUser(func(user *models.User) error { + _, err := storage.Copy(dstStorage, user.CustomAvatarRelativePath(), storage.Avatars, user.CustomAvatarRelativePath()) + return err + }) +} + +func migrateRepoAvatars(dstStorage storage.ObjectStorage) error { + return models.IterateRepository(func(repo *models.Repository) error { + _, err := storage.Copy(dstStorage, repo.CustomAvatarRelativePath(), storage.RepoAvatars, repo.CustomAvatarRelativePath()) + return err + }) +} + func runMigrateStorage(ctx *cli.Context) error { if err := initDB(); err != nil { return err @@ -142,9 +156,8 @@ func runMigrateStorage(ctx *cli.Context) error { UseSSL: ctx.Bool("minio-use-ssl"), }) default: - return fmt.Errorf("Unsupported attachments storage type: %s", ctx.String("storage")) + return fmt.Errorf("Unsupported storage type: %s", ctx.String("storage")) } - if err != nil { return err } @@ -159,6 +172,14 @@ func runMigrateStorage(ctx *cli.Context) error { if err := migrateLFS(dstStorage); err != nil { return err } + case "avatars": + if err := migrateAvatars(dstStorage); err != nil { + return err + } + case "repo-avatars": + if err := migrateRepoAvatars(dstStorage); err != nil { + return err + } default: return fmt.Errorf("Unsupported storage: %s", ctx.String("type")) } diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index 3bd667be6..bdd872b0b 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -564,16 +564,21 @@ Define allowed algorithms and their minimum key length (use -1 to disable a type - `DISABLE_GRAVATAR`: **false**: Enable this to use local avatars only. - `ENABLE_FEDERATED_AVATAR`: **false**: Enable support for federated avatars (see [http://www.libravatar.org](http://www.libravatar.org)). + +- `AVATAR_STORAGE_TYPE`: **default**: Storage type defined in `[storage.xxx]`. Default is `default` which will read `[storage]` if no section `[storage]` will be a type `local`. - `AVATAR_UPLOAD_PATH`: **data/avatars**: Path to store user avatar image files. +- `AVATAR_MAX_WIDTH`: **4096**: Maximum avatar image width in pixels. +- `AVATAR_MAX_HEIGHT`: **3072**: Maximum avatar image height in pixels. +- `AVATAR_MAX_FILE_SIZE`: **1048576** (1Mb): Maximum avatar image file size in bytes. + +- `REPOSITORY_AVATAR_STORAGE_TYPE`: **default**: Storage type defined in `[storage.xxx]`. Default is `default` which will read `[storage]` if no section `[storage]` will be a type `local`. - `REPOSITORY_AVATAR_UPLOAD_PATH`: **data/repo-avatars**: Path to store repository avatar image files. - `REPOSITORY_AVATAR_FALLBACK`: **none**: How Gitea deals with missing repository avatars - none = no avatar will be displayed - random = random avatar will be generated - - image = default image will be used (which is set in `REPOSITORY_AVATAR_DEFAULT_IMAGE`) + - image = default image will be used (which is set in `REPOSITORY_AVATAR_FALLBACK_IMAGE`) - `REPOSITORY_AVATAR_FALLBACK_IMAGE`: **/img/repo_default.png**: Image used as default repository avatar (if `REPOSITORY_AVATAR_FALLBACK` is set to image and none was uploaded) -- `AVATAR_MAX_WIDTH`: **4096**: Maximum avatar image width in pixels. -- `AVATAR_MAX_HEIGHT`: **3072**: Maximum avatar image height in pixels. -- `AVATAR_MAX_FILE_SIZE`: **1048576** (1Mb): Maximum avatar image file size in bytes. + ## Project (`project`) diff --git a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md index efa390bfb..505fdcdf7 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md +++ b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md @@ -182,6 +182,20 @@ menu: - `DISABLE_GRAVATAR`: 开启则只使用内部头像。 - `ENABLE_FEDERATED_AVATAR`: 启用头像联盟支持 (参见 http://www.libravatar.org) +- `AVATAR_STORAGE_TYPE`: **local**: 头像存储类型，可以为 `local` 或 `minio`，分别支持本地文件系统和 minio 兼容的API。 +- `AVATAR_UPLOAD_PATH`: **data/avatars**: 存储头像的文件系统路径。 +- `AVATAR_MAX_WIDTH`: **4096**: 头像最大宽度，单位像素。 +- `AVATAR_MAX_HEIGHT`: **3072**: 头像最大高度，单位像素。 +- `AVATAR_MAX_FILE_SIZE`: **1048576** (1Mb): 头像最大大小。 + +- `REPOSITORY_AVATAR_STORAGE_TYPE`: **local**: 仓库头像存储类型，可以为 `local` 或 `minio`，分别支持本地文件系统和 minio 兼容的API。 +- `REPOSITORY_AVATAR_UPLOAD_PATH`: **data/repo-avatars**: 存储仓库头像的路径。 +- `REPOSITORY_AVATAR_FALLBACK`: **none**: 当头像丢失时的处理方式 + - none = 不显示头像 + - random = 显示随机生成的头像 + - image = 显示默认头像，通过 `REPOSITORY_AVATAR_FALLBACK_IMAGE` 设置 +- `REPOSITORY_AVATAR_FALLBACK_IMAGE`: **/img/repo_default.png**: 默认仓库头像 + ## Attachment (`attachment`) - `ENABLED`: 是否允许用户上传附件。 diff --git a/integrations/mysql.ini.tmpl b/integrations/mysql.ini.tmpl index f10ea9f9d..b546748d1 100644 --- a/integrations/mysql.ini.tmpl +++ b/integrations/mysql.ini.tmpl @@ -58,7 +58,7 @@ LFS_MINIO_BASE_PATH = lfs/ LFS_MINIO_USE_SSL = false [attachment] -STORE_TYPE = minio +STORAGE_TYPE = minio SERVE_DIRECT = false MINIO_ENDPOINT = minio:9000 MINIO_ACCESS_KEY_ID = 123456 @@ -87,6 +87,7 @@ ENABLE_NOTIFY_MAIL = true [picture] DISABLE_GRAVATAR = false ENABLE_FEDERATED_AVATAR = false + AVATAR_UPLOAD_PATH = integrations/gitea-integration-mysql/data/avatars REPOSITORY_AVATAR_UPLOAD_PATH = integrations/gitea-integration-mysql/data/repo-avatars diff --git a/models/migrations/v115.go b/models/migrations/v115.go index fe3b08611..fcec1f549 100644 --- a/models/migrations/v115.go +++ b/models/migrations/v115.go @@ -61,7 +61,7 @@ func renameExistingUserAvatarName(x *xorm.Engine) error { for _, user := range users { oldAvatar := user.Avatar - if stat, err := os.Stat(filepath.Join(setting.AvatarUploadPath, oldAvatar)); err != nil || !stat.Mode().IsRegular() { + if stat, err := os.Stat(filepath.Join(setting.Avatar.Path, oldAvatar)); err != nil || !stat.Mode().IsRegular() { if err == nil { err = fmt.Errorf("Error: \"%s\" is not a regular file", oldAvatar) } @@ -86,7 +86,7 @@ func renameExistingUserAvatarName(x *xorm.Engine) error { return fmt.Errorf("[user: %s] user table update: %v", user.LowerName, err) } - deleteList[filepath.Join(setting.AvatarUploadPath, oldAvatar)] = struct{}{} + deleteList[filepath.Join(setting.Avatar.Path, oldAvatar)] = struct{}{} migrated++ select { case <-ticker.C: @@ -135,7 +135,7 @@ func renameExistingUserAvatarName(x *xorm.Engine) error { // copyOldAvatarToNewLocation copies oldAvatar to newAvatarLocation // and returns newAvatar location func copyOldAvatarToNewLocation(userID int64, oldAvatar string) (string, error) { - fr, err := os.Open(filepath.Join(setting.AvatarUploadPath, oldAvatar)) + fr, err := os.Open(filepath.Join(setting.Avatar.Path, oldAvatar)) if err != nil { return "", fmt.Errorf("os.Open: %v", err) } @@ -151,7 +151,7 @@ func copyOldAvatarToNewLocation(userID int64, oldAvatar string) (string, error) return newAvatar, nil } - if err := ioutil.WriteFile(filepath.Join(setting.AvatarUploadPath, newAvatar), data, 0666); err != nil { + if err := ioutil.WriteFile(filepath.Join(setting.Avatar.Path, newAvatar), data, 0666); err != nil { return "", fmt.Errorf("ioutil.WriteFile: %v", err) } diff --git a/models/org.go b/models/org.go index 31e5cf81c..b24db935a 100644 --- a/models/org.go +++ b/models/org.go @@ -11,10 +11,10 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" - "github.com/unknwon/com" "xorm.io/builder" "xorm.io/xorm" ) @@ -310,11 +310,9 @@ func deleteOrg(e *xorm.Session, u *User) error { } if len(u.Avatar) > 0 { - avatarPath := u.CustomAvatarPath() - if com.IsExist(avatarPath) { - if err := util.Remove(avatarPath); err != nil { - return fmt.Errorf("Failed to remove %s: %v", avatarPath, err) - } + avatarPath := u.CustomAvatarRelativePath() + if err := storage.Avatars.Delete(avatarPath); err != nil { + return fmt.Errorf("Failed to remove %s: %v", avatarPath, err) } } diff --git a/models/repo.go b/models/repo.go index f505412e0..efdd7049d 100644 --- a/models/repo.go +++ b/models/repo.go @@ -7,7 +7,6 @@ package models import ( "context" - "crypto/md5" "errors" "fmt" "html/template" @@ -15,7 +14,6 @@ import ( // Needed for jpeg support _ "image/jpeg" - "image/png" "io/ioutil" "net" "net/url" @@ -27,7 +25,6 @@ import ( "strings" "time" - "code.gitea.io/gitea/modules/avatar" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/options" @@ -1796,11 +1793,8 @@ func DeleteRepository(doer *User, uid, repoID int64) error { } if len(repo.Avatar) > 0 { - avatarPath := repo.CustomAvatarPath() - if com.IsExist(avatarPath) { - if err := util.Remove(avatarPath); err != nil { - return fmt.Errorf("Failed to remove %s: %v", avatarPath, err) - } + if err := storage.RepoAvatars.Delete(repo.CustomAvatarRelativePath()); err != nil { + return fmt.Errorf("Failed to remove %s: %v", repo.Avatar, err) } } @@ -2239,187 +2233,6 @@ func (repo *Repository) GetUserFork(userID int64) (*Repository, error) { return &forkedRepo, nil } -// CustomAvatarPath returns repository custom avatar file path. -func (repo *Repository) CustomAvatarPath() string { - // Avatar empty by default - if len(repo.Avatar) == 0 { - return "" - } - return filepath.Join(setting.RepositoryAvatarUploadPath, repo.Avatar) -} - -// generateRandomAvatar generates a random avatar for repository. -func (repo *Repository) generateRandomAvatar(e Engine) error { - idToString := fmt.Sprintf("%d", repo.ID) - - seed := idToString - img, err := avatar.RandomImage([]byte(seed)) - if err != nil { - return fmt.Errorf("RandomImage: %v", err) - } - - repo.Avatar = idToString - if err = os.MkdirAll(filepath.Dir(repo.CustomAvatarPath()), os.ModePerm); err != nil { - return fmt.Errorf("MkdirAll: %v", err) - } - fw, err := os.Create(repo.CustomAvatarPath()) - if err != nil { - return fmt.Errorf("Create: %v", err) - } - defer fw.Close() - - if err = png.Encode(fw, img); err != nil { - return fmt.Errorf("Encode: %v", err) - } - log.Info("New random avatar created for repository: %d", repo.ID) - - if _, err := e.ID(repo.ID).Cols("avatar").NoAutoTime().Update(repo); err != nil { - return err - } - - return nil -} - -// RemoveRandomAvatars removes the randomly generated avatars that were created for repositories -func RemoveRandomAvatars(ctx context.Context) error { - return x. - Where("id > 0").BufferSize(setting.Database.IterateBufferSize). - Iterate(new(Repository), - func(idx int, bean interface{}) error { - repository := bean.(*Repository) - select { - case <-ctx.Done(): - return ErrCancelledf("before random avatars removed for %s", repository.FullName()) - default: - } - stringifiedID := strconv.FormatInt(repository.ID, 10) - if repository.Avatar == stringifiedID { - return repository.DeleteAvatar() - } - return nil - }) -} - -// RelAvatarLink returns a relative link to the repository's avatar. -func (repo *Repository) RelAvatarLink() string { - return repo.relAvatarLink(x) -} - -func (repo *Repository) relAvatarLink(e Engine) string { - // If no avatar - path is empty - avatarPath := repo.CustomAvatarPath() - if len(avatarPath) == 0 || !com.IsFile(avatarPath) { - switch mode := setting.RepositoryAvatarFallback; mode { - case "image": - return setting.RepositoryAvatarFallbackImage - case "random": - if err := repo.generateRandomAvatar(e); err != nil { - log.Error("generateRandomAvatar: %v", err) - } - default: - // default behaviour: do not display avatar - return "" - } - } - return setting.AppSubURL + "/repo-avatars/" + repo.Avatar -} - -// AvatarLink returns a link to the repository's avatar. -func (repo *Repository) AvatarLink() string { - return repo.avatarLink(x) -} - -// avatarLink returns user avatar absolute link. -func (repo *Repository) avatarLink(e Engine) string { - link := repo.relAvatarLink(e) - // link may be empty! - if len(link) > 0 { - if link[0] == '/' && link[1] != '/' { - return setting.AppURL + strings.TrimPrefix(link, setting.AppSubURL)[1:] - } - } - return link -} - -// UploadAvatar saves custom avatar for repository. -// FIXME: split uploads to different subdirs in case we have massive number of repos. -func (repo *Repository) UploadAvatar(data []byte) error { - m, err := avatar.Prepare(data) - if err != nil { - return err - } - - sess := x.NewSession() - defer sess.Close() - if err = sess.Begin(); err != nil { - return err - } - - oldAvatarPath := repo.CustomAvatarPath() - - // Users can upload the same image to other repo - prefix it with ID - // Then repo will be removed - only it avatar file will be removed - repo.Avatar = fmt.Sprintf("%d-%x", repo.ID, md5.Sum(data)) - if _, err := sess.ID(repo.ID).Cols("avatar").Update(repo); err != nil { - return fmt.Errorf("UploadAvatar: Update repository avatar: %v", err) - } - - if err := os.MkdirAll(setting.RepositoryAvatarUploadPath, os.ModePerm); err != nil { - return fmt.Errorf("UploadAvatar: Failed to create dir %s: %v", setting.RepositoryAvatarUploadPath, err) - } - - fw, err := os.Create(repo.CustomAvatarPath()) - if err != nil { - return fmt.Errorf("UploadAvatar: Create file: %v", err) - } - defer fw.Close() - - if err = png.Encode(fw, *m); err != nil { - return fmt.Errorf("UploadAvatar: Encode png: %v", err) - } - - if len(oldAvatarPath) > 0 && oldAvatarPath != repo.CustomAvatarPath() { - if err := util.Remove(oldAvatarPath); err != nil { - return fmt.Errorf("UploadAvatar: Failed to remove old repo avatar %s: %v", oldAvatarPath, err) - } - } - - return sess.Commit() -} - -// DeleteAvatar deletes the repos's custom avatar. -func (repo *Repository) DeleteAvatar() error { - - // Avatar not exists - if len(repo.Avatar) == 0 { - return nil - } - - avatarPath := repo.CustomAvatarPath() - log.Trace("DeleteAvatar[%d]: %s", repo.ID, avatarPath) - - sess := x.NewSession() - defer sess.Close() - if err := sess.Begin(); err != nil { - return err - } - - repo.Avatar = "" - if _, err := sess.ID(repo.ID).Cols("avatar").Update(repo); err != nil { - return fmt.Errorf("DeleteAvatar: Update repository avatar: %v", err) - } - - if _, err := os.Stat(avatarPath); err == nil { - if err := util.Remove(avatarPath); err != nil { - return fmt.Errorf("DeleteAvatar: Failed to remove %s: %v", avatarPath, err) - } - } else { - // // Schrodinger: file may or may not exist. See err for details. - log.Trace("DeleteAvatar[%d]: %v", err) - } - return sess.Commit() -} - // GetOriginalURLHostname returns the hostname of a URL or the URL func (repo *Repository) GetOriginalURLHostname() string { u, err := url.Parse(repo.OriginalURL) @@ -2502,3 +2315,25 @@ func DoctorUserStarNum() (err error) { return } + +// IterateRepository iterate repositories +func IterateRepository(f func(repo *Repository) error) error { + var start int + var batchSize = setting.Database.IterateBufferSize + for { + var repos = make([]*Repository, 0, batchSize) + if err := x.Limit(batchSize, start).Find(&repos); err != nil { + return err + } + if len(repos) == 0 { + return nil + } + start += len(repos) + + for _, repo := range repos { + if err := f(repo); err != nil { + return err + } + } + } +} diff --git a/models/repo_avatar.go b/models/repo_avatar.go new file mode 100644 index 000000000..6f8f55f9e --- /dev/null +++ b/models/repo_avatar.go @@ -0,0 +1,190 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package models + +import ( + "context" + "crypto/md5" + "fmt" + "image/png" + "io" + "strconv" + "strings" + + "code.gitea.io/gitea/modules/avatar" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" +) + +// CustomAvatarRelativePath returns repository custom avatar file path. +func (repo *Repository) CustomAvatarRelativePath() string { + return repo.Avatar +} + +// generateRandomAvatar generates a random avatar for repository. +func (repo *Repository) generateRandomAvatar(e Engine) error { + idToString := fmt.Sprintf("%d", repo.ID) + + seed := idToString + img, err := avatar.RandomImage([]byte(seed)) + if err != nil { + return fmt.Errorf("RandomImage: %v", err) + } + + repo.Avatar = idToString + + if err := storage.SaveFrom(storage.RepoAvatars, repo.CustomAvatarRelativePath(), func(w io.Writer) error { + if err := png.Encode(w, img); err != nil { + log.Error("Encode: %v", err) + } + return err + }); err != nil { + return fmt.Errorf("Failed to create dir %s: %v", repo.CustomAvatarRelativePath(), err) + } + + log.Info("New random avatar created for repository: %d", repo.ID) + + if _, err := e.ID(repo.ID).Cols("avatar").NoAutoTime().Update(repo); err != nil { + return err + } + + return nil +} + +// RemoveRandomAvatars removes the randomly generated avatars that were created for repositories +func RemoveRandomAvatars(ctx context.Context) error { + return x. + Where("id > 0").BufferSize(setting.Database.IterateBufferSize). + Iterate(new(Repository), + func(idx int, bean interface{}) error { + repository := bean.(*Repository) + select { + case <-ctx.Done(): + return ErrCancelledf("before random avatars removed for %s", repository.FullName()) + default: + } + stringifiedID := strconv.FormatInt(repository.ID, 10) + if repository.Avatar == stringifiedID { + return repository.DeleteAvatar() + } + return nil + }) +} + +// RelAvatarLink returns a relative link to the repository's avatar. +func (repo *Repository) RelAvatarLink() string { + return repo.relAvatarLink(x) +} + +func (repo *Repository) relAvatarLink(e Engine) string { + // If no avatar - path is empty + avatarPath := repo.CustomAvatarRelativePath() + if len(avatarPath) == 0 { + switch mode := setting.RepoAvatar.Fallback; mode { + case "image": + return setting.RepoAvatar.FallbackImage + case "random": + if err := repo.generateRandomAvatar(e); err != nil { + log.Error("generateRandomAvatar: %v", err) + } + default: + // default behaviour: do not display avatar + return "" + } + } + return setting.AppSubURL + "/repo-avatars/" + repo.Avatar +} + +// AvatarLink returns a link to the repository's avatar. +func (repo *Repository) AvatarLink() string { + return repo.avatarLink(x) +} + +// avatarLink returns user avatar absolute link. +func (repo *Repository) avatarLink(e Engine) string { + link := repo.relAvatarLink(e) + // link may be empty! + if len(link) > 0 { + if link[0] == '/' && link[1] != '/' { + return setting.AppURL + strings.TrimPrefix(link, setting.AppSubURL)[1:] + } + } + return link +} + +// UploadAvatar saves custom avatar for repository. +// FIXME: split uploads to different subdirs in case we have massive number of repos. +func (repo *Repository) UploadAvatar(data []byte) error { + m, err := avatar.Prepare(data) + if err != nil { + return err + } + + newAvatar := fmt.Sprintf("%d-%x", repo.ID, md5.Sum(data)) + if repo.Avatar == newAvatar { // upload the same picture + return nil + } + + sess := x.NewSession() + defer sess.Close() + if err = sess.Begin(); err != nil { + return err + } + + oldAvatarPath := repo.CustomAvatarRelativePath() + + // Users can upload the same image to other repo - prefix it with ID + // Then repo will be removed - only it avatar file will be removed + repo.Avatar = newAvatar + if _, err := sess.ID(repo.ID).Cols("avatar").Update(repo); err != nil { + return fmt.Errorf("UploadAvatar: Update repository avatar: %v", err) + } + + if err := storage.SaveFrom(storage.RepoAvatars, repo.CustomAvatarRelativePath(), func(w io.Writer) error { + if err := png.Encode(w, *m); err != nil { + log.Error("Encode: %v", err) + } + return err + }); err != nil { + return fmt.Errorf("UploadAvatar %s failed: Failed to remove old repo avatar %s: %v", repo.RepoPath(), newAvatar, err) + } + + if len(oldAvatarPath) > 0 { + if err := storage.RepoAvatars.Delete(oldAvatarPath); err != nil { + return fmt.Errorf("UploadAvatar: Failed to remove old repo avatar %s: %v", oldAvatarPath, err) + } + } + + return sess.Commit() +} + +// DeleteAvatar deletes the repos's custom avatar. +func (repo *Repository) DeleteAvatar() error { + // Avatar not exists + if len(repo.Avatar) == 0 { + return nil + } + + avatarPath := repo.CustomAvatarRelativePath() + log.Trace("DeleteAvatar[%d]: %s", repo.ID, avatarPath) + + sess := x.NewSession() + defer sess.Close() + if err := sess.Begin(); err != nil { + return err + } + + repo.Avatar = "" + if _, err := sess.ID(repo.ID).Cols("avatar").Update(repo); err != nil { + return fmt.Errorf("DeleteAvatar: Update repository avatar: %v", err) + } + + if err := storage.RepoAvatars.Delete(avatarPath); err != nil { + return fmt.Errorf("DeleteAvatar: Failed to remove %s: %v", avatarPath, err) + } + + return sess.Commit() +} diff --git a/models/repo_generate.go b/models/repo_generate.go index 480683cd4..0b234d8e3 100644 --- a/models/repo_generate.go +++ b/models/repo_generate.go @@ -10,10 +10,10 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/util" "github.com/gobwas/glob" - "github.com/unknwon/com" ) // GenerateRepoOptions contains the template units to generate @@ -139,7 +139,7 @@ func GenerateWebhooks(ctx DBContext, templateRepo, generateRepo *Repository) err // GenerateAvatar generates the avatar from a template repository func GenerateAvatar(ctx DBContext, templateRepo, generateRepo *Repository) error { generateRepo.Avatar = strings.Replace(templateRepo.Avatar, strconv.FormatInt(templateRepo.ID, 10), strconv.FormatInt(generateRepo.ID, 10), 1) - if err := com.Copy(templateRepo.CustomAvatarPath(), generateRepo.CustomAvatarPath()); err != nil { + if _, err := storage.Copy(storage.RepoAvatars, generateRepo.CustomAvatarRelativePath(), storage.RepoAvatars, templateRepo.CustomAvatarRelativePath()); err != nil { return err } diff --git a/models/unit_tests.go b/models/unit_tests.go index 031744629..7254cbf66 100644 --- a/models/unit_tests.go +++ b/models/unit_tests.go @@ -70,6 +70,11 @@ func MainTest(m *testing.M, pathToGiteaRoot string) { setting.Attachment.Storage.Path = filepath.Join(setting.AppDataPath, "attachments") setting.LFS.Storage.Path = filepath.Join(setting.AppDataPath, "lfs") + + setting.Avatar.Storage.Path = filepath.Join(setting.AppDataPath, "avatars") + + setting.RepoAvatar.Storage.Path = filepath.Join(setting.AppDataPath, "repo-avatars") + if err = storage.Init(); err != nil { fatalTestError("storage.Init: %v\n", err) } diff --git a/models/user.go b/models/user.go index 6c57dd473..7248db533 100644 --- a/models/user.go +++ b/models/user.go @@ -8,29 +8,26 @@ package models import ( "container/list" "context" - "crypto/md5" "crypto/sha256" "crypto/subtle" "encoding/hex" "errors" "fmt" _ "image/jpeg" // Needed for jpeg support - "image/png" "os" "path/filepath" "regexp" - "strconv" "strings" "time" "unicode/utf8" - "code.gitea.io/gitea/modules/avatar" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/generate" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/public" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/structs" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/timeutil" @@ -347,104 +344,6 @@ func (u *User) GenerateActivateCode() string { return u.GenerateEmailActivateCode(u.Email) } -// CustomAvatarPath returns user custom avatar file path. -func (u *User) CustomAvatarPath() string { - return filepath.Join(setting.AvatarUploadPath, u.Avatar) -} - -// GenerateRandomAvatar generates a random avatar for user. -func (u *User) GenerateRandomAvatar() error { - return u.generateRandomAvatar(x) -} - -func (u *User) generateRandomAvatar(e Engine) error { - seed := u.Email - if len(seed) == 0 { - seed = u.Name - } - - img, err := avatar.RandomImage([]byte(seed)) - if err != nil { - return fmt.Errorf("RandomImage: %v", err) - } - // NOTICE for random avatar, it still uses id as avatar name, but custom avatar use md5 - // since random image is not a user's photo, there is no security for enumable - if u.Avatar == "" { - u.Avatar = fmt.Sprintf("%d", u.ID) - } - if err = os.MkdirAll(filepath.Dir(u.CustomAvatarPath()), os.ModePerm); err != nil { - return fmt.Errorf("MkdirAll: %v", err) - } - fw, err := os.Create(u.CustomAvatarPath()) - if err != nil { - return fmt.Errorf("Create: %v", err) - } - defer fw.Close() - - if _, err := e.ID(u.ID).Cols("avatar").Update(u); err != nil { - return err - } - - if err = png.Encode(fw, img); err != nil { - return fmt.Errorf("Encode: %v", err) - } - - log.Info("New random avatar created: %d", u.ID) - return nil -} - -// SizedRelAvatarLink returns a link to the user's avatar via -// the local explore page. Function returns immediately. -// When applicable, the link is for an avatar of the indicated size (in pixels). -func (u *User) SizedRelAvatarLink(size int) string { - return strings.TrimSuffix(setting.AppSubURL, "/") + "/user/avatar/" + u.Name + "/" + strconv.Itoa(size) -} - -// RealSizedAvatarLink returns a link to the user's avatar. When -// applicable, the link is for an avatar of the indicated size (in pixels). -// -// This function make take time to return when federated avatars -// are in use, due to a DNS lookup need -// -func (u *User) RealSizedAvatarLink(size int) string { - if u.ID == -1 { - return base.DefaultAvatarLink() - } - - switch { - case u.UseCustomAvatar: - if !com.IsFile(u.CustomAvatarPath()) { - return base.DefaultAvatarLink() - } - return setting.AppSubURL + "/avatars/" + u.Avatar - case setting.DisableGravatar, setting.OfflineMode: - if !com.IsFile(u.CustomAvatarPath()) { - if err := u.GenerateRandomAvatar(); err != nil { - log.Error("GenerateRandomAvatar: %v", err) - } - } - - return setting.AppSubURL + "/avatars/" + u.Avatar - } - return base.SizedAvatarLink(u.AvatarEmail, size) -} - -// RelAvatarLink returns a relative link to the user's avatar. The link -// may either be a sub-URL to this site, or a full URL to an external avatar -// service. -func (u *User) RelAvatarLink() string { - return u.SizedRelAvatarLink(base.DefaultAvatarSize) -} - -// AvatarLink returns user avatar absolute link. -func (u *User) AvatarLink() string { - link := u.RelAvatarLink() - if link[0] == '/' && link[1] != '/' { - return setting.AppURL + strings.TrimPrefix(link, setting.AppSubURL)[1:] - } - return link -} - // GetFollowers returns range of user's followers. func (u *User) GetFollowers(listOptions ListOptions) ([]*User, error) { sess := x. @@ -537,64 +436,6 @@ func (u *User) IsPasswordSet() bool { return !u.ValidatePassword("") } -// UploadAvatar saves custom avatar for user. -// FIXME: split uploads to different subdirs in case we have massive users. -func (u *User) UploadAvatar(data []byte) error { - m, err := avatar.Prepare(data) - if err != nil { - return err - } - - sess := x.NewSession() - defer sess.Close() - if err = sess.Begin(); err != nil { - return err - } - - u.UseCustomAvatar = true - // Different users can upload same image as avatar - // If we prefix it with u.ID, it will be separated - // Otherwise, if any of the users delete his avatar - // Other users will lose their avatars too. - u.Avatar = fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("%d-%x", u.ID, md5.Sum(data))))) - if err = updateUser(sess, u); err != nil { - return fmt.Errorf("updateUser: %v", err) - } - - if err := os.MkdirAll(setting.AvatarUploadPath, os.ModePerm); err != nil { - return fmt.Errorf("Failed to create dir %s: %v", setting.AvatarUploadPath, err) - } - - fw, err := os.Create(u.CustomAvatarPath()) - if err != nil { - return fmt.Errorf("Create: %v", err) - } - defer fw.Close() - - if err = png.Encode(fw, *m); err != nil { - return fmt.Errorf("Encode: %v", err) - } - - return sess.Commit() -} - -// DeleteAvatar deletes the user's custom avatar. -func (u *User) DeleteAvatar() error { - log.Trace("DeleteAvatar[%d]: %s", u.ID, u.CustomAvatarPath()) - if len(u.Avatar) > 0 { - if err := util.Remove(u.CustomAvatarPath()); err != nil { - return fmt.Errorf("Failed to remove %s: %v", u.CustomAvatarPath(), err) - } - } - - u.UseCustomAvatar = false - u.Avatar = "" - if _, err := x.ID(u.ID).Cols("avatar, use_custom_avatar").Update(u); err != nil { - return fmt.Errorf("UpdateUser: %v", err) - } - return nil -} - // IsOrganization returns true if user is actually a organization. func (u *User) IsOrganization() bool { return u.Type == UserTypeOrganization @@ -1285,17 +1126,14 @@ func deleteUser(e *xorm.Session, u *User) error { // Note: There are something just cannot be roll back, // so just keep error logs of those operations. path := UserPath(u.Name) - if err := util.RemoveAll(path); err != nil { return fmt.Errorf("Failed to RemoveAll %s: %v", path, err) } if len(u.Avatar) > 0 { - avatarPath := u.CustomAvatarPath() - if com.IsExist(avatarPath) { - if err := util.Remove(avatarPath); err != nil { - return fmt.Errorf("Failed to remove %s: %v", avatarPath, err) - } + avatarPath := u.CustomAvatarRelativePath() + if err := storage.Avatars.Delete(avatarPath); err != nil { + return fmt.Errorf("Failed to remove %s: %v", avatarPath, err) } } @@ -2034,3 +1872,25 @@ func SyncExternalUsers(ctx context.Context, updateExisting bool) error { } return nil } + +// IterateUser iterate users +func IterateUser(f func(user *User) error) error { + var start int + var batchSize = setting.Database.IterateBufferSize + for { + var users = make([]*User, 0, batchSize) + if err := x.Limit(batchSize, start).Find(&users); err != nil { + return err + } + if len(users) == 0 { + return nil + } + start += len(users) + + for _, user := range users { + if err := f(user); err != nil { + return err + } + } + } +} diff --git a/models/user_avatar.go b/models/user_avatar.go new file mode 100644 index 000000000..0a03ca770 --- /dev/null +++ b/models/user_avatar.go @@ -0,0 +1,169 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package models + +import ( + "crypto/md5" + "fmt" + "image/png" + "io" + "strconv" + "strings" + + "code.gitea.io/gitea/modules/avatar" + "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" +) + +// CustomAvatarRelativePath returns user custom avatar relative path. +func (u *User) CustomAvatarRelativePath() string { + return u.Avatar +} + +// GenerateRandomAvatar generates a random avatar for user. +func (u *User) GenerateRandomAvatar() error { + return u.generateRandomAvatar(x) +} + +func (u *User) generateRandomAvatar(e Engine) error { + seed := u.Email + if len(seed) == 0 { + seed = u.Name + } + + img, err := avatar.RandomImage([]byte(seed)) + if err != nil { + return fmt.Errorf("RandomImage: %v", err) + } + // NOTICE for random avatar, it still uses id as avatar name, but custom avatar use md5 + // since random image is not a user's photo, there is no security for enumable + if u.Avatar == "" { + u.Avatar = fmt.Sprintf("%d", u.ID) + } + + if err := storage.SaveFrom(storage.Avatars, u.CustomAvatarRelativePath(), func(w io.Writer) error { + if err := png.Encode(w, img); err != nil { + log.Error("Encode: %v", err) + } + return err + }); err != nil { + return fmt.Errorf("Failed to create dir %s: %v", u.CustomAvatarRelativePath(), err) + } + + if _, err := e.ID(u.ID).Cols("avatar").Update(u); err != nil { + return err + } + + log.Info("New random avatar created: %d", u.ID) + return nil +} + +// SizedRelAvatarLink returns a link to the user's avatar via +// the local explore page. Function returns immediately. +// When applicable, the link is for an avatar of the indicated size (in pixels). +func (u *User) SizedRelAvatarLink(size int) string { + return strings.TrimSuffix(setting.AppSubURL, "/") + "/user/avatar/" + u.Name + "/" + strconv.Itoa(size) +} + +// RealSizedAvatarLink returns a link to the user's avatar. When +// applicable, the link is for an avatar of the indicated size (in pixels). +// +// This function make take time to return when federated avatars +// are in use, due to a DNS lookup need +// +func (u *User) RealSizedAvatarLink(size int) string { + if u.ID == -1 { + return base.DefaultAvatarLink() + } + + switch { + case u.UseCustomAvatar: + if u.Avatar == "" { + return base.DefaultAvatarLink() + } + return setting.AppSubURL + "/avatars/" + u.Avatar + case setting.DisableGravatar, setting.OfflineMode: + if u.Avatar == "" { + if err := u.GenerateRandomAvatar(); err != nil { + log.Error("GenerateRandomAvatar: %v", err) + } + } + + return setting.AppSubURL + "/avatars/" + u.Avatar + } + return base.SizedAvatarLink(u.AvatarEmail, size) +} + +// RelAvatarLink returns a relative link to the user's avatar. The link +// may either be a sub-URL to this site, or a full URL to an external avatar +// service. +func (u *User) RelAvatarLink() string { + return u.SizedRelAvatarLink(base.DefaultAvatarSize) +} + +// AvatarLink returns user avatar absolute link. +func (u *User) AvatarLink() string { + link := u.RelAvatarLink() + if link[0] == '/' && link[1] != '/' { + return setting.AppURL + strings.TrimPrefix(link, setting.AppSubURL)[1:] + } + return link +} + +// UploadAvatar saves custom avatar for user. +// FIXME: split uploads to different subdirs in case we have massive users. +func (u *User) UploadAvatar(data []byte) error { + m, err := avatar.Prepare(data) + if err != nil { + return err + } + + sess := x.NewSession() + defer sess.Close() + if err = sess.Begin(); err != nil { + return err + } + + u.UseCustomAvatar = true + // Different users can upload same image as avatar + // If we prefix it with u.ID, it will be separated + // Otherwise, if any of the users delete his avatar + // Other users will lose their avatars too. + u.Avatar = fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("%d-%x", u.ID, md5.Sum(data))))) + if err = updateUser(sess, u); err != nil { + return fmt.Errorf("updateUser: %v", err) + } + + if err := storage.SaveFrom(storage.Avatars, u.CustomAvatarRelativePath(), func(w io.Writer) error { + if err := png.Encode(w, *m); err != nil { + log.Error("Encode: %v", err) + } + return err + }); err != nil { + return fmt.Errorf("Failed to create dir %s: %v", u.CustomAvatarRelativePath(), err) + } + + return sess.Commit() +} + +// DeleteAvatar deletes the user's custom avatar. +func (u *User) DeleteAvatar() error { + aPath := u.CustomAvatarRelativePath() + log.Trace("DeleteAvatar[%d]: %s", u.ID, aPath) + if len(u.Avatar) > 0 { + if err := storage.Avatars.Delete(aPath); err != nil { + return fmt.Errorf("Failed to remove %s: %v", aPath, err) + } + } + + u.UseCustomAvatar = false + u.Avatar = "" + if _, err := x.ID(u.ID).Cols("avatar, use_custom_avatar").Update(u); err != nil { + return fmt.Errorf("UpdateUser: %v", err) + } + return nil +} diff --git a/modules/avatar/avatar.go b/modules/avatar/avatar.go index f4c0655fa..44b56c26c 100644 --- a/modules/avatar/avatar.go +++ b/modules/avatar/avatar.go @@ -9,6 +9,7 @@ import ( "fmt" "image" "image/color/palette" + // Enable PNG support: _ "image/png" "math/rand" @@ -57,11 +58,11 @@ func Prepare(data []byte) (*image.Image, error) { if err != nil { return nil, fmt.Errorf("DecodeConfig: %v", err) } - if imgCfg.Width > setting.AvatarMaxWidth { - return nil, fmt.Errorf("Image width is too large: %d > %d", imgCfg.Width, setting.AvatarMaxWidth) + if imgCfg.Width > setting.Avatar.MaxWidth { + return nil, fmt.Errorf("Image width is too large: %d > %d", imgCfg.Width, setting.Avatar.MaxWidth) } - if imgCfg.Height > setting.AvatarMaxHeight { - return nil, fmt.Errorf("Image height is too large: %d > %d", imgCfg.Height, setting.AvatarMaxHeight) + if imgCfg.Height > setting.Avatar.MaxHeight { + return nil, fmt.Errorf("Image height is too large: %d > %d", imgCfg.Height, setting.Avatar.MaxHeight) } img, _, err := image.Decode(bytes.NewReader(data)) diff --git a/modules/avatar/avatar_test.go b/modules/avatar/avatar_test.go index 662d50fad..853560565 100644 --- a/modules/avatar/avatar_test.go +++ b/modules/avatar/avatar_test.go @@ -22,8 +22,8 @@ func Test_RandomImage(t *testing.T) { } func Test_PrepareWithPNG(t *testing.T) { - setting.AvatarMaxWidth = 4096 - setting.AvatarMaxHeight = 4096 + setting.Avatar.MaxWidth = 4096 + setting.Avatar.MaxHeight = 4096 data, err := ioutil.ReadFile("testdata/avatar.png") assert.NoError(t, err) @@ -36,8 +36,8 @@ func Test_PrepareWithPNG(t *testing.T) { } func Test_PrepareWithJPEG(t *testing.T) { - setting.AvatarMaxWidth = 4096 - setting.AvatarMaxHeight = 4096 + setting.Avatar.MaxWidth = 4096 + setting.Avatar.MaxHeight = 4096 data, err := ioutil.ReadFile("testdata/avatar.jpeg") assert.NoError(t, err) @@ -50,15 +50,15 @@ func Test_PrepareWithJPEG(t *testing.T) { } func Test_PrepareWithInvalidImage(t *testing.T) { - setting.AvatarMaxWidth = 5 - setting.AvatarMaxHeight = 5 + setting.Avatar.MaxWidth = 5 + setting.Avatar.MaxHeight = 5 _, err := Prepare([]byte{}) assert.EqualError(t, err, "DecodeConfig: image: unknown format") } func Test_PrepareWithInvalidImageSize(t *testing.T) { - setting.AvatarMaxWidth = 5 - setting.AvatarMaxHeight = 5 + setting.Avatar.MaxWidth = 5 + setting.Avatar.MaxHeight = 5 data, err := ioutil.ReadFile("testdata/avatar.png") assert.NoError(t, err) diff --git a/modules/setting/database.go b/modules/setting/database.go index d5d03c2a3..7d082d137 100644 --- a/modules/setting/database.go +++ b/modules/setting/database.go @@ -47,7 +47,8 @@ var ( ConnMaxLifetime time.Duration IterateBufferSize int }{ - Timeout: 500, + Timeout: 500, + IterateBufferSize: 50, } ) diff --git a/modules/setting/picture.go b/modules/setting/picture.go new file mode 100644 index 000000000..fa97245aa --- /dev/null +++ b/modules/setting/picture.go @@ -0,0 +1,114 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package setting + +import ( + "net/url" + + "code.gitea.io/gitea/modules/log" + + "strk.kbt.io/projects/go/libravatar" +) + +// settings +var ( + // Picture settings + Avatar = struct { + Storage + + MaxWidth int + MaxHeight int + MaxFileSize int64 + }{ + MaxWidth: 4096, + MaxHeight: 3072, + MaxFileSize: 1048576, + } + + GravatarSource string + GravatarSourceURL *url.URL + DisableGravatar bool + EnableFederatedAvatar bool + LibravatarService *libravatar.Libravatar + + RepoAvatar = struct { + Storage + + Fallback string + FallbackImage string + }{} +) + +func newPictureService() { + sec := Cfg.Section("picture") + + avatarSec := Cfg.Section("avatar") + storageType := sec.Key("AVATAR_STORAGE_TYPE").MustString("") + // Specifically default PATH to AVATAR_UPLOAD_PATH + avatarSec.Key("PATH").MustString( + sec.Key("AVATAR_UPLOAD_PATH").String()) + + Avatar.Storage = getStorage("avatars", storageType, avatarSec) + + Avatar.MaxWidth = sec.Key("AVATAR_MAX_WIDTH").MustInt(4096) + Avatar.MaxHeight = sec.Key("AVATAR_MAX_HEIGHT").MustInt(3072) + Avatar.MaxFileSize = sec.Key("AVATAR_MAX_FILE_SIZE").MustInt64(1048576) + + switch source := sec.Key("GRAVATAR_SOURCE").MustString("gravatar"); source { + case "duoshuo": + GravatarSource = "http://gravatar.duoshuo.com/avatar/" + case "gravatar": + GravatarSource = "https://secure.gravatar.com/avatar/" + case "libravatar": + GravatarSource = "https://seccdn.libravatar.org/avatar/" + default: + GravatarSource = source + } + DisableGravatar = sec.Key("DISABLE_GRAVATAR").MustBool() + EnableFederatedAvatar = sec.Key("ENABLE_FEDERATED_AVATAR").MustBool(!InstallLock) + if OfflineMode { + DisableGravatar = true + EnableFederatedAvatar = false + } + if DisableGravatar { + EnableFederatedAvatar = false + } + if EnableFederatedAvatar || !DisableGravatar { + var err error + GravatarSourceURL, err = url.Parse(GravatarSource) + if err != nil { + log.Fatal("Failed to parse Gravatar URL(%s): %v", + GravatarSource, err) + } + } + + if EnableFederatedAvatar { + LibravatarService = libravatar.New() + if GravatarSourceURL.Scheme == "https" { + LibravatarService.SetUseHTTPS(true) + LibravatarService.SetSecureFallbackHost(GravatarSourceURL.Host) + } else { + LibravatarService.SetUseHTTPS(false) + LibravatarService.SetFallbackHost(GravatarSourceURL.Host) + } + } + + newRepoAvatarService() +} + +func newRepoAvatarService() { + sec := Cfg.Section("picture") + + repoAvatarSec := Cfg.Section("repo-avatar") + storageType := sec.Key("REPOSITORY_AVATAR_STORAGE_TYPE").MustString("") + // Specifically default PATH to AVATAR_UPLOAD_PATH + repoAvatarSec.Key("PATH").MustString( + sec.Key("REPOSITORY_AVATAR_UPLOAD_PATH").String()) + + RepoAvatar.Storage = getStorage("repo-avatars", storageType, repoAvatarSec) + + RepoAvatar.Fallback = sec.Key("REPOSITORY_AVATAR_FALLBACK").MustString("none") + RepoAvatar.FallbackImage = sec.Key("REPOSITORY_AVATAR_FALLBACK_IMAGE").MustString("/img/repo_default.png") +} diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 4d8e02b9b..7ae8bb352 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -30,7 +30,6 @@ import ( "github.com/unknwon/com" gossh "golang.org/x/crypto/ssh" ini "gopkg.in/ini.v1" - "strk.kbt.io/projects/go/libravatar" ) // Scheme describes protocol types @@ -272,20 +271,6 @@ var ( DefaultEmailNotification string } - // Picture settings - AvatarUploadPath string - AvatarMaxWidth int - AvatarMaxHeight int - GravatarSource string - GravatarSourceURL *url.URL - DisableGravatar bool - EnableFederatedAvatar bool - LibravatarService *libravatar.Libravatar - AvatarMaxFileSize int64 - RepositoryAvatarUploadPath string - RepositoryAvatarFallback string - RepositoryAvatarFallbackImage string - // Log settings LogLevel string StacktraceLogLevel string @@ -864,59 +849,7 @@ func NewContext() { newRepository() - sec = Cfg.Section("picture") - AvatarUploadPath = sec.Key("AVATAR_UPLOAD_PATH").MustString(path.Join(AppDataPath, "avatars")) - forcePathSeparator(AvatarUploadPath) - if !filepath.IsAbs(AvatarUploadPath) { - AvatarUploadPath = path.Join(AppWorkPath, AvatarUploadPath) - } - RepositoryAvatarUploadPath = sec.Key("REPOSITORY_AVATAR_UPLOAD_PATH").MustString(path.Join(AppDataPath, "repo-avatars")) - forcePathSeparator(RepositoryAvatarUploadPath) - if !filepath.IsAbs(RepositoryAvatarUploadPath) { - RepositoryAvatarUploadPath = path.Join(AppWorkPath, RepositoryAvatarUploadPath) - } - RepositoryAvatarFallback = sec.Key("REPOSITORY_AVATAR_FALLBACK").MustString("none") - RepositoryAvatarFallbackImage = sec.Key("REPOSITORY_AVATAR_FALLBACK_IMAGE").MustString("/img/repo_default.png") - AvatarMaxWidth = sec.Key("AVATAR_MAX_WIDTH").MustInt(4096) - AvatarMaxHeight = sec.Key("AVATAR_MAX_HEIGHT").MustInt(3072) - AvatarMaxFileSize = sec.Key("AVATAR_MAX_FILE_SIZE").MustInt64(1048576) - switch source := sec.Key("GRAVATAR_SOURCE").MustString("gravatar"); source { - case "duoshuo": - GravatarSource = "http://gravatar.duoshuo.com/avatar/" - case "gravatar": - GravatarSource = "https://secure.gravatar.com/avatar/" - case "libravatar": - GravatarSource = "https://seccdn.libravatar.org/avatar/" - default: - GravatarSource = source - } - DisableGravatar = sec.Key("DISABLE_GRAVATAR").MustBool() - EnableFederatedAvatar = sec.Key("ENABLE_FEDERATED_AVATAR").MustBool(!InstallLock) - if OfflineMode { - DisableGravatar = true - EnableFederatedAvatar = false - } - if DisableGravatar { - EnableFederatedAvatar = false - } - if EnableFederatedAvatar || !DisableGravatar { - GravatarSourceURL, err = url.Parse(GravatarSource) - if err != nil { - log.Fatal("Failed to parse Gravatar URL(%s): %v", - GravatarSource, err) - } - } - - if EnableFederatedAvatar { - LibravatarService = libravatar.New() - if GravatarSourceURL.Scheme == "https" { - LibravatarService.SetUseHTTPS(true) - LibravatarService.SetSecureFallbackHost(GravatarSourceURL.Host) - } else { - LibravatarService.SetUseHTTPS(false) - LibravatarService.SetFallbackHost(GravatarSourceURL.Host) - } - } + newPictureService() if err = Cfg.Section("ui").MapTo(&UI); err != nil { log.Fatal("Failed to map UI settings: %v", err) diff --git a/modules/storage/storage.go b/modules/storage/storage.go index 8b1c336ae..1fa04119c 100644 --- a/modules/storage/storage.go +++ b/modules/storage/storage.go @@ -82,12 +82,32 @@ func Copy(dstStorage ObjectStorage, dstPath string, srcStorage ObjectStorage, sr return dstStorage.Save(dstPath, f) } +// SaveFrom saves data to the ObjectStorage with path p from the callback +func SaveFrom(objStorage ObjectStorage, p string, callback func(w io.Writer) error) error { + pr, pw := io.Pipe() + defer pr.Close() + go func() { + defer pw.Close() + if err := callback(pw); err != nil { + _ = pw.CloseWithError(err) + } + }() + + _, err := objStorage.Save(p, pr) + return err +} + var ( // Attachments represents attachments storage Attachments ObjectStorage // LFS represents lfs storage LFS ObjectStorage + + // Avatars represents user avatars storage + Avatars ObjectStorage + // RepoAvatars represents repository avatars storage + RepoAvatars ObjectStorage ) // Init init the stoarge @@ -96,6 +116,14 @@ func Init() error { return err } + if err := initAvatars(); err != nil { + return err + } + + if err := initRepoAvatars(); err != nil { + return err + } + return initLFS() } @@ -112,6 +140,11 @@ func NewStorage(typStr string, cfg interface{}) (ObjectStorage, error) { return fn(context.Background(), cfg) } +func initAvatars() (err error) { + Avatars, err = NewStorage(setting.Avatar.Storage.Type, setting.Avatar.Storage) + return +} + func initAttachments() (err error) { Attachments, err = NewStorage(setting.Attachment.Storage.Type, setting.Attachment.Storage) return @@ -121,3 +154,8 @@ func initLFS() (err error) { LFS, err = NewStorage(setting.LFS.Storage.Type, setting.LFS.Storage) return } + +func initRepoAvatars() (err error) { + RepoAvatars, err = NewStorage(setting.RepoAvatar.Storage.Type, setting.RepoAvatar.Storage) + return +} diff --git a/routers/repo/setting.go b/routers/repo/setting.go index 1b7552295..e4f8adc38 100644 --- a/routers/repo/setting.go +++ b/routers/repo/setting.go @@ -30,7 +30,6 @@ import ( mirror_service "code.gitea.io/gitea/services/mirror" repo_service "code.gitea.io/gitea/services/repository" - "github.com/unknwon/com" "mvdan.cc/xurls/v2" ) @@ -928,7 +927,7 @@ func UpdateAvatarSetting(ctx *context.Context, form auth.AvatarForm) error { // No avatar is uploaded and we not removing it here. // No random avatar generated here. // Just exit, no action. - if !com.IsFile(ctxRepo.CustomAvatarPath()) { + if ctxRepo.CustomAvatarRelativePath() == "" { log.Trace("No avatar was uploaded for repo: %d. Default icon will appear instead.", ctxRepo.ID) } return nil @@ -940,7 +939,7 @@ func UpdateAvatarSetting(ctx *context.Context, form auth.AvatarForm) error { } defer r.Close() - if form.Avatar.Size > setting.AvatarMaxFileSize { + if form.Avatar.Size > setting.Avatar.MaxFileSize { return errors.New(ctx.Tr("settings.uploaded_avatar_is_too_big")) } diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 97f4e5aea..a09e53efc 100644 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -7,8 +7,10 @@ package routes import ( "bytes" "encoding/gob" + "io" "net/http" "path" + "strings" "text/template" "time" @@ -21,6 +23,7 @@ import ( "code.gitea.io/gitea/modules/options" "code.gitea.io/gitea/modules/public" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/templates" "code.gitea.io/gitea/modules/validation" "code.gitea.io/gitea/routers" @@ -107,6 +110,61 @@ func RouterHandler(level log.Level) func(ctx *macaron.Context) { } } +func storageHandler(storageSetting setting.Storage, prefix string, objStore storage.ObjectStorage) macaron.Handler { + if storageSetting.ServeDirect { + return func(ctx *macaron.Context) { + req := ctx.Req.Request + if req.Method != "GET" && req.Method != "HEAD" { + return + } + + if !strings.HasPrefix(req.RequestURI, "/"+prefix) { + return + } + + rPath := strings.TrimPrefix(req.RequestURI, "/"+prefix) + u, err := objStore.URL(rPath, path.Base(rPath)) + if err != nil { + ctx.Error(500, err.Error()) + return + } + http.Redirect( + ctx.Resp, + req, + u.String(), + 301, + ) + } + } + + return func(ctx *macaron.Context) { + req := ctx.Req.Request + if req.Method != "GET" && req.Method != "HEAD" { + return + } + + if !strings.HasPrefix(req.RequestURI, "/"+prefix) { + return + } + + rPath := strings.TrimPrefix(req.RequestURI, "/"+prefix) + rPath = strings.TrimPrefix(rPath, "/") + //If we have matched and access to release or issue + fr, err := objStore.Open(rPath) + if err != nil { + ctx.Error(500, err.Error()) + return + } + defer fr.Close() + + _, err = io.Copy(ctx.Resp, fr) + if err != nil { + ctx.Error(500, err.Error()) + return + } + } +} + // NewMacaron initializes Macaron instance. func NewMacaron() *macaron.Macaron { gob.Register(&u2f.Challenge{}) @@ -149,22 +207,9 @@ func NewMacaron() *macaron.Macaron { ExpiresAfter: setting.StaticCacheTime, }, )) - m.Use(public.StaticHandler( - setting.AvatarUploadPath, - &public.Options{ - Prefix: "avatars", - SkipLogging: setting.DisableRouterLog, - ExpiresAfter: setting.StaticCacheTime, - }, - )) - m.Use(public.StaticHandler( - setting.RepositoryAvatarUploadPath, - &public.Options{ - Prefix: "repo-avatars", - SkipLogging: setting.DisableRouterLog, - ExpiresAfter: setting.StaticCacheTime, - }, - )) + + m.Use(storageHandler(setting.Avatar.Storage, "avatars", storage.Avatars)) + m.Use(storageHandler(setting.RepoAvatar.Storage, "repo-avatars", storage.RepoAvatars)) m.Use(templates.HTMLRenderer()) mailer.InitMailRender(templates.Mailer()) diff --git a/routers/user/setting/profile.go b/routers/user/setting/profile.go index fe0506946..1cb00aa77 100644 --- a/routers/user/setting/profile.go +++ b/routers/user/setting/profile.go @@ -20,7 +20,6 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" - "github.com/unknwon/com" "github.com/unknwon/i18n" ) @@ -133,7 +132,7 @@ func UpdateAvatarSetting(ctx *context.Context, form auth.AvatarForm, ctxUser *mo } defer fr.Close() - if form.Avatar.Size > setting.AvatarMaxFileSize { + if form.Avatar.Size > setting.Avatar.MaxFileSize { return errors.New(ctx.Tr("settings.uploaded_avatar_is_too_big")) } @@ -147,7 +146,7 @@ func UpdateAvatarSetting(ctx *context.Context, form auth.AvatarForm, ctxUser *mo if err = ctxUser.UploadAvatar(data); err != nil { return fmt.Errorf("UploadAvatar: %v", err) } - } else if ctxUser.UseCustomAvatar && !com.IsFile(ctxUser.CustomAvatarPath()) { + } else if ctxUser.UseCustomAvatar && ctxUser.Avatar == "" { // No avatar is uploaded but setting has been changed to enable, // generate a random one when needed. if err := ctxUser.GenerateRandomAvatar(); err != nil {