Merge pull request 'add bucket lookup type' (#2482) from jilen/forgejo:minio_bucket_lookup into forgejo

Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/2482
Reviewed-by: Earl Warren <earl-warren@noreply.codeberg.org>

.forgejo/workflows/testing.yml

@@ -43,8 +43,11 @@ jobs:
       image: 'docker.io/node:20-bookworm'
     services:
       minio:
-        image: 'docker.io/bitnami/minio:2023.8.31'
+        image: bitnami/minio:2024.2.26
+        options: >-
+          --hostname gitea.minio
         env:
+          MINIO_DOMAIN: minio
           MINIO_ROOT_USER: 123456
           MINIO_ROOT_PASSWORD: 12345678
     steps:
@@ -130,10 +133,10 @@ jobs:
       image: 'docker.io/node:20-bookworm'
     services:
       minio:
-        image: bitnami/minio:2021.3.17
+        image: bitnami/minio:2024.2.26
         env:
-          MINIO_ACCESS_KEY: 123456
+          MINIO_ROOT_USER: 123456
-          MINIO_SECRET_KEY: 12345678
+          MINIO_ROOT_PASSWORD: 12345678
       pgsql:
         image: 'docker.io/postgres:15'
         env:

docs/content/administration/config-cheat-sheet.en-us.md

@@ -842,6 +842,10 @@ Default templates for project boards:
 - `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when STORAGE_TYPE is `minio`
 - `MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey to connect only available when STORAGE_TYPE is `minio`
 - `MINIO_BUCKET`: **gitea**: Minio bucket to store the attachments only available when STORAGE_TYPE is `minio`
+- `MINIO_BUCKET_LOOKUP`: **auto**: Minio bucket lookup type only available when `STORAGE_TYPE` is `minio`
+  - `auto` Auto detect
+  - `dns`  Virtual Host Style bucket lookup
+  - `path` Path style bucket lookup
 - `MINIO_LOCATION`: **us-east-1**: Minio location to create bucket only available when STORAGE_TYPE is `minio`
 - `MINIO_BASE_PATH`: **attachments/**: Minio base path on the bucket only available when STORAGE_TYPE is `minio`
 - `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when STORAGE_TYPE is `minio`
@@ -1272,6 +1276,10 @@ is `data/lfs` and the default of `MINIO_BASE_PATH` is `lfs/`.
 - `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when `STORAGE_TYPE` is `minio`
 - `MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey to connect only available when `STORAGE_TYPE is` `minio`
 - `MINIO_BUCKET`: **gitea**: Minio bucket to store the lfs only available when `STORAGE_TYPE` is `minio`
+- `MINIO_BUCKET_LOOKUP`: **auto**: Minio bucket lookup type only available when `STORAGE_TYPE` is `minio`
+  - `auto` Auto detect
+  - `dns`  Virtual Host Style bucket lookup
+  - `path` Path style bucket lookup
 - `MINIO_LOCATION`: **us-east-1**: Minio location to create bucket only available when `STORAGE_TYPE` is `minio`
 - `MINIO_BASE_PATH`: **lfs/**: Minio base path on the bucket only available when `STORAGE_TYPE` is `minio`
 - `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when `STORAGE_TYPE` is `minio`
@@ -1287,6 +1295,10 @@ Default storage configuration for attachments, lfs, avatars, repo-avatars, repo-
 - `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when `STORAGE_TYPE` is `minio`
 - `MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey to connect only available when `STORAGE_TYPE is` `minio`
 - `MINIO_BUCKET`: **gitea**: Minio bucket to store the data only available when `STORAGE_TYPE` is `minio`
+- `MINIO_BUCKET_LOOKUP`: **auto**: Minio bucket lookup type only available when `STORAGE_TYPE` is `minio`
+  - `auto` Auto detect
+  - `dns`  Virtual Host Style bucket lookup
+  - `path` Path style bucket lookup
 - `MINIO_LOCATION`: **us-east-1**: Minio location to create bucket only available when `STORAGE_TYPE` is `minio`
 - `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when `STORAGE_TYPE` is `minio`
 - `MINIO_INSECURE_SKIP_VERIFY`: **false**: Minio skip SSL verification available when STORAGE_TYPE is `minio`
@@ -1372,6 +1384,10 @@ is `data/repo-archive` and the default of `MINIO_BASE_PATH` is `repo-archive/`.
 - `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when `STORAGE_TYPE` is `minio`
 - `MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey to connect only available when `STORAGE_TYPE is` `minio`
 - `MINIO_BUCKET`: **gitea**: Minio bucket to store the lfs only available when `STORAGE_TYPE` is `minio`
+- `MINIO_BUCKET_LOOKUP`: **auto**: Minio bucket lookup type only available when `STORAGE_TYPE` is `minio`
+  - `auto` Auto detect
+  - `dns`  Virtual Host Style bucket lookup
+  - `path` Path style bucket lookup
 - `MINIO_LOCATION`: **us-east-1**: Minio location to create bucket only available when `STORAGE_TYPE` is `minio`
 - `MINIO_BASE_PATH`: **repo-archive/**: Minio base path on the bucket only available when `STORAGE_TYPE` is `minio`
 - `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when `STORAGE_TYPE` is `minio`

docs/content/administration/config-cheat-sheet.zh-cn.md

@@ -792,6 +792,10 @@ Gitea 创建以下非唯一队列：
 - `MINIO_ACCESS_KEY_ID`: Minio accessKeyID 以连接，仅当 STORAGE_TYPE 为 `minio` 时可用。
 - `MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey 以连接，仅当 STORAGE_TYPE 为 `minio` 时可用。
 - `MINIO_BUCKET`: **gitea**: Minio 存储附件的存储桶，仅当 STORAGE_TYPE 为 `minio` 时可用。
+- `MINIO_BUCKET_LOOKUP`: **auto**: Minio 存储桶寻址方式, 仅当 `STORAGE_TYPE` 为 `minio` 时可用。
+  - `auto` 自动检测
+  - `dns`  子域名寻址
+  - `path` 路径寻址
 - `MINIO_LOCATION`: **us-east-1**: Minio 存储桶的位置以创建，仅当 STORAGE_TYPE 为 `minio` 时可用。
 - `MINIO_BASE_PATH`: **attachments/**: Minio 存储桶上的基本路径，仅当 STORAGE_TYPE 为 `minio` 时可用。
 - `MINIO_USE_SSL`: **false**: Minio 启用 SSL，仅当 STORAGE_TYPE 为 `minio` 时可用。
@@ -1207,6 +1211,10 @@ ALLOW_DATA_URI_IMAGES = true
 - `MINIO_ACCESS_KEY_ID`：Minio 的 accessKeyID，仅在 `STORAGE_TYPE` 为 `minio` 时可用。
 - `MINIO_SECRET_ACCESS_KEY`：Minio 的 secretAccessKey，仅在 `STORAGE_TYPE` 为 `minio` 时可用。
 - `MINIO_BUCKET`：**gitea**：用于存储 lfs 的 Minio 桶，仅在 `STORAGE_TYPE` 为 `minio` 时可用。
+- `MINIO_BUCKET_LOOKUP`: **auto**: Minio 存储桶寻址方式,可选值为 `auto`, `dns`, `path` 仅当 `STORAGE_TYPE` 为 `minio` 时可用。
+  - `auto` 自动检测
+  - `dns`  子域名寻址
+  - `path` 路径寻址
 - `MINIO_LOCATION`：**us-east-1**：创建桶的 Minio 位置，仅在 `STORAGE_TYPE` 为 `minio` 时可用。
 - `MINIO_BASE_PATH`：**lfs/**：桶上的 Minio 基本路径，仅在 `STORAGE_TYPE` 为 `minio` 时可用。
 - `MINIO_USE_SSL`：**false**：Minio 启用 ssl，仅在 `STORAGE_TYPE` 为 `minio` 时可用。
@@ -1222,6 +1230,10 @@ ALLOW_DATA_URI_IMAGES = true
 - `MINIO_ACCESS_KEY_ID`：Minio 的 accessKeyID，仅在 `STORAGE_TYPE` 为 `minio` 时可用。
 - `MINIO_SECRET_ACCESS_KEY`：Minio 的 secretAccessKey，仅在 `STORAGE_TYPE` 为 `minio` 时可用。
 - `MINIO_BUCKET`：**gitea**：用于存储数据的 Minio 桶，仅在 `STORAGE_TYPE` 为 `minio` 时可用。
+- `MINIO_BUCKET_LOOKUP`: **auto**: Minio 存储桶寻址方式,可选值为 `auto`, `dns`, `path` 仅当 `STORAGE_TYPE` 为 `minio` 时可用。
+  - `auto` 自动检测
+  - `dns`  子域名寻址
+  - `path` 路径寻址
 - `MINIO_LOCATION`：**us-east-1**：创建桶的 Minio 位置，仅在 `STORAGE_TYPE` 为 `minio` 时可用。
 - `MINIO_USE_SSL`：**false**：Minio 启用 ssl，仅在 `STORAGE_TYPE` 为 `minio` 时可用。
 - `MINIO_INSECURE_SKIP_VERIFY`：**false**：Minio 跳过 SSL 验证，仅在 `STORAGE_TYPE` 为 `minio` 时可用。
@@ -1305,6 +1317,10 @@ MINIO_INSECURE_SKIP_VERIFY = false
 - `MINIO_ACCESS_KEY_ID`: Minio的accessKeyID，仅在`STORAGE_TYPE`为`minio`时可用。
 - `MINIO_SECRET_ACCESS_KEY`: Minio的secretAccessKey，仅在`STORAGE_TYPE`为`minio`时可用。
 - `MINIO_BUCKET`: **gitea**：用于存储归档的Minio存储桶，仅在`STORAGE_TYPE`为`minio`时可用。
+- `MINIO_BUCKET_LOOKUP`: **auto**: Minio 存储桶寻址方式,可选值为 `auto`, `dns`, `path` 仅当 `STORAGE_TYPE` 为 `minio` 时可用。
+  - `auto` 自动检测
+  - `dns`  子域名寻址
+  - `path` 路径寻址
 - `MINIO_LOCATION`: **us-east-1**：用于创建存储桶的Minio位置，仅在`STORAGE_TYPE`为`minio`时可用。
 - `MINIO_BASE_PATH`: **repo-archive/**：存储桶上的Minio基本路径，仅在`STORAGE_TYPE`为`minio`时可用。
 - `MINIO_USE_SSL`: **false**：启用Minio的SSL，仅在`STORAGE_TYPE`为`minio`时可用。

modules/setting/storage.go

@@ -41,6 +41,7 @@ type MinioStorageConfig struct {
 	AccessKeyID        string `ini:"MINIO_ACCESS_KEY_ID" json:",omitempty"`
 	SecretAccessKey    string `ini:"MINIO_SECRET_ACCESS_KEY" json:",omitempty"`
 	Bucket             string `ini:"MINIO_BUCKET" json:",omitempty"`
+	BucketLookup       string `ini:"MINIO_BUCKET_LOOKUP" json:",omitempty"`
 	Location           string `ini:"MINIO_LOCATION" json:",omitempty"`
 	BasePath           string `ini:"MINIO_BASE_PATH" json:",omitempty"`
 	UseSSL             bool   `ini:"MINIO_USE_SSL"`
@@ -78,6 +79,7 @@ func getDefaultStorageSection(rootCfg ConfigProvider) ConfigSection {
 	storageSec.Key("MINIO_ACCESS_KEY_ID").MustString("")
 	storageSec.Key("MINIO_SECRET_ACCESS_KEY").MustString("")
 	storageSec.Key("MINIO_BUCKET").MustString("gitea")
+	storageSec.Key("MINIO_BUCKET_LOOKUP").MustString("auto")
 	storageSec.Key("MINIO_LOCATION").MustString("us-east-1")
 	storageSec.Key("MINIO_USE_SSL").MustBool(false)
 	storageSec.Key("MINIO_INSECURE_SKIP_VERIFY").MustBool(false)

modules/storage/minio.go

@@ -82,14 +82,26 @@ func NewMinioStorage(ctx context.Context, cfg *setting.Storage) (ObjectStorage,
 	if config.ChecksumAlgorithm != "" && config.ChecksumAlgorithm != "default" && config.ChecksumAlgorithm != "md5" {
 		return nil, fmt.Errorf("invalid minio checksum algorithm: %s", config.ChecksumAlgorithm)
 	}
+	var lookup minio.BucketLookupType
+	switch config.BucketLookup {
+	case "auto", "":
+		lookup = minio.BucketLookupAuto
+	case "dns":
+		lookup = minio.BucketLookupDNS
+	case "path":
+		lookup = minio.BucketLookupPath
+	default:
+		return nil, fmt.Errorf("invalid minio bucket lookup type %s", config.BucketLookup)
+	}
 
 	log.Info("Creating Minio storage at %s:%s with base path %s", config.Endpoint, config.Bucket, config.BasePath)
 
 	minioClient, err := minio.New(config.Endpoint, &minio.Options{
-		Creds:     credentials.NewStaticV4(config.AccessKeyID, config.SecretAccessKey, ""),
+		Creds:        credentials.NewStaticV4(config.AccessKeyID, config.SecretAccessKey, ""),
-		Secure:    config.UseSSL,
+		Secure:       config.UseSSL,
-		Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: config.InsecureSkipVerify}},
+		Transport:    &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: config.InsecureSkipVerify}},
-		Region:    config.Location,
+		Region:       config.Location,
+		BucketLookup: lookup,
 	})
 	if err != nil {
 		return nil, convertMinioErr(err)

modules/storage/minio_test.go

@@ -31,6 +31,23 @@ func TestMinioStorageIterator(t *testing.T) {
 	})
 }
 
+func TestVirtualHostMinioStorage(t *testing.T) {
+	if os.Getenv("CI") == "" {
+		t.Skip("minioStorage not present outside of CI")
+		return
+	}
+	testStorageIterator(t, setting.MinioStorageType, &setting.Storage{
+		MinioConfig: setting.MinioStorageConfig{
+			Endpoint:        "minio:9000",
+			AccessKeyID:     "123456",
+			SecretAccessKey: "12345678",
+			Bucket:          "gitea",
+			Location:        "us-east-1",
+			BucketLookup:    "dns",
+		},
+	})
+}
+
 func TestMinioStoragePath(t *testing.T) {
 	m := &MinioStorage{basePath: ""}
 	assert.Equal(t, "", m.buildMinioPath("/"))