Add package registry cleanup rules (#21658)
Fixes #20514 Fixes #20766 Fixes #20631 This PR adds Cleanup Rules for the package registry. This allows to delete unneeded packages automatically. Cleanup rules can be set up from the user or org settings. Please have a look at the documentation because I'm not a native english speaker. Rule Form  Rule List  Rule Preview  Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
This commit is contained in:
KN4CK3R
GitHub
parent
d3f850cc0e
commit
32db62515f
|
|
@ -0,0 +1,84 @@
|
|||
---
|
||||
date: "2022-11-01T00:00:00+00:00"
|
||||
title: "Storage"
|
||||
slug: "packages/storage"
|
||||
draft: false
|
||||
toc: false
|
||||
menu:
|
||||
sidebar:
|
||||
parent: "packages"
|
||||
name: "storage"
|
||||
weight: 5
|
||||
identifier: "storage"
|
||||
---
|
||||
|
||||
# Storage
|
||||
|
||||
This document describes the storage of the package registry and how it can be managed.
|
||||
|
||||
**Table of Contents**
|
||||
|
||||
{{< toc >}}
|
||||
|
||||
## Deduplication
|
||||
|
||||
The package registry has a build-in deduplication of uploaded blobs.
|
||||
If two identical files are uploaded only one blob is saved on the filesystem.
|
||||
This ensures no space is wasted for duplicated files.
|
||||
|
||||
If two packages are uploaded with identical files, both packages will display the same size but on the filesystem they require only half of the size.
|
||||
Whenever a package gets deleted only the references to the underlaying blobs are removed.
|
||||
The blobs get not removed at this moment, so they still require space on the filesystem.
|
||||
When a new package gets uploaded the existing blobs may get referenced again.
|
||||
|
||||
These unreferenced blobs get deleted by a [clean up job]({{< relref "doc/advanced/config-cheat-sheet.en-us.md#cron---cleanup-expired-packages-croncleanup_packages" >}}).
|
||||
The config setting `OLDER_THAN` configures how long unreferenced blobs are kept before they get deleted.
|
||||
|
||||
## Cleanup Rules
|
||||
|
||||
Package registries can become large over time without cleanup.
|
||||
It's recommended to delete unnecessary packages and set up cleanup rules to automatically manage the package registry usage.
|
||||
Every package owner (user or organization) manages the cleanup rules which are applied to their packages.
|
||||
|
||||
|Setting|Description|
|
||||
|-|-|
|
||||
|Enabled|Turn the cleanup rule on or off.|
|
||||
|Type|Every rule manages a specific package type.|
|
||||
|Apply pattern to full package name|If enabled, the patterns below are applied to the full package name (`package/version`). Otherwise only the version (`version`) is used.|
|
||||
|Keep the most recent|How many versions to *always* keep for each package.|
|
||||
|Keep versions matching|The regex pattern that determines which versions to keep. An empty pattern keeps no version while `.+` keeps all versions. The container registry will always keep the `latest` version even if not configured.|
|
||||
|Remove versions older than|Remove only versions older than the selected days.|
|
||||
|Remove versions matching|The regex pattern that determines which versions to remove. An empty pattern or `.+` leads to the removal of every package if no other setting tells otherwise.|
|
||||
|
||||
Every cleanup rule can show a preview of the affected packages.
|
||||
This can be used to check if the cleanup rules is proper configured.
|
||||
|
||||
### Regex examples
|
||||
|
||||
Regex patterns are automatically surrounded with `\A` and `\z` anchors.
|
||||
Do not include any `\A`, `\z`, `^` or `$` token in the regex patterns as they are not necessary.
|
||||
The patterns are case-insensitive which matches the behaviour of the package registry in Gitea.
|
||||
|
||||
|Pattern|Description|
|
||||
|-|-|
|
||||
|`.*`|Match every possible version.|
|
||||
|`v.+`|Match versions that start with `v`.|
|
||||
|`release`|Match only the version `release`.|
|
||||
|`release.*`|Match versions that are either named or start with `release`.|
|
||||
|`.+-temp-.+`|Match versions that contain `-temp-`.|
|
||||
|`v.+\|release`|Match versions that either start with `v` or are named `release`.|
|
||||
|`package/v.+\|other/release`|Match versions of the package `package` that start with `v` or the version `release` of the package `other`. This needs the setting *Apply pattern to full package name* enabled.|
|
||||
|
||||
### How the cleanup rules work
|
||||
|
||||
The cleanup rules are part of the [clean up job]({{< relref "doc/advanced/config-cheat-sheet.en-us.md#cron---cleanup-expired-packages-croncleanup_packages" >}}) and run periodicly.
|
||||
|
||||
The cleanup rule:
|
||||
|
||||
1. Collects all packages of the package type for the owners registry.
|
||||
1. For every package it collects all versions.
|
||||
1. Excludes from the list the # versions based on the *Keep the most recent* value.
|
||||
1. Excludes from the list any versions matching the *Keep versions matching* value.
|
||||
1. Excludes from the list the versions more recent than the *Remove versions older than* value.
|
||||
1. Excludes from the list any versions not matching the *Remove versions matching* value.
|
||||
1. Deletes the remaining versions.
|
||||
|
|
@ -439,6 +439,8 @@ var migrations = []Migration{
|
|||
NewMigration("Alter package_version.metadata_json to LONGTEXT", v1_19.AlterPackageVersionMetadataToLongText),
|
||||
// v233 -> v234
|
||||
NewMigration("Add header_authorization_encrypted column to webhook table", v1_19.AddHeaderAuthorizationEncryptedColWebhook),
|
||||
// v234 -> v235
|
||||
NewMigration("Add package cleanup rule table", v1_19.CreatePackageCleanupRuleTable),
|
||||
}
|
||||
|
||||
// GetCurrentDBVersion returns the current db version
|
||||
|
|
|
|||
|
|
@ -0,0 +1,29 @@
|
|||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package v1_19 //nolint
|
||||
|
||||
import (
|
||||
"code.gitea.io/gitea/modules/timeutil"
|
||||
|
||||
"xorm.io/xorm"
|
||||
)
|
||||
|
||||
func CreatePackageCleanupRuleTable(x *xorm.Engine) error {
|
||||
type PackageCleanupRule struct {
|
||||
ID int64 `xorm:"pk autoincr"`
|
||||
Enabled bool `xorm:"INDEX NOT NULL DEFAULT false"`
|
||||
OwnerID int64 `xorm:"UNIQUE(s) INDEX NOT NULL DEFAULT 0"`
|
||||
Type string `xorm:"UNIQUE(s) INDEX NOT NULL"`
|
||||
KeepCount int `xorm:"NOT NULL DEFAULT 0"`
|
||||
KeepPattern string `xorm:"NOT NULL DEFAULT ''"`
|
||||
RemoveDays int `xorm:"NOT NULL DEFAULT 0"`
|
||||
RemovePattern string `xorm:"NOT NULL DEFAULT ''"`
|
||||
MatchFullName bool `xorm:"NOT NULL DEFAULT false"`
|
||||
CreatedUnix timeutil.TimeStamp `xorm:"created NOT NULL DEFAULT 0"`
|
||||
UpdatedUnix timeutil.TimeStamp `xorm:"updated NOT NULL DEFAULT 0"`
|
||||
}
|
||||
|
||||
return x.Sync2(new(PackageCleanupRule))
|
||||
}
|
||||
|
|
@ -45,6 +45,21 @@ const (
|
|||
TypeVagrant Type = "vagrant"
|
||||
)
|
||||
|
||||
var TypeList = []Type{
|
||||
TypeComposer,
|
||||
TypeConan,
|
||||
TypeContainer,
|
||||
TypeGeneric,
|
||||
TypeHelm,
|
||||
TypeMaven,
|
||||
TypeNpm,
|
||||
TypeNuGet,
|
||||
TypePub,
|
||||
TypePyPI,
|
||||
TypeRubyGems,
|
||||
TypeVagrant,
|
||||
}
|
||||
|
||||
// Name gets the name of the package type
|
||||
func (pt Type) Name() string {
|
||||
switch pt {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,110 @@
|
|||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package packages
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
|
||||
"code.gitea.io/gitea/models/db"
|
||||
"code.gitea.io/gitea/modules/timeutil"
|
||||
|
||||
"xorm.io/builder"
|
||||
)
|
||||
|
||||
var ErrPackageCleanupRuleNotExist = errors.New("Package blob does not exist")
|
||||
|
||||
func init() {
|
||||
db.RegisterModel(new(PackageCleanupRule))
|
||||
}
|
||||
|
||||
// PackageCleanupRule represents a rule which describes when to clean up package versions
|
||||
type PackageCleanupRule struct {
|
||||
ID int64 `xorm:"pk autoincr"`
|
||||
Enabled bool `xorm:"INDEX NOT NULL DEFAULT false"`
|
||||
OwnerID int64 `xorm:"UNIQUE(s) INDEX NOT NULL DEFAULT 0"`
|
||||
Type Type `xorm:"UNIQUE(s) INDEX NOT NULL"`
|
||||
KeepCount int `xorm:"NOT NULL DEFAULT 0"`
|
||||
KeepPattern string `xorm:"NOT NULL DEFAULT ''"`
|
||||
KeepPatternMatcher *regexp.Regexp `xorm:"-"`
|
||||
RemoveDays int `xorm:"NOT NULL DEFAULT 0"`
|
||||
RemovePattern string `xorm:"NOT NULL DEFAULT ''"`
|
||||
RemovePatternMatcher *regexp.Regexp `xorm:"-"`
|
||||
MatchFullName bool `xorm:"NOT NULL DEFAULT false"`
|
||||
CreatedUnix timeutil.TimeStamp `xorm:"created NOT NULL DEFAULT 0"`
|
||||
UpdatedUnix timeutil.TimeStamp `xorm:"updated NOT NULL DEFAULT 0"`
|
||||
}
|
||||
|
||||
func (pcr *PackageCleanupRule) CompiledPattern() error {
|
||||
if pcr.KeepPatternMatcher != nil || pcr.RemovePatternMatcher != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if pcr.KeepPattern != "" {
|
||||
var err error
|
||||
pcr.KeepPatternMatcher, err = regexp.Compile(fmt.Sprintf(`(?i)\A%s\z`, pcr.KeepPattern))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if pcr.RemovePattern != "" {
|
||||
var err error
|
||||
pcr.RemovePatternMatcher, err = regexp.Compile(fmt.Sprintf(`(?i)\A%s\z`, pcr.RemovePattern))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func InsertCleanupRule(ctx context.Context, pcr *PackageCleanupRule) (*PackageCleanupRule, error) {
|
||||
return pcr, db.Insert(ctx, pcr)
|
||||
}
|
||||
|
||||
func GetCleanupRuleByID(ctx context.Context, id int64) (*PackageCleanupRule, error) {
|
||||
pcr := &PackageCleanupRule{}
|
||||
|
||||
has, err := db.GetEngine(ctx).ID(id).Get(pcr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !has {
|
||||
return nil, ErrPackageCleanupRuleNotExist
|
||||
}
|
||||
return pcr, nil
|
||||
}
|
||||
|
||||
func UpdateCleanupRule(ctx context.Context, pcr *PackageCleanupRule) error {
|
||||
_, err := db.GetEngine(ctx).ID(pcr.ID).AllCols().Update(pcr)
|
||||
return err
|
||||
}
|
||||
|
||||
func GetCleanupRulesByOwner(ctx context.Context, ownerID int64) ([]*PackageCleanupRule, error) {
|
||||
pcrs := make([]*PackageCleanupRule, 0, 10)
|
||||
return pcrs, db.GetEngine(ctx).Where("owner_id = ?", ownerID).Find(&pcrs)
|
||||
}
|
||||
|
||||
func DeleteCleanupRuleByID(ctx context.Context, ruleID int64) error {
|
||||
_, err := db.GetEngine(ctx).ID(ruleID).Delete(&PackageCleanupRule{})
|
||||
return err
|
||||
}
|
||||
|
||||
func HasOwnerCleanupRuleForPackageType(ctx context.Context, ownerID int64, packageType Type) (bool, error) {
|
||||
return db.GetEngine(ctx).
|
||||
Where("owner_id = ? AND type = ?", ownerID, packageType).
|
||||
Exist(&PackageCleanupRule{})
|
||||
}
|
||||
|
||||
func IterateEnabledCleanupRules(ctx context.Context, callback func(context.Context, *PackageCleanupRule) error) error {
|
||||
return db.Iterate(
|
||||
ctx,
|
||||
builder.Eq{"enabled": true},
|
||||
callback,
|
||||
)
|
||||
}
|
||||
|
|
@ -320,6 +320,15 @@ func SearchLatestVersions(ctx context.Context, opts *PackageSearchOptions) ([]*P
|
|||
return pvs, count, err
|
||||
}
|
||||
|
||||
// ExistVersion checks if a version matching the search options exist
|
||||
func ExistVersion(ctx context.Context, opts *PackageSearchOptions) (bool, error) {
|
||||
return db.GetEngine(ctx).
|
||||
Where(opts.toConds()).
|
||||
Table("package_version").
|
||||
Join("INNER", "package", "package.id = package_version.package_id").
|
||||
Exist(new(PackageVersion))
|
||||
}
|
||||
|
||||
// CountVersions counts all versions of packages matching the search options
|
||||
func CountVersions(ctx context.Context, opts *PackageSearchOptions) (int64, error) {
|
||||
return db.GetEngine(ctx).
|
||||
|
|
|
|||
|
|
@ -86,6 +86,9 @@ remove = Remove
|
|||
remove_all = Remove All
|
||||
edit = Edit
|
||||
|
||||
enabled = Enabled
|
||||
disabled = Disabled
|
||||
|
||||
copy = Copy
|
||||
copy_url = Copy URL
|
||||
copy_content = Copy content
|
||||
|
|
@ -3186,3 +3189,23 @@ settings.delete.description = Deleting a package is permanent and cannot be undo
|
|||
settings.delete.notice = You are about to delete %s (%s). This operation is irreversible, are you sure?
|
||||
settings.delete.success = The package has been deleted.
|
||||
settings.delete.error = Failed to delete the package.
|
||||
owner.settings.cleanuprules.title = Manage Cleanup Rules
|
||||
owner.settings.cleanuprules.add = Add Cleanup Rule
|
||||
owner.settings.cleanuprules.edit = Edit Cleanup Rule
|
||||
owner.settings.cleanuprules.none = No cleanup rules available. Read the docs to learn more.
|
||||
owner.settings.cleanuprules.preview = Cleanup Rule Preview
|
||||
owner.settings.cleanuprules.preview.overview = %d packages are scheduled to be removed.
|
||||
owner.settings.cleanuprules.preview.none = Cleanup rule does not match any packages.
|
||||
owner.settings.cleanuprules.enabled = Enabled
|
||||
owner.settings.cleanuprules.pattern_full_match = Apply pattern to full package name
|
||||
owner.settings.cleanuprules.keep.title = Versions that match these rules are kept, even if they match a removal rule below.
|
||||
owner.settings.cleanuprules.keep.count = Keep the most recent
|
||||
owner.settings.cleanuprules.keep.count.1 = 1 version per package
|
||||
owner.settings.cleanuprules.keep.count.n = %d versions per package
|
||||
owner.settings.cleanuprules.keep.pattern = Keep versions matching
|
||||
owner.settings.cleanuprules.keep.pattern.container = The <code>latest</code> version is always kept for Container packages.
|
||||
owner.settings.cleanuprules.remove.title = Versions that match these rules are removed, unless a rule above says to keep them.
|
||||
owner.settings.cleanuprules.remove.days = Remove versions older than
|
||||
owner.settings.cleanuprules.remove.pattern = Remove versions matching
|
||||
owner.settings.cleanuprules.success.update = Cleanup rule has been updated.
|
||||
owner.settings.cleanuprules.success.delete = Cleanup rule has been deleted.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,87 @@
|
|||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package org
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"code.gitea.io/gitea/modules/base"
|
||||
"code.gitea.io/gitea/modules/context"
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
shared "code.gitea.io/gitea/routers/web/shared/packages"
|
||||
)
|
||||
|
||||
const (
|
||||
tplSettingsPackages base.TplName = "org/settings/packages"
|
||||
tplSettingsPackagesRuleEdit base.TplName = "org/settings/packages_cleanup_rules_edit"
|
||||
tplSettingsPackagesRulePreview base.TplName = "org/settings/packages_cleanup_rules_preview"
|
||||
)
|
||||
|
||||
func Packages(ctx *context.Context) {
|
||||
ctx.Data["Title"] = ctx.Tr("packages.title")
|
||||
ctx.Data["PageIsOrgSettings"] = true
|
||||
ctx.Data["PageIsSettingsPackages"] = true
|
||||
|
||||
shared.SetPackagesContext(ctx, ctx.ContextUser)
|
||||
|
||||
ctx.HTML(http.StatusOK, tplSettingsPackages)
|
||||
}
|
||||
|
||||
func PackagesRuleAdd(ctx *context.Context) {
|
||||
ctx.Data["Title"] = ctx.Tr("packages.title")
|
||||
ctx.Data["PageIsOrgSettings"] = true
|
||||
ctx.Data["PageIsSettingsPackages"] = true
|
||||
|
||||
shared.SetRuleAddContext(ctx)
|
||||
|
||||
ctx.HTML(http.StatusOK, tplSettingsPackagesRuleEdit)
|
||||
}
|
||||
|
||||
func PackagesRuleEdit(ctx *context.Context) {
|
||||
ctx.Data["Title"] = ctx.Tr("packages.title")
|
||||
ctx.Data["PageIsOrgSettings"] = true
|
||||
ctx.Data["PageIsSettingsPackages"] = true
|
||||
|
||||
shared.SetRuleEditContext(ctx, ctx.ContextUser)
|
||||
|
||||
ctx.HTML(http.StatusOK, tplSettingsPackagesRuleEdit)
|
||||
}
|
||||
|
||||
func PackagesRuleAddPost(ctx *context.Context) {
|
||||
ctx.Data["Title"] = ctx.Tr("packages.title")
|
||||
ctx.Data["PageIsOrgSettings"] = true
|
||||
ctx.Data["PageIsSettingsPackages"] = true
|
||||
|
||||
shared.PerformRuleAddPost(
|
||||
ctx,
|
||||
ctx.ContextUser,
|
||||
fmt.Sprintf("%s/org/%s/settings/packages", setting.AppSubURL, ctx.ContextUser.Name),
|
||||
tplSettingsPackagesRuleEdit,
|
||||
)
|
||||
}
|
||||
|
||||
func PackagesRuleEditPost(ctx *context.Context) {
|
||||
ctx.Data["Title"] = ctx.Tr("packages.title")
|
||||
ctx.Data["PageIsOrgSettings"] = true
|
||||
ctx.Data["PageIsSettingsPackages"] = true
|
||||
|
||||
shared.PerformRuleEditPost(
|
||||
ctx,
|
||||
ctx.ContextUser,
|
||||
fmt.Sprintf("%s/org/%s/settings/packages", setting.AppSubURL, ctx.ContextUser.Name),
|
||||
tplSettingsPackagesRuleEdit,
|
||||
)
|
||||
}
|
||||
|
||||
func PackagesRulePreview(ctx *context.Context) {
|
||||
ctx.Data["Title"] = ctx.Tr("packages.title")
|
||||
ctx.Data["PageIsOrgSettings"] = true
|
||||
ctx.Data["PageIsSettingsPackages"] = true
|
||||
|
||||
shared.SetRulePreviewContext(ctx, ctx.ContextUser)
|
||||
|
||||
ctx.HTML(http.StatusOK, tplSettingsPackagesRulePreview)
|
||||
}
|
||||
|
|
@ -0,0 +1,226 @@
|
|||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package packages
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"code.gitea.io/gitea/models/db"
|
||||
packages_model "code.gitea.io/gitea/models/packages"
|
||||
user_model "code.gitea.io/gitea/models/user"
|
||||
"code.gitea.io/gitea/modules/base"
|
||||
"code.gitea.io/gitea/modules/context"
|
||||
"code.gitea.io/gitea/modules/util"
|
||||
"code.gitea.io/gitea/modules/web"
|
||||
"code.gitea.io/gitea/services/forms"
|
||||
container_service "code.gitea.io/gitea/services/packages/container"
|
||||
)
|
||||
|
||||
func SetPackagesContext(ctx *context.Context, owner *user_model.User) {
|
||||
pcrs, err := packages_model.GetCleanupRulesByOwner(ctx, owner.ID)
|
||||
if err != nil {
|
||||
ctx.ServerError("GetCleanupRulesByOwner", err)
|
||||
return
|
||||
}
|
||||
|
||||
ctx.Data["CleanupRules"] = pcrs
|
||||
}
|
||||
|
||||
func SetRuleAddContext(ctx *context.Context) {
|
||||
setRuleEditContext(ctx, nil)
|
||||
}
|
||||
|
||||
func SetRuleEditContext(ctx *context.Context, owner *user_model.User) {
|
||||
pcr := getCleanupRuleByContext(ctx, owner)
|
||||
if pcr == nil {
|
||||
return
|
||||
}
|
||||
|
||||
setRuleEditContext(ctx, pcr)
|
||||
}
|
||||
|
||||
func setRuleEditContext(ctx *context.Context, pcr *packages_model.PackageCleanupRule) {
|
||||
ctx.Data["IsEditRule"] = pcr != nil
|
||||
|
||||
if pcr == nil {
|
||||
pcr = &packages_model.PackageCleanupRule{}
|
||||
}
|
||||
ctx.Data["CleanupRule"] = pcr
|
||||
ctx.Data["AvailableTypes"] = packages_model.TypeList
|
||||
}
|
||||
|
||||
func PerformRuleAddPost(ctx *context.Context, owner *user_model.User, redirectURL string, template base.TplName) {
|
||||
performRuleEditPost(ctx, owner, nil, redirectURL, template)
|
||||
}
|
||||
|
||||
func PerformRuleEditPost(ctx *context.Context, owner *user_model.User, redirectURL string, template base.TplName) {
|
||||
pcr := getCleanupRuleByContext(ctx, owner)
|
||||
if pcr == nil {
|
||||
return
|
||||
}
|
||||
|
||||
form := web.GetForm(ctx).(*forms.PackageCleanupRuleForm)
|
||||
|
||||
if form.Action == "remove" {
|
||||
if err := packages_model.DeleteCleanupRuleByID(ctx, pcr.ID); err != nil {
|
||||
ctx.ServerError("DeleteCleanupRuleByID", err)
|
||||
return
|
||||
}
|
||||
|
||||
ctx.Flash.Success(ctx.Tr("packages.owner.settings.cleanuprules.success.delete"))
|
||||
ctx.Redirect(redirectURL)
|
||||
} else {
|
||||
performRuleEditPost(ctx, owner, pcr, redirectURL, template)
|
||||
}
|
||||
}
|
||||
|
||||
func performRuleEditPost(ctx *context.Context, owner *user_model.User, pcr *packages_model.PackageCleanupRule, redirectURL string, template base.TplName) {
|
||||
isEditRule := pcr != nil
|
||||
|
||||
if pcr == nil {
|
||||
pcr = &packages_model.PackageCleanupRule{}
|
||||
}
|
||||
|
||||
form := web.GetForm(ctx).(*forms.PackageCleanupRuleForm)
|
||||
|
||||
pcr.Enabled = form.Enabled
|
||||
pcr.OwnerID = owner.ID
|
||||
pcr.KeepCount = form.KeepCount
|
||||
pcr.KeepPattern = form.KeepPattern
|
||||
pcr.RemoveDays = form.RemoveDays
|
||||
pcr.RemovePattern = form.RemovePattern
|
||||
pcr.MatchFullName = form.MatchFullName
|
||||
|
||||
ctx.Data["IsEditRule"] = isEditRule
|
||||
ctx.Data["CleanupRule"] = pcr
|
||||
ctx.Data["AvailableTypes"] = packages_model.TypeList
|
||||
|
||||
if ctx.HasError() {
|
||||
ctx.HTML(http.StatusOK, template)
|
||||
return
|
||||
}
|
||||
|
||||
if isEditRule {
|
||||
if err := packages_model.UpdateCleanupRule(ctx, pcr); err != nil {
|
||||
ctx.ServerError("UpdateCleanupRule", err)
|
||||
return
|
||||
}
|
||||
} else {
|
||||
pcr.Type = packages_model.Type(form.Type)
|
||||
|
||||
if has, err := packages_model.HasOwnerCleanupRuleForPackageType(ctx, owner.ID, pcr.Type); err != nil {
|
||||
ctx.ServerError("HasOwnerCleanupRuleForPackageType", err)
|
||||
return
|
||||
} else if has {
|
||||
ctx.Data["Err_Type"] = true
|
||||
ctx.HTML(http.StatusOK, template)
|
||||
return
|
||||
}
|
||||
|
||||
var err error
|
||||
if pcr, err = packages_model.InsertCleanupRule(ctx, pcr); err != nil {
|
||||
ctx.ServerError("InsertCleanupRule", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
ctx.Flash.Success(ctx.Tr("packages.owner.settings.cleanuprules.success.update"))
|
||||
ctx.Redirect(fmt.Sprintf("%s/rules/%d", redirectURL, pcr.ID))
|
||||
}
|
||||
|
||||
func SetRulePreviewContext(ctx *context.Context, owner *user_model.User) {
|
||||
pcr := getCleanupRuleByContext(ctx, owner)
|
||||
if pcr == nil {
|
||||
return
|
||||
}
|
||||
|
||||
if err := pcr.CompiledPattern(); err != nil {
|
||||
ctx.ServerError("CompiledPattern", err)
|
||||
return
|
||||
}
|
||||
|
||||
olderThan := time.Now().AddDate(0, 0, -pcr.RemoveDays)
|
||||
|
||||
packages, err := packages_model.GetPackagesByType(ctx, pcr.OwnerID, pcr.Type)
|
||||
if err != nil {
|
||||
ctx.ServerError("GetPackagesByType", err)
|
||||
return
|
||||
}
|
||||
|
||||
versionsToRemove := make([]*packages_model.PackageDescriptor, 0, 10)
|
||||
|
||||
for _, p := range packages {
|
||||
pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{
|
||||
PackageID: p.ID,
|
||||
IsInternal: util.OptionalBoolFalse,
|
||||
Sort: packages_model.SortCreatedDesc,
|
||||
Paginator: db.NewAbsoluteListOptions(pcr.KeepCount, 200),
|
||||
})
|
||||
if err != nil {
|
||||
ctx.ServerError("SearchVersions", err)
|
||||
return
|
||||
}
|
||||
for _, pv := range pvs {
|
||||
if skip, err := container_service.ShouldBeSkipped(ctx, pcr, p, pv); err != nil {
|
||||
ctx.ServerError("ShouldBeSkipped", err)
|
||||
return
|
||||
} else if skip {
|
||||
continue
|
||||
}
|
||||
|
||||
toMatch := pv.LowerVersion
|
||||
if pcr.MatchFullName {
|
||||
toMatch = p.LowerName + "/" + pv.LowerVersion
|
||||
}
|
||||
|
||||
if pcr.KeepPatternMatcher != nil && pcr.KeepPatternMatcher.MatchString(toMatch) {
|
||||
continue
|
||||
}
|
||||
if pv.CreatedUnix.AsLocalTime().After(olderThan) {
|
||||
continue
|
||||
}
|
||||
if pcr.RemovePatternMatcher != nil && !pcr.RemovePatternMatcher.MatchString(toMatch) {
|
||||
continue
|
||||
}
|
||||
|
||||
pd, err := packages_model.GetPackageDescriptor(ctx, pv)
|
||||
if err != nil {
|
||||
ctx.ServerError("GetPackageDescriptor", err)
|
||||
return
|
||||
}
|
||||
versionsToRemove = append(versionsToRemove, pd)
|
||||
}
|
||||
}
|
||||
|
||||
ctx.Data["CleanupRule"] = pcr
|
||||
ctx.Data["VersionsToRemove"] = versionsToRemove
|
||||
}
|
||||
|
||||
func getCleanupRuleByContext(ctx *context.Context, owner *user_model.User) *packages_model.PackageCleanupRule {
|
||||
id := ctx.FormInt64("id")
|
||||
if id == 0 {
|
||||
id = ctx.ParamsInt64("id")
|
||||
}
|
||||
|
||||
pcr, err := packages_model.GetCleanupRuleByID(ctx, id)
|
||||
if err != nil {
|
||||
if err == packages_model.ErrPackageCleanupRuleNotExist {
|
||||
ctx.NotFound("", err)
|
||||
} else {
|
||||
ctx.ServerError("GetCleanupRuleByID", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if pcr != nil && pcr.OwnerID == owner.ID {
|
||||
return pcr
|
||||
}
|
||||
|
||||
ctx.NotFound("", fmt.Errorf("PackageCleanupRule[%v] not associated to owner %v", id, owner))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,80 @@
|
|||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package setting
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"code.gitea.io/gitea/modules/base"
|
||||
"code.gitea.io/gitea/modules/context"
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
shared "code.gitea.io/gitea/routers/web/shared/packages"
|
||||
)
|
||||
|
||||
const (
|
||||
tplSettingsPackages base.TplName = "user/settings/packages"
|
||||
tplSettingsPackagesRuleEdit base.TplName = "user/settings/packages_cleanup_rules_edit"
|
||||
tplSettingsPackagesRulePreview base.TplName = "user/settings/packages_cleanup_rules_preview"
|
||||
)
|
||||
|
||||
func Packages(ctx *context.Context) {
|
||||
ctx.Data["Title"] = ctx.Tr("packages.title")
|
||||
ctx.Data["PageIsSettingsPackages"] = true
|
||||
|
||||
shared.SetPackagesContext(ctx, ctx.Doer)
|
||||
|
||||
ctx.HTML(http.StatusOK, tplSettingsPackages)
|
||||
}
|
||||
|
||||
func PackagesRuleAdd(ctx *context.Context) {
|
||||
ctx.Data["Title"] = ctx.Tr("packages.title")
|
||||
ctx.Data["PageIsSettingsPackages"] = true
|
||||
|
||||
shared.SetRuleAddContext(ctx)
|
||||
|
||||
ctx.HTML(http.StatusOK, tplSettingsPackagesRuleEdit)
|
||||
}
|
||||
|
||||
func PackagesRuleEdit(ctx *context.Context) {
|
||||
ctx.Data["Title"] = ctx.Tr("packages.title")
|
||||
ctx.Data["PageIsSettingsPackages"] = true
|
||||
|
||||
shared.SetRuleEditContext(ctx, ctx.Doer)
|
||||
|
||||
ctx.HTML(http.StatusOK, tplSettingsPackagesRuleEdit)
|
||||
}
|
||||
|
||||
func PackagesRuleAddPost(ctx *context.Context) {
|
||||
ctx.Data["Title"] = ctx.Tr("settings")
|
||||
ctx.Data["PageIsSettingsPackages"] = true
|
||||
|
||||
shared.PerformRuleAddPost(
|
||||
ctx,
|
||||
ctx.Doer,
|
||||
setting.AppSubURL+"/user/settings/packages",
|
||||
tplSettingsPackagesRuleEdit,
|
||||
)
|
||||
}
|
||||
|
||||
func PackagesRuleEditPost(ctx *context.Context) {
|
||||
ctx.Data["Title"] = ctx.Tr("packages.title")
|
||||
ctx.Data["PageIsSettingsPackages"] = true
|
||||
|
||||
shared.PerformRuleEditPost(
|
||||
ctx,
|
||||
ctx.Doer,
|
||||
setting.AppSubURL+"/user/settings/packages",
|
||||
tplSettingsPackagesRuleEdit,
|
||||
)
|
||||
}
|
||||
|
||||
func PackagesRulePreview(ctx *context.Context) {
|
||||
ctx.Data["Title"] = ctx.Tr("packages.title")
|
||||
ctx.Data["PageIsSettingsPackages"] = true
|
||||
|
||||
shared.SetRulePreviewContext(ctx, ctx.Doer)
|
||||
|
||||
ctx.HTML(http.StatusOK, tplSettingsPackagesRulePreview)
|
||||
}
|
||||
|
|
@ -303,6 +303,13 @@ func RegisterRoutes(m *web.Route) {
|
|||
}
|
||||
}
|
||||
|
||||
packagesEnabled := func(ctx *context.Context) {
|
||||
if !setting.Packages.Enabled {
|
||||
ctx.Error(http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// FIXME: not all routes need go through same middleware.
|
||||
// Especially some AJAX requests, we can reduce middleware number to improve performance.
|
||||
// Routers.
|
||||
|
|
@ -443,12 +450,27 @@ func RegisterRoutes(m *web.Route) {
|
|||
m.Combo("/keys").Get(user_setting.Keys).
|
||||
Post(bindIgnErr(forms.AddKeyForm{}), user_setting.KeysPost)
|
||||
m.Post("/keys/delete", user_setting.DeleteKey)
|
||||
m.Group("/packages", func() {
|
||||
m.Get("", user_setting.Packages)
|
||||
m.Group("/rules", func() {
|
||||
m.Group("/add", func() {
|
||||
m.Get("", user_setting.PackagesRuleAdd)
|
||||
m.Post("", bindIgnErr(forms.PackageCleanupRuleForm{}), user_setting.PackagesRuleAddPost)
|
||||
})
|
||||
m.Group("/{id}", func() {
|
||||
m.Get("", user_setting.PackagesRuleEdit)
|
||||
m.Post("", bindIgnErr(forms.PackageCleanupRuleForm{}), user_setting.PackagesRuleEditPost)
|
||||
m.Get("/preview", user_setting.PackagesRulePreview)
|
||||
})
|
||||
})
|
||||
}, packagesEnabled)
|
||||
m.Get("/organization", user_setting.Organization)
|
||||
m.Get("/repos", user_setting.Repos)
|
||||
m.Post("/repos/unadopted", user_setting.AdoptOrDeleteRepository)
|
||||
}, reqSignIn, func(ctx *context.Context) {
|
||||
ctx.Data["PageIsUserSettings"] = true
|
||||
ctx.Data["AllThemes"] = setting.UI.Themes
|
||||
ctx.Data["EnablePackages"] = setting.Packages.Enabled
|
||||
})
|
||||
|
||||
m.Group("/user", func() {
|
||||
|
|
@ -526,12 +548,10 @@ func RegisterRoutes(m *web.Route) {
|
|||
m.Post("/delete", admin.DeleteRepo)
|
||||
})
|
||||
|
||||
if setting.Packages.Enabled {
|
||||
m.Group("/packages", func() {
|
||||
m.Get("", admin.Packages)
|
||||
m.Post("/delete", admin.DeletePackageVersion)
|
||||
})
|
||||
}
|
||||
m.Group("/packages", func() {
|
||||
m.Get("", admin.Packages)
|
||||
m.Post("/delete", admin.DeletePackageVersion)
|
||||
}, packagesEnabled)
|
||||
|
||||
m.Group("/hooks", func() {
|
||||
m.Get("", admin.DefaultOrSystemWebhooks)
|
||||
|
|
@ -750,8 +770,24 @@ func RegisterRoutes(m *web.Route) {
|
|||
})
|
||||
|
||||
m.Route("/delete", "GET,POST", org.SettingsDelete)
|
||||
|
||||
m.Group("/packages", func() {
|
||||
m.Get("", org.Packages)
|
||||
m.Group("/rules", func() {
|
||||
m.Group("/add", func() {
|
||||
m.Get("", org.PackagesRuleAdd)
|
||||
m.Post("", bindIgnErr(forms.PackageCleanupRuleForm{}), org.PackagesRuleAddPost)
|
||||
})
|
||||
m.Group("/{id}", func() {
|
||||
m.Get("", org.PackagesRuleEdit)
|
||||
m.Post("", bindIgnErr(forms.PackageCleanupRuleForm{}), org.PackagesRuleEditPost)
|
||||
m.Get("/preview", org.PackagesRulePreview)
|
||||
})
|
||||
})
|
||||
}, packagesEnabled)
|
||||
}, func(ctx *context.Context) {
|
||||
ctx.Data["EnableOAuth2"] = setting.OAuth2.Enable
|
||||
ctx.Data["EnablePackages"] = setting.Packages.Enabled
|
||||
})
|
||||
}, context.OrgAssignment(true, true))
|
||||
}, reqSignIn)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,31 @@
|
|||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package forms
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"code.gitea.io/gitea/modules/context"
|
||||
"code.gitea.io/gitea/modules/web/middleware"
|
||||
|
||||
"gitea.com/go-chi/binding"
|
||||
)
|
||||
|
||||
type PackageCleanupRuleForm struct {
|
||||
ID int64
|
||||
Enabled bool
|
||||
Type string `binding:"Required;In(composer,conan,container,generic,helm,maven,npm,nuget,pub,pypi,rubygems,vagrant)"`
|
||||
KeepCount int `binding:"In(0,1,5,10,25,50,100)"`
|
||||
KeepPattern string `binding:"RegexPattern"`
|
||||
RemoveDays int `binding:"In(0,7,14,30,60,90,180)"`
|
||||
RemovePattern string `binding:"RegexPattern"`
|
||||
MatchFullName bool
|
||||
Action string `binding:"Required;In(save,remove)"`
|
||||
}
|
||||
|
||||
func (f *PackageCleanupRuleForm) Validate(req *http.Request, errs binding.Errors) binding.Errors {
|
||||
ctx := context.GetContext(req)
|
||||
return middleware.Validate(errs, ctx.Data, f, ctx.Locale)
|
||||
}
|
||||
|
|
@ -6,13 +6,12 @@ package container
|
|||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
packages_model "code.gitea.io/gitea/models/packages"
|
||||
container_model "code.gitea.io/gitea/models/packages/container"
|
||||
user_model "code.gitea.io/gitea/models/user"
|
||||
container_module "code.gitea.io/gitea/modules/packages/container"
|
||||
"code.gitea.io/gitea/modules/packages/container/oci"
|
||||
"code.gitea.io/gitea/modules/util"
|
||||
)
|
||||
|
||||
|
|
@ -82,24 +81,30 @@ func cleanupExpiredUploadedBlobs(ctx context.Context, olderThan time.Duration) e
|
|||
return nil
|
||||
}
|
||||
|
||||
// UpdateRepositoryNames updates the repository name property for all packages of the specific owner
|
||||
func UpdateRepositoryNames(ctx context.Context, owner *user_model.User, newOwnerName string) error {
|
||||
ps, err := packages_model.GetPackagesByType(ctx, owner.ID, packages_model.TypeContainer)
|
||||
if err != nil {
|
||||
return err
|
||||
func ShouldBeSkipped(ctx context.Context, pcr *packages_model.PackageCleanupRule, p *packages_model.Package, pv *packages_model.PackageVersion) (bool, error) {
|
||||
// Always skip the "latest" tag
|
||||
if pv.LowerVersion == "latest" {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
newOwnerName = strings.ToLower(newOwnerName)
|
||||
|
||||
for _, p := range ps {
|
||||
if err := packages_model.DeletePropertyByName(ctx, packages_model.PropertyTypePackage, p.ID, container_module.PropertyRepository); err != nil {
|
||||
return err
|
||||
// Check if the version is a digest (or untagged)
|
||||
if oci.Digest(pv.LowerVersion).Validate() {
|
||||
// Check if there is another manifest referencing this version
|
||||
has, err := packages_model.ExistVersion(ctx, &packages_model.PackageSearchOptions{
|
||||
PackageID: p.ID,
|
||||
Properties: map[string]string{
|
||||
container_module.PropertyManifestReference: pv.LowerVersion,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypePackage, p.ID, container_module.PropertyRepository, newOwnerName+"/"+p.LowerName); err != nil {
|
||||
return err
|
||||
// Skip it if the version is referenced
|
||||
if has {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
return false, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,36 @@
|
|||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package container
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
packages_model "code.gitea.io/gitea/models/packages"
|
||||
user_model "code.gitea.io/gitea/models/user"
|
||||
container_module "code.gitea.io/gitea/modules/packages/container"
|
||||
)
|
||||
|
||||
// UpdateRepositoryNames updates the repository name property for all packages of the specific owner
|
||||
func UpdateRepositoryNames(ctx context.Context, owner *user_model.User, newOwnerName string) error {
|
||||
ps, err := packages_model.GetPackagesByType(ctx, owner.ID, packages_model.TypeContainer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
newOwnerName = strings.ToLower(newOwnerName)
|
||||
|
||||
for _, p := range ps {
|
||||
if err := packages_model.DeletePropertyByName(ctx, packages_model.PropertyTypePackage, p.ID, container_module.PropertyRepository); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypePackage, p.ID, container_module.PropertyRepository, newOwnerName+"/"+p.LowerName); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
@ -443,13 +443,80 @@ func DeletePackageFile(ctx context.Context, pf *packages_model.PackageFile) erro
|
|||
}
|
||||
|
||||
// Cleanup removes expired package data
|
||||
func Cleanup(unused context.Context, olderThan time.Duration) error {
|
||||
ctx, committer, err := db.TxContext(db.DefaultContext)
|
||||
func Cleanup(taskCtx context.Context, olderThan time.Duration) error {
|
||||
ctx, committer, err := db.TxContext(taskCtx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer committer.Close()
|
||||
|
||||
err = packages_model.IterateEnabledCleanupRules(ctx, func(ctx context.Context, pcr *packages_model.PackageCleanupRule) error {
|
||||
select {
|
||||
case <-taskCtx.Done():
|
||||
return db.ErrCancelledf("While processing package cleanup rules")
|
||||
default:
|
||||
}
|
||||
|
||||
if err := pcr.CompiledPattern(); err != nil {
|
||||
return fmt.Errorf("CleanupRule [%d]: CompilePattern failed: %w", pcr.ID, err)
|
||||
}
|
||||
|
||||
olderThan := time.Now().AddDate(0, 0, -pcr.RemoveDays)
|
||||
|
||||
packages, err := packages_model.GetPackagesByType(ctx, pcr.OwnerID, pcr.Type)
|
||||
if err != nil {
|
||||
return fmt.Errorf("CleanupRule [%d]: GetPackagesByType failed: %w", pcr.ID, err)
|
||||
}
|
||||
|
||||
for _, p := range packages {
|
||||
pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{
|
||||
PackageID: p.ID,
|
||||
IsInternal: util.OptionalBoolFalse,
|
||||
Sort: packages_model.SortCreatedDesc,
|
||||
Paginator: db.NewAbsoluteListOptions(pcr.KeepCount, 200),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("CleanupRule [%d]: SearchVersions failed: %w", pcr.ID, err)
|
||||
}
|
||||
for _, pv := range pvs {
|
||||
if skip, err := container_service.ShouldBeSkipped(ctx, pcr, p, pv); err != nil {
|
||||
return fmt.Errorf("CleanupRule [%d]: container.ShouldBeSkipped failed: %w", pcr.ID, err)
|
||||
} else if skip {
|
||||
log.Debug("Rule[%d]: keep '%s/%s' (container)", pcr.ID, p.Name, pv.Version)
|
||||
continue
|
||||
}
|
||||
|
||||
toMatch := pv.LowerVersion
|
||||
if pcr.MatchFullName {
|
||||
toMatch = p.LowerName + "/" + pv.LowerVersion
|
||||
}
|
||||
|
||||
if pcr.KeepPatternMatcher != nil && pcr.KeepPatternMatcher.MatchString(toMatch) {
|
||||
log.Debug("Rule[%d]: keep '%s/%s' (keep pattern)", pcr.ID, p.Name, pv.Version)
|
||||
continue
|
||||
}
|
||||
if pv.CreatedUnix.AsLocalTime().After(olderThan) {
|
||||
log.Debug("Rule[%d]: keep '%s/%s' (remove days)", pcr.ID, p.Name, pv.Version)
|
||||
continue
|
||||
}
|
||||
if pcr.RemovePatternMatcher != nil && !pcr.RemovePatternMatcher.MatchString(toMatch) {
|
||||
log.Debug("Rule[%d]: keep '%s/%s' (remove pattern)", pcr.ID, p.Name, pv.Version)
|
||||
continue
|
||||
}
|
||||
|
||||
log.Debug("Rule[%d]: remove '%s/%s'", pcr.ID, p.Name, pv.Version)
|
||||
|
||||
if err := DeletePackageVersionAndReferences(ctx, pv); err != nil {
|
||||
return fmt.Errorf("CleanupRule [%d]: DeletePackageVersionAndReferences failed: %w", pcr.ID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := container_service.Cleanup(ctx, olderThan); err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,6 +17,11 @@
|
|||
{{.locale.Tr "settings.applications"}}
|
||||
</a>
|
||||
{{end}}
|
||||
{{if .EnablePackages}}
|
||||
<a class="{{if .PageIsSettingsPackages}}active{{end}} item" href="{{.OrgLink}}/settings/packages">
|
||||
{{.locale.Tr "packages.title"}}
|
||||
</a>
|
||||
{{end}}
|
||||
<a class="{{if .PageIsSettingsDelete}}active{{end}} item" href="{{.OrgLink}}/settings/delete">
|
||||
{{.locale.Tr "org.settings.delete"}}
|
||||
</a>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,14 @@
|
|||
{{template "base/head" .}}
|
||||
<div class="page-content organization settings packages">
|
||||
{{template "org/header" .}}
|
||||
<div class="ui container">
|
||||
<div class="ui grid">
|
||||
{{template "org/settings/navbar" .}}
|
||||
<div class="twelve wide column content">
|
||||
{{template "base/alert" .}}
|
||||
{{template "package/shared/cleanup_rules/list" .}}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{{template "base/footer" .}}
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
{{template "base/head" .}}
|
||||
<div class="page-content organization settings packages">
|
||||
{{template "org/header" .}}
|
||||
<div class="ui container">
|
||||
<div class="ui grid">
|
||||
{{template "org/settings/navbar" .}}
|
||||
<div class="twelve wide column content">
|
||||
{{template "base/alert" .}}
|
||||
{{template "package/shared/cleanup_rules/edit" .}}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{{template "base/footer" .}}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
{{template "base/head" .}}
|
||||
<div class="page-content organization settings packages admin">
|
||||
{{template "org/header" .}}
|
||||
<div class="ui container">
|
||||
<div class="ui grid">
|
||||
{{template "org/settings/navbar" .}}
|
||||
<div class="twelve wide column content">
|
||||
{{template "package/shared/cleanup_rules/preview" .}}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{{template "base/footer" .}}
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
<h4 class="ui top attached header">{{if .IsEditRule}}{{.locale.Tr "packages.owner.settings.cleanuprules.edit"}}{{else}}{{.locale.Tr "packages.owner.settings.cleanuprules.add"}}{{end}}</h4>
|
||||
<div class="ui attached segment">
|
||||
<form class="ui form" action="{{.Link}}" method="post">
|
||||
{{.CsrfTokenHtml}}
|
||||
<input name="id" type="hidden" value="{{.CleanupRule.ID}}">
|
||||
<div class="field">
|
||||
<div class="ui checkbox">
|
||||
<label>{{.locale.Tr "enabled"}}</label>
|
||||
<input type="checkbox" name="enabled" {{if .CleanupRule.Enabled}}checked{{end}}>
|
||||
</div>
|
||||
</div>
|
||||
<div class="{{if .IsEditRule}}disabled {{end}}field {{if .Err_Type}}error{{end}}">
|
||||
<label>{{.locale.Tr "packages.filter.type"}}</label>
|
||||
<select class="ui selection dropdown" name="type">
|
||||
{{range $type := .AvailableTypes}}
|
||||
<option{{if eq $.CleanupRule.Type $type}} selected="selected"{{end}} value="{{$type}}">{{$type.Name}}</option>
|
||||
{{end}}
|
||||
</select>
|
||||
</div>
|
||||
<div class="field">
|
||||
<div class="ui checkbox">
|
||||
<label>{{.locale.Tr "packages.owner.settings.cleanuprules.pattern_full_match"}}</label>
|
||||
<input type="checkbox" name="match_full_name" {{if .CleanupRule.MatchFullName}}checked{{end}}>
|
||||
</div>
|
||||
</div>
|
||||
<div class="ui divider"></div>
|
||||
<p>{{.locale.Tr "packages.owner.settings.cleanuprules.keep.title"}}</p>
|
||||
<div class="field {{if .Err_KeepCount}}error{{end}}">
|
||||
<label>{{.locale.Tr "packages.owner.settings.cleanuprules.keep.count"}}:</label>
|
||||
<select class="ui selection dropdown" name="keep_count">
|
||||
<option{{if eq .CleanupRule.KeepCount 0}} selected="selected"{{end}} value="0"></option>
|
||||
<option{{if eq .CleanupRule.KeepCount 1}} selected="selected"{{end}} value="1">{{.locale.Tr "packages.owner.settings.cleanuprules.keep.count.1"}}</option>
|
||||
<option{{if eq .CleanupRule.KeepCount 5}} selected="selected"{{end}} value="5">{{.locale.Tr "packages.owner.settings.cleanuprules.keep.count.n" 5}}</option>
|
||||
<option{{if eq .CleanupRule.KeepCount 10}} selected="selected"{{end}} value="10">{{.locale.Tr "packages.owner.settings.cleanuprules.keep.count.n" 10}}</option>
|
||||
<option{{if eq .CleanupRule.KeepCount 25}} selected="selected"{{end}} value="25">{{.locale.Tr "packages.owner.settings.cleanuprules.keep.count.n" 25}}</option>
|
||||
<option{{if eq .CleanupRule.KeepCount 50}} selected="selected"{{end}} value="50">{{.locale.Tr "packages.owner.settings.cleanuprules.keep.count.n" 50}}</option>
|
||||
<option{{if eq .CleanupRule.KeepCount 100}} selected="selected"{{end}} value="100">{{.locale.Tr "packages.owner.settings.cleanuprules.keep.count.n" 100}}</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="field {{if .Err_KeepPattern}}error{{end}}">
|
||||
<label>{{.locale.Tr "packages.owner.settings.cleanuprules.keep.pattern"}}:</label>
|
||||
<input name="keep_pattern" type="text" value="{{.CleanupRule.KeepPattern}}">
|
||||
<p>{{.locale.Tr "packages.owner.settings.cleanuprules.keep.pattern.container" | Safe}}</p>
|
||||
</div>
|
||||
<div class="ui divider"></div>
|
||||
<p>{{.locale.Tr "packages.owner.settings.cleanuprules.remove.title"}}</p>
|
||||
<div class="field {{if .Err_RemoveDays}}error{{end}}">
|
||||
<label>{{.locale.Tr "packages.owner.settings.cleanuprules.remove.days"}}:</label>
|
||||
<select class="ui selection dropdown" name="remove_days">
|
||||
<option{{if eq .CleanupRule.RemoveDays 0}} selected="selected"{{end}} value="0"></option>
|
||||
<option{{if eq .CleanupRule.RemoveDays 7}} selected="selected"{{end}} value="7">{{.locale.Tr "tool.days" 7}}</option>
|
||||
<option{{if eq .CleanupRule.RemoveDays 14}} selected="selected"{{end}} value="14">{{.locale.Tr "tool.days" 14}}</option>
|
||||
<option{{if eq .CleanupRule.RemoveDays 30}} selected="selected"{{end}} value="30">{{.locale.Tr "tool.days" 30}}</option>
|
||||
<option{{if eq .CleanupRule.RemoveDays 60}} selected="selected"{{end}} value="60">{{.locale.Tr "tool.days" 60}}</option>
|
||||
<option{{if eq .CleanupRule.RemoveDays 90}} selected="selected"{{end}} value="90">{{.locale.Tr "tool.days" 90}}</option>
|
||||
<option{{if eq .CleanupRule.RemoveDays 180}} selected="selected"{{end}} value="180">{{.locale.Tr "tool.days" 180}}</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="field {{if .Err_RemovePattern}}error{{end}}">
|
||||
<label>{{.locale.Tr "packages.owner.settings.cleanuprules.remove.pattern"}}:</label>
|
||||
<input name="remove_pattern" type="text" value="{{.CleanupRule.RemovePattern}}">
|
||||
</div>
|
||||
<div class="field">
|
||||
{{if .IsEditRule}}
|
||||
<button class="ui green button" name="action" value="save">{{.locale.Tr "save"}}</button>
|
||||
<button class="ui red button" name="action" value="remove">{{.locale.Tr "remove"}}</button>
|
||||
<a class="ui button" href="{{.Link}}/preview">{{.locale.Tr "packages.owner.settings.cleanuprules.preview"}}</a>
|
||||
{{else}}
|
||||
<button class="ui green button" name="action" value="save">{{.locale.Tr "add"}}</button>
|
||||
{{end}}
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
<h4 class="ui top attached header">
|
||||
{{.locale.Tr "packages.owner.settings.cleanuprules.title"}}
|
||||
<div class="ui right">
|
||||
<a class="ui primary tiny button" href="{{.Link}}/rules/add">{{.locale.Tr "packages.owner.settings.cleanuprules.add"}}</a>
|
||||
</div>
|
||||
</h4>
|
||||
<div class="ui attached segment">
|
||||
<div class="ui key list">
|
||||
{{range .CleanupRules}}
|
||||
<div class="item">
|
||||
<div class="right floated content">
|
||||
<div class="ui dropdown tiny basic button icon-button">
|
||||
{{svg "octicon-kebab-horizontal"}}
|
||||
<div class="menu">
|
||||
<a class="item" href="{{$.Link}}/rules/{{.ID}}">{{$.locale.Tr "edit"}}</a>
|
||||
<a class="item" href="{{$.Link}}/rules/{{.ID}}/preview">{{$.locale.Tr "packages.owner.settings.cleanuprules.preview"}}</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<i class="icon">{{svg .Type.SVGName 36}}</i>
|
||||
<div class="content">
|
||||
<a class="item" href="{{$.Link}}/rules/{{.ID}}"><strong>{{.Type.Name}}</strong></a>
|
||||
<div><i>{{if .Enabled}}{{$.locale.Tr "enabled"}}{{else}}{{$.locale.Tr "disabled"}}{{end}}</i></div>
|
||||
{{if .KeepCount}}<div><i>{{$.locale.Tr "packages.owner.settings.cleanuprules.keep.count"}}:</i> {{if eq .KeepCount 1}}{{$.locale.Tr "packages.owner.settings.cleanuprules.keep.count.1"}}{{else}}{{$.locale.Tr "packages.owner.settings.cleanuprules.keep.count.n" .KeepCount}}{{end}}</div>{{end}}
|
||||
{{if .KeepPattern}}<div><i>{{$.locale.Tr "packages.owner.settings.cleanuprules.keep.pattern"}}:</i> {{EllipsisString .KeepPattern 100}}</div>{{end}}
|
||||
{{if .RemoveDays}}<div><i>{{$.locale.Tr "packages.owner.settings.cleanuprules.remove.days"}}:</i> {{$.locale.Tr "tool.days" .RemoveDays}}</div>{{end}}
|
||||
{{if .RemovePattern}}<div><i>{{$.locale.Tr "packages.owner.settings.cleanuprules.remove.pattern"}}:</i> {{EllipsisString .RemovePattern 100}}</div>{{end}}
|
||||
</div>
|
||||
</div>
|
||||
{{else}}
|
||||
<div class="item">{{.locale.Tr "packages.owner.settings.cleanuprules.none"}}</div>
|
||||
{{end}}
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
<h4 class="ui top attached header">{{.locale.Tr "packages.owner.settings.cleanuprules.preview"}}</h4>
|
||||
<div class="ui attached segment">
|
||||
<p>{{.locale.Tr "packages.owner.settings.cleanuprules.preview.overview" (len .VersionsToRemove)}}</p>
|
||||
</div>
|
||||
<div class="ui attached table segment">
|
||||
<table class="ui very basic striped table unstackable">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>{{.locale.Tr "admin.packages.type"}}</th>
|
||||
<th>{{.locale.Tr "admin.packages.name"}}</th>
|
||||
<th>{{.locale.Tr "admin.packages.version"}}</th>
|
||||
<th>{{.locale.Tr "admin.packages.creator"}}</th>
|
||||
<th>{{.locale.Tr "admin.packages.size"}}</th>
|
||||
<th>{{.locale.Tr "admin.packages.published"}}</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{{range .VersionsToRemove}}
|
||||
<tr>
|
||||
<td>{{.Package.Type.Name}}</td>
|
||||
<td>{{.Package.Name}}</td>
|
||||
<td><a href="{{.FullWebLink}}">{{.Version.Version}}</a></td>
|
||||
<td><a href="{{.Creator.HomeLink}}">{{.Creator.Name}}</a></td>
|
||||
<td>{{FileSize .CalculateBlobSize}}</td>
|
||||
<td><span title="{{.Version.CreatedUnix.FormatLong}}"><time data-format="short-date" datetime="{{.Version.CreatedUnix.FormatLong}}">{{.Version.CreatedUnix.FormatShort}}</time></span></td>
|
||||
</tr>
|
||||
{{else}}
|
||||
<tr>
|
||||
<td colspan="6">{{.locale.Tr "packages.owner.settings.cleanuprules.preview.none"}}</td>
|
||||
</tr>
|
||||
{{end}}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
|
@ -18,6 +18,11 @@
|
|||
<a class="{{if .PageIsSettingsKeys}}active{{end}} item" href="{{AppSubUrl}}/user/settings/keys">
|
||||
{{.locale.Tr "settings.ssh_gpg_keys"}}
|
||||
</a>
|
||||
{{if .EnablePackages}}
|
||||
<a class="{{if .PageIsSettingsPackages}}active{{end}} item" href="{{AppSubUrl}}/user/settings/packages">
|
||||
{{.locale.Tr "packages.title"}}
|
||||
</a>
|
||||
{{end}}
|
||||
<a class="{{if .PageIsSettingsOrganization}}active{{end}} item" href="{{AppSubUrl}}/user/settings/organization">
|
||||
{{.locale.Tr "settings.organization"}}
|
||||
</a>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,9 @@
|
|||
{{template "base/head" .}}
|
||||
<div class="page-content user settings packages">
|
||||
{{template "user/settings/navbar" .}}
|
||||
<div class="ui container">
|
||||
{{template "base/alert" .}}
|
||||
{{template "package/shared/cleanup_rules/list" .}}
|
||||
</div>
|
||||
</div>
|
||||
{{template "base/footer" .}}
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
{{template "base/head" .}}
|
||||
<div class="page-content user settings packages">
|
||||
{{template "user/settings/navbar" .}}
|
||||
<div class="ui container">
|
||||
{{template "base/alert" .}}
|
||||
{{template "package/shared/cleanup_rules/edit" .}}
|
||||
</div>
|
||||
</div>
|
||||
{{template "base/footer" .}}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
{{template "base/head" .}}
|
||||
<div class="page-content user settings packages admin">
|
||||
{{template "user/settings/navbar" .}}
|
||||
<div class="ui container">
|
||||
{{template "package/shared/cleanup_rules/preview" .}}
|
||||
</div>
|
||||
</div>
|
||||
{{template "base/footer" .}}
|
||||
|
|
@ -203,22 +203,171 @@ func TestPackageQuota(t *testing.T) {
|
|||
func TestPackageCleanup(t *testing.T) {
|
||||
defer tests.PrepareTestEnv(t)()
|
||||
|
||||
time.Sleep(time.Second)
|
||||
duration, _ := time.ParseDuration("-1h")
|
||||
|
||||
pbs, err := packages_model.FindExpiredUnreferencedBlobs(db.DefaultContext, time.Duration(0))
|
||||
assert.NoError(t, err)
|
||||
assert.NotEmpty(t, pbs)
|
||||
t.Run("Common", func(t *testing.T) {
|
||||
defer tests.PrintCurrentTest(t)()
|
||||
|
||||
_, err = packages_model.GetInternalVersionByNameAndVersion(db.DefaultContext, 2, packages_model.TypeContainer, "test", container_model.UploadVersion)
|
||||
assert.NoError(t, err)
|
||||
pbs, err := packages_model.FindExpiredUnreferencedBlobs(db.DefaultContext, duration)
|
||||
assert.NoError(t, err)
|
||||
assert.NotEmpty(t, pbs)
|
||||
|
||||
err = packages_service.Cleanup(nil, time.Duration(0))
|
||||
assert.NoError(t, err)
|
||||
_, err = packages_model.GetInternalVersionByNameAndVersion(db.DefaultContext, 2, packages_model.TypeContainer, "test", container_model.UploadVersion)
|
||||
assert.NoError(t, err)
|
||||
|
||||
pbs, err = packages_model.FindExpiredUnreferencedBlobs(db.DefaultContext, time.Duration(0))
|
||||
assert.NoError(t, err)
|
||||
assert.Empty(t, pbs)
|
||||
err = packages_service.Cleanup(db.DefaultContext, duration)
|
||||
assert.NoError(t, err)
|
||||
|
||||
_, err = packages_model.GetInternalVersionByNameAndVersion(db.DefaultContext, 2, packages_model.TypeContainer, "test", container_model.UploadVersion)
|
||||
assert.ErrorIs(t, err, packages_model.ErrPackageNotExist)
|
||||
pbs, err = packages_model.FindExpiredUnreferencedBlobs(db.DefaultContext, duration)
|
||||
assert.NoError(t, err)
|
||||
assert.Empty(t, pbs)
|
||||
|
||||
_, err = packages_model.GetInternalVersionByNameAndVersion(db.DefaultContext, 2, packages_model.TypeContainer, "test", container_model.UploadVersion)
|
||||
assert.ErrorIs(t, err, packages_model.ErrPackageNotExist)
|
||||
})
|
||||
|
||||
t.Run("CleanupRules", func(t *testing.T) {
|
||||
defer tests.PrintCurrentTest(t)()
|
||||
|
||||
user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
|
||||
|
||||
type version struct {
|
||||
Version string
|
||||
ShouldExist bool
|
||||
Created int64
|
||||
}
|
||||
|
||||
cases := []struct {
|
||||
Name string
|
||||
Versions []version
|
||||
Rule *packages_model.PackageCleanupRule
|
||||
}{
|
||||
{
|
||||
Name: "Disabled",
|
||||
Versions: []version{
|
||||
{Version: "keep", ShouldExist: true},
|
||||
},
|
||||
Rule: &packages_model.PackageCleanupRule{
|
||||
Enabled: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "KeepCount",
|
||||
Versions: []version{
|
||||
{Version: "keep", ShouldExist: true},
|
||||
{Version: "v1.0", ShouldExist: true},
|
||||
{Version: "test-3", ShouldExist: false, Created: 1},
|
||||
{Version: "test-4", ShouldExist: false, Created: 1},
|
||||
},
|
||||
Rule: &packages_model.PackageCleanupRule{
|
||||
Enabled: true,
|
||||
KeepCount: 2,
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "KeepPattern",
|
||||
Versions: []version{
|
||||
{Version: "keep", ShouldExist: true},
|
||||
{Version: "v1.0", ShouldExist: false},
|
||||
},
|
||||
Rule: &packages_model.PackageCleanupRule{
|
||||
Enabled: true,
|
||||
KeepPattern: "k.+p",
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "RemoveDays",
|
||||
Versions: []version{
|
||||
{Version: "keep", ShouldExist: true},
|
||||
{Version: "v1.0", ShouldExist: false, Created: 1},
|
||||
},
|
||||
Rule: &packages_model.PackageCleanupRule{
|
||||
Enabled: true,
|
||||
RemoveDays: 60,
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "RemovePattern",
|
||||
Versions: []version{
|
||||
{Version: "test", ShouldExist: true},
|
||||
{Version: "test-3", ShouldExist: false},
|
||||
{Version: "test-4", ShouldExist: false},
|
||||
},
|
||||
Rule: &packages_model.PackageCleanupRule{
|
||||
Enabled: true,
|
||||
RemovePattern: `t[e]+st-\d+`,
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "MatchFullName",
|
||||
Versions: []version{
|
||||
{Version: "keep", ShouldExist: true},
|
||||
{Version: "test", ShouldExist: false},
|
||||
},
|
||||
Rule: &packages_model.PackageCleanupRule{
|
||||
Enabled: true,
|
||||
RemovePattern: `package/test|different/keep`,
|
||||
MatchFullName: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "Mixed",
|
||||
Versions: []version{
|
||||
{Version: "keep", ShouldExist: true, Created: time.Now().Add(time.Duration(10000)).Unix()},
|
||||
{Version: "dummy", ShouldExist: true, Created: 1},
|
||||
{Version: "test-3", ShouldExist: true},
|
||||
{Version: "test-4", ShouldExist: false, Created: 1},
|
||||
},
|
||||
Rule: &packages_model.PackageCleanupRule{
|
||||
Enabled: true,
|
||||
KeepCount: 1,
|
||||
KeepPattern: `dummy`,
|
||||
RemoveDays: 7,
|
||||
RemovePattern: `t[e]+st-\d+`,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
t.Run(c.Name, func(t *testing.T) {
|
||||
defer tests.PrintCurrentTest(t)()
|
||||
|
||||
for _, v := range c.Versions {
|
||||
url := fmt.Sprintf("/api/packages/%s/generic/package/%s/file.bin", user.Name, v.Version)
|
||||
req := NewRequestWithBody(t, "PUT", url, bytes.NewReader([]byte{1}))
|
||||
AddBasicAuthHeader(req, user.Name)
|
||||
MakeRequest(t, req, http.StatusCreated)
|
||||
|
||||
if v.Created != 0 {
|
||||
pv, err := packages_model.GetVersionByNameAndVersion(db.DefaultContext, user.ID, packages_model.TypeGeneric, "package", v.Version)
|
||||
assert.NoError(t, err)
|
||||
_, err = db.GetEngine(db.DefaultContext).Exec("UPDATE package_version SET created_unix = ? WHERE id = ?", v.Created, pv.ID)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
}
|
||||
|
||||
c.Rule.OwnerID = user.ID
|
||||
c.Rule.Type = packages_model.TypeGeneric
|
||||
|
||||
pcr, err := packages_model.InsertCleanupRule(db.DefaultContext, c.Rule)
|
||||
assert.NoError(t, err)
|
||||
|
||||
err = packages_service.Cleanup(db.DefaultContext, duration)
|
||||
assert.NoError(t, err)
|
||||
|
||||
for _, v := range c.Versions {
|
||||
pv, err := packages_model.GetVersionByNameAndVersion(db.DefaultContext, user.ID, packages_model.TypeGeneric, "package", v.Version)
|
||||
if v.ShouldExist {
|
||||
assert.NoError(t, err)
|
||||
err = packages_service.DeletePackageVersionAndReferences(db.DefaultContext, pv)
|
||||
assert.NoError(t, err)
|
||||
} else {
|
||||
assert.ErrorIs(t, err, packages_model.ErrPackageNotExist)
|
||||
}
|
||||
}
|
||||
|
||||
assert.NoError(t, packages_model.DeleteCleanupRuleByID(db.DefaultContext, pcr.ID))
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue