Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add package registry quota limits #21584

Merged
merged 8 commits into from
Nov 9, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions cmd/migrate_storage_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,9 @@ func TestMigratePackages(t *testing.T) {
PackageFileInfo: packages_service.PackageFileInfo{
Filename: "a.go",
},
Data: buf,
IsLead: true,
Creator: creator,
Data: buf,
IsLead: true,
})
assert.NoError(t, err)
assert.NotNil(t, v)
Expand Down
29 changes: 29 additions & 0 deletions custom/conf/app.example.ini
Original file line number Diff line number Diff line change
Expand Up @@ -2335,6 +2335,35 @@ ROUTER = console
;;
;; Path for chunked uploads. Defaults to APP_DATA_PATH + `tmp/package-upload`
;CHUNKED_UPLOAD_PATH = tmp/package-upload
;;
;; Maxmimum count of package versions a single owner can have (`-1` means no limits)
;LIMIT_TOTAL_OWNER_COUNT = -1
;; Maxmimum size of packages a single owner can use (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
;LIMIT_TOTAL_OWNER_SIZE = -1
;; Maxmimum size of a Composer upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
;LIMIT_SIZE_COMPOSER = -1
;; Maxmimum size of a Conan upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
;LIMIT_SIZE_CONAN = -1
;; Maxmimum size of a Container upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
;LIMIT_SIZE_CONTAINER = -1
;; Maxmimum size of a Generic upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
;LIMIT_SIZE_GENERIC = -1
;; Maxmimum size of a Helm upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
;LIMIT_SIZE_HELM = -1
;; Maxmimum size of a Maven upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
;LIMIT_SIZE_MAVEN = -1
;; Maxmimum size of a npm upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
;LIMIT_SIZE_NPM = -1
;; Maxmimum size of a NuGet upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
;LIMIT_SIZE_NUGET = -1
;; Maxmimum size of a Pub upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
;LIMIT_SIZE_PUB = -1
;; Maxmimum size of a PyPI upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
;LIMIT_SIZE_PYPI = -1
;; Maxmimum size of a RubyGems upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
;LIMIT_SIZE_RUBYGEMS = -1
;; Maxmimum size of a Vagrant upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
;LIMIT_SIZE_VAGRANT = -1

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
Expand Down
14 changes: 14 additions & 0 deletions docs/content/doc/advanced/config-cheat-sheet.en-us.md
Original file line number Diff line number Diff line change
Expand Up @@ -1138,6 +1138,20 @@ Task queue configuration has been moved to `queue.task`. However, the below conf

- `ENABLED`: **true**: Enable/Disable package registry capabilities
- `CHUNKED_UPLOAD_PATH`: **tmp/package-upload**: Path for chunked uploads. Defaults to `APP_DATA_PATH` + `tmp/package-upload`
- `LIMIT_TOTAL_OWNER_COUNT`: **-1**: Maxmimum count of package versions a single owner can have (`-1` means no limits)
- `LIMIT_TOTAL_OWNER_SIZE`: **-1**: Maxmimum size of packages a single owner can use (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
- `LIMIT_SIZE_COMPOSER`: **-1**: Maxmimum size of a Composer upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
- `LIMIT_SIZE_CONAN`: **-1**: Maxmimum size of a Conan upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
- `LIMIT_SIZE_CONTAINER`: **-1**: Maxmimum size of a Container upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
- `LIMIT_SIZE_GENERIC`: **-1**: Maxmimum size of a Generic upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
- `LIMIT_SIZE_HELM`: **-1**: Maxmimum size of a Helm upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
- `LIMIT_SIZE_MAVEN`: **-1**: Maxmimum size of a Maven upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
- `LIMIT_SIZE_NPM`: **-1**: Maxmimum size of a npm upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
- `LIMIT_SIZE_NUGET`: **-1**: Maxmimum size of a NuGet upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
- `LIMIT_SIZE_PUB`: **-1**: Maxmimum size of a Pub upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
- `LIMIT_SIZE_PYPI`: **-1**: Maxmimum size of a PyPI upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
- `LIMIT_SIZE_RUBYGEMS`: **-1**: Maxmimum size of a RubyGems upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
- `LIMIT_SIZE_VAGRANT`: **-1**: Maxmimum size of a Vagrant upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)

## Mirror (`mirror`)

Expand Down
10 changes: 10 additions & 0 deletions models/packages/package_file.go
Original file line number Diff line number Diff line change
Expand Up @@ -199,3 +199,13 @@ func SearchFiles(ctx context.Context, opts *PackageFileSearchOptions) ([]*Packag
count, err := sess.FindAndCount(&pfs)
return pfs, count, err
}

// CalculateBlobSize sums up all blob sizes matching the search options.
// It does NOT respect the deduplication of blobs.
func CalculateBlobSize(ctx context.Context, opts *PackageFileSearchOptions) (int64, error) {
return db.GetEngine(ctx).
Table("package_file").
Where(opts.toConds()).
Join("INNER", "package_blob", "package_blob.id = package_file.blob_id").
SumInt(new(PackageBlob), "size")
}
9 changes: 9 additions & 0 deletions models/packages/package_version.go
Original file line number Diff line number Diff line change
Expand Up @@ -319,3 +319,12 @@ func SearchLatestVersions(ctx context.Context, opts *PackageSearchOptions) ([]*P
count, err := sess.FindAndCount(&pvs)
return pvs, count, err
}

// CountVersions counts all versions of packages matching the search options
func CountVersions(ctx context.Context, opts *PackageSearchOptions) (int64, error) {
return db.GetEngine(ctx).
Where(opts.toConds()).
Table("package_version").
KN4CK3R marked this conversation as resolved.
Show resolved Hide resolved
Join("INNER", "package", "package.id = package_version.package_id").
Count(new(PackageVersion))
}
50 changes: 49 additions & 1 deletion modules/setting/packages.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,15 @@
package setting

import (
"math"
"net/url"
"os"
"path/filepath"

"code.gitea.io/gitea/modules/log"

"github.com/dustin/go-humanize"
ini "gopkg.in/ini.v1"
)

// Package registry settings
Expand All @@ -19,8 +23,24 @@ var (
Enabled bool
ChunkedUploadPath string
RegistryHost string

LimitTotalOwnerCount int64
LimitTotalOwnerSize int64
LimitSizeComposer int64
LimitSizeConan int64
LimitSizeContainer int64
LimitSizeGeneric int64
LimitSizeHelm int64
LimitSizeMaven int64
LimitSizeNpm int64
LimitSizeNuGet int64
LimitSizePub int64
LimitSizePyPI int64
LimitSizeRubyGems int64
LimitSizeVagrant int64
}{
Enabled: true,
Enabled: true,
LimitTotalOwnerCount: -1,
}
Comment on lines +28 to 44
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Aren't there a few default values missing so that we now violate the assumptions in mustBytes by default?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

func mustBytes(section *ini.Section, key string) int64 {
	const noLimit = "-1"

	// MustString returns "-1" if the key is missing
	value := section.Key(key).MustString(noLimit)
	if value == noLimit { // and that is noLimit
		return -1 // which returns -1 as default
	}
	bytes, err := humanize.ParseBytes(value) // all other values
	if err != nil || bytes > math.MaxInt64 { // may be invalid
		return -1 // and default to -1
	}
	return int64(bytes) // or are valid
}

Copy link
Member

@delvh delvh Nov 9, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That is not what I meant!
What I meant is: You don't assign a default value to i.e. LimitSizeComposer at the moment, meaning its default is 0, even though we documented that it is -1. Or am I missing something?
And the assumptions are violated because your default value is then 0 instead of -1.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The values are overwritten/initialized in

func newPackages() {
sec := Cfg.Section("packages")
if err := sec.MapTo(&Packages); err != nil {
log.Fatal("Failed to map Packages settings: %v", err)
}
Packages.Storage = getStorage("packages", "", nil)
appURL, _ := url.Parse(AppURL)
Packages.RegistryHost = appURL.Host
Packages.ChunkedUploadPath = filepath.ToSlash(sec.Key("CHUNKED_UPLOAD_PATH").MustString("tmp/package-upload"))
if !filepath.IsAbs(Packages.ChunkedUploadPath) {
Packages.ChunkedUploadPath = filepath.ToSlash(filepath.Join(AppDataPath, Packages.ChunkedUploadPath))
}
if err := os.MkdirAll(Packages.ChunkedUploadPath, os.ModePerm); err != nil {
log.Error("Unable to create chunked upload directory: %s (%v)", Packages.ChunkedUploadPath, err)
}
Packages.LimitTotalOwnerSize = mustBytes(sec, "LIMIT_TOTAL_OWNER_SIZE")
Packages.LimitSizeComposer = mustBytes(sec, "LIMIT_SIZE_COMPOSER")
Packages.LimitSizeConan = mustBytes(sec, "LIMIT_SIZE_CONAN")
Packages.LimitSizeContainer = mustBytes(sec, "LIMIT_SIZE_CONTAINER")
Packages.LimitSizeGeneric = mustBytes(sec, "LIMIT_SIZE_GENERIC")
Packages.LimitSizeHelm = mustBytes(sec, "LIMIT_SIZE_HELM")
Packages.LimitSizeMaven = mustBytes(sec, "LIMIT_SIZE_MAVEN")
Packages.LimitSizeNpm = mustBytes(sec, "LIMIT_SIZE_NPM")
Packages.LimitSizeNuGet = mustBytes(sec, "LIMIT_SIZE_NUGET")
Packages.LimitSizePub = mustBytes(sec, "LIMIT_SIZE_PUB")
Packages.LimitSizePyPI = mustBytes(sec, "LIMIT_SIZE_PYPI")
Packages.LimitSizeRubyGems = mustBytes(sec, "LIMIT_SIZE_RUBYGEMS")
Packages.LimitSizeVagrant = mustBytes(sec, "LIMIT_SIZE_VAGRANT")

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, I always overlooked // MustString returns "-1" if the key is missing

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, there must be no default value be set because the "constructor" newPackages sets a valid value. 👍

)

Expand All @@ -43,4 +63,32 @@ func newPackages() {
if err := os.MkdirAll(Packages.ChunkedUploadPath, os.ModePerm); err != nil {
log.Error("Unable to create chunked upload directory: %s (%v)", Packages.ChunkedUploadPath, err)
}

Packages.LimitTotalOwnerSize = mustBytes(sec, "LIMIT_TOTAL_OWNER_SIZE")
Packages.LimitSizeComposer = mustBytes(sec, "LIMIT_SIZE_COMPOSER")
Packages.LimitSizeConan = mustBytes(sec, "LIMIT_SIZE_CONAN")
Packages.LimitSizeContainer = mustBytes(sec, "LIMIT_SIZE_CONTAINER")
Packages.LimitSizeGeneric = mustBytes(sec, "LIMIT_SIZE_GENERIC")
Packages.LimitSizeHelm = mustBytes(sec, "LIMIT_SIZE_HELM")
Packages.LimitSizeMaven = mustBytes(sec, "LIMIT_SIZE_MAVEN")
Packages.LimitSizeNpm = mustBytes(sec, "LIMIT_SIZE_NPM")
Packages.LimitSizeNuGet = mustBytes(sec, "LIMIT_SIZE_NUGET")
Packages.LimitSizePub = mustBytes(sec, "LIMIT_SIZE_PUB")
Packages.LimitSizePyPI = mustBytes(sec, "LIMIT_SIZE_PYPI")
Packages.LimitSizeRubyGems = mustBytes(sec, "LIMIT_SIZE_RUBYGEMS")
Packages.LimitSizeVagrant = mustBytes(sec, "LIMIT_SIZE_VAGRANT")
}

func mustBytes(section *ini.Section, key string) int64 {
const noLimit = "-1"

value := section.Key(key).MustString(noLimit)
if value == noLimit {
return -1
}
bytes, err := humanize.ParseBytes(value)
if err != nil || bytes > math.MaxInt64 {
return -1
}
return int64(bytes)
}
31 changes: 31 additions & 0 deletions modules/setting/packages_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package setting

import (
"testing"

"github.com/stretchr/testify/assert"
ini "gopkg.in/ini.v1"
)

func TestMustBytes(t *testing.T) {
test := func(value string) int64 {
sec, _ := ini.Empty().NewSection("test")
sec.NewKey("VALUE", value)

return mustBytes(sec, "VALUE")
}

assert.EqualValues(t, -1, test(""))
assert.EqualValues(t, -1, test("-1"))
assert.EqualValues(t, 0, test("0"))
assert.EqualValues(t, 1, test("1"))
assert.EqualValues(t, 10000, test("10000"))
assert.EqualValues(t, 1000000, test("1 mb"))
assert.EqualValues(t, 1048576, test("1mib"))
assert.EqualValues(t, 1782579, test("1.7mib"))
assert.EqualValues(t, -1, test("1 yib")) // too large
}
14 changes: 9 additions & 5 deletions routers/api/packages/composer/composer.go
Original file line number Diff line number Diff line change
Expand Up @@ -235,16 +235,20 @@ func UploadPackage(ctx *context.Context) {
PackageFileInfo: packages_service.PackageFileInfo{
Filename: strings.ToLower(fmt.Sprintf("%s.%s.zip", strings.ReplaceAll(cp.Name, "/", "-"), cp.Version)),
},
Data: buf,
IsLead: true,
Creator: ctx.Doer,
Data: buf,
IsLead: true,
},
)
if err != nil {
if err == packages_model.ErrDuplicatePackageVersion {
switch err {
case packages_model.ErrDuplicatePackageVersion:
apiError(ctx, http.StatusBadRequest, err)
return
case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
apiError(ctx, http.StatusForbidden, err)
default:
apiError(ctx, http.StatusInternalServerError, err)
}
apiError(ctx, http.StatusInternalServerError, err)
return
}

Expand Down
14 changes: 9 additions & 5 deletions routers/api/packages/conan/conan.go
Original file line number Diff line number Diff line change
Expand Up @@ -348,8 +348,9 @@ func uploadFile(ctx *context.Context, fileFilter container.Set[string], fileKey
Filename: strings.ToLower(filename),
CompositeKey: fileKey,
},
Data: buf,
IsLead: isConanfileFile,
Creator: ctx.Doer,
Data: buf,
IsLead: isConanfileFile,
Properties: map[string]string{
conan_module.PropertyRecipeUser: rref.User,
conan_module.PropertyRecipeChannel: rref.Channel,
Expand Down Expand Up @@ -416,11 +417,14 @@ func uploadFile(ctx *context.Context, fileFilter container.Set[string], fileKey
pfci,
)
if err != nil {
if err == packages_model.ErrDuplicatePackageFile {
switch err {
case packages_model.ErrDuplicatePackageFile:
apiError(ctx, http.StatusBadRequest, err)
return
case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
apiError(ctx, http.StatusForbidden, err)
default:
apiError(ctx, http.StatusInternalServerError, err)
}
apiError(ctx, http.StatusInternalServerError, err)
return
}

Expand Down
14 changes: 9 additions & 5 deletions routers/api/packages/generic/generic.go
Original file line number Diff line number Diff line change
Expand Up @@ -104,16 +104,20 @@ func UploadPackage(ctx *context.Context) {
PackageFileInfo: packages_service.PackageFileInfo{
Filename: filename,
},
Data: buf,
IsLead: true,
Creator: ctx.Doer,
Data: buf,
IsLead: true,
},
)
if err != nil {
if err == packages_model.ErrDuplicatePackageFile {
switch err {
case packages_model.ErrDuplicatePackageFile:
apiError(ctx, http.StatusConflict, err)
return
case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
apiError(ctx, http.StatusForbidden, err)
default:
apiError(ctx, http.StatusInternalServerError, err)
}
apiError(ctx, http.StatusInternalServerError, err)
return
}

Expand Down
10 changes: 7 additions & 3 deletions routers/api/packages/helm/helm.go
Original file line number Diff line number Diff line change
Expand Up @@ -186,17 +186,21 @@ func UploadPackage(ctx *context.Context) {
PackageFileInfo: packages_service.PackageFileInfo{
Filename: createFilename(metadata),
},
Creator: ctx.Doer,
Data: buf,
IsLead: true,
OverwriteExisting: true,
},
)
if err != nil {
if err == packages_model.ErrDuplicatePackageVersion {
switch err {
case packages_model.ErrDuplicatePackageVersion:
apiError(ctx, http.StatusConflict, err)
return
case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
apiError(ctx, http.StatusForbidden, err)
default:
apiError(ctx, http.StatusInternalServerError, err)
}
apiError(ctx, http.StatusInternalServerError, err)
return
}

Expand Down
10 changes: 7 additions & 3 deletions routers/api/packages/maven/maven.go
Original file line number Diff line number Diff line change
Expand Up @@ -266,6 +266,7 @@ func UploadPackageFile(ctx *context.Context) {
PackageFileInfo: packages_service.PackageFileInfo{
Filename: params.Filename,
},
Creator: ctx.Doer,
Data: buf,
IsLead: false,
OverwriteExisting: params.IsMeta,
Expand Down Expand Up @@ -312,11 +313,14 @@ func UploadPackageFile(ctx *context.Context) {
pfci,
)
if err != nil {
if err == packages_model.ErrDuplicatePackageFile {
switch err {
case packages_model.ErrDuplicatePackageFile:
apiError(ctx, http.StatusBadRequest, err)
return
case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
apiError(ctx, http.StatusForbidden, err)
default:
apiError(ctx, http.StatusInternalServerError, err)
}
apiError(ctx, http.StatusInternalServerError, err)
return
}

Expand Down
14 changes: 9 additions & 5 deletions routers/api/packages/npm/npm.go
Original file line number Diff line number Diff line change
Expand Up @@ -180,16 +180,20 @@ func UploadPackage(ctx *context.Context) {
PackageFileInfo: packages_service.PackageFileInfo{
Filename: npmPackage.Filename,
},
Data: buf,
IsLead: true,
Creator: ctx.Doer,
Data: buf,
IsLead: true,
},
)
if err != nil {
if err == packages_model.ErrDuplicatePackageVersion {
switch err {
case packages_model.ErrDuplicatePackageVersion:
apiError(ctx, http.StatusBadRequest, err)
return
case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
apiError(ctx, http.StatusForbidden, err)
default:
apiError(ctx, http.StatusInternalServerError, err)
}
apiError(ctx, http.StatusInternalServerError, err)
return
}

Expand Down
Loading