Merge remote-tracking branch 'giteaofficial/main'

* giteaofficial/main: Workaround for container registry push/pull errors (go-gitea#21862) Fix scroll over mermaid frame (go-gitea#21925) Add support for HEAD requests in Maven registry (go-gitea#21834) Fix button in branch list, avoid unexpected page jump before restore branch actually done (go-gitea#21562) Fix typo in sidebar (go-gitea#21922) Fix table misalignments and tweak webhook and githook lists (go-gitea#21917)
zjjhot · Nov 25, 2022 · 1a3fa0d · 1a3fa0d
2 parents 490381a + a1ae83f
commit 1a3fa0d
Show file tree

Hide file tree

Showing 30 changed files with 294 additions and 69 deletions.
diff --git a/docs/content/doc/packages/vagrant.en-us.md b/docs/content/doc/packages/vagrant.en-us.md
@@ -7,7 +7,7 @@ toc: false
 menu:
   sidebar:
     parent: "packages"
-    name: "vagrant"
+    name: "Vagrant"
     weight: 120
     identifier: "vagrant"
 ---

diff --git a/modules/context/context.go b/modules/context/context.go
@@ -349,9 +349,11 @@ func (ctx *Context) RespHeader() http.Header {
 type ServeHeaderOptions struct {
 	ContentType        string // defaults to "application/octet-stream"
 	ContentTypeCharset string
+	ContentLength      *int64
 	Disposition        string // defaults to "attachment"
 	Filename           string
 	CacheDuration      time.Duration // defaults to 5 minutes
+	LastModified       time.Time
 }
 
 // SetServeHeaders sets necessary content serve headers
@@ -369,6 +371,10 @@ func (ctx *Context) SetServeHeaders(opts *ServeHeaderOptions) {
 	header.Set("Content-Type", contentType)
 	header.Set("X-Content-Type-Options", "nosniff")
 
+	if opts.ContentLength != nil {
+		header.Set("Content-Length", strconv.FormatInt(*opts.ContentLength, 10))
+	}
+
 	if opts.Filename != "" {
 		disposition := opts.Disposition
 		if disposition == "" {
@@ -385,14 +391,16 @@ func (ctx *Context) SetServeHeaders(opts *ServeHeaderOptions) {
 		duration = 5 * time.Minute
 	}
 	httpcache.AddCacheControlToHeader(header, duration)
+
+	if !opts.LastModified.IsZero() {
+		header.Set("Last-Modified", opts.LastModified.UTC().Format(http.TimeFormat))
+	}
 }
 
 // ServeContent serves content to http request
-func (ctx *Context) ServeContent(name string, r io.ReadSeeker, modTime time.Time) {
-	ctx.SetServeHeaders(&ServeHeaderOptions{
-		Filename: name,
-	})
-	http.ServeContent(ctx.Resp, ctx.Req, name, modTime, r)
+func (ctx *Context) ServeContent(r io.ReadSeeker, opts *ServeHeaderOptions) {
+	ctx.SetServeHeaders(opts)
+	http.ServeContent(ctx.Resp, ctx.Req, opts.Filename, opts.LastModified, r)
 }
 
 // UploadStream returns the request body or the first form file

diff --git a/modules/packages/content_store.go b/modules/packages/content_store.go
@@ -32,6 +32,13 @@ func (s *ContentStore) Get(key BlobHash256Key) (storage.Object, error) {
 	return s.store.Open(KeyToRelativePath(key))
 }
 
+// FIXME: Workaround to be removed in v1.20
+// https://github.com/go-gitea/gitea/issues/19586
+func (s *ContentStore) Has(key BlobHash256Key) error {
+	_, err := s.store.Stat(KeyToRelativePath(key))
+	return err
+}
+
 // Save stores a package blob
 func (s *ContentStore) Save(key BlobHash256Key, r io.Reader, size int64) error {
 	_, err := s.store.Save(KeyToRelativePath(key), r, size)

diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go
@@ -181,6 +181,7 @@ func CommonRoutes(ctx gocontext.Context) *web.Route {
 		r.Group("/maven", func() {
 			r.Put("/*", reqPackageAccess(perm.AccessModeWrite), maven.UploadPackageFile)
 			r.Get("/*", maven.DownloadPackageFile)
+			r.Head("/*", maven.ProvidePackageFileHeader)
 		}, reqPackageAccess(perm.AccessModeRead))
 		r.Group("/nuget", func() {
 			r.Group("", func() { // Needs to be unauthenticated for the NuGet client.

diff --git a/routers/api/packages/composer/composer.go b/routers/api/packages/composer/composer.go
@@ -184,7 +184,10 @@ func DownloadPackageFile(ctx *context.Context) {
 	}
 	defer s.Close()
 
-	ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime())
+	ctx.ServeContent(s, &context.ServeHeaderOptions{
+		Filename:     pf.Name,
+		LastModified: pf.CreatedUnix.AsLocalTime(),
+	})
 }
 
 // UploadPackage creates a new package

diff --git a/routers/api/packages/conan/conan.go b/routers/api/packages/conan/conan.go
@@ -477,7 +477,10 @@ func downloadFile(ctx *context.Context, fileFilter container.Set[string], fileKe
 	}
 	defer s.Close()
 
-	ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime())
+	ctx.ServeContent(s, &context.ServeHeaderOptions{
+		Filename:     pf.Name,
+		LastModified: pf.CreatedUnix.AsLocalTime(),
+	})
 }
 
 // DeleteRecipeV1 deletes the requested recipe(s)

diff --git a/routers/api/packages/container/blob.go b/routers/api/packages/container/blob.go
@@ -7,18 +7,24 @@ package container
 import (
 	"context"
 	"encoding/hex"
+	"errors"
 	"fmt"
+	"os"
 	"strings"
+	"sync"
 
 	"code.gitea.io/gitea/models/db"
 	packages_model "code.gitea.io/gitea/models/packages"
 	container_model "code.gitea.io/gitea/models/packages/container"
 	"code.gitea.io/gitea/modules/log"
 	packages_module "code.gitea.io/gitea/modules/packages"
 	container_module "code.gitea.io/gitea/modules/packages/container"
+	"code.gitea.io/gitea/modules/util"
 	packages_service "code.gitea.io/gitea/services/packages"
 )
 
+var uploadVersionMutex sync.Mutex
+
 // saveAsPackageBlob creates a package blob from an upload
 // The uploaded blob gets stored in a special upload version to link them to the package/image
 func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pi *packages_service.PackageInfo) (*packages_model.PackageBlob, error) {
@@ -28,6 +34,11 @@ func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pi *packages_servic
 
 	contentStore := packages_module.NewContentStore()
 
+	var uploadVersion *packages_model.PackageVersion
+
+	// FIXME: Replace usage of mutex with database transaction
+	// https://github.com/go-gitea/gitea/pull/21862
+	uploadVersionMutex.Lock()
 	err := db.WithTx(db.DefaultContext, func(ctx context.Context) error {
 		created := true
 		p := &packages_model.Package{
@@ -68,11 +79,30 @@ func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pi *packages_servic
 			}
 		}
 
+		uploadVersion = pv
+
+		return nil
+	})
+	uploadVersionMutex.Unlock()
+	if err != nil {
+		return nil, err
+	}
+
+	err = db.WithTx(db.DefaultContext, func(ctx context.Context) error {
 		pb, exists, err = packages_model.GetOrInsertBlob(ctx, pb)
 		if err != nil {
 			log.Error("Error inserting package blob: %v", err)
 			return err
 		}
+		// FIXME: Workaround to be removed in v1.20
+		// https://github.com/go-gitea/gitea/issues/19586
+		if exists {
+			err = contentStore.Has(packages_module.BlobHash256Key(pb.HashSHA256))
+			if err != nil && (errors.Is(err, util.ErrNotExist) || errors.Is(err, os.ErrNotExist)) {
+				log.Debug("Package registry inconsistent: blob %s does not exist on file system", pb.HashSHA256)
+				exists = false
+			}
+		}
 		if !exists {
 			if err := contentStore.Save(packages_module.BlobHash256Key(pb.HashSHA256), hsr, hsr.Size()); err != nil {
 				log.Error("Error saving package blob in content store: %v", err)
@@ -83,7 +113,7 @@ func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pi *packages_servic
 		filename := strings.ToLower(fmt.Sprintf("sha256_%s", pb.HashSHA256))
 
 		pf := &packages_model.PackageFile{
-			VersionID:    pv.ID,
+			VersionID:    uploadVersion.ID,
 			BlobID:       pb.ID,
 			Name:         filename,
 			LowerName:    filename,

diff --git a/routers/api/packages/container/container.go b/routers/api/packages/container/container.go
@@ -10,6 +10,7 @@ import (
 	"io"
 	"net/http"
 	"net/url"
+	"os"
 	"regexp"
 	"strconv"
 	"strings"
@@ -24,6 +25,7 @@ import (
 	container_module "code.gitea.io/gitea/modules/packages/container"
 	"code.gitea.io/gitea/modules/packages/container/oci"
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/util"
 	"code.gitea.io/gitea/routers/api/packages/helper"
 	packages_service "code.gitea.io/gitea/services/packages"
 	container_service "code.gitea.io/gitea/services/packages/container"
@@ -193,7 +195,7 @@ func InitiateUploadBlob(ctx *context.Context) {
 	mount := ctx.FormTrim("mount")
 	from := ctx.FormTrim("from")
 	if mount != "" {
-		blob, _ := container_model.GetContainerBlob(ctx, &container_model.BlobSearchOptions{
+		blob, _ := workaroundGetContainerBlob(ctx, &container_model.BlobSearchOptions{
 			Image:  from,
 			Digest: mount,
 		})
@@ -406,7 +408,7 @@ func getBlobFromContext(ctx *context.Context) (*packages_model.PackageFileDescri
 		return nil, container_model.ErrContainerBlobNotExist
 	}
 
-	return container_model.GetContainerBlob(ctx, &container_model.BlobSearchOptions{
+	return workaroundGetContainerBlob(ctx, &container_model.BlobSearchOptions{
 		OwnerID: ctx.Package.Owner.ID,
 		Image:   ctx.Params("image"),
 		Digest:  digest,
@@ -548,7 +550,7 @@ func getManifestFromContext(ctx *context.Context) (*packages_model.PackageFileDe
 		return nil, container_model.ErrContainerBlobNotExist
 	}
 
-	return container_model.GetContainerBlob(ctx, opts)
+	return workaroundGetContainerBlob(ctx, opts)
 }
 
 // https://github.com/opencontainers/distribution-spec/blob/main/spec.md#checking-if-content-exists-in-the-registry
@@ -688,3 +690,23 @@ func GetTagList(ctx *context.Context) {
 		Tags: tags,
 	})
 }
+
+// FIXME: Workaround to be removed in v1.20
+// https://github.com/go-gitea/gitea/issues/19586
+func workaroundGetContainerBlob(ctx *context.Context, opts *container_model.BlobSearchOptions) (*packages_model.PackageFileDescriptor, error) {
+	blob, err := container_model.GetContainerBlob(ctx, opts)
+	if err != nil {
+		return nil, err
+	}
+
+	err = packages_module.NewContentStore().Has(packages_module.BlobHash256Key(blob.Blob.HashSHA256))
+	if err != nil {
+		if errors.Is(err, util.ErrNotExist) || errors.Is(err, os.ErrNotExist) {
+			log.Debug("Package registry inconsistent: blob %s does not exist on file system", blob.Blob.HashSHA256)
+			return nil, container_model.ErrContainerBlobNotExist
+		}
+		return nil, err
+	}
+
+	return blob, nil
+}
diff --git a/routers/api/packages/container/manifest.go b/routers/api/packages/container/manifest.go
@@ -6,8 +6,10 @@ package container
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
+	"os"
 	"strings"
 
 	"code.gitea.io/gitea/models/db"
@@ -19,6 +21,7 @@ import (
 	packages_module "code.gitea.io/gitea/modules/packages"
 	container_module "code.gitea.io/gitea/modules/packages/container"
 	"code.gitea.io/gitea/modules/packages/container/oci"
+	"code.gitea.io/gitea/modules/util"
 	packages_service "code.gitea.io/gitea/services/packages"
 )
 
@@ -403,6 +406,15 @@ func createManifestBlob(ctx context.Context, mci *manifestCreationInfo, pv *pack
 		log.Error("Error inserting package blob: %v", err)
 		return nil, false, "", err
 	}
+	// FIXME: Workaround to be removed in v1.20
+	// https://github.com/go-gitea/gitea/issues/19586
+	if exists {
+		err = packages_module.NewContentStore().Has(packages_module.BlobHash256Key(pb.HashSHA256))
+		if err != nil && (errors.Is(err, util.ErrNotExist) || errors.Is(err, os.ErrNotExist)) {
+			log.Debug("Package registry inconsistent: blob %s does not exist on file system", pb.HashSHA256)
+			exists = false
+		}
+	}
 	if !exists {
 		contentStore := packages_module.NewContentStore()
 		if err := contentStore.Save(packages_module.BlobHash256Key(pb.HashSHA256), buf, buf.Size()); err != nil {

diff --git a/routers/api/packages/generic/generic.go b/routers/api/packages/generic/generic.go
@@ -53,7 +53,10 @@ func DownloadPackageFile(ctx *context.Context) {
 	}
 	defer s.Close()
 
-	ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime())
+	ctx.ServeContent(s, &context.ServeHeaderOptions{
+		Filename:     pf.Name,
+		LastModified: pf.CreatedUnix.AsLocalTime(),
+	})
 }
 
 // UploadPackage uploads the specific generic package.

diff --git a/routers/api/packages/helm/helm.go b/routers/api/packages/helm/helm.go
@@ -138,7 +138,10 @@ func DownloadPackageFile(ctx *context.Context) {
 	}
 	defer s.Close()
 
-	ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime())
+	ctx.ServeContent(s, &context.ServeHeaderOptions{
+		Filename:     pf.Name,
+		LastModified: pf.CreatedUnix.AsLocalTime(),
+	})
 }
 
 // UploadPackage creates a new package

diff --git a/routers/api/packages/maven/api.go b/routers/api/packages/maven/api.go
@@ -6,7 +6,6 @@ package maven
 
 import (
 	"encoding/xml"
-	"sort"
 	"strings"
 
 	packages_model "code.gitea.io/gitea/models/packages"
@@ -23,12 +22,8 @@ type MetadataResponse struct {
 	Version    []string `xml:"versioning>versions>version"`
 }
 
+// pds is expected to be sorted ascending by CreatedUnix
 func createMetadataResponse(pds []*packages_model.PackageDescriptor) *MetadataResponse {
-	sort.Slice(pds, func(i, j int) bool {
-		// Maven and Gradle order packages by their creation timestamp and not by their version string
-		return pds[i].Version.CreatedUnix < pds[j].Version.CreatedUnix
-	})
-
 	var release *packages_model.PackageDescriptor
 
 	versions := make([]string, 0, len(pds))