implement artifact registry with rpm support

Signed-off-by: Adphi <philippe.adrien.nousse@gmail.com>

.gitignore

@@ -0,0 +1,3 @@
+.idea
+tmp
+.env

cmd/artifact-registry/main.go

@@ -0,0 +1,115 @@
+package main
+
+import (
+	"context"
+	"crypto/sha256"
+	"errors"
+	"net/http"
+	"os"
+	"time"
+
+	"github.com/gorilla/mux"
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+
+	"go.linka.cloud/artifact-registry/pkg/packages"
+)
+
+const (
+	EnvAddr    = "ARTIFACT_REGISTRY_ADDRESS"
+	EnvBackend = "ARTIFACT_REGISTRY_BACKEND"
+	EnvKey     = "ARTIFACT_REGISTRY_KEY"
+)
+
+var (
+	addr = ":9887"
+	// backend = "192.168.10.11:5000"
+	backend = "docker.io"
+
+	key = ""
+
+	cmd = &cobra.Command{
+		Use:          "artifact-registry",
+		SilenceUsage: true,
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := run(cmd.Context()); err != nil {
+				logrus.Fatal(err)
+			}
+		},
+	}
+)
+
+func main() {
+	cmd.Flags().StringVar(&addr, "addr", envDefault(EnvAddr, addr), "address to listen on [$"+EnvAddr+"]")
+	cmd.Flags().StringVar(&backend, "backend", envDefault(EnvBackend, backend), "registry backend [$"+EnvBackend+"]")
+	cmd.Flags().StringVar(&key, "key", envDefault(EnvKey, key), "key to encrypt the repositories keys [$"+EnvKey+"]")
+	if err := cmd.Execute(); err != nil {
+		logrus.Fatal(err)
+	}
+}
+
+type wrapWriter struct {
+	http.ResponseWriter
+	status int
+	size   int
+}
+
+func (w *wrapWriter) WriteHeader(statusCode int) {
+	w.ResponseWriter.WriteHeader(statusCode)
+	w.status = statusCode
+}
+
+func (w *wrapWriter) Write(b []byte) (int, error) {
+	n, err := w.ResponseWriter.Write(b)
+	if err != nil {
+		return 0, err
+	}
+	w.size += n
+	return n, nil
+}
+
+func run(ctx context.Context) error {
+	if key == "" {
+		return errors.New("key must be set")
+	}
+	logrus.Infof("intializing artifact registry using backend %s", backend)
+	r := mux.NewRouter()
+	r.Use(func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			wrap := &wrapWriter{ResponseWriter: w}
+			start := time.Now()
+			next.ServeHTTP(wrap, r)
+			time.Since(start)
+			status := wrap.status
+			if status == 0 {
+				status = 200
+			}
+			logrus.WithFields(logrus.Fields{
+				"method":    r.Method,
+				"path":      r.URL.Path,
+				"remote":    r.RemoteAddr,
+				"duration":  time.Since(start),
+				"status":    status,
+				"size":      wrap.size,
+				"userAgent": r.UserAgent(),
+			}).Infof("")
+		})
+	})
+	h := sha256.New()
+	h.Write([]byte(key))
+	if err := packages.Init(ctx, r, backend, h.Sum(nil)); err != nil {
+		return err
+	}
+	logrus.Infof("starting server at %s", addr)
+	if err := http.ListenAndServe(addr, r); err != nil {
+		return err
+	}
+	return nil
+}
+
+func envDefault(key string, def string) string {
+	if v, ok := os.LookupEnv(key); ok {
+		return v
+	}
+	return def
+}

cmd/artifact-registry/packages.go

@@ -0,0 +1,5 @@
+package main
+
+import (
+	_ "go.linka.cloud/artifact-registry/pkg/packages/rpm"
+)

go.mod

@@ -0,0 +1,29 @@
+module go.linka.cloud/artifact-registry
+
+go 1.21
+
+replace code.gitea.io/gitea => github.com/go-gitea/gitea v1.20.4
+
+require (
+	github.com/dustin/go-humanize v1.0.1
+	github.com/gorilla/mux v1.8.0
+	github.com/opencontainers/go-digest v1.0.0
+	github.com/opencontainers/image-spec v1.1.0-rc4
+	github.com/sassoftware/go-rpmutils v0.2.0
+	github.com/sirupsen/logrus v1.9.3
+	github.com/spf13/cobra v1.7.0
+	golang.org/x/crypto v0.7.0
+	oras.land/oras-go/v2 v2.3.0
+)
+
+require (
+	github.com/DataDog/zstd v1.4.5 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/klauspost/compress v1.11.7 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/ulikunitz/xz v0.5.9 // indirect
+	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
+	golang.org/x/sync v0.3.0 // indirect
+	golang.org/x/sys v0.6.0 // indirect
+	golang.org/x/tools v0.6.0 // indirect
+)

go.sum

@@ -0,0 +1,74 @@
+github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ=
+github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
+github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
+github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/klauspost/compress v1.11.7 h1:0hzRabrMN4tSTvMfnL3SCv1ZGeAP23ynzodBgaHeMeg=
+github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
+github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
+github.com/opencontainers/image-spec v1.1.0-rc4 h1:oOxKUJWnFC4YGHCCMNql1x4YaDfYBTS5Y4x/Cgeo1E0=
+github.com/opencontainers/image-spec v1.1.0-rc4/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/sassoftware/go-rpmutils v0.2.0 h1:pKW0HDYMFWQ5b4JQPiI3WI12hGsVoW0V8+GMoZiI/JE=
+github.com/sassoftware/go-rpmutils v0.2.0/go.mod h1:TJJQYtLe/BeEmEjelI3b7xNZjzAukEkeWKmoakvaOoI=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
+github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/ulikunitz/xz v0.5.9 h1:RsKRIA2MO8x56wkkcd3LbtcE/uMszhb6DpRf+3uwa3I=
+github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
+github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo=
+github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
+go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0=
+go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A=
+golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
+golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+oras.land/oras-go/v2 v2.3.0 h1:lqX1aXdN+DAmDTKjiDyvq85cIaI4RkIKp/PghWlAGIU=
+oras.land/oras-go/v2 v2.3.0/go.mod h1:GeAwLuC4G/JpNwkd+bSZ6SkDMGaaYglt6YK2WvZP7uQ=

pkg/buffer/file_buffer.go

@@ -0,0 +1,155 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package buffer
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"math"
+	"os"
+)
+
+var (
+	// ErrInvalidMemorySize occurs if the memory size is not in a valid range
+	ErrInvalidMemorySize = errors.New("Memory size must be greater 0 and lower math.MaxInt32")
+	// ErrWriteAfterRead occurs if Write is called after a read operation
+	ErrWriteAfterRead = errors.New("Write is unsupported after a read operation")
+)
+
+type readAtSeeker interface {
+	io.ReadSeeker
+	io.ReaderAt
+}
+
+// FileBackedBuffer uses a memory buffer with a fixed size.
+// If more data is written a temporary file is used instead.
+// It implements io.ReadWriteCloser, io.ReadSeekCloser and io.ReaderAt
+type FileBackedBuffer struct {
+	maxMemorySize int64
+	size          int64
+	buffer        bytes.Buffer
+	file          *os.File
+	reader        readAtSeeker
+}
+
+// NewFile creates a file backed buffer with a specific maximum memory size
+func NewFile(maxMemorySize int) (*FileBackedBuffer, error) {
+	if maxMemorySize < 0 || maxMemorySize > math.MaxInt32 {
+		return nil, ErrInvalidMemorySize
+	}
+
+	return &FileBackedBuffer{
+		maxMemorySize: int64(maxMemorySize),
+	}, nil
+}
+
+// CreateFromReader creates a file backed buffer and copies the provided reader data into it.
+func CreateFromReader(r io.Reader, maxMemorySize int) (*FileBackedBuffer, error) {
+	b, err := NewFile(maxMemorySize)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = io.Copy(b, r)
+	if err != nil {
+		return nil, err
+	}
+
+	return b, nil
+}
+
+// Write implements io.Writer
+func (b *FileBackedBuffer) Write(p []byte) (int, error) {
+	if b.reader != nil {
+		return 0, ErrWriteAfterRead
+	}
+
+	var n int
+	var err error
+
+	if b.file != nil {
+		n, err = b.file.Write(p)
+	} else {
+		if b.size+int64(len(p)) > b.maxMemorySize {
+			b.file, err = os.CreateTemp("", "gitea-buffer-")
+			if err != nil {
+				return 0, err
+			}
+
+			_, err = io.Copy(b.file, &b.buffer)
+			if err != nil {
+				return 0, err
+			}
+
+			return b.Write(p)
+		}
+
+		n, err = b.buffer.Write(p)
+	}
+
+	if err != nil {
+		return n, err
+	}
+	b.size += int64(n)
+	return n, nil
+}
+
+// Size returns the byte size of the buffered data
+func (b *FileBackedBuffer) Size() int64 {
+	return b.size
+}
+
+func (b *FileBackedBuffer) switchToReader() error {
+	if b.reader != nil {
+		return nil
+	}
+
+	if b.file != nil {
+		if _, err := b.file.Seek(0, io.SeekStart); err != nil {
+			return err
+		}
+		b.reader = b.file
+	} else {
+		b.reader = bytes.NewReader(b.buffer.Bytes())
+	}
+	return nil
+}
+
+// Read implements io.Reader
+func (b *FileBackedBuffer) Read(p []byte) (int, error) {
+	if err := b.switchToReader(); err != nil {
+		return 0, err
+	}
+
+	return b.reader.Read(p)
+}
+
+// ReadAt implements io.ReaderAt
+func (b *FileBackedBuffer) ReadAt(p []byte, off int64) (int, error) {
+	if err := b.switchToReader(); err != nil {
+		return 0, err
+	}
+
+	return b.reader.ReadAt(p, off)
+}
+
+// Seek implements io.Seeker
+func (b *FileBackedBuffer) Seek(offset int64, whence int) (int64, error) {
+	if err := b.switchToReader(); err != nil {
+		return 0, err
+	}
+
+	return b.reader.Seek(offset, whence)
+}
+
+// Close implements io.Closer
+func (b *FileBackedBuffer) Close() error {
+	if b.file != nil {
+		err := b.file.Close()
+		os.Remove(b.file.Name())
+		return err
+	}
+	return nil
+}