Merge pull request #460 from fluxcd/oci-media-type

oci: Introduce Flux media types

oci/client/push.go

@@ -28,6 +28,10 @@ import (
 	gcrv1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/empty"
 	"github.com/google/go-containerregistry/pkg/v1/mutate"
+	"github.com/google/go-containerregistry/pkg/v1/tarball"
+	"github.com/google/go-containerregistry/pkg/v1/types"
+
+	"github.com/fluxcd/pkg/oci"
 )
 
 // Push creates an artifact from the given directory, uploads the artifact
@@ -50,14 +54,22 @@ func (c *Client) Push(ctx context.Context, url, sourceDir string, meta Metadata,
 		return "", err
 	}
 
-	img, err := crane.Append(empty.Image, tmpFile)
+	ct := time.Now().UTC()
+	meta.Created = ct.Format(time.RFC3339)
+
+	img := mutate.MediaType(empty.Image, types.OCIManifestSchema1)
+	img = mutate.ConfigMediaType(img, oci.ConfigMediaType)
+	img = mutate.Annotations(img, meta.ToAnnotations()).(gcrv1.Image)
+
+	layer, err := tarball.LayerFromFile(tmpFile, tarball.WithMediaType(oci.ContentMediaType))
 	if err != nil {
-		return "", fmt.Errorf("appeding content to artifact failed: %w", err)
+		return "", fmt.Errorf("creating content layer failed: %w", err)
 	}
 
-	ct := time.Now()
-	meta.Created = ct.Format(time.RFC3339)
-	img = mutate.Annotations(img, meta.ToAnnotations()).(gcrv1.Image)
+	img, err = mutate.Append(img, mutate.Addendum{Layer: layer})
+	if err != nil {
+		return "", fmt.Errorf("appeding content to artifact failed: %w", err)
+	}
 
 	if err := crane.Push(img, url, c.optionsWithContext(ctx)...); err != nil {
 		return "", fmt.Errorf("pushing artifact failed: %w", err)

oci/client/push_pull_test.go

@@ -22,9 +22,11 @@ import (
 	"io/fs"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 
 	"github.com/google/go-containerregistry/pkg/crane"
+	"github.com/google/go-containerregistry/pkg/v1/types"
 	. "github.com/onsi/gomega"
 
 	"github.com/fluxcd/pkg/oci"
@@ -34,44 +36,63 @@ func Test_Push_Pull(t *testing.T) {
 	g := NewWithT(t)
 	ctx := context.Background()
 	c := NewLocalClient()
+	testDir := "testdata/artifact"
 	tag := "v0.0.1"
+	source := "github.com/fluxcd/flux2"
+	revision := "rev"
 	repo := "test-push" + randStringRunes(5)
 
 	url := fmt.Sprintf("%s/%s:%s", dockerReg, repo, tag)
 	metadata := Metadata{
-		Source:   "github.com/fluxcd/flux2",
-		Revision: "rev",
+		Source:   source,
+		Revision: revision,
 	}
 
-	testDir := "testdata/artifact"
+	// Build and push the artifact to registry
 	_, err := c.Push(ctx, url, testDir, metadata, nil)
 	g.Expect(err).ToNot(HaveOccurred())
 
+	// Verify that the artifact and its tag is present in the registry
 	tags, err := crane.ListTags(fmt.Sprintf("%s/%s", dockerReg, repo))
 	g.Expect(err).ToNot(HaveOccurred())
 	g.Expect(len(tags)).To(BeEquivalentTo(1))
+	g.Expect(tags[0]).To(BeEquivalentTo(tag))
 
+	// Pull the artifact from registry
 	image, err := crane.Pull(fmt.Sprintf("%s/%s:%s", dockerReg, repo, tag))
 	g.Expect(err).ToNot(HaveOccurred())
 
+	// Extract the manifest from the pulled artifact
 	manifest, err := image.Manifest()
 	g.Expect(err).ToNot(HaveOccurred())
 
+	// Verify that annotations exist in manifest
 	g.Expect(manifest.Annotations[oci.CreatedAnnotation]).ToNot(BeEmpty())
-	g.Expect(manifest.Annotations[oci.SourceAnnotation]).ToNot(BeEmpty())
-	g.Expect(manifest.Annotations[oci.RevisionAnnotation]).ToNot(BeEmpty())
+	g.Expect(manifest.Annotations[oci.SourceAnnotation]).To(BeEquivalentTo(source))
+	g.Expect(manifest.Annotations[oci.RevisionAnnotation]).To(BeEquivalentTo(revision))
+
+	// Verify media types
+	g.Expect(manifest.MediaType).To(Equal(types.OCIManifestSchema1))
+	g.Expect(manifest.Config.MediaType).To(BeEquivalentTo(oci.ConfigMediaType))
+	g.Expect(len(manifest.Layers)).To(BeEquivalentTo(1))
+	g.Expect(manifest.Layers[0].MediaType).To(BeEquivalentTo(oci.ContentMediaType))
 
 	tmpDir := t.TempDir()
+
+	// Pull the artifact from registry and extract its contents to tmp
 	_, err = c.Pull(ctx, url, tmpDir)
 	g.Expect(err).ToNot(HaveOccurred())
 
-	// Walk directory the test directory and check that all paths exists in the extracted archive
-	err = filepath.Walk(testDir, func(path string, info fs.FileInfo, err error) error {
-		tmpPath := filepath.Join(tmpDir, path)
-		if _, err := os.Stat(tmpPath); err != nil && os.IsNotExist(err) {
-			return fmt.Errorf("path '%s' doesn't exist in archive", path)
+	// Walk the test directory and check that all files exist in the pulled artifact
+	fsErr := filepath.Walk(testDir, func(path string, info fs.FileInfo, err error) error {
+		if !info.IsDir() {
+			tmpPath := filepath.Join(tmpDir, strings.TrimPrefix(path, testDir))
+			if _, err := os.Stat(tmpPath); err != nil && os.IsNotExist(err) {
+				return fmt.Errorf("path '%s' doesn't exist in archive", path)
+			}
 		}
 
 		return nil
 	})
+	g.Expect(fsErr).ToNot(HaveOccurred())
 }

oci/client/suite_test.go

@@ -19,16 +19,19 @@ package client
 import (
 	"context"
 	"fmt"
+	"io"
 	"math/rand"
 	"os"
 	"testing"
 	"time"
 
 	"github.com/distribution/distribution/v3/configuration"
+	dcontext "github.com/distribution/distribution/v3/context"
 	"github.com/distribution/distribution/v3/registry"
 	_ "github.com/distribution/distribution/v3/registry/auth/htpasswd"
 	_ "github.com/distribution/distribution/v3/registry/storage/driver/inmemory"
 	"github.com/phayes/freeport"
+	"github.com/sirupsen/logrus"
 	ctrl "sigs.k8s.io/controller-runtime"
 )
 
@@ -43,6 +46,11 @@ func init() {
 func setupRegistryServer(ctx context.Context) error {
 	// Registry config
 	config := &configuration.Configuration{}
+	config.Log.AccessLog.Disabled = true
+	config.Log.Level = "error"
+	logger := logrus.New()
+	logger.SetOutput(io.Discard)
+	dcontext.SetDefaultLogger(logrus.NewEntry(logger))
 	port, err := freeport.GetFreePort()
 	if err != nil {
 		return fmt.Errorf("failed to get free port: %s", err)

oci/constants.go

@@ -37,6 +37,12 @@ const (
 )
 
 const (
+	// ConfigMediaType is the OCI media type for the config layer.
+	ConfigMediaType = "application/vnd.cncf.flux.config.v1+json"
+
+	// ContentMediaType is the OCI media type for the content layer.
+	ContentMediaType = "application/vnd.cncf.flux.content.v1.tar+gzip"
+
 	// SourceAnnotation is the OpenContainers annotation for specifying
 	// the upstream source of an OCI artifact.
 	SourceAnnotation = "org.opencontainers.image.source"

oci/go.mod

@@ -9,36 +9,37 @@ replace (
 )
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.1
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.1
 	github.com/Masterminds/semver/v3 v3.2.0
-	github.com/aws/aws-sdk-go-v2 v1.17.3
-	github.com/aws/aws-sdk-go-v2/config v1.18.10
-	github.com/aws/aws-sdk-go-v2/credentials v1.13.10
-	github.com/aws/aws-sdk-go-v2/service/ecr v1.18.1
-	github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2
+	github.com/aws/aws-sdk-go-v2 v1.17.4
+	github.com/aws/aws-sdk-go-v2/config v1.18.12
+	github.com/aws/aws-sdk-go-v2/credentials v1.13.12
+	github.com/aws/aws-sdk-go-v2/service/ecr v1.18.2
+	github.com/distribution/distribution/v3 v3.0.0-20230131081513-cf87e8d07e8d
 	github.com/fluxcd/pkg/sourceignore v0.3.0
 	github.com/fluxcd/pkg/tar v0.2.0
 	github.com/fluxcd/pkg/version v0.2.0
 	github.com/google/go-containerregistry v0.13.0
 	github.com/onsi/gomega v1.26.0
 	github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5
-	sigs.k8s.io/controller-runtime v0.14.1
+	github.com/sirupsen/logrus v1.9.0
+	sigs.k8s.io/controller-runtime v0.14.4
 )
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.1 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v0.8.1 // indirect
 	github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d // indirect
 	github.com/acomagu/bufpipe v1.0.3 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.21 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.12.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.18.2 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.22 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.28 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.22 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.29 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.22 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.12.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.18.3 // indirect
 	github.com/aws/smithy-go v1.13.5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bshuster-repo/logrus-logstash-hook v1.0.0 // indirect
@@ -101,7 +102,6 @@ require (
 	github.com/prometheus/client_model v0.3.0 // indirect
 	github.com/prometheus/common v0.37.0 // indirect
 	github.com/prometheus/procfs v0.8.0 // indirect
-	github.com/sirupsen/logrus v1.9.0 // indirect
 	github.com/spf13/cobra v1.6.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/vbatts/tar-split v0.11.2 // indirect
@@ -123,11 +123,11 @@ require (
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/api v0.26.0 // indirect
-	k8s.io/apiextensions-apiserver v0.26.0 // indirect
-	k8s.io/apimachinery v0.26.0 // indirect
-	k8s.io/client-go v0.26.0 // indirect
-	k8s.io/component-base v0.26.0 // indirect
+	k8s.io/api v0.26.1 // indirect
+	k8s.io/apiextensions-apiserver v0.26.1 // indirect
+	k8s.io/apimachinery v0.26.1 // indirect
+	k8s.io/client-go v0.26.1 // indirect
+	k8s.io/component-base v0.26.1 // indirect
 	k8s.io/klog/v2 v2.80.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect
 	k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 // indirect