DO_NOT_MERGE - CFE-977: Implementation of a blob gatherer for images in the local cache

v2/go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/google/go-containerregistry v0.15.2
 	github.com/google/uuid v1.3.0
 	github.com/microlib/simple v1.0.2
+	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.1.0-rc3
 	github.com/openshift/api v0.0.0-20230223193310-d964c7a58d75
 	github.com/openshift/library-go v0.0.0-20230308200407-f3277c772011
@@ -107,7 +108,6 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/oklog/ulid v1.3.1 // indirect
-	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/runc v1.1.7 // indirect
 	github.com/opencontainers/runtime-spec v1.1.0-rc.3 // indirect
 	github.com/opencontainers/selinux v1.11.0 // indirect

v2/pkg/additional/const.go

@@ -3,4 +3,5 @@ package additional
 const (
 	dockerProtocol string = "docker://"
 	ociProtocol    string = "oci://"
+	hashTruncLen   int    = 12
 )

v2/pkg/additional/local_stored_collector.go

@@ -13,10 +13,6 @@ import (
 	"github.com/openshift/oc-mirror/v2/pkg/mirror"
 )
 
-const (
-	hashTruncLen int = 12
-)
-
 type LocalStorageCollector struct {
 	Log              clog.PluggableLoggerInterface
 	Mirror           mirror.MirrorInterface
@@ -35,27 +31,19 @@ func (o LocalStorageCollector) AdditionalImagesCollector(ctx context.Context) ([
 
 	if o.Opts.IsMirrorToDisk() {
 		for _, img := range o.Config.ImageSetConfigurationSpec.Mirror.AdditionalImages {
-			imgRef := img.Name
-			var src string
-			var dest string
-			if !strings.Contains(imgRef, "://") {
-				src = dockerProtocol + imgRef
-			} else {
-				src = imgRef
-				transportAndRef := strings.Split(imgRef, "://")
-				imgRef = transportAndRef[1]
-			}
-
-			pathWithoutDNS, err := image.PathWithoutDNS(imgRef)
+			imgSpec, err := image.ParseRef(img.Name)
 			if err != nil {
 				o.Log.Error("%s", err.Error())
 				return nil, err
 			}
+			var src string
+			var dest string
+			src = imgSpec.ReferenceWithTransport
 
-			if image.IsImageByDigest(imgRef) {
-				dest = dockerProtocol + strings.Join([]string{o.LocalStorageFQDN, pathWithoutDNS + ":" + image.Hash(imgRef)[:hashTruncLen]}, "/")
+			if imgSpec.IsImageByDigest() {
+				dest = dockerProtocol + strings.Join([]string{o.LocalStorageFQDN, imgSpec.PathComponent + ":" + imgSpec.Digest[:hashTruncLen]}, "/")
 			} else {
-				dest = dockerProtocol + strings.Join([]string{o.LocalStorageFQDN, pathWithoutDNS}, "/")
+				dest = dockerProtocol + strings.Join([]string{o.LocalStorageFQDN, imgSpec.PathComponent}, "/") + ":" + imgSpec.Tag
 			}
 
 			o.Log.Debug("source %s", src)
@@ -71,25 +59,18 @@ func (o LocalStorageCollector) AdditionalImagesCollector(ctx context.Context) ([
 			var dest string
 
 			if !strings.HasPrefix(img.Name, ociProtocol) {
-
-				imgRef := img.Name
-				transportAndRef := strings.Split(imgRef, "://")
-				if len(transportAndRef) > 1 {
-					imgRef = transportAndRef[1]
-				}
-
-				pathWithoutDNS, err := image.PathWithoutDNS(imgRef)
+				imgSpec, err := image.ParseRef(img.Name)
 				if err != nil {
 					o.Log.Error("%s", err.Error())
 					return nil, err
 				}
 
-				if image.IsImageByDigest(imgRef) {
-					src = dockerProtocol + strings.Join([]string{o.LocalStorageFQDN, pathWithoutDNS + ":" + image.Hash(imgRef)[:hashTruncLen]}, "/")
-					dest = strings.Join([]string{o.Opts.Destination, pathWithoutDNS + ":" + image.Hash(imgRef)[:hashTruncLen]}, "/")
+				if imgSpec.IsImageByDigest() {
+					src = dockerProtocol + strings.Join([]string{o.LocalStorageFQDN, imgSpec.PathComponent + ":" + imgSpec.Digest[:hashTruncLen]}, "/")
+					dest = strings.Join([]string{o.Opts.Destination, imgSpec.PathComponent + ":" + imgSpec.Digest[:hashTruncLen]}, "/")
 				} else {
-					src = dockerProtocol + strings.Join([]string{o.LocalStorageFQDN, pathWithoutDNS}, "/")
-					dest = strings.Join([]string{o.Opts.Destination, pathWithoutDNS}, "/")
+					src = dockerProtocol + strings.Join([]string{o.LocalStorageFQDN, imgSpec.PathComponent}, "/") + ":" + imgSpec.Tag
+					dest = strings.Join([]string{o.Opts.Destination, imgSpec.PathComponent}, "/") + ":" + imgSpec.Tag
 				}
 
 			} else {

v2/pkg/archive/blob-gatherer.go

@@ -0,0 +1,55 @@
+package archive
+
+import (
+	"os"
+	"path/filepath"
+
+	digest "github.com/opencontainers/go-digest"
+
+	"github.com/openshift/oc-mirror/v2/pkg/image"
+)
+
+const (
+	repositoriesSubFolder = "docker/registry/v2/repositories"
+)
+
+type StoreBlobGatherer struct {
+	localStorage string
+}
+
+func NewStoreBlobGatherer(localStorageLocation string) BlobsGatherer {
+	return StoreBlobGatherer{
+		localStorage: localStorageLocation,
+	}
+}
+func (o StoreBlobGatherer) GatherBlobs(imgRef string) (map[string]string, error) {
+	blobs := map[string]string{}
+	imgSpec, err := image.ParseRef(imgRef)
+	if err != nil {
+		return nil, err
+	}
+
+	imagePath := filepath.Join(o.localStorage, repositoriesSubFolder, imgSpec.PathComponent)
+
+	err = filepath.Walk(imagePath, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() && info.Name() == "link" {
+			possibleDigest := filepath.Base(filepath.Dir(path))
+
+			if _, err := digest.Parse("sha256:" + possibleDigest); err == nil {
+				blobs[possibleDigest] = ""
+			} else {
+				// this was for a tag
+				return nil
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return blobs, nil
+}

v2/pkg/archive/blob-gatherer_test.go

@@ -0,0 +1,36 @@
+package archive
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestStoreBlobGatherer_GatherBlobs(t *testing.T) {
+
+	// Create a new StoreBlobGatherer with the temporary directory as the local storage location
+	gatherer := NewStoreBlobGatherer("../../tests/cache-fake")
+
+	// Call GatherBlobs with a test image reference
+	actualBlobs, err := gatherer.GatherBlobs("docker://localhost:5000/ubi8/ubi:latest")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Check that the returned blobs map contains the expected key-value pair
+	expectedBlobs := map[string]string{
+		"db870970ba330193164dacc88657df261d75bce1552ea474dbc7cf08b2fae2ed": "",
+		"e6c589cf5f402a60a83a01653304d7a8dcdd47b93a395a797b5622a18904bd66": "",
+		"9b6fa335dba394d437930ad79e308e01da4f624328e49d00c0ff44775d2e4769": "",
+		"6376a0276facf61d87fdf7c6f21d761ee25ba8ceba934d64752d43e84fe0cb98": "",
+		"2e39d55595ea56337b5b788e96e6afdec3db09d2759d903cbe120468187c4644": "",
+		"4c0f6aace7053de3b9c1476b33c9a763e45a099c8c7ae9117773c9a8e5b8506b": "",
+		"53c56977ccd20c0d87df0ad52036c55b27201e1a63874c2644383d0c532f5aee": "",
+		"6e1ac33d11e06db5e850fec4a1ec07f6c2ab15f130c2fdf0f9d0d0a5c83651e7": "",
+		"94343313ec1512ab02267e4bc3ce09eecb01fda5bf26c56e2f028ecc72e80b18": "",
+		"cfaa7496ab546c36ab14859f93fbd2d8a3588b344b18d5fbe74dd834e4a6f7eb": "",
+		"e1bb0572465a9e03d7af5024abb36d7227b5bf133c448b54656d908982127874": "",
+		"f992cb38fce665360a4d07f6f78db864a1f6e20a7ad304219f7f81d7fe608d97": "",
+	}
+	assert.Equal(t, expectedBlobs, actualBlobs)
+}

v2/pkg/archive/interfaces.go

@@ -0,0 +1,5 @@
+package archive
+
+type BlobsGatherer interface {
+	GatherBlobs(imgRef string) (map[string]string, error)
+}

v2/pkg/cli/executor.go

@@ -23,6 +23,7 @@ import (
 	"github.com/openshift/oc-mirror/v2/pkg/additional"
 	"github.com/openshift/oc-mirror/v2/pkg/api/v1alpha2"
 	"github.com/openshift/oc-mirror/v2/pkg/api/v1alpha3"
+	"github.com/openshift/oc-mirror/v2/pkg/archive"
 	"github.com/openshift/oc-mirror/v2/pkg/batch"
 	"github.com/openshift/oc-mirror/v2/pkg/clusterresources"
 	"github.com/openshift/oc-mirror/v2/pkg/config"
@@ -183,6 +184,15 @@ func (o ExecutorSchema) Validate(dest []string) error {
 		return fmt.Errorf("destination must have either file:// (mirror to disk) or docker:// (diskToMirror) protocol prefixes")
 	}
 }
+func (o *ExecutorSchema) CacheRootDir() string {
+	rootDir := ""
+	if o.Opts.IsMirrorToDisk() {
+		rootDir = strings.TrimPrefix(o.Opts.Destination, fileProtocol)
+	} else {
+		rootDir = strings.TrimPrefix(o.Opts.Global.From, fileProtocol)
+	}
+	return rootDir
+}
 
 func (o *ExecutorSchema) PrepareStorageAndLogs() error {
 
@@ -226,13 +236,7 @@ health:
     threshold: 3
 `
 
-	rootDir := ""
-
-	if o.Opts.IsMirrorToDisk() {
-		rootDir = strings.TrimPrefix(o.Opts.Destination, fileProtocol)
-	} else {
-		rootDir = strings.TrimPrefix(o.Opts.Global.From, fileProtocol)
-	}
+	rootDir := o.CacheRootDir()
 
 	if rootDir == "" {
 		// something went wrong
@@ -361,6 +365,9 @@ func (o *ExecutorSchema) Run(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	// make sure we always get multi-arch images
+	o.Opts.MultiArch = "all"
+
 	if o.Opts.IsMirrorToDisk() {
 
 		// ensure working dir exists
@@ -435,9 +442,16 @@ func (o *ExecutorSchema) Run(cmd *cobra.Command, args []string) error {
 	allRelatedImages = mergeImages(allRelatedImages, imgs)
 
 	collectionFinish := time.Now()
-
+	ctx := cmd.Context()
+	blobGatherer := archive.NewStoreBlobGatherer(o.CacheRootDir())
+	blobs, err := blobGatherer.GatherBlobs(allRelatedImages[0].Destination)
+	if err != nil {
+		cleanUp()
+		return err
+	}
+	o.Log.Info("blobs for %s:\n %v ", allRelatedImages[0].Destination, blobs)
 	//call the batch worker
-	err = o.Batch.Worker(cmd.Context(), allRelatedImages, o.Opts)
+	err = o.Batch.Worker(ctx, allRelatedImages, o.Opts)
 	if err != nil {
 		cleanUp()
 		return err

v2/pkg/image/image.go

@@ -5,35 +5,94 @@ import (
 	"strings"
 )
 
-func IsImageByDigest(imgRef string) bool {
-	return strings.Contains(imgRef, "@")
+// specification is sourced from github.com/containers/image/blob/main/docker/reference/reference.go
+// Grammar
+//
+//	reference                       := name [ ":" tag ] [ "@" digest ]
+//	name                            := [domain '/'] path-component ['/' path-component]*
+//	domain                          := domain-component ['.' domain-component]* [':' port-number]
+//	domain-component                := /([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])/
+//	port-number                     := /[0-9]+/
+//	path-component                  := alphanumeric [separator alphanumeric]*
+//	alphanumeric                   := /[a-z0-9]+/
+//	separator                       := /[_.]|__|[-]*/
+//
+//	tag                             := /[\w][\w.-]{0,127}/
+//
+//	digest                          := digest-algorithm ":" digest-hex
+//	digest-algorithm                := digest-algorithm-component [ digest-algorithm-separator digest-algorithm-component ]*
+//	digest-algorithm-separator      := /[+.-_]/
+//	digest-algorithm-component      := /[A-Za-z][A-Za-z0-9]*/
+//	digest-hex                      := /[0-9a-fA-F]{32,}/ ; At least 128 bit digest value
+//
+//	identifier                      := /[a-f0-9]{64}/
+//	short-identifier                := /[a-f0-9]{6,64}/
+type ImageSpec struct {
+	Transport              string
+	Reference              string
+	ReferenceWithTransport string
+	Name                   string
+	Domain                 string
+	PathComponent          string
+	Tag                    string
+	Digest                 string
 }
 
-func PathWithoutDNS(imgRef string) (string, error) {
-
-	var imageName []string
-	if IsImageByDigest(imgRef) {
-		imageNameSplit := strings.Split(imgRef, "@")
-		imageName = strings.Split(imageNameSplit[0], "/")
+// It expects the image reference not to have the transport prefix.
+// Otherwise, it will return an error.
+func ParseRef(imgRef string) (ImageSpec, error) {
+	var imgSpec ImageSpec
+	if strings.Contains(imgRef, "://") {
+		imgSpec.ReferenceWithTransport = imgRef
+		imgSplit := strings.Split(imgRef, "://")
+		imgSpec.Transport = imgSplit[0] + "://"
+		imgSpec.Reference = imgSplit[1]
+		imgSpec.Name = imgSplit[1]
 	} else {
-		imageName = strings.Split(imgRef, "/")
+		imgSpec.Transport = "docker://"
+		imgSpec.Reference = imgRef
+		imgSpec.Name = imgRef
+		imgSpec.ReferenceWithTransport = imgSpec.Transport + imgRef
+	}
+	if strings.Contains(imgSpec.Name, "@") {
+		imgSplit := strings.Split(imgSpec.Name, "@")
+		if len(imgSplit) > 1 {
+			imgSpec.Digest = strings.Split(imgSplit[1], ":")[1]
+			imgSpec.Name = imgSplit[0]
+		}
+	} else if strings.Contains(imgSpec.Name, ":") {
+		lastColonIndex := strings.LastIndex(imgSpec.Name, ":")
+		imgSpec.Tag = imgSpec.Name[lastColonIndex+1:]
+		imgSpec.Name = imgSpec.Name[:lastColonIndex]
 	}
 
-	if len(imageName) > 2 {
-		return strings.Join(imageName[1:], "/"), nil
-	} else if len(imageName) == 1 {
-		return imageName[0], nil
-	} else {
-		return "", fmt.Errorf("unable to parse image %s correctly", imgRef)
+	if imgSpec.Name == "" {
+		return ImageSpec{}, fmt.Errorf("unable to parse image %s correctly", imgRef)
+	}
+	if imgSpec.Transport == "docker://" && imgSpec.Tag == "" && imgSpec.Digest == "" {
+		return ImageSpec{}, fmt.Errorf("unable to parse image %s correctly", imgRef)
 	}
-}
 
-func Hash(imgRef string) string {
-	var hash string
-	imgSplit := strings.Split(imgRef, "@")
-	if len(imgSplit) > 1 {
-		hash = strings.Split(imgSplit[1], ":")[1]
+	if imgSpec.Transport == "docker://" {
+		imageNameComponents := strings.Split(imgSpec.Name, "/")
+		if len(imageNameComponents) > 2 {
+			imgSpec.PathComponent = strings.Join(imageNameComponents[1:], "/")
+			imgSpec.Domain = imageNameComponents[0]
+		} else if len(imageNameComponents) == 1 {
+			imgSpec.PathComponent = imageNameComponents[0]
+		} else {
+			return ImageSpec{}, fmt.Errorf("unable to parse image %s correctly", imgRef)
+		}
+	} else {
+		imgSpec.PathComponent = imgSpec.Name
 	}
 
-	return hash
+	return imgSpec, nil
+}
+
+// TODO this might need to change when implementing OCI images
+// because the digest is not in the ImageSpec, but there should be
+// a way to find it
+func (i ImageSpec) IsImageByDigest() bool {
+	return i.Digest != ""
 }