Implementation of a blob gatherer for images in the local cache

v2/go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/google/go-containerregistry v0.15.2
 	github.com/google/uuid v1.3.0
 	github.com/microlib/simple v1.0.2
+	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.1.0-rc3
 	github.com/openshift/api v0.0.0-20230223193310-d964c7a58d75
 	github.com/openshift/library-go v0.0.0-20230308200407-f3277c772011
@@ -107,7 +108,6 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/oklog/ulid v1.3.1 // indirect
-	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/runc v1.1.7 // indirect
 	github.com/opencontainers/runtime-spec v1.1.0-rc.3 // indirect
 	github.com/opencontainers/selinux v1.11.0 // indirect

v2/pkg/archive/blob-gatherer.go

@@ -0,0 +1,97 @@
+package archive
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	digest "github.com/opencontainers/go-digest"
+)
+
+const (
+	repositoriesSubFolder = "docker/registry/v2/repositories"
+)
+
+type StoreBlobGatherer struct {
+	localStorage string
+}
+
+func NewStoreBlobGatherer(localStorageLocation string) BlobsGatherer {
+	return StoreBlobGatherer{
+		localStorage: localStorageLocation,
+	}
+}
+func (o StoreBlobGatherer) GatherBlobs(imgRef string) (map[string]string, error) {
+	blobs := map[string]string{}
+	imageSubPath, err := imagePathComponents(imgRef)
+	if err != nil {
+		return nil, err
+	}
+	imagePath := filepath.Join(o.localStorage, repositoriesSubFolder, imageSubPath)
+
+	err = filepath.Walk(imagePath, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() && info.Name() == "link" {
+			possibleDigest := filepath.Base(filepath.Dir(path))
+
+			if _, err := digest.Parse("sha256:" + possibleDigest); err == nil {
+				blobs[possibleDigest] = ""
+			} else {
+				// this was for a tag
+				return nil
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return blobs, nil
+}
+
+func imagePathComponents(imgRef string) (string, error) {
+	var imageName []string
+	if strings.Contains(imgRef, "://") {
+		transportAndRef := strings.Split(imgRef, "://")
+		imgRef = transportAndRef[1]
+	}
+
+	imageName = strings.Split(imgRef, "/")
+
+	if len(imageName) > 2 {
+		imgRef = strings.Join(imageName[1:], "/")
+	} else if len(imageName) == 1 {
+		imgRef = imageName[0]
+	} else {
+		return "", fmt.Errorf("unable to parse image %s correctly", imgRef)
+	}
+
+	if strings.Contains(imgRef, "@") {
+		nameAndDigest := strings.Split(imgRef, "@")
+		imgRef = nameAndDigest[0]
+	}
+
+	if strings.Contains(imgRef, ":") {
+		nameAndTag := strings.Split(imgRef, ":")
+		imgRef = nameAndTag[0]
+	}
+	return imgRef, nil
+}
+
+func isImageByDigest(imgRef string) bool {
+	return strings.Contains(imgRef, "@")
+}
+
+// func imageHash(imgRef string) string {
+// 	var hash string
+// 	imgSplit := strings.Split(imgRef, "@")
+// 	if len(imgSplit) > 1 {
+// 		hash = strings.Split(imgSplit[1], ":")[1]
+// 	}
+
+// 	return hash
+// }

v2/pkg/archive/blob-gatherer_test.go

@@ -0,0 +1,88 @@
+package archive
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestStoreBlobGatherer_imagePathComponents(t *testing.T) {
+	imgRefs := []string{
+		"docker://localhost:5000/ubi8/ubi:latest",
+		"docker://localhost:5000/ubi8/ubi@sha256:db870970ba330193164dacc88657df261d75bce1552ea474dbc7cf08b2fae2ed",
+		"docker://registry.redhat.io/ubi8/ubi:latest",
+		"docker://registry.redhat.io/ubi8/ubi@sha256:db870970ba330193164dacc88657df261d75bce1552ea474dbc7cf08b2fae2ed",
+		"localhost:5000/ubi8/ubi:latest",
+		"localhost:5000/ubi8/ubi@sha256:db870970ba330193164dacc88657df261d75bce1552ea474dbc7cf08b2fae2ed",
+		"file:///tmp/ubi8/ubi",
+		"oci:///tmp/ubi8/ubi",
+	}
+	expectedPathComponents := []string{
+		"ubi8/ubi",
+		"ubi8/ubi",
+		"ubi8/ubi",
+		"ubi8/ubi",
+		"ubi8/ubi",
+		"ubi8/ubi",
+		"tmp/ubi8/ubi",
+		"tmp/ubi8/ubi",
+	}
+
+	for i, imgRef := range imgRefs {
+		actualPathComponents, err := imagePathComponents(imgRef)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if actualPathComponents != expectedPathComponents[i] {
+			t.Errorf("imagePathComponents() returned unexpected value for %q: got %q, want %q", imgRef, actualPathComponents, expectedPathComponents[i])
+		}
+	}
+}
+
+func TestStoreBlobGatherer_isImageByDigest(t *testing.T) {
+	imgRefs := []string{
+		"docker://localhost:5000/ubi8/ubi:latest",
+		"docker://localhost:5000/ubi8/ubi@sha256:db870970ba330193164dacc88657df261d75bce1552ea474dbc7cf08b2fae2ed",
+	}
+	expectedisByDigest := []bool{
+		false,
+		true,
+	}
+
+	for i, imgRef := range imgRefs {
+		actualIsByDigest := isImageByDigest(imgRef)
+
+		if actualIsByDigest != expectedisByDigest[i] {
+			t.Errorf("isImageByDigest() returned unexpected value for %q: got %v, want %v", imgRef, actualIsByDigest, expectedisByDigest[i])
+		}
+	}
+}
+
+func TestStoreBlobGatherer_GatherBlobs(t *testing.T) {
+
+	// Create a new StoreBlobGatherer with the temporary directory as the local storage location
+	gatherer := NewStoreBlobGatherer("../../tests/cache-fake")
+
+	// Call GatherBlobs with a test image reference
+	actualBlobs, err := gatherer.GatherBlobs("docker://localhost:5000/ubi8/ubi:latest")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Check that the returned blobs map contains the expected key-value pair
+	expectedBlobs := map[string]string{
+		"db870970ba330193164dacc88657df261d75bce1552ea474dbc7cf08b2fae2ed": "",
+		"e6c589cf5f402a60a83a01653304d7a8dcdd47b93a395a797b5622a18904bd66": "",
+		"9b6fa335dba394d437930ad79e308e01da4f624328e49d00c0ff44775d2e4769": "",
+		"6376a0276facf61d87fdf7c6f21d761ee25ba8ceba934d64752d43e84fe0cb98": "",
+		"2e39d55595ea56337b5b788e96e6afdec3db09d2759d903cbe120468187c4644": "",
+		"4c0f6aace7053de3b9c1476b33c9a763e45a099c8c7ae9117773c9a8e5b8506b": "",
+		"53c56977ccd20c0d87df0ad52036c55b27201e1a63874c2644383d0c532f5aee": "",
+		"6e1ac33d11e06db5e850fec4a1ec07f6c2ab15f130c2fdf0f9d0d0a5c83651e7": "",
+		"94343313ec1512ab02267e4bc3ce09eecb01fda5bf26c56e2f028ecc72e80b18": "",
+		"cfaa7496ab546c36ab14859f93fbd2d8a3588b344b18d5fbe74dd834e4a6f7eb": "",
+		"e1bb0572465a9e03d7af5024abb36d7227b5bf133c448b54656d908982127874": "",
+		"f992cb38fce665360a4d07f6f78db864a1f6e20a7ad304219f7f81d7fe608d97": "",
+	}
+	assert.Equal(t, expectedBlobs, actualBlobs)
+}

v2/pkg/archive/interfaces.go

@@ -0,0 +1,5 @@
+package archive
+
+type BlobsGatherer interface {
+	GatherBlobs(imgRef string) (map[string]string, error)
+}

v2/pkg/cli/executor.go

@@ -23,6 +23,7 @@ import (
 	"github.com/openshift/oc-mirror/v2/pkg/additional"
 	"github.com/openshift/oc-mirror/v2/pkg/api/v1alpha2"
 	"github.com/openshift/oc-mirror/v2/pkg/api/v1alpha3"
+	"github.com/openshift/oc-mirror/v2/pkg/archive"
 	"github.com/openshift/oc-mirror/v2/pkg/batch"
 	"github.com/openshift/oc-mirror/v2/pkg/clusterresources"
 	"github.com/openshift/oc-mirror/v2/pkg/config"
@@ -186,6 +187,15 @@ func (o *ExecutorSchema) Validate(dest []string) error {
 		return fmt.Errorf("destination must have either file:// (mirror to disk) or docker:// (diskToMirror) protocol prefixes")
 	}
 }
+func (o *ExecutorSchema) CacheRootDir() string {
+	rootDir := ""
+	if o.Opts.Mode == mirrorToDisk {
+		rootDir = strings.TrimPrefix(o.Opts.Destination, fileProtocol)
+	} else {
+		rootDir = strings.TrimPrefix(o.Opts.Global.From, fileProtocol)
+	}
+	return rootDir
+}
 
 func (o *ExecutorSchema) PrepareStorageAndLogs() error {
 
@@ -229,13 +239,7 @@ health:
     threshold: 3
 `
 
-	rootDir := ""
-
-	if o.Opts.Mode == mirrorToDisk {
-		rootDir = strings.TrimPrefix(o.Opts.Destination, fileProtocol)
-	} else {
-		rootDir = strings.TrimPrefix(o.Opts.Global.From, fileProtocol)
-	}
+	rootDir := o.CacheRootDir()
 
 	if rootDir == "" {
 		// something went wrong
@@ -364,6 +368,9 @@ func (o *ExecutorSchema) Run(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	// make sure we always get multi-arch images
+	o.Opts.MultiArch = "all"
+
 	if o.Opts.Mode == mirrorToDisk {
 
 		// ensure working dir exists
@@ -438,9 +445,16 @@ func (o *ExecutorSchema) Run(cmd *cobra.Command, args []string) error {
 	allRelatedImages = mergeImages(allRelatedImages, imgs)
 
 	collectionFinish := time.Now()
-
+	ctx := cmd.Context()
+	blobGatherer := archive.NewStoreBlobGatherer(o.CacheRootDir())
+	blobs, err := blobGatherer.GatherBlobs(allRelatedImages[0].Destination)
+	if err != nil {
+		cleanUp()
+		return err
+	}
+	o.Log.Info("blobs for %s:\n %v ", allRelatedImages[0].Destination, blobs)
 	//call the batch worker
-	err = o.Batch.Worker(cmd.Context(), allRelatedImages, o.Opts)
+	err = o.Batch.Worker(ctx, allRelatedImages, o.Opts)
 	if err != nil {
 		cleanUp()
 		return err

v2/pkg/mirror/mirror.go

@@ -356,7 +356,3 @@ func parseMultiArch(multiArch string) (copy.ImageListSelection, error) {
 		return copy.CopySystemImage, fmt.Errorf("unknown multi-arch option %q. Choose one of the supported options: 'system', 'all', or 'index-only'", multiArch)
 	}
 }
-
-func HelloFromV2() string {
-	return "Hello From V2"
-}

v2/pkg/release/graph_test.go

@@ -15,9 +15,9 @@ func TestCreateGraphImage(t *testing.T) {
 
 	log := clog.New("trace")
 	globalM2D := &mirror.GlobalOptions{
-		TlsVerify:      false,
-		InsecurePolicy: true,
-		Dir:            t.TempDir(),
+		TlsVerify:    false,
+		SecurePolicy: false,
+		Dir:          t.TempDir(),
 	}
 
 	_, sharedOpts := mirror.SharedImageFlags()

v2/tests/cache-fake/docker/registry/v2/repositories/ubi8/ubi/_layers/sha256/2e39d55595ea56337b5b788e96e6afdec3db09d2759d903cbe120468187c4644/link

@@ -0,0 +1 @@
+sha256:2e39d55595ea56337b5b788e96e6afdec3db09d2759d903cbe120468187c4644

v2/tests/cache-fake/docker/registry/v2/repositories/ubi8/ubi/_layers/sha256/4c0f6aace7053de3b9c1476b33c9a763e45a099c8c7ae9117773c9a8e5b8506b/link

@@ -0,0 +1 @@
+sha256:4c0f6aace7053de3b9c1476b33c9a763e45a099c8c7ae9117773c9a8e5b8506b

v2/tests/cache-fake/docker/registry/v2/repositories/ubi8/ubi/_layers/sha256/53c56977ccd20c0d87df0ad52036c55b27201e1a63874c2644383d0c532f5aee/link

@@ -0,0 +1 @@
+sha256:53c56977ccd20c0d87df0ad52036c55b27201e1a63874c2644383d0c532f5aee

v2/tests/cache-fake/docker/registry/v2/repositories/ubi8/ubi/_layers/sha256/6e1ac33d11e06db5e850fec4a1ec07f6c2ab15f130c2fdf0f9d0d0a5c83651e7/link

@@ -0,0 +1 @@
+sha256:6e1ac33d11e06db5e850fec4a1ec07f6c2ab15f130c2fdf0f9d0d0a5c83651e7

v2/tests/cache-fake/docker/registry/v2/repositories/ubi8/ubi/_layers/sha256/94343313ec1512ab02267e4bc3ce09eecb01fda5bf26c56e2f028ecc72e80b18/link

@@ -0,0 +1 @@
+sha256:94343313ec1512ab02267e4bc3ce09eecb01fda5bf26c56e2f028ecc72e80b18

v2/tests/cache-fake/docker/registry/v2/repositories/ubi8/ubi/_layers/sha256/cfaa7496ab546c36ab14859f93fbd2d8a3588b344b18d5fbe74dd834e4a6f7eb/link

@@ -0,0 +1 @@
+sha256:cfaa7496ab546c36ab14859f93fbd2d8a3588b344b18d5fbe74dd834e4a6f7eb

v2/tests/cache-fake/docker/registry/v2/repositories/ubi8/ubi/_layers/sha256/e1bb0572465a9e03d7af5024abb36d7227b5bf133c448b54656d908982127874/link

@@ -0,0 +1 @@
+sha256:e1bb0572465a9e03d7af5024abb36d7227b5bf133c448b54656d908982127874

v2/tests/cache-fake/docker/registry/v2/repositories/ubi8/ubi/_layers/sha256/f992cb38fce665360a4d07f6f78db864a1f6e20a7ad304219f7f81d7fe608d97/link

@@ -0,0 +1 @@
+sha256:f992cb38fce665360a4d07f6f78db864a1f6e20a7ad304219f7f81d7fe608d97

v2/tests/cache-fake/docker/registry/v2/repositories/ubi8/ubi/_manifests/revisions/sha256/6376a0276facf61d87fdf7c6f21d761ee25ba8ceba934d64752d43e84fe0cb98/link

@@ -0,0 +1 @@
+sha256:6376a0276facf61d87fdf7c6f21d761ee25ba8ceba934d64752d43e84fe0cb98

v2/tests/cache-fake/docker/registry/v2/repositories/ubi8/ubi/_manifests/revisions/sha256/9b6fa335dba394d437930ad79e308e01da4f624328e49d00c0ff44775d2e4769/link

@@ -0,0 +1 @@
+sha256:9b6fa335dba394d437930ad79e308e01da4f624328e49d00c0ff44775d2e4769

v2/tests/cache-fake/docker/registry/v2/repositories/ubi8/ubi/_manifests/revisions/sha256/db870970ba330193164dacc88657df261d75bce1552ea474dbc7cf08b2fae2ed/link

@@ -0,0 +1 @@
+sha256:db870970ba330193164dacc88657df261d75bce1552ea474dbc7cf08b2fae2ed

v2/tests/cache-fake/docker/registry/v2/repositories/ubi8/ubi/_manifests/revisions/sha256/e6c589cf5f402a60a83a01653304d7a8dcdd47b93a395a797b5622a18904bd66/link

@@ -0,0 +1 @@
+sha256:e6c589cf5f402a60a83a01653304d7a8dcdd47b93a395a797b5622a18904bd66

v2/tests/cache-fake/docker/registry/v2/repositories/ubi8/ubi/_manifests/tags/latest/current/link

@@ -0,0 +1 @@
+sha256:db870970ba330193164dacc88657df261d75bce1552ea474dbc7cf08b2fae2ed

v2/tests/cache-fake/docker/registry/v2/repositories/ubi8/ubi/_manifests/tags/latest/index/sha256/db870970ba330193164dacc88657df261d75bce1552ea474dbc7cf08b2fae2ed/link

@@ -0,0 +1 @@
+sha256:db870970ba330193164dacc88657df261d75bce1552ea474dbc7cf08b2fae2ed

v2/tests/cache-fake/docker/registry/v2/repositories/ubi8/ubi/_uploads/97e2891e-4cb2-4289-a87a-e9b8cd006d20/startedat

@@ -0,0 +1 @@
+2023-11-09T17:47:04Z