feat(image): adds method to create Associations from remote sources

Signed-off-by: Jennifer Power <barnabei.jennifer@gmail.com>

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/containerd/containerd v1.5.8
 	github.com/containers/image/v5 v5.16.0
 	github.com/docker/cli v20.10.12+incompatible
+	github.com/docker/distribution v2.7.1+incompatible
 	github.com/go-git/go-git/v5 v5.4.2 // indirect
 	github.com/google/go-containerregistry v0.8.0
 	github.com/google/uuid v1.3.0

pkg/bundle/image.go

@@ -21,10 +21,10 @@ func (e ErrBlocked) Error() string {
 }
 
 // IsBlocked will return a boolean value on whether an image
-// is specified as blocked in the BundleSpec
-func IsBlocked(cfg v1alpha2.ImageSetConfiguration, imgRef reference.DockerImageReference) bool {
+// is specified as blocked in the ImageSetConfigSpec
+func IsBlocked(blocked []v1alpha2.BlockedImages, imgRef reference.DockerImageReference) bool {
 
-	for _, block := range cfg.Mirror.BlockedImages {
+	for _, block := range blocked {
 
 		logrus.Debugf("Checking if image %s is blocked", imgRef.Exact())
 

pkg/bundle/image_test.go

@@ -70,7 +70,7 @@ func TestImageBlocking(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		actual := IsBlocked(cfg, img.Ref)
+		actual := IsBlocked(cfg.Mirror.BlockedImages, img.Ref)
 
 		if actual != tt.want {
 			t.Errorf("Test %s: Expected '%v', got '%v'", tt.name, tt.want, actual)

pkg/cli/mirror/mirror.go

@@ -182,7 +182,7 @@ func (o *MirrorOptions) Validate() error {
 
 func (o *MirrorOptions) Run(cmd *cobra.Command, f kcmdutil.Factory) (err error) {
 	if o.OutputDir != "" {
-		if err := os.MkdirAll(o.OutputDir, 0755); err != nil {
+		if err := os.MkdirAll(o.OutputDir, 0750); err != nil {
 			return err
 		}
 	}
@@ -232,7 +232,7 @@ func (o *MirrorOptions) Run(cmd *cobra.Command, f kcmdutil.Factory) (err error)
 
 		// Create assocations
 		assocDir := filepath.Join(o.Dir, config.SourceDir)
-		assocs, errs := image.AssociateImageLayers(assocDir, mapping)
+		assocs, errs := image.AssociateLocalImageLayers(assocDir, mapping)
 		if errs != nil {
 			return errs
 		}
@@ -303,6 +303,7 @@ func (o *MirrorOptions) Run(cmd *cobra.Command, f kcmdutil.Factory) (err error)
 		if err := o.mirrorMappings(cfg, mapping, destInsecure); err != nil {
 			return err
 		}
+
 		// Process any catalog images
 		dir, err := o.createResultsDir()
 		if err != nil {
@@ -418,7 +419,7 @@ func (o *MirrorOptions) mirrorMappings(cfg v1alpha2.ImageSetConfiguration, image
 	// Create mapping from source and destination images
 	var mappings []mirror.Mapping
 	for srcRef, dstRef := range images {
-		if bundle.IsBlocked(cfg, srcRef.Ref) {
+		if bundle.IsBlocked(cfg.Mirror.BlockedImages, srcRef.Ref) {
 			logrus.Warnf("skipping blocked images %s", srcRef.String())
 			continue
 		}
@@ -451,7 +452,7 @@ func (o *MirrorOptions) newMirrorImageOptions(insecure bool) (*mirror.MirrorImag
 	a.KeepManifestList = true
 	a.SkipMultipleScopes = true
 	a.ParallelOptions = imagemanifest.ParallelOptions{MaxPerRegistry: 2}
-	regctx, err := config.CreateDefaultContext(insecure)
+	regctx, err := image.CreateDefaultContext(insecure)
 	if err != nil {
 		return a, fmt.Errorf("error creating registry context: %v", err)
 	}

pkg/cli/mirror/operator.go

@@ -446,7 +446,7 @@ func (o *OperatorOptions) newMirrorCatalogOptions(ctlgRef imgreference.DockerIma
 	opts.SecurityOptions.Insecure = insecure
 	opts.SecurityOptions.SkipVerification = o.SkipVerification
 
-	regctx, err := config.CreateDefaultContext(insecure)
+	regctx, err := image.CreateDefaultContext(insecure)
 	if err != nil {
 		return nil, fmt.Errorf("error creating registry context: %v", err)
 	}

pkg/cli/mirror/publish.go

@@ -418,7 +418,7 @@ func (o *MirrorOptions) fetchBlobs(ctx context.Context, meta v1alpha2.Metadata,
 	if o.DestPlainHTTP || o.DestSkipTLS {
 		insecure = true
 	}
-	restctx, err := config.CreateDefaultContext(insecure)
+	restctx, err := image.CreateDefaultContext(insecure)
 	if err != nil {
 		return err
 	}
@@ -509,7 +509,7 @@ func (o *MirrorOptions) publishImage(mappings []imgmirror.Mapping, fromDir strin
 		}
 		logrus.Debugf("mirroring generic images: %q", srcs)
 	}
-	regctx, err := config.CreateDefaultContext(insecure)
+	regctx, err := image.CreateDefaultContext(insecure)
 	if err != nil {
 		return err
 	}

pkg/cli/mirror/release.go

@@ -224,7 +224,7 @@ func (o *ReleaseOptions) newMirrorReleaseOptions(fileDir string) (*release.Mirro
 	opts.SecurityOptions.Insecure = o.insecure
 	opts.SecurityOptions.SkipVerification = o.SkipVerification
 
-	regctx, err := config.CreateDefaultContext(o.insecure)
+	regctx, err := image.CreateDefaultContext(o.insecure)
 	if err != nil {
 		return nil, fmt.Errorf("error creating registry context: %v", err)
 	}

pkg/image/association.go

@@ -5,26 +5,11 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/fs"
-	"io/ioutil"
-	"os"
 	"path/filepath"
 
-	ctrsimgmanifest "github.com/containers/image/v5/manifest"
-	imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
-	"github.com/openshift/oc/pkg/cli/image/imagesource"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 )
 
-type ErrInvalidComponent struct {
-	image string
-	tag   string
-}
-
-func (e *ErrInvalidComponent) Error() string {
-	return fmt.Sprintf("image %q has invalid component %q", e.image, e.tag)
-}
-
 // Associations is a map for Association
 // searching
 type Associations map[string]Association
@@ -37,7 +22,7 @@ type AssociationSet map[string]Associations
 type Association struct {
 	// Name of the image.
 	Name string `json:"name"`
-	// Path to image data within archive.
+	// Path to image in new location (archive or registry)
 	Path string `json:"path"`
 	// ID of the image. Joining this value with "manifests" and Path
 	// will produce a path to the image's manifest.
@@ -144,9 +129,8 @@ func (as AssociationSet) ContainsKey(setKey, key string) (found bool) {
 
 // Merge Associations into the receiver.
 func (as AssociationSet) Merge(in AssociationSet) {
-	for _, imageName := range in.Keys() {
-		values, _ := in.Search(imageName)
-		for _, value := range values {
+	for imageName, assocs := range in {
+		for _, value := range assocs {
 			as.Add(imageName, value)
 		}
 	}
@@ -172,8 +156,7 @@ func (as *AssociationSet) Decode(r io.Reader) error {
 		return fmt.Errorf("error decoding image associations: %v", err)
 	}
 	// Update paths for local usage.
-	for _, imageName := range as.Keys() {
-		assocs, _ := as.Search(imageName)
+	for imageName, assocs := range *as {
 		for _, assoc := range assocs {
 			assoc.Path = filepath.FromSlash(assoc.Path)
 			if err := as.UpdateValue(imageName, assoc); err != nil {
@@ -184,6 +167,34 @@ func (as *AssociationSet) Decode(r io.Reader) error {
 	return nil
 }
 
+// UpdatePath path will update path values for local
+// AssociationSet use
+func (as *AssociationSet) UpdatePath() error {
+	// Update paths for local usage.
+	for imageName, assocs := range *as {
+		for _, assoc := range assocs {
+			assoc.Path = filepath.FromSlash(assoc.Path)
+			if err := as.UpdateValue(imageName, assoc); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// GetDigests will return all layer and manifest digests in the association set
+func (as *AssociationSet) GetDigests() []string {
+	var digests []string
+	for _, assocs := range *as {
+		for _, assoc := range assocs {
+			digests = append(digests, assoc.LayerDigests...)
+			digests = append(digests, assoc.ManifestDigests...)
+			digests = append(digests, assoc.ID)
+		}
+	}
+	return digests
+}
+
 func (as AssociationSet) validate() error {
 	var errs []error
 	for _, imageName := range as.Keys() {
@@ -219,146 +230,16 @@ func (as AssociationSet) validate() error {
 	return utilerrors.NewAggregate(errs)
 }
 
-func AssociateImageLayers(rootDir string, imgMappings TypedImageMapping) (AssociationSet, utilerrors.Aggregate) {
-	errs := []error{}
-	bundleAssociations := AssociationSet{}
-
-	skipParse := func(ref string) bool {
-		seen := bundleAssociations.SetContainsKey(ref)
-		return seen
-	}
-
-	localRoot := filepath.Join(rootDir, "v2")
-	for image, diskLoc := range imgMappings {
-		if diskLoc.Type != imagesource.DestinationFile {
-			errs = append(errs, fmt.Errorf("image destination for %q is not type file", image.Ref.Exact()))
-			continue
-		}
-		dirRef := diskLoc.Ref.AsRepository().String()
-		imagePath := filepath.Join(localRoot, dirRef)
-
-		// Verify that the dirRef exists before proceeding
-		if _, err := os.Stat(imagePath); err != nil {
-			errs = append(errs, fmt.Errorf("image %q mapping %q: %v", image, dirRef, err))
-			continue
-		}
-
-		var tagOrID string
-		if diskLoc.Ref.Tag != "" {
-			tagOrID = diskLoc.Ref.Tag
-		} else {
-			tagOrID = diskLoc.Ref.ID
-		}
-
-		if tagOrID == "" {
-			errs = append(errs, &ErrInvalidComponent{image.String(), tagOrID})
-			continue
-		}
-
-		// TODO(estroz): parallelize
-		associations, err := associateImageLayers(image.Ref.String(), localRoot, dirRef, tagOrID, "oc-mirror", image.Category, skipParse)
-		if err != nil {
-			errs = append(errs, err)
-			continue
-		}
-		for _, association := range associations {
-			bundleAssociations.Add(image.Ref.String(), association)
-		}
-	}
-
-	return bundleAssociations, utilerrors.NewAggregate(errs)
-}
-
-func associateImageLayers(image, localRoot, dirRef, tagOrID, defaultTag string, typ ImageType, skipParse func(string) bool) (associations []Association, err error) {
-	if skipParse(image) {
-		return nil, nil
-	}
-
-	manifestPath := filepath.Join(localRoot, filepath.FromSlash(dirRef), "manifests", tagOrID)
-	// TODO(estroz): this file mode checking block is likely only necessary
-	// for the first recursion leaf since image manifest layers always contain id's,
-	// so unroll this component into AssociateImageLayers.
-
-	info, err := os.Lstat(manifestPath)
-	if errors.Is(err, os.ErrNotExist) {
-		return nil, &ErrInvalidComponent{image, tagOrID}
-	} else if err != nil {
-		return nil, err
-	}
-	// Tags are always symlinks due to how `oc` libraries mirror manifest files.
-	id, tag := tagOrID, tagOrID
-	switch m := info.Mode(); {
-	case m&fs.ModeSymlink != 0:
-		// Tag is the file name, so follow the symlink to the layer ID-named file.
-		dst, err := os.Readlink(manifestPath)
-		if err != nil {
-			return nil, fmt.Errorf("error evaluating image tag symlink: %v", err)
-		}
-		id = filepath.Base(dst)
-	case m.IsRegular():
-		// Layer ID is the file name, and no tag exists.
-		tag = defaultTag
-		if defaultTag != "" {
-			// If set, add a subset of the digest to randomize the
-			// tag in the event multiple digests are pulled for the same
-			// image
-			tag = defaultTag + id[7:13]
-			manifestDir := filepath.Dir(manifestPath)
-			symlink := filepath.Join(manifestDir, tag)
-			if err := os.Symlink(info.Name(), symlink); err != nil {
-				return nil, err
-			}
-		}
-	default:
-		return nil, fmt.Errorf("expected symlink or regular file mode, got: %b", m)
-	}
-	manifestBytes, err := ioutil.ReadFile(filepath.Clean(manifestPath))
-	if err != nil {
-		return nil, fmt.Errorf("error reading image manifest file: %v", err)
-	}
+func GetImageFromBlob(as AssociationSet, digest string) string {
+	for imageName, assocs := range as {
+		for _, assoc := range assocs {
+			for _, dgst := range assoc.LayerDigests {
+				if dgst == digest {
+					return imageName
+				}
 
-	association := Association{
-		Name:       image,
-		Path:       dirRef,
-		ID:         id,
-		TagSymlink: tag,
-		Type:       typ,
-	}
-	switch mt := ctrsimgmanifest.GuessMIMEType(manifestBytes); mt {
-	case "":
-		return nil, errors.New("unparseable manifest mediaType")
-	case imgspecv1.MediaTypeImageIndex, ctrsimgmanifest.DockerV2ListMediaType:
-		list, err := ctrsimgmanifest.ListFromBlob(manifestBytes, mt)
-		if err != nil {
-			return nil, err
-		}
-		for _, instance := range list.Instances() {
-			digestStr := instance.String()
-			// Add manifest references so publish can recursively look up image layers
-			// for the manifests of this list.
-			association.ManifestDigests = append(association.ManifestDigests, digestStr)
-			// Recurse on child manifests, which should be in the same directory
-			// with the same file name as it's digest.
-			childAssocs, err := associateImageLayers(digestStr, localRoot, dirRef, digestStr, "", typ, skipParse)
-			if err != nil {
-				return nil, err
 			}
-			associations = append(associations, childAssocs...)
-		}
-	default:
-		// Treat all others as image manifests.
-		manifest, err := ctrsimgmanifest.FromBlob(manifestBytes, mt)
-		if err != nil {
-			return nil, err
 		}
-		for _, layerInfo := range manifest.LayerInfos() {
-			association.LayerDigests = append(association.LayerDigests, layerInfo.Digest.String())
-		}
-		// The config is just another blob, so associate it opaquely.
-		association.LayerDigests = append(association.LayerDigests, manifest.ConfigInfo().Digest.String())
 	}
-
-	associations = append(associations, association)
-
-	return associations, nil
+	return ""
 }