diff --git a/go.mod b/go.mod index 2a80c75b5..9af8eb665 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/containerd/containerd v1.5.8 github.com/containers/image/v5 v5.16.0 github.com/docker/cli v20.10.12+incompatible + github.com/docker/distribution v2.7.1+incompatible github.com/go-git/go-git/v5 v5.4.2 // indirect github.com/google/go-containerregistry v0.8.0 github.com/google/uuid v1.3.0 diff --git a/pkg/bundle/image.go b/pkg/bundle/image.go index 72fb545f9..fefb5e37f 100644 --- a/pkg/bundle/image.go +++ b/pkg/bundle/image.go @@ -21,10 +21,10 @@ func (e ErrBlocked) Error() string { } // IsBlocked will return a boolean value on whether an image -// is specified as blocked in the BundleSpec -func IsBlocked(cfg v1alpha2.ImageSetConfiguration, imgRef reference.DockerImageReference) bool { +// is specified as blocked in the ImageSetConfigSpec +func IsBlocked(blocked []v1alpha2.BlockedImages, imgRef reference.DockerImageReference) bool { - for _, block := range cfg.Mirror.BlockedImages { + for _, block := range blocked { logrus.Debugf("Checking if image %s is blocked", imgRef.Exact()) diff --git a/pkg/bundle/image_test.go b/pkg/bundle/image_test.go index 02e6dae0b..90c884d76 100644 --- a/pkg/bundle/image_test.go +++ b/pkg/bundle/image_test.go @@ -70,7 +70,7 @@ func TestImageBlocking(t *testing.T) { t.Fatal(err) } - actual := IsBlocked(cfg, img.Ref) + actual := IsBlocked(cfg.Mirror.BlockedImages, img.Ref) if actual != tt.want { t.Errorf("Test %s: Expected '%v', got '%v'", tt.name, tt.want, actual) diff --git a/pkg/cli/mirror/mirror.go b/pkg/cli/mirror/mirror.go index 803a423d7..b8e721ec6 100644 --- a/pkg/cli/mirror/mirror.go +++ b/pkg/cli/mirror/mirror.go @@ -182,7 +182,7 @@ func (o *MirrorOptions) Validate() error { func (o *MirrorOptions) Run(cmd *cobra.Command, f kcmdutil.Factory) (err error) { if o.OutputDir != "" { - if err := os.MkdirAll(o.OutputDir, 0755); err != nil { + if err := os.MkdirAll(o.OutputDir, 0750); err != nil { return err } } @@ -232,7 +232,7 @@ func (o *MirrorOptions) Run(cmd *cobra.Command, f kcmdutil.Factory) (err error) // Create assocations assocDir := filepath.Join(o.Dir, config.SourceDir) - assocs, errs := image.AssociateImageLayers(assocDir, mapping) + assocs, errs := image.AssociateLocalImageLayers(assocDir, mapping) if errs != nil { return errs } @@ -303,6 +303,7 @@ func (o *MirrorOptions) Run(cmd *cobra.Command, f kcmdutil.Factory) (err error) if err := o.mirrorMappings(cfg, mapping, destInsecure); err != nil { return err } + // Process any catalog images dir, err := o.createResultsDir() if err != nil { @@ -418,7 +419,7 @@ func (o *MirrorOptions) mirrorMappings(cfg v1alpha2.ImageSetConfiguration, image // Create mapping from source and destination images var mappings []mirror.Mapping for srcRef, dstRef := range images { - if bundle.IsBlocked(cfg, srcRef.Ref) { + if bundle.IsBlocked(cfg.Mirror.BlockedImages, srcRef.Ref) { logrus.Warnf("skipping blocked images %s", srcRef.String()) continue } @@ -451,7 +452,7 @@ func (o *MirrorOptions) newMirrorImageOptions(insecure bool) (*mirror.MirrorImag a.KeepManifestList = true a.SkipMultipleScopes = true a.ParallelOptions = imagemanifest.ParallelOptions{MaxPerRegistry: 2} - regctx, err := config.CreateDefaultContext(insecure) + regctx, err := image.CreateDefaultContext(insecure) if err != nil { return a, fmt.Errorf("error creating registry context: %v", err) } diff --git a/pkg/cli/mirror/operator.go b/pkg/cli/mirror/operator.go index 0a99ae99f..47d3aa64b 100644 --- a/pkg/cli/mirror/operator.go +++ b/pkg/cli/mirror/operator.go @@ -446,7 +446,7 @@ func (o *OperatorOptions) newMirrorCatalogOptions(ctlgRef imgreference.DockerIma opts.SecurityOptions.Insecure = insecure opts.SecurityOptions.SkipVerification = o.SkipVerification - regctx, err := config.CreateDefaultContext(insecure) + regctx, err := image.CreateDefaultContext(insecure) if err != nil { return nil, fmt.Errorf("error creating registry context: %v", err) } diff --git a/pkg/cli/mirror/publish.go b/pkg/cli/mirror/publish.go index 274db5ece..2742702be 100644 --- a/pkg/cli/mirror/publish.go +++ b/pkg/cli/mirror/publish.go @@ -418,7 +418,7 @@ func (o *MirrorOptions) fetchBlobs(ctx context.Context, meta v1alpha2.Metadata, if o.DestPlainHTTP || o.DestSkipTLS { insecure = true } - restctx, err := config.CreateDefaultContext(insecure) + restctx, err := image.CreateDefaultContext(insecure) if err != nil { return err } @@ -509,7 +509,7 @@ func (o *MirrorOptions) publishImage(mappings []imgmirror.Mapping, fromDir strin } logrus.Debugf("mirroring generic images: %q", srcs) } - regctx, err := config.CreateDefaultContext(insecure) + regctx, err := image.CreateDefaultContext(insecure) if err != nil { return err } diff --git a/pkg/cli/mirror/release.go b/pkg/cli/mirror/release.go index ab0e2fd20..db0b2a5ed 100644 --- a/pkg/cli/mirror/release.go +++ b/pkg/cli/mirror/release.go @@ -224,7 +224,7 @@ func (o *ReleaseOptions) newMirrorReleaseOptions(fileDir string) (*release.Mirro opts.SecurityOptions.Insecure = o.insecure opts.SecurityOptions.SkipVerification = o.SkipVerification - regctx, err := config.CreateDefaultContext(o.insecure) + regctx, err := image.CreateDefaultContext(o.insecure) if err != nil { return nil, fmt.Errorf("error creating registry context: %v", err) } diff --git a/pkg/image/association.go b/pkg/image/association.go index 1c5f5e117..9f3a9dfe2 100644 --- a/pkg/image/association.go +++ b/pkg/image/association.go @@ -5,26 +5,11 @@ import ( "errors" "fmt" "io" - "io/fs" - "io/ioutil" - "os" "path/filepath" - ctrsimgmanifest "github.com/containers/image/v5/manifest" - imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/openshift/oc/pkg/cli/image/imagesource" utilerrors "k8s.io/apimachinery/pkg/util/errors" ) -type ErrInvalidComponent struct { - image string - tag string -} - -func (e *ErrInvalidComponent) Error() string { - return fmt.Sprintf("image %q has invalid component %q", e.image, e.tag) -} - // Associations is a map for Association // searching type Associations map[string]Association @@ -37,7 +22,7 @@ type AssociationSet map[string]Associations type Association struct { // Name of the image. Name string `json:"name"` - // Path to image data within archive. + // Path to image in new location (archive or registry) Path string `json:"path"` // ID of the image. Joining this value with "manifests" and Path // will produce a path to the image's manifest. @@ -144,9 +129,8 @@ func (as AssociationSet) ContainsKey(setKey, key string) (found bool) { // Merge Associations into the receiver. func (as AssociationSet) Merge(in AssociationSet) { - for _, imageName := range in.Keys() { - values, _ := in.Search(imageName) - for _, value := range values { + for imageName, assocs := range in { + for _, value := range assocs { as.Add(imageName, value) } } @@ -172,8 +156,7 @@ func (as *AssociationSet) Decode(r io.Reader) error { return fmt.Errorf("error decoding image associations: %v", err) } // Update paths for local usage. - for _, imageName := range as.Keys() { - assocs, _ := as.Search(imageName) + for imageName, assocs := range *as { for _, assoc := range assocs { assoc.Path = filepath.FromSlash(assoc.Path) if err := as.UpdateValue(imageName, assoc); err != nil { @@ -184,6 +167,34 @@ func (as *AssociationSet) Decode(r io.Reader) error { return nil } +// UpdatePath path will update path values for local +// AssociationSet use +func (as *AssociationSet) UpdatePath() error { + // Update paths for local usage. + for imageName, assocs := range *as { + for _, assoc := range assocs { + assoc.Path = filepath.FromSlash(assoc.Path) + if err := as.UpdateValue(imageName, assoc); err != nil { + return err + } + } + } + return nil +} + +// GetDigests will return all layer and manifest digests in the association set +func (as *AssociationSet) GetDigests() []string { + var digests []string + for _, assocs := range *as { + for _, assoc := range assocs { + digests = append(digests, assoc.LayerDigests...) + digests = append(digests, assoc.ManifestDigests...) + digests = append(digests, assoc.ID) + } + } + return digests +} + func (as AssociationSet) validate() error { var errs []error for _, imageName := range as.Keys() { @@ -219,146 +230,16 @@ func (as AssociationSet) validate() error { return utilerrors.NewAggregate(errs) } -func AssociateImageLayers(rootDir string, imgMappings TypedImageMapping) (AssociationSet, utilerrors.Aggregate) { - errs := []error{} - bundleAssociations := AssociationSet{} - - skipParse := func(ref string) bool { - seen := bundleAssociations.SetContainsKey(ref) - return seen - } - - localRoot := filepath.Join(rootDir, "v2") - for image, diskLoc := range imgMappings { - if diskLoc.Type != imagesource.DestinationFile { - errs = append(errs, fmt.Errorf("image destination for %q is not type file", image.Ref.Exact())) - continue - } - dirRef := diskLoc.Ref.AsRepository().String() - imagePath := filepath.Join(localRoot, dirRef) - - // Verify that the dirRef exists before proceeding - if _, err := os.Stat(imagePath); err != nil { - errs = append(errs, fmt.Errorf("image %q mapping %q: %v", image, dirRef, err)) - continue - } - - var tagOrID string - if diskLoc.Ref.Tag != "" { - tagOrID = diskLoc.Ref.Tag - } else { - tagOrID = diskLoc.Ref.ID - } - - if tagOrID == "" { - errs = append(errs, &ErrInvalidComponent{image.String(), tagOrID}) - continue - } - - // TODO(estroz): parallelize - associations, err := associateImageLayers(image.Ref.String(), localRoot, dirRef, tagOrID, "oc-mirror", image.Category, skipParse) - if err != nil { - errs = append(errs, err) - continue - } - for _, association := range associations { - bundleAssociations.Add(image.Ref.String(), association) - } - } - - return bundleAssociations, utilerrors.NewAggregate(errs) -} - -func associateImageLayers(image, localRoot, dirRef, tagOrID, defaultTag string, typ ImageType, skipParse func(string) bool) (associations []Association, err error) { - if skipParse(image) { - return nil, nil - } - - manifestPath := filepath.Join(localRoot, filepath.FromSlash(dirRef), "manifests", tagOrID) - // TODO(estroz): this file mode checking block is likely only necessary - // for the first recursion leaf since image manifest layers always contain id's, - // so unroll this component into AssociateImageLayers. - - info, err := os.Lstat(manifestPath) - if errors.Is(err, os.ErrNotExist) { - return nil, &ErrInvalidComponent{image, tagOrID} - } else if err != nil { - return nil, err - } - // Tags are always symlinks due to how `oc` libraries mirror manifest files. - id, tag := tagOrID, tagOrID - switch m := info.Mode(); { - case m&fs.ModeSymlink != 0: - // Tag is the file name, so follow the symlink to the layer ID-named file. - dst, err := os.Readlink(manifestPath) - if err != nil { - return nil, fmt.Errorf("error evaluating image tag symlink: %v", err) - } - id = filepath.Base(dst) - case m.IsRegular(): - // Layer ID is the file name, and no tag exists. - tag = defaultTag - if defaultTag != "" { - // If set, add a subset of the digest to randomize the - // tag in the event multiple digests are pulled for the same - // image - tag = defaultTag + id[7:13] - manifestDir := filepath.Dir(manifestPath) - symlink := filepath.Join(manifestDir, tag) - if err := os.Symlink(info.Name(), symlink); err != nil { - return nil, err - } - } - default: - return nil, fmt.Errorf("expected symlink or regular file mode, got: %b", m) - } - manifestBytes, err := ioutil.ReadFile(filepath.Clean(manifestPath)) - if err != nil { - return nil, fmt.Errorf("error reading image manifest file: %v", err) - } +func GetImageFromBlob(as AssociationSet, digest string) string { + for imageName, assocs := range as { + for _, assoc := range assocs { + for _, dgst := range assoc.LayerDigests { + if dgst == digest { + return imageName + } - association := Association{ - Name: image, - Path: dirRef, - ID: id, - TagSymlink: tag, - Type: typ, - } - switch mt := ctrsimgmanifest.GuessMIMEType(manifestBytes); mt { - case "": - return nil, errors.New("unparseable manifest mediaType") - case imgspecv1.MediaTypeImageIndex, ctrsimgmanifest.DockerV2ListMediaType: - list, err := ctrsimgmanifest.ListFromBlob(manifestBytes, mt) - if err != nil { - return nil, err - } - for _, instance := range list.Instances() { - digestStr := instance.String() - // Add manifest references so publish can recursively look up image layers - // for the manifests of this list. - association.ManifestDigests = append(association.ManifestDigests, digestStr) - // Recurse on child manifests, which should be in the same directory - // with the same file name as it's digest. - childAssocs, err := associateImageLayers(digestStr, localRoot, dirRef, digestStr, "", typ, skipParse) - if err != nil { - return nil, err } - associations = append(associations, childAssocs...) - } - default: - // Treat all others as image manifests. - manifest, err := ctrsimgmanifest.FromBlob(manifestBytes, mt) - if err != nil { - return nil, err } - for _, layerInfo := range manifest.LayerInfos() { - association.LayerDigests = append(association.LayerDigests, layerInfo.Digest.String()) - } - // The config is just another blob, so associate it opaquely. - association.LayerDigests = append(association.LayerDigests, manifest.ConfigInfo().Digest.String()) } - - associations = append(associations, association) - - return associations, nil + return "" } diff --git a/pkg/image/association_builder.go b/pkg/image/association_builder.go new file mode 100644 index 000000000..52e1446e6 --- /dev/null +++ b/pkg/image/association_builder.go @@ -0,0 +1,291 @@ +package image + +import ( + "context" + "errors" + "fmt" + "io/fs" + "io/ioutil" + "os" + "path/filepath" + + ctrsimgmanifest "github.com/containers/image/v5/manifest" + "github.com/docker/distribution" + "github.com/opencontainers/go-digest" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/openshift/oc/pkg/cli/image/imagesource" + utilerrors "k8s.io/apimachinery/pkg/util/errors" +) + +type ErrInvalidComponent struct { + image string + tag string +} + +func (e *ErrInvalidComponent) Error() string { + return fmt.Sprintf("image %q has invalid component %q", e.image, e.tag) +} + +// AssociateLocalImageLayers traverses a V2 directory and gathers all child manifests and layer digest information +// for mirrored images +func AssociateLocalImageLayers(rootDir string, imgMappings TypedImageMapping) (AssociationSet, error) { + errs := []error{} + bundleAssociations := AssociationSet{} + + skipParse := func(ref string) bool { + seen := bundleAssociations.SetContainsKey(ref) + return seen + } + + localRoot := filepath.Join(rootDir, "v2") + for image, diskLoc := range imgMappings { + if diskLoc.Type != imagesource.DestinationFile { + errs = append(errs, fmt.Errorf("image destination for %q is not type file", image.Ref.Exact())) + continue + } + dirRef := diskLoc.Ref.AsRepository().String() + imagePath := filepath.Join(localRoot, dirRef) + + // Verify that the dirRef exists before proceeding + if _, err := os.Stat(imagePath); err != nil { + errs = append(errs, fmt.Errorf("image %q mapping %q: %v", image, dirRef, err)) + continue + } + + var tagOrID string + if diskLoc.Ref.Tag != "" { + tagOrID = diskLoc.Ref.Tag + } else { + tagOrID = diskLoc.Ref.ID + } + + if tagOrID == "" { + errs = append(errs, &ErrInvalidComponent{image.String(), tagOrID}) + continue + } + + // TODO(estroz): parallelize + associations, err := associateLocalImageLayers(image.Ref.String(), localRoot, dirRef, tagOrID, "oc-mirror", image.Category, skipParse) + if err != nil { + errs = append(errs, err) + continue + } + for _, association := range associations { + bundleAssociations.Add(image.Ref.String(), association) + } + } + + return bundleAssociations, utilerrors.NewAggregate(errs) +} + +func associateLocalImageLayers(image, localRoot, dirRef, tagOrID, defaultTag string, typ ImageType, skipParse func(string) bool) (associations []Association, err error) { + if skipParse(image) { + return nil, nil + } + + manifestPath := filepath.Join(localRoot, filepath.FromSlash(dirRef), "manifests", tagOrID) + // TODO(estroz): this file mode checking block is likely only necessary + // for the first recursion leaf since image manifest layers always contain id's, + // so unroll this component into AssociateImageLayers. + + info, err := os.Lstat(manifestPath) + if errors.Is(err, os.ErrNotExist) { + return nil, &ErrInvalidComponent{image, tagOrID} + } else if err != nil { + return nil, err + } + // Tags are always symlinks due to how `oc` libraries mirror manifest files. + id, tag := tagOrID, tagOrID + switch m := info.Mode(); { + case m&fs.ModeSymlink != 0: + // Tag is the file name, so follow the symlink to the layer ID-named file. + dst, err := os.Readlink(manifestPath) + if err != nil { + return nil, fmt.Errorf("error evaluating image tag symlink: %v", err) + } + id = filepath.Base(dst) + case m.IsRegular(): + // Layer ID is the file name, and no tag exists. + tag = defaultTag + if defaultTag != "" { + // If set, add a subset of the digest to randomize the + // tag in the event multiple digests are pulled for the same + // image + tag = defaultTag + id[7:13] + manifestDir := filepath.Dir(manifestPath) + symlink := filepath.Join(manifestDir, tag) + if err := os.Symlink(info.Name(), symlink); err != nil { + return nil, err + } + } + default: + return nil, fmt.Errorf("expected symlink or regular file mode, got: %b", m) + } + manifestBytes, err := ioutil.ReadFile(filepath.Clean(manifestPath)) + if err != nil { + return nil, fmt.Errorf("error reading image manifest file: %v", err) + } + + association := Association{ + Name: image, + Path: dirRef, + ID: id, + TagSymlink: tag, + Type: typ, + } + switch mt := ctrsimgmanifest.GuessMIMEType(manifestBytes); mt { + case "": + return nil, errors.New("unparseable manifest mediaType") + case imgspecv1.MediaTypeImageIndex, ctrsimgmanifest.DockerV2ListMediaType: + list, err := ctrsimgmanifest.ListFromBlob(manifestBytes, mt) + if err != nil { + return nil, err + } + for _, instance := range list.Instances() { + digestStr := instance.String() + // Add manifest references so publish can recursively look up image layers + // for the manifests of this list. + association.ManifestDigests = append(association.ManifestDigests, digestStr) + // Recurse on child manifests, which should be in the same directory + // with the same file name as it's digest. + childAssocs, err := associateLocalImageLayers(digestStr, localRoot, dirRef, digestStr, "", typ, skipParse) + if err != nil { + return nil, err + } + associations = append(associations, childAssocs...) + } + default: + // Treat all others as image manifests. + manifest, err := ctrsimgmanifest.FromBlob(manifestBytes, mt) + if err != nil { + return nil, err + } + for _, layerInfo := range manifest.LayerInfos() { + association.LayerDigests = append(association.LayerDigests, layerInfo.Digest.String()) + } + // The config is just another blob, so associate it opaquely. + association.LayerDigests = append(association.LayerDigests, manifest.ConfigInfo().Digest.String()) + } + + associations = append(associations, association) + + return associations, nil +} + +// AssociateRemoteImageLayers queries remote manifests and gathers all child manifests and layer digest information +// for mirrored images +func AssociateRemoteImageLayers(ctx context.Context, imgMappings TypedImageMapping, insecure bool) (AssociationSet, error) { + errs := []error{} + bundleAssociations := AssociationSet{} + + skipParse := func(ref string) bool { + seen := bundleAssociations.SetContainsKey(ref) + return seen + } + + for srcImg, dstImg := range imgMappings { + if dstImg.Type != imagesource.DestinationRegistry { + errs = append(errs, fmt.Errorf("image destination for %q is not type registry", srcImg.Ref.Exact())) + continue + } + + if srcImg.Ref.ID == "" { + errs = append(errs, &ErrInvalidComponent{srcImg.String(), srcImg.Ref.ID}) + continue + } + + regctx, err := CreateDefaultContext(insecure) + if err != nil { + return nil, err + } + + repo, err := regctx.RepositoryForRef(ctx, srcImg.Ref, insecure) + if err != nil { + return nil, fmt.Errorf("create repo for %s: %v", srcImg.Ref.Exact(), err) + } + + ms, err := repo.Manifests(ctx) + if err != nil { + return nil, fmt.Errorf("open blob: %v", err) + } + + // TODO(estroz): parallelize + associations, err := associateRemoteImageLayers(ctx, srcImg.String(), dstImg.String(), srcImg, ms, skipParse, insecure) + if err != nil { + errs = append(errs, err) + continue + } + for _, association := range associations { + bundleAssociations.Add(srcImg.String(), association) + } + } + + return bundleAssociations, utilerrors.NewAggregate(errs) +} + +func associateRemoteImageLayers(ctx context.Context, srcImg, dstImg string, srcInfo TypedImage, ms distribution.ManifestService, skipParse func(string) bool, insecure bool) (associations []Association, err error) { + if skipParse(srcImg) { + return nil, nil + } + + dgst, err := digest.Parse(srcInfo.Ref.ID) + if err != nil { + return nil, err + } + mn, err := ms.Get(ctx, dgst) + if err != nil { + return nil, fmt.Errorf("error getting manifest %s: %v", dgst, err) + } + mt, payload, err := mn.Payload() + if err != nil { + return nil, err + } + + association := Association{ + Name: srcImg, + Path: dstImg, + ID: srcInfo.Ref.ID, + TagSymlink: srcInfo.Ref.Tag, + Type: srcInfo.Category, + } + switch mt { + case "": + return nil, errors.New("unparseable manifest mediaType") + case imgspecv1.MediaTypeImageIndex, ctrsimgmanifest.DockerV2ListMediaType: + list, err := ctrsimgmanifest.ListFromBlob(payload, mt) + if err != nil { + return nil, err + } + for _, instance := range list.Instances() { + digestStr := instance.String() + // Add manifest references so publish can recursively look up image layers + // for the manifests of this list. + association.ManifestDigests = append(association.ManifestDigests, digestStr) + // Recurse on child manifests, which should be in the same directory + // with the same file name as it's digest. + childInfo := srcInfo + childInfo.Ref.ID = digestStr + childInfo.Ref.Tag = "" + childAssocs, err := associateRemoteImageLayers(ctx, digestStr, dstImg, childInfo, ms, skipParse, insecure) + if err != nil { + return nil, err + } + associations = append(associations, childAssocs...) + } + default: + // Treat all others as image manifests. + manifest, err := ctrsimgmanifest.FromBlob(payload, mt) + if err != nil { + return nil, err + } + for _, layerInfo := range manifest.LayerInfos() { + association.LayerDigests = append(association.LayerDigests, layerInfo.Digest.String()) + } + // The config is just another blob, so associate it opaquely. + association.LayerDigests = append(association.LayerDigests, manifest.ConfigInfo().Digest.String()) + } + + associations = append(associations, association) + + return associations, nil +} diff --git a/pkg/image/association_builder_test.go b/pkg/image/association_builder_test.go new file mode 100644 index 000000000..fc988ebbe --- /dev/null +++ b/pkg/image/association_builder_test.go @@ -0,0 +1,502 @@ +package image + +import ( + "context" + "encoding/json" + "fmt" + "io/fs" + "io/ioutil" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path" + "path/filepath" + "strings" + "testing" + + "github.com/docker/distribution/manifest" + "github.com/openshift/library-go/pkg/image/reference" + "github.com/openshift/oc/pkg/cli/image/imagesource" + "github.com/stretchr/testify/require" +) + +func TestAssociateLocalImageLayers(t *testing.T) { + tests := []struct { + name string + imgTyp ImageType + imgMapping TypedImageMapping + expResult AssociationSet + expError error + wantErr bool + }{ + { + name: "Valid/ManifestWithTag", + imgTyp: TypeGeneric, + imgMapping: map[TypedImage]TypedImage{ + { + TypedImageReference: imagesource.TypedImageReference{ + Ref: reference.DockerImageReference{ + Name: "imgname", + Tag: "latest", + }}, + Category: TypeGeneric}: { + TypedImageReference: imagesource.TypedImageReference{ + Ref: reference.DockerImageReference{ + Name: "single_manifest", + Tag: "latest", + }, + Type: imagesource.DestinationFile, + }, + Category: TypeGeneric}}, + expResult: AssociationSet{"imgname:latest": map[string]Association{ + "imgname:latest": { + Name: "imgname:latest", + Path: "single_manifest", + TagSymlink: "latest", + ID: "sha256:d31c6ea5c50be93d6eb94d2b508f0208e84a308c011c6454ebf291d48b37df19", + Type: TypeGeneric, + ManifestDigests: nil, + LayerDigests: []string{ + "sha256:e8614d09b7bebabd9d8a450f44e88a8807c98a438a2ddd63146865286b132d1b", + "sha256:601401253d0aac2bc95cccea668761a6e69216468809d1cee837b2e8b398e241", + "sha256:211941188a4f55ffc6bcefa4f69b69b32c13fafb65738075de05808bbfcec086", + "sha256:f0fd5be261dfd2e36d01069a387a3e5125f5fd5adfec90f3cb190d1d5f1d1ad9", + "sha256:0c0beb258254c0566315c641b4107b080a96fa78d4f96833453dd6c5b9edf2b7", + "sha256:30c794a11b4c340c77238c5b7ca845752904bd8b74b73a9b16d31253234da031", + }, + }, + }}, + }, + { + name: "Valid/ManifestWithDigest", + imgTyp: TypeGeneric, + imgMapping: map[TypedImage]TypedImage{ + { + TypedImageReference: imagesource.TypedImageReference{ + Ref: reference.DockerImageReference{ + Name: "imgname", + ID: "sha256:d31c6ea5c50be93d6eb94d2b508f0208e84a308c011c6454ebf291d48b37df19", + }}, + Category: TypeGeneric}: { + TypedImageReference: imagesource.TypedImageReference{ + Ref: reference.DockerImageReference{ + Name: "single_manifest", + ID: "sha256:d31c6ea5c50be93d6eb94d2b508f0208e84a308c011c6454ebf291d48b37df19", + }, + Type: imagesource.DestinationFile, + }, + Category: TypeGeneric}}, + expResult: AssociationSet{"imgname@sha256:d31c6ea5c50be93d6eb94d2b508f0208e84a308c011c6454ebf291d48b37df19": map[string]Association{ + "imgname@sha256:d31c6ea5c50be93d6eb94d2b508f0208e84a308c011c6454ebf291d48b37df19": { + Name: "imgname@sha256:d31c6ea5c50be93d6eb94d2b508f0208e84a308c011c6454ebf291d48b37df19", + Path: "single_manifest", + TagSymlink: "oc-mirrord31c6e", + ID: "sha256:d31c6ea5c50be93d6eb94d2b508f0208e84a308c011c6454ebf291d48b37df19", + Type: TypeGeneric, + ManifestDigests: nil, + LayerDigests: []string{ + "sha256:e8614d09b7bebabd9d8a450f44e88a8807c98a438a2ddd63146865286b132d1b", + "sha256:601401253d0aac2bc95cccea668761a6e69216468809d1cee837b2e8b398e241", + "sha256:211941188a4f55ffc6bcefa4f69b69b32c13fafb65738075de05808bbfcec086", + "sha256:f0fd5be261dfd2e36d01069a387a3e5125f5fd5adfec90f3cb190d1d5f1d1ad9", + "sha256:0c0beb258254c0566315c641b4107b080a96fa78d4f96833453dd6c5b9edf2b7", + "sha256:30c794a11b4c340c77238c5b7ca845752904bd8b74b73a9b16d31253234da031", + }, + }, + }}, + }, + { + name: "Valid/IndexManifest", + imgTyp: TypeGeneric, + imgMapping: map[TypedImage]TypedImage{ + { + TypedImageReference: imagesource.TypedImageReference{ + Ref: reference.DockerImageReference{ + Name: "imgname", + Tag: "latest", + }}, + Category: TypeGeneric}: { + TypedImageReference: imagesource.TypedImageReference{ + Ref: reference.DockerImageReference{ + Name: "index_manifest", + Tag: "latest", + }, + Type: imagesource.DestinationFile, + }, + Category: TypeGeneric}}, + expResult: AssociationSet{"imgname:latest": map[string]Association{ + "imgname:latest": { + Name: "imgname:latest", + Path: "index_manifest", + TagSymlink: "latest", + ID: "sha256:d15a206e4ee462e82ab722ed84dfa514ab9ed8d85100d591c04314ae7c2162ee", + Type: TypeGeneric, + ManifestDigests: []string{ + "sha256:bab3a6153010b614c8764548f0dbe34c4a7dce4ea278a94713c3e9a936bb74e6", + "sha256:9574416689665a82cb4eaf43463da5b6156071ebbec117262eef7fa32b4d7021", + "sha256:b8a825862d73b2f1110dd9c5fc0631f47117c7cd99e42efa34244cd82bd6742f", + "sha256:60f5921e0f6a21a485a0a4e9415761afb5b60814bbe8a6864cb12b90ae24c1d0", + }, + LayerDigests: nil, + }, + "sha256:60f5921e0f6a21a485a0a4e9415761afb5b60814bbe8a6864cb12b90ae24c1d0": { + Name: "sha256:60f5921e0f6a21a485a0a4e9415761afb5b60814bbe8a6864cb12b90ae24c1d0", + Path: "index_manifest", + TagSymlink: "", + ID: "sha256:60f5921e0f6a21a485a0a4e9415761afb5b60814bbe8a6864cb12b90ae24c1d0", + Type: TypeGeneric, + LayerDigests: []string{ + "sha256:b538f80385f9b48122e3da068c932a96ea5018afa3c7be79da00437414bd18cd", + "sha256:342a15c43afd15b4d93051022ecf020ea6fde1e14d34599f5b4c10a8a5bae3c6", + "sha256:70660e39ee11b715823a96729d7f1b8964ecd6ca2b7c0e3fd5cde284e34758eb", + "sha256:f553d3748799c35aa60227875706f727a526a1d4c7840a5d550cdb4ba6cd5196", + "sha256:c5338ca295456f5c677bf8910ac94765be2f53977af6bd792f18a2298054d6be", + "sha256:af94dd630ca5e3e15d15502c2a03e386f4c1ef5a59def62e84ede35a009c4110", + "sha256:337fc839f463fd6b6d1773e0b8f2f9d40b3a8dff6963008193344cd29466a3d1", + "sha256:4d4b85daa42ca075d8aff8563d14434799268a4b823e74737171ed438f8c60ad", + }, + }, + "sha256:9574416689665a82cb4eaf43463da5b6156071ebbec117262eef7fa32b4d7021": { + Name: "sha256:9574416689665a82cb4eaf43463da5b6156071ebbec117262eef7fa32b4d7021", + Path: "index_manifest", + TagSymlink: "", + ID: "sha256:9574416689665a82cb4eaf43463da5b6156071ebbec117262eef7fa32b4d7021", + Type: TypeGeneric, + LayerDigests: []string{ + "sha256:b4b72e716706d29f5d2351709c20bf737b94f876a5472a43ff1b6e203c65d27f", + "sha256:8d0157f7a4ed4136f430f737f0f79d650248e19ebd87371f1ae1735536f0eaf2", + "sha256:46f9bc09f2ae8c0a95a69d77cd91527281cf54cd466dbee5ba6b28e05ee68a77", + "sha256:21d0f0a83af189ace4e566f1520e8ac5a404adda15edb534ee79a994bdd94abe", + "sha256:61a5adb16b8c308ed6481d3abac7e08035f09d936f2a1ecad0bd2000a18464b9", + "sha256:a92dcc7bd9c9c1369ef92728f7649e3ec868b53b7b38ab2a4bddc525f74896a8", + "sha256:317a9dc239a3310e2010e6e1c4f2a87b4b2c53f49ca5231c031227540ef91d0b", + "sha256:d476ce7797cc1558919a31a1cccd9b09f48ea2787982ccd3c2576252450d2d51", + }, + }, + "sha256:b8a825862d73b2f1110dd9c5fc0631f47117c7cd99e42efa34244cd82bd6742f": { + Name: "sha256:b8a825862d73b2f1110dd9c5fc0631f47117c7cd99e42efa34244cd82bd6742f", + Path: "index_manifest", + TagSymlink: "", + ID: "sha256:b8a825862d73b2f1110dd9c5fc0631f47117c7cd99e42efa34244cd82bd6742f", + Type: TypeGeneric, + LayerDigests: []string{ + "sha256:52278dd8e57993669c5b72a9620e89bebdc098f2af2379caaa8945f7403f77a2", + "sha256:1dc2a2c4dd124cf83f27e6d8852303f7874507b71a3f7b6a1265837b43279092", + "sha256:26100ac97b3237b89768d0dac0150c6a2b483a16b0662160df98d03ba25fa474", + "sha256:7c120a97d24392c377b955ca42f09fc04942aecff3f0a007d31ebd20c185958a", + "sha256:87875760340f78f13107842911184c55308475062940399772e7944138879704", + "sha256:5ad5a4942ddf238ce385d4b29eaa3b2d5f8836de538918d7da9a839c8313fd46", + "sha256:6121cb3c461255702c8b8ef03ed4b13061c0c600b20c7664ce82815ed15febbd", + "sha256:c72bf53b697715cd03c3f3dc6fd6d2bccb4b10e511c2847eb98e312d28850e48", + }, + }, + "sha256:bab3a6153010b614c8764548f0dbe34c4a7dce4ea278a94713c3e9a936bb74e6": { + Name: "sha256:bab3a6153010b614c8764548f0dbe34c4a7dce4ea278a94713c3e9a936bb74e6", + Path: "index_manifest", + TagSymlink: "", + ID: "sha256:bab3a6153010b614c8764548f0dbe34c4a7dce4ea278a94713c3e9a936bb74e6", + Type: TypeGeneric, + LayerDigests: []string{ + "sha256:df20fa9351a15782c64e6dddb2d4a6f50bf6d3688060a34c4014b0d9a752eb4c", + "sha256:58445347cff86791f89717f3bf79ec6f597d146397d9e78136cf9e937f363555", + "sha256:49f791cfca3e59c6094ec94d091473ddd9fe206e9860c0eb37dacbc3bbcccafd", + "sha256:b83c8811a2df5586918135a8bab5304c9c6f0c0a3b103c4b3ceb4515d2c480a5", + "sha256:36821795adb1d93e34b9835d2cd738738e0a7fb99b6232f00f69a0146f6db7fa", + "sha256:f31bf23bf137d6210ce78d1b133bab25ae0daffda0bfff172476479dfcc0b3a1", + "sha256:59064015f738a38367ca0ef7083840f3f1dbc579aa208071b4fb6b022a48d89a", + "sha256:3f161edc88f5ebe6db761902c3e563f450a8f373f58f6f9f59a13a7954f57d90", + }, + }, + }}, + }, + { + name: "Invalid/InvalidComponent", + imgTyp: TypeGeneric, + imgMapping: map[TypedImage]TypedImage{ + { + TypedImageReference: imagesource.TypedImageReference{ + Ref: reference.DockerImageReference{ + Name: "imgname", + }}, + Category: TypeGeneric}: { + TypedImageReference: imagesource.TypedImageReference{ + Ref: reference.DockerImageReference{ + Name: "single_manifest", + }, + Type: imagesource.DestinationFile, + }, + Category: TypeGeneric}}, + wantErr: true, + expError: &ErrInvalidComponent{}, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + tmpdir := t.TempDir() + require.NoError(t, copyV2("testdata", tmpdir)) + asSet, err := AssociateLocalImageLayers(tmpdir, test.imgMapping) + if !test.wantErr { + require.NoError(t, err) + require.Equal(t, test.expResult, asSet) + } else { + require.ErrorAs(t, err, &test.expError) + } + }) + } +} + +func TestAssociateRemoteImageLayers(t *testing.T) { + + server := httptest.NewServer(mirrorV2("testdata")) + t.Cleanup(server.Close) + u, err := url.Parse(server.URL) + require.NoError(t, err) + + tests := []struct { + name string + imgTyp ImageType + imgMapping TypedImageMapping + expResult AssociationSet + expError error + wantErr bool + }{ + { + name: "Valid/ManifestWithDigest", + imgTyp: TypeGeneric, + imgMapping: map[TypedImage]TypedImage{ + { + TypedImageReference: imagesource.TypedImageReference{ + Ref: reference.DockerImageReference{ + Name: "single_manifest", + ID: "sha256:d31c6ea5c50be93d6eb94d2b508f0208e84a308c011c6454ebf291d48b37df19", + Tag: "latest", + Registry: u.Host, + }}, + Category: TypeGeneric}: { + TypedImageReference: imagesource.TypedImageReference{ + Ref: reference.DockerImageReference{ + Name: "single_manifest", + ID: "sha256:d31c6ea5c50be93d6eb94d2b508f0208e84a308c011c6454ebf291d48b37df19", + Registry: "test-registry", + }, + Type: imagesource.DestinationRegistry, + }, + Category: TypeGeneric}}, + expResult: AssociationSet{fmt.Sprintf("%s/single_manifest@sha256:d31c6ea5c50be93d6eb94d2b508f0208e84a308c011c6454ebf291d48b37df19", u.Host): map[string]Association{ + fmt.Sprintf("%s/single_manifest@sha256:d31c6ea5c50be93d6eb94d2b508f0208e84a308c011c6454ebf291d48b37df19", u.Host): { + Name: fmt.Sprintf("%s/single_manifest@sha256:d31c6ea5c50be93d6eb94d2b508f0208e84a308c011c6454ebf291d48b37df19", u.Host), + Path: "test-registry/single_manifest@sha256:d31c6ea5c50be93d6eb94d2b508f0208e84a308c011c6454ebf291d48b37df19", + TagSymlink: "latest", + ID: "sha256:d31c6ea5c50be93d6eb94d2b508f0208e84a308c011c6454ebf291d48b37df19", + Type: TypeGeneric, + ManifestDigests: nil, + LayerDigests: []string{ + "sha256:e8614d09b7bebabd9d8a450f44e88a8807c98a438a2ddd63146865286b132d1b", + "sha256:601401253d0aac2bc95cccea668761a6e69216468809d1cee837b2e8b398e241", + "sha256:211941188a4f55ffc6bcefa4f69b69b32c13fafb65738075de05808bbfcec086", + "sha256:f0fd5be261dfd2e36d01069a387a3e5125f5fd5adfec90f3cb190d1d5f1d1ad9", + "sha256:0c0beb258254c0566315c641b4107b080a96fa78d4f96833453dd6c5b9edf2b7", + "sha256:30c794a11b4c340c77238c5b7ca845752904bd8b74b73a9b16d31253234da031", + }, + }, + }}, + }, + { + name: "Valid/IndexManifest", + imgTyp: TypeGeneric, + imgMapping: map[TypedImage]TypedImage{ + { + TypedImageReference: imagesource.TypedImageReference{ + Ref: reference.DockerImageReference{ + Name: "index_manifest", + Tag: "latest", + ID: "sha256:d15a206e4ee462e82ab722ed84dfa514ab9ed8d85100d591c04314ae7c2162ee", + Registry: u.Host, + }}, + Category: TypeGeneric}: { + TypedImageReference: imagesource.TypedImageReference{ + Ref: reference.DockerImageReference{ + Name: "index_manifest", + Tag: "latest", + Registry: "test-registry", + }, + Type: imagesource.DestinationRegistry, + }, + Category: TypeGeneric}}, + expResult: AssociationSet{fmt.Sprintf("%s/index_manifest@sha256:d15a206e4ee462e82ab722ed84dfa514ab9ed8d85100d591c04314ae7c2162ee", u.Host): map[string]Association{ + fmt.Sprintf("%s/index_manifest@sha256:d15a206e4ee462e82ab722ed84dfa514ab9ed8d85100d591c04314ae7c2162ee", u.Host): { + Name: fmt.Sprintf("%s/index_manifest@sha256:d15a206e4ee462e82ab722ed84dfa514ab9ed8d85100d591c04314ae7c2162ee", u.Host), + Path: "test-registry/index_manifest:latest", + TagSymlink: "latest", + ID: "sha256:d15a206e4ee462e82ab722ed84dfa514ab9ed8d85100d591c04314ae7c2162ee", + Type: TypeGeneric, + ManifestDigests: []string{ + "sha256:bab3a6153010b614c8764548f0dbe34c4a7dce4ea278a94713c3e9a936bb74e6", + "sha256:9574416689665a82cb4eaf43463da5b6156071ebbec117262eef7fa32b4d7021", + "sha256:b8a825862d73b2f1110dd9c5fc0631f47117c7cd99e42efa34244cd82bd6742f", + "sha256:60f5921e0f6a21a485a0a4e9415761afb5b60814bbe8a6864cb12b90ae24c1d0", + }, + LayerDigests: nil, + }, + "sha256:60f5921e0f6a21a485a0a4e9415761afb5b60814bbe8a6864cb12b90ae24c1d0": { + Name: "sha256:60f5921e0f6a21a485a0a4e9415761afb5b60814bbe8a6864cb12b90ae24c1d0", + Path: "test-registry/index_manifest:latest", + TagSymlink: "", + ID: "sha256:60f5921e0f6a21a485a0a4e9415761afb5b60814bbe8a6864cb12b90ae24c1d0", + Type: TypeGeneric, + LayerDigests: []string{ + "sha256:b538f80385f9b48122e3da068c932a96ea5018afa3c7be79da00437414bd18cd", + "sha256:342a15c43afd15b4d93051022ecf020ea6fde1e14d34599f5b4c10a8a5bae3c6", + "sha256:70660e39ee11b715823a96729d7f1b8964ecd6ca2b7c0e3fd5cde284e34758eb", + "sha256:f553d3748799c35aa60227875706f727a526a1d4c7840a5d550cdb4ba6cd5196", + "sha256:c5338ca295456f5c677bf8910ac94765be2f53977af6bd792f18a2298054d6be", + "sha256:af94dd630ca5e3e15d15502c2a03e386f4c1ef5a59def62e84ede35a009c4110", + "sha256:337fc839f463fd6b6d1773e0b8f2f9d40b3a8dff6963008193344cd29466a3d1", + "sha256:4d4b85daa42ca075d8aff8563d14434799268a4b823e74737171ed438f8c60ad", + }, + }, + "sha256:9574416689665a82cb4eaf43463da5b6156071ebbec117262eef7fa32b4d7021": { + Name: "sha256:9574416689665a82cb4eaf43463da5b6156071ebbec117262eef7fa32b4d7021", + Path: "test-registry/index_manifest:latest", + TagSymlink: "", + ID: "sha256:9574416689665a82cb4eaf43463da5b6156071ebbec117262eef7fa32b4d7021", + Type: TypeGeneric, + LayerDigests: []string{ + "sha256:b4b72e716706d29f5d2351709c20bf737b94f876a5472a43ff1b6e203c65d27f", + "sha256:8d0157f7a4ed4136f430f737f0f79d650248e19ebd87371f1ae1735536f0eaf2", + "sha256:46f9bc09f2ae8c0a95a69d77cd91527281cf54cd466dbee5ba6b28e05ee68a77", + "sha256:21d0f0a83af189ace4e566f1520e8ac5a404adda15edb534ee79a994bdd94abe", + "sha256:61a5adb16b8c308ed6481d3abac7e08035f09d936f2a1ecad0bd2000a18464b9", + "sha256:a92dcc7bd9c9c1369ef92728f7649e3ec868b53b7b38ab2a4bddc525f74896a8", + "sha256:317a9dc239a3310e2010e6e1c4f2a87b4b2c53f49ca5231c031227540ef91d0b", + "sha256:d476ce7797cc1558919a31a1cccd9b09f48ea2787982ccd3c2576252450d2d51", + }, + }, + "sha256:b8a825862d73b2f1110dd9c5fc0631f47117c7cd99e42efa34244cd82bd6742f": { + Name: "sha256:b8a825862d73b2f1110dd9c5fc0631f47117c7cd99e42efa34244cd82bd6742f", + Path: "test-registry/index_manifest:latest", + TagSymlink: "", + ID: "sha256:b8a825862d73b2f1110dd9c5fc0631f47117c7cd99e42efa34244cd82bd6742f", + Type: TypeGeneric, + LayerDigests: []string{ + "sha256:52278dd8e57993669c5b72a9620e89bebdc098f2af2379caaa8945f7403f77a2", + "sha256:1dc2a2c4dd124cf83f27e6d8852303f7874507b71a3f7b6a1265837b43279092", + "sha256:26100ac97b3237b89768d0dac0150c6a2b483a16b0662160df98d03ba25fa474", + "sha256:7c120a97d24392c377b955ca42f09fc04942aecff3f0a007d31ebd20c185958a", + "sha256:87875760340f78f13107842911184c55308475062940399772e7944138879704", + "sha256:5ad5a4942ddf238ce385d4b29eaa3b2d5f8836de538918d7da9a839c8313fd46", + "sha256:6121cb3c461255702c8b8ef03ed4b13061c0c600b20c7664ce82815ed15febbd", + "sha256:c72bf53b697715cd03c3f3dc6fd6d2bccb4b10e511c2847eb98e312d28850e48", + }, + }, + "sha256:bab3a6153010b614c8764548f0dbe34c4a7dce4ea278a94713c3e9a936bb74e6": { + Name: "sha256:bab3a6153010b614c8764548f0dbe34c4a7dce4ea278a94713c3e9a936bb74e6", + Path: "test-registry/index_manifest:latest", + TagSymlink: "", + ID: "sha256:bab3a6153010b614c8764548f0dbe34c4a7dce4ea278a94713c3e9a936bb74e6", + Type: TypeGeneric, + LayerDigests: []string{ + "sha256:df20fa9351a15782c64e6dddb2d4a6f50bf6d3688060a34c4014b0d9a752eb4c", + "sha256:58445347cff86791f89717f3bf79ec6f597d146397d9e78136cf9e937f363555", + "sha256:49f791cfca3e59c6094ec94d091473ddd9fe206e9860c0eb37dacbc3bbcccafd", + "sha256:b83c8811a2df5586918135a8bab5304c9c6f0c0a3b103c4b3ceb4515d2c480a5", + "sha256:36821795adb1d93e34b9835d2cd738738e0a7fb99b6232f00f69a0146f6db7fa", + "sha256:f31bf23bf137d6210ce78d1b133bab25ae0daffda0bfff172476479dfcc0b3a1", + "sha256:59064015f738a38367ca0ef7083840f3f1dbc579aa208071b4fb6b022a48d89a", + "sha256:3f161edc88f5ebe6db761902c3e563f450a8f373f58f6f9f59a13a7954f57d90", + }, + }, + }}, + }, + { + name: "Invalid/InvalidComponent", + imgTyp: TypeGeneric, + imgMapping: map[TypedImage]TypedImage{ + { + TypedImageReference: imagesource.TypedImageReference{ + Ref: reference.DockerImageReference{ + Name: "imgname", + }}, + Category: TypeGeneric}: { + TypedImageReference: imagesource.TypedImageReference{ + Ref: reference.DockerImageReference{ + Name: "single_manifest", + }, + Type: imagesource.DestinationRegistry, + }, + Category: TypeGeneric}}, + wantErr: true, + expError: &ErrInvalidComponent{}, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + asSet, err := AssociateRemoteImageLayers(context.TODO(), test.imgMapping, true) + if !test.wantErr { + require.NoError(t, err) + require.Equal(t, test.expResult, asSet) + } else { + require.ErrorAs(t, err, &test.expError) + } + }) + } +} + +func mirrorV2(v2Dir string) http.HandlerFunc { + dir := http.Dir(v2Dir) + fileHandler := http.FileServer(dir) + handler := func(w http.ResponseWriter, req *http.Request) { + if req.Method == "GET" && req.URL.Path == "/v2/" { + w.Header().Set("Docker-Distribution-API-Version", "2.0") + } + if req.Method == "GET" { + switch path.Base(path.Dir(req.URL.Path)) { + case "blobs": + w.Header().Set("Content-Type", "application/octet-stream") + case "manifests": + if f, err := dir.Open(req.URL.Path); err == nil { + defer f.Close() + if data, err := ioutil.ReadAll(f); err == nil { + var versioned manifest.Versioned + if err = json.Unmarshal(data, &versioned); err == nil { + w.Header().Set("Content-Type", versioned.MediaType) + } + } + } + } + } + fileHandler.ServeHTTP(w, req) + } + return http.HandlerFunc(handler) +} + +func copyV2(source, destination string) error { + err := filepath.Walk(source, func(path string, info os.FileInfo, err error) error { + relPath := strings.Replace(path, source, "", 1) + if relPath == "" { + return nil + } + switch m := info.Mode(); { + case m&fs.ModeSymlink != 0: // Tag is the file name, so follow the symlink to the layer ID-named file. + dst, err := os.Readlink(path) + if err != nil { + return err + } + id := filepath.Base(dst) + if err := os.Symlink(id, filepath.Join(destination, relPath)); err != nil { + return err + } + case m.IsDir(): + return os.Mkdir(filepath.Join(destination, relPath), 0755) + default: + data, err := ioutil.ReadFile(filepath.Join(source, relPath)) + if err != nil { + return err + } + return ioutil.WriteFile(filepath.Join(destination, relPath), data, 0777) + } + return nil + }) + return err +} diff --git a/pkg/image/association_test.go b/pkg/image/association_test.go index ff3e080a8..215607dc7 100644 --- a/pkg/image/association_test.go +++ b/pkg/image/association_test.go @@ -1,15 +1,8 @@ package image import ( - "io/fs" - "io/ioutil" - "os" - "path/filepath" - "strings" "testing" - "github.com/openshift/library-go/pkg/image/reference" - "github.com/openshift/oc/pkg/cli/image/imagesource" "github.com/stretchr/testify/require" ) @@ -18,261 +11,6 @@ const ( setTestKeyName = "setTestKey" ) -func TestAssociateImageLayers(t *testing.T) { - tests := []struct { - name string - imgTyp ImageType - imgMapping TypedImageMapping - expResult AssociationSet - expError error - wantErr bool - }{ - { - name: "Valid/ManifestWithTag", - imgTyp: TypeGeneric, - imgMapping: map[TypedImage]TypedImage{ - { - TypedImageReference: imagesource.TypedImageReference{ - Ref: reference.DockerImageReference{ - Name: "imgname", - Tag: "latest", - }}, - Category: TypeGeneric}: { - TypedImageReference: imagesource.TypedImageReference{ - Ref: reference.DockerImageReference{ - Name: "single_manifest", - Tag: "latest", - }, - Type: imagesource.DestinationFile, - }, - Category: TypeGeneric}}, - expResult: AssociationSet{"imgname:latest": map[string]Association{ - "imgname:latest": { - Name: "imgname:latest", - Path: "single_manifest", - TagSymlink: "latest", - ID: "sha256:d31c6ea5c50be93d6eb94d2b508f0208e84a308c011c6454ebf291d48b37df19", - Type: TypeGeneric, - ManifestDigests: nil, - LayerDigests: []string{ - "sha256:e8614d09b7bebabd9d8a450f44e88a8807c98a438a2ddd63146865286b132d1b", - "sha256:601401253d0aac2bc95cccea668761a6e69216468809d1cee837b2e8b398e241", - "sha256:211941188a4f55ffc6bcefa4f69b69b32c13fafb65738075de05808bbfcec086", - "sha256:f0fd5be261dfd2e36d01069a387a3e5125f5fd5adfec90f3cb190d1d5f1d1ad9", - "sha256:0c0beb258254c0566315c641b4107b080a96fa78d4f96833453dd6c5b9edf2b7", - "sha256:30c794a11b4c340c77238c5b7ca845752904bd8b74b73a9b16d31253234da031", - }, - }, - }}, - }, - { - name: "Valid/ManifestWithDigest", - imgTyp: TypeGeneric, - imgMapping: map[TypedImage]TypedImage{ - { - TypedImageReference: imagesource.TypedImageReference{ - Ref: reference.DockerImageReference{ - Name: "imgname", - ID: "sha256:d31c6ea5c50be93d6eb94d2b508f0208e84a308c011c6454ebf291d48b37df19", - }}, - Category: TypeGeneric}: { - TypedImageReference: imagesource.TypedImageReference{ - Ref: reference.DockerImageReference{ - Name: "single_manifest", - ID: "sha256:d31c6ea5c50be93d6eb94d2b508f0208e84a308c011c6454ebf291d48b37df19", - }, - Type: imagesource.DestinationFile, - }, - Category: TypeGeneric}}, - expResult: AssociationSet{"imgname@sha256:d31c6ea5c50be93d6eb94d2b508f0208e84a308c011c6454ebf291d48b37df19": map[string]Association{ - "imgname@sha256:d31c6ea5c50be93d6eb94d2b508f0208e84a308c011c6454ebf291d48b37df19": { - Name: "imgname@sha256:d31c6ea5c50be93d6eb94d2b508f0208e84a308c011c6454ebf291d48b37df19", - Path: "single_manifest", - TagSymlink: "oc-mirrord31c6e", - ID: "sha256:d31c6ea5c50be93d6eb94d2b508f0208e84a308c011c6454ebf291d48b37df19", - Type: TypeGeneric, - ManifestDigests: nil, - LayerDigests: []string{ - "sha256:e8614d09b7bebabd9d8a450f44e88a8807c98a438a2ddd63146865286b132d1b", - "sha256:601401253d0aac2bc95cccea668761a6e69216468809d1cee837b2e8b398e241", - "sha256:211941188a4f55ffc6bcefa4f69b69b32c13fafb65738075de05808bbfcec086", - "sha256:f0fd5be261dfd2e36d01069a387a3e5125f5fd5adfec90f3cb190d1d5f1d1ad9", - "sha256:0c0beb258254c0566315c641b4107b080a96fa78d4f96833453dd6c5b9edf2b7", - "sha256:30c794a11b4c340c77238c5b7ca845752904bd8b74b73a9b16d31253234da031", - }, - }, - }}, - }, - { - name: "Valid/IndexManifest", - imgTyp: TypeGeneric, - imgMapping: map[TypedImage]TypedImage{ - { - TypedImageReference: imagesource.TypedImageReference{ - Ref: reference.DockerImageReference{ - Name: "imgname", - Tag: "latest", - }}, - Category: TypeGeneric}: { - TypedImageReference: imagesource.TypedImageReference{ - Ref: reference.DockerImageReference{ - Name: "index_manifest", - Tag: "latest", - }, - Type: imagesource.DestinationFile, - }, - Category: TypeGeneric}}, - expResult: AssociationSet{"imgname:latest": map[string]Association{ - "imgname:latest": { - Name: "imgname:latest", - Path: "index_manifest", - TagSymlink: "latest", - ID: "sha256:d15a206e4ee462e82ab722ed84dfa514ab9ed8d85100d591c04314ae7c2162ee", - Type: TypeGeneric, - ManifestDigests: []string{ - "sha256:bab3a6153010b614c8764548f0dbe34c4a7dce4ea278a94713c3e9a936bb74e6", - "sha256:9574416689665a82cb4eaf43463da5b6156071ebbec117262eef7fa32b4d7021", - "sha256:b8a825862d73b2f1110dd9c5fc0631f47117c7cd99e42efa34244cd82bd6742f", - "sha256:60f5921e0f6a21a485a0a4e9415761afb5b60814bbe8a6864cb12b90ae24c1d0", - }, - LayerDigests: nil, - }, - "sha256:60f5921e0f6a21a485a0a4e9415761afb5b60814bbe8a6864cb12b90ae24c1d0": { - Name: "sha256:60f5921e0f6a21a485a0a4e9415761afb5b60814bbe8a6864cb12b90ae24c1d0", - Path: "index_manifest", - TagSymlink: "", - ID: "sha256:60f5921e0f6a21a485a0a4e9415761afb5b60814bbe8a6864cb12b90ae24c1d0", - Type: TypeGeneric, - LayerDigests: []string{ - "sha256:b538f80385f9b48122e3da068c932a96ea5018afa3c7be79da00437414bd18cd", - "sha256:342a15c43afd15b4d93051022ecf020ea6fde1e14d34599f5b4c10a8a5bae3c6", - "sha256:70660e39ee11b715823a96729d7f1b8964ecd6ca2b7c0e3fd5cde284e34758eb", - "sha256:f553d3748799c35aa60227875706f727a526a1d4c7840a5d550cdb4ba6cd5196", - "sha256:c5338ca295456f5c677bf8910ac94765be2f53977af6bd792f18a2298054d6be", - "sha256:af94dd630ca5e3e15d15502c2a03e386f4c1ef5a59def62e84ede35a009c4110", - "sha256:337fc839f463fd6b6d1773e0b8f2f9d40b3a8dff6963008193344cd29466a3d1", - "sha256:4d4b85daa42ca075d8aff8563d14434799268a4b823e74737171ed438f8c60ad", - }, - }, - "sha256:9574416689665a82cb4eaf43463da5b6156071ebbec117262eef7fa32b4d7021": { - Name: "sha256:9574416689665a82cb4eaf43463da5b6156071ebbec117262eef7fa32b4d7021", - Path: "index_manifest", - TagSymlink: "", - ID: "sha256:9574416689665a82cb4eaf43463da5b6156071ebbec117262eef7fa32b4d7021", - Type: TypeGeneric, - LayerDigests: []string{ - "sha256:b4b72e716706d29f5d2351709c20bf737b94f876a5472a43ff1b6e203c65d27f", - "sha256:8d0157f7a4ed4136f430f737f0f79d650248e19ebd87371f1ae1735536f0eaf2", - "sha256:46f9bc09f2ae8c0a95a69d77cd91527281cf54cd466dbee5ba6b28e05ee68a77", - "sha256:21d0f0a83af189ace4e566f1520e8ac5a404adda15edb534ee79a994bdd94abe", - "sha256:61a5adb16b8c308ed6481d3abac7e08035f09d936f2a1ecad0bd2000a18464b9", - "sha256:a92dcc7bd9c9c1369ef92728f7649e3ec868b53b7b38ab2a4bddc525f74896a8", - "sha256:317a9dc239a3310e2010e6e1c4f2a87b4b2c53f49ca5231c031227540ef91d0b", - "sha256:d476ce7797cc1558919a31a1cccd9b09f48ea2787982ccd3c2576252450d2d51", - }, - }, - "sha256:b8a825862d73b2f1110dd9c5fc0631f47117c7cd99e42efa34244cd82bd6742f": { - Name: "sha256:b8a825862d73b2f1110dd9c5fc0631f47117c7cd99e42efa34244cd82bd6742f", - Path: "index_manifest", - TagSymlink: "", - ID: "sha256:b8a825862d73b2f1110dd9c5fc0631f47117c7cd99e42efa34244cd82bd6742f", - Type: TypeGeneric, - LayerDigests: []string{ - "sha256:52278dd8e57993669c5b72a9620e89bebdc098f2af2379caaa8945f7403f77a2", - "sha256:1dc2a2c4dd124cf83f27e6d8852303f7874507b71a3f7b6a1265837b43279092", - "sha256:26100ac97b3237b89768d0dac0150c6a2b483a16b0662160df98d03ba25fa474", - "sha256:7c120a97d24392c377b955ca42f09fc04942aecff3f0a007d31ebd20c185958a", - "sha256:87875760340f78f13107842911184c55308475062940399772e7944138879704", - "sha256:5ad5a4942ddf238ce385d4b29eaa3b2d5f8836de538918d7da9a839c8313fd46", - "sha256:6121cb3c461255702c8b8ef03ed4b13061c0c600b20c7664ce82815ed15febbd", - "sha256:c72bf53b697715cd03c3f3dc6fd6d2bccb4b10e511c2847eb98e312d28850e48", - }, - }, - "sha256:bab3a6153010b614c8764548f0dbe34c4a7dce4ea278a94713c3e9a936bb74e6": { - Name: "sha256:bab3a6153010b614c8764548f0dbe34c4a7dce4ea278a94713c3e9a936bb74e6", - Path: "index_manifest", - TagSymlink: "", - ID: "sha256:bab3a6153010b614c8764548f0dbe34c4a7dce4ea278a94713c3e9a936bb74e6", - Type: TypeGeneric, - LayerDigests: []string{ - "sha256:df20fa9351a15782c64e6dddb2d4a6f50bf6d3688060a34c4014b0d9a752eb4c", - "sha256:58445347cff86791f89717f3bf79ec6f597d146397d9e78136cf9e937f363555", - "sha256:49f791cfca3e59c6094ec94d091473ddd9fe206e9860c0eb37dacbc3bbcccafd", - "sha256:b83c8811a2df5586918135a8bab5304c9c6f0c0a3b103c4b3ceb4515d2c480a5", - "sha256:36821795adb1d93e34b9835d2cd738738e0a7fb99b6232f00f69a0146f6db7fa", - "sha256:f31bf23bf137d6210ce78d1b133bab25ae0daffda0bfff172476479dfcc0b3a1", - "sha256:59064015f738a38367ca0ef7083840f3f1dbc579aa208071b4fb6b022a48d89a", - "sha256:3f161edc88f5ebe6db761902c3e563f450a8f373f58f6f9f59a13a7954f57d90", - }, - }, - }}, - }, - { - name: "Invalid/InvalidComponent", - imgTyp: TypeGeneric, - imgMapping: map[TypedImage]TypedImage{ - { - TypedImageReference: imagesource.TypedImageReference{ - Ref: reference.DockerImageReference{ - Name: "imgname", - }}, - Category: TypeGeneric}: { - TypedImageReference: imagesource.TypedImageReference{ - Ref: reference.DockerImageReference{ - Name: "single_manifest", - }, - Type: imagesource.DestinationFile, - }, - Category: TypeGeneric}}, - wantErr: true, - expError: &ErrInvalidComponent{}, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - tmpdir := t.TempDir() - require.NoError(t, copyV2("testdata", tmpdir)) - asSet, err := AssociateImageLayers(tmpdir, test.imgMapping) - if !test.wantErr { - require.NoError(t, err) - require.Equal(t, test.expResult, asSet) - } else { - require.ErrorAs(t, err, &test.expError) - } - }) - } -} - -func copyV2(source, destination string) error { - err := filepath.Walk(source, func(path string, info os.FileInfo, err error) error { - relPath := strings.Replace(path, source, "", 1) - if relPath == "" { - return nil - } - switch m := info.Mode(); { - case m&fs.ModeSymlink != 0: // Tag is the file name, so follow the symlink to the layer ID-named file. - dst, err := os.Readlink(path) - if err != nil { - return err - } - id := filepath.Base(dst) - if err := os.Symlink(id, filepath.Join(destination, relPath)); err != nil { - return err - } - case m.IsDir(): - return os.Mkdir(filepath.Join(destination, relPath), 0755) - default: - data, err := ioutil.ReadFile(filepath.Join(source, relPath)) - if err != nil { - return err - } - return ioutil.WriteFile(filepath.Join(destination, relPath), data, 0777) - } - return nil - }) - return err -} - func TestUpdateKey(t *testing.T) { asSet := makeTestAssocationSet() testAssocs, ok := asSet[setTestKeyName] @@ -342,6 +80,12 @@ func TestSearch(t *testing.T) { require.Len(t, assocs, 1) } +func TestGetDigests(t *testing.T) { + asSet := makeTestAssocationSet() + digests := asSet.GetDigests() + require.Len(t, digests, 2) +} + func TestAdd(t *testing.T) { asSet := AssociationSet{} newAssoc := Association{ @@ -363,15 +107,24 @@ func TestAdd(t *testing.T) { require.Equal(t, newAssoc, assoc) } +func TestGetImageFromBlob(t *testing.T) { + asSet := makeTestAssocationSet() + ref := GetImageFromBlob(asSet, "test-layer") + require.Equal(t, setTestKeyName, ref) + ref = GetImageFromBlob(asSet, "fake") + require.Equal(t, "", ref) +} + func makeTestAssocationSet() AssociationSet { asSet := AssociationSet{} assocs := Associations{} association := Association{ - Name: testKeyName, - Path: "test", - ID: "test-id", - TagSymlink: "test-tag", - Type: TypeGeneric, + Name: testKeyName, + Path: "test", + ID: "test-id", + TagSymlink: "test-tag", + Type: TypeGeneric, + LayerDigests: []string{"test-layer"}, } assocs[testKeyName] = association asSet[setTestKeyName] = assocs diff --git a/pkg/config/credentials.go b/pkg/image/credentials.go similarity index 98% rename from pkg/config/credentials.go rename to pkg/image/credentials.go index 18314ae7f..62b76fa89 100644 --- a/pkg/config/credentials.go +++ b/pkg/image/credentials.go @@ -1,4 +1,4 @@ -package config +package image import ( "errors" diff --git a/vendor/modules.txt b/vendor/modules.txt index bb5fcaa28..597fe152c 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -183,6 +183,7 @@ github.com/docker/cli/cli/config/configfile github.com/docker/cli/cli/config/credentials github.com/docker/cli/cli/config/types # github.com/docker/distribution v2.7.1+incompatible +## explicit github.com/docker/distribution github.com/docker/distribution/digestset github.com/docker/distribution/manifest