CLID-17: feat: docker v2s2 manifest list (fat manifest) / oci image i…

…ndex implementation

v2/pkg/additional/const.go

@@ -3,5 +3,4 @@ package additional
 const (
 	dockerProtocol string = "docker://"
 	ociProtocol    string = "oci://"
-	hashTruncLen   int    = 12
 )

v2/pkg/additional/local_stored_collector.go

@@ -41,7 +41,7 @@ func (o LocalStorageCollector) AdditionalImagesCollector(ctx context.Context) ([
 			src = imgSpec.ReferenceWithTransport
 
 			if imgSpec.IsImageByDigest() {
-				dest = dockerProtocol + strings.Join([]string{o.LocalStorageFQDN, imgSpec.PathComponent + ":" + imgSpec.Digest[:hashTruncLen]}, "/")
+				dest = dockerProtocol + strings.Join([]string{o.LocalStorageFQDN, imgSpec.PathComponent + ":" + imgSpec.Digest}, "/")
 			} else {
 				dest = dockerProtocol + strings.Join([]string{o.LocalStorageFQDN, imgSpec.PathComponent}, "/") + ":" + imgSpec.Tag
 			}
@@ -66,8 +66,8 @@ func (o LocalStorageCollector) AdditionalImagesCollector(ctx context.Context) ([
 				}
 
 				if imgSpec.IsImageByDigest() {
-					src = dockerProtocol + strings.Join([]string{o.LocalStorageFQDN, imgSpec.PathComponent + ":" + imgSpec.Digest[:hashTruncLen]}, "/")
-					dest = strings.Join([]string{o.Opts.Destination, imgSpec.PathComponent + ":" + imgSpec.Digest[:hashTruncLen]}, "/")
+					src = dockerProtocol + strings.Join([]string{o.LocalStorageFQDN, imgSpec.PathComponent + ":" + imgSpec.Digest}, "/")
+					dest = strings.Join([]string{o.Opts.Destination, imgSpec.PathComponent + ":" + imgSpec.Digest}, "/")
 				} else {
 					src = dockerProtocol + strings.Join([]string{o.LocalStorageFQDN, imgSpec.PathComponent}, "/") + ":" + imgSpec.Tag
 					dest = strings.Join([]string{o.Opts.Destination, imgSpec.PathComponent}, "/") + ":" + imgSpec.Tag

v2/pkg/additional/local_stored_collector_test.go

@@ -186,3 +186,7 @@ func (o MockManifest) ExtractLayersOCI(filePath, toPath, label string, oci *v1al
 func (o MockManifest) ExtractLayers(filePath, name, label string) error {
 	return nil
 }
+
+func (o MockManifest) ConvertIndexToSingleManifest(dir string, oci *v1alpha3.OCISchema) error {
+	return nil
+}

v2/pkg/api/v1alpha3/types.go

@@ -82,16 +82,16 @@ type Status struct {
 // OCISchema
 type OCISchema struct {
 	SchemaVersion int           `json:"schemaVersion"`
-	MediaType     string        `json:"mediaType"`
+	MediaType     string        `json:"mediaType,omitempty"`
 	Manifests     []OCIManifest `json:"manifests"`
-	Config        OCIManifest   `json:"config"`
-	Layers        []OCIManifest `json:"layers"`
+	Config        OCIManifest   `json:"config,omitempty"`
+	Layers        []OCIManifest `json:"layers,omitempty"`
 }
 
 type OCIManifest struct {
-	MediaType string `json:"mediaType"`
-	Digest    string `json:"digest"`
-	Size      int    `json:"size"`
+	MediaType string `json:"mediaType,omitempty"`
+	Digest    string `json:"digest,omitempty"`
+	Size      int    `json:"size,omitempty"`
 }
 
 // OperatorConfigSchema

v2/pkg/batch/worker.go

@@ -90,7 +90,6 @@ func (o *Batch) Worker(ctx context.Context, images []v1alpha3.CopyImageSchema, o
 			index := (i * b.BatchSize) + x
 			o.Log.Debug("source %s ", images[index].Source)
 			o.Log.Debug("destination %s ", images[index].Destination)
-			opts.MultiArch = "all"
 			go func(ctx context.Context, src, dest string, opts *mirror.CopyOptions, writer bufio.Writer) {
 				defer wg.Done()
 				err := o.Mirror.Run(ctx, src, dest, "copy", opts, writer)

v2/pkg/batch/worker_test.go

@@ -65,22 +65,22 @@ func TestWorker(t *testing.T) {
 type Mirror struct{}
 type Manifest struct{}
 
-func (o *Mirror) Run(ctx context.Context, src, dest string, mode mirror.Mode, opts *mirror.CopyOptions, stdout bufio.Writer) error {
+func (o Mirror) Run(ctx context.Context, src, dest string, mode mirror.Mode, opts *mirror.CopyOptions, stdout bufio.Writer) error {
 	return nil
 }
 
-func (o *Mirror) Check(ctx context.Context, image string, opts *mirror.CopyOptions) (bool, error) {
+func (o Mirror) Check(ctx context.Context, image string, opts *mirror.CopyOptions) (bool, error) {
 	return true, nil
 }
 
-func (o *Manifest) GetOperatorConfig(file string) (*v1alpha3.OperatorConfigSchema, error) {
+func (o Manifest) GetOperatorConfig(file string) (*v1alpha3.OperatorConfigSchema, error) {
 	opcl := v1alpha3.OperatorLabels{OperatorsOperatorframeworkIoIndexConfigsV1: "/configs"}
 	opc := v1alpha3.OperatorConfig{Labels: opcl}
 	ocs := &v1alpha3.OperatorConfigSchema{Config: opc}
 	return ocs, nil
 }
 
-func (o *Manifest) GetReleaseSchema(filePath string) ([]v1alpha3.RelatedImage, error) {
+func (o Manifest) GetReleaseSchema(filePath string) ([]v1alpha3.RelatedImage, error) {
 	relatedImages := []v1alpha3.RelatedImage{
 		{Name: "testA", Image: "registry/name/namespace/sometestimage-a@sha256:f30638f60452062aba36a26ee6c036feead2f03b28f2c47f2b0a991e41baebea"},
 		{Name: "testB", Image: "registry/name/namespace/sometestimage-b@sha256:f30638f60452062aba36a26ee6c036feead2f03b28f2c47f2b0a991e41baebea"},
@@ -90,7 +90,7 @@ func (o *Manifest) GetReleaseSchema(filePath string) ([]v1alpha3.RelatedImage, e
 	return relatedImages, nil
 }
 
-func (o *Manifest) GetImageIndex(name string) (*v1alpha3.OCISchema, error) {
+func (o Manifest) GetImageIndex(name string) (*v1alpha3.OCISchema, error) {
 	return &v1alpha3.OCISchema{
 		SchemaVersion: 2,
 		Manifests: []v1alpha3.OCIManifest{
@@ -103,7 +103,7 @@ func (o *Manifest) GetImageIndex(name string) (*v1alpha3.OCISchema, error) {
 	}, nil
 }
 
-func (o *Manifest) GetImageManifest(name string) (*v1alpha3.OCISchema, error) {
+func (o Manifest) GetImageManifest(name string) (*v1alpha3.OCISchema, error) {
 	return &v1alpha3.OCISchema{
 		SchemaVersion: 2,
 		Manifests: []v1alpha3.OCIManifest{
@@ -125,7 +125,7 @@ func (o Manifest) GetCatalog(filePath string) (manifest.OperatorCatalog, error)
 	return manifest.OperatorCatalog{}, nil
 }
 
-func (o *Manifest) GetRelatedImagesFromCatalog(operatorCatalog manifest.OperatorCatalog, ctlgInIsc v1alpha2.Operator) (map[string][]v1alpha3.RelatedImage, error) {
+func (o Manifest) GetRelatedImagesFromCatalog(operatorCatalog manifest.OperatorCatalog, ctlgInIsc v1alpha2.Operator) (map[string][]v1alpha3.RelatedImage, error) {
 	relatedImages := make(map[string][]v1alpha3.RelatedImage)
 	relatedImages["abc"] = []v1alpha3.RelatedImage{
 		{Name: "testA", Image: "sometestimage-a@sha256:f30638f60452062aba36a26ee6c036feead2f03b28f2c47f2b0a991e41baebea"},
@@ -134,10 +134,14 @@ func (o *Manifest) GetRelatedImagesFromCatalog(operatorCatalog manifest.Operator
 	return relatedImages, nil
 }
 
-func (o *Manifest) ExtractLayersOCI(filePath, toPath, label string, oci *v1alpha3.OCISchema) error {
+func (o Manifest) ExtractLayersOCI(filePath, toPath, label string, oci *v1alpha3.OCISchema) error {
 	return nil
 }
 
-func (o *Manifest) ExtractLayers(filePath, name, label string) error {
+func (o Manifest) ExtractLayers(filePath, name, label string) error {
+	return nil
+}
+
+func (o Manifest) ConvertIndexToSingleManifest(dir string, oci *v1alpha3.OCISchema) error {
 	return nil
 }

v2/pkg/cli/executor.go

@@ -83,7 +83,7 @@ type ExecutorSchema struct {
 	LogsDir                      string
 	registryLogFile              *os.File
 	Config                       v1alpha2.ImageSetConfiguration
-	Opts                         mirror.CopyOptions
+	Opts                         *mirror.CopyOptions
 	Operator                     operator.CollectorInterface
 	Release                      release.CollectorInterface
 	AdditionalImages             additional.CollectorInterface
@@ -126,7 +126,7 @@ func NewMirrorCmd(log clog.PluggableLoggerInterface) *cobra.Command {
 	flagDestOpts, destOpts := mirror.ImageDestFlags(global, sharedOpts, deprecatedTLSVerifyOpt, "dest-", "dcreds")
 	flagRetryOpts, retryOpts := mirror.RetryFlags()
 
-	opts := mirror.CopyOptions{
+	opts := &mirror.CopyOptions{
 		Global:              global,
 		DeprecatedTLSVerify: deprecatedTLSVerifyOpt,
 		SrcImage:            srcOpts,
@@ -189,6 +189,7 @@ func NewMirrorCmd(log clog.PluggableLoggerInterface) *cobra.Command {
 	cmd.Flags().IntVar(&opts.Global.MaxNestedPaths, "max-nested-paths", 0, "Number of nested paths, for destination registries that limit nested paths")
 	cmd.Flags().BoolVar(&opts.Global.StrictArchiving, "strict-archive", false, "// If set, generates archives that are strictly less than `archiveSize`, failing for files that exceed that limit.")
 	cmd.Flags().StringVar(&opts.Global.SinceString, "since", "", "Include all new content since specified date (format yyyy-MM-dd). When not provided, new content since previous mirroring is mirrored")
+	cmd.Flags().BoolVar(&opts.All, "all", opts.All, "copy all available architectures images in case the image is multi-arch (manifest list / index)")
 	// nolint: errcheck
 	cmd.Flags().AddFlagSet(&flagSharedOpts)
 	cmd.Flags().AddFlagSet(&flagRetryOpts)
@@ -324,24 +325,24 @@ func (o *ExecutorSchema) Complete(args []string) error {
 
 	client, _ := release.NewOCPClient(uuid.New())
 
-	o.ImageBuilder = imagebuilder.NewBuilder(o.Log, o.Opts)
+	o.ImageBuilder = imagebuilder.NewBuilder(o.Log, *o.Opts)
 
-	signature := release.NewSignatureClient(o.Log, o.Config, o.Opts)
-	cn := release.NewCincinnati(o.Log, &o.Config, o.Opts, client, false, signature)
-	o.Release = release.New(o.Log, o.LogsDir, o.Config, o.Opts, o.Mirror, o.Manifest, cn, o.LocalStorageFQDN, o.ImageBuilder)
-	o.Operator = operator.New(o.Log, o.LogsDir, o.Config, o.Opts, o.Mirror, o.Manifest, o.LocalStorageFQDN)
-	o.AdditionalImages = additional.New(o.Log, o.Config, o.Opts, o.Mirror, o.Manifest, o.LocalStorageFQDN)
+	signature := release.NewSignatureClient(o.Log, o.Config, *o.Opts)
+	cn := release.NewCincinnati(o.Log, &o.Config, *o.Opts, client, false, signature)
+	o.Release = release.New(o.Log, o.LogsDir, o.Config, *o.Opts, o.Mirror, o.Manifest, cn, o.LocalStorageFQDN, o.ImageBuilder)
+	o.Operator = operator.New(o.Log, o.LogsDir, o.Config, *o.Opts, o.Mirror, o.Manifest, o.LocalStorageFQDN)
+	o.AdditionalImages = additional.New(o.Log, o.Config, *o.Opts, o.Mirror, o.Manifest, o.LocalStorageFQDN)
 	o.ClusterResources = clusterresources.New(o.Log, o.Opts.Global.WorkingDir, o.Config)
 	o.Batch = batch.New(o.Log, o.LogsDir, o.Mirror, o.Manifest)
 
 	if o.Opts.IsMirrorToDisk() {
 		if o.Opts.Global.StrictArchiving {
-			o.MirrorArchiver, err = archive.NewMirrorArchive(&o.Opts, rootDir, o.Opts.Global.ConfigPath, o.Opts.Global.WorkingDir, o.LocalStorageDisk, o.Config.ImageSetConfigurationSpec.ArchiveSize, o.Log)
+			o.MirrorArchiver, err = archive.NewMirrorArchive(o.Opts, rootDir, o.Opts.Global.ConfigPath, o.Opts.Global.WorkingDir, o.LocalStorageDisk, o.Config.ImageSetConfigurationSpec.ArchiveSize, o.Log)
 			if err != nil {
 				return err
 			}
 		} else {
-			o.MirrorArchiver, err = archive.NewPermissiveMirrorArchive(&o.Opts, rootDir, o.Opts.Global.ConfigPath, o.Opts.Global.WorkingDir, o.LocalStorageDisk, o.Config.ImageSetConfigurationSpec.ArchiveSize, o.Log)
+			o.MirrorArchiver, err = archive.NewPermissiveMirrorArchive(o.Opts, rootDir, o.Opts.Global.ConfigPath, o.Opts.Global.WorkingDir, o.LocalStorageDisk, o.Config.ImageSetConfigurationSpec.ArchiveSize, o.Log)
 			if err != nil {
 				return err
 			}
@@ -357,9 +358,6 @@ func (o *ExecutorSchema) Complete(args []string) error {
 
 // Run - start the mirror functionality
 func (o *ExecutorSchema) Run(cmd *cobra.Command, args []string) error {
-
-	// make sure we always get multi-arch images
-	o.Opts.MultiArch = "all"
 	var err error
 	if o.Opts.IsMirrorToDisk() {
 		err = o.RunMirrorToDisk(cmd, args)
@@ -573,7 +571,7 @@ func (o *ExecutorSchema) RunMirrorToDisk(cmd *cobra.Command, args []string) erro
 	collectionFinish := time.Now()
 
 	//call the batch worker
-	err = o.Batch.Worker(cmd.Context(), allImages, o.Opts)
+	err = o.Batch.Worker(cmd.Context(), allImages, *o.Opts)
 	if err != nil {
 		return err
 	}
@@ -629,7 +627,7 @@ func (o *ExecutorSchema) RunDiskToMirror(cmd *cobra.Command, args []string) erro
 		}
 	}
 	//call the batch worker
-	err = o.Batch.Worker(cmd.Context(), allImages, o.Opts)
+	err = o.Batch.Worker(cmd.Context(), allImages, *o.Opts)
 	if err != nil {
 		return err
 	}
@@ -770,7 +768,7 @@ func NewPrepareCommand(log clog.PluggableLoggerInterface) *cobra.Command {
 	flagDestOpts, destOpts := mirror.ImageDestFlags(global, sharedOpts, deprecatedTLSVerifyOpt, "dest-", "dcreds")
 	flagRetryOpts, retryOpts := mirror.RetryFlags()
 
-	opts := mirror.CopyOptions{
+	opts := &mirror.CopyOptions{
 		Global:              global,
 		DeprecatedTLSVerify: deprecatedTLSVerifyOpt,
 		SrcImage:            srcOpts,
@@ -882,11 +880,11 @@ func (o *ExecutorSchema) CompletePrepare(args []string) error {
 	}
 	client, _ := release.NewOCPClient(uuid.New())
 
-	signature := release.NewSignatureClient(o.Log, o.Config, o.Opts)
-	cn := release.NewCincinnati(o.Log, &o.Config, o.Opts, client, false, signature)
-	o.Release = release.New(o.Log, o.LogsDir, o.Config, o.Opts, o.Mirror, o.Manifest, cn, o.LocalStorageFQDN, o.ImageBuilder)
-	o.Operator = operator.New(o.Log, o.LogsDir, o.Config, o.Opts, o.Mirror, o.Manifest, o.LocalStorageFQDN)
-	o.AdditionalImages = additional.New(o.Log, o.Config, o.Opts, o.Mirror, o.Manifest, o.LocalStorageFQDN)
+	signature := release.NewSignatureClient(o.Log, o.Config, *o.Opts)
+	cn := release.NewCincinnati(o.Log, &o.Config, *o.Opts, client, false, signature)
+	o.Release = release.New(o.Log, o.LogsDir, o.Config, *o.Opts, o.Mirror, o.Manifest, cn, o.LocalStorageFQDN, o.ImageBuilder)
+	o.Operator = operator.New(o.Log, o.LogsDir, o.Config, *o.Opts, o.Mirror, o.Manifest, o.LocalStorageFQDN)
+	o.AdditionalImages = additional.New(o.Log, o.Config, *o.Opts, o.Mirror, o.Manifest, o.LocalStorageFQDN)
 	return nil
 }
 
@@ -914,7 +912,7 @@ func (o *ExecutorSchema) RunPrepare(cmd *cobra.Command, args []string) error {
 	var buff bytes.Buffer
 	for _, img := range allImages {
 		buff.WriteString(img.Destination + "\n")
-		exists, err := o.Mirror.Check(cmd.Context(), img.Destination, &o.Opts)
+		exists, err := o.Mirror.Check(cmd.Context(), img.Destination, o.Opts)
 		if err != nil {
 			o.Log.Warn("unable to check existence of %s in local cache: %v", img.Destination, err)
 		}