Merge pull request #818 from awgreene/split-large-icsp

Bug 1951203: Allow users to set a limit on ICSP file size

contrib/completions/bash/oc

@@ -614,6 +614,10 @@ _oc_adm_catalog_mirror()
     two_word_flags+=("--max-components")
     local_nonpersistent_flags+=("--max-components")
     local_nonpersistent_flags+=("--max-components=")
+    flags+=("--max-icsp-size=")
+    two_word_flags+=("--max-icsp-size")
+    local_nonpersistent_flags+=("--max-icsp-size")
+    local_nonpersistent_flags+=("--max-icsp-size=")
     flags+=("--max-per-registry=")
     two_word_flags+=("--max-per-registry")
     local_nonpersistent_flags+=("--max-per-registry")

contrib/completions/zsh/oc

@@ -714,6 +714,10 @@ _oc_adm_catalog_mirror()
     two_word_flags+=("--max-components")
     local_nonpersistent_flags+=("--max-components")
     local_nonpersistent_flags+=("--max-components=")
+    flags+=("--max-icsp-size=")
+    two_word_flags+=("--max-icsp-size")
+    local_nonpersistent_flags+=("--max-icsp-size")
+    local_nonpersistent_flags+=("--max-icsp-size=")
     flags+=("--max-per-registry=")
     two_word_flags+=("--max-per-registry")
     local_nonpersistent_flags+=("--max-per-registry")

go.mod

@@ -53,7 +53,7 @@ require (
 	google.golang.org/genproto v0.0.0-20210219173056-d891e3cb3b5b // indirect
 	google.golang.org/grpc v1.35.0 // indirect
 	gopkg.in/ldap.v2 v2.5.1
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 	k8s.io/api v0.21.1
 	k8s.io/apimachinery v0.21.1
 	k8s.io/apiserver v0.21.1

pkg/cli/admin/catalog/mirror.go

@@ -8,6 +8,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"text/tabwriter"
 	"time"
@@ -68,10 +69,15 @@ var (
 		# Edit the mirroring mappings and mirror with "oc image mirror" manually
 		oc adm catalog mirror --manifests-only quay.io/my/image:latest myregistry.com
 		oc image mirror -f manifests/mapping.txt
+
+		# Delete all ImageContentSourcePolicies generated by oc adm catalog mirror
+		oc delete imagecontentsourcepolicy -l operators.openshift.org/catalog=true
 	`)
 )
 
-const IndexLocationLabelKey = "operators.operatorframework.io.index.database.v1"
+const (
+	IndexLocationLabelKey = "operators.operatorframework.io.index.database.v1"
+)
 
 func init() {
 	subCommands = append(subCommands, NewMirrorCatalog)
@@ -87,6 +93,7 @@ type MirrorCatalogOptions struct {
 
 	FromFileDir string
 	FileDir     string
+	MaxICSPSize int
 
 	IcspScope string
 
@@ -145,6 +152,7 @@ func NewMirrorCatalog(f kcmdutil.Factory, streams genericclioptions.IOStreams) *
 	flags.StringVar(&o.FromFileDir, "from-dir", o.FromFileDir, "The directory on disk that file:// images will be read from. Overrides --dir")
 	flags.IntVar(&o.MaxPathComponents, "max-components", 2, "The maximum number of path components allowed in a destination mapping. Example: `quay.io/org/repo` has two path components.")
 	flags.StringVar(&o.IcspScope, "icsp-scope", o.IcspScope, "Scope of registry mirrors in imagecontentsourcepolicy file. Allowed values: repository, registry. Defaults to: repository")
+	flags.IntVar(&o.MaxICSPSize, "max-icsp-size", 250000, "The maximum number of bytes for the generated ICSP yaml(s). Defaults to 250000")
 	return cmd
 }
 
@@ -382,10 +390,49 @@ func (o *MirrorCatalogOptions) Run() error {
 		fmt.Fprintf(o.IOStreams.ErrOut, "errors during mirroring. the full contents of the catalog may not have been mirrored: %s\n", err.Error())
 	}
 
-	return WriteManifests(o.IOStreams.Out, o.SourceRef, o.DestRef, o.ManifestDir, o.IcspScope, mapping)
+	return WriteManifests(o.IOStreams.Out, o.SourceRef, o.DestRef, o.ManifestDir, o.IcspScope, o.MaxICSPSize, mapping)
+}
+
+func getRegistryMapping(out io.Writer, icspScope string, mapping map[imagesource.TypedImageReference]imagesource.TypedImageReference) map[string]string {
+	registryMapping := map[string]string{}
+	for k, v := range mapping {
+		if len(v.Ref.ID) == 0 {
+			fmt.Fprintf(out, "no digest mapping available for %s, skip writing to ImageContentSourcePolicy\n", k)
+			continue
+		}
+		if icspScope == "registry" {
+			registryMapping[k.Ref.Registry] = v.Ref.Registry
+		} else {
+			registryMapping[k.Ref.AsRepository().String()] = v.Ref.AsRepository().String()
+		}
+	}
+	return registryMapping
+}
+
+func generateICSPs(out io.Writer, source string, icspScope string, maxICSPSize int, mapping map[imagesource.TypedImageReference]imagesource.TypedImageReference) ([][]byte, error) {
+	registryMapping := getRegistryMapping(out, icspScope, mapping)
+	icsps := [][]byte{}
+
+	for i := 0; len(registryMapping) != 0; i++ {
+		icsp, err := generateICSP(out, source+"-"+strconv.Itoa(i), maxICSPSize, registryMapping)
+		if err != nil {
+			return nil, err
+		}
+		icsps = append(icsps, icsp)
+	}
+	return icsps, nil
 }
 
-func WriteManifests(out io.Writer, source, dest imagesource.TypedImageReference, dir, icspScope string, mapping map[imagesource.TypedImageReference]imagesource.TypedImageReference) error {
+func aggregateICSPs(icsps [][]byte) []byte {
+	aggregation := []byte{}
+	for _, icsp := range icsps {
+		aggregation = append(aggregation, []byte("---\n")...)
+		aggregation = append(aggregation, icsp...)
+	}
+	return aggregation
+}
+
+func WriteManifests(out io.Writer, source, dest imagesource.TypedImageReference, dir, icspScope string, maxICSPSize int, mapping map[imagesource.TypedImageReference]imagesource.TypedImageReference) error {
 	f, err := os.Create(filepath.Join(dir, "mapping.txt"))
 	if err != nil {
 		return err
@@ -401,12 +448,12 @@ func WriteManifests(out io.Writer, source, dest imagesource.TypedImageReference,
 	}
 
 	if dest.Type != imagesource.DestinationFile {
-		icsp, err := generateICSP(out, source.Ref.Name, icspScope, mapping)
+		icsps, err := generateICSPs(out, source.Ref.Name, icspScope, maxICSPSize, mapping)
 		if err != nil {
 			return err
 		}
 
-		if err := ioutil.WriteFile(filepath.Join(dir, "imageContentSourcePolicy.yaml"), icsp, os.ModePerm); err != nil {
+		if err := ioutil.WriteFile(filepath.Join(dir, "imageContentSourcePolicy.yaml"), aggregateICSPs(icsps), os.ModePerm); err != nil {
 			return fmt.Errorf("error writing ImageContentSourcePolicy")
 		}
 
@@ -472,36 +519,39 @@ func generateCatalogSource(source imagesource.TypedImageReference, mapping map[i
 	return csExample, nil
 }
 
-func generateICSP(out io.Writer, name string, icspScope string, mapping map[imagesource.TypedImageReference]imagesource.TypedImageReference) ([]byte, error) {
+func generateICSP(out io.Writer, name string, byteLimit int, registryMapping map[string]string) ([]byte, error) {
 	icsp := operatorv1alpha1.ImageContentSourcePolicy{
 		TypeMeta: metav1.TypeMeta{
 			APIVersion: operatorv1alpha1.GroupVersion.String(),
 			Kind:       "ImageContentSourcePolicy"},
 		ObjectMeta: metav1.ObjectMeta{
 			Name: strings.Join(strings.Split(name, "/"), "-"),
+			Labels: map[string]string{
+				"operators.openshift.org/catalog": "true",
+			},
 		},
 		Spec: operatorv1alpha1.ImageContentSourcePolicySpec{
 			RepositoryDigestMirrors: []operatorv1alpha1.RepositoryDigestMirrors{},
 		},
 	}
 
-	registryMapping := map[string]string{}
-	for k, v := range mapping {
-		if len(v.Ref.ID) == 0 {
-			fmt.Fprintf(out, "no digest mapping available for %s, skip writing to ImageContentSourcePolicy\n", k)
-			continue
-		}
-		if icspScope == "registry" {
-			registryMapping[k.Ref.Registry] = v.Ref.Registry
-		} else {
-			registryMapping[k.Ref.AsRepository().String()] = v.Ref.AsRepository().String()
-		}
-	}
 	for key := range registryMapping {
 		icsp.Spec.RepositoryDigestMirrors = append(icsp.Spec.RepositoryDigestMirrors, operatorv1alpha1.RepositoryDigestMirrors{
 			Source:  key,
 			Mirrors: []string{registryMapping[key]},
 		})
+		y, err := yaml.Marshal(icsp)
+		if err != nil {
+			return nil, fmt.Errorf("unable to marshal ImageContentSourcePolicy yaml: %v", err)
+		}
+
+		if len(y) > byteLimit {
+			if lenMirrors := len(icsp.Spec.RepositoryDigestMirrors); lenMirrors > 0 {
+				icsp.Spec.RepositoryDigestMirrors = icsp.Spec.RepositoryDigestMirrors[:lenMirrors-1]
+			}
+			break
+		}
+		delete(registryMapping, key)
 	}
 
 	// Create an unstructured object for removing creationTimestamp