Skip to content

Commit

Permalink
make the number of path components configuration for catalog mirror
Browse files Browse the repository at this point in the history
defaults to 2, which is what quay.io requires
  • Loading branch information
ecordell committed Apr 10, 2020
1 parent 041fa6a commit 649a5d4
Show file tree
Hide file tree
Showing 6 changed files with 213 additions and 43 deletions.
3 changes: 3 additions & 0 deletions contrib/completions/bash/oc
Original file line number Diff line number Diff line change
Expand Up @@ -604,6 +604,9 @@ _oc_adm_catalog_mirror()
local_nonpersistent_flags+=("--insecure")
flags+=("--manifests-only")
local_nonpersistent_flags+=("--manifests-only")
flags+=("--max-components=")
two_word_flags+=("--max-components")
local_nonpersistent_flags+=("--max-components=")
flags+=("--max-per-registry=")
two_word_flags+=("--max-per-registry")
local_nonpersistent_flags+=("--max-per-registry=")
Expand Down
3 changes: 3 additions & 0 deletions contrib/completions/zsh/oc
Original file line number Diff line number Diff line change
Expand Up @@ -746,6 +746,9 @@ _oc_adm_catalog_mirror()
local_nonpersistent_flags+=("--insecure")
flags+=("--manifests-only")
local_nonpersistent_flags+=("--manifests-only")
flags+=("--max-components=")
two_word_flags+=("--max-components")
local_nonpersistent_flags+=("--max-components=")
flags+=("--max-per-registry=")
two_word_flags+=("--max-per-registry")
local_nonpersistent_flags+=("--max-per-registry=")
Expand Down
61 changes: 32 additions & 29 deletions pkg/cli/admin/catalog/mirror.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (
"io"
"io/ioutil"
"os"
"path"
"path/filepath"
"strings"

Expand Down Expand Up @@ -46,6 +45,9 @@ var (
# Mirror an operator-registry image and its contents to a registry
%[1]s quay.io/my/image:latest myregistry.com
# Mirror an operator-registry image and its contents to a particular namespace in a registry
%[1]s quay.io/my/image:latest myregistry.com/my-namespace
# Configure a cluster to use a mirrored registry
oc apply -f manifests/imageContentSourcePolicy.yaml
Expand Down Expand Up @@ -112,6 +114,7 @@ func NewMirrorCatalog(streams genericclioptions.IOStreams) *cobra.Command {
flags.BoolVar(&o.ManifestOnly, "manifests-only", o.ManifestOnly, "Calculate the manifests required for mirroring, but do not actually mirror image content.")
flags.StringVar(&o.FileDir, "dir", o.FileDir, "The directory on disk that file:// images will be copied under.")
flags.StringVar(&o.FromFileDir, "from-dir", o.FromFileDir, "The directory on disk that file:// images will be read from. Overrides --dir")
flags.IntVar(&o.MaxPathComponents, "max-components", 2, "The maximum number of path components allowed in a destination mapping. Example: `quay.io/org/repo` has two path components.")
return cmd
}

Expand Down Expand Up @@ -152,7 +155,7 @@ func (o *MirrorCatalogOptions) Complete(cmd *cobra.Command, args []string) error
if len(dir) < 2 {
return fmt.Errorf("invalid path")
}
if err := os.MkdirAll(path.Dir(dir[1]), os.ModePerm); err != nil {
if err := os.MkdirAll(filepath.Dir(dir[1]), os.ModePerm); err != nil {
return err
}
}
Expand Down Expand Up @@ -257,6 +260,33 @@ func (o *MirrorCatalogOptions) Run() error {
return WriteManifests(o.SourceRef.Ref.Name, o.ManifestDir, mapping)
}

func WriteManifests(name, dir string, mapping map[string]Target) error {
f, err := os.Create(filepath.Join(dir, "mapping.txt"))
if err != nil {
return err
}
defer func() {
if err := f.Close(); err != nil {
klog.Warningf("error closing file")
}
}()

if err := writeToMapping(f, mapping); err != nil {
return err
}

icsp, err := generateICSP(name, mapping)
if err != nil {
return err
}

if err := ioutil.WriteFile(filepath.Join(dir, "imageContentSourcePolicy.yaml"), icsp, os.ModePerm); err != nil {
return fmt.Errorf("error writing ImageContentSourcePolicy")
}
klog.Infof("wrote mirroring manifests to %s", dir)
return nil
}

func writeToMapping(w io.StringWriter, mapping map[string]Target) error {
for k, v := range mapping {
if _, err := w.WriteString(fmt.Sprintf("%s=%s\n", k, v.WithTag)); err != nil {
Expand Down Expand Up @@ -312,30 +342,3 @@ func generateICSP(name string, mapping map[string]Target) ([]byte, error) {

return icspExample, nil
}

func WriteManifests(name, dir string, mapping map[string]Target) error {
f, err := os.Create(filepath.Join(dir, "mapping.txt"))
if err != nil {
return err
}
defer func() {
if err := f.Close(); err != nil {
klog.Warningf("error closing file")
}
}()

if err := writeToMapping(f, mapping); err != nil {
return err
}

icsp, err := generateICSP(name, mapping)
if err != nil {
return err
}

if err := ioutil.WriteFile(filepath.Join(dir, "imageContentSourcePolicy.yaml"), icsp, os.ModePerm); err != nil {
return fmt.Errorf("error writing ImageContentSourcePolicy")
}
klog.Infof("wrote mirroring manifests to %s", dir)
return nil
}
20 changes: 15 additions & 5 deletions pkg/cli/admin/catalog/mirror_options.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,12 @@ type IndexImageMirrorerOptions struct {
ImageMirrorer ImageMirrorer
DatabaseExtractor DatabaseExtractor

Source, Dest imagesource.TypedImageReference
ManifestDir string
Source, Dest imagesource.TypedImageReference
ManifestDir string
MaxPathComponents int
}

func (o *IndexImageMirrorerOptions) Validate() error {
// TODO: better validation

if o.ImageMirrorer == nil {
return fmt.Errorf("can't mirror without a mirrorer configured")
}
Expand All @@ -35,7 +34,9 @@ func (o *IndexImageMirrorerOptions) Validate() error {
if o.ManifestDir == "" {
return fmt.Errorf("must have directory to write manifests to")
}

if o.MaxPathComponents < 0 {
return fmt.Errorf("max path components must be a positive integer, or 0 for no limit")
}
return nil
}

Expand Down Expand Up @@ -72,6 +73,9 @@ func (c *IndexImageMirrorerOptions) ToOption() ImageIndexMirrorOption {
if c.ManifestDir != "" {
o.ManifestDir = c.ManifestDir
}
if c.MaxPathComponents > 0 {
o.MaxPathComponents = c.MaxPathComponents
}
}
}

Expand Down Expand Up @@ -112,3 +116,9 @@ func WithManifestDir(d string) ImageIndexMirrorOption {
o.ManifestDir = d
}
}

func WithMaxPathComponents(i int) ImageIndexMirrorOption {
return func(o *IndexImageMirrorerOptions) {
o.MaxPathComponents = i
}
}
21 changes: 15 additions & 6 deletions pkg/cli/admin/catalog/mirrorer.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,8 @@ type IndexImageMirrorer struct {
DatabaseExtractor DatabaseExtractor

// options
Source, Dest imagesource.TypedImageReference
Source, Dest imagesource.TypedImageReference
MaxPathComponents int
}

var _ Mirrorer = &IndexImageMirrorer{}
Expand All @@ -69,6 +70,7 @@ func NewIndexImageMirror(options ...ImageIndexMirrorOption) (*IndexImageMirrorer
DatabaseExtractor: config.DatabaseExtractor,
Source: config.Source,
Dest: config.Dest,
MaxPathComponents: config.MaxPathComponents,
}, nil
}

Expand All @@ -84,7 +86,7 @@ func (b *IndexImageMirrorer) Mirror() (map[string]Target, error) {
}

var errs = make([]error, 0)
mapping, mapErrs := mappingForImages(images, b.Dest)
mapping, mapErrs := mappingForImages(images, b.Dest, b.MaxPathComponents)
if len(mapErrs) > 0 {
errs = append(errs, mapErrs...)
}
Expand Down Expand Up @@ -128,7 +130,7 @@ func imagesFromDb(file string) (map[string]struct{}, error) {
return images, nil
}

func mappingForImages(images map[string]struct{}, dest imagesource.TypedImageReference) (mapping map[string]Target, errs []error) {
func mappingForImages(images map[string]struct{}, dest imagesource.TypedImageReference, maxComponents int) (mapping map[string]Target, errs []error) {
domain := dest.Ref.Registry

// handle bare repository targets
Expand Down Expand Up @@ -161,9 +163,16 @@ func mappingForImages(images map[string]struct{}, dest imagesource.TypedImageRef
continue
}

// calculate a new path in the target registry, where only the first path component is allowed
// the rest of the path components are collapsed into a longer name
name := domain + "/" + components[0] + "/" + strings.Join(components[1:], "-")
// calculate a new path in the target registry, where only the first (max path components - 1) components are
// allowed, and the rest of the path components are collapsed into a single longer component
parts := []string{domain}
if maxComponents < 1 {
parts = append(parts, components...)
} else {
parts = append(parts, components[0:maxComponents-1]...)
parts = append(parts, strings.Join(components[maxComponents-1:], "-"))
}
name := strings.TrimSuffix(strings.Join(parts, "/"), "/")

var target Target
// if ref has a tag, generate a target with the same tag
Expand Down

0 comments on commit 649a5d4

Please sign in to comment.