Skip to content

Commit

Permalink
OCPBUGS-31536: Change default behavior to not rebuild catalogs for V1
Browse files Browse the repository at this point in the history
  • Loading branch information
@sherine-k
sherine-k committed May 7, 2024
1 parent 403c00e commit 90d3ec6
Show file tree
Hide file tree
Showing 6 changed files with 213 additions and 103 deletions.
226 changes: 159 additions & 67 deletions pkg/cli/mirror/catalog_images.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"fmt"
"io"
"io/fs"
"math/rand"
"os"
"os/exec"
"path"
Expand All @@ -22,6 +23,7 @@ import (
"github.com/openshift/oc/pkg/cli/image/imagesource"
"github.com/operator-framework/operator-registry/pkg/containertools"
"github.com/operator-framework/operator-registry/pkg/image/containerdregistry"
"github.com/otiai10/copy"
"k8s.io/klog/v2"

"github.com/openshift/oc-mirror/pkg/api/v1alpha2"
Expand All @@ -35,6 +37,7 @@ const (
opmBinarySuffix = "opm"
cacheFolderUID = 1001
cacheFolderGID = 0
tempFolder = "/tmp/temp_folder"
)

type NoCacheArgsErrorType struct{}
Expand Down Expand Up @@ -88,80 +91,163 @@ func (o *MirrorOptions) rebuildCatalogs(ctx context.Context, dstDir string) (ima

dstDir = filepath.Clean(dstDir)
catalogsByImage := map[image.TypedImage]string{}
if err := filepath.Walk(dstDir, func(fpath string, info fs.FileInfo, err error) error {
if o.RebuildCatalogs {
if err := filepath.Walk(dstDir, func(fpath string, info fs.FileInfo, err error) error {

// Skip the layouts dir because we only need
// to process the parent directory one time
if filepath.Base(fpath) == config.LayoutsDir {
return filepath.SkipDir
}
// Skip the layouts dir because we only need
// to process the parent directory one time
if filepath.Base(fpath) == config.LayoutsDir {
return filepath.SkipDir
}

if err != nil || info == nil {
return err
if err != nil || info == nil {
return err
}

// From the index path determine the artifacts (index and layout) directory.
// Using that path to determine the corresponding catalog image for processing.
slashPath := filepath.ToSlash(fpath)
if base := path.Base(slashPath); base == "index.json" {
// remove the index.json from the path
// results in <some path>/src/catalogs/<repoPath>/index
slashPath = path.Dir(slashPath)
// remove the index folder from the path
// results in <some path>/src/catalogs/<repoPath>
slashPath = strings.TrimSuffix(slashPath, config.IndexDir)

// remove the <some path>/src/catalogs from the path to arrive at <repoPath>
repoPath := strings.TrimPrefix(slashPath, fmt.Sprintf("%s/%s/", dstDir, config.CatalogsDir))
// get the repo namespace and id (where ID is a SHA or tag)
// example: foo.com/foo/bar/<id>
regRepoNs, id := path.Split(path.Dir(repoPath))
regRepoNs = path.Clean(regRepoNs)
// reconstitute the path into a valid docker ref
var img string
if strings.Contains(id, ":") {
// Digest.
img = fmt.Sprintf("%s@%s", regRepoNs, id)
} else {
// Tag.
img = fmt.Sprintf("%s:%s", regRepoNs, id)
}
ctlgRef := image.TypedImage{}
ctlgRef.Type = imagesource.DestinationRegistry
sourceRef, err := image.ParseReference(img)
// since we can't really tell if the "img" reference originated from an actual docker
// reference or from an OCI file path that approximates a docker reference, ParseReference
// might not lowercase the name and namespace values which is required by the
// docker reference spec (see https://github.com/distribution/distribution/blob/main/reference/reference.go).
// Therefore we lower case name and namespace here to make sure it's done.
sourceRef.Ref.Name = strings.ToLower(sourceRef.Ref.Name)
sourceRef.Ref.Namespace = strings.ToLower(sourceRef.Ref.Namespace)

if err != nil {
return fmt.Errorf("error parsing index dir path %q as image %q: %v", fpath, img, err)
}
ctlgRef.Ref = sourceRef.Ref
// Update registry so the existing catalog image can be pulled.
ctlgRef.Ref.Registry = mirrorRef.Ref.Registry
ctlgRef.Ref.Namespace = path.Join(o.UserNamespace, ctlgRef.Ref.Namespace)
ctlgRef = ctlgRef.SetDefaults()
// Unset the ID when passing to the image builder.
// Tags are needed here since the digest will be recalculated.
ctlgRef.Ref.ID = ""

catalogsByImage[ctlgRef] = slashPath

// Add to mapping for ICSP generation
refs.Add(sourceRef, ctlgRef.TypedImageReference, v1alpha2.TypeOperatorCatalog)
}
return nil
}); err != nil {
return nil, err
}

// From the index path determine the artifacts (index and layout) directory.
// Using that path to determine the corresponding catalog image for processing.
slashPath := filepath.ToSlash(fpath)
if base := path.Base(slashPath); base == "index.json" {
// remove the index.json from the path
// results in <some path>/src/catalogs/<repoPath>/index
slashPath = path.Dir(slashPath)
// remove the index folder from the path
// results in <some path>/src/catalogs/<repoPath>
slashPath = strings.TrimSuffix(slashPath, config.IndexDir)

// remove the <some path>/src/catalogs from the path to arrive at <repoPath>
repoPath := strings.TrimPrefix(slashPath, fmt.Sprintf("%s/%s/", dstDir, config.CatalogsDir))
// get the repo namespace and id (where ID is a SHA or tag)
// example: foo.com/foo/bar/<id>
regRepoNs, id := path.Split(path.Dir(repoPath))
regRepoNs = path.Clean(regRepoNs)
// reconstitute the path into a valid docker ref
var img string
if strings.Contains(id, ":") {
// Digest.
img = fmt.Sprintf("%s@%s", regRepoNs, id)
} else {
// Tag.
img = fmt.Sprintf("%s:%s", regRepoNs, id)
// update the catalogs in the OCI layout directory and push them to their destination
if err := o.processCatalogRefs(ctx, catalogsByImage); err != nil {
return nil, err
}
} else {
if err := filepath.Walk(dstDir, func(fpath string, info fs.FileInfo, err error) error {
if err != nil || info == nil {
return err
}
ctlgRef := image.TypedImage{}
ctlgRef.Type = imagesource.DestinationRegistry
sourceRef, err := image.ParseReference(img)
// since we can't really tell if the "img" reference originated from an actual docker
// reference or from an OCI file path that approximates a docker reference, ParseReference
// might not lowercase the name and namespace values which is required by the
// docker reference spec (see https://github.com/distribution/distribution/blob/main/reference/reference.go).
// Therefore we lower case name and namespace here to make sure it's done.
sourceRef.Ref.Name = strings.ToLower(sourceRef.Ref.Name)
sourceRef.Ref.Namespace = strings.ToLower(sourceRef.Ref.Namespace)
slashPath := filepath.ToSlash(fpath)

if filepath.Base(fpath) == config.LayoutsDir {

// results in <some path>/src/catalogs/<repoPath>/layout
slashPath = path.Dir(slashPath)

// remove the <some path>/src/catalogs from the path to arrive at <repoPath>
repoPath := strings.TrimPrefix(slashPath, fmt.Sprintf("%s/%s/", dstDir, config.CatalogsDir))
// get the repo namespace and id (where ID is a SHA or tag)
// example: foo.com/foo/bar/<id>
regRepoNs, id := path.Split(repoPath)
regRepoNs = path.Clean(regRepoNs)
// reconstitute the path into a valid docker ref
var img string
if strings.Contains(id, ":") {
// Digest.
img = fmt.Sprintf("%s@%s", regRepoNs, id)
} else {
// Tag.
img = fmt.Sprintf("%s:%s", regRepoNs, id)
}

ctlgRef := image.TypedImage{}
ctlgRef.Type = imagesource.DestinationRegistry
originRef, err := image.ParseReference(img)
// since we can't really tell if the "img" reference originated from an actual docker
// reference or from an OCI file path that approximates a docker reference, ParseReference
// might not lowercase the name and namespace values which is required by the
// docker reference spec (see https://github.com/distribution/distribution/blob/main/reference/reference.go).
// Therefore we lower case name and namespace here to make sure it's done.
originRef.Ref.Name = strings.ToLower(originRef.Ref.Name)
originRef.Ref.Namespace = strings.ToLower(originRef.Ref.Namespace)

if err != nil {
return fmt.Errorf("error parsing index dir path %q as image %q: %v", fpath, img, err)
}
ctlgRef.Ref = originRef.Ref
// Update registry so the existing catalog image can be pulled.
ctlgRef.Ref.Registry = mirrorRef.Ref.Registry
ctlgRef.Ref.Namespace = path.Join(o.UserNamespace, ctlgRef.Ref.Namespace)
ctlgRef = ctlgRef.SetDefaults()
// Unset the ID when passing to the image builder.
// Tags are needed here since the digest will be recalculated.
ctlgRef.Ref.ID = ""

catalogOCIDir := fpath
// Add to mapping for ICSP generation
if strings.Contains(fpath, "sha256:") {
randomNumber := (string)(rand.Int())
catalogOCIDir = tempFolder + randomNumber
err := os.Mkdir(catalogOCIDir, 0755)
if err != nil {
klog.Warningf("unable to create temp folder facilitating mirroring of catalog image %s: %v", fpath, err)
}
defer os.RemoveAll(catalogOCIDir)
err = copy.Copy(fpath, catalogOCIDir)
if err != nil {
klog.Warningf("unable to copy catalog to temp folder while mirroring of catalog image %s: %v", fpath, err)
}
}
_, err = o.copyImage(ctx, "oci://"+catalogOCIDir, "docker://"+ctlgRef.Ref.String(), o.remoteRegFuncs)
if err != nil {
klog.Warningf("error copying image %s to %s: %v", "oci://"+fpath, "docker://"+ctlgRef.Ref.String(), err)
klog.Warning("trying to copy directly from docker source")
_, err = o.copyImage(ctx, "docker://"+originRef.Ref.String(), "docker://"+ctlgRef.Ref.String(), o.remoteRegFuncs)
if err != nil {
return fmt.Errorf("error copying image %s to %s: %v", "docker://"+originRef.Ref.String(), "docker://"+ctlgRef.Ref.String(), err)
}
}

if err != nil {
return fmt.Errorf("error parsing index dir path %q as image %q: %v", fpath, img, err)
}
ctlgRef.Ref = sourceRef.Ref
// Update registry so the existing catalog image can be pulled.
ctlgRef.Ref.Registry = mirrorRef.Ref.Registry
ctlgRef.Ref.Namespace = path.Join(o.UserNamespace, ctlgRef.Ref.Namespace)
ctlgRef = ctlgRef.SetDefaults()
// Unset the ID when passing to the image builder.
// Tags are needed here since the digest will be recalculated.
ctlgRef.Ref.ID = ""

catalogsByImage[ctlgRef] = slashPath

// Add to mapping for ICSP generation
refs.Add(sourceRef, ctlgRef.TypedImageReference, v1alpha2.TypeOperatorCatalog)
return nil
}); err != nil {
return nil, err
}
return nil
}); err != nil {
return nil, err
}

// update the catalogs in the OCI layout directory and push them to their destination
if err := o.processCatalogRefs(ctx, catalogsByImage); err != nil {
return nil, err
}

// use the resolver to obtain the digests of the newly pushed images
Expand Down Expand Up @@ -241,11 +327,17 @@ func (o *MirrorOptions) processCatalogRefs(ctx context.Context, catalogsByImage

if withCacheRegeneration {

opmCmdPath := filepath.Join(artifactDir, config.OpmBinDir, "opm")
opmCmdPath := ""
if opmBinary := os.Getenv("OPM_BINARY"); opmBinary != "" {
opmCmdPath = opmBinary
} else {
opmCmdPath = filepath.Join(artifactDir, config.OpmBinDir, "opm")
}
_, err = os.Stat(opmCmdPath)
if err != nil {
return fmt.Errorf("cannot find opm in the extracted catalog %v for %s on %s: %v", ctlgRef, runtime.GOOS, runtime.GOARCH, err)
}

absConfigPath, err := filepath.Abs(filepath.Join(artifactDir, config.IndexDir))
if err != nil {
return fmt.Errorf("error getting absolute path for catalog's index %v: %v", filepath.Join(artifactDir, config.IndexDir), err)
Expand Down
15 changes: 13 additions & 2 deletions pkg/cli/mirror/fbc_operators.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -476,13 +476,24 @@ func findFirstAvailableMirror(ctx context.Context, sysCtx *types.SystemContext,
func (o *MirrorOptions) copyImage(ctx context.Context, from, to string, funcs RemoteRegFuncs) (digest.Digest, error) {
if !strings.HasPrefix(from, "docker") {
// find absolute path if from is a relative path
fromPath := v1alpha2.TrimProtocol(from)
protocolAndPath := strings.Split(from, "://")
var protocol, fromPath string
if len(protocolAndPath) == 2 {
protocol = protocolAndPath[0]
fromPath = protocolAndPath[1]
} else {
protocol = "oci://"
fromPath = from
}

if !strings.HasPrefix(fromPath, "/") {
absolutePath, err := filepath.Abs(fromPath)
if err != nil {
return digest.Digest(""), fmt.Errorf("unable to get absolute path of oci image %s: %v", from, err)
}
from = "oci://" + absolutePath
from = protocol + "://" + absolutePath
} else {
from = protocol + "://" + fromPath
}
}

Expand Down
5 changes: 5 additions & 0 deletions pkg/cli/mirror/mirror.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ import (
"path/filepath"
"strings"

imagecopy "github.com/containers/image/v5/copy"
"github.com/containers/image/v5/signature"
"github.com/containers/image/v5/types"
"github.com/google/go-containerregistry/pkg/authn"
"github.com/google/go-containerregistry/pkg/name"
Expand Down Expand Up @@ -340,6 +342,9 @@ func (o *MirrorOptions) Run(cmd *cobra.Command, f kcmdutil.Factory) (err error)
func (o *MirrorOptions) mirrorImages(ctx context.Context, cleanup cleanupFunc) error {

o.remoteRegFuncs = RemoteRegFuncs{
copy: func(ctx context.Context, policyContext *signature.PolicyContext, destRef types.ImageReference, srcRef types.ImageReference, options *imagecopy.Options) (copiedManifest []byte, retErr error) {
return imagecopy.Image(ctx, policyContext, destRef, srcRef, options)
},
newImageSource: func(ctx context.Context, sys *types.SystemContext, imgRef types.ImageReference) (types.ImageSource, error) {
return imgRef.NewImageSource(ctx, sys)
},
Expand Down
2 changes: 2 additions & 0 deletions pkg/cli/mirror/options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ type MirrorOptions struct {
OCIRegistriesConfig string // Registries config file location (it works only with local oci catalogs)
OCIInsecureSignaturePolicy bool // If set, OCI catalog push will not try to push signatures
MaxNestedPaths int
RebuildCatalogs bool // If set, rebuilds catalogs based on filtered declarative config, and regenerates the cache of that catalog
// cancelCh is a channel listening for command cancellations
cancelCh <-chan struct{}
once sync.Once
Expand Down Expand Up @@ -76,6 +77,7 @@ func (o *MirrorOptions) BindFlags(fs *pflag.FlagSet) {
fs.BoolVar(&o.OCIInsecureSignaturePolicy, "oci-insecure-signature-policy", o.OCIInsecureSignaturePolicy, "If set, OCI catalog push will not try to push signatures")
fs.BoolVar(&o.SkipPruning, "skip-pruning", o.SkipPruning, "If set, will disable pruning globally")
fs.IntVar(&o.MaxNestedPaths, "max-nested-paths", 0, "Number of nested paths, for destination registries that limit nested paths")
fs.BoolVar(&o.RebuildCatalogs, "rebuild-catalogs", o.RebuildCatalogs, "If set (defaults to false), rebuilds catalogs based on filtered declarative config, and regenerates the cache of that catalog")
}

func (o *MirrorOptions) init() {
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/lib/workflow.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ function workflow_m2d2m_oci_catalog() {
if !$DIFF; then
cleanup_conn
fi
run_cmd --from "${CREATE_FULL_DIR}/mirror_seq1_000000.tar" "docker://${remote_image}"
run_cmd --from "${CREATE_FULL_DIR}/mirror_seq1_000000.tar" "docker://${remote_image}" $PUBLISH_FLAGS

popd
}
Expand Down
Loading

0 comments on commit 90d3ec6

Please sign in to comment.