TEMPORARY: use a different repository for source catalogs

Fix E2E test catalog_full

go.work

@@ -6,3 +6,5 @@ use (
 )
 
 replace github.com/docker/docker => github.com/docker/docker v20.10.21+incompatible
+replace github.com/containerd/stargz-snapshotter/estargz => github.com/containerd/stargz-snapshotter/estargz v0.12.0
+replace github.com/google/go-containerregistry => github.com/google/go-containerregistry v0.10.0

pkg/cli/mirror/catalog_images.go

@@ -35,6 +35,8 @@ const (
 	opmCachePrefix  = "/tmp/cache"
 	opmBinarySuffix = "opm"
 	opmBinaryDir    = "usr/bin/registry"
+	cacheFolderUID  = 1001
+	cacheFolderGID  = 0
 )
 
 // unpackCatalog will unpack file-based catalogs if they exists
@@ -219,10 +221,9 @@ func (o *MirrorOptions) processCatalogRefs(ctx context.Context, catalogsByImage
 			return fmt.Errorf("error creating deleted layer: %v", err)
 		}
 
-		//TODO how do you know which is the cache dir
-
-		//TODO white out layer /tmp
-		deletedCacheLayer, err := deleteLayer("/tmp/.wh.cache")
+		// white out layer /tmp
+		// TODO : any label in the manifests that might indicate more accurately where the cache is?
+		deletedCacheLayer, err := deleteLayer("/.wh.tmp")
 		if err != nil {
 			return fmt.Errorf("error creating deleted cache layer: %v", err)
 		}
@@ -246,8 +247,7 @@ func (o *MirrorOptions) processCatalogRefs(ctx context.Context, catalogsByImage
 			return fmt.Errorf("error regenerating the cache for %v: %v", ctlgRef, err)
 		}
 		//TODO check the tmp folder is not empty
-		//TODO create a new tmp layer from reconstructed cache
-		cacheLayerToAdd, err := builder.LayerFromPath("/tmp/cache", filepath.Join(artifactDir, config.TmpDir))
+		cacheLayerToAdd, err := builder.LayerFromPathWithUidGid("/tmp/cache", filepath.Join(artifactDir, config.TmpDir), cacheFolderUID, cacheFolderGID)
 		if err != nil {
 			return fmt.Errorf("error creating add layer: %v", err)
 		}
@@ -268,6 +268,10 @@ func (o *MirrorOptions) processCatalogRefs(ctx context.Context, catalogsByImage
 				containertools.ConfigsLocationLabel: "/configs",
 			}
 			cfg.Config.Labels = labels
+			// Although it was prefered to keep the entrypoint and command as it was
+			// we couldnt guarantie that the cache-dir was /tmp/cache, and therefore
+			// we had to specify the command for the newly built catalog
+			cfg.Config.Cmd = []string{"serve", "/configs", "--cache-dir=/tmp/cache"}
 		}
 		if err := imgBuilder.Run(ctx, refExact, layoutPath, update, layers...); err != nil {
 			return fmt.Errorf("error building catalog layers: %v", err)
@@ -277,7 +281,6 @@ func (o *MirrorOptions) processCatalogRefs(ctx context.Context, catalogsByImage
 }
 
 func findOpmCmd(artifactDir string) (string, error) {
-	//TODO guess the opmCmdPath
 	wd, err := os.Getwd()
 	if err != nil {
 		return "", fmt.Errorf("error finding current working directory while preparing to run opm to regenerate cache: %v", err)

pkg/cli/mirror/operator.go

@@ -185,8 +185,6 @@ func (o *OperatorOptions) run(
 			return nil, o.checkValidationErr(err)
 		}
 
-		//TODO CFE-825 extract opm from catalog
-		//TODO deal with case of ID in ctlgRef
 		ctlgSrcDir := filepath.Join(o.Dir, config.SourceDir, config.CatalogsDir, targetCtlg.Ref.Registry, targetCtlg.Ref.Namespace, targetCtlg.Ref.Name)
 		if targetCtlg.Ref.ID != "" {
 			ctlgSrcDir = filepath.Join(ctlgSrcDir, targetCtlg.Ref.ID, config.OpmBinDir)

pkg/image/builder/image_builder.go

@@ -342,6 +342,13 @@ func (b *ImageBuilder) CreateLayout(srcRef, dir string) (layout.Path, error) {
 // LayerFromPath will write the contents of the path(s) the target
 // directory and build a v1.Layer
 func LayerFromPath(targetPath, path string) (v1.Layer, error) {
+	return LayerFromPathWithUidGid(targetPath, path, -1, -1)
+}
+
+// LayerFromPath will write the contents of the path(s) the target
+// directory specifying the target UID/GID and build a v1.Layer.
+// Use gid = -1 , uid = -1 if you don't want to override.
+func LayerFromPathWithUidGid(targetPath, path string, uid int, gid int) (v1.Layer, error) {
 	var b bytes.Buffer
 	tw := tar.NewWriter(&b)
 
@@ -396,6 +403,12 @@ func LayerFromPath(targetPath, path string) (v1.Layer, error) {
 				Name: filepath.Join(targetPath, filepath.ToSlash(rel)),
 				Mode: int64(info.Mode()),
 			}
+			if uid != -1 {
+				hdr.Uid = uid
+			}
+			if gid != -1 {
+				hdr.Gid = gid
+			}
 			if err := processPaths(hdr, info, fp); err != nil {
 				return err
 			}
@@ -412,6 +425,12 @@ func LayerFromPath(targetPath, path string) (v1.Layer, error) {
 			Name: filepath.Join(targetPath, filepath.ToSlash(base)),
 			Mode: int64(pathInfo.Mode()),
 		}
+		if uid != -1 { // uid was specified in the input param
+			hdr.Uid = uid
+		}
+		if gid != -1 { // gid was specified in the input param
+			hdr.Gid = gid
+		}
 		if err := processPaths(hdr, pathInfo, path); err != nil {
 			return nil, err
 		}

test/e2e/e2e-simple.sh

@@ -22,8 +22,8 @@ MIRROR_OCI_DIR="${DATA_TMP}/mirror_oci"
 OCI_CTLG_PATH="oc-mirror-dev.tgz"
 WORKSPACE="oc-mirror-workspace"
 CATALOGREGISTRY="quay.io"
-CATALOGORG="redhatgov"
-CATALOGNAMESPACE="redhatgov/oc-mirror-dev"
+CATALOGORG="skhoury"
+CATALOGNAMESPACE="skhoury/test-catalog"
 REGISTRY_CONN_PORT=5000
 REGISTRY_DISCONN_PORT=5001
 METADATA_REGISTRY="localhost.localdomain:$REGISTRY_CONN_PORT"

test/e2e/lib/catalog-extract/main.go

@@ -0,0 +1,84 @@
+package main
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"time"
+
+	"github.com/google/go-containerregistry/pkg/authn"
+	"github.com/google/go-containerregistry/pkg/crane"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/mutate"
+)
+
+func createRT(insecure bool) http.RoundTripper {
+	return &http.Transport{
+		Proxy: http.ProxyFromEnvironment,
+		DialContext: (&net.Dialer{
+			// By default, we wrap the transport in retries, so reduce the
+			// default dial timeout to 5s to avoid 5x 30s of connection
+			// timeouts when doing the "ping" on certain http registries.
+			Timeout:   5 * time.Second,
+			KeepAlive: 30 * time.Second,
+		}).DialContext,
+		ForceAttemptHTTP2:     true,
+		MaxIdleConns:          100,
+		IdleConnTimeout:       90 * time.Second,
+		TLSHandshakeTimeout:   10 * time.Second,
+		ExpectContinueTimeout: 1 * time.Second,
+		TLSClientConfig: &tls.Config{
+			InsecureSkipVerify: insecure,
+			MinVersion:         tls.VersionTLS12,
+		},
+	}
+}
+
+func getCraneOpts(ctx context.Context, insecure bool) []crane.Option {
+	opts := []crane.Option{
+		crane.WithAuthFromKeychain(authn.DefaultKeychain),
+		crane.WithTransport(createRT(insecure)),
+		crane.WithContext(ctx),
+	}
+	if insecure {
+		opts = append(opts, crane.Insecure)
+	}
+	return opts
+}
+
+func toTar(img v1.Image, filename string) error {
+	tr := mutate.Extract(img)
+	outFile, err := os.Create(filename)
+	// handle err
+	if err != nil {
+		return err
+	}
+	defer outFile.Close()
+	_, err = io.Copy(outFile, tr)
+	return err
+}
+
+func main() {
+	argsWithoutProg := os.Args[1:]
+	var img v1.Image
+	var err error
+
+	remoteOpts := getCraneOpts(context.TODO(), true)
+	img, err = crane.Pull(argsWithoutProg[0], remoteOpts...)
+	if err != nil {
+		fmt.Printf("unable to pull image from %s: %v", argsWithoutProg[0], err)
+	}
+
+	// if we get here and no image was found bail out
+	if img == nil {
+		fmt.Printf("unable to obtain image for %v", argsWithoutProg[0])
+	}
+	err = toTar(img, argsWithoutProg[1])
+	if err != nil {
+		fmt.Printf("%v", err)
+	}
+}

test/e2e/lib/check.sh

@@ -8,21 +8,40 @@ function check_bundles() {
   local disconn_registry="${3:?disconnected registry host name must be set}"
   local ns="${4:-""}"
 
-  crane export --insecure $catalog_image temp.tar
+  # extracted directory is used to extract opm binary and cache from the catalog
+  local extraction_dir="${DATA_TMP}/extracted"
+  mkdir -p "$extraction_dir"
+
+  # unpacked directory is used to extract the declarative config from the catalog
   local index_dir="${DATA_TMP}/unpacked"
-  local tmp_cache_dir="${DATA_TMP}/unpacked_cache"
-  mkdir -p "$index_dir"
-  mkdir -p "$tmp_cache_dir"
   local index_path="${index_dir}/index.json"
-  tar xvf temp.tar /configs/index.json --strip-components=1 
-  tar xvf temp.tar /tmp/cache 
-  mv index.json $index_dir
-  mv /tmp/cache $tmp_cache_dir
+  mkdir -p "$index_dir"
+
+  # catalog-extract go code replaces crane extract, which was facing an issue
+  # it extracts the contents of the catalog image to a tar archive.
+  go run -mod=readonly test/e2e/lib/catalog-extract/main.go $catalog_image $extraction_dir/temp.tar
 
-  rm -f temp.tar
+  # extract declarative config from tar file
+  tar xvf $extraction_dir/temp.tar /configs/index.json --strip-components=1 
+  mv index.json $index_dir
+
+  # extract the cache from the tar file
+  local cache_dir="${extraction_dir}/tmp/cache"
+  tar xvf $extraction_dir/temp.tar /tmp
+  mv tmp "${extraction_dir}"
+
+  # extract the opm binary from the tar file
+  local opm_path="${extraction_dir}/opm"
+  tar xvf $extraction_dir/temp.tar bin/opm
+  mv bin/opm "${extraction_dir}"
+  chmod +x $opm_path
+
+
+  $opm_path validate $index_dir
 
-  opm validate $index_dir
-  opm serve $index_dir --cache-dir=$tmp_cache_dir --cache-only --cache-enforce-integrity
+  # validate cache integrity
+  $opm_path serve $index_dir --cache-dir=$cache_dir --cache-only --cache-enforce-integrity
+  rm -fr "$extraction_dir"
 
   declare -A exp_bundles_set
   for bundle in $exp_bundles_list; do