Skip to content

Commit

Permalink
TEMPORARY: use a different repository for source catalogs
Browse files Browse the repository at this point in the history
Fix E2E test catalog_full
  • Loading branch information
sherine-k committed Aug 7, 2023
1 parent 67bdd15 commit 0009a06
Show file tree
Hide file tree
Showing 210 changed files with 4,377 additions and 8,059 deletions.
7 changes: 7 additions & 0 deletions .gitignore
Expand Up @@ -10,6 +10,13 @@ sha256sum.txt
**/test-output
**/.metadata.json
oc-mirror-workspace/
v2/working-dir/
v2/tests/hold-test-untar
v2/build/
v2/cmd/mirror/logs/
v2/cmd/mirror/working-dir/
v2/launch.json
v2/logs/
pkg/image/testdata/v2/single_manifest/manifests/oc-mirror
olm_artifacts/

Expand Down
6 changes: 5 additions & 1 deletion go.mod
Expand Up @@ -254,4 +254,8 @@ replace github.com/apcera/gssapi => github.com/openshift/gssapi v0.0.0-201610102

replace github.com/docker/docker => github.com/docker/docker v20.10.21+incompatible

replace github.com/openshift/oc-mirror/v2 v2.0.0-20230802085830-e81c913cc044 => ./v2
replace github.com/openshift/oc-mirror/v2 => ./v2

replace github.com/containerd/stargz-snapshotter/estargz => github.com/containerd/stargz-snapshotter/estargz v0.12.0

replace github.com/google/go-containerregistry => github.com/google/go-containerregistry v0.10.0
515 changes: 493 additions & 22 deletions go.sum

Large diffs are not rendered by default.

48 changes: 28 additions & 20 deletions pkg/cli/mirror/catalog_images.go
Expand Up @@ -35,6 +35,8 @@ const (
opmCachePrefix = "/tmp/cache"
opmBinarySuffix = "opm"
opmBinaryDir = "usr/bin/registry"
cacheFolderUID = 1001
cacheFolderGID = 0
)

// unpackCatalog will unpack file-based catalogs if they exists
Expand Down Expand Up @@ -219,10 +221,8 @@ func (o *MirrorOptions) processCatalogRefs(ctx context.Context, catalogsByImage
return fmt.Errorf("error creating deleted layer: %v", err)
}

//TODO how do you know which is the cache dir

//TODO white out layer /tmp
deletedCacheLayer, err := deleteLayer("/tmp/.wh.cache")
// white out layer /tmp
deletedCacheLayer, err := deleteLayer("/.wh.tmp")
if err != nil {
return fmt.Errorf("error creating deleted cache layer: %v", err)
}
Expand All @@ -239,15 +239,11 @@ func (o *MirrorOptions) processCatalogRefs(ctx context.Context, catalogsByImage
if err != nil {
return fmt.Errorf("error getting absolute path for catalog's cache %v: %v", filepath.Join(artifactDir, config.TmpDir), err)
}
//TODO call opm serve /configs –-cache-dir /tmp/cache –-cache-only
cmd := exec.Command(opmCmdPath, "serve", absConfigPath, "--cache-dir", absCachePath, "--cache-only")
fmt.Printf("%s\n", cmd.String())
if err := cmd.Run(); err != nil {
return fmt.Errorf("error regenerating the cache for %v: %v", ctlgRef, err)
}
//TODO check the tmp folder is not empty
//TODO create a new tmp layer from reconstructed cache
cacheLayerToAdd, err := builder.LayerFromPath("/tmp/cache", filepath.Join(artifactDir, config.TmpDir))
cacheLayerToAdd, err := builder.LayerFromPathWithUidGid("/tmp/cache", filepath.Join(artifactDir, config.TmpDir), cacheFolderUID, cacheFolderGID)
if err != nil {
return fmt.Errorf("error creating add layer: %v", err)
}
Expand All @@ -268,6 +264,10 @@ func (o *MirrorOptions) processCatalogRefs(ctx context.Context, catalogsByImage
containertools.ConfigsLocationLabel: "/configs",
}
cfg.Config.Labels = labels
// Although it was prefered to keep the entrypoint and command as it was
// we couldnt guarantie that the cache-dir was /tmp/cache, and therefore
// we had to specify the command for the newly built catalog
cfg.Config.Cmd = []string{"serve", "/configs", "--cache-dir=/tmp/cache"}
}
if err := imgBuilder.Run(ctx, refExact, layoutPath, update, layers...); err != nil {
return fmt.Errorf("error building catalog layers: %v", err)
Expand All @@ -276,8 +276,10 @@ func (o *MirrorOptions) processCatalogRefs(ctx context.Context, catalogsByImage
return nil
}

// findOpmCmd attempts to find the opm binary within the extracted contents of the catalog container image.
// The exact location of the opm binary within the image is not certain: It depends among other things on
// the version of the catalog, on the decisions by maintainers of the catalog, on the platform...
func findOpmCmd(artifactDir string) (string, error) {
//TODO guess the opmCmdPath
wd, err := os.Getwd()
if err != nil {
return "", fmt.Errorf("error finding current working directory while preparing to run opm to regenerate cache: %v", err)
Expand All @@ -293,7 +295,6 @@ func findOpmCmd(artifactDir string) (string, error) {
binaryDir := filepath.Join(wd, artifactDir, config.OpmBinDir)
err = filepath.Walk(binaryDir, func(path string, info fs.FileInfo, err error) error {
if err != nil {
fmt.Printf("prevent panic by handling failure accessing a path %q: %v\n", path, err)
return err
}

Expand All @@ -317,6 +318,11 @@ func findOpmCmd(artifactDir string) (string, error) {
return opmCmdPath, nil
}

// extractOPMBinary is usually called after rendering catalog's declarative config.
// it uses crane modules to pull the catalog image, select the manifest that corresponds to the
// current platform. It then extracts from that image any files that are suffixed `*opm` for later
// use upon rebuilding the catalog: This is because the opm binary can be called `opm` but also
// `darwin-amd64-opm` etc.
func extractOPMBinary(ctx context.Context, srcRef image.TypedImageReference, outDir string, insecure bool) error {
var img v1.Image
var err error
Expand All @@ -342,7 +348,7 @@ func extractOPMBinary(ctx context.Context, srcRef image.TypedImageReference, out
return err
}

// attempt to find the first image reference in the layout...
// attempt to find the first image reference in the layout that corresponds to the platform...
// for a manifest list only search one level deep.

loop:
Expand All @@ -359,8 +365,8 @@ func extractOPMBinary(ctx context.Context, srcRef image.TypedImageReference, out
return err
}

// at this point, find the first image and store it for later if possible
//TODO extract the child index that corresponds to this machine's architecture
// at this point, find and extract the child index that corresponds to this machine's
// architecture
for _, childDescriptor := range childIndexManifest.Manifests {
if childDescriptor.MediaType.IsImage() && childDescriptor.Platform.Architecture == runtime.GOARCH && childDescriptor.Platform.OS == runtime.GOOS {
img, err = childIndex.Image(childDescriptor.Digest)
Expand Down Expand Up @@ -397,7 +403,7 @@ func extractOPMBinary(ctx context.Context, srcRef image.TypedImageReference, out
break
}

// skip the file if it is a directory or not in the bin dir
// skip the file if it is a directory or if file name does not end with `opm`
if !strings.HasSuffix(header.Name, opmBinarySuffix) || header.FileInfo().IsDir() {
continue
}
Expand Down Expand Up @@ -429,12 +435,14 @@ func extractOPMBinary(ctx context.Context, srcRef image.TypedImageReference, out
return err
}
opmBinaryExtracted = targetFileName

// check for the extracted opm file (it should exist if we found something)
_, err = os.Stat(opmBinaryExtracted)
if errors.Is(err, os.ErrNotExist) {
return fmt.Errorf("opm binary not found after extracting %q from catalog image %v", opmBinaryExtracted, srcRef)
}
}
// check for the folder (it should exist if we found something)
_, err = os.Stat(opmBinaryExtracted)
if errors.Is(err, os.ErrNotExist) {
return fmt.Errorf("opm binary not found after extracting %q from catalog image %v", opmBinaryExtracted, srcRef)
}

return nil
}

Expand Down
2 changes: 0 additions & 2 deletions pkg/cli/mirror/operator.go
Expand Up @@ -185,8 +185,6 @@ func (o *OperatorOptions) run(
return nil, o.checkValidationErr(err)
}

//TODO CFE-825 extract opm from catalog
//TODO deal with case of ID in ctlgRef
ctlgSrcDir := filepath.Join(o.Dir, config.SourceDir, config.CatalogsDir, targetCtlg.Ref.Registry, targetCtlg.Ref.Namespace, targetCtlg.Ref.Name)
if targetCtlg.Ref.ID != "" {
ctlgSrcDir = filepath.Join(ctlgSrcDir, targetCtlg.Ref.ID, config.OpmBinDir)
Expand Down
19 changes: 19 additions & 0 deletions pkg/image/builder/image_builder.go
Expand Up @@ -342,6 +342,13 @@ func (b *ImageBuilder) CreateLayout(srcRef, dir string) (layout.Path, error) {
// LayerFromPath will write the contents of the path(s) the target
// directory and build a v1.Layer
func LayerFromPath(targetPath, path string) (v1.Layer, error) {
return LayerFromPathWithUidGid(targetPath, path, -1, -1)
}

// LayerFromPath will write the contents of the path(s) the target
// directory specifying the target UID/GID and build a v1.Layer.
// Use gid = -1 , uid = -1 if you don't want to override.
func LayerFromPathWithUidGid(targetPath, path string, uid int, gid int) (v1.Layer, error) {
var b bytes.Buffer
tw := tar.NewWriter(&b)

Expand Down Expand Up @@ -396,6 +403,12 @@ func LayerFromPath(targetPath, path string) (v1.Layer, error) {
Name: filepath.Join(targetPath, filepath.ToSlash(rel)),
Mode: int64(info.Mode()),
}
if uid != -1 {
hdr.Uid = uid
}
if gid != -1 {
hdr.Gid = gid
}
if err := processPaths(hdr, info, fp); err != nil {
return err
}
Expand All @@ -412,6 +425,12 @@ func LayerFromPath(targetPath, path string) (v1.Layer, error) {
Name: filepath.Join(targetPath, filepath.ToSlash(base)),
Mode: int64(pathInfo.Mode()),
}
if uid != -1 { // uid was specified in the input param
hdr.Uid = uid
}
if gid != -1 { // gid was specified in the input param
hdr.Gid = gid
}
if err := processPaths(hdr, pathInfo, path); err != nil {
return nil, err
}
Expand Down
Binary file modified test/e2e/artifacts/oc-mirror-dev.tgz
Binary file not shown.
4 changes: 2 additions & 2 deletions test/e2e/e2e-simple.sh
Expand Up @@ -22,8 +22,8 @@ MIRROR_OCI_DIR="${DATA_TMP}/mirror_oci"
OCI_CTLG_PATH="oc-mirror-dev.tgz"
WORKSPACE="oc-mirror-workspace"
CATALOGREGISTRY="quay.io"
CATALOGORG="redhatgov"
CATALOGNAMESPACE="redhatgov/oc-mirror-dev"
CATALOGORG="skhoury"
CATALOGNAMESPACE="skhoury/test-catalog"
REGISTRY_CONN_PORT=5000
REGISTRY_DISCONN_PORT=5001
METADATA_REGISTRY="localhost.localdomain:$REGISTRY_CONN_PORT"
Expand Down
84 changes: 84 additions & 0 deletions test/e2e/lib/catalog-extract/main.go
@@ -0,0 +1,84 @@
package main

import (
"context"
"crypto/tls"
"fmt"
"io"
"net"
"net/http"
"os"
"time"

"github.com/google/go-containerregistry/pkg/authn"
"github.com/google/go-containerregistry/pkg/crane"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/mutate"
)

func createRT(insecure bool) http.RoundTripper {
return &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
// By default, we wrap the transport in retries, so reduce the
// default dial timeout to 5s to avoid 5x 30s of connection
// timeouts when doing the "ping" on certain http registries.
Timeout: 5 * time.Second,
KeepAlive: 30 * time.Second,
}).DialContext,
ForceAttemptHTTP2: true,
MaxIdleConns: 100,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: insecure,
MinVersion: tls.VersionTLS12,
},
}
}

func getCraneOpts(ctx context.Context, insecure bool) []crane.Option {
opts := []crane.Option{
crane.WithAuthFromKeychain(authn.DefaultKeychain),
crane.WithTransport(createRT(insecure)),
crane.WithContext(ctx),
}
if insecure {
opts = append(opts, crane.Insecure)
}
return opts
}

func toTar(img v1.Image, filename string) error {
tr := mutate.Extract(img)
outFile, err := os.Create(filename)
// handle err
if err != nil {
return err
}
defer outFile.Close()
_, err = io.Copy(outFile, tr)
return err
}

func main() {
argsWithoutProg := os.Args[1:]
var img v1.Image
var err error

remoteOpts := getCraneOpts(context.TODO(), true)
img, err = crane.Pull(argsWithoutProg[0], remoteOpts...)
if err != nil {
fmt.Printf("unable to pull image from %s: %v", argsWithoutProg[0], err)
}

// if we get here and no image was found bail out
if img == nil {
fmt.Printf("unable to obtain image for %v", argsWithoutProg[0])
}
err = toTar(img, argsWithoutProg[1])
if err != nil {
fmt.Printf("%v", err)
}
}
41 changes: 30 additions & 11 deletions test/e2e/lib/check.sh
Expand Up @@ -8,21 +8,40 @@ function check_bundles() {
local disconn_registry="${3:?disconnected registry host name must be set}"
local ns="${4:-""}"

crane export --insecure $catalog_image temp.tar
# extracted directory is used to extract opm binary and cache from the catalog
local extraction_dir="${DATA_TMP}/extracted"
mkdir -p "$extraction_dir"

# unpacked directory is used to extract the declarative config from the catalog
local index_dir="${DATA_TMP}/unpacked"
local tmp_cache_dir="${DATA_TMP}/unpacked_cache"
mkdir -p "$index_dir"
mkdir -p "$tmp_cache_dir"
local index_path="${index_dir}/index.json"
tar xvf temp.tar /configs/index.json --strip-components=1
tar xvf temp.tar /tmp/cache
mv index.json $index_dir
mv /tmp/cache $tmp_cache_dir
mkdir -p "$index_dir"

# catalog-extract go code replaces crane extract, which was facing an issue
# it extracts the contents of the catalog image to a tar archive.
go run -mod=readonly test/e2e/lib/catalog-extract/main.go $catalog_image $extraction_dir/temp.tar

rm -f temp.tar
# extract declarative config from tar file
tar xvf $extraction_dir/temp.tar /configs/index.json --strip-components=1
mv index.json $index_dir

# extract the cache from the tar file
local cache_dir="${extraction_dir}/tmp/cache"
tar xvf $extraction_dir/temp.tar /tmp
mv tmp "${extraction_dir}"

# extract the opm binary from the tar file
local opm_path="${extraction_dir}/opm"
tar xvf $extraction_dir/temp.tar bin/opm
mv bin/opm "${extraction_dir}"
chmod +x $opm_path


$opm_path validate $index_dir

opm validate $index_dir
opm serve $index_dir --cache-dir=$tmp_cache_dir --cache-only --cache-enforce-integrity
# validate cache integrity
$opm_path serve $index_dir --cache-dir=$cache_dir --cache-only --cache-enforce-integrity
rm -fr "$extraction_dir"

declare -A exp_bundles_set
for bundle in $exp_bundles_list; do
Expand Down

0 comments on commit 0009a06

Please sign in to comment.