WIP: [V2] CFE-971: create and mirror a graph image for ocp releases

CONTRIBUTING.md

@@ -8,6 +8,7 @@ Contributing to `oc-mirror`
     - [Requesting Enhancements](#requesting-enhancements)
     - [Your First Code Contribution](#your-first-code-contribution)
       - [Getting Started](#getting-started)
+      - [Debugging oc-mirror](#debugging-oc-mirror)
     - [Pull Requests](#pull-requests)
     - [Docs Contributions](#docs-contributions)
   - [Testing](#testing)
@@ -42,6 +43,17 @@ Please submit bug reports as GitHub Issues. When submitting bug reports, please
 #### Getting Started
 Please refer to the [developer docs](./docs/dev/getting-started.md) for information on getting started with developing on `oc-mirror`.
 
+#### Debugging oc-mirror
+Since end of November 2023, and PR [#721](https://github.com/openshift/oc-mirror/pull/721), oc-mirror's v2 implementation is relying on [Buildah](https://github.com/containers/buildah) in order to
+build Cincinnati graph images. 
+
+Since oc-mirror needs to run rootless, it is now re-executing itself in a separate user namespace where buildah can have root privileges.
+
+This modifies the debugging process: In order to debug you should launch dlv debugger for oc-mirror in a user namespace: 
+```
+podman unshare dlv debug cmd/oc-mirror/main.go -- --v2 -c cfe-969.yml --from file:///home/skhoury/release docker://localhost:5000/cfe971/ --dest-tls-verify=false
+```
+
 ### Pull Requests
 When submitting pull requests, please ensure the following:
 1. Include unit tests if applicable

cmd/oc-mirror/main.go

@@ -1,11 +1,21 @@
 package main
 
 import (
+	"github.com/containers/buildah"
+	"github.com/containers/storage/pkg/unshare"
 	"github.com/openshift/oc-mirror/pkg/cli/mirror"
 	kcmdutil "k8s.io/kubectl/pkg/cmd/util"
 )
 
 func main() {
+	// oc-mirror runs in rootless mode. It is therefore necessary to
+	// ensure that oc-mirror is re-executed in a user namespace where
+	// it has root privileges.
+	if buildah.InitReexec() {
+		return
+	}
+	unshare.MaybeReexecUsingUserNamespace(false)
+
 	rootCmd := mirror.NewMirrorCmd()
 	checkErr(rootCmd.Execute())
 }

go.mod

@@ -5,10 +5,10 @@ go 1.20
 require (
 	github.com/blang/semver/v4 v4.0.0
 	github.com/bshuster-repo/logrus-logstash-hook v1.0.2 // indirect
-	github.com/containerd/containerd v1.6.22
-	github.com/containers/image/v5 v5.26.0
+	github.com/containerd/containerd v1.7.6
+	github.com/containers/image/v5 v5.26.1
 	github.com/docker/cli v23.0.5+incompatible
-	github.com/docker/distribution v2.8.2+incompatible
+	github.com/docker/distribution v2.8.3+incompatible
 	github.com/go-git/go-git/v5 v5.4.2 // indirect
 	github.com/go-logr/logr v1.2.4
 	github.com/google/go-containerregistry v0.15.2
@@ -44,6 +44,8 @@ require (
 require golang.org/x/sys v0.13.0 // indirect
 
 require (
+	github.com/containers/buildah v1.30.0
+	github.com/containers/storage v1.48.0
 	github.com/mitchellh/hashstructure/v2 v2.0.2
 	github.com/openshift/oc-mirror/v2 v2.0.0-20230802085830-e81c913cc044
 	github.com/otiai10/copy v1.2.0
@@ -60,7 +62,7 @@ require (
 	github.com/Masterminds/semver/v3 v3.2.0 // indirect
 	github.com/Masterminds/sprig/v3 v3.2.3 // indirect
 	github.com/Microsoft/go-winio v0.6.1 // indirect
-	github.com/Microsoft/hcsshim v0.9.9 // indirect
+	github.com/Microsoft/hcsshim v0.11.0 // indirect
 	github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d // indirect
 	github.com/VividCortex/ewma v1.2.0 // indirect
 	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
@@ -78,21 +80,26 @@ require (
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/chai2010/gettext-go v1.0.2 // indirect
-	github.com/containerd/cgroups v1.0.4 // indirect
-	github.com/containerd/continuity v0.3.0 // indirect
+	github.com/chzyer/readline v1.5.1 // indirect
+	github.com/container-orchestrated-devices/container-device-interface v0.5.4 // indirect
+	github.com/containerd/cgroups v1.1.0 // indirect
+	github.com/containerd/continuity v0.4.2 // indirect
 	github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
-	github.com/containerd/ttrpc v1.1.2 // indirect
-	github.com/containers/common v0.51.0 // indirect
+	github.com/containerd/ttrpc v1.2.2 // indirect
+	github.com/containernetworking/cni v1.1.2 // indirect
+	github.com/containernetworking/plugins v1.3.0 // indirect
+	github.com/containers/common v0.55.1 // indirect
 	github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 // indirect
 	github.com/containers/ocicrypt v1.1.7 // indirect
-	github.com/containers/storage v1.47.0 // indirect
 	github.com/cyberphone/json-canonicalization v0.0.0-20230514072755-504adb8a8af1 // indirect
 	github.com/cyphar/filepath-securejoin v0.2.3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/disiqueira/gotree/v3 v3.0.2 // indirect
 	github.com/distribution/distribution/v3 v3.0.0-20230722181636-7b502560cad4 // indirect
-	github.com/docker/docker v24.0.2+incompatible // indirect
+	github.com/distribution/reference v0.5.0 // indirect
+	github.com/docker/docker v24.0.6+incompatible // indirect
 	github.com/docker/docker-credential-helpers v0.7.0 // indirect
-	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-connections v0.4.1-0.20210727194412-58542c764a11 // indirect
 	github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
 	github.com/docker/go-metrics v0.0.1 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
@@ -102,6 +109,8 @@ require (
 	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
 	github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
 	github.com/felixge/httpsnoop v1.0.3 // indirect
+	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/fsouza/go-dockerclient v1.9.7 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-errors/errors v1.4.2 // indirect
 	github.com/go-git/gcfg v1.5.0 // indirect
@@ -142,9 +151,10 @@ require (
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/huandu/xstrings v1.4.0 // indirect
-	github.com/imdario/mergo v0.3.13 // indirect
+	github.com/imdario/mergo v0.3.15 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
+	github.com/jinzhu/copier v0.3.5 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
@@ -154,6 +164,7 @@ require (
 	github.com/letsencrypt/boulder v0.0.0-20230213213521-fdfea0d469b6 // indirect
 	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/manifoldco/promptui v0.9.0 // indirect
 	github.com/mattn/go-runewidth v0.0.14 // indirect
 	github.com/mattn/go-shellwords v1.0.12 // indirect
 	github.com/mattn/go-sqlite3 v1.14.16 // indirect
@@ -166,9 +177,9 @@ require (
 	github.com/mitchellh/go-wordwrap v1.0.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
-	github.com/moby/locker v1.0.1 // indirect
+	github.com/moby/patternmatcher v0.5.0 // indirect
 	github.com/moby/spdystream v0.2.0 // indirect
-	github.com/moby/sys/mount v0.2.0 // indirect
+	github.com/moby/sys/mount v0.3.3 // indirect
 	github.com/moby/sys/mountinfo v0.6.2 // indirect
 	github.com/moby/term v0.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
@@ -178,12 +189,14 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/nwaples/rardecode v1.1.0 // indirect
 	github.com/oklog/ulid v1.3.1 // indirect
-	github.com/onsi/gomega v1.27.7 // indirect
+	github.com/onsi/gomega v1.27.8 // indirect
 	github.com/opencontainers/runc v1.1.7 // indirect
 	github.com/opencontainers/runtime-spec v1.1.0-rc.3 // indirect
+	github.com/opencontainers/runtime-tools v0.9.1-0.20230317050512-e931285f4b69 // indirect
 	github.com/opencontainers/selinux v1.11.0 // indirect
 	github.com/openshift/client-go v0.0.0-20230120202327-72f107311084 // indirect
 	github.com/openshift/custom-resource-status v1.1.0 // indirect
+	github.com/openshift/imagebuilder v1.2.5 // indirect
 	github.com/operator-framework/api v0.17.7 // indirect
 	github.com/ostreedev/ostree-go v0.0.0-20210805093236-719684c64e4f // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
@@ -197,6 +210,7 @@ require (
 	github.com/prometheus/procfs v0.9.0 // indirect
 	github.com/rivo/uniseg v0.4.4 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/seccomp/libseccomp-golang v0.10.0 // indirect
 	github.com/shopspring/decimal v1.3.1 // indirect
 	github.com/sigstore/fulcio v1.3.1 // indirect
 	github.com/sigstore/rekor v1.2.2-0.20230601122533-4c81ff246d12 // indirect
@@ -212,6 +226,8 @@ require (
 	github.com/ulikunitz/xz v0.5.11 // indirect
 	github.com/vbatts/tar-split v0.11.3 // indirect
 	github.com/vbauerster/mpb/v8 v8.4.0 // indirect
+	github.com/vishvananda/netlink v1.2.1-beta.2 // indirect
+	github.com/vishvananda/netns v0.0.4 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
@@ -229,7 +245,7 @@ require (
 	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.15.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.15.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.15.0 // indirect
-	go.opentelemetry.io/otel/metric v0.31.0 // indirect
+	go.opentelemetry.io/otel/metric v0.37.0 // indirect
 	go.opentelemetry.io/otel/sdk v1.15.0 // indirect
 	go.opentelemetry.io/otel/trace v1.15.0 // indirect
 	go.opentelemetry.io/proto/otlp v0.19.0 // indirect
@@ -240,7 +256,7 @@ require (
 	golang.org/x/term v0.13.0 // indirect
 	golang.org/x/text v0.13.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
-	golang.org/x/tools v0.9.1 // indirect
+	golang.org/x/tools v0.9.3 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20230525154841-bd750badd5c6 // indirect
 	google.golang.org/grpc v1.55.0 // indirect
@@ -276,4 +292,11 @@ replace github.com/openshift/oc-mirror/v2/pkg/cli => ./v2/pkg/cli
 
 replace github.com/google/go-containerregistry => github.com/google/go-containerregistry v0.10.0
 
-// replace github.com/vbatts/tar-split =>	github.com/vbatts/tar-split v0.11.2
+// CFE-971 - introducing use of buildah in v2
+replace go.opentelemetry.io/otel => go.opentelemetry.io/otel v1.15.0
+
+replace go.opentelemetry.io/otel/metric => go.opentelemetry.io/otel/metric v0.31.0
+
+replace github.com/containerd/containerd => github.com/containerd/containerd v1.4.4
+
+// End CFE-971