Cirrus: Run checks directly on the host

In order to meet achievable deadlines converting from Travis to Cirrus
CI, one significant artifact was carried forward (instead of fixing):

Depending on a `--privileged` container to execute all/most automated
checks/tests.

Prior attempts to remove this aspect resulted in several test failures.
Fixing the problems was viewed as more time-consuming than simply
preserving this runtime environment.

Time has passed, and the code has since moved on.  This commit removes
the legacy need to execute most operations in a `--privileged`
container, instead running them directly on the host.  At the same time,
the necessary test binaries are obtained from the same container used
for development purposes.  This ensures the two experiences are
virtually always identical.

Signed-off-by: Chris Evich <cevich@redhat.com>

.cirrus.yml

@@ -41,8 +41,9 @@ env:
     UBUNTU_CONTAINER_FQIN: "quay.io/libpod/ubuntu_podman:${IMAGE_SUFFIX}"
     PRIOR_UBUNTU_CONTAINER_FQIN: "quay.io/libpod/prior-ubuntu_podman:${IMAGE_SUFFIX}"
 
-    # Equivilent to image produced by 'make build-container'
-    SKOPEO_CI_CONTAINER_FQIN: "quay.io/skopeo/ci:${DEST_BRANCH}"
+    # Equivilent to image produced by 'make build-container'. Auto-built by quay.
+    # TODO:SKOPEO_CI_CONTAINER_FQIN: "quay.io/libpod/skopeo_cidev:${IMAGE_SUFFIX}"
+    SKOPEO_CI_CONTAINER_FQIN: "quay.io/libpod/skopeo_cidev:c5636194336768000"
 
 
 # Default timeout for each task
@@ -57,8 +58,8 @@ validate_task:
     # under Cirrus-CI, due to challenges obtaining the starting commit ID.
     # Only do validation for PRs.
     only_if: $CIRRUS_PR != ''
-    container:  &build_container
-        image: "${SKOPEO_CI_CONTAINER_FQIN}"
+    container:
+        image: '${SKOPEO_CI_CONTAINER_FQIN}'
         cpu: 4
         memory: 8
     script: |
@@ -140,8 +141,6 @@ test_skopeo_task:
         "${SKOPEO_PATH}/${SCRIPT_BASE}/runner.sh" vendor
     build_script: >-
         "${SKOPEO_PATH}/${SCRIPT_BASE}/runner.sh" build
-    validate_script: >-
-        "${SKOPEO_PATH}/${SCRIPT_BASE}/runner.sh" validate
     unit_script: >-
         "${SKOPEO_PATH}/${SCRIPT_BASE}/runner.sh" unit
     integration_script: >-

Makefile

@@ -1,4 +1,4 @@
-.PHONY: all binary build-container docs docs-in-container build-local clean install install-binary install-completions shell test-integration .install.vndr vendor vendor-in-container
+.PHONY: all binary docs docs-in-container build-local clean install install-binary install-completions shell test-integration .install.vndr vendor vendor-in-container
 
 export GOPROXY=https://proxy.golang.org
 
@@ -54,9 +54,9 @@ ifeq ($(GOOS), linux)
 endif
 
 GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null)
-IMAGE := skopeo-dev$(if $(GIT_BRANCH),:$(GIT_BRANCH))
+IMAGE := quay.io/libpod/skopeo_cidev:c5636194336768000  # TODO: Lookup tag from .cirrus.yml
 # set env like gobuildtag?
-CONTAINER_CMD := ${CONTAINER_RUNTIME} run --rm -i -e TESTFLAGS="$(TESTFLAGS)" #$(CONTAINER_ENVS)
+CONTAINER_CMD := ${CONTAINER_RUNTIME} run --rm -i -e TESTFLAGS="$(TESTFLAGS)"
 # if this session isn't interactive, then we don't want to allocate a
 # TTY, which would fail, but if it is interactive, we do want to attach
 # so that the user can send e.g. ^C through.
@@ -105,10 +105,8 @@ help:
 	@echo " * 'shell' - Run the built image and attach to a shell"
 	@echo " * 'clean' - Clean artifacts"
 
-# Build a container image (skopeobuild) that has everything we need to build.
-# Then do the build and the output (skopeo) should appear in current dir
+# Do the build and the output (skopeo) should appear in current dir
 binary: cmd/skopeo
-	${CONTAINER_RUNTIME} build ${BUILD_ARGS} -f Dockerfile.build -t skopeobuildimage .
 	${CONTAINER_RUNTIME} run --rm --security-opt label=disable -v $$(pwd):/src/github.com/containers/skopeo \
 		skopeobuildimage make bin/skopeo $(if $(DEBUG),DEBUG=$(DEBUG)) BUILDTAGS='$(BUILDTAGS)'
 
@@ -136,9 +134,6 @@ bin/skopeo.%:
 	GOOS=$(word 2,$(subst ., ,$@)) GOARCH=$(word 3,$(subst ., ,$@)) $(GO) build $(MOD_VENDOR) ${SKOPEO_LDFLAGS} -tags "containers_image_openpgp $(BUILDTAGS)" -o $@ ./cmd/skopeo
 local-cross: bin/skopeo.darwin.amd64 bin/skopeo.linux.arm bin/skopeo.linux.arm64 bin/skopeo.windows.386.exe bin/skopeo.windows.amd64.exe
 
-build-container:
-	${CONTAINER_RUNTIME} build ${BUILD_ARGS} -t "$(IMAGE)" .
-
 $(MANPAGES): %: %.md
 	sed -e 's/\((skopeo.*\.md)\)//' -e 's/\[\(skopeo.*\)\]/\1/' $<  | $(GOMD2MAN) -in /dev/stdin -out $@
 
@@ -171,21 +166,21 @@ install-completions:
 	install -m 755 -d ${DESTDIR}${BASHCOMPLETIONSDIR}
 	install -m 644 completions/bash/skopeo ${DESTDIR}${BASHCOMPLETIONSDIR}/skopeo
 
-shell: build-container
+shell:
 	$(CONTAINER_RUN) bash
 
 check: validate test-unit test-integration test-system
 
 # The tests can run out of entropy and block in containers, so replace /dev/random.
-test-integration: build-container
+test-integration:
 	$(CONTAINER_RUN) bash -c 'rm -f /dev/random; ln -sf /dev/urandom /dev/random; SKOPEO_CONTAINER_TESTS=1 BUILDTAGS="$(BUILDTAGS)" $(MAKE) test-integration-local'
 
-# Intended for CI, shortcut 'build-container' since already running inside container.
+# Intended for CI, assumed to already be running in quay.io/libpod/skopeo_cidev container.
 test-integration-local:
 	hack/make.sh test-integration
 
 # complicated set of options needed to run podman-in-podman
-test-system: build-container
+test-system:
 	DTEMP=$(shell mktemp -d --tmpdir=/var/tmp podman-tmp.XXXXXX); \
 	$(CONTAINER_CMD) --privileged \
 	    -v $$DTEMP:/var/lib/containers:Z -v /run/systemd/journal/socket:/run/systemd/journal/socket \
@@ -195,15 +190,15 @@ test-system: build-container
 	$(RM) -rf $$DTEMP; \
 	exit $$rc
 
-# Intended for CI, shortcut 'build-container' since already running inside container.
+# Intended for CI, assumed to already be running in quay.io/libpod/skopeo_cidev container.
 test-system-local:
 	hack/make.sh test-system
 
-test-unit: build-container
+test-unit:
 	# Just call (make test unit-local) here instead of worrying about environment differences
 	$(CONTAINER_RUN) make test-unit-local BUILDTAGS='$(BUILDTAGS)'
 
-validate: build-container
+validate:
 	$(CONTAINER_RUN) make validate-local
 
 # This target is only intended for development, e.g. executing it from an IDE. Use (make test) for CI or pre-release testing.

contrib/cirrus/runner.sh

@@ -25,59 +25,69 @@ OS_REL_VER="${OS_RELEASE_ID}-${OS_RELEASE_VER}"
 
 export "PATH=$PATH:$GOPATH/bin"
 
-podmanmake() {
-    req_env_vars GOPATH SKOPEO_PATH SKOPEO_CI_CONTAINER_FQIN
-    warn "Accumulated technical-debt requires execution inside a --privileged container.  This is very likely hiding bugs!"
-    showrun podman run -it --rm --privileged \
-        -e GOPATH=$GOPATH \
-        -v $GOPATH:$GOPATH:Z \
-        -w $SKOPEO_PATH \
-        $SKOPEO_CI_CONTAINER_FQIN \
-            make "$@"
-}
-
 _run_setup() {
+    local mnt
+    local errmsg
+    req_env_vars SKOPEO_CI_CONTAINER_FQIN
     if [[ "$OS_RELEASE_ID" == "fedora" ]]; then
         # This is required as part of the standard Fedora VM setup
         growpart /dev/sda 1
         resize2fs /dev/sda1
 
-        # VM's come with the distro. skopeo pre-installed
+        # VM's come with the distro. skopeo package pre-installed
         dnf erase -y skopeo
+
+        # A slew of compiled binaries are pre-built and distributed
+        # within the CI/Dev container image, but we want to run
+        # things directly on the host VM.  Fortunately they're all
+        # located in the container under /usr/local/bin
+        podman pull --quiet $SKOPEO_CI_CONTAINER_FQIN
+        mnt=$(podman mount $(podman create $SKOPEO_CI_CONTAINER_FQIN))
+        errmsg="Somehow $SKOPEO_CI_CONTAINER_FQIN is not based on $OS_REL_VER."
+        if fgrep -qx "ID=$OS_RELEASE_ID" $mnt/etc/os-release; then
+            if ! fgrep -qx "VERSION_ID=$OS_RELEASE_VER" $mnt/etc/os-release; then
+                die "$errmsg"
+            fi
+        else
+            die "$errmsg"
+        fi
+        cp -a "$mnt/usr/local/bin/* /usr/local/bin/"
+        podman umount --latest
+        podman rm --latest
     else
         die "Unknown/unsupported distro. $OS_REL_VER"
     fi
 }
 
 _run_vendor() {
-    podmanmake vendor BUILDTAGS="$BUILDTAGS"
+    make vendor BUILDTAGS="$BUILDTAGS"
 }
 
 _run_build() {
-    podmanmake bin/skopeo BUILDTAGS="$BUILDTAGS"
+    make bin/skopeo BUILDTAGS="$BUILDTAGS"
 }
 
 _run_cross() {
-    podmanmake local-cross BUILDTAGS="$BUILDTAGS"
+    make local-cross BUILDTAGS="$BUILDTAGS"
 }
 
 _run_validate() {
-    podmanmake validate-local BUILDTAGS="$BUILDTAGS"
+    make validate-local BUILDTAGS="$BUILDTAGS"
 }
 
 _run_unit() {
-    podmanmake test-unit-local BUILDTAGS="$BUILDTAGS"
+    make test-unit-local BUILDTAGS="$BUILDTAGS"
 }
 
 _run_integration() {
-    podmanmake test-integration-local BUILDTAGS="$BUILDTAGS"
+    make test-integration-local BUILDTAGS="$BUILDTAGS"
 }
 
 _run_system() {
     # Ensure we start with a clean-slate
     podman system reset --force
     # Executes with containers required for testing.
-    showrun make test-system-local BUILDTAGS="$BUILDTAGS"
+    make test-system-local BUILDTAGS="$BUILDTAGS"
 }
 
 req_env_vars SKOPEO_PATH BUILDTAGS

hack/make.sh

@@ -23,22 +23,9 @@ export SKOPEO_PKG='github.com/containers/skopeo'
 export SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 export MAKEDIR="$SCRIPTDIR/make"
 
-# We're a nice, sexy, little shell script, and people might try to run us;
-# but really, they shouldn't. We want to be in a container!
-# The magic value is defined inside our Dockerfile.
-if [[ "$container_magic" != "85531765-346b-4316-bdb8-358e4cca9e5d" ]]; then
-	{
-		echo "# WARNING! I don't seem to be running in a Docker container."
-		echo "# The result of this command might be an incorrect build, and will not be"
-		echo "# officially supported."
-		echo "#"
-		echo "# Try this instead: make all"
-		echo "#"
-	} >&2
-else
-    echo "# I appear to be running inside my designated container image, good!"
-    export SKOPEO_CONTAINER_TESTS=1
-fi
+# Override this to "0" to disable tests which may fail w/o
+# having applied hack/test_env_setup.sh
+export SKOPEO_CONTAINER_TESTS=${SKOPEO_CONTAINER_TESTS:-1}
 
 echo