diff --git a/.cirrus.yml b/.cirrus.yml index f00f419656..ec78bcaada 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -41,8 +41,9 @@ env: UBUNTU_CONTAINER_FQIN: "quay.io/libpod/ubuntu_podman:${IMAGE_SUFFIX}" PRIOR_UBUNTU_CONTAINER_FQIN: "quay.io/libpod/prior-ubuntu_podman:${IMAGE_SUFFIX}" - # Equivilent to image produced by 'make build-container' - SKOPEO_CI_CONTAINER_FQIN: "quay.io/skopeo/ci:${DEST_BRANCH}" + # Equivilent to image produced by 'make build-container'. Auto-built by quay. + # TODO:SKOPEO_CI_CONTAINER_FQIN: "quay.io/libpod/skopeo_cidev:${IMAGE_SUFFIX}" + SKOPEO_CI_CONTAINER_FQIN: "quay.io/libpod/skopeo_cidev:c5636194336768000" # Default timeout for each task @@ -57,8 +58,8 @@ validate_task: # under Cirrus-CI, due to challenges obtaining the starting commit ID. # Only do validation for PRs. only_if: $CIRRUS_PR != '' - container: &build_container - image: "${SKOPEO_CI_CONTAINER_FQIN}" + container: + image: '${SKOPEO_CI_CONTAINER_FQIN}' cpu: 4 memory: 8 script: | @@ -140,8 +141,6 @@ test_skopeo_task: "${SKOPEO_PATH}/${SCRIPT_BASE}/runner.sh" vendor build_script: >- "${SKOPEO_PATH}/${SCRIPT_BASE}/runner.sh" build - validate_script: >- - "${SKOPEO_PATH}/${SCRIPT_BASE}/runner.sh" validate unit_script: >- "${SKOPEO_PATH}/${SCRIPT_BASE}/runner.sh" unit integration_script: >- diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 8696393628..0000000000 --- a/Dockerfile +++ /dev/null @@ -1,56 +0,0 @@ -FROM registry.fedoraproject.org/fedora:latest - -RUN dnf -y update && dnf install -y make git golang golang-github-cpuguy83-md2man \ - # storage deps - btrfs-progs-devel \ - device-mapper-devel \ - # gpgme bindings deps - libassuan-devel gpgme-devel \ - gnupg \ - # htpasswd for system tests - httpd-tools \ - # OpenShift deps - which tar wget hostname util-linux bsdtar socat ethtool device-mapper iptables tree findutils nmap-ncat e2fsprogs xfsprogs lsof docker iproute \ - bats jq podman runc \ - golint \ - openssl \ - && dnf clean all - -# Install two versions of the registry. The first is an older version that -# only supports schema1 manifests. The second is a newer version that supports -# both. This allows integration-cli tests to cover push/pull with both schema1 -# and schema2 manifests. -RUN set -x \ - && export GO111MODULE=off \ - && REGISTRY_COMMIT_SCHEMA1=ec87e9b6971d831f0eff752ddb54fb64693e51cd \ - && REGISTRY_COMMIT=47a064d4195a9b56133891bbb13620c3ac83a827 \ - && export GOPATH="$(mktemp -d)" \ - && git clone https://github.com/docker/distribution.git "$GOPATH/src/github.com/docker/distribution" \ - && (cd "$GOPATH/src/github.com/docker/distribution" && git checkout -q "$REGISTRY_COMMIT") \ - && GOPATH="$GOPATH/src/github.com/docker/distribution/Godeps/_workspace:$GOPATH" \ - go build -o /usr/local/bin/registry-v2 github.com/docker/distribution/cmd/registry \ - && (cd "$GOPATH/src/github.com/docker/distribution" && git checkout -q "$REGISTRY_COMMIT_SCHEMA1") \ - && GOPATH="$GOPATH/src/github.com/docker/distribution/Godeps/_workspace:$GOPATH" \ - go build -o /usr/local/bin/registry-v2-schema1 github.com/docker/distribution/cmd/registry \ - && rm -rf "$GOPATH" - -RUN set -x \ - && export GO111MODULE=off \ - && export GOPATH=$(mktemp -d) \ - && git clone --depth 1 -b v1.5.0-alpha.3 git://github.com/openshift/origin "$GOPATH/src/github.com/openshift/origin" \ - # The sed edits out a "go < 1.5" check which works incorrectly with go ≥ 1.10. \ - && sed -i -e 's/\[\[ "\${go_version\[2]}" < "go1.5" ]]/false/' "$GOPATH/src/github.com/openshift/origin/hack/common.sh" \ - && (cd "$GOPATH/src/github.com/openshift/origin" && make clean build && make all WHAT=cmd/dockerregistry) \ - && cp -a "$GOPATH/src/github.com/openshift/origin/_output/local/bin/linux"/*/* /usr/local/bin \ - && cp "$GOPATH/src/github.com/openshift/origin/images/dockerregistry/config.yml" /atomic-registry-config.yml \ - && rm -rf "$GOPATH" \ - && mkdir /registry - -ENV GOPATH /usr/share/gocode:/go -ENV PATH $GOPATH/bin:/usr/share/gocode/bin:$PATH -ENV container_magic 85531765-346b-4316-bdb8-358e4cca9e5d -RUN go version -WORKDIR /go/src/github.com/containers/skopeo -COPY . /go/src/github.com/containers/skopeo - -#ENTRYPOINT ["hack/dind"] diff --git a/Dockerfile.build b/Dockerfile.build deleted file mode 100644 index 2aad56f2c8..0000000000 --- a/Dockerfile.build +++ /dev/null @@ -1,12 +0,0 @@ -FROM registry.fedoraproject.org/fedora:33 - -RUN dnf update -y && \ - dnf install -y \ - btrfs-progs-devel \ - device-mapper-devel \ - golang \ - gpgme-devel \ - make - -ENV GOPATH=/ -WORKDIR /src/github.com/containers/skopeo diff --git a/Makefile b/Makefile index a7949fbccd..7288d74fe5 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all binary build-container docs docs-in-container build-local clean install install-binary install-completions shell test-integration .install.vndr vendor vendor-in-container +.PHONY: all binary docs docs-in-container build-local clean install install-binary install-completions shell test-integration .install.vndr vendor vendor-in-container export GOPROXY=https://proxy.golang.org @@ -54,9 +54,9 @@ ifeq ($(GOOS), linux) endif GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null) -IMAGE := skopeo-dev$(if $(GIT_BRANCH),:$(GIT_BRANCH)) +IMAGE := quay.io/libpod/skopeo_cidev:c5636194336768000 # TODO: Lookup tag from .cirrus.yml # set env like gobuildtag? -CONTAINER_CMD := ${CONTAINER_RUNTIME} run --rm -i -e TESTFLAGS="$(TESTFLAGS)" #$(CONTAINER_ENVS) +CONTAINER_CMD := ${CONTAINER_RUNTIME} run --rm -i -e TESTFLAGS="$(TESTFLAGS)" # if this session isn't interactive, then we don't want to allocate a # TTY, which would fail, but if it is interactive, we do want to attach # so that the user can send e.g. ^C through. @@ -105,10 +105,8 @@ help: @echo " * 'shell' - Run the built image and attach to a shell" @echo " * 'clean' - Clean artifacts" -# Build a container image (skopeobuild) that has everything we need to build. -# Then do the build and the output (skopeo) should appear in current dir +# Do the build and the output (skopeo) should appear in current dir binary: cmd/skopeo - ${CONTAINER_RUNTIME} build ${BUILD_ARGS} -f Dockerfile.build -t skopeobuildimage . ${CONTAINER_RUNTIME} run --rm --security-opt label=disable -v $$(pwd):/src/github.com/containers/skopeo \ skopeobuildimage make bin/skopeo $(if $(DEBUG),DEBUG=$(DEBUG)) BUILDTAGS='$(BUILDTAGS)' @@ -136,9 +134,6 @@ bin/skopeo.%: GOOS=$(word 2,$(subst ., ,$@)) GOARCH=$(word 3,$(subst ., ,$@)) $(GO) build $(MOD_VENDOR) ${SKOPEO_LDFLAGS} -tags "containers_image_openpgp $(BUILDTAGS)" -o $@ ./cmd/skopeo local-cross: bin/skopeo.darwin.amd64 bin/skopeo.linux.arm bin/skopeo.linux.arm64 bin/skopeo.windows.386.exe bin/skopeo.windows.amd64.exe -build-container: - ${CONTAINER_RUNTIME} build ${BUILD_ARGS} -t "$(IMAGE)" . - $(MANPAGES): %: %.md sed -e 's/\((skopeo.*\.md)\)//' -e 's/\[\(skopeo.*\)\]/\1/' $< | $(GOMD2MAN) -in /dev/stdin -out $@ @@ -171,21 +166,21 @@ install-completions: install -m 755 -d ${DESTDIR}${BASHCOMPLETIONSDIR} install -m 644 completions/bash/skopeo ${DESTDIR}${BASHCOMPLETIONSDIR}/skopeo -shell: build-container +shell: $(CONTAINER_RUN) bash check: validate test-unit test-integration test-system # The tests can run out of entropy and block in containers, so replace /dev/random. -test-integration: build-container +test-integration: $(CONTAINER_RUN) bash -c 'rm -f /dev/random; ln -sf /dev/urandom /dev/random; SKOPEO_CONTAINER_TESTS=1 BUILDTAGS="$(BUILDTAGS)" $(MAKE) test-integration-local' -# Intended for CI, shortcut 'build-container' since already running inside container. +# Intended for CI, assumed to already be running in quay.io/libpod/skopeo_cidev container. test-integration-local: hack/make.sh test-integration # complicated set of options needed to run podman-in-podman -test-system: build-container +test-system: DTEMP=$(shell mktemp -d --tmpdir=/var/tmp podman-tmp.XXXXXX); \ $(CONTAINER_CMD) --privileged \ -v $$DTEMP:/var/lib/containers:Z -v /run/systemd/journal/socket:/run/systemd/journal/socket \ @@ -195,15 +190,15 @@ test-system: build-container $(RM) -rf $$DTEMP; \ exit $$rc -# Intended for CI, shortcut 'build-container' since already running inside container. +# Intended for CI, assumed to already be running in quay.io/libpod/skopeo_cidev container. test-system-local: hack/make.sh test-system -test-unit: build-container +test-unit: # Just call (make test unit-local) here instead of worrying about environment differences $(CONTAINER_RUN) make test-unit-local BUILDTAGS='$(BUILDTAGS)' -validate: build-container +validate: $(CONTAINER_RUN) make validate-local # This target is only intended for development, e.g. executing it from an IDE. Use (make test) for CI or pre-release testing. diff --git a/contrib/cirrus/runner.sh b/contrib/cirrus/runner.sh index 5c76725a73..5522434daa 100755 --- a/contrib/cirrus/runner.sh +++ b/contrib/cirrus/runner.sh @@ -25,59 +25,69 @@ OS_REL_VER="${OS_RELEASE_ID}-${OS_RELEASE_VER}" export "PATH=$PATH:$GOPATH/bin" -podmanmake() { - req_env_vars GOPATH SKOPEO_PATH SKOPEO_CI_CONTAINER_FQIN - warn "Accumulated technical-debt requires execution inside a --privileged container. This is very likely hiding bugs!" - showrun podman run -it --rm --privileged \ - -e GOPATH=$GOPATH \ - -v $GOPATH:$GOPATH:Z \ - -w $SKOPEO_PATH \ - $SKOPEO_CI_CONTAINER_FQIN \ - make "$@" -} - _run_setup() { + local mnt + local errmsg + req_env_vars SKOPEO_CI_CONTAINER_FQIN if [[ "$OS_RELEASE_ID" == "fedora" ]]; then # This is required as part of the standard Fedora VM setup growpart /dev/sda 1 resize2fs /dev/sda1 - # VM's come with the distro. skopeo pre-installed + # VM's come with the distro. skopeo package pre-installed dnf erase -y skopeo + + # A slew of compiled binaries are pre-built and distributed + # within the CI/Dev container image, but we want to run + # things directly on the host VM. Fortunately they're all + # located in the container under /usr/local/bin + podman pull --quiet $SKOPEO_CI_CONTAINER_FQIN + mnt=$(podman mount $(podman create $SKOPEO_CI_CONTAINER_FQIN)) + errmsg="Somehow $SKOPEO_CI_CONTAINER_FQIN is not based on $OS_REL_VER." + if fgrep -qx "ID=$OS_RELEASE_ID" $mnt/etc/os-release; then + if ! fgrep -qx "VERSION_ID=$OS_RELEASE_VER" $mnt/etc/os-release; then + die "$errmsg" + fi + else + die "$errmsg" + fi + cp -a "$mnt/usr/local/bin/* /usr/local/bin/" + podman umount --latest + podman rm --latest else die "Unknown/unsupported distro. $OS_REL_VER" fi } _run_vendor() { - podmanmake vendor BUILDTAGS="$BUILDTAGS" + make vendor BUILDTAGS="$BUILDTAGS" } _run_build() { - podmanmake bin/skopeo BUILDTAGS="$BUILDTAGS" + make bin/skopeo BUILDTAGS="$BUILDTAGS" } _run_cross() { - podmanmake local-cross BUILDTAGS="$BUILDTAGS" + make local-cross BUILDTAGS="$BUILDTAGS" } _run_validate() { - podmanmake validate-local BUILDTAGS="$BUILDTAGS" + make validate-local BUILDTAGS="$BUILDTAGS" } _run_unit() { - podmanmake test-unit-local BUILDTAGS="$BUILDTAGS" + make test-unit-local BUILDTAGS="$BUILDTAGS" } _run_integration() { - podmanmake test-integration-local BUILDTAGS="$BUILDTAGS" + make test-integration-local BUILDTAGS="$BUILDTAGS" } _run_system() { # Ensure we start with a clean-slate podman system reset --force # Executes with containers required for testing. - showrun make test-system-local BUILDTAGS="$BUILDTAGS" + make test-system-local BUILDTAGS="$BUILDTAGS" } req_env_vars SKOPEO_PATH BUILDTAGS diff --git a/hack/make.sh b/hack/make.sh index 6850f36f24..e886475b18 100755 --- a/hack/make.sh +++ b/hack/make.sh @@ -23,22 +23,9 @@ export SKOPEO_PKG='github.com/containers/skopeo' export SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" export MAKEDIR="$SCRIPTDIR/make" -# We're a nice, sexy, little shell script, and people might try to run us; -# but really, they shouldn't. We want to be in a container! -# The magic value is defined inside our Dockerfile. -if [[ "$container_magic" != "85531765-346b-4316-bdb8-358e4cca9e5d" ]]; then - { - echo "# WARNING! I don't seem to be running in a Docker container." - echo "# The result of this command might be an incorrect build, and will not be" - echo "# officially supported." - echo "#" - echo "# Try this instead: make all" - echo "#" - } >&2 -else - echo "# I appear to be running inside my designated container image, good!" - export SKOPEO_CONTAINER_TESTS=1 -fi +# Override this to "0" to disable tests which may fail w/o +# having applied hack/test_env_setup.sh +export SKOPEO_CONTAINER_TESTS=${SKOPEO_CONTAINER_TESTS:-1} echo