From 8e46106f420dfc6125750c12e13c5ae39be9d6f1 Mon Sep 17 00:00:00 2001 From: Chris Evich Date: Wed, 31 Jul 2019 09:19:06 -0400 Subject: [PATCH 1/2] Readme: Links for automatic binary releases Signed-off-by: Chris Evich --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index f7c592671158..0ef388d46212 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,14 @@ popularized by Kubernetes. Libpod also contains the Pod Manager tool `(Podman)` * [Latest Version: 1.4.4](https://github.com/containers/libpod/releases/latest) * [Continuous Integration:](contrib/cirrus/README.md) [![Build Status](https://api.cirrus-ci.com/github/containers/libpod.svg)](https://cirrus-ci.com/github/containers/libpod/master) * [GoDoc: ![GoDoc](https://godoc.org/github.com/containers/libpod/libpod?status.svg)](https://godoc.org/github.com/containers/libpod/libpod) +* Latest, automatic binary release downloads: + * Master Branch: + * [Fedora 30](https://storage.cloud.google.com/libpod-master-releases/libpod-master-fedora-30-amd64.zip) + * [Fedora 29](https://storage.cloud.google.com/libpod-master-releases/libpod-master-fedora-29-amd64.zip) + * [Ubuntu 18](https://storage.cloud.google.com/libpod-master-releases/libpod-master-ubuntu-18-amd64.zip) + * [Ubuntu 19](https://storage.cloud.google.com/libpod-master-releases/libpod-master-ubuntu-19-amd64.zip) + * [Windows Client](https://storage.cloud.google.com/libpod-master-releases/podman-remote-master-windows-cc-amd64.zip) + * [MacOS Client](https://storage.cloud.google.com/libpod-master-releases/podman-remote-master-darwin-cc-amd64.zip) ## Overview and scope From 370b1a887cbf6db8ac893c39118cf8c6c2fd663c Mon Sep 17 00:00:00 2001 From: Chris Evich Date: Thu, 1 Aug 2019 07:31:04 -0400 Subject: [PATCH 2/2] Cirrus: Reimplement release archive + upload The initial implementation was far more complicated than necessary. Strip out the complexities in favor of a simpler and more direct approach. Signed-off-by: Chris Evich --- .cirrus.yml | 70 +++-------- .gitignore | 3 +- Makefile | 106 +++++++++------- README.md | 11 +- contrib/cirrus/build_release.sh | 30 +++++ contrib/cirrus/cache_release_archive.sh | 140 --------------------- contrib/cirrus/cirrus_yaml_test.py | 43 +++---- contrib/cirrus/integration_test.sh | 4 - contrib/cirrus/lib.sh | 4 +- contrib/cirrus/setup_environment.sh | 5 +- contrib/cirrus/uncache_release_archives.sh | 1 - contrib/cirrus/unit_test.sh | 4 - contrib/cirrus/upload_release_archive.sh | 52 ++++++++ contrib/imgts/lib_entrypoint.sh | 15 ++- contrib/upldrel/Dockerfile | 9 ++ contrib/upldrel/README.md | 9 ++ contrib/upldrel/entrypoint.sh | 62 +++++++++ hack/get_release_info.sh | 54 ++++++++ 18 files changed, 328 insertions(+), 294 deletions(-) create mode 100755 contrib/cirrus/build_release.sh delete mode 100755 contrib/cirrus/cache_release_archive.sh delete mode 120000 contrib/cirrus/uncache_release_archives.sh create mode 100755 contrib/cirrus/upload_release_archive.sh create mode 100644 contrib/upldrel/Dockerfile create mode 100644 contrib/upldrel/README.md create mode 100755 contrib/upldrel/entrypoint.sh create mode 100755 hack/get_release_info.sh diff --git a/.cirrus.yml b/.cirrus.yml index 9f718477f656..3c50332e2a73 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -62,6 +62,10 @@ env: GCE_SSH_USERNAME: cirrus-ci # Name where this repositories cloud resources are located GCP_PROJECT_ID: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f] + RELEASE_GCPJSON: ENCRYPTED[789d8f7e9a5972ce350fd8e60f1032ccbf4a35c3938b604774b711aad280e12c21faf10e25af1e0ba33597ffb9e39e46] + RELEASE_GCPNAME: ENCRYPTED[417d50488a4bd197bcc925ba6574de5823b97e68db1a17e3a5fde4bcf26576987345e75f8d9ea1c15a156b4612c072a1] + RELEASE_GCPROJECT: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f] + # Default VM to use unless set or modified by task @@ -336,9 +340,8 @@ testing_task: unit_test_script: '$SCRIPT_BASE/unit_test.sh |& ${TIMESTAMP}' integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}' system_test_script: '$SCRIPT_BASE/system_test.sh |& ${TIMESTAMP}' - cache_release_archive_script: >- - [[ "$TEST_REMOTE_CLIENT" == "false" ]] || \ - $SCRIPT_BASE/cache_release_archive.sh |& ${TIMESTAMP} + build_release_script: '$SCRIPT_BASE/build_release.sh |& ${TIMESTAMP}' + upload_release_archive_script: '$SCRIPT_BASE/upload_release_archive.sh |& ${TIMESTAMP}' on_failure: failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh' @@ -376,9 +379,6 @@ testing_crun_task: unit_test_script: '$SCRIPT_BASE/unit_test.sh |& ${TIMESTAMP}' integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}' system_test_script: '$SCRIPT_BASE/system_test.sh |& ${TIMESTAMP}' - cache_release_archive_script: >- - [[ "$TEST_REMOTE_CLIENT" == "false" ]] || \ - $SCRIPT_BASE/cache_release_archive.sh |& ${TIMESTAMP} on_failure: failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh' @@ -459,14 +459,15 @@ special_testing_cross_task: env: matrix: - SPECIALMODE: 'windows' # See docs - SPECIALMODE: 'darwin' + CROSS_PLATFORM: 'windows' + CROSS_PLATFORM: 'darwin' timeout_in: 20m networking_script: '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/networking.sh' setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' - cache_release_archive_script: '$SCRIPT_BASE/cache_release_archive.sh |& ${TIMESTAMP}' + build_release_script: '$SCRIPT_BASE/build_release.sh |& ${TIMESTAMP}' + upload_release_archive_script: '$SCRIPT_BASE/upload_release_archive.sh |& ${TIMESTAMP}' on_failure: failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh' @@ -615,6 +616,9 @@ verify_test_built_images_task: integration_test_script: >- [[ "$PACKER_BUILDER_NAME" == "xfedora-30" ]] || \ $SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP} + build_release_script: >- + [[ "$PACKER_BUILDER_NAME" == "xfedora-30" ]] || \ + '$SCRIPT_BASE/build_release.sh |& ${TIMESTAMP}' system_test_script: >- [[ "$PACKER_BUILDER_NAME" == "xfedora-30" ]] || \ $SCRIPT_BASE/system_test.sh |& ${TIMESTAMP} @@ -629,7 +633,7 @@ success_task: # it blocks PRs from merging if a depends_on task fails only_if: $CIRRUS_BRANCH != $DEST_BRANCH - # ignores any dependent task conditions, include everything except 'release' + # ignores any dependent task conditions depends_on: - "gating" - "vendor" @@ -660,49 +664,3 @@ success_task: memory: 1 success_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/success.sh |& ${TIMESTAMP}' - - -release_task: - - # Never do this when building images - only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' - - # TODO: Uncomment both to not affect pass/fail status of entire job? - # allow_failures: $CI == "true" - # skip_notifications: $CI == "true" - - # Must include everything (YAML anchor/alias cannot be used here) - depends_on: - - "gating" - - "vendor" - - "varlink_api" - - "build_each_commit" - - "build_without_cgo" - - "meta" - - "image_prune" - - "testing" - - "testing_crun" - - "special_testing_rootless" - - "special_testing_in_podman" - - "special_testing_cgroupv2" - - "special_testing_cross" - - "special_testing_endpoint" - - "test_build_cache_images" - - "test_building_snap" - - "verify_test_built_images" - - "success" - - gce_instance: - image_name: "${IMAGE_BUILDER_CACHE_IMAGE_NAME}" - - timeout_in: 30m - - env: - GCPJSON: ENCRYPTED[789d8f7e9a5972ce350fd8e60f1032ccbf4a35c3938b604774b711aad280e12c21faf10e25af1e0ba33597ffb9e39e46] - GCPNAME: ENCRYPTED[417d50488a4bd197bcc925ba6574de5823b97e68db1a17e3a5fde4bcf26576987345e75f8d9ea1c15a156b4612c072a1] - GCPROJECT: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f] - - uncache_release_archives_script: '$SCRIPT_BASE/uncache_release_archives.sh |& ${TIMESTAMP}' - - on_failure: - failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh' diff --git a/.gitignore b/.gitignore index 2d984cb5a913..d3e56ecdf9d3 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,7 @@ __pycache__ /cmd/podman/varlink/iopodman.go .gopathok test/e2e/e2e.coverprofile -/podman*zip +release.txt +podman-remote*.zip podman*.tar.gz .idea* diff --git a/Makefile b/Makefile index 82a24306d688..1a3d03f8a3a8 100644 --- a/Makefile +++ b/Makefile @@ -81,13 +81,12 @@ LIBSECCOMP_COMMIT := release-2.3 # caller may override in special circumstances if needed. GINKGOTIMEOUT ?= -timeout=90m -RELEASE_VERSION ?= $(shell git fetch --tags && git describe HEAD 2> /dev/null) -RELEASE_NUMBER ?= $(shell echo $(RELEASE_VERSION) | sed 's/-.*//') -RELEASE_DIST ?= $(shell ( source /etc/os-release; echo $$ID )) -RELEASE_DIST_VER ?= $(shell ( source /etc/os-release; echo $$VERSION_ID | cut -d '.' -f 1)) -RELEASE_ARCH ?= $(shell go env GOARCH 2> /dev/null) -RELEASE_BASENAME := $(shell basename $(PROJECT)) - +RELEASE_VERSION ?= $(shell hack/get_release_info.sh VERSION) +RELEASE_NUMBER ?= $(shell hack/get_release_info.sh NUMBER) +RELEASE_DIST ?= $(shell hack/get_release_info.sh DIST) +RELEASE_DIST_VER ?= $(shell hack/get_release_info.sh DIST_VER) +RELEASE_ARCH ?= $(shell hack/get_release_info.sh ARCH) +RELEASE_BASENAME := $(shell hack/get_release_info.sh BASENAME) # If GOPATH not specified, use one in the local directory ifeq ($(GOPATH),) @@ -164,11 +163,9 @@ podman: .gopathok $(PODMAN_VARLINK_DEPENDENCIES) ## Build with podman podman-remote: .gopathok $(PODMAN_VARLINK_DEPENDENCIES) ## Build with podman on remote environment $(GO_BUILD) $(BUILDFLAGS) -gcflags '$(GCFLAGS)' -asmflags '$(ASMFLAGS)' -ldflags '$(LDFLAGS_PODMAN)' -tags "$(BUILDTAGS) remoteclient" -o bin/$@ $(PROJECT)/cmd/podman -podman-remote-darwin: .gopathok $(PODMAN_VARLINK_DEPENDENCIES) ## Build with podman on remote OSX environment - CGO_ENABLED=0 GOOS=darwin $(GO_BUILD) -gcflags '$(GCFLAGS)' -asmflags '$(ASMFLAGS)' -ldflags '$(LDFLAGS_PODMAN)' -tags "remoteclient containers_image_openpgp exclude_graphdriver_devicemapper" -o bin/$@ $(PROJECT)/cmd/podman - -podman-remote-windows: .gopathok $(PODMAN_VARLINK_DEPENDENCIES) ## Build with podman for a remote windows environment - CGO_ENABLED=0 GOOS=windows $(GO_BUILD) -gcflags '$(GCFLAGS)' -asmflags '$(ASMFLAGS)' -ldflags '$(LDFLAGS_PODMAN)' -tags "remoteclient containers_image_openpgp exclude_graphdriver_devicemapper" -o bin/$@.exe $(PROJECT)/cmd/podman +podman-remote-%: .gopathok $(PODMAN_VARLINK_DEPENDENCIES) ## Build podman for a specific GOOS + $(eval BINSFX := $(shell test "$*" != "windows" || echo ".exe")) + CGO_ENABLED=0 GOOS=$* $(GO_BUILD) -gcflags '$(GCFLAGS)' -asmflags '$(ASMFLAGS)' -ldflags '$(LDFLAGS_PODMAN)' -tags "remoteclient containers_image_openpgp exclude_graphdriver_devicemapper" -o bin/$@$(BINSFX) $(PROJECT)/cmd/podman local-cross: $(CROSS_BUILD_TARGETS) ## Cross local compilation @@ -182,8 +179,9 @@ clean: ## Clean artifacts rm -rf \ .gopathok \ _output \ - podman*.zip \ - podman*.tar.gz \ + release.txt + $(wildcard podman-remote*.zip) \ + $(wildcard podman*.tar.gz) \ bin \ build \ docs/remote \ @@ -300,23 +298,6 @@ vagrant-check: binaries: varlink_generate podman podman-remote ## Build podman -# Zip archives are supported on all platforms + allows embedding metadata -podman.zip: binaries docs - $(eval TMPDIR := $(shell mktemp -d -p '' $@_XXXX)) - test -n "$(TMPDIR)" - $(MAKE) install "DESTDIR=$(TMPDIR)" "PREFIX=$(TMPDIR)/usr" - # Encoded RELEASE_INFO format depended upon by CI tooling - # X-RELEASE-INFO format depended upon by CI tooling - cd "$(TMPDIR)" && echo \ - "X-RELEASE-INFO: $(RELEASE_BASENAME) $(RELEASE_VERSION) $(RELEASE_DIST) $(RELEASE_DIST_VER) $(RELEASE_ARCH)" | \ - zip --recurse-paths --archive-comment "$(CURDIR)/$@" "./" - -rm -rf "$(TMPDIR)" - -podman-remote-%.zip: podman-remote-% - # Don't label darwin/windows cros-compiles with local distribution & version - echo "X-RELEASE-INFO: podman-remote $(RELEASE_VERSION) $* cc $(RELEASE_ARCH)" | \ - zip --archive-comment "$(CURDIR)/$@" ./bin/$<* - install.catatonit: ./hack/install_catatonit.sh @@ -333,19 +314,58 @@ docs: $(MANPAGES) ## Generate documentation install-podman-remote-docs: docs @(cd docs; ./podman-remote.sh ./remote) +# When publishing releases include critical build-time details +.PHONY: release.txt +release.txt: + # X-RELEASE-INFO format depended upon by automated tooling + echo -n "X-RELEASE-INFO:" > "$@" + for field in "$(RELEASE_BASENAME)" "$(RELEASE_VERSION)" \ + "$(RELEASE_DIST)" "$(RELEASE_DIST_VER)" "$(RELEASE_ARCH)"; do \ + echo -n " $$field"; done >> "$@" + echo "" >> "$@" + +podman-$(RELEASE_NUMBER).tar.gz: binaries docs release.txt + $(eval TMPDIR := $(shell mktemp -d -p '' podman_XXXX)) + $(eval SUBDIR := podman-$(RELEASE_NUMBER)) + mkdir -p "$(TMPDIR)/$(SUBDIR)" + $(MAKE) install.bin install.man install.cni install.systemd "DESTDIR=$(TMPDIR)/$(SUBDIR)" "PREFIX=/usr" + # release.txt location and content depended upon by automated tooling + cp release.txt "$(TMPDIR)/" + tar -czvf $@ --xattrs -C "$(TMPDIR)" "./release.txt" "./$(SUBDIR)" + -rm -rf "$(TMPDIR)" + +# Must call make in-line: Dependency-spec. w/ wild-card also consumes variable value. +podman-remote-$(RELEASE_NUMBER)-%.zip: + $(MAKE) podman-remote-$* install-podman-remote-docs release.txt \ + RELEASE_BASENAME=$(shell hack/get_release_info.sh REMOTENAME) \ + RELEASE_DIST=$* RELEASE_DIST_VER="-" + $(eval TMPDIR := $(shell mktemp -d -p '' $podman_remote_XXXX)) + $(eval SUBDIR := podman-$(RELEASE_VERSION)) + $(eval BINSFX := $(shell test "$*" != "windows" || echo ".exe")) + mkdir -p "$(TMPDIR)/$(SUBDIR)" + # release.txt location and content depended upon by automated tooling + cp release.txt "$(TMPDIR)/" + cp ./bin/podman-remote-$*$(BINSFX) "$(TMPDIR)/$(SUBDIR)/podman$(BINSFX)" + cp -r ./docs/remote "$(TMPDIR)/$(SUBDIR)/docs/" + $(eval DOCFILE := $(TMPDIR)/$(SUBDIR)/docs/podman.1) + cp docs/podman-remote.1 "$(DOCFILE)" + sed -i 's/podman\\*-remote/podman/g' "$(DOCFILE)" + sed -i 's/Podman\\*-remote/Podman\ for\ $*/g' "$(DOCFILE)" + sed -i 's/podman\.conf/podman\-remote\.conf/g' "$(DOCFILE)" + sed -i 's/A\ remote\ CLI\ for\ Podman\:\ //g' "$(DOCFILE)" + cd "$(TMPDIR)" && \ + zip --recurse-paths "$(CURDIR)/$@" "./release.txt" "./" + -rm -rf "$(TMPDIR)" + +.PHONY: podman-release +podman-release: + rm -f release.txt + $(MAKE) podman-$(RELEASE_NUMBER).tar.gz -brew-pkg: install-podman-remote-docs podman-remote-darwin - @mkdir -p ./brew - @cp ./bin/podman-remote-darwin ./brew/podman - @cp -r ./docs/remote ./brew/docs/ - @cp docs/podman-remote.1 ./brew/docs/podman.1 - @cp docs/podman-remote.conf.5 ./brew/docs/podman-remote.conf.5 - @sed -i 's/podman\\*-remote/podman/g' ./brew/docs/podman.1 - @sed -i 's/Podman\\*-remote/Podman\ for\ Mac/g' ./brew/docs/podman.1 - @sed -i 's/podman\.conf/podman\-remote\.conf/g' ./brew/docs/podman.1 - @sed -i 's/A\ remote\ CLI\ for\ Podman\:\ //g' ./brew/docs/podman.1 - tar -czvf podman-${RELEASE_NUMBER}.tar.gz ./brew - @rm -rf ./brew +.PHONY: podman-remote-%-release +podman-remote-%-release: + rm -f release.txt + $(MAKE) podman-remote-$(RELEASE_NUMBER)-$*.zip docker-docs: docs (cd docs; ./dckrman.sh *.1) diff --git a/README.md b/README.md index 0ef388d46212..34ed6d9e123a 100644 --- a/README.md +++ b/README.md @@ -8,14 +8,9 @@ popularized by Kubernetes. Libpod also contains the Pod Manager tool `(Podman)` * [Latest Version: 1.4.4](https://github.com/containers/libpod/releases/latest) * [Continuous Integration:](contrib/cirrus/README.md) [![Build Status](https://api.cirrus-ci.com/github/containers/libpod.svg)](https://cirrus-ci.com/github/containers/libpod/master) * [GoDoc: ![GoDoc](https://godoc.org/github.com/containers/libpod/libpod?status.svg)](https://godoc.org/github.com/containers/libpod/libpod) -* Latest, automatic binary release downloads: - * Master Branch: - * [Fedora 30](https://storage.cloud.google.com/libpod-master-releases/libpod-master-fedora-30-amd64.zip) - * [Fedora 29](https://storage.cloud.google.com/libpod-master-releases/libpod-master-fedora-29-amd64.zip) - * [Ubuntu 18](https://storage.cloud.google.com/libpod-master-releases/libpod-master-ubuntu-18-amd64.zip) - * [Ubuntu 19](https://storage.cloud.google.com/libpod-master-releases/libpod-master-ubuntu-19-amd64.zip) - * [Windows Client](https://storage.cloud.google.com/libpod-master-releases/podman-remote-master-windows-cc-amd64.zip) - * [MacOS Client](https://storage.cloud.google.com/libpod-master-releases/podman-remote-master-darwin-cc-amd64.zip) +* Automated continuous release downloads (including remote-client): + * Master Branch: [https://storage.cloud.google.com/libpod-master-releases/](https://storage.cloud.google.com/libpod-master-releases/) + * Pull-requests: [https://storage.cloud.google.com/libpod-pr-releases/](https://storage.cloud.google.com/libpod-pr-releases/) ## Overview and scope diff --git a/contrib/cirrus/build_release.sh b/contrib/cirrus/build_release.sh new file mode 100755 index 000000000000..287643f47d9b --- /dev/null +++ b/contrib/cirrus/build_release.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +source $(dirname $0)/lib.sh + +req_env_var TEST_REMOTE_CLIENT OS_RELEASE_ID GOSRC + +cd $GOSRC + +if [[ "$TEST_REMOTE_CLIENT" == "true" ]] && [[ -z "$CROSS_PLATFORM" ]] +then + CROSS_PLATFORM=linux +fi + +if [[ -n "$CROSS_PLATFORM" ]] +then + echo "Compiling podman-remote release archive for ${CROSS_PLATFORM}" + case "$CROSS_PLATFORM" in + linux) ;& + windows) ;& + darwin) + make podman-remote-${CROSS_PLATFORM}-release + ;; + *) + die 1 "Unknown/unsupported cross-compile platform '$CROSS_PLATFORM'" + ;; + esac +else + echo "Compiling release archive for $OS_RELEASE_ID" + make podman-release +fi diff --git a/contrib/cirrus/cache_release_archive.sh b/contrib/cirrus/cache_release_archive.sh deleted file mode 100755 index 2365f759350b..000000000000 --- a/contrib/cirrus/cache_release_archive.sh +++ /dev/null @@ -1,140 +0,0 @@ -#!/bin/bash - -set -eo pipefail - -source $(dirname $0)/lib.sh - -req_env_var GOSRC - -RELEASE_ARCHIVE_NAMES="" - -handle_archive() { # Assumed to be called with set +e - TASK_NUMBER=$1 - PR_OR_BRANCH=$2 - CACHE_URL=$3 - ARCHIVE_NAME="$(basename $CACHE_URL)" - req_env_var TASK_NUMBER PR_OR_BRANCH CACHE_URL ARCHIVE_NAME - - cd /tmp - curl -sO "$CACHE_URL" || return $(warn 0 "Couldn't download file, skipping.") - [[ -r "/tmp/$ARCHIVE_NAME" ]] || return $(warn 0 "Unreadable archive '/tmp/$ARCHIVE_NAME', skipping.") - - ZIPCOMMENT=$(unzip -qqz "$ARCHIVE_NAME" 2>/dev/null) # noisy bugger - if [[ "$?" -ne "0" ]] || [[ -z "$ZIPCOMMENT" ]] - then - return $(warn 0 "Could not unzip metadata from downloaded '/tmp/$ARCHIVE_NAME', skipping.") - fi - - RELEASE_INFO=$(echo "$ZIPCOMMENT" | grep -m 1 'X-RELEASE-INFO:' | sed -r -e 's/X-RELEASE-INFO:\s*(.+)/\1/') - if [[ "$?" -ne "0" ]] || [[ -z "$RELEASE_INFO" ]] - then - return $(warn 0 "Metadata empty or invalid: '$ZIPCOMMENT', skipping.") - fi - - # e.g. libpod v1.3.1-166-g60df124e fedora 29 amd64 - # or libpod v1.3.1-166-g60df124e amd64 - FIELDS="RELEASE_BASENAME RELEASE_VERSION RELEASE_DIST RELEASE_DIST_VER RELEASE_ARCH" - read $FIELDS <<< $RELEASE_INFO - for f in $FIELDS - do - [[ -n "${!f}" ]] || return $(warn 0 "Expecting $f to be non-empty in metadata: '$RELEASE_INFO', skipping.") - done - - echo -n "Preparing $RELEASE_BASENAME archive: " - # Drop version number to enable "latest" representation - # (version available w/in zip-file comment) - RELEASE_ARCHIVE_NAME="${RELEASE_BASENAME}-${PR_OR_BRANCH}-${RELEASE_DIST}-${RELEASE_DIST_VER}-${RELEASE_ARCH}.zip" - # Allow uploading all gathered files in parallel, later with gsutil. - mv -v "$ARCHIVE_NAME" "/$RELEASE_ARCHIVE_NAME" - RELEASE_ARCHIVE_NAMES="$RELEASE_ARCHIVE_NAMES $RELEASE_ARCHIVE_NAME" -} - -make_release() { - ARCHIVE_NAME="$1" - req_env_var ARCHIVE_NAME - - # There's no actual testing of windows/darwin targets yet - # but we still want to cross-compile and publish binaries - if [[ "$SPECIALMODE" == "windows" ]] || [[ "$SPECIALMODE" == "darwin" ]] - then - RELFILE="podman-remote-${SPECIALMODE}.zip" - elif [[ "$SPECIALMODE" == "none" ]] - then - RELFILE="podman.zip" - else - die 55 "$(basename $0) unable to handle \$SPECIALMODE=$SPECIALMODE for $ARCHIVE_NAME" - fi - echo "Calling make $RELFILE" - cd $GOSRC - make "$RELFILE" - echo "Renaming archive so it can be identified/downloaded for publishing" - mv -v "$RELFILE" "$ARCHIVE_NAME" - echo "Success!" -} - -[[ "$CI" == "true" ]] || \ - die 56 "$0 requires a Cirrus-CI cross-task cache to function" - -cd $GOSRC -# Same script re-used for both uploading and downloading to avoid duplication -if [[ "$(basename $0)" == "cache_release_archive.sh" ]] -then - # ref: https://cirrus-ci.org/guide/writing-tasks/#environment-variables - req_env_var CI_NODE_INDEX CIRRUS_BUILD_ID - # Use unique names for uncache_release_archives.sh to find/download them all - ARCHIVE_NAME="build-${CIRRUS_BUILD_ID}-task-${CI_NODE_INDEX}.zip" - make_release "$ARCHIVE_NAME" - - # ref: https://cirrus-ci.org/guide/writing-tasks/#http-cache - URL="http://$CIRRUS_HTTP_CACHE_HOST/${ARCHIVE_NAME}" - echo "Uploading $ARCHIVE_NAME to Cirrus-CI cache at $URL" - curl -s -X POST --data-binary "@$ARCHIVE_NAME" "$URL" -elif [[ "$(basename $0)" == "uncache_release_archives.sh" ]] -then - req_env_var CIRRUS_BUILD_ID CI_NODE_TOTAL GCPJSON GCPNAME GCPROJECT - [[ "${CI_NODE_INDEX}" -eq "$[CI_NODE_TOTAL-1]" ]] || \ - die 0 "WARNING: This task depends on cache data from other tasks, otherwise it is a no-op." - - if [[ -n "$CIRRUS_PR" ]] - then - PR_OR_BRANCH="pr$CIRRUS_PR" - BUCKET="libpod-pr-releases" - elif [[ -n "$CIRRUS_BRANCH" ]] - then - PR_OR_BRANCH="$CIRRUS_BRANCH" - BUCKET="libpod-$CIRRUS_BRANCH-releases" - else - die 10 "Expecting either \$CIRRUS_PR or \$CIRRUS_BRANCH to be non-empty." - fi - - echo "Blindly downloading Cirrus-CI cache files for task (some will fail)." - set +e # Don't stop looping until all task's cache is attempted - for (( task_number = 0 ; task_number < $CI_NODE_TOTAL ; task_number++ )) - do - ARCHIVE_NAME="build-${CIRRUS_BUILD_ID}-task-${task_number}.zip" - URL="http://$CIRRUS_HTTP_CACHE_HOST/${ARCHIVE_NAME}" - echo "Attempting to download cached archive from $URL" - handle_archive "$task_number" "$PR_OR_BRANCH" "$URL" - echo "----------------------------------------" - done - set -e - - [[ -n "$RELEASE_ARCHIVE_NAMES" ]] || \ - die 67 "Error: No release archives found in CI cache, expecting at least one." - - echo "Preparing to upload release archives." - gcloud config set project "$GCPROJECT" - echo "$GCPJSON" > /tmp/gcp.json - gcloud auth activate-service-account --key-file=/tmp/gcp.json - rm /tmp/gcp.json - # handle_archive() placed all uploadable files under / - gsutil -m cp /*.zip "gs://$BUCKET" # Upload in parallel - echo "Successfully uploaded archives:" - for ARCHIVE_NAME in $RELEASE_ARCHIVE_NAMES - do - echo " https://storage.cloud.google.com/$BUCKET/$ARCHIVE_NAME" - done - echo "These will remain available until automatic pruning by bucket policy." -else - die 9 "I don't know what to do when called $0" -fi diff --git a/contrib/cirrus/cirrus_yaml_test.py b/contrib/cirrus/cirrus_yaml_test.py index c8faee65fec9..c2ff8e69e80f 100755 --- a/contrib/cirrus/cirrus_yaml_test.py +++ b/contrib/cirrus/cirrus_yaml_test.py @@ -26,7 +26,6 @@ def setUp(self): class TestDependsOn(TestCaseBase): ALL_TASK_NAMES = None - SUCCESS_RELEASE = set(['success', 'release']) def setUp(self): super().setUp() @@ -34,34 +33,22 @@ def setUp(self): for key, _ in self.CIRRUS_YAML.items() if key.endswith('_task')]) - def test_dicts(self): + def test_00_dicts(self): """Expected dictionaries are present and non-empty""" - for name in ('success_task', 'release_task'): - # tests all names then show specific failures - with self.subTest(name=name): - self.assertIn(name, self.CIRRUS_YAML) - self.assertIn(name.replace('_task', ''), self.ALL_TASK_NAMES) - self.assertIn('depends_on', self.CIRRUS_YAML[name]) - self.assertGreater(len(self.CIRRUS_YAML[name]['depends_on']), 0) - - def _check_dep(self, name, task_name, deps): - # name includes '_task' suffix, task_name does not - msg=('Please add "{0}" to the "depends_on" list in "{1}"' - "".format(task_name, name)) - self.assertIn(task_name, deps, msg=msg) - - def test_depends(self): - """Success and Release tasks depend on all other tasks""" - for name in ('success_task', 'release_task'): - deps = set(self.CIRRUS_YAML[name]['depends_on']) - for task_name in self.ALL_TASK_NAMES - self.SUCCESS_RELEASE: - with self.subTest(name=name, task_name=task_name): - self._check_dep(name, task_name, deps) - - def test_release(self): - """Release task must always execute last""" - deps = set(self.CIRRUS_YAML['release_task']['depends_on']) - self._check_dep('release_task', 'success', deps) + self.assertIn('success_task', self.CIRRUS_YAML) + self.assertIn('success_task'.replace('_task', ''), self.ALL_TASK_NAMES) + self.assertIn('depends_on', self.CIRRUS_YAML['success_task']) + self.assertGreater(len(self.CIRRUS_YAML['success_task']['depends_on']), 0) + + def test_01_depends(self): + """Success task depends on all other tasks""" + success_deps = set(self.CIRRUS_YAML['success_task']['depends_on']) + for task_name in self.ALL_TASK_NAMES - set(['success']): + with self.subTest(task_name=task_name): + msg=('Please add "{0}" to the "depends_on" list in "success_task"' + "".format(task_name)) + self.assertIn(task_name, success_deps, msg=msg) + if __name__ == "__main__": diff --git a/contrib/cirrus/integration_test.sh b/contrib/cirrus/integration_test.sh index a3d18d4401f0..552f2ba731bc 100755 --- a/contrib/cirrus/integration_test.sh +++ b/contrib/cirrus/integration_test.sh @@ -65,10 +65,6 @@ case "$SPECIALMODE" in make local${TESTSUITE} fi ;; - windows) ;& # for podman-remote building only - darwin) - warn '' "No $SPECIALMODE remote client integration tests configured" - ;; *) die 110 "Unsupported \$SPECIALMODE: $SPECIALMODE" esac diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index a20ee5a62043..555f3e717513 100644 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -64,6 +64,8 @@ export PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-29-1-2-1559164849" export BUILT_IMAGE_SUFFIX="${BUILT_IMAGE_SUFFIX:--$CIRRUS_REPO_NAME-${CIRRUS_BUILD_ID}}" # IN_PODMAN container image IN_PODMAN_IMAGE="quay.io/libpod/in_podman:latest" +# Image for uploading releases +UPLDREL_IMAGE="quay.io/libpod/upldrel:latest" # Avoid getting stuck waiting for user input export DEBIAN_FRONTEND="noninteractive" @@ -76,7 +78,7 @@ BIGTO="timeout_attempt_delay_command 300s 5 30s" # Safe env. vars. to transfer from root -> $ROOTLESS_USER (go env handled separetly) ROOTLESS_ENV_RE='(CIRRUS_.+)|(ROOTLESS_.+)|(.+_IMAGE.*)|(.+_BASE)|(.*DIRPATH)|(.*FILEPATH)|(SOURCE.*)|(DEPEND.*)|(.+_DEPS_.+)|(OS_REL.*)|(.+_ENV_RE)|(TRAVIS)|(CI.+)|(TEST_REMOTE.*)' # Unsafe env. vars for display -SECRET_ENV_RE='(IRCID)|(ACCOUNT)|(^GC[EP]..+)|(SSH)' +SECRET_ENV_RE='(IRCID)|(ACCOUNT)|(GC[EP]..+)|(SSH)' # Names of systemd units which should never be running EVIL_UNITS="cron crond atd apt-daily-upgrade apt-daily fstrim motd-news systemd-tmpfiles-clean" diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh index 2579229a5bde..323a05489207 100755 --- a/contrib/cirrus/setup_environment.sh +++ b/contrib/cirrus/setup_environment.sh @@ -66,7 +66,8 @@ case "$SPECIALMODE" in remove_packaged_podman_files # we're building from source ;; none) - remove_packaged_podman_files + [[ -n "$CROSS_PLATFORM" ]] || \ + remove_packaged_podman_files ;; endpoint) remove_packaged_podman_files @@ -88,8 +89,6 @@ case "$SPECIALMODE" in in_podman) # Assumed to be Fedora $SCRIPT_BASE/setup_container_environment.sh ;; - windows) ;& # for podman-remote building only - darwin) ;; *) die 111 "Unsupported \$SPECIALMODE: $SPECIALMODE" esac diff --git a/contrib/cirrus/uncache_release_archives.sh b/contrib/cirrus/uncache_release_archives.sh deleted file mode 120000 index e9fc6edffef0..000000000000 --- a/contrib/cirrus/uncache_release_archives.sh +++ /dev/null @@ -1 +0,0 @@ -cache_release_archive.sh \ No newline at end of file diff --git a/contrib/cirrus/unit_test.sh b/contrib/cirrus/unit_test.sh index 004839f17556..c6c77d17e355 100755 --- a/contrib/cirrus/unit_test.sh +++ b/contrib/cirrus/unit_test.sh @@ -16,10 +16,6 @@ case "$SPECIALMODE" in none) make ;; - windows) ;& - darwin) - make podman-remote-$SPECIALMODE - ;; *) die 109 "Unsupported \$SPECIAL_MODE: $SPECIALMODE" esac diff --git a/contrib/cirrus/upload_release_archive.sh b/contrib/cirrus/upload_release_archive.sh new file mode 100755 index 000000000000..942255821729 --- /dev/null +++ b/contrib/cirrus/upload_release_archive.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +set -eo pipefail + +source $(dirname $0)/lib.sh + +req_env_var CI UPLDREL_IMAGE CIRRUS_BUILD_ID GOSRC RELEASE_GCPJSON RELEASE_GCPNAME RELEASE_GCPROJECT + +[[ "$CI" == "true" ]] || \ + die 56 "$0 must be run under Cirrus-CI to function" + +unset PR_OR_BRANCH BUCKET +if [[ -n "$CIRRUS_PR" ]] +then + PR_OR_BRANCH="pr$CIRRUS_PR" + BUCKET="libpod-pr-releases" +elif [[ -n "$CIRRUS_BRANCH" ]] +then + PR_OR_BRANCH="$CIRRUS_BRANCH" + BUCKET="libpod-$CIRRUS_BRANCH-releases" +else + die 1 "Expecting either \$CIRRUS_PR or \$CIRRUS_BRANCH to be non-empty." +fi + +# Functional local podman required for uploading a release +cd $GOSRC +[[ -n "$(type -P podman)" ]] || \ + make install || \ + die 57 "$0 requires working podman binary on path to function" + +TMPF=$(mktemp -p '' $(basename $0)_XXXX.json) +trap "rm -f $TMPF" EXIT +set +x +echo "$RELEASE_GCPJSON" > "$TMPF" +unset RELEASE_GCPJSON + +cd $GOSRC +for filename in $(ls -1 *.tar.gz *.zip) +do + echo "Running podman ... $UPLDREL_IMAGE $filename" + podman run -i --rm \ + -e "GCPNAME=$RELEASE_GCPNAME" \ + -e "GCPPROJECT=$RELEASE_GCPROJECT" \ + -e "GCPJSON_FILEPATH=$TMPF" \ + -e "REL_ARC_FILEPATH=/tmp/$filename" \ + -e "PR_OR_BRANCH=$PR_OR_BRANCH" \ + -e "BUCKET=$BUCKET" \ + --security-opt label=disable \ + -v "$TMPF:$TMPF:ro" \ + -v "$GOSRC/$filename:/tmp/$filename:ro" \ + $UPLDREL_IMAGE +done diff --git a/contrib/imgts/lib_entrypoint.sh b/contrib/imgts/lib_entrypoint.sh index 7b76c823feea..3f6b1112848d 100644 --- a/contrib/imgts/lib_entrypoint.sh +++ b/contrib/imgts/lib_entrypoint.sh @@ -35,10 +35,15 @@ req_env_var() { gcloud_init() { set +xe - TMPF=$(mktemp -p '' .$(uuidgen)XXXX) - trap "rm -f $TMPF" EXIT - echo "$GCPJSON" > $TMPF && \ - $GCLOUD auth activate-service-account --project "$GCPPROJECT" --key-file=$TMPF || \ + if [[ -n "$1" ]] && [[ -r "$1" ]] + then + TMPF="$1" + else + TMPF=$(mktemp -p '' .$(uuidgen)_XXXX.json) + trap "rm -f $TMPF &> /dev/null" EXIT + echo "$GCPJSON" > $TMPF + fi + $GCLOUD auth activate-service-account --project="$GCPPROJECT" --key-file="$TMPF" || \ die 5 FATAL auth - rm -f $TMPF + rm -f $TMPF &> /dev/null || true # ignore any read-only error } diff --git a/contrib/upldrel/Dockerfile b/contrib/upldrel/Dockerfile new file mode 100644 index 000000000000..54a58c52141c --- /dev/null +++ b/contrib/upldrel/Dockerfile @@ -0,0 +1,9 @@ +FROM quay.io/libpod/imgts:latest + +RUN yum -y update && \ + yum -y install unzip && \ + rpm -V unzip && \ + yum clean all + +COPY /contrib/upldrel/entrypoint.sh /usr/local/bin/entrypoint.sh +RUN chmod 755 /usr/local/bin/entrypoint.sh diff --git a/contrib/upldrel/README.md b/contrib/upldrel/README.md new file mode 100644 index 000000000000..41f5ffef0aa4 --- /dev/null +++ b/contrib/upldrel/README.md @@ -0,0 +1,9 @@ +![PODMAN logo](../../logo/podman-logo-source.svg) + +A container image for canonical-naming and uploading of +libpod and remote-client archives. Only intended to ever +be used by CI/CD, and depends heavily on an embedded +`release.txt` file produced by `make`. + +Build script: [../cirrus/build_release.sh](../cirrus/build_release.sh) +Upload script: [../cirrus/upload_release_archive.sh](../cirrus/upload_release_archive.sh) diff --git a/contrib/upldrel/entrypoint.sh b/contrib/upldrel/entrypoint.sh new file mode 100755 index 000000000000..985b828a014d --- /dev/null +++ b/contrib/upldrel/entrypoint.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +set -e + +source /usr/local/bin/lib_entrypoint.sh + +req_env_var GCPJSON_FILEPATH GCPNAME GCPPROJECT REL_ARC_FILEPATH PR_OR_BRANCH BUCKET + +[[ -r "$REL_ARC_FILEPATH" ]] || \ + die 2 ERROR Cannot read release archive file: "$REL_ARC_FILEPATH" + +[[ -r "$GCPJSON_FILEPATH" ]] || \ + die 3 ERROR Cannot read GCP credentials file: "$GCPJSON_FILEPATH" + +cd $TMPDIR +echo "Attempting to extract release.txt from tar or zip $REL_ARC_FILEPATH" +unset SFX +if tar xzf "$REL_ARC_FILEPATH" "./release.txt" +then + echo "It's a tarball" + SFX="tar.gz" +elif unzip "$REL_ARC_FILEPATH" release.txt +then + echo "It's a zip" + SFX="zip" +else + die 5 ERROR Could not extract release.txt from $REL_ARC_FILEPATH +fi + +echo "Parsing release.txt contents" +RELEASETXT=$( /dev/stderr + exit 2 +fi