Merge pull request #15153 from SimonRichardson/content-addressable-deps

[JUJU-2627] Enable content addressable downloads

Makefile

@@ -386,9 +386,12 @@ test: run-tests
 # Can't make the length of the TMP dir too long or it hits socket name length issues.
 run-tests: musl-install-if-missing dqlite-install-if-missing
 ## run-tests: Run the unit tests
+	$(eval OS = $(shell go env GOOS))
+	$(eval ARCH = $(shell go env GOARCH))
+	$(eval BUILD_ARCH = $(subst ppc64el,ppc64le,${ARCH}))
 	$(eval TMP := $(shell mktemp -d $${TMPDIR:-/tmp}/jj-XXX))
 	$(eval TEST_PACKAGES := $(shell go list $(PROJECT)/... | sort | ([ -f "$(TEST_PACKAGE_LIST)" ] && comm -12 "$(TEST_PACKAGE_LIST)" - || cat) | grep -v $(PROJECT)$$ | grep -v $(PROJECT)/vendor/ | grep -v $(PROJECT)/acceptancetests/ | grep -v $(PROJECT)/generate/ | grep -v mocks))
-	@echo 'go test -mod=$(JUJU_GOMOD_MODE) -tags "$(BUILD_TAGS)" $(TEST_ARGS) $(CHECK_ARGS) -test.timeout=$(TEST_TIMEOUT) $$TEST_PACKAGES -check.v'
+	@echo 'GOOS=${OS} GOARCH=${BUILD_ARCH} go test -mod=$(JUJU_GOMOD_MODE) -tags "$(BUILD_TAGS)" $(TEST_ARGS) $(CHECK_ARGS) -test.timeout=$(TEST_TIMEOUT) $$TEST_PACKAGES -check.v'
 	@TMPDIR=$(TMP) \
 		PATH=${PATH}:${MUSL_BIN_PATH} \
 		CC="musl-gcc" \
@@ -397,22 +400,29 @@ run-tests: musl-install-if-missing dqlite-install-if-missing
 		CGO_LDFLAGS_ALLOW="(-Wl,-wrap,pthread_create)|(-Wl,-z,now)" \
 		LD_LIBRARY_PATH="${DQLITE_EXTRACTED_DEPS_ARCHIVE_PATH}" \
 		CGO_ENABLED=1 \
-		go test -mod=$(JUJU_GOMOD_MODE) -tags "$(BUILD_TAGS)" $(TEST_ARGS) $(CHECK_ARGS) -test.timeout=$(TEST_TIMEOUT) $(TEST_PACKAGES) -check.v
+		GOOS=${OS} \
+		GOARCH=${BUILD_ARCH} \
+		go test -v -mod=$(JUJU_GOMOD_MODE) -tags "$(BUILD_TAGS)" $(TEST_ARGS) $(CHECK_ARGS) -test.timeout=$(TEST_TIMEOUT) $(TEST_PACKAGES) -check.v
 	@rm -r $(TMP)
 
 run-go-tests: musl-install-if-missing dqlite-install-if-missing
 ## run-go-tests: Run the unit tests
+	$(eval OS = $(shell go env GOOS))
+	$(eval ARCH = $(shell go env GOARCH))
+	$(eval BUILD_ARCH = $(subst ppc64el,ppc64le,${ARCH}))
 	$(eval TEST_PACKAGES ?= "./...")
 	$(eval TEST_FILTER ?= "")
-	@echo 'go test -mod=$(JUJU_GOMOD_MODE) -tags "$(BUILD_TAGS)" $(TEST_ARGS) $(CHECK_ARGS) -test.timeout=$(TEST_TIMEOUT) $$TEST_PACKAGES -check.v -check.f $(TEST_FILTER)'
+	@echo 'GOOS=${OS} GOARCH=${BUILD_ARCH} go test -mod=$(JUJU_GOMOD_MODE) -tags "$(BUILD_TAGS)" $(TEST_ARGS) $(CHECK_ARGS) -test.timeout=$(TEST_TIMEOUT) $$TEST_PACKAGES -check.v -check.f $(TEST_FILTER)'
 	@PATH=${PATH}:${MUSL_BIN_PATH} \
 		CC="musl-gcc" \
 		CGO_CFLAGS="-I${DQLITE_EXTRACTED_DEPS_ARCHIVE_PATH}/include" \
 		CGO_LDFLAGS="-L${DQLITE_EXTRACTED_DEPS_ARCHIVE_PATH} -luv -lraft -ldqlite -llz4 -lsqlite3" \
 		CGO_LDFLAGS_ALLOW="(-Wl,-wrap,pthread_create)|(-Wl,-z,now)" \
 		LD_LIBRARY_PATH="${DQLITE_EXTRACTED_DEPS_ARCHIVE_PATH}" \
 		CGO_ENABLED=1 \
-		go test -mod=$(JUJU_GOMOD_MODE) -tags "$(BUILD_TAGS)" $(TEST_ARGS) $(CHECK_ARGS) -test.timeout=$(TEST_TIMEOUT) ${TEST_PACKAGES} -check.v -check.f $(TEST_FILTER)
+		GOOS=${OS} \
+		GOARCH=${BUILD_ARCH} \
+		go test -v -mod=$(JUJU_GOMOD_MODE) -tags "$(BUILD_TAGS)" $(TEST_ARGS) $(CHECK_ARGS) -test.timeout=$(TEST_TIMEOUT) ${TEST_PACKAGES} -check.v -check.f $(TEST_FILTER)
 
 .PHONY: install
 install: rebuild-schema go-install

scripts/dqlite/Makefile

@@ -1,6 +1,3 @@
-SHELL = bash
-.ONESHELL:
-
 PROJECT_DIR := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
 
 DQLITE_BUILD_MACHINE ?= $(shell uname -m)
@@ -10,10 +7,6 @@ DQLITE_ARCHIVE_DEPS_PATH=${PROJECT_DIR}/scripts/dqlite
 DQLITE_ARCHIVE_NAME=dqlite-deps
 DQLITE_ARCHIVE_PATH=${DQLITE_ARCHIVE_DEPS_PATH}/${DQLITE_ARCHIVE_NAME}.tar.bz2
 
-DQLITE_S3_BUCKET=s3://dqlite-static-libs
-DQLITE_S3_ARCHIVE_NAME=$(shell date -u +"%Y-%m-%d")-dqlite-deps-${DQLITE_BUILD_ARCH}.tar.bz2
-DQLITE_S3_ARCHIVE_PATH=${DQLITE_S3_BUCKET}/${DQLITE_S3_ARCHIVE_NAME}
-
 DQLITE_EXTRACTED_DEPS_PATH=${PROJECT_DIR}/_deps
 DQLITE_EXTRACTED_DEPS_ARCHIVE_PATH=${DQLITE_EXTRACTED_DEPS_PATH}/dqlite-deps-${DQLITE_BUILD_ARCH}
 
@@ -28,14 +21,8 @@ dqlite-build-lxd: ${DQLITE_ARCHIVE_PATH}
 dqlite-build:
 	@./scripts/dqlite/scripts/dqlite/build.sh
 
-# s3 puts here are without an ACL.
-# The bucket uses policies that allow GetObject to anyone,
-# and PutObject to select Juju team accounts.
-dqlite-deps-push: ${DQLITE_ARCHIVE_PATH}
-	@echo "DQLITE: Pushing ${DQLITE_S3_ARCHIVE_PATH} to s3"
-	aws s3 cp $< ${DQLITE_S3_ARCHIVE_PATH}
-	@echo "DQLITE: Pushing latest-dqlite-deps-${DQLITE_BUILD_ARCH}.tar.bz2 to s3"
-	aws s3 cp ${DQLITE_S3_ARCHIVE_PATH} ${DQLITE_S3_BUCKET}/latest-dqlite-deps-${DQLITE_BUILD_ARCH}.tar.bz2
+dqlite-push: ${DQLITE_ARCHIVE_PATH}
+	@./scripts/dqlite/scripts/dqlite/push.sh
 
 dqlite-install:
 	@./scripts/dqlite/scripts/dqlite/install.sh
@@ -49,16 +36,8 @@ musl-install:
 musl-install-if-missing:
 	@./scripts/dqlite/scripts/musl/install-if-missing.sh
 
-################################################################################
-# REPL
-#
-# Accessing the dqlite repl on a given controller for debugging purposes.
-################################################################################
-
-juju-dqlite-repl-deps-on-controller:
-	@juju exec -m controller --machine=0 'sudo which rlwrap &>/dev/null || sudo apt-get -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install rlwrap'
-	@juju exec -m controller --machine=0 'sudo which socat &>/dev/null || sudo apt-get -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install socat'
+repl-install:
+	@./scripts/dqlite/script/repl/install.sh
 
-juju-dqlite-repl: juju-dqlite-repl-deps-on-controller
-	@echo "[+] Connecting to REPL interface to controller machine 0"
-	@juju ssh -m controller 0 'sudo rlwrap -H /root/.dqlite_repl.history socat - /var/lib/juju/dqlite/juju.sock'
+repl: repl-install
+	@./script/dqlite/script/repl/repl.sh

scripts/dqlite/scripts/dqlite/build-lxd.sh

@@ -19,5 +19,6 @@ lxc file push $(dirname $0)/dqlite-build.sh ${BUILD_CONTAINER}/root/dqlite/dqlit
 
 lxc exec -t ${BUILD_CONTAINER} bash /root/dqlite/build.sh
 
-lxc file pull ${BUILD_CONTAINER}/root/build/juju-dqlite-static-lib-deps.tar.bz2 ${EXTRACTED_DEPS_PATH}/dqlite-deps-${BUILD_ARCH}.tar.bz2
+mkdir -p ${ARCHIVE_DEPS_PATH}
+lxc file pull ${BUILD_CONTAINER}/root/_build/dqlite-deps.tar.bz2 ${ARCHIVE_PATH}
 lxc delete -f ${BUILD_CONTAINER}

scripts/dqlite/scripts/dqlite/dqlite-build.sh

@@ -22,8 +22,8 @@ build() {
     sudo apt-get -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install \
         gcc automake libtool make gettext autopoint pkg-config tclsh tcl libsqlite3-dev wget git
 
-    mkdir -p build
-    cd build
+    mkdir -p "${ARCHIVE_DEPS_PATH}"
+    cd "${ARCHIVE_DEPS_PATH}"
 
     # Checkout and build musl. We will use this to avoid depending
     # on the hosts libc.
@@ -160,5 +160,5 @@ build() {
     echo "sqlite ${TAG_SQLITE}" >> juju-dqlite-static-lib-deps/BOM
     echo "dqlite ${TAG_DQLITE}" >> juju-dqlite-static-lib-deps/BOM
 
-    tar cjvf juju-dqlite-static-lib-deps.tar.bz2 juju-dqlite-static-lib-deps
+    tar cjvf ${ARCHIVE_PATH} juju-dqlite-static-lib-deps
 }

scripts/dqlite/scripts/dqlite/dqlite-install.sh

@@ -14,13 +14,35 @@ sha() {
 	esac
 }
 
-FILE="${EXTRACTED_DEPS_PATH}/latest-dqlite-deps-${BUILD_ARCH}.tar.bz2"
+FILE="${EXTRACTED_DEPS_PATH}/dqlite-deps-${BUILD_ARCH}.tar.bz2"
+
+retrieve() {
+	local filenames sha
+
+	sha=${1}
+
+	filenames=( "${sha}.tar.bz2" "latest-dqlite-deps-${BUILD_ARCH}.tar.bz2" )
+	for name in "${filenames[@]}"; do
+		echo "Retrieving ${name}"
+		curl --fail -o ${FILE} -s https://dqlite-static-libs.s3.amazonaws.com/${name} && return || {
+			echo " + Failed to retrieve ${name}";
+			rm -f ${FILE} || true;
+			true;
+		}
+	done
+}
 
 install() {
 	mkdir -p ${EXTRACTED_DEPS_PATH}
-	curl -o ${FILE} -s https://dqlite-static-libs.s3.amazonaws.com/latest-dqlite-deps-${BUILD_ARCH}.tar.bz2
+	SHA=$(sha)
+	retrieve ${SHA}
+	if [ ! -f ${FILE} ]; then
+		echo "Failed to retrieve dqlite static libs"
+		exit 1
+	fi
+
     SUM=$(sha256sum ${FILE} | awk '{print $1}')
-    if [ "${SUM}" != $(sha) ]; then
+    if [ "${SUM}" != ${SHA} ]; then
         echo "sha256sum mismatch (${SUM}, expected $(sha))"
         exit 1
     fi

scripts/dqlite/scripts/dqlite/push.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -e
+
+source "$(dirname $0)/../env.sh"
+
+# s3 puts here are without an ACL.
+# The bucket uses policies that allow GetObject to anyone,
+# and PutObject to select Juju team accounts.
+
+echo "Pushing ${S3_ARCHIVE_PATH} to s3"
+aws s3 cp ${ARCHIVE_PATH} ${S3_ARCHIVE_PATH}
+
+SUM=$(sha256sum ${FILE} | awk '{print $1}')
+echo "Pushing ${SUM}.tar.bz2 to s3"
+aws s3 cp ${S3_ARCHIVE_PATH} ${S3_BUCKET}/${SUM}.tar.bz2
+
+# This is the old way and is deprecated.
+echo "Pushing latest-dqlite-deps-${BUILD_ARCH}.tar.bz2 to s3"
+aws s3 cp ${S3_ARCHIVE_PATH} ${S3_BUCKET}/latest-dqlite-deps-${BUILD_ARCH}.tar.bz2

scripts/dqlite/scripts/env.sh

@@ -32,3 +32,11 @@ TAG_LIBLZ4=v1.9.4
 TAG_RAFT=v0.16.0
 TAG_SQLITE=version-3.40.0
 TAG_DQLITE=v1.12.0
+
+S3_BUCKET=s3://dqlite-static-libs
+S3_ARCHIVE_NAME=$(date -u +"%Y-%m-%d")-dqlite-deps-${BUILD_ARCH}.tar.bz2
+S3_ARCHIVE_PATH=${S3_BUCKET}/${S3_ARCHIVE_NAME}
+
+ARCHIVE_DEPS_PATH=${PROJECT_DIR}/_build
+ARCHIVE_NAME=dqlite-deps
+ARCHIVE_PATH=${ARCHIVE_DEPS_PATH}/${ARCHIVE_NAME}.tar.bz2

scripts/dqlite/scripts/repl/install.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -e
+
+MACHINE=${MACHINE:-0}
+
+juju exec -m controller --machine=${MACHINE} 'sudo which rlwrap &>/dev/null || sudo apt-get -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install rlwrap socat'

scripts/dqlite/scripts/repl/repl.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -e
+
+MACHINE=${MACHINE:-0}
+
+juju ssh -m controller ${MACHINE} 'sudo rlwrap -H /root/.dqlite_repl.history socat - /var/lib/juju/dqlite/juju.sock'