From ae4a1908dd1eb038b12c81f1efc266bfc5f8ece6 Mon Sep 17 00:00:00 2001 From: Oliver Gugger Date: Mon, 21 Dec 2020 12:15:43 +0100 Subject: [PATCH 1/4] GitHub: fetch full history for release build To make sure git has the full history, including all annotated tags, we need to explicitly fetch everything on checkout. --- .github/workflows/release.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index c1aed51968..c2240598f4 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -10,9 +10,6 @@ defaults: shell: bash env: - DOCKER_REPO: lightninglabs - DOCKER_IMAGE: lnd - # If you change this value, please change it in the following files as well: # /.travis.yml # /Dockerfile @@ -27,6 +24,8 @@ jobs: steps: - name: git checkout uses: actions/checkout@v2 + with: + fetch-depth: 0 - name: go cache uses: actions/cache@v1 From 330f3c4dd0be89333e323ff11fbbd076294f59c5 Mon Sep 17 00:00:00 2001 From: Oliver Gugger Date: Mon, 21 Dec 2020 13:03:14 +0100 Subject: [PATCH 2/4] make: use --tags to describe commit --- Makefile | 2 +- scripts/release.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 4b10a2ffa8..1bffec5908 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ IOS_BUILD := $(IOS_BUILD_DIR)/Lndmobile.framework ANDROID_BUILD_DIR := $(MOBILE_BUILD_DIR)/android ANDROID_BUILD := $(ANDROID_BUILD_DIR)/Lndmobile.aar -COMMIT := $(shell git describe --abbrev=40 --dirty) +COMMIT := $(shell git describe --tags --dirty) COMMIT_HASH := $(shell git rev-parse HEAD) BTCD_COMMIT := $(shell cat go.mod | \ diff --git a/scripts/release.sh b/scripts/release.sh index e73ab3ba22..2d12949b04 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -37,7 +37,7 @@ function check_tag_correct() { fi # If a tag is specified, ensure that that tag is present and checked out. - if [[ $tag != $(git describe) ]]; then + if [[ $tag != $(git describe --tags) ]]; then red "tag $tag not checked out" exit 1 fi From 2686391c315afa1b0d60827c719d9fd9e8840e81 Mon Sep 17 00:00:00 2001 From: Oliver Gugger Date: Mon, 21 Dec 2020 17:33:37 +0100 Subject: [PATCH 3/4] scripts: package release artifacts deterministically A number of tricks need to be applied to the tar and zip commands respectively to make sure they always produce deterministic archives. This includes setting the file timestamps to a fixed date and setting specific ownership attributes (tar) or no attributes at all (zip). --- scripts/release.sh | 84 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 72 insertions(+), 12 deletions(-) diff --git a/scripts/release.sh b/scripts/release.sh index 2d12949b04..4b89bbac4e 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -13,6 +13,61 @@ LND_VERSION_REGEX="lnd version (.+) commit" PKG="github.com/lightningnetwork/lnd" PACKAGE=lnd +# Needed for setting file timestamps to get reproducible archives. +BUILD_DATE="2020-01-01 00:00:00" +BUILD_DATE_STAMP="202001010000.00" + +# reproducible_tar_gzip creates a reproducible tar.gz file of a directory. This +# includes setting all file timestamps and ownership settings uniformly. +function reproducible_tar_gzip() { + local dir=$1 + local tar_cmd=tar + + # MacOS has a version of BSD tar which doesn't support setting the --mtime + # flag. We need gnu-tar, or gtar for short to be installed for this script to + # work properly. + tar_version=$(tar --version) + if [[ ! "$tar_version" =~ "GNU tar" ]]; then + if ! command -v "gtar"; then + echo "GNU tar is required but cannot be found!" + echo "On MacOS please run 'brew install gnu-tar' to install gtar." + exit 1 + fi + + # We have gtar installed, use that instead. + tar_cmd=gtar + fi + + # Pin down the timestamp time zone. + export TZ=UTC + + find "${dir}" -print0 | LC_ALL=C sort -r -z | $tar_cmd \ + "--mtime=${BUILD_DATE}" --no-recursion --null --mode=u+rw,go+r-w,a+X \ + --owner=0 --group=0 --numeric-owner -c -T - | gzip -9n > "${dir}.tar.gz" + + rm -r "${dir}" +} + +# reproducible_zip creates a reproducible zip file of a directory. This +# includes setting all file timestamps. +function reproducible_zip() { + local dir=$1 + + # Pin down file name encoding and timestamp time zone. + export TZ=UTC + + # Set the date of each file in the directory that's about to be packaged to + # the same timestamp and make sure the same permissions are used everywhere. + chmod -R 0755 "${dir}" + touch -t "${BUILD_DATE_STAMP}" "${dir}" + find "${dir}" -print0 | LC_ALL=C sort -r -z | xargs -0r touch \ + -t "${BUILD_DATE_STAMP}" + + find "${dir}" | LC_ALL=C sort -r | zip -o -X -r -@ "${dir}.zip" + + rm -r "${dir}" +} + # green prints one line of green text (if the terminal supports it). function green() { echo -e "\e[0;32m${1}\e[0m" @@ -82,20 +137,27 @@ function build_release() { green " - Packaging vendor" go mod vendor - tar -czf vendor.tar.gz vendor + reproducible_tar_gzip vendor maindir=$PACKAGE-$tag mkdir -p $maindir + mv vendor.tar.gz "${maindir}/" - cp vendor.tar.gz $maindir/ - rm vendor.tar.gz - rm -r vendor + # Don't use tag in source directory, otherwise our file names get too long and + # tar starts to package them non-deterministically. + package_source="${PACKAGE}-source" - package_source="${maindir}/${PACKAGE}-source-${tag}.tar" - git archive -o "${package_source}" HEAD - gzip -f "${package_source}" >"${package_source}.gz" + # The git archive command doesn't support setting timestamps and file + # permissions. That's why we unpack the tar again, then use our reproducible + # method to create the final archive. + git archive -o "${maindir}/${package_source}.tar" HEAD cd "${maindir}" + mkdir -p ${package_source} + tar -xf "${package_source}.tar" -C ${package_source} + rm "${package_source}.tar" + reproducible_tar_gzip ${package_source} + mv "${package_source}.tar.gz" "${package_source}-$tag.tar.gz" for i in $sys; do os=$(echo $i | cut -f1 -d-) @@ -120,15 +182,13 @@ function build_release() { popd if [[ $os == "windows" ]]; then - zip -r "${dir}.zip" "${dir}" + reproducible_zip "${dir}" else - tar -cvzf "${dir}.tar.gz" "${dir}" + reproducible_tar_gzip "${dir}" fi - - rm -r "${dir}" done - shasum -a 256 * >manifest-$tag.txt + sha256sum * >manifest-$tag.txt } # usage prints the usage of the whole script. From 4c56f3cacba3b85a130b675116cd524b89450e18 Mon Sep 17 00:00:00 2001 From: Oliver Gugger Date: Fri, 8 Jan 2021 10:40:17 +0100 Subject: [PATCH 4/4] multi: add release helper docker file --- .github/workflows/main.yml | 1 + .github/workflows/release.yaml | 1 + .travis.yml | 1 + Dockerfile | 6 ++++++ Makefile | 8 ++++++++ dev.Dockerfile | 6 ++++++ docs/release.md | 21 +++++++++++++++++++-- make/builder.Dockerfile | 34 ++++++++++++++++++++++++++++++++++ make/release_flags.mk | 10 ++++++++++ 9 files changed, 86 insertions(+), 2 deletions(-) create mode 100644 make/builder.Dockerfile diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index abeed51201..c9a0538853 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,6 +23,7 @@ env: # /.travis.yml # /Dockerfile # /dev.Dockerfile + # /make/builder.Dockerfile # /.github/workflows/release.yml GO_VERSION: 1.15.6 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index c2240598f4..e40deccd6e 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -14,6 +14,7 @@ env: # /.travis.yml # /Dockerfile # /dev.Dockerfile + # /make/builder.Dockerfile # /.github/workflows/main.yml GO_VERSION: 1.15.6 diff --git a/.travis.yml b/.travis.yml index cd1c1cb729..eb4243f6cb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,6 +19,7 @@ go: # If you change this value, please change it in the following files as well: # /Dockerfile # /dev.Dockerfile + # /make/builder.Dockerfile # /.github/workflows/main.yml # /.github/workflows/release.yml - "1.15.6" diff --git a/Dockerfile b/Dockerfile index b544c0b84b..9fdcb4a3db 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,9 @@ +# If you change this value, please change it in the following files as well: +# /.travis.yml +# /dev.Dockerfile +# /make/builder.Dockerfile +# /.github/workflows/main.yml +# /.github/workflows/release.yml FROM golang:1.15.6-alpine as builder # Force Go to use the cgo based DNS resolver. This is required to ensure DNS diff --git a/Makefile b/Makefile index 1bffec5908..cfe1de3457 100644 --- a/Makefile +++ b/Makefile @@ -156,6 +156,14 @@ release: $(VERSION_CHECK) ./scripts/release.sh build-release "$(VERSION_TAG)" "$(BUILD_SYSTEM)" "$(RELEASE_TAGS)" "$(RELEASE_LDFLAGS)" +docker-release: + @$(call print, "Building release helper docker image.") + if [ "$(tag)" = "" ]; then echo "Must specify tag=!"; exit 1; fi + + docker build -t lnd-release-helper -f make/builder.Dockerfile make/ + $(DOCKER_RELEASE_HELPER) scripts/release.sh check-tag "$(VERSION_TAG)" + $(DOCKER_RELEASE_HELPER) scripts/release.sh build-release "$(VERSION_TAG)" "$(BUILD_SYSTEM)" "$(RELEASE_TAGS)" "$(RELEASE_LDFLAGS)" + scratch: build diff --git a/dev.Dockerfile b/dev.Dockerfile index cd3be28aee..f876febca4 100644 --- a/dev.Dockerfile +++ b/dev.Dockerfile @@ -1,3 +1,9 @@ +# If you change this value, please change it in the following files as well: +# /.travis.yml +# /Dockerfile +# /make/builder.Dockerfile +# /.github/workflows/main.yml +# /.github/workflows/release.yml FROM golang:1.15.6-alpine as builder LABEL maintainer="Olaoluwa Osuntokun " diff --git a/docs/release.md b/docs/release.md index e6591ea471..58d82d8455 100644 --- a/docs/release.md +++ b/docs/release.md @@ -10,7 +10,23 @@ utilize a work around needed until `go1.13.2`. ## Building a New Release -### macOS/Linux/Windows (WSL) +### MacOS + +The first requirement is to have [`docker`](https://www.docker.com/) +installed locally and running. The second requirement is to have `make` +installed. Everything else (including `golang`) is included in the release +helper image. + +To build a release, run the following commands: + +1. `git clone https://github.com/lightningnetwork/lnd.git` +2. `cd lnd` +3. `git checkout # is the name of the next release/tag` +4. `make docker-release tag=` + +Where `` is the name of the next release of `lnd`. + +### Linux/Windows (WSL) No prior set up is needed on Linux or macOS is required in order to build the release binaries. However, on Windows, the only way to build the release @@ -19,7 +35,8 @@ the release binaries following these steps: 1. `git clone https://github.com/lightningnetwork/lnd.git` 2. `cd lnd` -3. `make release tag= # is the name of the next release/tag` +3. `git checkout # is the name of the next release/tag` +4. `make release tag=` This will then create a directory of the form `lnd-` containing archives of the release binaries for each supported operating system and architecture, diff --git a/make/builder.Dockerfile b/make/builder.Dockerfile new file mode 100644 index 0000000000..08ba05c604 --- /dev/null +++ b/make/builder.Dockerfile @@ -0,0 +1,34 @@ +# If you change this value, please change it in the following files as well: +# /.travis.yml +# /Dockerfile +# /dev.Dockerfile +# /.github/workflows/main.yml +# /.github/workflows/release.yml +FROM golang:1.15.6-buster + +MAINTAINER Olaoluwa Osuntokun + +# Golang build related environment variables that are static and used for all +# architectures/OSes. +ENV GODEBUG netdns=cgo +ENV GO111MODULE=auto +ENV CGO_ENABLED=0 + +# Set up cache directories. Those will be mounted from the host system to speed +# up builds. If go isn't installed on the host system, those will fall back to +# temp directories during the build (see make/release_flags.mk). +ENV GOCACHE=/tmp/build/.cache +ENV GOMODCACHE=/tmp/build/.modcache + +RUN apt-get update && apt-get install -y \ + git \ + make \ + tar \ + zip \ + bash \ + && mkdir -p /tmp/build/lnd \ + && mkdir -p /tmp/build/.cache \ + && mkdir -p /tmp/build/.modcache \ + && chmod -R 777 /tmp/build/ + +WORKDIR /tmp/build/lnd diff --git a/make/release_flags.mk b/make/release_flags.mk index e0058133e6..8a00f0898e 100644 --- a/make/release_flags.mk +++ b/make/release_flags.mk @@ -1,6 +1,16 @@ VERSION_TAG = $(shell date +%Y%m%d)-01 VERSION_CHECK = @$(call print, "Building master with date version tag") +DOCKER_RELEASE_HELPER = docker run \ + -it \ + --rm \ + --user $(shell id -u):$(shell id -g) \ + -v $(shell pwd):/tmp/build/lnd \ + -v $(shell bash -c "go env GOCACHE || (mkdir -p /tmp/go-cache; echo /tmp/go-cache)"):/tmp/build/.cache \ + -v $(shell bash -c "go env GOMODCACHE || (mkdir -p /tmp/go-modcache; echo /tmp/go-modcache)"):/tmp/build/.modcache \ + -e SKIP_VERSION_CHECK \ + lnd-release-helper + BUILD_SYSTEM = darwin-amd64 \ dragonfly-amd64 \ freebsd-386 \