From f70678f8631b1bddbcbe4eca7c7185577d86b52c Mon Sep 17 00:00:00 2001 From: Boris Nagaev Date: Fri, 26 Sep 2025 00:45:57 -0300 Subject: [PATCH 01/16] make: add target "docker-release" "make docker-release tag=..." produces the same binaries regardless on the platform and the installed Go version, because it uses Docker. --- Makefile | 23 +++++++++++++++++++++-- release.Dockerfile | 23 +++++++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 release.Dockerfile diff --git a/Makefile b/Makefile index 33096d3f0..426a8a3ef 100644 --- a/Makefile +++ b/Makefile @@ -38,10 +38,18 @@ endif DOCKER_TOOLS = docker run \ --rm \ - -v $(shell bash -c "go env GOCACHE || (mkdir -p /tmp/go-cache; echo /tmp/go-cache)"):/tmp/build/.cache \ - -v $(shell bash -c "go env GOMODCACHE || (mkdir -p /tmp/go-modcache; echo /tmp/go-modcache)"):/tmp/build/.modcache \ + -v $(shell bash -c "go env GOCACHE 2>/dev/null || (mkdir -p /tmp/go-cache; echo /tmp/go-cache)"):/tmp/build/.cache \ + -v $(shell bash -c "go env GOMODCACHE 2>/dev/null || (mkdir -p /tmp/go-modcache; echo /tmp/go-modcache)"):/tmp/build/.modcache \ -v $$(pwd):/build loop-tools +DOCKER_RELEASE_BUILDER = docker run \ + --rm \ + -v $(shell bash -c "go env GOCACHE 2>/dev/null || (mkdir -p /tmp/go-cache; echo /tmp/go-cache)"):/tmp/build/.cache \ + -v $(shell bash -c "go env GOMODCACHE 2>/dev/null || (mkdir -p /tmp/go-modcache; echo /tmp/go-modcache)"):/tmp/build/.modcache \ + -v $$(pwd):/repo \ + -e LOOPBUILDSYS='$(buildsys)' \ + loop-release-builder + GREEN := "\\033[0;32m" NC := "\\033[0m" define print @@ -71,6 +79,13 @@ install: $(GOINSTALL) -tags="${tags}" $(LDFLAGS) $(PKG)/cmd/loop $(GOINSTALL) -tags="${tags}" $(LDFLAGS) $(PKG)/cmd/loopd +# docker-release: Same as release.sh but within a docker container to support +# reproducible builds on any platform. +docker-release: docker-release-builder + @$(call print, "Building release binaries in docker.") + @if [ "$(tag)" = "" ]; then echo "Must specify tag=!"; exit 1; fi + $(DOCKER_RELEASE_BUILDER) bash release.sh $(tag) + rpc: @$(call print, "Compiling protos.") cd ./swapserverrpc; ./gen_protos_docker.sh @@ -131,6 +146,10 @@ docker-tools: @$(call print, "Building tools docker image.") docker build -q -t loop-tools $(TOOLS_DIR) +docker-release-builder: + @$(call print, "Building release builder docker image.") + docker build -q -t loop-release-builder -f release.Dockerfile . + mod-tidy: @$(call print, "Tidying modules.") $(GOMOD) tidy diff --git a/release.Dockerfile b/release.Dockerfile new file mode 100644 index 000000000..a42588934 --- /dev/null +++ b/release.Dockerfile @@ -0,0 +1,23 @@ +FROM golang:1.24.6 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + git ca-certificates zip gpg && rm -rf /var/lib/apt/lists/* + +# Add GPG key of Alex Bosworth to verify release tag signature. +RUN gpg --keyserver keys.openpgp.org \ + --recv-keys DE23E73BFA8A0AD5587D2FCDE80D2F3F311FD87E + +# Mark the repo directory safe for git. User ID may be different inside +# Docker and Git might refuse to work without this setting. +RUN git config --global --add safe.directory /repo +RUN git config --global --add safe.directory /repo/.git + +# Set GO build time environment variables. +ENV GOCACHE=/tmp/build/.cache \ + GOMODCACHE=/tmp/build/.modcache + +# Create directories to which host's Go caches are mounted. +RUN mkdir -p /tmp/build/.cache /tmp/build/.modcache \ + && chmod -R 777 /tmp/build/ + +WORKDIR /repo From 8a357059f00bdcd601643249ff76880299c2a614 Mon Sep 17 00:00:00 2001 From: Boris Nagaev Date: Fri, 26 Sep 2025 00:32:29 -0300 Subject: [PATCH 02/16] make: fix green prints Previous version produces escapped sequnces as text (Debian 12). --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 426a8a3ef..505ceba1e 100644 --- a/Makefile +++ b/Makefile @@ -50,10 +50,10 @@ DOCKER_RELEASE_BUILDER = docker run \ -e LOOPBUILDSYS='$(buildsys)' \ loop-release-builder -GREEN := "\\033[0;32m" -NC := "\\033[0m" +GREEN=\033[0;32m +NC=\033[0m define print - echo $(GREEN)$1$(NC) + @printf '%b%s%b\n' '${GREEN}' $1 '${NC}' endef # ============ From 3150aa56412c85c3333a98b3e46c80fbcb5ede82 Mon Sep 17 00:00:00 2001 From: Boris Nagaev Date: Fri, 26 Sep 2025 00:36:57 -0300 Subject: [PATCH 03/16] release.sh: fix building untagged commits To test "make docker-release" it is convenient to pass a regular commit to it (an output of `git describe`). In this commit release.sh is changed to detect such "tags" and skip tag signature verification for it. Also if release.sh is used without an argument, a unique directory name is generated from the current time. --- release.sh | 40 +++++++++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/release.sh b/release.sh index 94a7d11c1..af22e46e7 100755 --- a/release.sh +++ b/release.sh @@ -10,12 +10,16 @@ # Exit on errors. set -e -# If no tag specified, use date + version otherwise use tag. -if [[ $1x = x ]]; then - DATE=`date +%Y%m%d` - VERSION="01" - TAG=$DATE-$VERSION -else +TAG='' + +check_tag() { + # If no tag specified, use date + version otherwise use tag. + if [[ $1x = x ]]; then + TAG=`date +%Y%m%d-%H%M%S` + + return + fi + TAG=$1 # If a tag is specified, ensure that tag is present and checked out. @@ -24,8 +28,24 @@ else exit 1 fi - # Verify that it is signed. - if ! git verify-tag $TAG; then + # Verify that it is signed if it is a real tag. If the tag looks like the + # output of "git describe" for an untagged commit, skip verification. + # The pattern is: --g + # Example: "v0.31.2-beta-122-g8c6b73c". + if [[ $TAG =~ -[0-9]+-g([0-9a-f]+)$ ]]; then + # This looks like a "git describe" output. Make sure the hash + # described is a prefix of the current commit. + DESCRIBED_HASH=${BASH_REMATCH[1]} + CURRENT_HASH=$(git rev-parse HEAD) + if [[ $CURRENT_HASH != $DESCRIBED_HASH* ]]; then + echo "Described hash $DESCRIBED_HASH is not a prefix of current commit $CURRENT_HASH" + exit 1 + fi + + return + fi + + if ! git verify-tag $TAG; then echo "tag $TAG not signed" exit 1 fi @@ -52,7 +72,9 @@ else echo "malformed loop version output" exit 1 fi -fi +} + +check_tag $1 go mod vendor tar -cvzf vendor.tar.gz vendor From 0e30dc92c2c1a7c0f1a848f347550ce684711d1c Mon Sep 17 00:00:00 2001 From: Boris Nagaev Date: Fri, 26 Sep 2025 01:11:56 -0300 Subject: [PATCH 04/16] release.sh: git clone before building Create a temporary directory inside loop/ directory and make a clone of Git repo to that dir. This ensures that the version built is exactly what is committed without any unstaged files and not dirty. --- release.sh | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/release.sh b/release.sh index af22e46e7..001a69d49 100755 --- a/release.sh +++ b/release.sh @@ -10,6 +10,16 @@ # Exit on errors. set -e +# Get the directory of the script +SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + +# Checkout the repo to a subdir to clean from clean from unstaged files and +# build exactly what is committed. +BUILD_DIR="${SCRIPT_DIR}/tmp-build-$(date +%Y%m%d-%H%M%S)" +mkdir -p $BUILD_DIR +cd $BUILD_DIR +git clone --tags "$SCRIPT_DIR" . + TAG='' check_tag() { @@ -80,25 +90,21 @@ go mod vendor tar -cvzf vendor.tar.gz vendor PACKAGE=loop -MAINDIR=$PACKAGE-$TAG -mkdir -p $MAINDIR +ARTIFACTS_DIR="${SCRIPT_DIR}/${PACKAGE}-${TAG}" +mkdir -p $ARTIFACTS_DIR -cp vendor.tar.gz $MAINDIR/ +cp vendor.tar.gz $ARTIFACTS_DIR/ rm vendor.tar.gz rm -r vendor -PACKAGESRC="$MAINDIR/$PACKAGE-source-$TAG.tar" +PACKAGESRC="${ARTIFACTS_DIR}/${PACKAGE}-source-${TAG}.tar" git archive -o $PACKAGESRC HEAD gzip -f $PACKAGESRC > "$PACKAGESRC.gz" -cd $MAINDIR - # If LOOPBUILDSYS is set the default list is ignored. Useful to release # for a subset of systems/architectures. SYS=${LOOPBUILDSYS:-"windows-amd64 linux-386 linux-amd64 linux-armv6 linux-armv7 linux-arm64 darwin-arm64 darwin-amd64 freebsd-amd64 freebsd-arm"} -# Use the first element of $GOPATH in the case where GOPATH is a list -# (something that is totally allowed). PKG="github.com/lightninglabs/loop" COMMIT=$(git describe --abbrev=40 --dirty) COMMITFLAGS="-X $PKG/build.Commit=$COMMIT" @@ -126,12 +132,14 @@ for i in $SYS; do cd .. if [[ $OS = "windows" ]]; then - zip -r $PACKAGE-$i-$TAG.zip $PACKAGE-$i-$TAG + zip -r "${ARTIFACTS_DIR}/${PACKAGE}-${i}-${TAG}.zip" "${PACKAGE}-${i}-${TAG}" else - tar -cvzf $PACKAGE-$i-$TAG.tar.gz $PACKAGE-$i-$TAG + tar -cvzf "${ARTIFACTS_DIR}/${PACKAGE}-${i}-${TAG}.tar.gz" "${PACKAGE}-${i}-${TAG}" fi rm -r $PACKAGE-$i-$TAG done +cd "$ARTIFACTS_DIR" + shasum -a 256 * > manifest-$TAG.txt From 9e8cde5556589b66a18a7463b81d2b76e1fc2ba6 Mon Sep 17 00:00:00 2001 From: Boris Nagaev Date: Fri, 26 Sep 2025 13:58:50 -0300 Subject: [PATCH 05/16] release.sh: make reproducible .zip and .tar.gz Took the code for reproducible packing from LND. --- release.sh | 134 ++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 113 insertions(+), 21 deletions(-) diff --git a/release.sh b/release.sh index 001a69d49..f4cad1644 100755 --- a/release.sh +++ b/release.sh @@ -16,9 +16,16 @@ SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) # Checkout the repo to a subdir to clean from clean from unstaged files and # build exactly what is committed. BUILD_DIR="${SCRIPT_DIR}/tmp-build-$(date +%Y%m%d-%H%M%S)" -mkdir -p $BUILD_DIR -cd $BUILD_DIR -git clone --tags "$SCRIPT_DIR" . + +# green prints one line of green text (if the terminal supports it). +function green() { + echo -e "\e[0;32m${1}\e[0m" +} + +# red prints one line of red text (if the terminal supports it). +function red() { + echo -e "\e[0;31m${1}\e[0m" +} TAG='' @@ -26,6 +33,7 @@ check_tag() { # If no tag specified, use date + version otherwise use tag. if [[ $1x = x ]]; then TAG=`date +%Y%m%d-%H%M%S` + green "No tag specified, using ${TAG} as tag" return fi @@ -34,7 +42,7 @@ check_tag() { # If a tag is specified, ensure that tag is present and checked out. if [[ $TAG != $(git describe) ]]; then - echo "tag $TAG not checked out" + red "tag $TAG not checked out" exit 1 fi @@ -48,7 +56,7 @@ check_tag() { DESCRIBED_HASH=${BASH_REMATCH[1]} CURRENT_HASH=$(git rev-parse HEAD) if [[ $CURRENT_HASH != $DESCRIBED_HASH* ]]; then - echo "Described hash $DESCRIBED_HASH is not a prefix of current commit $CURRENT_HASH" + red "Described hash $DESCRIBED_HASH is not a prefix of current commit $CURRENT_HASH" exit 1 fi @@ -56,7 +64,7 @@ check_tag() { fi if ! git verify-tag $TAG; then - echo "tag $TAG not signed" + red "tag $TAG not signed" exit 1 fi @@ -75,31 +83,111 @@ check_tag() { # Match git tag with loop version. if [[ $TAG != $LOOP_VERSION ]]; then - echo "loop version $LOOP_VERSION does not match tag $TAG" + red "loop version $LOOP_VERSION does not match tag $TAG" exit 1 fi else - echo "malformed loop version output" + red "malformed loop version output" exit 1 fi } -check_tag $1 +# Needed for setting file timestamps to get reproducible archives. +BUILD_DATE="2020-01-01 00:00:00" +BUILD_DATE_STAMP="202001010000.00" + +# reproducible_tar_gzip creates a reproducible tar.gz file of a directory. This +# includes setting all file timestamps and ownership settings uniformly. +function reproducible_tar_gzip() { + local dir=$1 + local dst=$2 + local tar_cmd=tar + + # MacOS has a version of BSD tar which doesn't support setting the --mtime + # flag. We need gnu-tar, or gtar for short to be installed for this script to + # work properly. + tar_version=$(tar --version) + if [[ ! "$tar_version" =~ "GNU tar" ]]; then + if ! command -v "gtar"; then + red "GNU tar is required but cannot be found!" + red "On MacOS please run 'brew install gnu-tar' to install gtar." + exit 1 + fi -go mod vendor -tar -cvzf vendor.tar.gz vendor + # We have gtar installed, use that instead. + tar_cmd=gtar + fi + + # Pin down the timestamp time zone. + export TZ=UTC + + find "${dir}" -print0 | LC_ALL=C sort -r -z | $tar_cmd \ + "--mtime=${BUILD_DATE}" --no-recursion --null --mode=u+rw,go+r-w,a+X \ + --owner=0 --group=0 --numeric-owner -c -T - | gzip -9n > "$dst" +} + +# reproducible_zip creates a reproducible zip file of a directory. This +# includes setting all file timestamps. +function reproducible_zip() { + local dir=$1 + local dst=$2 + + # Pin down file name encoding and timestamp time zone. + export TZ=UTC + + # Set the date of each file in the directory that's about to be packaged to + # the same timestamp and make sure the same permissions are used everywhere. + chmod -R 0755 "${dir}" + touch -t "${BUILD_DATE_STAMP}" "${dir}" + find "${dir}" -print0 | LC_ALL=C sort -r -z | xargs -0r touch \ + -t "${BUILD_DATE_STAMP}" + + find "${dir}" | LC_ALL=C sort -r | zip -o -X -r -@ "$dst" +} + +################## +# Start Building # +################## + +if [ -d "$BUILD_DIR" ]; then + red "Build directory ${BUILD_DIR} already exists!" + exit 1 +fi + +green " - Cloning to subdir ${BUILD_DIR} to get clean Git" +mkdir -p "$BUILD_DIR" +cd "$BUILD_DIR" +git clone --tags "$SCRIPT_DIR" . + +green " - Checking tag $1" +check_tag $1 PACKAGE=loop ARTIFACTS_DIR="${SCRIPT_DIR}/${PACKAGE}-${TAG}" -mkdir -p $ARTIFACTS_DIR - -cp vendor.tar.gz $ARTIFACTS_DIR/ -rm vendor.tar.gz +if [ -d "$ARTIFACTS_DIR" ]; then + red "artifacts directory ${ARTIFACTS_DIR} already exists!" + exit 1 +fi +green " - Creating artifacts directory ${ARTIFACTS_DIR}" +mkdir -p "$ARTIFACTS_DIR" +green " - Packaging vendor to ${ARTIFACTS_DIR}/vendor.tar.gz" +go mod vendor +reproducible_tar_gzip vendor "${ARTIFACTS_DIR}/vendor.tar.gz" rm -r vendor -PACKAGESRC="${ARTIFACTS_DIR}/${PACKAGE}-source-${TAG}.tar" -git archive -o $PACKAGESRC HEAD -gzip -f $PACKAGESRC > "$PACKAGESRC.gz" +PACKAGESRC="${ARTIFACTS_DIR}/${PACKAGE}-source-${TAG}.tar.gz" +green " - Creating source archive ${PACKAGESRC}" +TMPSOURCETAR="${ARTIFACTS_DIR}/tmp-${PACKAGE}-source-${TAG}.tar" +PKGSRC="${PACKAGE}-source" +git archive -o "$TMPSOURCETAR" HEAD +cd "$ARTIFACTS_DIR" +mkdir "$PKGSRC" +tar -xf "$TMPSOURCETAR" -C "$PKGSRC" +cd "$PKGSRC" +reproducible_tar_gzip . "$PACKAGESRC" +cd .. +rm -r "$PKGSRC" +rm "$TMPSOURCETAR" # If LOOPBUILDSYS is set the default list is ignored. Useful to release # for a subset of systems/architectures. @@ -109,6 +197,7 @@ PKG="github.com/lightninglabs/loop" COMMIT=$(git describe --abbrev=40 --dirty) COMMITFLAGS="-X $PKG/build.Commit=$COMMIT" +cd "$BUILD_DIR" for i in $SYS; do OS=$(echo $i | cut -f1 -d-) ARCH=$(echo $i | cut -f2 -d-) @@ -125,16 +214,18 @@ for i in $SYS; do mkdir $PACKAGE-$i-$TAG cd $PACKAGE-$i-$TAG - echo "Building:" $OS $ARCH $ARM + green "- Building: $OS $ARCH $ARM" for bin in loop loopd; do env CGO_ENABLED=0 GOOS=$OS GOARCH=$ARCH GOARM=$ARM go build -v -ldflags "$COMMITFLAGS" "github.com/lightninglabs/loop/cmd/$bin" done cd .. if [[ $OS = "windows" ]]; then - zip -r "${ARTIFACTS_DIR}/${PACKAGE}-${i}-${TAG}.zip" "${PACKAGE}-${i}-${TAG}" + green "- Producing ZIP file ${ARTIFACTS_DIR}/${PACKAGE}-${i}-${TAG}.zip" + reproducible_zip "${PACKAGE}-${i}-${TAG}" "${ARTIFACTS_DIR}/${PACKAGE}-${i}-${TAG}.zip" else - tar -cvzf "${ARTIFACTS_DIR}/${PACKAGE}-${i}-${TAG}.tar.gz" "${PACKAGE}-${i}-${TAG}" + green "- Producing TAR.GZ file ${ARTIFACTS_DIR}/${PACKAGE}-${i}-${TAG}.tar.gz" + reproducible_tar_gzip "${PACKAGE}-${i}-${TAG}" "${ARTIFACTS_DIR}/${PACKAGE}-${i}-${TAG}.tar.gz" fi rm -r $PACKAGE-$i-$TAG @@ -142,4 +233,5 @@ done cd "$ARTIFACTS_DIR" +green "- Producing manifest-$TAG.txt" shasum -a 256 * > manifest-$TAG.txt From 9c138039d2593837c895673f87a294f78896ac59 Mon Sep 17 00:00:00 2001 From: Boris Nagaev Date: Fri, 26 Sep 2025 15:54:46 -0300 Subject: [PATCH 06/16] release.sh: build Go in reproducible way Added -trimpath to remove absolute paths from the binary. --- release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release.sh b/release.sh index f4cad1644..3f36ecc8f 100755 --- a/release.sh +++ b/release.sh @@ -216,7 +216,7 @@ for i in $SYS; do green "- Building: $OS $ARCH $ARM" for bin in loop loopd; do - env CGO_ENABLED=0 GOOS=$OS GOARCH=$ARCH GOARM=$ARM go build -v -ldflags "$COMMITFLAGS" "github.com/lightninglabs/loop/cmd/$bin" + env CGO_ENABLED=0 GOOS=$OS GOARCH=$ARCH GOARM=$ARM go build -v -trimpath -ldflags "$COMMITFLAGS" "github.com/lightninglabs/loop/cmd/$bin" done cd .. From a889fbecb31d2a8bbc8d8bcca6d8ac8584ddf6d2 Mon Sep 17 00:00:00 2001 From: Boris Nagaev Date: Sat, 27 Sep 2025 22:04:06 -0300 Subject: [PATCH 07/16] release.sh: collect artifacts in intermediate dir Collect artifacts in "tmp-..." instead of final place. If build fails in the middle, we don't want to leave the artifacts directory with partial result. Also remove the build directory (with Git clone) in the end. --- release.sh | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/release.sh b/release.sh index 3f36ecc8f..343b7f7f2 100755 --- a/release.sh +++ b/release.sh @@ -163,11 +163,16 @@ green " - Checking tag $1" check_tag $1 PACKAGE=loop -ARTIFACTS_DIR="${SCRIPT_DIR}/${PACKAGE}-${TAG}" +FINAL_ARTIFACTS_DIR="${SCRIPT_DIR}/${PACKAGE}-${TAG}" +ARTIFACTS_DIR="${SCRIPT_DIR}/tmp-${PACKAGE}-${TAG}-$(date +%Y%m%d-%H%M%S)" if [ -d "$ARTIFACTS_DIR" ]; then red "artifacts directory ${ARTIFACTS_DIR} already exists!" exit 1 fi +if [ -d "$FINAL_ARTIFACTS_DIR" ]; then + red "final artifacts directory ${FINAL_ARTIFACTS_DIR} already exists!" + exit 1 +fi green " - Creating artifacts directory ${ARTIFACTS_DIR}" mkdir -p "$ARTIFACTS_DIR" green " - Packaging vendor to ${ARTIFACTS_DIR}/vendor.tar.gz" @@ -232,6 +237,13 @@ for i in $SYS; do done cd "$ARTIFACTS_DIR" - green "- Producing manifest-$TAG.txt" shasum -a 256 * > manifest-$TAG.txt +cd .. + +green "- Moving artifacts directory to final place ${FINAL_ARTIFACTS_DIR}" +mv "$ARTIFACTS_DIR" "$FINAL_ARTIFACTS_DIR" + +green "- Removing the subdir used for building ${BUILD_DIR}" + +rm -r "$BUILD_DIR" From ec53b0d2d445edf6f6384adaf48ea6613ec43334 Mon Sep 17 00:00:00 2001 From: Boris Nagaev Date: Sun, 28 Sep 2025 11:52:17 -0300 Subject: [PATCH 08/16] release.sh: make the list of tags reproducible When doing git-clone to a subdir, do not keep tags, since there can be local tags affecting `git describe` and buildvcs info. Instead pull the tags from upstream. --- release.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/release.sh b/release.sh index 343b7f7f2..37cdaee3a 100755 --- a/release.sh +++ b/release.sh @@ -157,7 +157,19 @@ fi green " - Cloning to subdir ${BUILD_DIR} to get clean Git" mkdir -p "$BUILD_DIR" cd "$BUILD_DIR" -git clone --tags "$SCRIPT_DIR" . +git clone --no-tags "$SCRIPT_DIR" . + +# It is cloned without tags from the local dir and tags are pulled later +# from the upstream to make sure we have the same set of tags. Otherwise +# we can endup with different `git describe` and buildvcs info depending +# on local tags. +green " - Pulling tags from upstream" +git pull --tags https://github.com/lightninglabs/loop + +# The cloned Git repo may be on wrong branch and commit. +commit=$(git --git-dir "${SCRIPT_DIR}/.git" rev-parse HEAD) +green " - Checkout commit ${commit} in ${BUILD_DIR}" +git checkout -b build-branch "$commit" green " - Checking tag $1" check_tag $1 From 2f3edb913b7be68977d8684cede45c8d930fce1f Mon Sep 17 00:00:00 2001 From: Boris Nagaev Date: Sun, 28 Sep 2025 12:20:03 -0300 Subject: [PATCH 09/16] release.sh: print sha256 hash of manifest file --- release.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/release.sh b/release.sh index 37cdaee3a..dd3d12e55 100755 --- a/release.sh +++ b/release.sh @@ -251,6 +251,7 @@ done cd "$ARTIFACTS_DIR" green "- Producing manifest-$TAG.txt" shasum -a 256 * > manifest-$TAG.txt +shasum -a 256 manifest-$TAG.txt cd .. green "- Moving artifacts directory to final place ${FINAL_ARTIFACTS_DIR}" From 35d0fa26acaf71ae3fe84454e1fd0ad087d65de4 Mon Sep 17 00:00:00 2001 From: Boris Nagaev Date: Fri, 26 Sep 2025 15:36:33 -0300 Subject: [PATCH 10/16] docs: describe "make docker-release" --- README.md | 5 +++ docs/release.md | 81 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 docs/release.md diff --git a/README.md b/README.md index 912df86f3..1f9938fa2 100644 --- a/README.md +++ b/README.md @@ -125,3 +125,8 @@ git clone https://github.com/lightninglabs/loop.git cd loop/cmd go install ./... ``` + +## Reproducible builds + +If you want to build release files yourself, follow +[the guide](./docs/release.md). diff --git a/docs/release.md b/docs/release.md new file mode 100644 index 000000000..5cad5a08a --- /dev/null +++ b/docs/release.md @@ -0,0 +1,81 @@ +# Reproducible Builds + +## Building with Docker + +To create a Loop release with binaries that are identical to an official +release, run the following command (available since release `v0.31.3-beta`): + +```bash +make docker-release tag= +``` + +This command will create a directory named `loop-` containing +the source archive, vendored dependencies, and the built binaries packaged in +`.tar.gz` or `.zip` format. It also creates a manifest file with `SHA-256` +checksums for all release files. + +For example: + +```bash +make docker-release tag=v0.31.3-beta +``` + +This will create the release artifacts in the `loop-v0.31.3-beta` directory. + +If you want to build from an untagged commit, first check it out, then use the +output of `git describe` as the tag: + +```bash +git describe +# v0.31.2-beta-128-gfa80357 + +make docker-release tag=v0.31.2-beta-128-gfa80357 +``` + +You can filter the target platforms to speed up the build process. For example, +to build only for `linux-amd64`: + +```bash +make docker-release buildsys=linux-amd64 tag=v0.31.3-beta +``` + +Or for multiple platforms: + +```bash +make docker-release buildsys='linux-amd64 windows-amd64' tag=v0.31.3-beta +``` + +Note: inside Docker the current directory is mapped as `/repo` and it might +mention `/repo` as parts of file paths. + +## Building on the Host + +You can also build a release on your host system without Docker. You will need +to install the Go version specified in the `go.mod` file, as well as a few +other tools: + +```bash +sudo apt-get install build-essential git make zip perl gpg +``` + +Add GPG key of Alex Bosworth to verify release tag signature: +```bash +gpg --keyserver keys.openpgp.org --recv-keys DE23E73BFA8A0AD5587D2FCDE80D2F3F311FD87E +``` + +Then, run the `release.sh` script directly: + +```bash +./release.sh +``` + +To filter the target platforms, pass them as a space-separated list in the +`LOOPBUILDSYS` environment variable: + +```bash +LOOPBUILDSYS='linux-amd64 windows-amd64' ./release.sh v0.31.3-beta +``` + +This will produce the same artifacts in a `loop-` directory as +the `make docker-release` command. The latter simply runs the `release.sh` +script inside a Docker container. From 2464e942ec541d0773e5ad4e8134f9f731dc73e4 Mon Sep 17 00:00:00 2001 From: Boris Nagaev Date: Mon, 29 Sep 2025 11:27:25 -0300 Subject: [PATCH 11/16] release.sh: use git describe --abbrev=10 Without --abbrev flag it may produce different number of hash characters on different instances. It depends on the state of Git repo itself. --- docs/release.md | 8 ++++---- release.sh | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/release.md b/docs/release.md index 5cad5a08a..539726e84 100644 --- a/docs/release.md +++ b/docs/release.md @@ -23,13 +23,13 @@ make docker-release tag=v0.31.3-beta This will create the release artifacts in the `loop-v0.31.3-beta` directory. If you want to build from an untagged commit, first check it out, then use the -output of `git describe` as the tag: +output of `git describe --abbrev=10` as the tag: ```bash -git describe -# v0.31.2-beta-128-gfa80357 +git describe --abbrev=10 +# v0.31.2-beta-135-g35d0fa26ac -make docker-release tag=v0.31.2-beta-128-gfa80357 +make docker-release tag=v0.31.2-beta-135-g35d0fa26ac ``` You can filter the target platforms to speed up the build process. For example, diff --git a/release.sh b/release.sh index dd3d12e55..d7cf0f830 100755 --- a/release.sh +++ b/release.sh @@ -41,7 +41,7 @@ check_tag() { TAG=$1 # If a tag is specified, ensure that tag is present and checked out. - if [[ $TAG != $(git describe) ]]; then + if [[ $TAG != $(git describe --abbrev=10) ]]; then red "tag $TAG not checked out" exit 1 fi @@ -50,7 +50,7 @@ check_tag() { # output of "git describe" for an untagged commit, skip verification. # The pattern is: --g # Example: "v0.31.2-beta-122-g8c6b73c". - if [[ $TAG =~ -[0-9]+-g([0-9a-f]+)$ ]]; then + if [[ $TAG =~ -[0-9]+-g([0-9a-f]{10})$ ]]; then # This looks like a "git describe" output. Make sure the hash # described is a prefix of the current commit. DESCRIBED_HASH=${BASH_REMATCH[1]} From a34b46c7601ed85aa0511df58fe63a74377b84f6 Mon Sep 17 00:00:00 2001 From: Boris Nagaev Date: Mon, 29 Sep 2025 12:10:31 -0300 Subject: [PATCH 12/16] release.sh: remove the clonned dir with -rf Otherwise it asks confirmation to remove tmp-.../.git dir. --- release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release.sh b/release.sh index d7cf0f830..2fcbda9ea 100755 --- a/release.sh +++ b/release.sh @@ -259,4 +259,4 @@ mv "$ARTIFACTS_DIR" "$FINAL_ARTIFACTS_DIR" green "- Removing the subdir used for building ${BUILD_DIR}" -rm -r "$BUILD_DIR" +rm -rf "$BUILD_DIR" From 6d8872f99590963978f8bc60cdc32f5098e1327b Mon Sep 17 00:00:00 2001 From: Boris Nagaev Date: Mon, 29 Sep 2025 22:22:58 -0300 Subject: [PATCH 13/16] release.sh: reproducible gzip on Mac Make sure we use GNU gzip. Mac by default uses BSD gzip. --- docs/release.md | 7 +++++++ release.sh | 21 ++++++++++++++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/docs/release.md b/docs/release.md index 539726e84..15782d1cc 100644 --- a/docs/release.md +++ b/docs/release.md @@ -58,6 +58,13 @@ other tools: sudo apt-get install build-essential git make zip perl gpg ``` +On MacOS, you will need to install GNU tar and GNU gzip, which can be done with +`brew`: + +```bash +brew install gnu-tar gzip +``` + Add GPG key of Alex Bosworth to verify release tag signature: ```bash gpg --keyserver keys.openpgp.org --recv-keys DE23E73BFA8A0AD5587D2FCDE80D2F3F311FD87E diff --git a/release.sh b/release.sh index 2fcbda9ea..6f8989288 100755 --- a/release.sh +++ b/release.sh @@ -102,13 +102,14 @@ function reproducible_tar_gzip() { local dir=$1 local dst=$2 local tar_cmd=tar + local gzip_cmd=gzip # MacOS has a version of BSD tar which doesn't support setting the --mtime # flag. We need gnu-tar, or gtar for short to be installed for this script to # work properly. - tar_version=$(tar --version) + tar_version=$(tar --version 2>&1 || true) if [[ ! "$tar_version" =~ "GNU tar" ]]; then - if ! command -v "gtar"; then + if ! command -v "gtar" >/dev/null 2>&1; then red "GNU tar is required but cannot be found!" red "On MacOS please run 'brew install gnu-tar' to install gtar." exit 1 @@ -118,12 +119,26 @@ function reproducible_tar_gzip() { tar_cmd=gtar fi + # On MacOS, the default BSD gzip produces a different output than the GNU + # gzip on Linux. To ensure reproducible builds, we need to use GNU gzip. + gzip_version=$(gzip --version 2>&1 || true) + if [[ ! "$gzip_version" =~ "GNU" ]]; then + if ! command -v "ggzip" >/dev/null 2>&1; then + red "GNU gzip is required but cannot be found!" + red "On MacOS please run 'brew install gzip' to install ggzip." + exit 1 + fi + + # We have ggzip installed, use that instead. + gzip_cmd=ggzip + fi + # Pin down the timestamp time zone. export TZ=UTC find "${dir}" -print0 | LC_ALL=C sort -r -z | $tar_cmd \ "--mtime=${BUILD_DATE}" --no-recursion --null --mode=u+rw,go+r-w,a+X \ - --owner=0 --group=0 --numeric-owner -c -T - | gzip -9n > "$dst" + --owner=0 --group=0 --numeric-owner -c -T - | $gzip_cmd -9n > "$dst" } # reproducible_zip creates a reproducible zip file of a directory. This From 260b90d1af1035803cbf6190cf011d0961def0cf Mon Sep 17 00:00:00 2001 From: Boris Nagaev Date: Tue, 30 Sep 2025 11:12:22 -0300 Subject: [PATCH 14/16] release.sh: check Go version, provide GO_CMD var --- docs/release.md | 19 +++++++++++++++++++ release.sh | 23 +++++++++++++++++++++-- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/docs/release.md b/docs/release.md index 15782d1cc..8088cc312 100644 --- a/docs/release.md +++ b/docs/release.md @@ -58,6 +58,25 @@ other tools: sudo apt-get install build-essential git make zip perl gpg ``` +You can [download](https://go.dev/dl/) and unpack Go somewhere and set variable +`GO_CMD=/path/to/go` (path to Go binary of the needed version). + +If you already have another Go version, you can install the Go version needed +for a release using the following commands: + +```bash +$ go version +go version go1.25.0 linux/amd64 +$ go install golang.org/dl/go1.24.6@latest +$ go1.24.6 download +Unpacking /home/user/sdk/go1.24.6/go1.24.6.linux-amd64.tar.gz ... +Success. You may now run 'go1.24.6' +$ go1.24.6 version +go version go1.24.6 linux/amd64 + +$ GO_CMD=/home/user/go/bin/go1.24.6 ./release.sh v0.31.3 +``` + On MacOS, you will need to install GNU tar and GNU gzip, which can be done with `brew`: diff --git a/release.sh b/release.sh index 6f8989288..90a607789 100755 --- a/release.sh +++ b/release.sh @@ -27,6 +27,25 @@ function red() { echo -e "\e[0;31m${1}\e[0m" } +# Use GO_CMD from env if set, otherwise default to "go". +GO_CMD="${GO_CMD:-go}" + +# Check if the command exists. +if ! command -v "$GO_CMD" >/dev/null 2>&1; then + red "Error: Go command '$GO_CMD' not found" + exit 1 +fi + +# Make sure we have the expected Go version installed. +EXPECTED_VERSION="go1.24.6" +INSTALLED_VERSION=$("$GO_CMD" version 2>/dev/null | awk '{print $3}') +if [ "$INSTALLED_VERSION" = "$EXPECTED_VERSION" ]; then + green "Go version matches expected: $INSTALLED_VERSION" +else + red "Error: Expected Go version $EXPECTED_VERSION but found $INSTALLED_VERSION" + exit 1 +fi + TAG='' check_tag() { @@ -203,7 +222,7 @@ fi green " - Creating artifacts directory ${ARTIFACTS_DIR}" mkdir -p "$ARTIFACTS_DIR" green " - Packaging vendor to ${ARTIFACTS_DIR}/vendor.tar.gz" -go mod vendor +"$GO_CMD" mod vendor reproducible_tar_gzip vendor "${ARTIFACTS_DIR}/vendor.tar.gz" rm -r vendor @@ -248,7 +267,7 @@ for i in $SYS; do green "- Building: $OS $ARCH $ARM" for bin in loop loopd; do - env CGO_ENABLED=0 GOOS=$OS GOARCH=$ARCH GOARM=$ARM go build -v -trimpath -ldflags "$COMMITFLAGS" "github.com/lightninglabs/loop/cmd/$bin" + env CGO_ENABLED=0 GOOS=$OS GOARCH=$ARCH GOARM=$ARM "$GO_CMD" build -v -trimpath -ldflags "$COMMITFLAGS" "github.com/lightninglabs/loop/cmd/$bin" done cd .. From 25c5618684228c4191e022c36b862515282ee0e9 Mon Sep 17 00:00:00 2001 From: Boris Nagaev Date: Tue, 30 Sep 2025 12:02:12 -0300 Subject: [PATCH 15/16] release.sh: fix green and red messages on Mac --- release.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release.sh b/release.sh index 90a607789..56df22779 100755 --- a/release.sh +++ b/release.sh @@ -19,12 +19,12 @@ BUILD_DIR="${SCRIPT_DIR}/tmp-build-$(date +%Y%m%d-%H%M%S)" # green prints one line of green text (if the terminal supports it). function green() { - echo -e "\e[0;32m${1}\e[0m" + printf "\e[0;32m%s\e[0m\n" "${1}" } # red prints one line of red text (if the terminal supports it). function red() { - echo -e "\e[0;31m${1}\e[0m" + printf "\e[0;31m%s\e[0m\n" "${1}" } # Use GO_CMD from env if set, otherwise default to "go". From aec03c1e0bc6332babfd5f8156f4bd90898fb7f1 Mon Sep 17 00:00:00 2001 From: Boris Nagaev Date: Tue, 30 Sep 2025 14:15:47 -0300 Subject: [PATCH 16/16] release.sh: set -buildid to '' If Go binary itself comes from `go install golang.org/dl/go1.24.6@latest` it embeds a different BuildID. We unset BuildID to enforce reproducibility. --- release.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release.sh b/release.sh index 56df22779..77dbfad67 100755 --- a/release.sh +++ b/release.sh @@ -246,7 +246,7 @@ SYS=${LOOPBUILDSYS:-"windows-amd64 linux-386 linux-amd64 linux-armv6 linux-armv7 PKG="github.com/lightninglabs/loop" COMMIT=$(git describe --abbrev=40 --dirty) -COMMITFLAGS="-X $PKG/build.Commit=$COMMIT" +GOLDFLAGS="-X $PKG/build.Commit=$COMMIT -buildid=" cd "$BUILD_DIR" for i in $SYS; do @@ -267,7 +267,7 @@ for i in $SYS; do green "- Building: $OS $ARCH $ARM" for bin in loop loopd; do - env CGO_ENABLED=0 GOOS=$OS GOARCH=$ARCH GOARM=$ARM "$GO_CMD" build -v -trimpath -ldflags "$COMMITFLAGS" "github.com/lightninglabs/loop/cmd/$bin" + env CGO_ENABLED=0 GOOS=$OS GOARCH=$ARCH GOARM=$ARM "$GO_CMD" build -v -trimpath -ldflags "$GOLDFLAGS" "github.com/lightninglabs/loop/cmd/$bin" done cd ..