diff --git a/Makefile b/Makefile index 33096d3f0..505ceba1e 100644 --- a/Makefile +++ b/Makefile @@ -38,14 +38,22 @@ endif DOCKER_TOOLS = docker run \ --rm \ - -v $(shell bash -c "go env GOCACHE || (mkdir -p /tmp/go-cache; echo /tmp/go-cache)"):/tmp/build/.cache \ - -v $(shell bash -c "go env GOMODCACHE || (mkdir -p /tmp/go-modcache; echo /tmp/go-modcache)"):/tmp/build/.modcache \ + -v $(shell bash -c "go env GOCACHE 2>/dev/null || (mkdir -p /tmp/go-cache; echo /tmp/go-cache)"):/tmp/build/.cache \ + -v $(shell bash -c "go env GOMODCACHE 2>/dev/null || (mkdir -p /tmp/go-modcache; echo /tmp/go-modcache)"):/tmp/build/.modcache \ -v $$(pwd):/build loop-tools -GREEN := "\\033[0;32m" -NC := "\\033[0m" +DOCKER_RELEASE_BUILDER = docker run \ + --rm \ + -v $(shell bash -c "go env GOCACHE 2>/dev/null || (mkdir -p /tmp/go-cache; echo /tmp/go-cache)"):/tmp/build/.cache \ + -v $(shell bash -c "go env GOMODCACHE 2>/dev/null || (mkdir -p /tmp/go-modcache; echo /tmp/go-modcache)"):/tmp/build/.modcache \ + -v $$(pwd):/repo \ + -e LOOPBUILDSYS='$(buildsys)' \ + loop-release-builder + +GREEN=\033[0;32m +NC=\033[0m define print - echo $(GREEN)$1$(NC) + @printf '%b%s%b\n' '${GREEN}' $1 '${NC}' endef # ============ @@ -71,6 +79,13 @@ install: $(GOINSTALL) -tags="${tags}" $(LDFLAGS) $(PKG)/cmd/loop $(GOINSTALL) -tags="${tags}" $(LDFLAGS) $(PKG)/cmd/loopd +# docker-release: Same as release.sh but within a docker container to support +# reproducible builds on any platform. +docker-release: docker-release-builder + @$(call print, "Building release binaries in docker.") + @if [ "$(tag)" = "" ]; then echo "Must specify tag=!"; exit 1; fi + $(DOCKER_RELEASE_BUILDER) bash release.sh $(tag) + rpc: @$(call print, "Compiling protos.") cd ./swapserverrpc; ./gen_protos_docker.sh @@ -131,6 +146,10 @@ docker-tools: @$(call print, "Building tools docker image.") docker build -q -t loop-tools $(TOOLS_DIR) +docker-release-builder: + @$(call print, "Building release builder docker image.") + docker build -q -t loop-release-builder -f release.Dockerfile . + mod-tidy: @$(call print, "Tidying modules.") $(GOMOD) tidy diff --git a/README.md b/README.md index 912df86f3..1f9938fa2 100644 --- a/README.md +++ b/README.md @@ -125,3 +125,8 @@ git clone https://github.com/lightninglabs/loop.git cd loop/cmd go install ./... ``` + +## Reproducible builds + +If you want to build release files yourself, follow +[the guide](./docs/release.md). diff --git a/docs/release.md b/docs/release.md new file mode 100644 index 000000000..8088cc312 --- /dev/null +++ b/docs/release.md @@ -0,0 +1,107 @@ +# Reproducible Builds + +## Building with Docker + +To create a Loop release with binaries that are identical to an official +release, run the following command (available since release `v0.31.3-beta`): + +```bash +make docker-release tag= +``` + +This command will create a directory named `loop-` containing +the source archive, vendored dependencies, and the built binaries packaged in +`.tar.gz` or `.zip` format. It also creates a manifest file with `SHA-256` +checksums for all release files. + +For example: + +```bash +make docker-release tag=v0.31.3-beta +``` + +This will create the release artifacts in the `loop-v0.31.3-beta` directory. + +If you want to build from an untagged commit, first check it out, then use the +output of `git describe --abbrev=10` as the tag: + +```bash +git describe --abbrev=10 +# v0.31.2-beta-135-g35d0fa26ac + +make docker-release tag=v0.31.2-beta-135-g35d0fa26ac +``` + +You can filter the target platforms to speed up the build process. For example, +to build only for `linux-amd64`: + +```bash +make docker-release buildsys=linux-amd64 tag=v0.31.3-beta +``` + +Or for multiple platforms: + +```bash +make docker-release buildsys='linux-amd64 windows-amd64' tag=v0.31.3-beta +``` + +Note: inside Docker the current directory is mapped as `/repo` and it might +mention `/repo` as parts of file paths. + +## Building on the Host + +You can also build a release on your host system without Docker. You will need +to install the Go version specified in the `go.mod` file, as well as a few +other tools: + +```bash +sudo apt-get install build-essential git make zip perl gpg +``` + +You can [download](https://go.dev/dl/) and unpack Go somewhere and set variable +`GO_CMD=/path/to/go` (path to Go binary of the needed version). + +If you already have another Go version, you can install the Go version needed +for a release using the following commands: + +```bash +$ go version +go version go1.25.0 linux/amd64 +$ go install golang.org/dl/go1.24.6@latest +$ go1.24.6 download +Unpacking /home/user/sdk/go1.24.6/go1.24.6.linux-amd64.tar.gz ... +Success. You may now run 'go1.24.6' +$ go1.24.6 version +go version go1.24.6 linux/amd64 + +$ GO_CMD=/home/user/go/bin/go1.24.6 ./release.sh v0.31.3 +``` + +On MacOS, you will need to install GNU tar and GNU gzip, which can be done with +`brew`: + +```bash +brew install gnu-tar gzip +``` + +Add GPG key of Alex Bosworth to verify release tag signature: +```bash +gpg --keyserver keys.openpgp.org --recv-keys DE23E73BFA8A0AD5587D2FCDE80D2F3F311FD87E +``` + +Then, run the `release.sh` script directly: + +```bash +./release.sh +``` + +To filter the target platforms, pass them as a space-separated list in the +`LOOPBUILDSYS` environment variable: + +```bash +LOOPBUILDSYS='linux-amd64 windows-amd64' ./release.sh v0.31.3-beta +``` + +This will produce the same artifacts in a `loop-` directory as +the `make docker-release` command. The latter simply runs the `release.sh` +script inside a Docker container. diff --git a/release.Dockerfile b/release.Dockerfile new file mode 100644 index 000000000..a42588934 --- /dev/null +++ b/release.Dockerfile @@ -0,0 +1,23 @@ +FROM golang:1.24.6 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + git ca-certificates zip gpg && rm -rf /var/lib/apt/lists/* + +# Add GPG key of Alex Bosworth to verify release tag signature. +RUN gpg --keyserver keys.openpgp.org \ + --recv-keys DE23E73BFA8A0AD5587D2FCDE80D2F3F311FD87E + +# Mark the repo directory safe for git. User ID may be different inside +# Docker and Git might refuse to work without this setting. +RUN git config --global --add safe.directory /repo +RUN git config --global --add safe.directory /repo/.git + +# Set GO build time environment variables. +ENV GOCACHE=/tmp/build/.cache \ + GOMODCACHE=/tmp/build/.modcache + +# Create directories to which host's Go caches are mounted. +RUN mkdir -p /tmp/build/.cache /tmp/build/.modcache \ + && chmod -R 777 /tmp/build/ + +WORKDIR /repo diff --git a/release.sh b/release.sh index 94a7d11c1..77dbfad67 100755 --- a/release.sh +++ b/release.sh @@ -10,23 +10,80 @@ # Exit on errors. set -e -# If no tag specified, use date + version otherwise use tag. -if [[ $1x = x ]]; then - DATE=`date +%Y%m%d` - VERSION="01" - TAG=$DATE-$VERSION +# Get the directory of the script +SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + +# Checkout the repo to a subdir to clean from clean from unstaged files and +# build exactly what is committed. +BUILD_DIR="${SCRIPT_DIR}/tmp-build-$(date +%Y%m%d-%H%M%S)" + +# green prints one line of green text (if the terminal supports it). +function green() { + printf "\e[0;32m%s\e[0m\n" "${1}" +} + +# red prints one line of red text (if the terminal supports it). +function red() { + printf "\e[0;31m%s\e[0m\n" "${1}" +} + +# Use GO_CMD from env if set, otherwise default to "go". +GO_CMD="${GO_CMD:-go}" + +# Check if the command exists. +if ! command -v "$GO_CMD" >/dev/null 2>&1; then + red "Error: Go command '$GO_CMD' not found" + exit 1 +fi + +# Make sure we have the expected Go version installed. +EXPECTED_VERSION="go1.24.6" +INSTALLED_VERSION=$("$GO_CMD" version 2>/dev/null | awk '{print $3}') +if [ "$INSTALLED_VERSION" = "$EXPECTED_VERSION" ]; then + green "Go version matches expected: $INSTALLED_VERSION" else + red "Error: Expected Go version $EXPECTED_VERSION but found $INSTALLED_VERSION" + exit 1 +fi + +TAG='' + +check_tag() { + # If no tag specified, use date + version otherwise use tag. + if [[ $1x = x ]]; then + TAG=`date +%Y%m%d-%H%M%S` + green "No tag specified, using ${TAG} as tag" + + return + fi + TAG=$1 # If a tag is specified, ensure that tag is present and checked out. - if [[ $TAG != $(git describe) ]]; then - echo "tag $TAG not checked out" + if [[ $TAG != $(git describe --abbrev=10) ]]; then + red "tag $TAG not checked out" exit 1 fi - # Verify that it is signed. - if ! git verify-tag $TAG; then - echo "tag $TAG not signed" + # Verify that it is signed if it is a real tag. If the tag looks like the + # output of "git describe" for an untagged commit, skip verification. + # The pattern is: --g + # Example: "v0.31.2-beta-122-g8c6b73c". + if [[ $TAG =~ -[0-9]+-g([0-9a-f]{10})$ ]]; then + # This looks like a "git describe" output. Make sure the hash + # described is a prefix of the current commit. + DESCRIBED_HASH=${BASH_REMATCH[1]} + CURRENT_HASH=$(git rev-parse HEAD) + if [[ $CURRENT_HASH != $DESCRIBED_HASH* ]]; then + red "Described hash $DESCRIBED_HASH is not a prefix of current commit $CURRENT_HASH" + exit 1 + fi + + return + fi + + if ! git verify-tag $TAG; then + red "tag $TAG not signed" exit 1 fi @@ -45,42 +102,153 @@ else # Match git tag with loop version. if [[ $TAG != $LOOP_VERSION ]]; then - echo "loop version $LOOP_VERSION does not match tag $TAG" + red "loop version $LOOP_VERSION does not match tag $TAG" exit 1 fi else - echo "malformed loop version output" + red "malformed loop version output" exit 1 fi +} + +# Needed for setting file timestamps to get reproducible archives. +BUILD_DATE="2020-01-01 00:00:00" +BUILD_DATE_STAMP="202001010000.00" + +# reproducible_tar_gzip creates a reproducible tar.gz file of a directory. This +# includes setting all file timestamps and ownership settings uniformly. +function reproducible_tar_gzip() { + local dir=$1 + local dst=$2 + local tar_cmd=tar + local gzip_cmd=gzip + + # MacOS has a version of BSD tar which doesn't support setting the --mtime + # flag. We need gnu-tar, or gtar for short to be installed for this script to + # work properly. + tar_version=$(tar --version 2>&1 || true) + if [[ ! "$tar_version" =~ "GNU tar" ]]; then + if ! command -v "gtar" >/dev/null 2>&1; then + red "GNU tar is required but cannot be found!" + red "On MacOS please run 'brew install gnu-tar' to install gtar." + exit 1 + fi + + # We have gtar installed, use that instead. + tar_cmd=gtar + fi + + # On MacOS, the default BSD gzip produces a different output than the GNU + # gzip on Linux. To ensure reproducible builds, we need to use GNU gzip. + gzip_version=$(gzip --version 2>&1 || true) + if [[ ! "$gzip_version" =~ "GNU" ]]; then + if ! command -v "ggzip" >/dev/null 2>&1; then + red "GNU gzip is required but cannot be found!" + red "On MacOS please run 'brew install gzip' to install ggzip." + exit 1 + fi + + # We have ggzip installed, use that instead. + gzip_cmd=ggzip + fi + + # Pin down the timestamp time zone. + export TZ=UTC + + find "${dir}" -print0 | LC_ALL=C sort -r -z | $tar_cmd \ + "--mtime=${BUILD_DATE}" --no-recursion --null --mode=u+rw,go+r-w,a+X \ + --owner=0 --group=0 --numeric-owner -c -T - | $gzip_cmd -9n > "$dst" +} + +# reproducible_zip creates a reproducible zip file of a directory. This +# includes setting all file timestamps. +function reproducible_zip() { + local dir=$1 + local dst=$2 + + # Pin down file name encoding and timestamp time zone. + export TZ=UTC + + # Set the date of each file in the directory that's about to be packaged to + # the same timestamp and make sure the same permissions are used everywhere. + chmod -R 0755 "${dir}" + touch -t "${BUILD_DATE_STAMP}" "${dir}" + find "${dir}" -print0 | LC_ALL=C sort -r -z | xargs -0r touch \ + -t "${BUILD_DATE_STAMP}" + + find "${dir}" | LC_ALL=C sort -r | zip -o -X -r -@ "$dst" +} + +################## +# Start Building # +################## + +if [ -d "$BUILD_DIR" ]; then + red "Build directory ${BUILD_DIR} already exists!" + exit 1 fi -go mod vendor -tar -cvzf vendor.tar.gz vendor +green " - Cloning to subdir ${BUILD_DIR} to get clean Git" +mkdir -p "$BUILD_DIR" +cd "$BUILD_DIR" +git clone --no-tags "$SCRIPT_DIR" . -PACKAGE=loop -MAINDIR=$PACKAGE-$TAG -mkdir -p $MAINDIR +# It is cloned without tags from the local dir and tags are pulled later +# from the upstream to make sure we have the same set of tags. Otherwise +# we can endup with different `git describe` and buildvcs info depending +# on local tags. +green " - Pulling tags from upstream" +git pull --tags https://github.com/lightninglabs/loop -cp vendor.tar.gz $MAINDIR/ -rm vendor.tar.gz -rm -r vendor +# The cloned Git repo may be on wrong branch and commit. +commit=$(git --git-dir "${SCRIPT_DIR}/.git" rev-parse HEAD) +green " - Checkout commit ${commit} in ${BUILD_DIR}" +git checkout -b build-branch "$commit" + +green " - Checking tag $1" +check_tag $1 -PACKAGESRC="$MAINDIR/$PACKAGE-source-$TAG.tar" -git archive -o $PACKAGESRC HEAD -gzip -f $PACKAGESRC > "$PACKAGESRC.gz" +PACKAGE=loop +FINAL_ARTIFACTS_DIR="${SCRIPT_DIR}/${PACKAGE}-${TAG}" +ARTIFACTS_DIR="${SCRIPT_DIR}/tmp-${PACKAGE}-${TAG}-$(date +%Y%m%d-%H%M%S)" +if [ -d "$ARTIFACTS_DIR" ]; then + red "artifacts directory ${ARTIFACTS_DIR} already exists!" + exit 1 +fi +if [ -d "$FINAL_ARTIFACTS_DIR" ]; then + red "final artifacts directory ${FINAL_ARTIFACTS_DIR} already exists!" + exit 1 +fi +green " - Creating artifacts directory ${ARTIFACTS_DIR}" +mkdir -p "$ARTIFACTS_DIR" +green " - Packaging vendor to ${ARTIFACTS_DIR}/vendor.tar.gz" +"$GO_CMD" mod vendor +reproducible_tar_gzip vendor "${ARTIFACTS_DIR}/vendor.tar.gz" +rm -r vendor -cd $MAINDIR +PACKAGESRC="${ARTIFACTS_DIR}/${PACKAGE}-source-${TAG}.tar.gz" +green " - Creating source archive ${PACKAGESRC}" +TMPSOURCETAR="${ARTIFACTS_DIR}/tmp-${PACKAGE}-source-${TAG}.tar" +PKGSRC="${PACKAGE}-source" +git archive -o "$TMPSOURCETAR" HEAD +cd "$ARTIFACTS_DIR" +mkdir "$PKGSRC" +tar -xf "$TMPSOURCETAR" -C "$PKGSRC" +cd "$PKGSRC" +reproducible_tar_gzip . "$PACKAGESRC" +cd .. +rm -r "$PKGSRC" +rm "$TMPSOURCETAR" # If LOOPBUILDSYS is set the default list is ignored. Useful to release # for a subset of systems/architectures. SYS=${LOOPBUILDSYS:-"windows-amd64 linux-386 linux-amd64 linux-armv6 linux-armv7 linux-arm64 darwin-arm64 darwin-amd64 freebsd-amd64 freebsd-arm"} -# Use the first element of $GOPATH in the case where GOPATH is a list -# (something that is totally allowed). PKG="github.com/lightninglabs/loop" COMMIT=$(git describe --abbrev=40 --dirty) -COMMITFLAGS="-X $PKG/build.Commit=$COMMIT" +GOLDFLAGS="-X $PKG/build.Commit=$COMMIT -buildid=" +cd "$BUILD_DIR" for i in $SYS; do OS=$(echo $i | cut -f1 -d-) ARCH=$(echo $i | cut -f2 -d-) @@ -97,19 +265,32 @@ for i in $SYS; do mkdir $PACKAGE-$i-$TAG cd $PACKAGE-$i-$TAG - echo "Building:" $OS $ARCH $ARM + green "- Building: $OS $ARCH $ARM" for bin in loop loopd; do - env CGO_ENABLED=0 GOOS=$OS GOARCH=$ARCH GOARM=$ARM go build -v -ldflags "$COMMITFLAGS" "github.com/lightninglabs/loop/cmd/$bin" + env CGO_ENABLED=0 GOOS=$OS GOARCH=$ARCH GOARM=$ARM "$GO_CMD" build -v -trimpath -ldflags "$GOLDFLAGS" "github.com/lightninglabs/loop/cmd/$bin" done cd .. if [[ $OS = "windows" ]]; then - zip -r $PACKAGE-$i-$TAG.zip $PACKAGE-$i-$TAG + green "- Producing ZIP file ${ARTIFACTS_DIR}/${PACKAGE}-${i}-${TAG}.zip" + reproducible_zip "${PACKAGE}-${i}-${TAG}" "${ARTIFACTS_DIR}/${PACKAGE}-${i}-${TAG}.zip" else - tar -cvzf $PACKAGE-$i-$TAG.tar.gz $PACKAGE-$i-$TAG + green "- Producing TAR.GZ file ${ARTIFACTS_DIR}/${PACKAGE}-${i}-${TAG}.tar.gz" + reproducible_tar_gzip "${PACKAGE}-${i}-${TAG}" "${ARTIFACTS_DIR}/${PACKAGE}-${i}-${TAG}.tar.gz" fi rm -r $PACKAGE-$i-$TAG done +cd "$ARTIFACTS_DIR" +green "- Producing manifest-$TAG.txt" shasum -a 256 * > manifest-$TAG.txt +shasum -a 256 manifest-$TAG.txt +cd .. + +green "- Moving artifacts directory to final place ${FINAL_ARTIFACTS_DIR}" +mv "$ARTIFACTS_DIR" "$FINAL_ARTIFACTS_DIR" + +green "- Removing the subdir used for building ${BUILD_DIR}" + +rm -rf "$BUILD_DIR"