Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
target
.cargo
build/
67 changes: 56 additions & 11 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,10 +1,22 @@
# syntax=docker/dockerfile:1

FROM stagex/pallet-rust@sha256:9c38bf1066dd9ad1b6a6b584974dd798c2bf798985bf82e58024fbe0515592ca AS pallet-rust
FROM stagex/user-protobuf@sha256:5e67b3d3a7e7e9db9aa8ab516ffa13e54acde5f0b3d4e8638f79880ab16da72c AS protobuf
FROM stagex/user-abseil-cpp@sha256:3dca99adfda0cb631bd3a948a99c2d5f89fab517bda034ce417f222721115aa2 AS abseil-cpp
FROM stagex/core-gcc@sha256:964ffd3793c5a38ca581e9faefd19918c259f1611c4cbf5dc8be612e3a8b72f5 AS gcc
FROM stagex/core-musl@sha256:d9af23284cca2e1002cd53159ada469dfe6d6791814e72d6163c7de18d4ae701 AS musl
FROM stagex/core-libunwind@sha256:eb66122d8fc543f5e2f335bb1616f8c3a471604383e2c0a9df4a8e278505d3bc AS libunwind
Comment on lines +3 to +8
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I did not review any of these pins.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll update these pins to latest. The real move is to clone stagex and use the latest tag, then make and compare results (you probably want a 16+ core CPU and at least 32GB of ram to do this in ~2 days or so). Or at least refer to the signatures repo where we sign everything (at least 2 maintainers sign each release).

FROM stagex/core-user-runtime@sha256:055ae534e1e01259449fb4e0226f035a7474674c7371a136298e8bdac65d90bb AS user-runtime

# --- Stage 1: Build with Rust --- (amd64)
FROM rust:bookworm AS builder
FROM pallet-rust AS builder
COPY --from=protobuf . /
COPY --from=abseil-cpp . /

RUN apt-get update && \
apt-get install -y --no-install-recommends \
libclang-dev
ENV SOURCE_DATE_EPOCH=1
ENV CXXFLAGS="-include cstdint"
ENV ROCKSDB_USE_PKG_CONFIG=0
ENV CARGO_HOME=/usr/local/cargo

# Make a fake Rust app to keep a cached layer of compiled crates
WORKDIR /usr/src/app
Expand All @@ -13,26 +25,59 @@ COPY zallet/Cargo.toml ./zallet/
# Needs at least a main.rs file with a main function
RUN mkdir -p zallet/src/bin/zallet && echo "fn main(){}" > zallet/src/bin/zallet/main.rs
RUN mkdir zallet/tests && touch zallet/tests/cli_tests.rs

ENV RUST_BACKTRACE=1
ENV RUSTFLAGS="-C codegen-units=1"
ENV RUSTFLAGS="${RUSTFLAGS} -C target-feature=+crt-static"
ENV RUSTFLAGS="${RUSTFLAGS} -C link-arg=-Wl,--build-id=none"
ENV CFLAGS="-D__GNUC_PREREQ(maj,min)=1"
ENV TARGET_ARCH="x86_64-unknown-linux-musl"

RUN --mount=type=cache,target=/usr/local/cargo/registry \
--mount=type=cache,target=/usr/local/cargo/git \
cargo fetch --locked --target $TARGET_ARCH

RUN --mount=type=cache,target=/usr/local/cargo/registry \
--mount=type=cache,target=/usr/local/cargo/git \
cargo metadata --locked --format-version=1 > /dev/null 2>&1

# Will build all dependent crates in release mode
RUN --mount=type=cache,target=/usr/local/cargo/registry \
--mount=type=cache,target=/usr/local/cargo/git \
--mount=type=cache,target=/usr/src/app/target \
cargo build --release --features rpc-cli,zcashd-import
--network=none \
cargo build --release --frozen \
--target ${TARGET_ARCH} \
--features rpc-cli,zcashd-import

# Copy the rest
COPY . .
# Build the zallet binary
# Compile & install offline
RUN --mount=type=cache,target=/usr/local/cargo/registry \
--mount=type=cache,target=/usr/local/cargo/git \
--mount=type=cache,target=/usr/src/app/target \
cargo install --locked --features rpc-cli,zcashd-import --path ./zallet --bins
--network=none \
cargo build --release --frozen \
--bin zallet \
--target ${TARGET_ARCH} \
--features rpc-cli,zcashd-import \
&& install -D -m 0755 /usr/src/app/target/${TARGET_ARCH}/release/zallet /usr/local/bin/zallet
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why does this need to use cargo build && install, instead of cargo install?

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

cargo install sometimes does wonky things by default, including ignoring any project-specific configuration, such as Cargo.lock. using cargo build is the most reliable way for it to respect your projects' configuration.

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the reason we must && install in the same step is because otherwise we lose access to the cached build object

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yep it breaks determinism if we only use cargo install



# --- Stage 2: Minimal runtime with distroless ---
FROM gcr.io/distroless/cc-debian12 AS runtime
# --- Stage 2: layer for local binary extraction ---
FROM scratch AS export

COPY --link --from=builder /usr/local/cargo/bin/zallet /usr/local/bin/
COPY --from=builder /usr/local/bin/zallet /zallet

# USER nonroot (UID 65532) — for K8s, use runAsUser: 65532
USER nonroot
# --- Stage 3: Minimal runtime with stagex ---
# `stagex/core-user-runtime` sets the user to non-root by default
FROM user-runtime AS runtime
COPY --from=gcc /usr/lib/libgcc_s.so.1 /usr/lib/
COPY --from=gcc /usr/lib/libstdc++.so.6 /usr/lib/
COPY --from=musl /lib/ld-musl-x86_64.so.1 /lib/
COPY --from=libunwind /lib/libunwind.so.8 /lib/
COPY --from=builder /usr/local/bin/zallet /usr/local/bin/zallet

WORKDIR /var/lib/zallet

Expand Down
27 changes: 27 additions & 0 deletions Makefile
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I will be looking into removing this Makefile during the Zallet alpha phase. I do not see the value it brings vs the requirement of GNU make that it imposes.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's fair, I'll just say that gnu make is one of the most minimal setup for wrapping commands that also happens to be what is commonly used by projects when building from source. It becomes more useful as the project grows and there are more and more commands to run. You can always use something like python but that is not preinstalled on all systems either, and is much larger than make, or you can resort to bash, but it's just a bit worse in terms of UX. Of course, ultimately it's up to you, I'll support whatever decision you make here. The idea is that make + docker beats most setups in terms of deps required + UX, and in this case also security.

Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
# Makefile
SHELL := /bin/bash
.SHELLFLAGS := -eu -o pipefail -c
IMAGE_NAME := zallet
IMAGE_TAG := latest

.PHONY: all build import
all: build import

.PHONY: build
build:
@echo "Running compat check..."
@out="$$(bash utils/compat.sh)"; \
if [[ -z "$$out" ]]; then \
echo "Compat produced no output; proceeding to build"; \
bash utils/build.sh; \
else \
echo "Compat produced output; not building."; \
printf '%s\n' "$$out"; \
exit 1; \
fi


.PHONY: import
import:
docker load -i build/oci/zallet.tar
docker tag $(IMAGE_NAME):latest $(IMAGE_NAME):$(IMAGE_TAG)
26 changes: 26 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,32 @@ contacting us in the `#wallet-dev` channel of the

See the [user guide](book/src/README.md) for information on how to set up a Zallet wallet.

## Reproducible Builds
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is not the correct place for this information; it should instead be in the book as one of the options for building from source. Having this here on its own suggests that this is the only supported build mechanism, which is not the case. At a minimum, regular cargo install builds will always be supported. The intention of reproducible build support is to enable the binaries that ECC publishes to be verifiable by others; individual users building from source should always be free to build as they wish.

Non-blocking, I will move it myself in a subsequent PR.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Getting people to build using Docker is both easier and more secure with this setup, so I'd try to motivate them to use the infra that's in place for this, and of course they can still build locally whatever way they like.

What do you think about clarifying more or less what you said "This is not the only way to build, but it is the recommended way as you still get the same binaries, just built with a secure toolchain", or something along those lines.

Also I'm happy to add more documentation on whatever you feel would be useful to have.


Zallet leverages [StageX](https://codeberg.org/stagex/stagex/) to provied a
full source bootstrapped, and deterministic/reproducible build and runtime
dependencies. This helps mitigate supply chain attacks, and especially trusting
trust style attacks and reduces trust in any single computer or individual.

### Requirements
* Docker 25+
* [`containerd` support](https://docs.docker.com/engine/storage/containerd/#enable-containerd-image-store-on-docker-engine)
* GNU Make

### Usage

* To `build` and `import` the image use the `make` command

* The `build` commmands uses the `utils/compat.sh` and `utils/builds.sh`
in order to ensure that the user has required dependencies installed and that
the [OCI](https://opencontainers.org/) image built is deterministic by using
the appropriate flags.

### Details

* `stagex/core-user-runtime` is used to set user to non-root and provide a
minimal filesystem

## License

All code in this workspace is licensed under either of
Expand Down
31 changes: 31 additions & 0 deletions utils/build.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
#!/bin/sh

set -e

DIR="$( cd "$( dirname "$0" )" && pwd )"
REPO_ROOT="$(git rev-parse --show-toplevel)"
PLATFORM="linux/amd64"
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This limitation will need to be lifted at some point; arm64 builds will likely be necessary.

Non-blocking, this can be figured out later on in the alpha or beta phases.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We are currently merging LLVM into StageX which will allow us to generate ARM, and Apache Software Foundation has interest in support of arm so this is very likely to happen over the next little while.

OCI_OUTPUT="$REPO_ROOT/build/oci"
DOCKERFILE="$REPO_ROOT/Dockerfile"

export DOCKER_BUILDKIT=1
export SOURCE_DATE_EPOCH=1

echo $DOCKERFILE
mkdir -p $OCI_OUTPUT

# Build runtime image for docker run
echo "Building runtime image..."
docker build -f "$DOCKERFILE" "$REPO_ROOT" \
--platform "$PLATFORM" \
--target runtime \
--output type=oci,rewrite-timestamp=true,force-compression=true,dest=$OCI_OUTPUT/zallet.tar,name=zallet \
"$@"

# Extract binary locally from export stage
echo "Extracting binary..."
docker build -f "$DOCKERFILE" "$REPO_ROOT" --quiet \
--platform "$PLATFORM" \
--target export \
--output type=local,dest="$REPO_ROOT/build" \
"$@"
81 changes: 81 additions & 0 deletions utils/compat.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
#!/usr/bin/env bash
set -e
readonly MIN_BASH_VERSION=5
readonly MIN_DOCKER_VERSION=26.0.0
readonly MIN_BUILDX_VERSION=0.13
### Exit with error message
die() {
echo "$@" >&2
exit 1
}
### Bail and instruct user on missing package to install for their platform
die_pkg() {
local -r package=${1?}
local -r version=${2?}
local install_cmd
case "$OSTYPE" in
linux*)
if command -v "apt" >/dev/null; then
install_cmd="apt install ${package}"
elif command -v "yum" >/dev/null; then
install_cmd="yum install ${package}"
elif command -v "pacman" >/dev/null; then
install_cmd="pacman -Ss ${package}"
elif command -v "emerge" >/dev/null; then
install_cmd="emerge ${package}"
elif command -v "nix-env" >/dev/null; then
install_cmd="nix-env -i ${package}"
fi
;;
bsd*) install_cmd="pkg install ${package}" ;;
darwin*) install_cmd="port install ${package}" ;;
*) die "Error: Your operating system is not supported" ;;
esac
echo "Error: ${package} ${version}+ does not appear to be installed." >&2
[ -n "$install_cmd" ] && echo "Try: \`${install_cmd}\`" >&2
exit 1
}
### Check if actual binary version is >= minimum version
check_version(){
local pkg="${1?}"
local have="${2?}"
local need="${3?}"
local i ver1 ver2 IFS='.'
[[ "$have" == "$need" ]] && return 0
read -r -a ver1 <<< "$have"
read -r -a ver2 <<< "$need"
for ((i=${#ver1[@]}; i<${#ver2[@]}; i++));
do ver1[i]=0;
done
for ((i=0; i<${#ver1[@]}; i++)); do
[[ -z ${ver2[i]} ]] && ver2[i]=0
((10#${ver1[i]} > 10#${ver2[i]})) && return 0
((10#${ver1[i]} < 10#${ver2[i]})) && die_pkg "${pkg}" "${need}"
done
}
### Check if required binaries are installed at appropriate versions
check_tools(){
if [ -z "${BASH_VERSINFO[0]}" ] \
|| [ "${BASH_VERSINFO[0]}" -lt "${MIN_BASH_VERSION}" ]; then
die_pkg "bash" "${MIN_BASH_VERSION}"
fi
for cmd in "$@"; do
case $cmd in
buildx)
docker buildx version >/dev/null 2>&1 || die "Error: buildx not found"
version=$(docker buildx version 2>/dev/null | grep -o 'v[0-9.]*' | sed 's/v//')
check_version "buildx" "${version}" "${MIN_BUILDX_VERSION}"
;;
docker)
command -v docker >/dev/null || die "Error: docker not found"
version=$(docker version -f '{{ .Server.Version }}')
check_version "docker" "${version}" "${MIN_DOCKER_VERSION}"
;;
esac
done
}
check_tools docker buildx;
docker info -f '{{ .DriverStatus }}' \
| grep "io.containerd.snapshotter.v1" >/dev/null \
|| die "Error: Docker Engine is not using containerd for image storage"

Loading