chore: reproducible builds

Based on #143

Changes:

* python2 is removed, as it seems to be unused at the moment
* Perl updated to 5.34.0, and build changed to official method
* Python3 updated to 3.9.6
* ncurses updated to 6.2
* img updated to the release

Signed-off-by: Andrey Smirnov <andrey.smirnov@talos-systems.com>

.gitignore

@@ -1 +1,2 @@
 bin/
+_out/

Makefile

@@ -7,12 +7,13 @@ REGISTRY_AND_USERNAME := $(REGISTRY)/$(USERNAME)
 
 # Sync bldr image with Pkgfile
 BLDR ?= docker run --rm --volume $(PWD):/tools --entrypoint=/bldr \
-	ghcr.io/talos-systems/bldr:v0.2.0-alpha.3-frontend graph --root=/tools
+	ghcr.io/talos-systems/bldr:v0.2.0-alpha.4-frontend graph --root=/tools
 
 BUILD := docker buildx build
 PLATFORM ?= linux/amd64,linux/arm64
 PROGRESS ?= auto
 PUSH ?= false
+DEST ?= _out
 COMMON_ARGS := --file=Pkgfile
 COMMON_ARGS += --progress=$(PROGRESS)
 COMMON_ARGS += --platform=$(PLATFORM)
@@ -23,21 +24,34 @@ all: $(TARGETS) ## Builds all known pkgs.
 
 .PHONY: help
 help: ## This help menu.
-	@grep -E '^[a-zA-Z%_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+	@grep -E '^[a-zA-Z0-9\.%_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
 target-%: ## Builds the specified target defined in the Pkgfile. The build result will only remain in the build cache.
 	@$(BUILD) \
 		--target=$* \
 		$(COMMON_ARGS) \
 		$(TARGET_ARGS) .
 
-docker-%: ## Builds the specified target defined in the Pkgfile using the docker output type. The build result will be loaded into docker.
+local-%: ## Builds the specified target defined in the Pkgfile using the local output type. The build result will be output to the specified local destination.
+	@$(MAKE) target-$* TARGET_ARGS="--output=type=local,dest=$(DEST) $(TARGET_ARGS)"
+
+rebuild-%: ## Builds the specified target twice into $(DEST)/build-1/2 and compares results.
+	@rm -fr $(DEST)/build-1 $(DEST)/build-2 $(DEST)/build-1.txt $(DEST)/build-2.txt
+	@$(MAKE) target-$* PROGRESS=plain TARGET_ARGS="--output=type=local,dest=$(DEST)/build-1 $(TARGET_ARGS)" 2>&1 | tee $(DEST)/build-1.txt
+	@docker buildx rm reproducer || true
+	@docker buildx create --driver docker-container --driver-opt network=host --name reproducer
+	@$(MAKE) target-$* PROGRESS=plain TARGET_ARGS="--output=type=local,dest=$(DEST)/build-2 --builder=reproducer $(TARGET_ARGS)" 2>&1 | tee $(DEST)/build-2.txt
+	@docker buildx rm reproducer
+	@find _out/ -exec touch -ch -t 202108110000 {} \;
+	@diffoscope _out/build-1 _out/build-2
+
+docker-%: ## Builds the specified target defined in the Pkgfile using the docker output type. The build result will be loaded into Docker.
 	@$(MAKE) target-$* TARGET_ARGS="$(TARGET_ARGS)"
 
 .PHONY: $(TARGETS)
 $(TARGETS):
-	@$(MAKE) docker-$@ TARGET_ARGS="--tag=${REGISTRY}/${USERNAME}/$@:$(TAG) --push=$(PUSH)"
+	@$(MAKE) docker-$@ TARGET_ARGS="--tag=$(REGISTRY_AND_USERNAME)/$@:$(TAG) --push=$(PUSH)"
 
 .PHONY: deps.png
-deps.png:
-	$(BLDR) graph | dot -Tpng > deps.png
+deps.png: ## Regenerate deps.png.
+	@$(BLDR) graph | dot -Tpng > deps.png

Pkgfile

@@ -5,7 +5,7 @@
 format: v1alpha2
 
 vars:
-  TOOLCHAIN_IMAGE: ghcr.io/talos-systems/toolchain:v0.2.0
+  TOOLCHAIN_IMAGE: ghcr.io/talos-systems/toolchain:v0.3.0-beta.3
 
 labels:
   org.opencontainers.image.source: https://github.com/talos-systems/tools

curl/pkg.yaml

@@ -4,6 +4,7 @@ dependencies:
   - stage: libressl
     runtime: true
   - stage: zlib
+    runtime: true
   - stage: pkg-config
 steps:
   - sources:

git/pkg.yaml

@@ -2,6 +2,7 @@ name: git
 dependencies:
   - stage: base
   - stage: zlib
+    runtime: true
   - stage: tcl
   - stage: gettext
   - stage: libressl

img/patches/reproducibility.patch

@@ -0,0 +1,13 @@
+diff --git a/Makefile b/Makefile
+index b336159f..7d1e071c 100644
+--- a/Makefile
++++ b/Makefile
+@@ -24,7 +24,7 @@ $(BUILDDIR)/bin/runc: $(RUNCBUILDDIR)
+ 	mkdir -p $(BUILDDIR)/bin
+ 	mv $(RUNCBUILDDIR)/runc $(BUILDDIR)/bin/runc
+
+-internal/binutils/runc.go: $(BUILDDIR)/bin/runc
++internal/binutils/runc.go:
+ 	go-bindata -tags \!noembed -pkg binutils -prefix "$(BUILDDIR)/bin" -o $(CURDIR)/internal/binutils/runc.go $(BUILDDIR)/bin
+ 	gofmt -s -w $(CURDIR)/internal/binutils/runc.go
+

img/pkg.yaml

@@ -7,34 +7,44 @@ dependencies:
   - stage: git
   - stage: pkg-config
   - stage: libseccomp
+  - stage: patch
 steps:
   - env:
-      GO111MODULE: off
       GOPATH: /go
       PKGPATH: src/github.com/genuinetools/img
       CGO_ENABLED: "1"
+      LD_LIBRARY_PATH: /toolchain/lib
     sources:
-      - url: https://github.com/genuinetools/img/archive/9856fcb98e30a8152a7f5f1445bb3ecc11598e75.tar.gz
+      - url: https://github.com/genuinetools/img/archive/refs/tags/v0.5.11.tar.gz
         destination: img.tar.gz
-        sha256: 73756b8d6d27d665ceeac8ae06678be477c53a832db2e2e24171672e46811442
-        sha512: bd18a8a83c6f99d63f7e520660ac772cbe80cc2281b71a18f71f447e8f6018f17c31c34143c7d7138e18b8d97a1ff6f70eb1b22f153b962f5ae2ba40778ea182
+        sha256: c6d805ab899db639fa08f0e33fe6673593be8336d288bb1cb4cb81a3229a916f
+        sha512: de80c2c26dd2fb780fb0447f07b5478e1571067ec934fb2adb32dff1aa3853ff872388b9c9c168c139194ab5d38672fb3b4c76c4f77fd8e61ce48bde8cae8c1b
     prepare:
       - |
+        export PATH=${GOPATH}/bin:${TOOLCHAIN}/go/bin:${PATH}
+
+        go install github.com/go-bindata/go-bindata/go-bindata@v3.1.2+incompatible
+
         mkdir -p ${GOPATH}/${PKGPATH}
         tar -xzf img.tar.gz --strip-components=1 -C ${GOPATH}/${PKGPATH}
+
+        cd ${GOPATH}/${PKGPATH}
+        patch -p1 < /pkg/patches/reproducibility.patch
     build:
       - |
         export PATH=${GOPATH}/bin:${TOOLCHAIN}/go/bin:${PATH}
+
         cd ${GOPATH}/${PKGPATH}
-        go get github.com/go-bindata/go-bindata/...
-        make
+        make $PWD/cross/bin/runc
+        touch -ch -t 202108110000 ./cross/bin/runc
+        make internal/binutils/runc.go
+        touch -ch -t 202108110000 internal/binutils/runc.go
+        make static
     install:
       - |
-        cd ${GOPATH}/${PKGPATH}
-        export PATH=${GOPATH}/bin:${TOOLCHAIN}/go/bin:${PATH}
-        make install
         mkdir -p /rootfs${TOOLCHAIN}/bin
-        mv ${GOPATH}/bin/img /rootfs${TOOLCHAIN}/bin
+        cd ${GOPATH}/${PKGPATH}
+        mv img /rootfs${TOOLCHAIN}/bin
 finalize:
   - from: /rootfs
     to: /

ncurses/pkg.yaml

@@ -3,10 +3,12 @@ dependencies:
   - stage: base
 steps:
   - sources:
-      - url: https://ftp.gnu.org/gnu/ncurses/ncurses-6.1.tar.gz
+      - url: https://ftp.gnu.org/gnu/ncurses/ncurses-6.2.tar.gz
         destination: ncurses.tar.gz
-        sha256: aa057eeeb4a14d470101eff4597d5833dcef5965331be3528c08d99cebaa0d17
-        sha512: e308af43f8b7e01e98a55f4f6c4ee4d1c39ce09d95399fa555b3f0cdf5fd0db0f4c4d820b4af78a63f6cf6d8627587114a40af48cfc066134b600520808a77ee
+        sha256: 30306e0c76e0f9f1f0de987cf1c82a5c21e1ce6568b9227f7da5b71cbea86c9d
+        sha512: 4c1333dcc30e858e8a9525d4b9aefb60000cfc727bc4a1062bace06ffc4639ad9f6e54f6bdda0e3a0e5ea14de995f96b52b3327d9ec633608792c99a1e8d840d
+    env:
+      SOURCE_DATE_EPOCH: "1"
     prepare:
       - |
         tar -xzf ncurses.tar.gz --strip-components=1
@@ -30,6 +32,8 @@ steps:
       - |
         cd build
         make DESTDIR=/rootfs install
+
+        rm /rootfs${TOOLCHAIN}/lib/*.a
 finalize:
   - from: /rootfs
     to: /

perl/patches/reproducible-build-date.patch

@@ -0,0 +1,52 @@
+Don't encode the current timestamp.
+
+This affects the output of `perl -V`, specifically the message "Compiled
+at [...]".
+
+The 'cf_time' and 'cf_by' values show up in 'config.h' and
+in 'Config_heavy.pl'.
+
+Use the output of 'uname -s' instead of 'uname -a' to avoid recording
+the kernel version ('uname -o' leads to directory names like
+'x86_64-gnulinux' instead of 'x86_64-linux', which might cause breakage
+down the road.)
+
+From: https://raw.githubusercontent.com/guix-mirror/guix/master/gnu/packages/patches/perl-reproducible-build-date.patch
+
+diff --git a/perl.c b/perl.c
+index 228a0d8..ed38313 100644
+--- a/perl.c
++++ b/perl.c
+@@ -1825,6 +1825,7 @@ S_Internals_V(pTHX_ CV *cv)
+     PUSHs(Perl_newSVpvn_flags(aTHX_ non_bincompat_options,
+ 			      sizeof(non_bincompat_options) - 1, SVs_TEMP));
+
++#define PERL_BUILD_DATE "Jan  1 1970 00:00:00"
+ #ifndef PERL_BUILD_DATE
+ #  ifdef __DATE__
+ #    ifdef __TIME__
+
+--- a/Configure	1970-01-01 01:00:00.000000000 +0100
++++ b/Configure	2016-10-01 14:47:20.017319739 +0200
+@@ -3276,7 +3276,7 @@ $eunicefix tr
+ : Try to determine whether config.sh was made on this system
+ case "$config_sh" in
+ '')
+-myuname=`$uname -a 2>/dev/null`
++myuname=`$uname -s 2>/dev/null`
+ $test -z "$myuname" && myuname=`hostname 2>/dev/null`
+ # Downcase everything to avoid ambiguity.
+ # Remove slashes and single quotes so we can use parts of this in
+@@ -3845,10 +3845,10 @@
+ . ./posthint.sh
+
+ : who configured the system
+-cf_time=`LC_ALL=C; LANGUAGE=C; export LC_ALL; export LANGUAGE; $date 2>&1`
++cf_time="1970-01-01"
+ case "$cf_by" in
+ "")
+-	cf_by=`(logname) 2>/dev/null`
++	cf_by="guix"
+ 	case "$cf_by" in
+ 	"")
+ 		cf_by=`(whoami) 2>/dev/null`

perl/pkg.yaml

@@ -4,35 +4,41 @@ dependencies:
   - stage: patch
 steps:
   - sources:
-      - url: https://www.cpan.org/src/5.0/perl-5.28.2.tar.xz
-        destination: perl.tar.xz
-        sha256: 0b0189bfa4b2da20e899b4bdd746ac402e8f746a58e4fcf5516484157f2aab07
-        sha512: 0f2e4f7cb5d8cf6e00054b3842907e29b6c85902d97fb881d5bea65edbc875fef4e15e064561fac7c8db4939586576dd76a225026c7cca9624261c887b1fdb08
-      - url: https://github.com/arsv/perl-cross/releases/download/1.2.3/perl-cross-1.2.3.tar.gz
-        destination: perl-cross.tar.gz
-        sha256: 84583a1bb5f3a2c64d13d031386c114382e5c62955c6f84a27cec8318536627c
-        sha512: bd4fb3fa835b807c3ffb613a7fe935f92df5d89f603bf3f92fb3d249928f3ba69f078cdd46de5749d53e47bc437756f8e3530d7adc40b6eaa21265295af85616
+    - url: https://www.cpan.org/src/5.0/perl-5.34.0.tar.xz
+      destination: perl.tar.xz
+      sha256: 82c2e5e5c71b0e10487a80d79140469ab1f8056349ca8545140a224dbbed7ded
+      sha512: 691b4b31eacec357191fba777612b4e3eae59e946a22998a50766697c0d61db1d42a9b3bc1e41abf0d1ca1893e4a7c06d7bf3290480cf03d7f79befd7a8a3267
+    env:
+      SOURCE_DATE_EPOCH: "1"
     prepare:
       - |
         tar -xJf perl.tar.xz --strip-components=1
-        tar -xzf perl-cross.tar.gz --strip-components=1
+
+        patch -p1 < /pkg/patches/reproducible-build-date.patch
 
         # This is hardcoded in ./ext/Errno/Errno_pm.P.
         # Copy it from our toolchain to where it is expected.
         mkdir -p /usr/include
         cp ${TOOLCHAIN}/include/errno.h /usr/include/
 
-        ./configure \
-            --prefix=${TOOLCHAIN}
+        ./Configure -des \
+          -Dcccdlflags='-fPIC' \
+          -Dcccdlflags='-fPIC' \
+          -Dccdlflags='-rdynamic' \
+          -Dprefix=${TOOLCHAIN} \
+          -Dvendorprefix=${TOOLCHAIN} \
+          -Duselargefiles \
+          -Dusethreads \
+          -Duseshrplib \
+          -Dd_semctl_semun \
+          -Ud_csh \
+          -Dusenm
     build:
       - |
         make -j $(nproc)
     install:
       - |
-        mkdir -p /rootfs${TOOLCHAIN}/bin
-        cp -v perl cpan/podlators/scripts/pod2man /rootfs${TOOLCHAIN}/bin
-        mkdir -pv /rootfs${TOOLCHAIN}/lib/perl5/5.28.2
-        cp -Rv lib/* /rootfs${TOOLCHAIN}/lib/perl5/5.28.2
+        make DESTDIR=/rootfs install.perl
 finalize:
   - from: /rootfs
     to: /

python3/pkg.yaml

@@ -2,12 +2,15 @@ name: python3
 dependencies:
   - stage: base
   - stage: zlib
+  - stage: patch
 steps:
   - sources:
-      - url: https://www.python.org/ftp/python/3.9.2/Python-3.9.2.tar.xz
+      - url: https://www.python.org/ftp/python/3.9.6/Python-3.9.6.tar.xz
         destination: python.tar.xz
-        sha256: 3c2034c54f811448f516668dce09d24008a0716c3a794dd8639b5388cbde247d
-        sha512: b204d865e4c974951830008c381678876987da267c37bce3b4e488c07aa744658e57b8dc5d248051d0391f3b580e69877f7772abc0a0de5288349d448ccf0789
+        sha256: 397920af33efc5b97f2e0b57e91923512ef89fc5b3c1d21dbfc8c4828ce0108a
+        sha512: 01c529e3207738d8771caeb5ed6217e10745c21aa39832fe3185e1c87fdb6aedead97ce38dbb11e02d873654028bd8071c3f345c18452769520723284efe9dc1
+    env:
+      SOURCE_DATE_EPOCH: "1"
     prepare:
       - |
         tar -xJf python.tar.xz --strip-components=1
@@ -28,6 +31,21 @@ steps:
       - |
         cd build
         make DESTDIR=/rootfs install
+
+        # needed for some packages, especially packages that backport functionality
+        # to 2.x from 3.x
+        for item in /rootfs${TOOLCHAIN}/lib/python3.9/test/*; do
+          if [[ "$item" != */test_support.py* ]]; then
+            rm -rf "$item"
+          else
+            echo $item
+          fi
+        done
+
+        touch /rootfs${TOOLCHAIN}/lib/python3.9/test/__init__.py
+
+        # Determinism: remove all bytecode
+        find /rootfs -type d -name __pycache__ -print0 | xargs -0 -I {} rm -rf "{}"
 finalize:
   - from: /rootfs
     to: /

tools/pkg.yaml

@@ -51,7 +51,6 @@ dependencies:
   - stage: protobuf
   - stage: protoc-gen-go
   - stage: protoc-gen-go-grpc
-  - stage: python2
   - stage: python3
   - stage: rhash
   - stage: sed