Tools for deterministic builds of standalone binaries and distribution archives

.python-version

@@ -1 +1 @@
-3.5.6
+3.6.8

.travis.yml

@@ -2,7 +2,7 @@ language: python
 os: linux
 dist: xenial
 python:
-    - '3.5'
+    - '3.6.8'
 cache:
     pip: true
     ccache: true
@@ -40,18 +40,41 @@ addons:
             - cython3
             - ccache
 install:
-    - pip install pipenv pysdl2 python-bitcoinrpc protobuf
+    - pip install pipenv pysdl2 python-bitcoinrpc protobuf poetry
     # From trezor-mcu to get the correct protobuf version
     - curl -LO "https://github.com/google/protobuf/releases/download/v3.4.0/protoc-3.4.0-linux-x86_64.zip"
     - unzip "protoc-3.4.0-linux-x86_64.zip" -d protoc
     - export PATH="$(pwd)/protoc/bin:$PATH"
     # Build emulators/simulators and bitcoind
     - cd test; ./setup_environment.sh; cd ..
     - pip uninstall -y trezor # Hack to get rid of master branch version of trezor that is installed for trezor-mcu build
-    - python setup.py install
+    - poetry install
 jobs:
     include:
         -   name: With process_commands interface
-            script: cd test; ./run_tests.py --interface=library
+            script: cd test; poetry run ./run_tests.py --interface=library
         -   name: With command line interface
-            script: cd test; ./run_tests.py --interface=cli
+            script: cd test; poetry run ./run_tests.py --interface=cli
+        -   name: With linux binary distribution command line interface
+            services: docker
+            before_script:
+                - docker build -t hwi-builder -f contrib/build.Dockerfile .
+            script:
+                - docker run -it --name hwi-builder -v $PWD:/opt/hwi --rm  --workdir /opt/hwi hwi-builder /bin/bash -c "contrib/build_bin.sh && contrib/build_dist.sh"
+                - sudo chown -R `whoami`:`whoami` dist/
+                - cd test; poetry run ./run_tests.py --interface=bindist
+                - cd ..; sha256sum dist/*
+        -   name: macOS binary distribution (no tests)
+            os: osx
+            osx_image: xcode7.3
+            language: generic
+            addons:
+                artifacts:
+                    working_dir: dist
+            install:
+                - brew update && brew upgrade pyenv
+                - brew install libusb
+                - cat contrib/reproducible-python.diff | PYTHON_CONFIGURE_OPTS="--enable-framework" BUILD_DATE="Jan  1 2019" BUILD_TIME="00:00:00" pyenv install -kp 3.6.8
+            script:
+                - contrib/build_bin.sh
+                - shasum -a 256 dist/*

contrib/README.md

@@ -0,0 +1,21 @@
+# Assorted tools
+
+## `build_bin.sh`
+
+Creates a virtualenv with the locked dependencies using Poetry. Then uses pyinstaller to create a standalone binary for the OS type currently running.
+
+## `build_dist.sh`
+
+Creates a virtualenv with the locked dependencies using Poetry. Then uses Poetry to produce deterministic builds of the wheel and sdist for upload to PyPi
+
+`faketime` needs to be installed
+
+## `build_wine.sh`
+
+Sets up Wine with Python and everything needed to build Windows binaries. Creates a virtualenv with the locked dependencies using Poetry. Then uses pyinstaller to create a standalone Windows binary.
+
+`wine` needs to be installed
+
+## `generate_setup.sh`
+
+Builds the source distribution and extracts the setup.py from it.

contrib/build.Dockerfile

@@ -0,0 +1,48 @@
+FROM debian:stretch-slim
+
+SHELL ["/bin/bash", "-c"]
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update
+RUN apt-get install -y \
+    apt-transport-https \
+    git \
+    make \
+    build-essential \
+    libssl-dev \
+    zlib1g-dev \
+    libbz2-dev \
+    libreadline-dev \
+    libsqlite3-dev \
+    wget \
+    curl \
+    llvm \
+    libncurses5-dev \
+    xz-utils \
+    libxml2-dev \
+    libxmlsec1-dev \
+    libffi-dev \
+    liblzma-dev \
+    libusb-1.0-0-dev \
+    libudev-dev \
+    faketime
+
+RUN curl https://pyenv.run | bash
+ENV PATH="/root/.pyenv/bin:$PATH"
+COPY contrib/reproducible-python.diff /opt/reproducible-python.diff
+ENV PYTHON_CONFIGURE_OPTS="--enable-shared"
+ENV BUILD_DATE="Jan  1 2019"
+ENV BUILD_TIME="00:00:00"
+RUN eval "$(pyenv init -)" && eval "$(pyenv virtualenv-init -)" && cat /opt/reproducible-python.diff | pyenv install -kp 3.6.8
+
+RUN dpkg --add-architecture i386 
+RUN wget -nc https://dl.winehq.org/wine-builds/winehq.key
+RUN apt-key add winehq.key
+RUN echo "deb https://dl.winehq.org/wine-builds/debian/ stretch main" >> /etc/apt/sources.list
+RUN apt-get update
+RUN apt-get install --install-recommends -y \
+    wine-stable-amd64 \
+    wine-stable-i386 \
+    wine-stable \
+    winehq-stable \
+    p7zip-full

contrib/build_bin.sh

@@ -0,0 +1,26 @@
+#! /bin/bash
+# Script for building standalone binary releases deterministically
+
+eval "$(pyenv init -)"
+eval "$(pyenv virtualenv-init -)"
+pip install -U pip
+pip install poetry
+
+# Setup poetry and install the dependencies
+poetry install
+
+# We now need to remove debugging symbols and build id from the hidapi SO file
+so_dir=`dirname $(dirname $(poetry run which python))`/lib/python3.6/site-packages
+find ${so_dir} -name '*.so' -type f -execdir strip '{}' \;
+if [[ $OSTYPE != *"darwin"* ]]; then
+    find ${so_dir} -name '*.so' -type f -execdir strip -R .note.gnu.build-id '{}' \;
+fi
+
+# We also need to change the timestamps of all of the base library files
+lib_dir=`pyenv root`/versions/3.6.8/lib/python3.6
+TZ=UTC find ${lib_dir} -name '*.py' -type f -execdir touch -t "201901010000.00" '{}' \;
+
+# Make the standalone binary
+export PYTHONHASHSEED=42
+poetry run pyinstaller hwi.spec
+unset PYTHONHASHSEED

contrib/build_dist.sh

@@ -0,0 +1,15 @@
+#! /bin/bash
+# Script for building pypi distribution archives deterministically
+
+eval "$(pyenv init -)"
+eval "$(pyenv virtualenv-init -)"
+pip install -U pip
+pip install poetry
+
+# Setup poetry and install the dependencies
+poetry install
+
+# Make the distribution archives for pypi
+poetry build -f wheel
+# faketime is needed to make sdist detereministic
+TZ=UTC faketime -f "2019-01-01 00:00:00" poetry build -f sdist

contrib/build_wine.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+# Script which sets up Wine and builds the Windows standalone binary
+
+set -e
+
+PYTHON_VERSION=3.6.8
+
+PYTHON_FOLDER="python3"
+PYHOME="c:/$PYTHON_FOLDER"
+PYTHON="wine $PYHOME/python.exe -OO -B"
+
+LIBUSB_URL=https://github.com/libusb/libusb/releases/download/v1.0.22/libusb-1.0.22.7z
+LIBUSB_HASH="671f1a420757b4480e7fadc8313d6fb3cbb75ca00934c417c1efa6e77fb8779b"
+
+wine 'wineboot'
+
+# Install Python
+# Get the PGP keys
+wget -N -c "https://www.python.org/static/files/pubkeys.txt"
+gpg --import pubkeys.txt
+rm pubkeys.txt
+
+# Install python components
+for msifile in core dev exe lib pip tools; do
+    wget -N -c "https://www.python.org/ftp/python/$PYTHON_VERSION/amd64/${msifile}.msi"
+    wget -N -c "https://www.python.org/ftp/python/$PYTHON_VERSION/amd64/${msifile}.msi.asc"
+    gpg --verify "${msifile}.msi.asc" "${msifile}.msi"
+    wine msiexec /i "${msifile}.msi" /qb TARGETDIR=$PYHOME
+    rm $msifile.msi*
+done
+
+# Get libusb
+wget -N -c -O libusb.7z "$LIBUSB_URL"
+echo "$LIBUSB_HASH  libusb.7z" | sha256sum -c
+7za x -olibusb libusb.7z -aoa
+cp libusb/MS64/dll/libusb-1.0.dll ~/.wine/drive_c/python3/
+rm -r libusb*
+
+# Update pip
+$PYTHON -m pip install -U pip
+
+# Install Poetry and things needed for pyinstaller
+$PYTHON -m pip install poetry
+
+# We also need to change the timestamps of all of the base library files
+lib_dir=~/.wine/drive_c/python3/Lib
+TZ=UTC find ${lib_dir} -name '*.py' -type f -execdir touch -t "201901010000.00" '{}' \;
+
+# Do the build
+POETRY="wine $PYHOME/Scripts/poetry.exe"
+$POETRY install -E windist
+export PYTHONHASHSEED=42
+$POETRY run pyinstaller hwi.spec
+unset PYTHONHASHSEED

contrib/generate_setup.sh

@@ -0,0 +1,32 @@
+#! /bin/bash
+# Generates the setup.py file
+
+set -e
+
+# Setup poetry and install the dependencies
+poetry install
+
+# Build the source distribution
+poetry build -f sdist
+
+# Extract setup.py from the distribution
+unset -v tarball
+for file in dist/*
+do
+    if [[ $file -nt $tarball && $file == *".tar.gz" ]]
+    then
+        tarball=$file
+    fi
+done
+unset -v toextract
+for file in `tar -tf $tarball`
+do
+    if [[ $file == *"setup.py" ]]
+    then
+        toextract=$file
+    fi
+done
+tar -xf $tarball $toextract
+mv $toextract .
+dir=`echo $toextract | cut -f1 -d"/"`
+rm -r $dir

contrib/pyinstaller-hooks/hook-hwilib.devices.py

@@ -0,0 +1,4 @@
+from hwilib.devices import __all__
+hiddenimports = []
+for d in __all__:
+    hiddenimports.append('hwilib.devices.' + d)

contrib/reproducible-python.diff

@@ -0,0 +1,13 @@
+# DP: Build getbuildinfo.o with DATE/TIME values when defined
+
+--- Makefile.pre.in
++++ Makefile.pre.in
+@@ -741,6 +741,8 @@ Modules/getbuildinfo.o: $(PARSER_OBJS) \
+ 	      -DGITVERSION="\"`LC_ALL=C $(GITVERSION)`\"" \
+ 	      -DGITTAG="\"`LC_ALL=C $(GITTAG)`\"" \
+ 	      -DGITBRANCH="\"`LC_ALL=C $(GITBRANCH)`\"" \
++	      $(if $(BUILD_DATE),-DDATE='"$(BUILD_DATE)"') \
++	      $(if $(BUILD_TIME),-DTIME='"$(BUILD_TIME)"') \
+ 	      -o $@ $(srcdir)/Modules/getbuildinfo.c
+
+ Modules/getpath.o: $(srcdir)/Modules/getpath.c Makefile

docs/release-process.md

@@ -0,0 +1,51 @@
+# Release Process
+
+1. Bump version number in `pyproject.toml`, generate the setup.py file, and git tag release
+2. Build distribution archives for PyPi with `contrib/build_dist.sh`
+3. For MacOS and Linux, use `contrib/build_bin.sh`. This needs to be run on a MacOS machine for the MacOS binary and on a Linux machine for the linux one.
+4. For Windows, use `contrib/build_wine.sh` to build the Windows binary using wine
+5. Upload distribution archives to PyPi
+6. Upload distribution archives and standalone binaries to Github
+
+## Deterministic builds with Docker
+
+Create the docker image:
+
+```
+docker build --no-cache -t hwi-builder -f contrib/build.Dockerfile .
+```
+
+Build everything
+
+```
+docker run -it --name hwi-builder -v $PWD:/opt/hwi --rm  --workdir /opt/hwi hwi-builder /bin/bash -c "contrib/build_bin.sh && contrib/build_dist.sh && contrib/build_wine.sh"
+```
+
+## Building macOS binary
+
+Note that the macOS build is non-deterministic.
+
+First install [pyenv](https://github.com/pyenv/pyenv) using whichever method you prefer.
+
+Then a deterministic build of Python 3.6.8 needs to be installed. This can be done with the patch in `contrib/reproducible-python.diff`. First `cd` into HWI's source tree. Then use:
+
+```
+cat contrib/reproducible-python.diff | PYTHON_CONFIGURE_OPTS="--enable-framework" BUILD_DATE="Jan  1 2019" BUILD_TIME="00:00:00" pyenv install -kp 3.6.8
+```
+
+Make sure that python 3.6.8 is active
+
+```
+$ python --version
+Python 3.6.8
+```
+
+Now install [Poetry](https://github.com/sdispater/poetry) with `pip install poetry`
+
+Additional dependencies can be installed with:
+
+```
+brew install libusb
+```
+
+Build the binaries by using `contrib/build_bin.sh`.

hwi.spec

@@ -0,0 +1,42 @@
+# -*- mode: python -*-
+import platform
+import subprocess
+
+block_cipher = None
+
+binaries = []
+if platform.system() == 'Windows':
+    binaries = [("c:/python3/libusb-1.0.dll", ".")]
+elif platform.system() == 'Linux':
+    binaries = [("/lib/x86_64-linux-gnu/libusb-1.0.so.0", ".")]
+elif platform.system() == 'Darwin':
+    find_brew_libusb_proc = subprocess.Popen(['brew', '--prefix', 'libusb'], stdout=subprocess.PIPE)
+    libusb_path = find_brew_libusb_proc.communicate()[0]
+    binaries = [(libusb_path.rstrip().decode() + "/lib/libusb-1.0.dylib", ".")]
+
+a = Analysis(['hwi.py'],
+             binaries=binaries,
+             datas=[],
+             hiddenimports=[],
+             hookspath=['contrib/pyinstaller-hooks/'],
+             runtime_hooks=[],
+             excludes=[],
+             win_no_prefer_redirects=False,
+             win_private_assemblies=False,
+             cipher=block_cipher,
+             noarchive=False)
+pyz = PYZ(a.pure, a.zipped_data,
+             cipher=block_cipher)
+exe = EXE(pyz,
+          a.scripts,
+          a.binaries,
+          a.zipfiles,
+          a.datas,
+          [],
+          name='hwi',
+          debug=False,
+          bootloader_ignore_signals=False,
+          strip=False,
+          upx=True,
+          runtime_tmpdir=None,
+          console=True )