Merge #689: Repair CI

9acf98c coldcard: Drop PyCoin usage entirely (Andrew Chow)
e956b5c tests, ledger: specify nanos as the model (Andrew Chow)
c553632 tests, jade: specify pc-bios dir to qemu (Andrew Chow)
0dd20b4 ci, speculos: install pytesseract as speculos dependency (Andrew Chow)
30b88cf tests: Use h as hardened marker (Andrew Chow)
a1b0d64 ci, trezor: Set protobuf version to 3.20 (Andrew Chow)
6818400 ci: Run dist tests on all versions (Andrew Chow)
7122515 ci: Patch keepkey googletest to build (Andrew Chow)
d59d452 ci: Add slirp dep for jade emulator (Andrew Chow)
0713b2c ci: Remove global container (Andrew Chow)
f8c6f1c ci: Update speculos cmake command (Andrew Chow)
735aba9 ci: Use python 3.8 for main image (Andrew Chow)
5e43ff6 ci: Use upstream ledger app builder images (Andrew Chow)
3ff8ccc ci: Consolidate lint tasks (Andrew Chow)
e24871f build: Use debian oldoldstable instead of fixed version (Andrew Chow)
e21c862 ci: Test against python 3.11 (Andrew Chow)
48baa1d ci: Break dockerfile build caches (Andrew Chow)
0f0caae common: enforce enum name is a str (Andrew Chow)
b7b9b3c bitbox02: implement missing encode_check (Andrew Chow)
7db43b9 flake8: exclude bitbox02_lib (Andrew Chow)

Pull request description:

  Several changes to the various simulators and python dependencies have left CI broken for a while. This omnibus PR contains some fixes for HWI itself, and the rest for getting CI running again.

  * Implements a missing base58 function used by BitBox02
  * Change keypath test to use `h` for hardened indicator since Core changed this
  * Some type checker and linter fixes
  * Updated the dockerfiles to include new simulator dependencies
  * Test against python 3.11
  * Update simulator building dockerfile to use python 3.8
  * Update distribution building dockerfile to use debian oldoldstable rather than fixed version
  * Another patch for Keepkey's simulator to make it build on modern compilers
  * Some simulator starting command updates so that they actually start

Top commit has no ACKs.

Tree-SHA512: 43c804438bfe0e8be179ae4c5e705f98ff02dba714ee8a2c42d1f8290914e7297540ccb0b429a4f27e3d352db886c615f37902b601f4368c949bbd9be5e4d3ee

.cirrus.yml

@@ -1,9 +1,28 @@
-container:
-  dockerfile: ci/cirrus.Dockerfile
-
 env:
   EMAIL: cirrus@cirrus-ci.org
 
+python_version_matrix_template: &PYTHON_VERSION_MATRIX_TEMPLATE
+  - container:
+      dockerfile: ci/py37.Dockerfile
+    env:
+      PYTHON: 3.7
+  - container:
+      dockerfile: ci/cirrus.Dockerfile
+    env:
+      PYTHON: 3.8
+  - container:
+      dockerfile: ci/py39.Dockerfile
+    env:
+      PYTHON: 3.9
+  - container:
+      dockerfile: ci/py310.Dockerfile
+    env:
+      PYTHON: 3.10
+  - container:
+      dockerfile: ci/py311.Dockerfile
+    env:
+      PYTHON: 3.11
+
 device_matrix_template: &DEVICE_MATRIX_TEMPLATE
   - env:
       DEVICE: --trezor-1
@@ -81,8 +100,8 @@ device_matrix_template: &DEVICE_MATRIX_TEMPLATE
       - wget -nv "https://api.cirrus-ci.com/v1/artifact/build/$CIRRUS_BUILD_ID/Ledger App Builder/built_ledger_app/app-bitcoin-new/bin/app.elf"
       - mv app.elf test/work/speculos/apps/btc-test.elf
     sim_install_script:
-      - poetry run pip install construct flask-restful jsonschema mnemonic pyelftools pillow requests
-      - pip install construct flask-restful jsonschema mnemonic pyelftools pillow requests
+      - poetry run pip install construct flask-restful jsonschema mnemonic pyelftools pillow requests pytesseract
+      - pip install construct flask-restful jsonschema mnemonic pyelftools pillow requests pytesseract
   - env:
       DEVICE: --ledger-legacy
     depends_on:
@@ -95,8 +114,8 @@ device_matrix_template: &DEVICE_MATRIX_TEMPLATE
       - wget -nv "https://api.cirrus-ci.com/v1/artifact/build/$CIRRUS_BUILD_ID/bitcoind_builder/bitcoin/bitcoin.tar.gz"
       - tar -xvf "bitcoin.tar.gz"
     sim_install_script:
-      - poetry run pip install construct flask-restful jsonschema mnemonic pyelftools pillow requests
-      - pip install construct flask-restful jsonschema mnemonic pyelftools pillow requests
+      - poetry run pip install construct flask-restful jsonschema mnemonic pyelftools pillow requests pytesseract
+      - pip install construct flask-restful jsonschema mnemonic pyelftools pillow requests pytesseract
   - env:
       DEVICE: --keepkey
     depends_on:
@@ -109,12 +128,11 @@ device_matrix_template: &DEVICE_MATRIX_TEMPLATE
       - wget -nv "https://api.cirrus-ci.com/v1/artifact/build/$CIRRUS_BUILD_ID/bitcoind_builder/bitcoin/bitcoin.tar.gz"
       - tar -xvf "bitcoin.tar.gz"
 
-lint_task:
-  test_script:
-    - flake8
-
 task:
+  container:
+    image: python:latest
   install_script:
+    - pip install poetry
     - poetry install
   matrix:
     - name: Type Check
@@ -146,6 +164,8 @@ task:
         hwilib/udevinstaller.py
     - name: Non-Device Tests
       test_script: cd test; poetry run ./run_tests.py; cd ..
+    - name: lint
+      test_script: poetry run flake8
 
 wine_builder_task:
   container:
@@ -155,6 +175,8 @@ wine_builder_task:
     - find dist -type f -exec sha256sum {} \;
 
 bitcoind_builder_task:
+  container:
+    dockerfile: ci/cirrus.Dockerfile
   bitcoind_cache:
     folder: test/work/bitcoin
   ccache_cache:
@@ -168,6 +190,8 @@ bitcoind_builder_task:
     path: "bitcoin.tar.gz"
 
 task:
+  container:
+    dockerfile: ci/cirrus.Dockerfile
   env:
     DEVICE: --trezor-1
   name: Trezor 1 Sim Builder
@@ -180,6 +204,8 @@ task:
     path: "trezor-firmware.tar.gz"
 
 task:
+  container:
+    dockerfile: ci/cirrus.Dockerfile
   env:
     DEVICE: --trezor-t
   name: Trezor T Sim Builder
@@ -192,6 +218,8 @@ task:
     path: "trezor-firmware.tar.gz"
 
 task:
+  container:
+    dockerfile: ci/cirrus.Dockerfile
   env:
     DEVICE: --coldcard
   name: Coldcard Sim Builder
@@ -204,6 +232,8 @@ task:
     path: "coldcard-mpy.tar.gz"
 
 task:
+  container:
+    dockerfile: ci/cirrus.Dockerfile
   env:
     DEVICE: --bitbox01
   name: Bitbox01 Sim Builder
@@ -216,6 +246,8 @@ task:
     path: "mcu.tar.gz"
 
 task:
+  container:
+    dockerfile: ci/cirrus.Dockerfile
   env:
     DEVICE: --jade
   name: Jade Sim Builder
@@ -228,6 +260,8 @@ task:
     path: "jade.tar.gz"
 
 task:
+  container:
+    dockerfile: ci/cirrus.Dockerfile
   env:
     DEVICE: --ledger
   name: Ledger Sim Builder
@@ -240,6 +274,8 @@ task:
     path: "speculos.tar.gz"
 
 task:
+  container:
+    dockerfile: ci/cirrus.Dockerfile
   env:
     DEVICE: --keepkey
   name: Keepkey Sim Builder
@@ -265,7 +301,7 @@ dist_builder_task:
 ledger_app_builder_task:
   name: Ledger App Builder
   container:
-    dockerfile: ci/ledger-app-builder.Dockerfile
+    image: ghcr.io/ledgerhq/ledger-app-builder/ledger-app-builder:latest
   build_script:
     - git clone https://github.com/LedgerHQ/app-bitcoin-new.git
     - cd app-bitcoin-new
@@ -274,19 +310,21 @@ ledger_app_builder_task:
     path: app-bitcoin-new/bin/app.elf
 
 task:
+  matrix:
+    << : *PYTHON_VERSION_MATRIX_TEMPLATE
   matrix:
     << : *DEVICE_MATRIX_TEMPLATE
   fetch_dist_script:
     - wget -nv "https://api.cirrus-ci.com/v1/artifact/build/$CIRRUS_BUILD_ID/dist_builder/built_dist.zip"
     - unzip built_dist.zip
   matrix:
-    - name: $DEVICE Wheel
+    - name: Python $PYTHON $DEVICE Wheel
       install_script: pip install dist/*.whl
       test_script: cd test; ./run_tests.py $DEVICE --interface=cli --device-only; cd ..
-    - name: $DEVICE Sdist
+    - name: Python $PYTHON $DEVICE Sdist
       install_script: pip install $(find dist -name "*.tar.gz" -a -not -name "*amd64*")
       test_script: cd test; ./run_tests.py $DEVICE --interface=cli --device-only; cd ..
-    - name: $DEVICE Bindist
+    - name: Python $PYTHON $DEVICE Bindist
       install_script: poetry install
       untar_bindist_script: cd dist; tar -xvf hwi*linux*.tar.gz; cd ..
       test_script: cd test; poetry run ./run_tests.py $DEVICE --interface=bindist --device-only; cd ..
@@ -295,22 +333,7 @@ task:
 
 task:
   matrix:
-    - container:
-        dockerfile: ci/cirrus.Dockerfile
-      env:
-        PYTHON: 3.7
-    - container:
-        dockerfile: ci/py38.Dockerfile
-      env:
-        PYTHON: 3.8
-    - container:
-        dockerfile: ci/py39.Dockerfile
-      env:
-        PYTHON: 3.9
-    - container:
-        dockerfile: ci/py310.Dockerfile
-      env:
-        PYTHON: 3.10
+    << : *PYTHON_VERSION_MATRIX_TEMPLATE
   install_script: poetry install
   matrix:
     << : *DEVICE_MATRIX_TEMPLATE

.flake8

@@ -1,4 +1,4 @@
 [flake8]
-exclude = *.pyc,__pycache__,hwilib/devices/ledger_bitcoin/,hwilib/devices/btchip,hwilib/devices/ckcc/,hwilib/devices/jadepy/,hwilib/devices/trezorlib/,test/work/,hwilib/ui
+exclude = *.pyc,__pycache__,hwilib/devices/ledger_bitcoin/,hwilib/devices/btchip,hwilib/devices/ckcc/,hwilib/devices/jadepy/,hwilib/devices/trezorlib/,test/work/,hwilib/ui,hwilib/devices/bitbox02_lib
 ignore = E261,E302,E305,E501,E722,W5
 per-file-ignores = setup.py:E122

ci/cirrus.Dockerfile

@@ -1,4 +1,6 @@
-FROM python:3.7
+# Cache break (modify this line to break cirrus' dockerfile build cache) 1
+
+FROM python:3.8
 
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update
@@ -30,6 +32,7 @@ RUN apt-get install -y \
     libsdl2-dev \
     libsdl2-image-dev \
     libssl-dev \
+    libslirp-dev \
     libtool \
     libudev-dev \
     libusb-1.0-0-dev \

ci/py310.Dockerfile

@@ -1,3 +1,5 @@
+# Cache break (modify this line to break cirrus' dockerfile build cache) 1
+
 FROM python:3.10
 
 ENV DEBIAN_FRONTEND=noninteractive
@@ -8,6 +10,7 @@ RUN apt-get install -y \
     libpcsclite-dev \
     libsdl2-dev \
     libsdl2-image-dev \
+    libslirp-dev \
     libudev-dev \
     libusb-1.0-0-dev \
     qemu-user-static \

ci/py38.Dockerfile → ci/py311.Dockerfile

@@ -1,4 +1,6 @@
-FROM python:3.8
+# Cache break (modify this line to break cirrus' dockerfile build cache)
+
+FROM python:3.11
 
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update
@@ -8,6 +10,7 @@ RUN apt-get install -y \
     libpcsclite-dev \
     libsdl2-dev \
     libsdl2-image-dev \
+    libslirp-dev \
     libudev-dev \
     libusb-1.0-0-dev \
     qemu-user-static \

ci/py37.Dockerfile

@@ -0,0 +1,23 @@
+# Cache break (modify this line to break cirrus' dockerfile build cache) 1
+
+FROM python:3.7
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update
+RUN apt-get install -y \
+    cython3 \
+    git \
+    libpcsclite-dev \
+    libsdl2-dev \
+    libsdl2-image-dev \
+    libslirp-dev \
+    libudev-dev \
+    libusb-1.0-0-dev \
+    qemu-user-static \
+    swig
+
+RUN pip install poetry flake8
+
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
+ENV LANGUAGE=C.UTF-8

ci/py39.Dockerfile

@@ -1,3 +1,5 @@
+# Cache break (modify this line to break cirrus' dockerfile build cache) 1
+
 FROM python:3.9
 
 ENV DEBIAN_FRONTEND=noninteractive
@@ -8,6 +10,7 @@ RUN apt-get install -y \
     libpcsclite-dev \
     libsdl2-dev \
     libsdl2-image-dev \
+    libslirp-dev \
     libudev-dev \
     libusb-1.0-0-dev \
     qemu-user-static \

contrib/build.Dockerfile

@@ -1,4 +1,4 @@
-FROM debian:stretch-slim
+FROM debian:oldoldstable-slim
 
 SHELL ["/bin/bash", "-c"]
 

hwilib/_base58.py

@@ -99,6 +99,11 @@ def decode_check(s: str) -> bytes:
         raise ValueError("Invalid checksum")
     return payload
 
+def encode_check(b: bytes) -> str:
+    checksum = hash256(b)[0:4]
+    data = b + checksum
+    return encode(data)
+
 def get_xpub_fingerprint(s: str) -> bytes:
     """
     Get the parent fingerprint from an extended public key