Build and generate Debian packages on multiple arch

.github/workflows/debian.yml

@@ -1,47 +1,116 @@
 # Build and publish Debian packages
+# for various architectures:
+# - amd64
+# - i386
+# - armhf
+# - arm64 / aarch64
+# - armel
+#
+# The native (amd64)
+# linux/386
+#linux/amd64
+#linux/arm
+#linux/arm64
 name: debian
 on:
+  pull_request:
   push:
     branches:
       - "master"
       - "release/**"
     tags:
       - "v*"
 jobs:
-  build:
-    runs-on: "ubuntu-20.04"
+  # Cross-build is currently blocked by sqlite3
+  #build_cross:
+  #  name: Cross-build using Go 1.16 on alpine
+  #  runs-on: "ubuntu-20.04"
+  #  strategy:
+  #    matrix:
+  #      arch: ["amd64", "i386", "aarch64", "armhf"]
+  #  steps:
+  #    - name: Docker
+  #      run: |
+  #        echo $'{\n    "experimental": true\n}' | sudo tee /etc/docker/daemon.json
+  #        sudo service docker restart
+  #    - uses: actions/setup-go@v2
+  #      with:
+  #        go-version: "1.16"
+  #    - uses: actions/checkout@v2
+  #    - run: docker pull --platform linux/${arch} golang:1.16-alpine
+  #    - run: docker run --platform linux/${arch} -v`pwd`:/ooni -w/ooni golang:1.16-alpine ./build.sh _alpine
+  #      #tar -cvzf ooniprobe_${v}_linux_amd64.tar.gz LICENSE.md Readme.md ooniprobe
+  #      #mv ooniprobe ./CLI/linux/amd64/
+  #    #- run: ./smoketest.sh ./CLI/linux/amd64/ooniprobe
+  #    - uses: actions/upload-artifact@v1
+  #      with:
+  #        name: ooniprobe-linux-amd64
+  #        path: ./CLI/linux/amd64/ooniprobe
+
+  #build_amd64:
+  #  runs-on: "ubuntu-20.04"
+  #  name: Build on native amd64 Buster
+  #  steps:
+  #    - name: Docker
+  #      run: |
+  #        echo $'{\n    "experimental": true\n}' | sudo tee /etc/docker/daemon.json
+  #        sudo service docker restart
+  #    - uses: actions/setup-go@v1
+  #      with:
+  #        go-version: "1.14"
+  #    - uses: actions/checkout@v2
+  #    - run: DOCKER_CLI_EXPERIMENTAL=enabled ./build.sh linux_amd64
+  #    - run: sudo apt-get update -q
+  #    - run: sudo apt-get build-dep -y --no-install-recommends .
+  #    - run: |
+  #        VER=$(./CLI/linux/amd64/ooniprobe version)
+  #        if [[ ! $GITHUB_REF =~ ^refs/tags/* ]]; then
+  #          VER="${VER}~${GITHUB_RUN_NUMBER}"
+  #          dch -v $VER "New test version"
+  #          BT_REPO="${BT_REPO}-test"
+  #        else
+  #          dch -v $VER "New release"
+  #        fi
+  #        dpkg-buildpackage -us -uc -b
+  #        find ../ -name "*.deb" -type f
+  #        DEB="../ooniprobe-cli_${VER}_amd64.deb"
+  #        echo no | sudo dpkg -i $DEB
+  #        BT_FNAME="ooniprobe-cli_${VER}_amd64.deb"
+  #        curl --upload-file "${DEB}" -u "${BT_APIUSER}:${BT_APIKEY}" \
+  #          "https://api.bintray.com/content/${BT_ORG}/${BT_REPO}/${BT_PKGNAME}/${VER}/${BT_FNAME};deb_distribution=${DEBDIST};deb_component=main;deb_architecture=amd64;publish=1"
+  #      env:
+  #        DEBDIST: unstable
+  #        BT_APIKEY: ${{ secrets.BT_APIKEY }}
+  #        BT_APIUSER: federicoceratto
+  #        BT_ORG: ooni
+  #        BT_PKGNAME: ooniprobe
+  #        BT_REPO: ooniprobe-debian
+
+  build_archs:
+    runs-on: "ubuntu-latest"
+    strategy:
+      fail-fast: false
+      matrix:
+        arch: ["armv7", "aarch64"]
+    name: Build on ${{ matrix.arch }} Buster
     steps:
-      - name: Docker
-        run: |
-          echo $'{\n    "experimental": true\n}' | sudo tee /etc/docker/daemon.json
-          sudo service docker restart
-      - uses: actions/setup-go@v1
+      - uses: actions/checkout@v2.1.0
+      - uses: uraimo/run-on-arch-action@v2.0.5
+        name: Build ooniprobe
+        id: runcmd
         with:
-          go-version: "1.16"
-      - uses: actions/checkout@v2
-      - run: DOCKER_CLI_EXPERIMENTAL=enabled ./build.sh linux_amd64
-      - run: sudo apt-get update -q
-      - run: sudo apt-get build-dep -y --no-install-recommends .
-      - run: |
-          VER=$(./CLI/linux/amd64/ooniprobe version)
-          if [[ ! $GITHUB_REF =~ ^refs/tags/* ]]; then
-            VER="${VER}~${GITHUB_RUN_NUMBER}"
-            dch -v $VER "New test version"
-            BT_REPO="${BT_REPO}-test"
-          else
-            dch -v $VER "New release"
-          fi
-          dpkg-buildpackage -us -uc -b
-          find ../ -name "*.deb" -type f
-          DEB="../ooniprobe-cli_${VER}_amd64.deb"
-          echo no | sudo dpkg -i $DEB
-          BT_FNAME="ooniprobe-cli_${VER}_amd64.deb"
-          curl --upload-file "${DEB}" -u "${BT_APIUSER}:${BT_APIKEY}" \
-            "https://api.bintray.com/content/${BT_ORG}/${BT_REPO}/${BT_PKGNAME}/${VER}/${BT_FNAME};deb_distribution=${DEBDIST};deb_component=main;deb_architecture=amd64;publish=1"
-        env:
-          DEBDIST: unstable
-          BT_APIKEY: ${{ secrets.BT_APIKEY }}
-          BT_APIUSER: federicoceratto
-          BT_ORG: ooni
-          BT_PKGNAME: ooniprobe
-          BT_REPO: ooniprobe-debian
+          arch: ${{ matrix.arch }}
+          distro: buster
+          githubToken: ${{ github.token }}
+
+          run: |
+            export ARCH=${{ matrix.arch }}
+            export BT_APIKEY=${{ secrets.BT_APIKEY }}
+            ./build.sh _ci_build_on_debian
+            ./build.sh _ci_build_upload_deb
+
+      - name: Upload executable
+        uses: actions/upload-artifact@v1
+        with:
+          name: ooniprobe-linux-${{ matrix.arch }}
+          path: ooniprobe

build.sh

@@ -1,8 +1,77 @@
-#!/bin/sh
+#!/bin/bash
 set -e
 
 # We don't have a git repository when running in github actions
-v=`git describe --tags || echo $GITHUB_SHA`
+if command -v git &> /dev/null; then
+  v=$(git describe --tags || echo $GITHUB_SHA)
+else
+  v=GITHUB_SHA
+fi
+
+ci_build_on_debian() {
+  # Install build dependencies, go, and build the probe
+  set -eux
+  case $ARCH in
+    armv6) HDR=armmp;;
+    armv7) HDR=armmp;;
+    aarch64) HDR=arm64;;
+    *) HDR=$ARCH;;
+  esac
+  uname -a
+  apt-get -qq update
+  apt-get install -qq -y --no-install-recommends gcc git linux-headers-$HDR wget musl-dev ca-certificates libc6-dev
+
+  case $ARCH in
+    armv6) GOARCH=armv6l;; #ARMv6
+    armv7) GOARCH=armv6l;; #ARMv7: not available, use v6
+    aarch64) GOARCH=arm64;; #ARMv8
+    *) GOARCH=$ARCH;;
+  esac
+  tarfn=go1.16.linux-$GOARCH.tar.gz
+  echo Downloading $tarfn
+  wget --no-verbose https://golang.org/dl/$tarfn
+  tar xfz $tarfn
+  export PATH=$PATH:$(pwd)/go/bin
+  go version
+  go env
+
+  go build -tags netgo -ldflags='-s -w -extldflags "-static"' ./cmd/ooniprobe
+}
+
+ci_build_upload_deb() {
+  # Build and upload .deb package
+  set -eu
+  for e in ARCH BT_APIKEY GITHUB_REF GITHUB_RUN_NUMBER; do
+    [[ -z "${!e}" ]] && echo "Please set env var $e" && exit 1
+  done
+  case $ARCH in
+    armv6) DARCH=armel;;
+    armv7) DARCH=armhf;;
+    *) DARCH=$ARCH;;
+  esac
+  DEBDIST=unstable
+  BT_APIUSER=federicoceratto
+  BT_ORG=ooni
+  BT_PKGNAME=ooniprobe
+  apt-get -qq update
+  # apt-get build-dep -y --no-install-recommends .
+  apt-get install -q  -y --no-install-recommends dpkg-dev build-essential devscripts debhelper
+  VER=$(./ooniprobe version)
+  if [[ ! $GITHUB_REF =~ ^refs/tags/* ]]; then
+    VER="${VER}~${GITHUB_RUN_NUMBER}"
+    dch -v $VER "New test version"
+    BT_REPO=ooniprobe-debian-test
+  else
+    dch -v $VER "New release"
+    BT_REPO=ooniprobe-debian
+  fi
+  dpkg-buildpackage -us -uc -b
+  find ../ -name "*.deb" -type f
+  DEB="../ooniprobe-cli_${VER}_${DARCH}.deb"
+  BT_FNAME="ooniprobe-cli_${VER}_${DARCH}.deb"
+  curl --upload-file "${DEB}" -u "${BT_APIUSER}:${BT_APIKEY}" \
+            "https://api.bintray.com/content/${BT_ORG}/${BT_REPO}/${BT_PKGNAME}/${VER}/${BT_FNAME};deb_distribution=${DEBDIST};deb_component=main;deb_architecture=${DARCH};publish=1"
+}
 
 case $1 in
   windows)
@@ -79,6 +148,14 @@ case $1 in
     shasum -a 256 ooniprobe_${v}_*_*.* > ooniprobe_checksums.txt
     ;;
 
+  _ci_build_on_debian)
+    ci_build_on_debian
+    ;;
+
+  _ci_build_upload_deb)
+    ci_build_upload_deb
+    ;;
+
   *)
 
     set +x

debian/ooniprobe-cli.install

@@ -1,2 +1,2 @@
-./CLI/linux/amd64/ooniprobe usr/bin
+ooniprobe usr/bin
 debian/ooniprobe.conf.disabled /etc/ooniprobe

debian/rules

@@ -11,7 +11,7 @@ override_dh_auto_clean:
 
 override_dh_auto_build:
 	# The ooniprobe binary is provided
-	./CLI/linux/amd64/ooniprobe --help-man > debian/ooniprobe.1
+	./ooniprobe --help-man > debian/ooniprobe.1
 
 override_dh_dwz:
 	true