github actions: build auth+rec+dnsdist; test auth-api, dnsdist-regres…

…sion, rec-api

.github/workflows/build-and-test-all.yml

@@ -0,0 +1,189 @@
+---
+name: 'Build and test everything'
+
+on:
+  push:
+  pull_request:
+  schedule:
+    - cron: '0 22 * * 3'
+
+jobs:
+  build-auth:
+    name: build auth
+    runs-on: ubuntu-20.04
+    env:
+      UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1:suppressions=/home/runner/work/pdns/pdns/build-scripts/UBSan.supp'
+      ASAN_OPTIONS: detect_leaks=0
+    steps:
+      - uses: actions/checkout@v2.3.4
+        with:
+          fetch-depth: 5
+          submodules: recursive
+      - run: build-scripts/gh-actions-setup-inv  # this runs apt update+upgrade
+      - run: inv install-clang
+      - run: inv install-auth-build-deps
+      - run: inv ci-autoconf
+      - run: inv ci-auth-configure
+      - run: inv ci-auth-make
+      # FIXME: save ccache here?
+      - run: inv ci-auth-install-remotebackend-ruby-deps
+      - run: inv ci-auth-run-unit-tests
+      - run: inv ci-make-install
+      - name: Store the binaries
+        uses: actions/upload-artifact@v2 # this takes 30 seconds, maybe we want to tar
+        with:
+          name: pdns-auth
+          path: /opt/pdns-auth
+
+  build-recursor:
+    name: build recursor
+    runs-on: ubuntu-20.04
+    env:
+      UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1:suppressions=/home/runner/work/pdns/pdns/build-scripts/UBSan.supp'
+      ASAN_OPTIONS: detect_leaks=0
+    defaults:
+      run:
+        working-directory: ./pdns/recursordist/
+    steps:
+      - uses: actions/checkout@v2.3.4
+        with:
+          fetch-depth: 5
+          submodules: recursive
+      - run: ../../build-scripts/gh-actions-setup-inv  # this runs apt update+upgrade
+      - run: inv apt-fresh
+      - run: inv install-clang
+      - run: inv install-rec-build-deps
+      - run: inv ci-autoconf
+      - run: inv ci-rec-configure
+      - run: inv ci-rec-make
+      # FIXME: save ccache here?
+      - run: inv ci-rec-run-unit-tests
+      - run: inv ci-make-install
+      - name: Store the binaries
+        uses: actions/upload-artifact@v2 # this takes 30 seconds, maybe we want to tar
+        with:
+          name: pdns-recursor
+          path: /opt/pdns-recursor
+  build-dnsdist:
+    name: build dnsdist
+    runs-on: ubuntu-20.04
+    env:
+      UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1:suppressions=/home/runner/work/pdns/pdns/build-scripts/UBSan.supp'
+      ASAN_OPTIONS: detect_leaks=0
+    defaults:
+      run:
+        working-directory: ./pdns/dnsdistdist/
+    steps:
+      - uses: actions/checkout@v2.3.4
+        with:
+          fetch-depth: 5
+          submodules: recursive
+      - run: ../../build-scripts/gh-actions-setup-inv  # this runs apt update+upgrade
+      - run: inv apt-fresh
+      - run: inv install-clang
+      - run: inv install-dnsdist-build-deps
+      - run: inv ci-autoconf
+      - run: inv ci-dnsdist-configure
+      - run: inv ci-dnsdist-make
+      # FIXME: save ccache here?
+      - run: inv ci-dnsdist-run-unit-tests
+      - run: inv ci-make-install
+      - name: Store the binaries
+        uses: actions/upload-artifact@v2 # this takes 30 seconds, maybe we want to tar
+        with:
+          name: dnsdist
+          path: /opt/dnsdist
+
+  test-auth-api:
+    needs: build-auth
+    runs-on: ubuntu-20.04
+    env:
+      UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1:suppressions=/home/runner/work/pdns/pdns/build-scripts/UBSan.supp'
+      ASAN_OPTIONS: detect_leaks=0
+    strategy:
+      matrix:
+        include:
+          - backend: gsqlite3
+            image: coscale/docker-sleep
+          - backend: gmysql
+            image: mysql:5
+          - backend: gpgsql
+            image: postgres:9
+          - backend: lmdb
+            image: coscale/docker-sleep
+      fail-fast: false
+    services:
+      database:
+        image: ${{ matrix.image }}
+        env:
+          POSTGRES_USER: runner
+          POSTGRES_HOST_AUTH_METHOD: trust
+          MYSQL_ALLOW_EMPTY_PASSWORD: 1
+        ports:
+          - 3306:3306
+          - 5432:5432
+        # FIXME: this works around dist-upgrade stopping all docker containers. dist-upgrade is huge on these images anyway. Perhaps we do want to run our tasks in a Docker container too.
+        options: >-
+          --restart always
+    steps:
+      - uses: actions/checkout@v2.3.4
+        with:
+          fetch-depth: 5
+          submodules: recursive
+      - name: Fetch the binaries
+        uses: actions/download-artifact@v2
+        with:
+          name: pdns-auth
+          path: /opt/pdns-auth
+      # - name: Setup upterm session
+      #   uses: lhotari/action-upterm@v1
+      - run: build-scripts/gh-actions-setup-inv  # this runs apt update+upgrade
+      - run: inv install-clang-runtime
+      - run: inv install-auth-test-deps -b ${{ matrix.backend }}
+      - run: inv test-api auth -b ${{ matrix.backend }}
+
+  test-recursor-api:
+    needs: build-recursor
+    runs-on: ubuntu-20.04
+    env:
+      UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1:suppressions=/home/runner/work/pdns/pdns/build-scripts/UBSan.supp'
+      ASAN_OPTIONS: detect_leaks=0
+    steps:
+      - uses: actions/checkout@v2.3.4
+        with:
+          fetch-depth: 5
+          submodules: recursive
+      - name: Fetch the binaries
+        uses: actions/download-artifact@v2
+        with:
+          name: pdns-recursor
+          path: /opt/pdns-recursor
+      - run: build-scripts/gh-actions-setup-inv  # this runs apt update+upgrade
+      - run: inv add-auth-repo  # FIXME: do we need this for rec API testing?
+      - run: inv install-clang-runtime
+      - run: inv install-rec-test-deps
+      - run: inv test-api recursor
+
+  test-dnsdist-regression:
+    needs: build-dnsdist
+    runs-on: ubuntu-20.04
+    env:
+      UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1:suppressions=/home/runner/work/pdns/pdns/build-scripts/UBSan.supp'
+      ASAN_OPTIONS: detect_leaks=0
+    steps:
+      - uses: actions/checkout@v2.3.4
+        with:
+          fetch-depth: 5
+          submodules: recursive
+      - name: Fetch the binaries
+        uses: actions/download-artifact@v2
+        with:
+          name: dnsdist
+          path: /opt/dnsdist
+      - run: build-scripts/gh-actions-setup-inv  # this runs apt update+upgrade
+      - run: inv install-clang-runtime
+      - run: inv install-dnsdist-test-deps
+      - run: inv test-dnsdist
+
+
+# FIXME: if we can make upload/download-artifact fasts, running unit tests outside of build can let regression tests start earlier

build-scripts/UBSan.supp

@@ -1 +1,2 @@
-vptr:/usr/include/boost/any.hpp
+vptr:/usr/include/boost/any.hpp
+null:/usr/include/boost/serialization/singleton.hpp

build-scripts/gh-actions-setup-inv

@@ -0,0 +1,5 @@
+#!/bin/bash -x
+sudo apt-get update
+sudo apt-get -qq -y dist-upgrade
+sudo apt-get -qq -y --no-install-recommends install python3-pip
+sudo pip3 install git+https://github.com/pyinvoke/invoke@faa5728a6f76199a3da1750ed952e7efee17c1da

invoke.yaml

@@ -0,0 +1,3 @@
+run:
+    echo: true
+    echo_format: "::endgroup::\n::group::{command}"

modules/remotebackend/Gemfile.lock

@@ -1,14 +1,14 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    ffi (1.10.0)
+    ffi (1.15.3)
     ffi-rzmq (2.0.7)
       ffi-rzmq-core (>= 1.0.7)
     ffi-rzmq-core (1.0.7)
       ffi
-    json (2.3.0)
-    sqlite3 (1.3.13)
-    webrick (1.4.2)
+    json (2.5.1)
+    sqlite3 (1.4.2)
+    webrick (1.7.0)
     zeromqrb (0.1.3)
       ffi-rzmq
 
@@ -22,4 +22,4 @@ DEPENDENCIES
   zeromqrb
 
 BUNDLED WITH
-   1.16.1
+   2.1.4

pdns/Makefile.am

@@ -1300,7 +1300,7 @@ dnspcap2protobuf_LDADD = \
 	$(RT_LIBS)
 
 pdns.conf-dist: pdns_server
-	$(AM_V_GEN)./pdns_server --config=default 2>/dev/null > $@
+	$(AM_V_GEN)./pdns_server --config=default > $@
 
 testrunner_SOURCES = \
 	arguments.cc \