From 99bb353016a50c954ead0195d81b1a23a8fc2acb Mon Sep 17 00:00:00 2001 From: Peter van Dijk Date: Thu, 25 Mar 2021 13:48:49 +0100 Subject: [PATCH] github actions: build auth+rec+dnsdist; test auth-api, dnsdist-regression, rec-api --- .github/workflows/build-and-test-all.yml | 189 ++++++++++++ build-scripts/UBSan.supp | 3 +- build-scripts/gh-actions-setup-inv | 5 + invoke.yaml | 3 + modules/remotebackend/Gemfile.lock | 10 +- pdns/Makefile.am | 2 +- tasks.py | 347 +++++++++++++++++++++++ 7 files changed, 552 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/build-and-test-all.yml create mode 100755 build-scripts/gh-actions-setup-inv create mode 100644 invoke.yaml create mode 100644 tasks.py diff --git a/.github/workflows/build-and-test-all.yml b/.github/workflows/build-and-test-all.yml new file mode 100644 index 000000000000..d82fd70a3940 --- /dev/null +++ b/.github/workflows/build-and-test-all.yml @@ -0,0 +1,189 @@ +--- +name: 'Build and test everything' + +on: + push: + pull_request: + schedule: + - cron: '0 22 * * 3' + +jobs: + build-auth: + name: build auth + runs-on: ubuntu-20.04 + env: + UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1:suppressions=/home/runner/work/pdns/pdns/build-scripts/UBSan.supp' + ASAN_OPTIONS: detect_leaks=0 + steps: + - uses: actions/checkout@v2.3.4 + with: + fetch-depth: 5 + submodules: recursive + - run: build-scripts/gh-actions-setup-inv # this runs apt update+upgrade + - run: inv install-clang + - run: inv install-auth-build-deps + - run: inv ci-autoconf + - run: inv ci-auth-configure + - run: inv ci-auth-make + # FIXME: save ccache here? + - run: inv ci-auth-install-remotebackend-ruby-deps + - run: inv ci-auth-run-unit-tests + - run: inv ci-make-install + - name: Store the binaries + uses: actions/upload-artifact@v2 # this takes 30 seconds, maybe we want to tar + with: + name: pdns-auth + path: /opt/pdns-auth + + build-recursor: + name: build recursor + runs-on: ubuntu-20.04 + env: + UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1:suppressions=/home/runner/work/pdns/pdns/build-scripts/UBSan.supp' + ASAN_OPTIONS: detect_leaks=0 + defaults: + run: + working-directory: ./pdns/recursordist/ + steps: + - uses: actions/checkout@v2.3.4 + with: + fetch-depth: 5 + submodules: recursive + - run: ../../build-scripts/gh-actions-setup-inv # this runs apt update+upgrade + - run: inv apt-fresh + - run: inv install-clang + - run: inv install-rec-build-deps + - run: inv ci-autoconf + - run: inv ci-rec-configure + - run: inv ci-rec-make + # FIXME: save ccache here? + - run: inv ci-rec-run-unit-tests + - run: inv ci-make-install + - name: Store the binaries + uses: actions/upload-artifact@v2 # this takes 30 seconds, maybe we want to tar + with: + name: pdns-recursor + path: /opt/pdns-recursor + build-dnsdist: + name: build dnsdist + runs-on: ubuntu-20.04 + env: + UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1:suppressions=/home/runner/work/pdns/pdns/build-scripts/UBSan.supp' + ASAN_OPTIONS: detect_leaks=0 + defaults: + run: + working-directory: ./pdns/dnsdistdist/ + steps: + - uses: actions/checkout@v2.3.4 + with: + fetch-depth: 5 + submodules: recursive + - run: ../../build-scripts/gh-actions-setup-inv # this runs apt update+upgrade + - run: inv apt-fresh + - run: inv install-clang + - run: inv install-dnsdist-build-deps + - run: inv ci-autoconf + - run: inv ci-dnsdist-configure + - run: inv ci-dnsdist-make + # FIXME: save ccache here? + - run: inv ci-dnsdist-run-unit-tests + - run: inv ci-make-install + - name: Store the binaries + uses: actions/upload-artifact@v2 # this takes 30 seconds, maybe we want to tar + with: + name: dnsdist + path: /opt/dnsdist + + test-auth-api: + needs: build-auth + runs-on: ubuntu-20.04 + env: + UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1:suppressions=/home/runner/work/pdns/pdns/build-scripts/UBSan.supp' + ASAN_OPTIONS: detect_leaks=0 + strategy: + matrix: + include: + - backend: gsqlite3 + image: coscale/docker-sleep + - backend: gmysql + image: mysql:5 + - backend: gpgsql + image: postgres:9 + - backend: lmdb + image: coscale/docker-sleep + fail-fast: false + services: + database: + image: ${{ matrix.image }} + env: + POSTGRES_USER: runner + POSTGRES_HOST_AUTH_METHOD: trust + MYSQL_ALLOW_EMPTY_PASSWORD: 1 + ports: + - 3306:3306 + - 5432:5432 + # FIXME: this works around dist-upgrade stopping all docker containers. dist-upgrade is huge on these images anyway. Perhaps we do want to run our tasks in a Docker container too. + options: >- + --restart always + steps: + - uses: actions/checkout@v2.3.4 + with: + fetch-depth: 5 + submodules: recursive + - name: Fetch the binaries + uses: actions/download-artifact@v2 + with: + name: pdns-auth + path: /opt/pdns-auth + # - name: Setup upterm session + # uses: lhotari/action-upterm@v1 + - run: build-scripts/gh-actions-setup-inv # this runs apt update+upgrade + - run: inv install-clang-runtime + - run: inv install-auth-test-deps -b ${{ matrix.backend }} + - run: inv test-api auth -b ${{ matrix.backend }} + + test-recursor-api: + needs: build-recursor + runs-on: ubuntu-20.04 + env: + UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1:suppressions=/home/runner/work/pdns/pdns/build-scripts/UBSan.supp' + ASAN_OPTIONS: detect_leaks=0 + steps: + - uses: actions/checkout@v2.3.4 + with: + fetch-depth: 5 + submodules: recursive + - name: Fetch the binaries + uses: actions/download-artifact@v2 + with: + name: pdns-recursor + path: /opt/pdns-recursor + - run: build-scripts/gh-actions-setup-inv # this runs apt update+upgrade + - run: inv add-auth-repo # FIXME: do we need this for rec API testing? + - run: inv install-clang-runtime + - run: inv install-rec-test-deps + - run: inv test-api recursor + + test-dnsdist-regression: + needs: build-dnsdist + runs-on: ubuntu-20.04 + env: + UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1:suppressions=/home/runner/work/pdns/pdns/build-scripts/UBSan.supp' + ASAN_OPTIONS: detect_leaks=0 + steps: + - uses: actions/checkout@v2.3.4 + with: + fetch-depth: 5 + submodules: recursive + - name: Fetch the binaries + uses: actions/download-artifact@v2 + with: + name: dnsdist + path: /opt/dnsdist + - run: build-scripts/gh-actions-setup-inv # this runs apt update+upgrade + - run: inv install-clang-runtime + - run: inv install-dnsdist-test-deps + - run: inv test-dnsdist + + +# FIXME: if we can make upload/download-artifact fasts, running unit tests outside of build can let regression tests start earlier diff --git a/build-scripts/UBSan.supp b/build-scripts/UBSan.supp index 4d1964485e14..e8875b08fb7c 100644 --- a/build-scripts/UBSan.supp +++ b/build-scripts/UBSan.supp @@ -1 +1,2 @@ -vptr:/usr/include/boost/any.hpp \ No newline at end of file +vptr:/usr/include/boost/any.hpp +null:/usr/include/boost/serialization/singleton.hpp diff --git a/build-scripts/gh-actions-setup-inv b/build-scripts/gh-actions-setup-inv new file mode 100755 index 000000000000..de9bf0c4b349 --- /dev/null +++ b/build-scripts/gh-actions-setup-inv @@ -0,0 +1,5 @@ +#!/bin/bash -x +sudo apt-get update +sudo apt-get -qq -y dist-upgrade +sudo apt-get -qq -y --no-install-recommends install python3-pip +sudo pip3 install git+https://github.com/pyinvoke/invoke@faa5728a6f76199a3da1750ed952e7efee17c1da \ No newline at end of file diff --git a/invoke.yaml b/invoke.yaml new file mode 100644 index 000000000000..7c0fa02faaef --- /dev/null +++ b/invoke.yaml @@ -0,0 +1,3 @@ +run: + echo: true + echo_format: "::endgroup::\n::group::{command}" diff --git a/modules/remotebackend/Gemfile.lock b/modules/remotebackend/Gemfile.lock index 974f6cf9a431..dbf6065483ae 100644 --- a/modules/remotebackend/Gemfile.lock +++ b/modules/remotebackend/Gemfile.lock @@ -1,14 +1,14 @@ GEM remote: https://rubygems.org/ specs: - ffi (1.10.0) + ffi (1.15.3) ffi-rzmq (2.0.7) ffi-rzmq-core (>= 1.0.7) ffi-rzmq-core (1.0.7) ffi - json (2.3.0) - sqlite3 (1.3.13) - webrick (1.4.2) + json (2.5.1) + sqlite3 (1.4.2) + webrick (1.7.0) zeromqrb (0.1.3) ffi-rzmq @@ -22,4 +22,4 @@ DEPENDENCIES zeromqrb BUNDLED WITH - 1.16.1 + 2.1.4 diff --git a/pdns/Makefile.am b/pdns/Makefile.am index 50bcccf691a4..137de97c83cf 100644 --- a/pdns/Makefile.am +++ b/pdns/Makefile.am @@ -1300,7 +1300,7 @@ dnspcap2protobuf_LDADD = \ $(RT_LIBS) pdns.conf-dist: pdns_server - $(AM_V_GEN)./pdns_server --config=default 2>/dev/null > $@ + $(AM_V_GEN)./pdns_server --config=default > $@ testrunner_SOURCES = \ arguments.cc \ diff --git a/tasks.py b/tasks.py new file mode 100644 index 000000000000..221e764b0fd3 --- /dev/null +++ b/tasks.py @@ -0,0 +1,347 @@ +from invoke import task +from invoke.exceptions import Failure, UnexpectedExit + +import sys +import time + +all_build_deps = [ + 'libboost-all-dev', + 'libluajit-5.1-dev', + 'libsodium-dev', + 'libssl-dev', + 'libsystemd-dev', + 'libtool', + 'make', + 'pkg-config', + 'python3-venv', + 'systemd', +] +git_build_deps = [ + 'autoconf', + 'automake', + 'bison', + 'bzip2', + 'curl', + 'flex', + 'git', + 'ragel' +] +auth_build_deps = [ # FIXME: perhaps we should be stealing these from the debian (Ubuntu) control file + 'default-libmysqlclient-dev', + 'libcdb-dev', + 'libcurl4-openssl-dev', + 'libgeoip-dev', + 'libkrb5-dev', + 'libldap2-dev', + 'liblmdb-dev', + 'libmaxminddb-dev', + 'libp11-kit-dev', + 'libpq-dev', + 'libsqlite3-dev', + 'libyaml-cpp-dev', + 'libzmq3-dev', + 'ruby-bundler', + 'ruby-dev', + 'sqlite3', +] +rec_build_deps = [ + 'libcap-dev', + 'libfstrm-dev', + 'libsnmp-dev', +] +dnsdist_build_deps = [ + 'libcap-dev', + 'libcdb-dev', + 'libedit-dev', + 'libfstrm-dev', + 'libh2o-evloop-dev', + 'liblmdb-dev', + 'libre2-dev', + 'libsnmp-dev', +] +auth_test_deps = [ # FIXME: we should be generating some of these from shlibdeps in build + 'authbind', + 'bc', + 'bind9utils', + 'curl', + 'default-jre-headless', + 'dnsutils', + 'gawk', + 'ldnsutils', + 'libboost-serialization1.71.0', + 'libcdb1', + 'libcurl4', + 'libgeoip1', + 'libkrb5-3', + 'libldap-2.4-2', + 'liblmdb0', + 'libluajit-5.1-2', + 'libmaxminddb0', + 'libnet-dns-perl', + 'libp11-kit0', + 'libpq5', + 'libsodium23', + 'libsqlite3-dev', + 'libssl1.1', + 'libsystemd0', + 'libyaml-cpp0.6', + 'libzmq3-dev', + 'pdns-recursor', + 'socat', + 'softhsm2', + 'unbound-host', + 'unixodbc', + 'wget' +] + +@task +def apt_fresh(c): + c.sudo('apt-get update') + c.sudo('apt-get dist-upgrade') + +@task +def install_clang(c): + """ + install clang-11 and llvm-11 + """ + c.sudo('apt-get -qq -y --no-install-recommends install clang-11 llvm-11') + +@task +def install_clang_runtime(c): + # this gives us the symbolizer, for symbols in asan/ubsan traces + c.sudo('apt-get -qq -y --no-install-recommends install clang-11') + +@task +def install_auth_build_deps(c): + c.sudo('apt-get install -qq -y --no-install-recommends ' + ' '.join(all_build_deps + git_build_deps + auth_build_deps)) + +def setup_authbind(c): + c.sudo('touch /etc/authbind/byport/53') + c.sudo('chmod 755 /etc/authbind/byport/53') + +auth_backend_test_deps = dict( + gsqlite3=['sqlite3'], + gmysql=['default-libmysqlclient-dev'], + gpgsql=['libpq-dev'], + lmdb=[] +) + +@task(help={'backend': 'Backend to install test deps for, e.g. gsqlite3; can be repeated'}, iterable=['backend'], optional=['backend']) +def install_auth_test_deps(c, backend): # FIXME: rename this, we do way more than apt-get + extra=[] + for b in backend: + extra.extend(auth_backend_test_deps[b]) + c.sudo('apt-get -y -qq install ' + ' '.join(extra+auth_test_deps)) + + c.run('chmod +x /opt/pdns-auth/bin/* /opt/pdns-auth/sbin/*') + # c.run('''if [ ! -e $HOME/bin/jdnssec-verifyzone ]; then + # wget https://github.com/dblacka/jdnssec-tools/releases/download/0.14/jdnssec-tools-0.14.tar.gz + # tar xfz jdnssec-tools-0.14.tar.gz -C $HOME + # rm jdnssec-tools-0.14.tar.gz + # fi + # echo 'export PATH=$HOME/jdnssec-tools-0.14/bin:$PATH' >> $BASH_ENV''') # FIXME: why did this fail with no error? + c.run('touch regression-tests/tests/verify-dnssec-zone/allow-missing') # FIXME: can this go? + # FIXME we need to start a background recursor here for some tests + setup_authbind(c) + +@task +def install_rec_test_deps(c): # FIXME: rename this, we do way more than apt-get + c.sudo('apt-get --no-install-recommends install -qq -y authbind python3-venv python3-dev default-libmysqlclient-dev libpq-dev pdns-tools libluajit-5.1-2 \ + libboost-all-dev \ + libcap2 \ + libssl1.1 \ + libsystemd0 \ + libsodium23 \ + libfstrm0 \ + libsnmp35') + + c.run('chmod +x /opt/pdns-recursor/bin/* /opt/pdns-recursor/sbin/*') + + setup_authbind(c) + +@task +def install_dnsdist_test_deps(c): # FIXME: rename this, we do way more than apt-get + c.sudo('apt-get install -qq -y \ + libluajit-5.1-2 \ + libboost-all-dev \ + libcap2 \ + libcdb1 \ + libcurl4-openssl-dev \ + libfstrm0 \ + libh2o-evloop0.13 \ + liblmdb0 \ + libre2-5 \ + libssl-dev \ + libsystemd0 \ + libsodium23 \ + patch \ + protobuf-compiler \ + python3-venv snmpd prometheus') + c.run('sed "s/agentxperms 0700 0755 dnsdist/agentxperms 0777 0755/g" regression-tests.dnsdist/snmpd.conf | sudo tee /etc/snmp/snmpd.conf') + c.sudo('systemctl restart snmpd') + time.sleep(5) + c.sudo('chmod 755 /var/agentx') + +@task +def install_rec_build_deps(c): + c.sudo('apt-get install -qq -y --no-install-recommends ' + ' '.join(all_build_deps + git_build_deps + rec_build_deps)) + +@task +def install_dnsdist_build_deps(c): + c.sudo('apt-get install -qq -y --no-install-recommends ' + ' '.join(all_build_deps + git_build_deps + dnsdist_build_deps)) + +@task +def ci_autoconf(c): + c.run('BUILDER_VERSION=0.0.0-git1 autoreconf -vfi') + +@task +def ci_auth_configure(c): + res = c.run('''CFLAGS="-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int" \ + CXXFLAGS="-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int -Wp,-D_GLIBCXX_ASSERTIONS" \ + ./configure \ + CC='clang-11' \ + CXX='clang++-11' \ + --enable-option-checking=fatal \ + --with-modules='bind geoip gmysql godbc gpgsql gsqlite3 ldap lmdb lua2 pipe random remote tinydns' \ + --enable-systemd \ + --enable-tools \ + --enable-unit-tests \ + --enable-backend-unit-tests \ + --enable-fuzz-targets \ + --enable-experimental-pkcs11 \ + --enable-remotebackend-zeromq \ + --with-lmdb=/usr \ + --with-libsodium \ + --prefix=/opt/pdns-auth \ + --enable-ixfrdist \ + --enable-asan \ + --enable-ubsan''', warn=True) + if res.exited != 0: + c.run('cat config.log') + raise UnexpectedExit(res) +@task +def ci_rec_configure(c): + res = c.run(''' CFLAGS="-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int" \ + CXXFLAGS="-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int -Wp,-D_GLIBCXX_ASSERTIONS" \ + ./configure \ + CC='clang-11' \ + CXX='clang++-11' \ + --enable-option-checking=fatal \ + --enable-unit-tests \ + --enable-nod \ + --enable-systemd \ + --prefix=/opt/pdns-recursor \ + --with-libsodium \ + --with-lua=luajit \ + --with-libcap \ + --with-net-snmp \ + --enable-dns-over-tls \ + --enable-asan \ + --enable-ubsan''', warn=True) + if res.exited != 0: + c.run('cat config.log') + raise UnexpectedExit(res) + +@task +def ci_dnsdist_configure(c): + res = c.run('''CFLAGS="-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int" \ + CXXFLAGS="-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int -Wp,-D_GLIBCXX_ASSERTIONS" \ + ./configure \ + CC='clang-11' \ + CXX='clang++-11' \ + --enable-option-checking=fatal \ + --enable-unit-tests \ + --enable-dnstap \ + --enable-dnscrypt \ + --enable-dns-over-tls \ + --enable-dns-over-https \ + --enable-systemd \ + --prefix=/opt/dnsdist \ + --with-libsodium \ + --with-lua=luajit \ + --with-libcap \ + --with-re2 \ + --enable-asan \ + --enable-ubsan''', warn=True) + if res.exited != 0: + c.run('cat config.log') + raise UnexpectedExit(res) + +@task +def ci_auth_make(c): + c.run('make -j8 -k V=1') + +@task +def ci_rec_make(c): + c.run('make -j8 -k V=1') + +@task +def ci_dnsdist_make(c): + c.run('make -j4 -k V=1') + +@task +def ci_auth_install_remotebackend_ruby_deps(c): + with c.cd('modules/remotebackend'): + c.run('bundle config set path vendor/bundle') + c.run('ruby -S bundle install') + +@task +def ci_auth_run_unit_tests(c): + res = c.run('make check', warn=True) + if res.exited != 0: + c.run('cat pdns/test-suite.log') + raise UnexpectedExit(res) + +@task +def ci_rec_run_unit_tests(c): + res = c.run('make check', warn=True) + if res.exited != 0: + c.run('cat test-suite.log') + raise UnexpectedExit(res) + +@task +def ci_dnsdist_run_unit_tests(c): + res = c.run('make check', warn=True) + if res.exited != 0: + c.run('cat test-suite.log') + raise UnexpectedExit(res) + +@task +def ci_make_install(c): + res = c.run('make install') # FIXME: this builds auth docs - again + +@task +def add_auth_repo(c): + dist = 'ubuntu' # FIXME take these from the caller? + release = 'focal' + version = '44' + + c.sudo('apt-get install -qq -y curl gnupg2') + if version == 'master': + c.sudo('curl -s -o /etc/apt/trusted.gpg.d/pdns-repo.asc https://repo.powerdns.com/CBC8B383-pub.asc') + else: + c.sudo('curl -s -o /etc/apt/trusted.gpg.d/pdns-repo.asc https://repo.powerdns.com/FD380FBB-pub.asc') + c.run(f"echo 'deb [arch=amd64] http://repo.powerdns.com/{dist} {release}-auth-{version} main' | sudo tee /etc/apt/sources.list.d/pdns.list") + c.run("echo 'Package: pdns-*' | sudo tee /etc/apt/preferences.d/pdns") + c.run("echo 'Pin: origin repo.powerdns.com' | sudo tee -a /etc/apt/preferences.d/pdns") + c.run("echo 'Pin-Priority: 600' | sudo tee -a /etc/apt/preferences.d/pdns") + c.sudo('apt-get update') + +@task +def test_api(c, product, backend=''): + if product == 'recursor': + with c.cd('regression-tests.api'): + c.run(f'PDNSRECURSOR=/opt/pdns-recursor/sbin/pdns_recursor ./runtests recursor {backend}') + elif product == 'auth': + with c.cd('regression-tests.api'): + c.run(f'PDNSSERVER=/opt/pdns-auth/sbin/pdns_server PDNSUTIL=/opt/pdns-auth/bin/pdnsutil SDIG=/opt/pdns-auth/bin/sdig MYSQL_HOST="127.0.0.1" PGHOST="127.0.0.1" PGPORT="5432" ./runtests authoritative {backend}') + else: + raise Failure('unknown product') + +@task +def test_dnsdist(c): + c.run('chmod +x /opt/dnsdist/bin/*') + c.run('ls -ald /var /var/agentx /var/agentx/master') + c.run('ls -al /var/agentx/master') + with c.cd('regression-tests.dnsdist'): + c.run('DNSDISTBIN=/opt/dnsdist/bin/dnsdist ./runtests')