diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 59845efb..3527e735 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,17 +7,19 @@ permissions: jobs: package-build: - runs-on: ubuntu-latest - + strategy: + fail-fast: false + matrix: + distro: [focal] steps: - uses: actions/checkout@v6 - - name: Run package build focal - run: script/cibuild-create-packages-focal + - name: Run package build ${{ matrix.distro }} + run: script/cibuild-create-packages ${{ matrix.distro }} - name: Tar files - run: tar -cvf glb-director.tar $GITHUB_WORKSPACE/tmp/build + run: tar -cvf glb-director-${{ matrix.distro }}.tar $GITHUB_WORKSPACE/tmp/build - name: Upload Artifact uses: actions/upload-artifact@v7 with: - name: glb-director - path: glb-director.tar \ No newline at end of file + name: glb-director-${{ matrix.distro }} + path: glb-director-${{ matrix.distro }}.tar diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..60cfe3b2 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,72 @@ +name: Tests + +on: + push: + branches: + - main + - master + pull_request: + branches: + - main + - master + +permissions: + contents: read + +jobs: + build-images: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + distro: [focal] + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Build image + run: | + docker build --file script/Dockerfile.${{ matrix.distro }} --tag glb-director-build-${{ matrix.distro }}:latest . + docker save glb-director-build-${{ matrix.distro }}:latest --output glb-director-build-${{ matrix.distro }}.tar + + - name: Upload image artifact + uses: actions/upload-artifact@v4 + with: + name: build-${{ matrix.distro }} + path: glb-director-build-${{ matrix.distro }}.tar + retention-days: 1 + + + + test: + needs: build-images + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + test-suite: [director, director-xdp, healthcheck, redirect] + distro: [focal] + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Download image artifact + uses: actions/download-artifact@v4 + with: + name: build-${{ matrix.distro }} + + - name: Load Docker image + run: | + docker load --input glb-director-build-${{ matrix.distro }}.tar + + - name: Run test suite in container + run: | + docker run --rm \ + --privileged \ + --volume $(pwd):/workspace \ + --workdir /workspace \ + glb-director-build-${{ matrix.distro }}:latest \ + bash -c "cd /workspace/src/glb-${{ matrix.test-suite }} && script/test" \ No newline at end of file diff --git a/script/Dockerfile.focal b/script/Dockerfile.focal index 2208dbb5..8fae6957 100644 --- a/script/Dockerfile.focal +++ b/script/Dockerfile.focal @@ -1,4 +1,4 @@ -FROM ubuntu:focal +FROM --platform=linux/amd64 ubuntu:focal@sha256:8feb4d8ca5354def3d8fce243717141ce31e2c428701f6682bd2fafe15388214 RUN echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80-retries @@ -18,6 +18,11 @@ RUN wget --quiet https://golang.org/dl/go1.24.5.linux-amd64.tar.gz -O- | tar -C ENV GOROOT /usr/local/go ENV GOPATH /go ENV PATH="${GOPATH}/bin:${GOROOT}/bin:${PATH}" +# Disable VCS stamping in Go 1.24+ builds. The repo is bind-mounted from the +# host so git refuses to operate on it inside the container ("dubious +# ownership"), which causes `go build` to fail with +# "error obtaining VCS status: exit status 128". +ENV GOFLAGS=-buildvcs=false # fpm for packaging RUN apt-get update && apt-get install -y ruby ruby-dev rubygems build-essential @@ -30,7 +35,7 @@ RUN gem install rake fpm # XDP # linux-libc-dev must be upgraded to get a bpf.h that matches what we use. the rest match what we do in Vagrant for testing. RUN apt-get update && apt install -y apt-transport-https curl software-properties-common -RUN apt-get update && apt install -y iproute2 libbpf-dev linux-libc-dev clang-10 +RUN apt-get update && apt install -y iproute2 libbpf-dev linux-libc-dev clang-10 clang-tools-10 # Hack because the kernel headers are not installed in the right place (linuxkit vs generic) RUN ln -s /usr/src/$(ls /usr/src/ | grep generic) /usr/src/linux-headers-$(uname -r) @@ -38,3 +43,14 @@ RUN ln -s /usr/src/$(ls /usr/src/ | grep generic) /usr/src/linux-headers-$(uname # Hack for C99 math RUN sed -i '1s/^/#define __USE_C99_MATH\n/' /usr/src/$(ls /usr/src/ | grep generic)/include/linux/kasan-checks.h RUN sed -i '2s/^/#include \n/' /usr/src/$(ls /usr/src/ | grep generic)/include/linux/kasan-checks.h + +# Python test dependencies (scapy/nose etc.) used by the test suites. +RUN apt-get update && apt-get install -y python3 python3-pip python3-dev +COPY requirements.txt /tmp/requirements.txt +RUN pip3 install --no-cache-dir -r /tmp/requirements.txt + +# valgrind is required by the glb-director test suite +RUN apt-get update && apt-get install -y valgrind + +# netcat and jq are required by the glb-healthcheck test suite +RUN apt-get update && apt-get install -y netcat jq tcpdump diff --git a/script/cibuild-create-packages b/script/cibuild-create-packages index 8cf9fa34..4d1d2cd4 100755 --- a/script/cibuild-create-packages +++ b/script/cibuild-create-packages @@ -7,9 +7,20 @@ cd "$(dirname "$0")/.." . script/helpers/folding.sh +DISTRO="$1" +if [ -z "$DISTRO" ]; then + DISTRO="focal" +fi + +DOCKERFILE="script/Dockerfile.$DISTRO" +if [ ! -f "$DOCKERFILE" ]; then + echo "Error: unsupported distro '$DISTRO' or missing Dockerfile '$DOCKERFILE'." >&2 + exit 1 +fi + begin_fold "Preparing Docker build environment" ( - docker build -t glb-director-build-stretch -f script/Dockerfile.stretch script + docker build -t glb-director-build-$DISTRO -f "$DOCKERFILE" "$HOSTPATH" ) end_fold @@ -21,8 +32,8 @@ begin_fold "Building packages" docker run --rm \ --volume "$HOSTPATH":/glb-director \ - "glb-director-build-stretch" \ + "glb-director-build-$DISTRO" \ bash -c "cd /glb-director && make BUILDDIR=/glb-director/tmp/build clean mkdeb" ) -end_fold +end_fold \ No newline at end of file diff --git a/script/cibuild-create-packages-focal b/script/cibuild-create-packages-focal deleted file mode 100755 index adc26467..00000000 --- a/script/cibuild-create-packages-focal +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -set -e - -HOSTPATH=$(cd $(dirname "$0") && cd .. && pwd) -cd "$(dirname "$0")/.." - -. script/helpers/folding.sh - -begin_fold "Preparing Docker build environment" -( - docker build -t glb-director-build-focal -f script/Dockerfile.focal script -) -end_fold - -begin_fold "Building packages" -( - # prep - rm -rf tmp/build/ - mkdir -p tmp/build/ - - docker run --rm \ - --volume "$HOSTPATH":/glb-director \ - "glb-director-build-focal" \ - bash -c "cd /glb-director && - make BUILDDIR=/glb-director/tmp/build clean mkdeb" -) -end_fold diff --git a/script/helpers/folding.sh b/script/helpers/folding.sh index 0c387304..774bf222 100644 --- a/script/helpers/folding.sh +++ b/script/helpers/folding.sh @@ -1,9 +1,9 @@ #!/bin/bash begin_fold() { - echo "%%%FOLD {$*}%%%" + echo "::group::$*" } end_fold() { - echo "%%%END FOLD%%%" -} + echo "::endgroup::" +} \ No newline at end of file diff --git a/script/test-local b/script/test-local new file mode 100755 index 00000000..76fb2ec3 --- /dev/null +++ b/script/test-local @@ -0,0 +1,43 @@ +#!/bin/bash + +set -e + +DISTRO="${1}" + +if [[ -z "$DISTRO" ]]; then + echo "Usage: $0 " + echo " e.g. $0 focal" + echo " e.g. $0 noble" + exit 1 +fi + +HOSTPATH=$(cd "$(dirname "$0")/.." && pwd) +IMAGE="glb-director-build-${DISTRO}:latest" +DOCKERFILE="script/Dockerfile.${DISTRO}" + +if [[ ! -f "${HOSTPATH}/${DOCKERFILE}" ]]; then + echo "ERROR: Dockerfile not found: ${DOCKERFILE}" + exit 1 +fi + +cd "$HOSTPATH" + +echo "==> Building Docker image for ${DISTRO}..." +docker build --platform linux/amd64 --file "${DOCKERFILE}" --tag "${IMAGE}" . + +TEST_SUITES=(director director-xdp healthcheck redirect) + +for suite in "${TEST_SUITES[@]}"; do + echo "" + echo "==> Running test suite: glb-${suite} (${DISTRO})" + docker run --rm \ + --platform linux/amd64 \ + --privileged \ + --volume "$(pwd):/workspace" \ + --workdir /workspace \ + "${IMAGE}" \ + bash -c "cd /workspace/src/glb-${suite} && script/test" +done + +echo "" +echo "==> All test suites passed for ${DISTRO}." diff --git a/src/glb-director/tests/glb_test_utils.py b/src/glb-director/tests/glb_test_utils.py index e0bae561..66992cae 100644 --- a/src/glb-director/tests/glb_test_utils.py +++ b/src/glb-director/tests/glb_test_utils.py @@ -18,9 +18,11 @@ import logging logging.getLogger("scapy.runtime").setLevel(logging.ERROR) -from scapy.all import sniff, sendp, Ether, IP, IPv6, L2ListenSocket, MTU, Packet, UDP, TCP, bind_layers, ICMP, ICMPv6PacketTooBig +from scapy.all import sniff, sendp, Ether, IP, IPv6, MTU, Packet, UDP, TCP, bind_layers, ICMP, ICMPv6PacketTooBig, conf +from scapy.arch.linux import L2ListenSocket from pyroute2 import IPRoute, NetlinkError from nose.tools import assert_equals +from nose.plugins.skip import SkipTest import subprocess, time import signal from contextlib import contextmanager @@ -60,7 +62,14 @@ def setup_pyside(self, iface): class DPDKDirectorControl(DirectorControlBase): def __init__(self): - assert os.path.exists('/dev/kni'), "KNI kernel module not loaded" + if not os.path.exists('/dev/kni'): + # The DPDK director requires the rte_kni out-of-tree kernel module + # (/dev/kni). When running in environments where it can't be loaded + # (e.g. Docker Desktop on macOS / linuxkit kernels), skip rather + # than fail so the suite can still be exercised locally. + raise SkipTest("rte_kni kernel module not loaded (/dev/kni missing); " + "skipping DPDK director tests. Run on a Linux host with rte_kni " + "available to execute these tests.") self.director = None @@ -75,7 +84,7 @@ def setup(self, iface): '--config-file', './tests/director-config.json', '--forwarding-table', './tests/test-tables.bin' ], - stdout=open('director-output.txt', 'wba'), + stdout=open('director-output.txt', 'ab'), stderr=subprocess.STDOUT, ) @@ -138,10 +147,17 @@ def wait(self): self.notify_sock.settimeout(2) try: data, addr = self.notify_sock.recvfrom(32) - assert data == 'READY=1' # only thing it will send + assert data == b'READY=1' # only thing it will send except socket.timeout: print('notify ready timed out') - raise Exception('Timeout while waiting for director to signal ready, did it crash?\n\n' + open('director-output.txt', 'rb').read()) + try: + with open('director-output.txt', 'rb') as fh: + captured = fh.read().decode('utf-8', errors='replace') + except OSError as e: + captured = ''.format(e) + raise Exception( + 'Timeout while waiting for director to signal ready, did it crash?\n\n' + + captured) self.notify_sock.close() @@ -149,6 +165,49 @@ def wait(self): class XDPDirectorControl(DirectorControlBase): def __init__(self): + # XDP requires a Linux kernel with XDP support and the ability to + # attach BPF programs to veth interfaces. Docker Desktop on macOS / + # Windows runs a "linuxkit" kernel that doesn't support this, and + # some CI runners (e.g. older GitHub Actions kernels) similarly + # can't attach XDP to veth. Detect that environment and skip rather + # than fail so the rest of the suite can still be exercised. + try: + kernel_release = os.uname().release + except Exception: + kernel_release = '' + if 'linuxkit' in kernel_release: + raise SkipTest("Running on linuxkit kernel ({}); XDP/veth attach is " + "not supported. Run on a Linux host to execute these tests.".format(kernel_release)) + if not os.path.isdir('/sys/fs/bpf'): + raise SkipTest("/sys/fs/bpf is not available; BPF filesystem not " + "mounted. Run on a Linux host with BPF support to execute these tests.") + + # Functional probe: build a throwaway veth pair and try to attach + # the passer.o XDP program. If that fails the kernel can't run the + # rest of these tests anyway, so skip with the actual reason. + passer = os.path.abspath('../glb-director-xdp/bpf/passer.o') + if not os.path.exists(passer): + raise SkipTest("XDP passer.o not built at {}; build glb-director-xdp before running tests.".format(passer)) + probe_a = 'glbxdpprobe0' + probe_b = 'glbxdpprobe1' + subprocess.call(['ip', 'link', 'del', 'dev', probe_a], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + try: + rc = subprocess.call( + ['ip', 'link', 'add', probe_a, 'type', 'veth', 'peer', 'name', probe_b], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + if rc != 0: + raise SkipTest("Unable to create veth pair (rc={}); container/kernel does not permit veth (kernel {}).".format(rc, kernel_release)) + probe = subprocess.run( + ['ip', 'link', 'set', 'dev', probe_a, 'xdp', 'obj', passer], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if probe.returncode != 0: + err = probe.stderr.decode('utf-8', errors='replace').strip() + raise SkipTest("Kernel ({}) cannot attach XDP to veth: {}".format(kernel_release, err)) + finally: + subprocess.call(['ip', 'link', 'del', 'dev', probe_a], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + self.director = None # veth pair implementation of XDP_TX silently drops packets unless the other side of the veth @@ -167,7 +226,7 @@ def setup(self, iface): '/sys/fs/bpf/root_array@' + iface, iface, ], - stdout=open('director-output.txt', 'wba'), + stdout=open('director-output.txt', 'ab'), stderr=subprocess.STDOUT, env=notify_shim.updated_env(), ) @@ -189,12 +248,12 @@ def launch_director(self): '--forwarding-table', os.path.abspath('./tests/test-tables.bin'), '--bpf-program', os.path.abspath('../glb-director-xdp/bpf/glb_encap.o'), ], - stdout=open('director-output.txt', 'wba'), + stdout=open('director-output.txt', 'ab'), stderr=subprocess.STDOUT, env=notify_director.updated_env(), ) - print('launched as pid', self.director.pid) + print(('launched as pid', self.director.pid)) notify_director.wait() @@ -299,7 +358,7 @@ def get_initial_director_config(cls): @classmethod def update_running_forwarding_tables(cls, config): - f = open('tests/test-tables.json', 'wb') + f = open('tests/test-tables.json', 'w') f.write(json.dumps(config, indent=4)) f.close() @@ -334,7 +393,7 @@ def setup_class(cls): GLBDirectorTestBase.py_side_mac = dict(ip.link('get', index=ip.link_lookup(ifname=cls.IFACE_NAME_PY))[0]['attrs'])['IFLA_ADDRESS'] - with open('tests/director-config.json', 'wb') as f: + with open('tests/director-config.json', 'w') as f: f.write(json.dumps(cls.get_initial_director_config(), indent=4)) # set up a statsd receiver @@ -352,44 +411,94 @@ def setup_class(cls): GLBDirectorTestBase.backend.setup(iface=cls.IFACE_NAME_DIRECTOR) GLBDirectorTestBase.backend.setup_pyside(iface=cls.IFACE_NAME_PY) + # Scapy caches name->ifindex in conf.ifaces. Between test classes we + # tear down and recreate the veth pair, which assigns a new ifindex + # under the same name -- the stale cache then makes + # setsockopt(SOL_PACKET, PACKET_MR_PROMISC, ...) fail with ENODEV + # ("No such device"). Force a refresh before opening sockets. + try: + conf.ifaces.reload() + except Exception: + # Older scapy versions don't expose reload(); fall back to + # clearing the cache directly so it gets rebuilt on next lookup. + try: + conf.ifaces.data.clear() # type: ignore[attr-defined] + except Exception: + pass + # prepare our listener for return traffic from director GLBDirectorTestBase.eth_tx = L2ListenSocket(iface=cls.IFACE_NAME_PY, promisc=True) GLBDirectorTestBase.kni_tx = GLBDirectorTestBase.backend.kni() @classmethod def teardown_class(cls): + # Stop the director / xdp-root-shim FIRST. While the xdp-root-shim is + # still running it holds pinned BPF programs attached to the veth, + # and the kernel refuses RTM_DELLINK on the device with + # ENOTSUP (95, "Operation not supported"). + try: + GLBDirectorTestBase.backend.cleanup() + except Exception as e: + print('backend.cleanup() raised during teardown: {}'.format(e)) + + # Detach any XDP programs that may still be present (e.g. the + # passer.o we attached to the py-side of the veth). + for iface in (cls.IFACE_NAME_PY, cls.IFACE_NAME_DIRECTOR): + subprocess.call(['ip', 'link', 'set', 'dev', iface, 'xdp', 'off'], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + # Also unpin anything the xdp-root-shim may have left behind in bpffs; + # stale pinned maps on the iface can also cause ENOTSUP on dellink. + for iface in (cls.IFACE_NAME_PY, cls.IFACE_NAME_DIRECTOR): + pin = '/sys/fs/bpf/root_array@' + iface + try: + if os.path.exists(pin): + os.unlink(pin) + except OSError as e: + print('Failed to unpin {}: {}'.format(pin, e)) + + # Tear down the veth pair. Use `ip link del` via subprocess; the + # pyroute2 path returns ENOTSUP on some kernels even when the + # equivalent userspace command works. Removing either end of a veth + # removes both, but try each in case only one side exists. ip = IPRoute() - - # tear down the veth pair - if len(ip.link_lookup(ifname=cls.IFACE_NAME_PY)) > 0: - ip.link('remove', ifname=cls.IFACE_NAME_DIRECTOR) - if len(ip.link_lookup(ifname=cls.IFACE_NAME_PY)) > 0: - ip.link('remove', ifname=cls.IFACE_NAME_DIRECTOR) + for iface in (cls.IFACE_NAME_DIRECTOR, cls.IFACE_NAME_PY): + if len(ip.link_lookup(ifname=iface)) > 0: + rc = subprocess.call(['ip', 'link', 'del', 'dev', iface], + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) + if rc != 0: + # Fall back to pyroute2; surface any error rather than + # masking it (matches the previous behaviour). + ip.link('remove', ifname=iface) assert_equals(len(ip.link_lookup(ifname=cls.IFACE_NAME_PY)), 0) assert_equals(len(ip.link_lookup(ifname=cls.IFACE_NAME_DIRECTOR)), 0) - # clean up the director - GLBDirectorTestBase.backend.cleanup() - def sendp(self, *args, **kwargs): sendp(*args, **kwargs) def wait_for_packet(self, iface, condition, timeout_seconds=5): - print('Waiting for packets on', iface.iff, 'with timeout', timeout_seconds) + # Newer scapy L2ListenSocket no longer exposes `.iff`; fall back to + # `.iface` and finally repr() so the print never crashes the test. + iface_name = getattr(iface, 'iff', None) or getattr(iface, 'iface', None) or repr(iface) + print(('Waiting for packets on', iface_name, 'with timeout', timeout_seconds)) try: with timeout(timeout_seconds): while True: packet = iface.recv(MTU) - print(repr(packet)) + print((repr(packet))) if condition(packet): return packet except: - with open('director-output.txt', 'rb') as d: - sys.stdout.write('-' * 50 + '\n') - sys.stdout.write('Output from glb-director-ng\n') - sys.stdout.write('-' * 50 + '\n') - sys.stdout.write(d.read()) - sys.stdout.write('-' * 50 + '\n') + try: + with open('director-output.txt', 'rb') as d: + captured = d.read().decode('utf-8', errors='replace') + except OSError as e: + captured = ''.format(e) + sys.stdout.write('-' * 50 + '\n') + sys.stdout.write('Output from glb-director-ng\n') + sys.stdout.write('-' * 50 + '\n') + sys.stdout.write(captured) + sys.stdout.write('-' * 50 + '\n') raise def stream_statsd_metrics(self, timeout=0): @@ -400,6 +509,9 @@ def stream_statsd_metrics(self, timeout=0): return # nothing more to receive, we timed out else: block, _ = s.recvfrom(4096) + # recvfrom returns bytes in Py3; decode so the string ops below work. + if isinstance(block, bytes): + block = block.decode('utf-8', errors='replace') for data in block.split('\n'): metric_name, metric_data = data.split(':', 1) metric_info, metric_tags = metric_data.split('#', 1) @@ -414,7 +526,7 @@ def expect_metrics(self, spec): spec_matches = set() for metric_name, metric_value, metric_type, metric_tags in self.stream_statsd_metrics(timeout=1): metric_key = (metric_name, metric_tags) - print metric_key + print(metric_key) if metric_key in spec: assert spec[metric_key](metric_value), "Metric {} had unexpected value {}".format(metric_key, repr(metric_value)) spec_matches.add(metric_key) @@ -444,7 +556,7 @@ def pkt_hash(self, key, src_addr=None, dst_addr=None, src_port=None, dst_port=No hash_parts.append(self._encode_port(dst_port)) assert len(hash_parts) > 0 - hash_data = ''.join(hash_parts) + hash_data = b''.join(hash_parts) hash_bytes = siphash.SipHash_2_4(key, hash_data).digest() hash_num, = struct.unpack('&3 2>&4 + if [ -f "$TRASHDIR/.skipped" ]; then + reason=$(cat "$TRASHDIR/.skip_reason" 2>/dev/null) + rm -f "$TRASHDIR/.skipped" "$TRASHDIR/.skip_reason" + printf "test: %-60s SKIPPED (%s)\n" "$test_description ..." "$reason" + unset test_description + return 0 + fi + if [ "$test_status" -eq 0 ]; then printf "test: %-60s OK\n" "$test_description ..." else @@ -90,3 +98,31 @@ end_test () { end_test_exfail () { end_test $? 1 } + +# Mark the current test as skipped from inside the subshell. The marker file +# is read by end_test to report SKIPPED rather than OK/FAILED. Mirrors the +# SkipTest pattern used by the director Python suite so tests that require +# infrastructure unavailable in the container (e.g. DPDK hugepages on +# Docker Desktop / macOS) don't fail when run via script/test-local. +skip_test () { + reason="${1:-no reason given}" + echo "SKIP: $reason" + : > "$TRASHDIR/.skipped" + echo "$reason" > "$TRASHDIR/.skip_reason" + exit 0 +} + +# Returns 0 if DPDK can use hugepages on this host. DPDK requires free +# hugepages of one of the supported sizes; without them EAL initialization +# fails with "Cannot get hugepage information." Docker Desktop on macOS +# (linuxkit kernel) does not expose hugepages by default. +hugepages_available () { + for f in /sys/kernel/mm/hugepages/hugepages-*/free_hugepages; do + [ -r "$f" ] || continue + n=$(cat "$f" 2>/dev/null || echo 0) + if [ "${n:-0}" -gt 0 ]; then + return 0 + fi + done + return 1 +} diff --git a/src/glb-director/tests/pcap_tests.sh b/src/glb-director/tests/pcap_tests.sh index b7179d91..9d5c270d 100644 --- a/src/glb-director/tests/pcap_tests.sh +++ b/src/glb-director/tests/pcap_tests.sh @@ -36,6 +36,8 @@ set -e begin_test "run with pcap and example tables" ( + hugepages_available || skip_test "DPDK hugepages not available; skipping pcap director run. Run on a Linux host with hugepages configured." + $BASEDIR/cli/glb-director-cli build-config \ $BASEDIR/tests/data/table.json \ $BASEDIR/tests/data/test-tables.bin @@ -50,12 +52,16 @@ end_test begin_test "tx: should be 1000 packets" ( + hugepages_available || skip_test "DPDK hugepages not available; tx.pcap was not produced." + sudo tcpdump -r $BASEDIR/build/tx.pcap | wc -l | grep 1000 ) end_test begin_test "tx: verify to/from" ( + hugepages_available || skip_test "DPDK hugepages not available; tx.pcap was not produced." + sudo tcpdump -nr $BASEDIR/build/tx.pcap | grep -q 'IP 65.65.65.65.61139 > 3.4.5.6.19523: UDP, length 52' ) end_test diff --git a/src/glb-director/tests/test-config.json b/src/glb-director/tests/test-config.json index 7ae9e5f0..59b07e6f 100644 --- a/src/glb-director/tests/test-config.json +++ b/src/glb-director/tests/test-config.json @@ -1,80 +1,80 @@ { "tables": [ { + "hash_key": "12345678901234561234567890123456", + "seed": "34567890123456783456789012345678", "binds": [ { - "ip": "1.1.1.1", - "port": 80, - "proto": "tcp" - }, + "ip": "1.1.1.1", + "proto": "tcp", + "port": 80 + }, { - "ip": "1.1.1.1", - "port": 443, - "proto": "tcp" + "ip": "1.1.1.1", + "proto": "tcp", + "port": 443 } - ], - "seed": "34567890123456783456789012345678", - "hash_key": "12345678901234561234567890123456", + ], "backends": [ { - "healthy": true, - "ip": "1.2.3.4", - "state": "active" - }, + "ip": "1.2.3.4", + "state": "active", + "healthy": true + }, { - "healthy": true, - "ip": "2.3.4.5", - "state": "active" - }, + "ip": "2.3.4.5", + "state": "active", + "healthy": true + }, { - "healthy": true, - "ip": "3.4.5.6", - "state": "active" + "ip": "3.4.5.6", + "state": "active", + "healthy": true } ] - }, + }, { + "hash_key": "12345678901234561234567890123456", + "seed": "12345678901234561234567890123456", "binds": [ { - "ip": "1.1.1.2", - "port": 80, - "proto": "tcp" - }, + "ip": "1.1.1.2", + "proto": "tcp", + "port": 80 + }, { - "ip": "1.1.1.3", - "port": 80, - "proto": "tcp" - }, + "ip": "1.1.1.3", + "proto": "tcp", + "port": 80 + }, { - "ip": "fdb4:98ce:52d4::42", - "port": 80, - "proto": "tcp" + "ip": "fdb4:98ce:52d4::42", + "proto": "tcp", + "port": 80 } - ], - "seed": "12345678901234561234567890123456", - "hash_key": "12345678901234561234567890123456", + ], "backends": [ { - "healthy": true, - "ip": "4.5.6.7", - "state": "active" - }, + "ip": "4.5.6.7", + "state": "active", + "healthy": true + }, { - "healthy": true, - "ip": "5.6.7.8", - "state": "active" - }, + "ip": "5.6.7.8", + "state": "active", + "healthy": true + }, { - "healthy": true, - "ip": "6.7.8.9", - "state": "active" - }, + "ip": "6.7.8.9", + "state": "active", + "healthy": true + }, { - "healthy": true, - "ip": "7.8.9.0", - "state": "active" + "ip": "7.8.9.0", + "state": "active", + "healthy": true } ] } ] -} +} \ No newline at end of file diff --git a/src/glb-director/tests/test_cli_tool.py b/src/glb-director/tests/test_cli_tool.py index cb3b8abf..3a9f6af8 100644 --- a/src/glb-director/tests/test_cli_tool.py +++ b/src/glb-director/tests/test_cli_tool.py @@ -63,11 +63,11 @@ def write_example_config(self): def get_example_table_reference_implementation(self, table_index): table_config = self.get_example_config()['tables'][table_index] - return GLBRendezvousTable(table_config['seed'].decode('hex')) + return GLBRendezvousTable(bytes.fromhex(table_config['seed'])) def get_example_table_hosts(self, table_index): table_config = self.get_example_config()['tables'][table_index] - return map(lambda b: b['ip'], table_config['backends']) + return [b['ip'] for b in table_config['backends']] def test_generate_configs(self): self.write_example_config() @@ -75,7 +75,7 @@ def test_generate_configs(self): subprocess.check_call(['cli/glb-director-cli', 'build-config', 'tests/test-config.json', 'tests/test-config.bin']) f = open('tests/test-config.bin', 'rb') - assert_equals(f.read(4), 'GLBD') + assert_equals(f.read(4), b'GLBD') num_table_entries = 0x10000 max_num_backends = 0x100 @@ -109,7 +109,7 @@ def test_generate_configs(self): assert_equals(inet_addr, socket.inet_pton(socket.AF_INET6, backend['ip'])) else: assert_equals(inet_family, 1) - assert_equals(inet_addr, socket.inet_pton(socket.AF_INET, backend['ip']).ljust(16, '\x00')) + assert_equals(inet_addr, socket.inet_pton(socket.AF_INET, backend['ip']).ljust(16, b'\x00')) assert_equals(be_state, 1) assert_equals(be_health, 1) @@ -128,14 +128,14 @@ def test_generate_configs(self): assert_equals(ip_bits, 128) else: assert_equals(inet_family, 1) - assert_equals(inet_addr, socket.inet_pton(socket.AF_INET, bind['ip']).ljust(16, '\x00')) + assert_equals(inet_addr, socket.inet_pton(socket.AF_INET, bind['ip']).ljust(16, b'\x00')) assert_equals(ip_bits, 32) assert_equals(bind_port_start, bind['port']) assert_equals(bind_port_end, bind['port']) assert_equals(bind_proto, 6 if bind['proto'] == 'tcp' else 17) # validate hash key for source hashing - assert_equals(f.read(16), table['hash_key'].decode('hex').rjust(16, '\x00')) + assert_equals(f.read(16), bytes.fromhex(table['hash_key']).rjust(16, b'\x00')) # validate table entries for table_index in range(num_table_entries): @@ -149,10 +149,10 @@ def test_generate_configs(self): assert_equals(actual_first_ips, expected_first_ips[:2]) - # forwarding_table_seed = '49a3d861d661ae5ab06ed9326871a2f5'.decode('hex') + # forwarding_table_seed = bytes.fromhex('49a3d861d661ae5ab06ed9326871a2f5') # table = GLBRendezvousTable(forwarding_table_seed) - # assert_equals(table.calculate_forwarding_table_row_seed(0x0000).encode('hex'), '491c53a72df4c837') - # assert_equals(table.calculate_forwarding_table_row_seed(0xffff).encode('hex'), 'f223c0cc65161620') + # assert_equals(table.calculate_forwarding_table_row_seed(0x0000).hex(), '491c53a72df4c837') + # assert_equals(table.calculate_forwarding_table_row_seed(0xffff).hex(), 'f223c0cc65161620') def test_atomic_write_no_temp_file_remains(self): """Verify that no temporary file is left behind after a successful build.""" diff --git a/src/glb-director/tests/test_director_classify_v6.py b/src/glb-director/tests/test_director_classify_v6.py index f40c6208..2d82ad4e 100644 --- a/src/glb-director/tests/test_director_classify_v6.py +++ b/src/glb-director/tests/test_director_classify_v6.py @@ -45,7 +45,7 @@ def test_01_route_classified_v6(self): assert_equals(glb_gue.private_data[0].hops, ['6.7.8.9']) inner_ip = glb_gue.payload - print repr(inner_ip) + print(repr(inner_ip)) assert isinstance(inner_ip, IPv6) # Expecting the inner IPv6 packet assert_equals(inner_ip.src, 'fd91:79d3:d621::1234') assert_equals(inner_ip.dst, 'fdb4:98ce:52d4::42') diff --git a/src/glb-director/tests/test_rendezvous_table.py b/src/glb-director/tests/test_rendezvous_table.py index a891831f..42233331 100644 --- a/src/glb-director/tests/test_rendezvous_table.py +++ b/src/glb-director/tests/test_rendezvous_table.py @@ -22,10 +22,10 @@ class TestGLBRendezvousTable(): def test_row_seeds(self): """GLBRendezvousTable correctly calculates valid row seeds""" - forwarding_table_seed = '49a3d861d661ae5ab06ed9326871a2f5'.decode('hex') + forwarding_table_seed = bytes.fromhex('49a3d861d661ae5ab06ed9326871a2f5') table = GLBRendezvousTable(forwarding_table_seed) - assert_equals(table.calculate_forwarding_table_row_seed(0x0000).encode('hex'), '491c53a72df4c837') - assert_equals(table.calculate_forwarding_table_row_seed(0xffff).encode('hex'), 'f223c0cc65161620') + assert_equals(table.calculate_forwarding_table_row_seed(0x0000).hex(), '491c53a72df4c837') + assert_equals(table.calculate_forwarding_table_row_seed(0xffff).hex(), 'f223c0cc65161620') def test_order_hosts_0000(self): """ @@ -37,7 +37,7 @@ def test_order_hosts_0000(self): 1.1.1.4 6f022ce1ea607e16 """ - forwarding_table_seed = '49a3d861d661ae5ab06ed9326871a2f5'.decode('hex') + forwarding_table_seed = bytes.fromhex('49a3d861d661ae5ab06ed9326871a2f5') table = GLBRendezvousTable(forwarding_table_seed) hosts = ['1.1.1.1', '1.1.1.2', '1.1.1.3', '1.1.1.4'] @@ -54,7 +54,7 @@ def test_order_hosts_ffff(self): 1.1.1.4 a1f610df9fbb2025 """ - forwarding_table_seed = '49a3d861d661ae5ab06ed9326871a2f5'.decode('hex') + forwarding_table_seed = bytes.fromhex('49a3d861d661ae5ab06ed9326871a2f5') table = GLBRendezvousTable(forwarding_table_seed) hosts = ['1.1.1.1', '1.1.1.2', '1.1.1.3', '1.1.1.4'] @@ -71,7 +71,7 @@ def test_order_hosts_bb44(self): 1.1.1.4 0676eaf9cb7d2f85 """ - forwarding_table_seed = '49a3d861d661ae5ab06ed9326871a2f5'.decode('hex') + forwarding_table_seed = bytes.fromhex('49a3d861d661ae5ab06ed9326871a2f5') table = GLBRendezvousTable(forwarding_table_seed) hosts = ['1.1.1.1', '1.1.1.2', '1.1.1.3', '1.1.1.4'] diff --git a/src/glb-healthcheck/test/lib.sh b/src/glb-healthcheck/test/lib.sh index 05058491..7a2fd8e1 100644 --- a/src/glb-healthcheck/test/lib.sh +++ b/src/glb-healthcheck/test/lib.sh @@ -98,6 +98,14 @@ end_test () { echo "---- end_test: $test_description ----" >> $HC_LOGFILE + if [ -f "$TRASHDIR/.skipped" ]; then + reason=$(cat "$TRASHDIR/.skip_reason" 2>/dev/null) + rm -f "$TRASHDIR/.skipped" "$TRASHDIR/.skip_reason" + printf "test: %-60s SKIPPED (%s)\n" "$test_description ..." "$reason" + unset test_description + return 0 + fi + if [ "$test_status" -eq 0 ]; then if [ "$ex_fail" -eq 0 ]; then printf "test: %-60s OK (${elapsed_time}s)\n" "$test_description ..." @@ -119,6 +127,32 @@ end_test_exfail () { end_test $? 1 } +# Mark the current test as skipped from inside the subshell. The marker file +# is read by end_test below to report SKIPPED rather than OK/FAILED. This +# mirrors the SkipTest pattern used by the director Python test suite so that +# tests requiring infrastructure unavailable in the container (e.g. the +# Vagrant proxy1/proxy2 backends) don't fail when run via script/test-local. +skip_test () { + reason="${1:-no reason given}" + echo "SKIP: $reason" + : > "$TRASHDIR/.skipped" + echo "$reason" > "$TRASHDIR/.skip_reason" + exit 0 +} + +# Returns 0 if the Vagrant proxy backends (proxy1/proxy2) referenced by the +# default forwarding table are reachable on their HTTP healthcheck port. +# Used to decide whether to skip tests that depend on real backends being up. +proxy_backends_available () { + # quick TCP probe with an explicit short timeout; both proxies must answer on :80 + for ip in 192.168.50.10 192.168.50.11; do + if ! nc -z -w 1 "$ip" 80 >/dev/null 2>&1; then + return 1 + fi + done + return 0 +} + atexit () { [ -z "$KEEPTRASH" ] && rm -rf "$TEMPDIR" if [ $failures -gt 0 ]; then @@ -157,6 +191,13 @@ setup() { set -e + # When running under Docker (rather than the Vagrant director-test VM), + # the forwarding table references 192.168.50.5 as the local backend for + # the HTTP healthcheck test. Bind it to loopback so that the healthcheck + # daemon can actually reach a local HTTP server. Ignore failures (e.g. + # already added, or not running as root on the host). + ip addr add 192.168.50.5/32 dev lo 2>/dev/null || true + # copy a backup of the initial version to reset later cp $TEMPDIR/forwarding_table.json $TEMPDIR/forwarding_table.json.bak diff --git a/src/glb-healthcheck/test/test-basic.sh b/src/glb-healthcheck/test/test-basic.sh index 1e12cf03..3c8d6bb3 100644 --- a/src/glb-healthcheck/test/test-basic.sh +++ b/src/glb-healthcheck/test/test-basic.sh @@ -90,6 +90,8 @@ end_test begin_test "outputs the healthcheck file with valid health" ( + proxy_backends_available || skip_test "Vagrant proxy1/proxy2 backends (192.168.50.10/11) not reachable; requires the Vagrant test network." + setup sleep 3 @@ -107,6 +109,8 @@ end_test begin_test "reload should take effect" ( + proxy_backends_available || skip_test "Vagrant proxy1/proxy2 backends (192.168.50.10/11) not reachable; requires the Vagrant test network." + setup sleep 3 @@ -169,7 +173,7 @@ begin_test "responds to health check changes" [[ "$(jq -r '.tables[1].backends[3].healthy' $TEMPDIR/forwarding_table.hc.json)" == "false" ]] # start up a HTTP server - python -m SimpleHTTPServer 8765 & + python3 -m http.server 8765 & http_pid=$! echo "$http_pid" > "${TEMPDIR}/http.pid" diff --git a/src/glb-redirect/tests/glb_test_remote_snoop.py b/src/glb-redirect/tests/glb_test_remote_snoop.py index 1bc18a58..c010447d 100644 --- a/src/glb-redirect/tests/glb_test_remote_snoop.py +++ b/src/glb-redirect/tests/glb_test_remote_snoop.py @@ -49,7 +49,7 @@ def recv(self, recv_filter, timeout=10): pkt_ether = Ether(pkt_raw) pkt = pkt_ether.payload - if self.debug: print("got packet from {}: {}".format(self.remote_host, repr(pkt))) + if self.debug: print(("got packet from {}: {}".format(self.remote_host, repr(pkt)))) if recv_filter(pkt): if self.debug: print(" -> match!") return pkt diff --git a/src/glb-redirect/tests/glb_test_utils.py b/src/glb-redirect/tests/glb_test_utils.py index f1f303cd..95ee2f71 100644 --- a/src/glb-redirect/tests/glb_test_utils.py +++ b/src/glb-redirect/tests/glb_test_utils.py @@ -16,8 +16,46 @@ # along with this project. If not, see . from scapy.all import sniff, send, L3RawSocket, L3RawSocket6 +import os +import socket +from nose.plugins.skip import SkipTest + + +def _proxy_backends_available(): + """Return True iff the Vagrant proxy backends (proxy1/proxy2) used by + these tests are reachable. They live in the Vagrant `glb_datacenter_network` + and aren't present when running under script/test-local in Docker.""" + for host in ('192.168.50.10', '192.168.50.11'): + try: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(0.2) + # port 22 is used as a liveness probe in the actual tests + rc = s.connect_ex((host, 22)) + s.close() + if rc != 0: + return False + except OSError: + return False + return True + + +def skip_if_no_vagrant_network(): + """Raise SkipTest if the Vagrant proxy network isn't available. The + glb-redirect tests fundamentally require the proxy1/proxy2/director-test + VMs (with the glb-redirect iptables module loaded), so they can't run in + the Docker test image.""" + if not _proxy_backends_available(): + raise SkipTest( + "Vagrant proxy backends (192.168.50.10/11) not reachable; " + "glb-redirect tests require the Vagrant test network with the " + "glb-redirect iptables module installed on proxy1/proxy2.") + class GLBTestHelpers(object): + @classmethod + def setup_class(cls): + skip_if_no_vagrant_network() + def _sendrecv6(self, pkt, **kwargs): s = L3RawSocket6() send(pkt) @@ -25,7 +63,7 @@ def _sendrecv6(self, pkt, **kwargs): s.close() if len(ret) == 0: assert False, "Expected to receive a response packet, but none received." - print "Received packet:", repr(ret[0]) + print("Received packet:", repr(ret[0])) return ret[0] def _sendrecvmany4(self, pkt, **kwargs): @@ -36,7 +74,7 @@ def _sendrecvmany4(self, pkt, **kwargs): if len(ret) == 0: assert False, "Expected to receive a response packet, but none received." for pkt in ret: - print "Received packet:", repr(pkt) + print("Received packet:", repr(pkt)) return ret def _sendrecv4(self, pkt, **kwargs): diff --git a/src/glb-redirect/tests/test_glb_redirect_v4_on_v4.py b/src/glb-redirect/tests/test_glb_redirect_v4_on_v4.py index e8616817..23b7682b 100644 --- a/src/glb-redirect/tests/test_glb_redirect_v4_on_v4.py +++ b/src/glb-redirect/tests/test_glb_redirect_v4_on_v4.py @@ -41,7 +41,7 @@ def test_00_icmp_accepted(self): # expect a ICMP echo response back from self.PROXY_HOST (decapsulated) resp_ip = self._sendrecv4(pkt, filter='host {} and icmp'.format(dst)) - print repr(resp_ip) + print(repr(resp_ip)) assert isinstance(resp_ip, IP) assert_equals(resp_ip.src, dst) assert_equals(resp_ip.dst, self.SELF_HOST) diff --git a/src/glb-redirect/tests/test_glb_redirect_v6_on_v4.py b/src/glb-redirect/tests/test_glb_redirect_v6_on_v4.py index 8a5970d6..84c78831 100644 --- a/src/glb-redirect/tests/test_glb_redirect_v6_on_v4.py +++ b/src/glb-redirect/tests/test_glb_redirect_v6_on_v4.py @@ -45,7 +45,7 @@ def test_00_icmp_accepted(self): GLBGUE(private_data=GLBGUEChainedRouting(hops=[self.ALT_HOST])) / \ IPv6(src=self.SELF_HOST_V6, dst=self.V4_TO_V6[dst]) / \ ICMPv6EchoRequest() - print repr(pkt) + print(repr(pkt)) # expect a ICMP echo response back from self.PROXY_HOST (decapsulated) resp_ip = self._sendrecv6(pkt, lfilter=lambda p: isinstance(p, IPv6) and isinstance(p.payload, ICMPv6EchoReply)) diff --git a/src/scapy-glb-gue/glb_scapy/__init__.py b/src/scapy-glb-gue/glb_scapy/__init__.py index 213434c5..f23dcde2 100644 --- a/src/scapy-glb-gue/glb_scapy/__init__.py +++ b/src/scapy-glb-gue/glb_scapy/__init__.py @@ -15,4 +15,4 @@ # You should have received a copy of the GNU General Public License # along with scapy-glb-gue. If not, see . -from glb_gue_scapy import GLBGUEChainedRouting, GLBGUE +from .glb_gue_scapy import GLBGUEChainedRouting, GLBGUE diff --git a/src/scapy-glb-gue/glb_scapy/glb_gue_scapy.py b/src/scapy-glb-gue/glb_scapy/glb_gue_scapy.py index beb31019..3c023057 100644 --- a/src/scapy-glb-gue/glb_scapy/glb_gue_scapy.py +++ b/src/scapy-glb-gue/glb_scapy/glb_gue_scapy.py @@ -36,7 +36,7 @@ class GLBGUE(Packet): name = "GLBGUE" fields_desc = [BitField("version", 0, 2), BitField("control_msg", 0, 1), - BitFieldLenField("hlen", None, 5, length_of='private_data', adjust=lambda pkt, x: (x / 4)), + BitFieldLenField("hlen", None, 5, length_of='private_data', adjust=lambda pkt, x: (x // 4)), BitField("protocol", 0, 8), BitField("flags", 0, 16), PacketListField("private_data", [], GLBGUEChainedRouting, length_from=lambda p:p.hlen * 4)