Add buildomat job for interop testing

.github/buildomat/jobs/build-interop.sh

@@ -0,0 +1,85 @@
+#!/bin/bash
+#:
+#: name = "build-interop"
+#: variety = "basic"
+#: target = "helios-2.0"
+#: rust_toolchain = "stable"
+#: access_repos = [
+#:   "oxidecomputer/testbed",
+#: ]
+#: output_rules = [
+#:   "=/work/testbed.tar.gz",
+#:   "=/work/dhcp-server",
+#: ]
+#:
+
+set -x
+set -e
+
+#
+# Allow this program to run either under buildomat, or in a local clone:
+#
+if [[ $CI == true ]]; then
+	top=$PWD
+	WORK=/work
+
+	pfexec pkg install protobuf git
+else
+	if [[ -z $WORK || ! -d $WORK ]]; then
+		printf 'ERROR: set WORK when running manually\n' >&2
+		exit 1
+	fi
+
+	#
+	# Be resilient against someone running this while not in the repository
+	# root directory:
+	#
+	top=$(cd "$(dirname "$0")"/../../.. && pwd)
+fi
+
+cargo --version
+rustc --version
+
+banner 'clone'
+mkdir "$top/ci"
+git clone https://github.com/oxidecomputer/testbed "$top/ci/testbed"
+
+banner 'build'
+cd "$top/ci/testbed"
+cargo build \
+    -p interop-lab \
+    -p wrangler
+cargo build --tests
+
+banner 'prep'
+
+mkdir out
+cp target/debug/{interop,wrangler} out/
+# grab just the file ending in the hash, not the file ending in ".d"
+TEST=$(find target/debug/deps -maxdepth 1 -type f -name 'baseline-*' -exec ls -t {} + | grep -v -E '.*\.d$' | head -1)
+mv "$TEST" 'out/baseline'
+
+banner 'archive'
+
+cd "$top/ci"
+cat <<EOF > exclude-file.txt
+testbed/.git
+testbed/a4x2
+testbed/archive
+testbed/target
+EOF
+tar cvzXf exclude-file.txt \
+    "$WORK/testbed.tar.gz" \
+    testbed
+
+banner 'dhcp-server'
+
+git clone https://github.com/oxidecomputer/omicron.git "$top/ci/omicron"
+cd "$top/ci/omicron"
+source env.sh
+if [[ $CI == true ]]; then
+	source .github/buildomat/ci-env.sh
+	./tools/install_builder_prerequisites.sh -y
+fi
+cargo build -p end-to-end-tests --bin dhcp-server --release
+cp target/release/dhcp-server "$WORK/"

.github/buildomat/jobs/test-bfd.sh

@@ -16,10 +16,15 @@ set -o xtrace
 set -o errexit
 set -o pipefail
 
+# NOTE: This version should be in sync with the recommended version in
+# .config/nextest.toml. (Maybe build an automated way to pull the recommended
+# version in the future.)
+NEXTEST_VERSION='0.9.97'
+PLATFORM='illumos'
+
 cargo --version
 rustc --version
-
-cargo install cargo-nextest
+curl -sSfL --retry 10 https://get.nexte.st/"$NEXTEST_VERSION"/"$PLATFORM" | gunzip | tar -xvf - -C ~/.cargo/bin
 
 banner bfd
 cargo nextest run -p bfd --nocapture

.github/buildomat/jobs/test-bgp.sh

@@ -15,10 +15,15 @@
 set -x
 set -e
 
+# NOTE: This version should be in sync with the recommended version in
+# .config/nextest.toml. (Maybe build an automated way to pull the recommended
+# version in the future.)
+NEXTEST_VERSION='0.9.97'
+PLATFORM='illumos'
+
 cargo --version
 rustc --version
-
-cargo install cargo-nextest
+curl -sSfL --retry 10 https://get.nexte.st/"$NEXTEST_VERSION"/"$PLATFORM" | gunzip | tar -xvf - -C ~/.cargo/bin
 
 pushd bgp
 cargo nextest run

.github/buildomat/jobs/test-interop.sh

@@ -0,0 +1,183 @@
+#!/bin/bash
+#:
+#: name = "test-interop"
+#: variety = "basic"
+#: target = "lab-2.0-gimlet"
+#: skip_clone = true
+#: output_rules = [
+#:   "/work/*",
+#: ]
+#:
+#: [dependencies.build-interop]
+#: job = "build-interop"
+#:
+#: [dependencies.image]
+#: job = "image"
+#:
+
+set -x
+set -e
+
+ssh() {
+	if [ -z "$SSH_BIN" ]; then
+		SSH_BIN=$(which ssh)
+	fi
+
+	$SSH_BIN -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no "$@"
+}
+
+net_info() {
+	#
+	# grab command output to see what's going on from the buildomat logs
+	#
+	if [ -z "$ARISTA_IP" ]; then
+		ARISTA_IF=$(pfexec ./interop exec arista "ip -4 -j route show default | jq '.[0][\"dev\"]' | tr -d '\"'")
+		ARISTA_IP=$(pfexec ./interop exec arista "ip -4 -br -j addr show dev $ARISTA_IF | jq '.[0][\"addr_info\"][0][\"local\"]' | tr -d '\"'")
+	fi
+	ssh root@"$ARISTA_IP" "docker exec -t ceos1 Cli -c 'show ip interface brief | no-more'"
+	ssh root@"$ARISTA_IP" "docker exec -t ceos1 Cli -c 'show ip bgp summary | no-more'"
+	ssh root@"$ARISTA_IP" "docker exec -t ceos1 Cli -c 'show ip bgp | no-more'"
+	ssh root@"$ARISTA_IP" "docker exec -t ceos1 Cli -c 'show ip route | no-more'"
+
+	if [ -z "$JUNIPER_IP" ]; then
+		JUNIPER_IF=$(pfexec ./interop exec juniper "ip -j route show default | jq '.[0][\"dev\"]' | tr -d '\"'")
+		JUNIPER_IP=$(pfexec ./interop exec juniper "ip -4 -br -j addr show dev $JUNIPER_IF | jq '.[0][\"addr_info\"][0][\"local\"]' | tr -d '\"'")
+	fi
+	ssh root@"$JUNIPER_IP" "docker exec -t crpd1 cli -c 'show interfaces terse | no-more'"
+	ssh root@"$JUNIPER_IP" "docker exec -t crpd1 cli -c 'show bgp summary | no-more'"
+	ssh root@"$JUNIPER_IP" "docker exec -t crpd1 cli -c 'show route | no-more'"
+
+	if [ -z "$MGD_IP" ]; then
+		MGD_IF=$(pfexec ./interop exec mgd "route get -inet default | grep interface | awk '{print \$NF}'")
+		MGD_IP=$(pfexec ./interop exec mgd "ipadm show-addr $MGD_IF/v4 -p -o addr | cut -d / -f 1")
+	fi
+	ssh root@"$MGD_IP" "/opt/cargo-bay/mgadm bgp status neighbors 65100"
+	ssh root@"$MGD_IP" "/opt/cargo-bay/mgadm bgp status imported 65100"
+	ssh root@"$MGD_IP" "/opt/cargo-bay/mgadm bgp status selected 65100"
+	ssh root@"$MGD_IP" "/opt/cargo-bay/mgadm bgp status exported 65100"
+}
+
+_exit_trap() {
+	local status=$?
+	[[ $status -eq 0 ]] && exit 0
+
+	set +o errexit
+
+	banner 'debug'
+
+	#
+	# collect general info about runner
+	#
+	pfexec df -h
+	pfexec diskinfo
+	pfexec zfs list
+	pfexec zpool list
+	pfexec ls -l /ci
+
+	#
+	# collect falcon info
+	#
+	find /ci/testbed/interop/.falcon -ls
+	cp /ci/testbed/interop/.falcon/{arista,juniper,mgd}* /work/
+
+	#
+	# check if propolis is running
+	#
+	pgrep -lf propolis-server
+
+	#
+	# get network info from nodes
+	#
+	net_info
+
+	#
+	# grab platform-specific logs
+	# variables set in prior call to net_info
+	#
+	ssh root@"$ARISTA_IP" "cp /tmp/init.log /tmp/arista.init.log"
+	ssh root@"$ARISTA_IP" "docker ps -a > /tmp/arista.docker-ps.log"
+	ssh root@"$ARISTA_IP" "docker logs ceos1 > /tmp/arista.docker.logs"
+	ssh root@"$ARISTA_IP" "docker exec -t ceos1 cat /var/log/account.log > /tmp/arista.account.log"
+	ssh root@"$ARISTA_IP" "docker exec -t ceos1 cat /var/log/messages > /tmp/arista.messages"
+	ssh root@"$ARISTA_IP" "docker exec -t ceos1 cat /var/log/nginx-error.log > /tmp/arista.nginx-error.log"
+	ssh root@"$ARISTA_IP" "docker exec -t ceos1 cat /var/log/nginx-access.log > /tmp/arista.nginx-access.log"
+	scp root@"$ARISTA_IP":/tmp/*.log /work
+
+	ssh root@"$JUNIPER_IP" "cp /tmp/init.log /tmp/juniper.init.log"
+	ssh root@"$JUNIPER_IP" "docker ps -a > /tmp/juniper.docker-ps.log"
+	ssh root@"$JUNIPER_IP" "docker logs crpd1 > /tmp/juniper.docker-logs.log"
+	ssh root@"$JUNIPER_IP" "docker exec -t crpd1 cat /var/log/messages > /tmp/juniper-messages.log"
+	ssh root@"$JUNIPER_IP" "docker exec -t crpd1 cat /var/log/na-grpcd > /tmp/juniper-na-grpcd.log"
+	scp root@"$JUNIPER_IP":/tmp/*.log /work
+
+	# /tmp filepaths chosen in testbed/interop/src/interop.rs and testbed/interop/cargo-bay/mgd/init.sh
+	ssh root@"$MGD_IP" "cp /tmp/init.log /tmp/mgd.init.log"
+	scp root@"$MGD_IP":/tmp/{mgd.init,mgd}.log /work
+
+	find /work -ls
+
+	exit 1
+}
+
+trap _exit_trap EXIT
+
+banner 'inputs'
+
+find /input -ls
+
+banner 'zpool'
+
+# pick the largest disk available
+DISK=$(pfexec diskinfo -pH | sort -k8 -n -r | head -1 | awk '{print $2}')
+export DISK
+pfexec zpool create -o ashift=12 -f cpool "$DISK"
+pfexec zfs create -o mountpoint=/ci cpool/ci
+
+if [[ $(curl -s http://catacomb.eng.oxide.computer:12346/trim-me) =~ "true" ]]; then
+		pfexec zpool trim cpool
+		while [[ ! $(zpool status -t cpool) =~ "100%" ]]; do sleep 10; done
+fi
+
+pfexec chown "$UID" /ci
+cd /ci
+export FALCON_DATASET="cpool/falcon"
+
+banner 'setup'
+
+tar xvfz /input/build-interop/work/testbed.tar.gz
+
+mkdir -p image/mgd
+(cd image/mgd && tar xvfz /input/image/out/mgd.tar.gz)
+for bin in mgadm mgd; do
+	mv "image/mgd/root/opt/oxide/mgd/bin/$bin" \
+	    "testbed/interop/cargo-bay/mgd/$bin"
+done
+cd testbed
+mkdir -p target/debug
+mv out/{interop,wrangler} target/debug
+mv out/baseline interop
+
+banner 'dhcp-server'
+
+export EXT_INTERFACE=${EXT_INTERFACE:-igb0}
+
+cp /input/build-interop/work/dhcp-server .
+chmod +x dhcp-server
+first=$(bmat address ls -f extra -Ho first)
+last=$(bmat address ls -f extra -Ho last)
+gw=$(bmat address ls -f extra -Ho gateway)
+server=$(ipadm show-addr "$EXT_INTERFACE"/dhcp -po ADDR | sed 's#/.*##g')
+pfexec ./dhcp-server "$first" "$last" "$gw" "$server" &> /work/dhcp-server.log &
+
+banner 'launch'
+
+cd interop
+pfexec ./interop launch
+
+banner 'status'
+
+net_info
+
+banner 'test'
+
+./baseline --show-output

.github/buildomat/jobs/test-rdb.sh

@@ -15,10 +15,15 @@
 set -x
 set -e
 
+# NOTE: This version should be in sync with the recommended version in
+# .config/nextest.toml. (Maybe build an automated way to pull the recommended
+# version in the future.)
+NEXTEST_VERSION='0.9.97'
+PLATFORM='illumos'
+
 cargo --version
 rustc --version
-
-cargo install cargo-nextest
+curl -sSfL --retry 10 https://get.nexte.st/"$NEXTEST_VERSION"/"$PLATFORM" | gunzip | tar -xvf - -C ~/.cargo/bin
 
 pushd rdb
 

.gitignore

@@ -1,4 +1,5 @@
 target
+ci
 *.sw*
 rusty-tags.vi
 cargo-bay