tests: Add a network swing test.

tests/TODO

@@ -178,6 +178,7 @@ killcluster.test
 register_linearizable.test
 sigstopcluster.test
 cinsert_linearizable.test
+netloss.test              -- does not pass most of the time; requires kernel 4.x & docker 17.x
 <END>
 
 # vim: set sw=4 ts=4 et:

tests/netloss.test/Dockerfile.client

@@ -0,0 +1,9 @@
+FROM @image@
+
+ENV PATH      $PATH:/opt/bb/bin
+
+COPY init.client /init.client
+COPY send /opt/bb/bin/send
+COPY dotran ./dotran
+
+ENTRYPOINT /init.client

tests/netloss.test/Dockerfile.db

@@ -0,0 +1,9 @@
+FROM @image@
+
+ENV PATH      $PATH:/opt/bb/bin
+
+COPY testdb /opt/bb/var/cdb2/testdb
+COPY init /init
+COPY send /opt/bb/bin/send
+
+ENTRYPOINT /init

tests/netloss.test/Dockerfile.dev

@@ -0,0 +1,43 @@
+FROM ubuntu:xenial
+
+RUN apt-get update && \
+  apt-get install -y \
+    cmake \
+    bc \
+    bison \
+    build-essential \
+    flex \
+    gawk \
+    liblz4-dev \
+    libprotobuf-c-dev \
+    libreadline-dev \
+    libsqlite3-dev \
+    libssl-dev \
+    libunwind-dev \
+    libz-dev \
+    make \
+    ncurses-dev \
+    protobuf-c-compiler \
+    tcl \
+    uuid-dev \
+    libz1 \
+    liblz4-tool \
+    libprotobuf-c1 \
+    libreadline6 \
+    libsqlite3-0 \
+    libuuid1 \
+    libz1 \
+    tzdata \
+    iputils-ping \
+    net-tools \
+    iproute2 \
+    socat \
+    time \
+    strace && \
+  rm -rf /var/lib/apt/lists/*
+
+COPY . /comdb2.build
+
+RUN cd /comdb2.build && make clean && make -j4 && make install && cd / && rm -fr comdb2.build
+
+ENV PATH      $PATH:/opt/bb/bin

tests/netloss.test/Makefile

@@ -0,0 +1,11 @@
+ifeq ($(TESTSROOTDIR),)
+  include ../testcase.mk
+else
+  include $(TESTSROOTDIR)/testcase.mk
+endif
+# Downloading base OS and packages can take quite a while.
+export TEST_TIMEOUT=20m
+
+# this is a local test, don't need cluster
+export CLUSTER=
+export COMDB2_UNITTEST=1

tests/netloss.test/README

@@ -0,0 +1,11 @@
+Test Comdb2 with 2 redundant networks where one fails.
+
+Software requirement:
+* docker
+* docker-composer
+
+Versions known to work:
+* docker 17.04.0-ce
+* docker-composer version 1.16.1
+
+To run the test, change directory into `tests' and run `make netloss'.

tests/netloss.test/docker-compose.yml

@@ -0,0 +1,95 @@
+version: '3'
+
+networks:
+    primary:
+    n3:
+    n4:
+
+services:
+    client:
+        cap_add:
+            - SYS_PTRACE
+        hostname: c1
+        networks:
+            - primary
+        build:
+            context: .
+            dockerfile: Dockerfile.client
+        volumes:
+            - ./log:/log
+    m1:
+        hostname: m1
+        build:
+            context: .
+            dockerfile: Dockerfile.db
+        networks:
+            primary:
+            n3:
+                aliases: 
+                    - m1-n3
+            n4:
+                aliases: 
+                    - m1-n4
+        volumes:
+            - ./log:/log
+    m2:
+        hostname: m2
+        build:
+            context: .
+            dockerfile: Dockerfile.db
+        networks:
+            primary:
+            n3:
+                aliases: 
+                    - m2-n3
+            n4:
+                aliases: 
+                    - m2-n4
+        volumes:
+            - ./log:/log
+    m3:
+        hostname: m3
+        build:
+            context: .
+            dockerfile: Dockerfile.db
+        networks:
+            primary:
+            n3:
+                aliases: 
+                    - m3-n3
+            n4:
+                aliases: 
+                    - m3-n4
+        volumes:
+            - ./log:/log
+    m4:
+        hostname: m4
+        build:
+            context: .
+            dockerfile: Dockerfile.db
+        networks:
+            primary:
+            n3:
+                aliases: 
+                    - m4-n3
+            n4:
+                aliases: 
+                    - m4-n4
+        volumes:
+            - ./log:/log
+
+    m5:
+        hostname: m5
+        build:
+            context: .
+            dockerfile: Dockerfile.db
+        networks:
+            primary:
+            n3:
+                aliases: 
+                    - m5-n3
+            n4:
+                aliases: 
+                    - m5-n4
+        volumes:
+            - ./log:/log

tests/netloss.test/dotran

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+(
+  echo "begin"
+  for i in $(seq 1 5000); do
+  echo "insert into t1(a) values($RANDOM)"
+  done
+  echo "commit"
+ ) | cdb2sql testdb dev - >/dev/null

tests/netloss.test/init

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+exec >/log/$(hostname).log 2>&1
+
+pmux -l &
+sleep 1
+/opt/bb/bin/comdb2 testdb 

tests/netloss.test/init.client

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+exec > /log/client.log 2>&1
+
+mkdir -p /opt/bb/etc/cdb2/config.d/
+echo "testdb m1 m2 m3 m4 m5" > /opt/bb/etc/cdb2/config.d/testdb.cfg
+
+rc=$(cdb2sql -tabs testdb dev "select 1")
+while :; do
+    if [[ "$rc" == "1" ]]; then
+        break
+    fi
+    echo "waiting for db to come up"
+    sleep 0.2
+    rc=$(cdb2sql -tabs testdb dev "select 1")
+done
+
+echo READY
+
+cdb2sql testdb dev "create table t1(a integer)"
+
+function writer  {
+me=$1
+echo started writer $me
+while :; do
+    (
+     date
+     /usr/bin/time -f '%E' ./dotran
+    ) >> /log/$me.log 2>&1
+done
+}
+
+writer 1 &
+writer 2 &
+writer 3 &
+
+while :; do
+    date
+    cdb2sql testdb dev "select comdb2_host()"
+    sleep 1
+done
+
+wait

tests/netloss.test/runit

@@ -0,0 +1,78 @@
+#!/bin/bash
+
+set -e
+set -x
+prefix=$(basename $(realpath $(pwd)/..))
+dir=$(pwd)
+echo test dir is $dir
+prefix=$(echo $prefix | sed "s/_//g")
+echo prefix for test is $prefix
+
+export COMPOSE_PROJECT_NAME=$prefix
+
+cd ../../../
+pwd
+
+# Build a container from the current source
+docker build -t comdb2test:$prefix -f ${dir}/Dockerfile.dev .
+cd $dir
+pwd
+rm -fr testdb
+rm -fr log
+mkdir testdb
+mkdir log
+chmod 777 log
+../../../comdb2 testdb --create --dir $(pwd)/testdb
+sed -i.bak 's!^dir.*!dir     /opt/bb/var/cdb2/testdb!' testdb/testdb.lrl
+cat >> testdb/testdb.lrl <<EOF
+cluster nodes m1 m2 m3 m4 m5
+SOSQL_MAX_COMMIT_WAIT_SEC 60
+dedicated_network_suffixes -n3 -n4
+EOF
+
+# We'll build our db and client containers from that container
+sed -i.bak "s/@image@/comdb2test:$prefix/" Dockerfile.client Dockerfile.db
+docker-compose build
+
+# Start, wait for it to signal that it's ready
+docker-compose up &
+(cat log/client.log ; tail -F log/client.log) | sed '/READY/q' >/dev/null
+
+# let it run for a bit
+sleep 30
+
+# break the network
+net=$((3 + $RANDOM % 2))
+echo "Disconnecting n${net}..."
+for m in $(seq 1 5); do 
+    docker network disconnect ${prefix}_n${net} ${prefix}_m${m}_1
+done
+
+# Let the test keep running, we'll look for errors later.
+sleep 120
+
+set +e
+
+rc=0
+# Let's see if we have errors:
+
+grep -q "Master.*failed to acknowledge session" log/m*.log
+[[ $? -eq 0 ]] && { echo 'master failed to ack session'; rc=$((rc + 1)); }
+
+grep -q "master.*lost transaction" log/*.log
+[[ $? -eq 0 ]] && { echo 'master lost txn'; rc=$((rc + 1)); }
+
+grep -q "Cannot connect to db" log/*.log
+[[ $? -eq 0 ]] && { echo 'client could not connect'; rc=$((rc + 1)); }
+
+# tear down
+docker network ls
+
+# shutting down instances
+docker-compose down --rmi 'all' --volumes --remove-orphans
+
+# try to clean up
+docker rmi -f comdb2test:$prefix >/dev/null 2>&1
+docker network rm ${prefix}_primary ${prefix}_n3 ${prefix}_n4 >/dev/null 2>&1
+
+exit $rc

tests/netloss.test/send

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+if [[ $# -lt 2 ]]; then
+    echo "Usage: db command"
+    exit 1
+fi
+
+db=$1
+shift
+cmd=$*
+
+cdb2sql -tabs $db @localhost "exec procedure sys.cmd.send('$cmd')"