tests: Check unit tests logs for errors.

System tests are checking for errors in logs at the end of the tests.
This patch implements the same feature for Unit tests.
However, many unit tests display errors in the logs.
Hence this patch also:
- fixes a few tests, when the error cause was identified to be due to the test.
- add exceptions to ignore most common errors (in the macro, common for all tests)
- add exceptions to ignore some errors unique to some tests.
Future patches should investigate those errors, and either fix
ovn, fix the test or add exceptions (some errors are expected, such as purposely passing wrong
parameters).

Signed-off-by: Xavier Simonart <xsimonar@redhat.com>
Signed-off-by: Mark Michelson <mmichels@redhat.com>

tests/ovn-controller.at

@@ -74,15 +74,15 @@ check_patches
 # won't be created until there's a localnet port on a logical switch with
 # another logical port bound to this chassis.
 ovn-sbctl \
-    -- --id=@dp101 create Datapath_Binding tunnel_key=101 \
+    -- --id=@dp101 create Datapath_Binding tunnel_key=101 external_ids:name=dp101 \
     -- create Port_Binding datapath=@dp101 logical_port=localnet1 tunnel_key=1 \
         type=localnet options:network_name=physnet1
 check_patches
 
 # Create a localnet port on a logical switch with a port bound to this chassis.
 # Now we should get some patch ports created.
 ovn-sbctl \
-    -- --id=@dp102 create Datapath_Binding tunnel_key=102 \
+    -- --id=@dp102 create Datapath_Binding tunnel_key=102 external_ids:name=dp102 \
     -- create Port_Binding datapath=@dp102 logical_port=localnet2 tunnel_key=1 \
         type=localnet options:network_name=physnet1 \
     -- create Port_Binding datapath=@dp102 logical_port=localvif2 tunnel_key=2
@@ -96,8 +96,8 @@ check_patches \
 # OVN no longer uses OVS patch ports to implement logical patch ports, so
 # the set of OVS patch ports doesn't change.
 AT_CHECK([ovn-sbctl \
-    -- --id=@dp1 create Datapath_Binding tunnel_key=1 \
-    -- --id=@dp2 create Datapath_Binding tunnel_key=2 \
+    -- --id=@dp1 create Datapath_Binding tunnel_key=1 external_ids:name=dp1 \
+    -- --id=@dp2 create Datapath_Binding tunnel_key=2 external_ids:name=dp2 \
     -- create Port_Binding datapath=@dp1 logical_port=foo tunnel_key=1 type=patch options:peer=bar \
     -- create Port_Binding datapath=@dp2 logical_port=bar tunnel_key=2 type=patch options:peer=foo \
     -- create Port_Binding datapath=@dp1 logical_port=dp1vif tunnel_key=3 \
@@ -241,7 +241,20 @@ wait_row_count Chassis_Private 1
 wait_row_count Chassis 1
 
 # Gracefully terminate daemons
-OVN_CLEANUP_SBOX([hv])
+# Ignore  following errors:
+# - Invalid ovn-bridge-mappings configuration: 'foo-mapping'
+# - unknown datapath type foo
+# - could not create datapath br-int of unknown type foo
+# - could not create datapath br-int of unknown type foobar
+# - unknown datapath type bar
+# - could not create datapath br-int of unknown type bar
+# - Transaction causes multiple rows ...
+# - failed to create bridge br-int: Address family not supported by protocol # due to previous errors.
+OVN_CLEANUP_SBOX([hv], ["/foo/d
+/bar/d
+/Transaction causes multiple rows/d
+/failed to create bridge/d"])
+
 OVN_CLEANUP_VSWITCH([main])
 as ovn-sb
 OVS_APP_EXIT_AND_WAIT([ovsdb-server])
@@ -319,6 +332,8 @@ OVN_CLEANUP_SBOX([hv])
 OVN_CLEANUP_VSWITCH([main])
 as ovn-sb
 OVS_APP_EXIT_AND_WAIT([ovsdb-server])
+as ovn-nb
+OVS_APP_EXIT_AND_WAIT([ovsdb-server])
 
 AT_CLEANUP
 ])
@@ -355,6 +370,8 @@ OVN_CLEANUP_SBOX([hv])
 OVN_CLEANUP_VSWITCH([main])
 as ovn-sb
 OVS_APP_EXIT_AND_WAIT([ovsdb-server])
+as ovn-nb
+OVS_APP_EXIT_AND_WAIT([ovsdb-server])
 
 AT_CLEANUP
 ])
@@ -386,6 +403,8 @@ OVN_CLEANUP_SBOX([hv])
 OVN_CLEANUP_VSWITCH([main])
 as ovn-sb
 OVS_APP_EXIT_AND_WAIT([ovsdb-server])
+as ovn-nb
+OVS_APP_EXIT_AND_WAIT([ovsdb-server])
 
 AT_CLEANUP
 
@@ -647,7 +666,9 @@ primary lport : [[lsp1]]
     done
 done
 
-OVN_CLEANUP([hv1])
+check as northd ovn-appctl -t ovn-northd resume
+OVN_CLEANUP([hv1
+/No distributed-port option set for chassisredirect port lsp1/d])
 AT_CLEANUP
 
 AT_SETUP([ovn-controller - ssl files change when using command line options])
@@ -704,7 +725,16 @@ OVS_WAIT_FOR_OUTPUT([ovn-appctl -t ovn-controller connection-status], [0], [conn
 
 cat hv1/ovn-controller.log
 
-OVN_CLEANUP([hv1])
+# Expect different errors as trying to connect with wrong keys/certificates, or with keys/certificates deleted
+OVN_CLEANUP([hv1
+/SSL_use_PrivateKey_file/d
+/SSL_use_certificate_file/d
+/Private key must be configured to use SSL/d
+/Certificate must be configured to use SSL/d
+/SSL_read: error/d
+/receive error: Input/d
+/connection dropped/d
+])
 AT_CLEANUP
 
 OVN_FOR_EACH_NORTHD([
@@ -832,10 +862,13 @@ tos_option=$(ovs-vsctl get interface ovn-fakech-0 options)
 AT_CHECK([test "$tos_option" = "$no_tos_options"], [0], [])
 
 # Gracefully terminate daemons
-OVN_CLEANUP_SBOX([hv])
+# Expect invalid TOS 61
+OVN_CLEANUP_SBOX([hv], ["/invalid TOS 61/d"])
 OVN_CLEANUP_VSWITCH([main])
 as ovn-sb
 OVS_APP_EXIT_AND_WAIT([ovsdb-server])
+as ovn-nb
+OVS_APP_EXIT_AND_WAIT([ovsdb-server])
 
 AT_CLEANUP
 ])
@@ -2120,6 +2153,8 @@ for i in $(seq 10); do
     OVS_WAIT_UNTIL([test $(as hv1 ovs-ofctl dump-flows br-int table=46 | grep -c "priority=1100") = "$(($i + 1))"])
 done
 
+# Resume northd to be able to properly cleanup
+check as northd ovn-appctl -t ovn-northd resume
 OVN_CLEANUP([hv1])
 AT_CLEANUP
 
@@ -2294,7 +2329,9 @@ AT_CHECK([grep "Parsing of ovn-chassis-mac-mappings failed" hv1/ovn-controller.l
 AT_CHECK([grep "Parsing of ovn-chassis-mac-mappings failed" hv1/ovn-controller.log | grep -q invalid2])
 AT_CHECK([grep "Parsing of ovn-chassis-mac-mappings failed" hv1/ovn-controller.log | grep -q br1], [1])
 
-OVN_CLEANUP([hv1])
+OVN_CLEANUP([hv1
+/Parsing of ovn-chassis-mac-mappings failed/d
+])
 AT_CLEANUP
 
 AT_SETUP([ovn-controller - localport can be recreated])
@@ -2631,7 +2668,10 @@ OVS_WAIT_UNTIL([grep -q 'Removing unsupported key "unsupported" from chassis rec
 
 AT_CHECK([ovn-sbctl get chassis $chassis_id other_config:unsupported], [1], [ignore], [ignore])
 
-OVN_CLEANUP([hv1])
+OVN_CLEANUP([hv1
+/Removing unsupported key/d
+])
+
 AT_CLEANUP
 ])
 
@@ -2723,7 +2763,11 @@ ovn-nbctl --wait=hv sync
 
 AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=OFTABLE_LOCAL_OUTPUT | grep -q controller], [1])
 
-OVN_CLEANUP([hv1])
+# Since we add many ports in one nbctl command, we might get some warnings on slow systems.
+OVN_CLEANUP([hv1
+/Unreasonably long/d
+/faults:/d
+/context switches/d])
 AT_CLEANUP
 ])
 

tests/ovn-ic.at

@@ -136,9 +136,9 @@ AT_CHECK([ovn-ic-sbctl show | grep -A2 lsp1], [0], [dnl
 # remove transit switch and check if port_binding is deleted
 check ovn-ic-nbctl ts-del ts1
 wait_row_count ic-sb:Port_Binding 0 logical_port=lsp1
-
 for i in 1 2; do
     az=az$i
+    ovn_as $az
     OVN_CLEANUP_SBOX(gw-$az)
     OVN_CLEANUP_AZ([$az])
 done
@@ -213,6 +213,7 @@ ovn-ic-sbctl list route
 
 for i in 1 2; do
     az=az$i
+    ovn_as $az
     OVN_CLEANUP_SBOX(gw-$az)
     OVN_CLEANUP_AZ([$az])
 done
@@ -273,6 +274,7 @@ wait_row_count ic-sb:route 1 ip_prefix=1.1.1.1/32
 
 for i in 1 2; do
     az=az$i
+    ovn_as $az
     OVN_CLEANUP_SBOX(gw-$az)
     OVN_CLEANUP_AZ([$az])
 done

tests/ovn-macros.at

@@ -10,32 +10,62 @@ m4_define([OVN_CLEANUP_VSWITCH],[
     OVS_APP_EXIT_AND_WAIT([ovsdb-server])
 ])
 
-# OVN_CLEANUP_SBOX(sbox)
+# OVN_CLEANUP_CONTROLLER(sbox)
 #
-# Gracefully terminate OVN daemons in the specified
+# Gracefully terminate ovn-controller in the specified
 # sandbox instance. The sandbox name "vtep" is treated
 # as a special case, and is assumed to have ovn-controller-vtep
 # and ovs-vtep daemons running instead of ovn-controller.
-m4_define([OVN_CLEANUP_SBOX],[
-    echo
-    echo "$1: clean up sandbox"
-    as $1
-    if test "$1" = "vtep"; then
+m4_define([OVN_CLEANUP_CONTROLLER],[
+    hv=$1
+    sbox=$2
+
+    ovn-nbctl --wait=hv sync
+    as $sbox
+    if test "$hv" = "vtep"; then
         OVS_APP_EXIT_AND_WAIT([ovn-controller-vtep])
         OVS_APP_EXIT_AND_WAIT([ovs-vtep])
     else
         OVS_APP_EXIT_AND_WAIT([ovn-controller])
     fi
-    OVN_CLEANUP_VSWITCH([$1])
+])
+
+# OVN_CLEANUP_SBOX(sbox)
+#
+# Gracefully terminate OVN daemons in the specified
+# sandbox instance. The sandbox name "vtep" is treated
+# as a special case, and is assumed to have ovn-controller-vtep
+# and ovs-vtep daemons running instead of ovn-controller.
+m4_define([OVN_CLEANUP_SBOX],[
+    sbox=$1
+    error=$2
+    echo "$sbox: clean up sandbox"
+    OVN_CLEANUP_CONTROLLER([$sbox], [$sbox])
+    OVN_CLEANUP_VSWITCH([$sbox])
+
+    # Check for errors in logs. Ignore following errors:
+    # "connection failed (No such file or directory)" as happens everytime we restart ovs-vswitchd.
+    # "has no network name*" as localnet network_name is often set "after" creating localnet.
+    # "receive tunnel port not found*" happening when closing other hv.
+    # "Failed to locate tunnel to reach main chassis" happening when controller started e.g. before ovn-bridge-mappings is set
+    AT_CHECK([check_logs "
+        $error
+        /connection failed (No such file or directory)/d
+        /has no network name*/d
+        /receive tunnel port not found*/d
+        /Failed to locate tunnel to reach main chassis/d
+    " $sbox])
 ])
 
 # OVN_CLEANUP(sim [, sim ...])
 #
 # Gracefully terminate all OVN daemons, including those in the
 # specified sandbox instances.
 m4_define([OVN_CLEANUP],[
-    m4_foreach([sbox], [$@], [
-        OVN_CLEANUP_SBOX([sbox])
+    m4_foreach([sbox_and_error], [$@], [
+        sbox=$(echo "sbox_and_error" |sed -n '1p')
+        error=$(echo "sbox_and_error" | sed -n '2,$p')
+        OVN_CLEANUP_SBOX([$sbox], [$error])
     ])
 
     echo