Update for Interopability Tests #2 (to be squashed later)

examples/oscore_testcases.sh Add in support for reporting Pass / Fail Make it more configurable by adding runtime options
obgm · Nov 7, 2022 · 04f275f · 04f275f
1 parent 762dce0
commit 04f275f
Show file tree

Hide file tree

Showing 2 changed files with 216 additions and 57 deletions.
diff --git a/examples/oscore-interop-server.c b/examples/oscore-interop-server.c
@@ -226,6 +226,11 @@ hnd_get_observe1(coap_resource_t *resource,
     break;
   default:
     coap_pdu_set_code(response, COAP_RESPONSE_CODE_INTERNAL_ERROR);
+    coap_add_data_large_response(resource, session, request, response,
+                                 query, COAP_MEDIATYPE_TEXT_PLAIN,
+                                 -1, 0, strlen("Terminate Observe"),
+                                 (const uint8_t *)"Terminate Observe",
+                                 NULL, NULL);
     r_observe_1 = NULL;
   }
 }

diff --git a/examples/oscore_testcases.sh b/examples/oscore_testcases.sh
@@ -1,105 +1,259 @@
 #!/bin/bash
 
-if [ ! -z "$1" ] ; then
-  TARGET_IP=$1
-  echo Target IP $TARGET_IP
-else
-  TARGET_IP=127.0.0.1
-fi
+#
+# This script is used to run the oscore interop tests as specified in
+# https://core-wg.github.io/oscore/test-spec5.html
+#
+# By default, this script should be run in the examples directory.
+#
+# Run as
+#  ./oscore_testcases.sh [-h remote-target-IP] [-B port-B-OSCORE] \
+#                        [-D port-D-OSCORE] [-N port-NO-OSCORE] \
+#                        [-s executable-for-interop-server] \
+#                        [-c executable-for-client] \
+#                        [-P] [-F]
+#
+# -h remote-target-IP
+#  Remote server hosting interop tests if not running the interop server on this host.
+#
+# -B port-B-OSCORE
+#  Port that the server listening on providing B OSCORE security profile
+#
+# -D port-D-OSCORE
+#  Port that the server listening on providing D OSCORE security profile
+#
+# -N port-N-OSCORE
+#  Port that the server listening on providing no security profile
+#
+# -s executable-for-interop-server
+#  Exectuable to use for the interop server if not the default of ./oscore-interop-server.
+#
+# -c executable-for-client
+#  Exectuable to use for the coap client if not the default of ./coap-client.
+#
+# -P
+#  Output partial client logs
+#
+# -F
+#  Output full client logs
+#
+
+INDIR=`dirname $0`
+
+# Defaults
 
-#Server with B OSCORE Security
+# host running oscore interop server
+TARGET_IP=127.0.0.1
+# Server with B OSCORE Security
 S_PORT_B=5683
-#Server with D OSCORE Security
+# Server with D OSCORE Security
 S_PORT_D=5685
-#Server with no Security
+# Server with no Security
 S_PORT_N=5687
+# Client app
+CLIENT=$INDIR/coap-client
+# SERVER app
+SERVER=$INDIR/oscore-interop-server
+# Partial Logs
+PARTIAL_LOGS=no
+# Full Logs
+FULL_LOGS=no
+
+while getopts "c:h:s:B:D:FN:P" OPTION; do
+  case $OPTION in
+    c)
+      CLIENT="$OPTARG"
+      ;;
+    h)
+      TARGET_IP="$OPTARG"
+      ;;
+    s)
+      SERVER="$OPTARG"
+      ;;
+    B)
+      S_PORT_B="$OPTARG"
+      ;;
+    D)
+      S_PORT_D="$OPTARG"
+      ;;
+    F)
+      FULL_LOGS=yes
+      ;;
+    N)
+      S_PORT_N="$OPTARG"
+      ;;
+    P)
+      PARTIAL_LOGS=yes
+      ;;
+    *)
+      echo Error in options detected
+      echo Run as
+      echo "$0 [-h remote-target-IP] [-B port-B-OSCORE]"
+      echo "      [-D port-D-OSCORE] [-N port-NO-OSCORE]"
+      echo "      [-s executable-for-interop-server]"
+      echo "      [-c executable-for-client]"
+      echo "      [-P] [-F]"
+      exit 1
+  esac
+done
+
+timecheck () {
+  timeout $*
+  if [ $? = 124 ] ; then
+    echo "****** Timed Out ******"
+  fi
+}
+
+NO_PASS=0
+NO_FAIL=0
+# passfail count egrep-expression
+passfail () {
+  PASS=`cat /tmp/client_out | egrep "$2" | wc -l`
+  if [ "$PASS" = "$1" ] ; then
+    echo Pass
+    let "NO_PASS=$NO_PASS+1"
+  else
+    echo Fail
+    let "NO_FAIL=$NO_FAIL+1"
+  fi
+  if [ "$FULL_LOGS" = yes ] ; then
+    cat /tmp/client_out
+  elif [ "$PARTIAL_LOGS" = yes ] ; then
+    cat /tmp/client_out | egrep -v " DEBG | OSC  "
+  fi
+}
 
-./oscore-interop-server -E interop/b_server.conf -v8 -p $S_PORT_B > /tmp/server_b 2>&1 &
-./oscore-interop-server -E interop/d_server.conf -v8 -p $S_PORT_D > /tmp/server_d 2>&1 &
-./oscore-interop-server                          -v8 -p $S_PORT_N > /tmp/server_n 2>&1 &
+$SERVER -E $INDIR/interop/b_server.conf -v8 -p $S_PORT_B > /tmp/server_b 2>&1 &
+$SERVER -E $INDIR/interop/d_server.conf -v8 -p $S_PORT_D > /tmp/server_d 2>&1 &
+$SERVER                                 -v8 -p $S_PORT_N > /tmp/server_n 2>&1 &
 
+# Reset sequence number counters
 rm -f /tmp/client_a
 rm -f /tmp/client_c
 
 # Test 0 General checkout
-echo Test 0
-./coap-client -w -v8 coap://$TARGET_IP:$S_PORT_B/oscore/hello/coap 2>&1 | egrep -v " DEBG | OSC  "
+echo -n "Test 0 - "
+timecheck 10 $CLIENT -w -v8 coap://$TARGET_IP:$S_PORT_B/oscore/hello/coap 2>&1 | egrep -v " DEBG | OSC  " > /tmp/client_out
+passfail 1 "^Hello World"
 
 # Test 1
-echo Test 1
-./coap-client -w -v8 -E interop/a_client.conf,/tmp/client_a coap://$TARGET_IP:$S_PORT_B/oscore/hello/1 2>&1 | egrep -v " DEBG | OSC  "
+echo -n "Test 1 - "
+timeout 10 $CLIENT -w -v8 -E $INDIR/interop/a_client.conf,/tmp/client_a coap://$TARGET_IP:$S_PORT_B/oscore/hello/1 > /tmp/client_out 2>&1
+passfail 1 "^Hello World"
 
 # Test 2
-echo Test 2
-./coap-client -w -v8 -E interop/c_client.conf,/tmp/client_c coap://$TARGET_IP:$S_PORT_D/oscore/hello/1 2>&1 | egrep -v " DEBG | OSC  "
+echo -n "Test 2 - "
+timeout 10 $CLIENT -w -v8 -E $INDIR/interop/c_client.conf,/tmp/client_c coap://$TARGET_IP:$S_PORT_D/oscore/hello/1 > /tmp/client_out 2>&1
+passfail 1 "^Hello World"
 
 # Test 3
-echo Test 3
-./coap-client -w -v8 -E interop/a_client.conf,/tmp/client_a coap://$TARGET_IP:$S_PORT_B/oscore/hello/2?first=1 2>&1 | egrep -v " DEBG | OSC  "
+echo -n "Test 3 - "
+timeout 10 $CLIENT -w -v8 -E $INDIR/interop/a_client.conf,/tmp/client_a coap://$TARGET_IP:$S_PORT_B/oscore/hello/2?first=1 > /tmp/client_out 2>&1
+passfail 1 "^Hello World"
 
 # Test 4
-echo Test 4
-./coap-client -w -v8 -E interop/a_client.conf,/tmp/client_a -A 0 coap://$TARGET_IP:$S_PORT_B/oscore/hello/3 2>&1 | egrep -v " DEBG | OSC  "
+echo -n "Test 4 - "
+timeout 10 $CLIENT -w -v8 -E $INDIR/interop/a_client.conf,/tmp/client_a -A 0 coap://$TARGET_IP:$S_PORT_B/oscore/hello/3 > /tmp/client_out 2>&1
+passfail 1 "^Hello World"
 
 # Test 5
-echo Test 5
-./coap-client -w -v8 -E interop/a_client.conf,/tmp/client_a -s 2 coap://$TARGET_IP:$S_PORT_B/oscore/hello/1 2>&1 | egrep -v " DEBG | OSC  "
+echo -n "Test 5 - "
+timeout 10 $CLIENT -w -v8 -E $INDIR/interop/a_client.conf,/tmp/client_a -s 2 coap://$TARGET_IP:$S_PORT_B/oscore/hello/1 > /tmp/client_out 2>&1
+passfail 1 "^Hello World"
 
 # Test 6
-echo Test 6
-./coap-client -w -v8 -E interop/a_client.conf,/tmp/client_a -s 4 coap://$TARGET_IP:$S_PORT_B/oscore/observe1 2>&1 | egrep -v " DEBG | OSC  "
+echo -n "Test 6 - "
+timeout 10 $CLIENT -w -v8 -E $INDIR/interop/a_client.conf,/tmp/client_a -s 4 coap://$TARGET_IP:$S_PORT_B/oscore/observe1 > /tmp/client_out > /tmp/client_out 2>&1
+passfail 3 "^one|^two|^5.00 Terminate Observe"
 
 # Test 7
-echo Test 7
-./coap-client -w -v8 -E interop/a_client.conf,/tmp/client_a -s 2 coap://$TARGET_IP:$S_PORT_B/oscore/observe2 2>&1 | egrep -v " DEBG | OSC  "
+echo -n "Test 7 - "
+timeout 10 $CLIENT -w -v8 -E $INDIR/interop/a_client.conf,/tmp/client_a -s 2 coap://$TARGET_IP:$S_PORT_B/oscore/observe2 > /tmp/client_out 2>&1
+passfail 3 "^one|^two"
 
 # Test 8
-echo Test 8
-./coap-client -w -v8 -E interop/a_client.conf,/tmp/client_a -m post -e "%4a" -t 0 coap://$TARGET_IP:$S_PORT_B/oscore/hello/6 2>&1 | egrep -v " DEBG | OSC  "
+echo -n "Test 8 - "
+timeout 10 $CLIENT -w -v8 -E $INDIR/interop/a_client.conf,/tmp/client_a -m post -e "%4a" -t 0 coap://$TARGET_IP:$S_PORT_B/oscore/hello/6 > /tmp/client_out 2>&1
+passfail 1 "^J"
 
 # Test 9
-echo Test 9
-./coap-client -w -v8 -E interop/a_client.conf,/tmp/client_a -m put -e "%7a" -t 0 -O 1,0x7b coap://$TARGET_IP:$S_PORT_B/oscore/hello/7 2>&1 | egrep -v " DEBG | OSC  "
+echo -n "Test 9 - "
+timeout 10 $CLIENT -w -v8 -E $INDIR/interop/a_client.conf,/tmp/client_a -m put -e "%7a" -t 0 -O 1,0x7b coap://$TARGET_IP:$S_PORT_B/oscore/hello/7 > /tmp/client_out 2>&1
+passfail 1 "^z"
 
 # Test 10
-echo Test 10
-./coap-client -w -v8 -E interop/a_client.conf,/tmp/client_a -m put -e "%8a" -t 0 -O 5 coap://$TARGET_IP:$S_PORT_B/oscore/hello/7 2>&1 | egrep -v " DEBG | OSC  "
+echo -n "Test 10 - "
+timeout 10 $CLIENT -w -v8 -E $INDIR/interop/a_client.conf,/tmp/client_a -m put -e "%8a" -t 0 -O 5 coap://$TARGET_IP:$S_PORT_B/oscore/hello/7 > /tmp/client_out 2>&1
+passfail 1 "^4.12 Precondition Failed"
 
 # Test 11
-echo
-echo Test 11
-./coap-client -w -v8 -E interop/a_client.conf,/tmp/client_a -m delete coap://$TARGET_IP:$S_PORT_B/oscore/test 2>&1 | egrep -v " DEBG | OSC  "
+if [ "$SUPPRESS" = no ] ; then
+  echo
+fi
+echo -n "Test 11 - "
+timeout 10 $CLIENT -w -v8 -E $INDIR/interop/a_client.conf,/tmp/client_a -m delete coap://$TARGET_IP:$S_PORT_B/oscore/test > /tmp/client_out 2>&1
+passfail 1 "^v:1 t:CON c:2.02 i:"
 
 # Test 12
-echo
-echo Test 12
-./coap-client -w -v8 -E interop/e_client.conf,/tmp/client_a coap://$TARGET_IP:$S_PORT_B/oscore/hello/1 2>&1 | egrep -v " DEBG | OSC  "
+if [ "$SUPPRESS" = no ] ; then
+  echo
+fi
+echo -n "Test 12 - "
+timeout 10 $CLIENT -w -v8 -E $INDIR/interop/e_client.conf,/tmp/client_a coap://$TARGET_IP:$S_PORT_B/oscore/hello/1 > /tmp/client_out 2>&1
+passfail 1 "^4.01 Security context not found"
 
 # Test 13
-echo
-echo Test 13
-./coap-client -w -v8 -E interop/f_client.conf,/tmp/client_a coap://$TARGET_IP:$S_PORT_B/oscore/hello/1 2>&1 | egrep -v " DEBG | OSC  "
+if [ "$SUPPRESS" = no ] ; then
+  echo
+fi
+echo -n "Test 13 - "
+timeout 10 $CLIENT -w -v8 -E $INDIR/interop/f_client.conf,/tmp/client_a coap://$TARGET_IP:$S_PORT_B/oscore/hello/1 > /tmp/client_out 2>&1
+passfail 1 "^4.00 Decryption failed"
 
 # Test 14
-echo
-echo Test 14
-./coap-client -w -v8 -E interop/g_client.conf,/tmp/client_a coap://$TARGET_IP:$S_PORT_B/oscore/hello/1 2>&1 | egrep -v " DEBG | OSC  "
+if [ "$SUPPRESS" = no ] ; then
+  echo
+fi
+echo -n "Test 14 - "
+timeout 10 $CLIENT -w -v8 -E $INDIR/interop/g_client.conf,/tmp/client_a coap://$TARGET_IP:$S_PORT_B/oscore/hello/1 > /tmp/client_out 2>&1
+passfail 1 "WARN OSCORE: Decryption Failure, result code: -5"
 
 # Test 15
-echo
-echo Test 15
-./coap-client -w -v8 -E interop/a_client.conf coap://$TARGET_IP:$S_PORT_B/oscore/hello/1 2>&1 | egrep -v " DEBG | OSC  "
-# ./coap-client -w -v8 -E interop/a_client.conf coap://$TARGET_IP:$S_PORT_B/oscore/hello/1 2>&1 | egrep -v " DEBG | OSC  "
+if [ "$SUPPRESS" = no ] ; then
+  echo
+fi
+echo -n "Test 15 - "
+timeout 10 $CLIENT -w -v8 -E $INDIR/interop/a_client.conf coap://$TARGET_IP:$S_PORT_B/oscore/hello/1 > /tmp/client_out 2>&1
+passfail 1 "^4.01 Replay detected"
 
 # Test 16
-echo
-echo Test 16
-./coap-client -w -v8 -E interop/e_client.conf,/tmp/client_a coap://$TARGET_IP:$S_PORT_N/oscore/hello/coap 2>&1 | egrep -v " DEBG | OSC  "
+if [ "$SUPPRESS" = no ] ; then
+  echo
+fi
+echo -n "Test 16 - "
+timeout 10 $CLIENT -w -v8 -E $INDIR/interop/e_client.conf,/tmp/client_a coap://$TARGET_IP:$S_PORT_N/oscore/hello/coap > /tmp/client_out 2>&1
+passfail 1 "^4.02 Bad Option"
 
 # Test 17
-echo
-echo Test 17
-./coap-client -w -v8 coap://$TARGET_IP:$S_PORT_N/oscore/hello/1 2>&1 | egrep -v " DEBG | OSC  "
+if [ "$SUPPRESS" = no ] ; then
+  echo
+fi
+echo -n "Test 17 - "
+timeout 10 $CLIENT -w -v8 coap://$TARGET_IP:$S_PORT_N/oscore/hello/1 > /tmp/client_out 2>&1
+passfail 1 "^4.01 Unauthorized"
+
+KILL_SERVER=`basename $SERVER`
+if [ ! -z "$KILL_SERVER" ] ; then
+  killall $KILL_SERVER
+fi
 
+echo
+echo ===============
+echo Pass:  $NO_PASS
+echo Fail:  $NO_FAIL
+#Starts with test 0
+echo Total: 18
 
-killall oscore-interop-server
+if [ "$NO_FAIL" != 0 ] ; then
+  exit 1
+fi