From 1ea3ef7894261d474f90cbdb12198496a56f6d1d Mon Sep 17 00:00:00 2001 From: Andy Green Date: Fri, 25 May 2018 15:33:28 +0800 Subject: [PATCH] travis: handle reduced max run time by splitting autobahn tests across two instances --- .travis.yml | 1 + ...tobahn-test.sh => autobahn-test-client.sh} | 43 ------- scripts/autobahn-test-server.sh | 107 ++++++++++++++++++ scripts/travis_control.sh | 32 ++++-- scripts/travis_install.sh | 6 +- 5 files changed, 131 insertions(+), 58 deletions(-) rename scripts/{autobahn-test.sh => autobahn-test-client.sh} (85%) create mode 100755 scripts/autobahn-test-server.sh diff --git a/.travis.yml b/.travis.yml index 6181435a05..cf439b4dec 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ env: - secure: "KhAdQ9ja+LBObWNQTYO7Df5J4DyOih6S+eerDMu8UPSO+CoWV2pWoQzbOfocjyOscGOwC+2PrrHDNZyGfqkCLDXg1BxynXPCFerHC1yc2IajvKpGXmAAygNIvp4KACDfGv/dkXrViqIzr/CdcNaU4vIMHSVb5xkeLi0W1dPnQOI=" matrix: - LWS_METHOD=lwsws CMAKE_ARGS="-DLWS_WITH_LWSWS=ON -DLWS_WITHOUT_EXTENSIONS=0 -DLWS_WITH_HTTP2=1 -DLWS_WITH_ACME=1 -DLWS_WITH_MINIMAL_EXAMPLES=1 -DCMAKE_BUILD_TYPE=DEBUG" + - LWS_METHOD=lwsws2 CMAKE_ARGS="-DLWS_WITH_LWSWS=ON -DLWS_WITHOUT_EXTENSIONS=0 -DLWS_WITH_HTTP2=1 -DLWS_WITH_ACME=1 -DLWS_WITH_MINIMAL_EXAMPLES=1 -DCMAKE_BUILD_TYPE=DEBUG" - LWS_METHOD=default CMAKE_ARGS="-DLWS_WITH_MINIMAL_EXAMPLES=1" - LWS_METHOD=mbedtls CMAKE_ARGS="-DLWS_WITH_MBEDTLS=1 -DLWS_WITH_HTTP2=1 -DLWS_WITH_LWSWS=1 -DLWS_WITH_MINIMAL_EXAMPLES=1 -DCMAKE_BUILD_TYPE=DEBUG" - LWS_METHOD=noserver CMAKE_ARGS="-DLWS_WITHOUT_SERVER=ON -DLWS_WITH_MINIMAL_EXAMPLES=1" diff --git a/scripts/autobahn-test.sh b/scripts/autobahn-test-client.sh similarity index 85% rename from scripts/autobahn-test.sh rename to scripts/autobahn-test-client.sh index f81368432f..e6db7dc321 100755 --- a/scripts/autobahn-test.sh +++ b/scripts/autobahn-test-client.sh @@ -142,49 +142,6 @@ else echo fi -# 2) lws-as-server tests - -echo -echo "----------------------------------------------" -echo "------- tests: autobahn as server" -echo - -$SERV -p 9001 -d7 & -wstest -m fuzzingclient -R=$? -echo "Autobahn client exit $R" - -killall lws-minimal-ws-server-echo -sleep 1s - -# repeat the client results - -R=`cat /tmp/ji | grep -v '"behavior": "OK"' | grep -v '"behavior": "NON-STRICT"' | grep -v '"behavior": "INFORMATIONAL"' | wc -l` -echo -n "AUTOBAHN SERVER / LWS CLIENT: Total tests: " `cat /tmp/ji | wc -l` " : " -if [ "$R" == "0" ] ;then - echo "All pass" -else - RESULT=1 - echo -n "$R FAIL : " - cat /tmp/ji | grep -v '"behavior": "OK"' | grep -v '"behavior": "NON-STRICT"' | grep -v '"behavior": "INFORMATIONAL"' | cut -d\" -f2 | tr '\n' ',' - echo -fi - -# and then the server results - -cat reports/servers/index.json | tr '\n' '!' | sed "s|\},\!|\n|g" | tr '!' ' ' | tr -s ' ' > /tmp/jis -R=`cat /tmp/jis | grep -v '"behavior": "OK"' | grep -v '"behavior": "NON-STRICT"' | grep -v '"behavior": "INFORMATIONAL"' | wc -l` - -echo -n "AUTOBAHN CLIENT / LWS SERVER: Total tests: " `cat /tmp/jis | wc -l` " : " -if [ "$R" == "0" ] ;then - echo "All pass" -else - RESULT=$(( $RESULT + 2 )) - echo -n "$R FAIL : " - cat /tmp/jis | grep -v '"behavior": "OK"' | grep -v '"behavior": "NON-STRICT"' | grep -v '"behavior": "INFORMATIONAL"' | cut -d\" -f2 | tr '\n' ',' - echo -fi - echo $RESULT exit $RESULT diff --git a/scripts/autobahn-test-server.sh b/scripts/autobahn-test-server.sh new file mode 100755 index 0000000000..d2afd68549 --- /dev/null +++ b/scripts/autobahn-test-server.sh @@ -0,0 +1,107 @@ +#!/bin/bash +# +# Requires pip install autobahntestsuite +# +# you should run this from ./build, after building with +# cmake .. -DLWS_WITH_MINIMAL_EXAMPLES=1 +# +# It will use the minimal echo client and server to run +# autobahn ws tests as both client and server. + +set -u + +PARALLEL=8 +N=1 +OS=`uname` + +CLIE=bin/lws-minimal-ws-client-echo +SERV=bin/lws-minimal-ws-server-echo + +RESULT=0 + +which wstest 2>/dev/null +if [ $? -ne 0 ]; then + echo "wstest is not installed" + exit 8 +fi + +killall wstest 2>/dev/null + +# +# 2.10 / 2.11: There is no requirement to handle multiple PING / PONG +# in flight in RFC6455. lws doesn't waste memory on it +# since it is useless. +# +# 12.3.1 / 12.3.2 +# 12.4.* / 12.5.*: Autobahn has been broken for these tests since Aug 2017 +# https://github.com/crossbario/autobahn-testsuite/issues/71 + + +cat << EOF >fuzzingserver.json +{ + "url": "ws://127.0.0.1:9001", + "outdir": "./reports/clients", + "cases": ["*"], + "exclude-cases": [ "2.10", "2.11", "12.3.1", "12.3.2", "12.4.*", "12.5.*"], + "exclude-agent-cases": {} +} +EOF + +cat << EOF >fuzzingclient.json +{ + "outdir": "./reports/servers", + "servers": [ + { + "url": "ws://127.0.0.1:9001" + } + ], + "cases": ["*"], + "exclude-cases": ["2.10", "2.11", "12.3.1", "12.3.2", "12.4.*", "12.5.*" ], + "exclude-agent-cases": {} +} +EOF + +echo +echo "----------------------------------------------" +echo "------- tests: autobahn as server" +echo + +$SERV -p 9001 -d7 & +wstest -m fuzzingclient +R=$? +echo "Autobahn client exit $R" + +killall lws-minimal-ws-server-echo +sleep 1s + +# repeat the client results + +R=`cat /tmp/ji | grep -v '"behavior": "OK"' | grep -v '"behavior": "NON-STRICT"' | grep -v '"behavior": "INFORMATIONAL"' | wc -l` +echo -n "AUTOBAHN SERVER / LWS CLIENT: Total tests: " `cat /tmp/ji | wc -l` " : " +if [ "$R" == "0" ] ;then + echo "All pass" +else + RESULT=1 + echo -n "$R FAIL : " + cat /tmp/ji | grep -v '"behavior": "OK"' | grep -v '"behavior": "NON-STRICT"' | grep -v '"behavior": "INFORMATIONAL"' | cut -d\" -f2 | tr '\n' ',' + echo +fi + +# and then the server results + +cat reports/servers/index.json | tr '\n' '!' | sed "s|\},\!|\n|g" | tr '!' ' ' | tr -s ' ' > /tmp/jis +R=`cat /tmp/jis | grep -v '"behavior": "OK"' | grep -v '"behavior": "NON-STRICT"' | grep -v '"behavior": "INFORMATIONAL"' | wc -l` + +echo -n "AUTOBAHN CLIENT / LWS SERVER: Total tests: " `cat /tmp/jis | wc -l` " : " +if [ "$R" == "0" ] ;then + echo "All pass" +else + RESULT=$(( $RESULT + 2 )) + echo -n "$R FAIL : " + cat /tmp/jis | grep -v '"behavior": "OK"' | grep -v '"behavior": "NON-STRICT"' | grep -v '"behavior": "INFORMATIONAL"' | cut -d\" -f2 | tr '\n' ',' + echo +fi + +echo $RESULT +exit $RESULT + diff --git a/scripts/travis_control.sh b/scripts/travis_control.sh index 6603953bea..92e086b490 100755 --- a/scripts/travis_control.sh +++ b/scripts/travis_control.sh @@ -18,25 +18,33 @@ else ../scripts/h2spec.sh && ../scripts/attack.sh && ../scripts/h2load.sh && - ../scripts/autobahn-test.sh + ../scripts/autobahn-test-client.sh else - if [ "$LWS_METHOD" = "smp" ] ; then + if [ "$LWS_METHOD" = "lwsws2" ] ; then cmake -DLWS_OPENSSL_LIBRARIES="/usr/local/lib/libssl.so;/usr/local/lib/libcrypto.so" \ -DLWS_OPENSSL_INCLUDE_DIRS="/usr/local/include/openssl" $CMAKE_ARGS .. && cmake --build . && - ../scripts/h2load-smp.sh + sudo make install && + ../scripts/autobahn-test-server.sh else - if [ "$LWS_METHOD" = "mbedtls" ] ; then - cmake $CMAKE_ARGS .. && + if [ "$LWS_METHOD" = "smp" ] ; then + cmake -DLWS_OPENSSL_LIBRARIES="/usr/local/lib/libssl.so;/usr/local/lib/libcrypto.so" \ + -DLWS_OPENSSL_INCLUDE_DIRS="/usr/local/include/openssl" $CMAKE_ARGS .. && cmake --build . && - sudo make install && - ../minimal-examples/selftests.sh && - ../scripts/h2spec.sh && - ../scripts/h2load.sh && - ../scripts/attack.sh + ../scripts/h2load-smp.sh else - cmake $CMAKE_ARGS .. && - cmake --build . + if [ "$LWS_METHOD" = "mbedtls" ] ; then + cmake $CMAKE_ARGS .. && + cmake --build . && + sudo make install && + ../minimal-examples/selftests.sh && + ../scripts/h2spec.sh && + ../scripts/h2load.sh && + ../scripts/attack.sh + else + cmake $CMAKE_ARGS .. && + cmake --build . + fi fi fi fi diff --git a/scripts/travis_install.sh b/scripts/travis_install.sh index c52bb444ad..4cdeccb7b7 100755 --- a/scripts/travis_install.sh +++ b/scripts/travis_install.sh @@ -6,7 +6,7 @@ if [ "$TRAVIS_OS_NAME" == "linux" ]; then sudo apt-get update -qq - if [ "$LWS_METHOD" == "lwsws" ]; + if [ "$LWS_METHOD" == "lwsws" -o "$LWS_METHOD" == "lwsws2" ]; then sudo apt-get install -y -qq realpath libjemalloc1 libev4 libuv-dev sudo apt-get remove python-six @@ -46,7 +46,7 @@ then sudo apt-get install -y -qq libev-dev; fi - if [ "$LWS_METHOD" == "libuv" -o "$LWS_METHOD" == "lwsws" ]; + if [ "$LWS_METHOD" == "libuv" -o "$LWS_METHOD" == "lwsws" -o "$LWS_METHOD" == "lwsws2" ]; then sudo apt-get install -y -qq libuv-dev; #libuv1 libuv1-dev; @@ -62,7 +62,7 @@ then brew install libev; fi - if [ "$LWS_METHOD" == "libuv" -o "$LWS_METHOD" == "lwsws" ]; + if [ "$LWS_METHOD" == "libuv" -o "$LWS_METHOD" == "lwsws" -o "$LWS_METHOD" == "lwsws2" ]; then brew update; brew install libuv;