travis: handle reduced max run time by splitting autobahn tests across two instances

.travis.yml

@@ -5,6 +5,7 @@ env:
     - secure: "KhAdQ9ja+LBObWNQTYO7Df5J4DyOih6S+eerDMu8UPSO+CoWV2pWoQzbOfocjyOscGOwC+2PrrHDNZyGfqkCLDXg1BxynXPCFerHC1yc2IajvKpGXmAAygNIvp4KACDfGv/dkXrViqIzr/CdcNaU4vIMHSVb5xkeLi0W1dPnQOI="
   matrix:
     - LWS_METHOD=lwsws CMAKE_ARGS="-DLWS_WITH_LWSWS=ON -DLWS_WITHOUT_EXTENSIONS=0 -DLWS_WITH_HTTP2=1 -DLWS_WITH_ACME=1 -DLWS_WITH_MINIMAL_EXAMPLES=1 -DCMAKE_BUILD_TYPE=DEBUG"
+    - LWS_METHOD=lwsws2 CMAKE_ARGS="-DLWS_WITH_LWSWS=ON -DLWS_WITHOUT_EXTENSIONS=0 -DLWS_WITH_HTTP2=1 -DLWS_WITH_ACME=1 -DLWS_WITH_MINIMAL_EXAMPLES=1 -DCMAKE_BUILD_TYPE=DEBUG"
     - LWS_METHOD=default CMAKE_ARGS="-DLWS_WITH_MINIMAL_EXAMPLES=1"
     - LWS_METHOD=mbedtls CMAKE_ARGS="-DLWS_WITH_MBEDTLS=1 -DLWS_WITH_HTTP2=1 -DLWS_WITH_LWSWS=1 -DLWS_WITH_MINIMAL_EXAMPLES=1 -DCMAKE_BUILD_TYPE=DEBUG"
     - LWS_METHOD=noserver CMAKE_ARGS="-DLWS_WITHOUT_SERVER=ON -DLWS_WITH_MINIMAL_EXAMPLES=1"

scripts/autobahn-test.sh → scripts/autobahn-test-client.sh

@@ -142,49 +142,6 @@ else
 	echo
 fi
 
-# 2) lws-as-server tests
-
-echo
-echo "----------------------------------------------"
-echo "-------   tests: autobahn as server"
-echo
-
-$SERV -p 9001 -d7 &
-wstest -m fuzzingclient
-R=$?
-echo "Autobahn client exit $R"
-
-killall lws-minimal-ws-server-echo
-sleep 1s
-
-# repeat the client results
-
-R=`cat /tmp/ji | grep -v '"behavior": "OK"' | grep -v '"behavior": "NON-STRICT"' | grep -v '"behavior": "INFORMATIONAL"' | wc -l`
-echo -n "AUTOBAHN SERVER / LWS CLIENT: Total tests: " `cat /tmp/ji | wc -l` " : "
-if [ "$R" == "0" ] ;then
-	echo "All pass"
-else
-	RESULT=1
-	echo -n "$R FAIL : "
-	cat /tmp/ji | grep -v '"behavior": "OK"' | grep -v '"behavior": "NON-STRICT"' | grep -v '"behavior": "INFORMATIONAL"' | cut -d\" -f2 | tr '\n' ','
-	echo
-fi
-
-# and then the server results
-
-cat reports/servers/index.json | tr '\n' '!' | sed "s|\},\!|\n|g" | tr '!' ' ' | tr -s ' ' > /tmp/jis
-R=`cat /tmp/jis | grep -v '"behavior": "OK"' | grep -v '"behavior": "NON-STRICT"' | grep -v '"behavior": "INFORMATIONAL"' | wc -l`
-
-echo -n "AUTOBAHN CLIENT / LWS SERVER: Total tests: " `cat /tmp/jis | wc -l` " : "
-if [ "$R" == "0" ] ;then
-	echo "All pass"
-else
-	RESULT=$(( $RESULT + 2 ))
-	echo -n "$R FAIL : "
-	cat /tmp/jis | grep -v '"behavior": "OK"' | grep -v '"behavior": "NON-STRICT"' | grep -v '"behavior": "INFORMATIONAL"' | cut -d\" -f2 | tr '\n' ','
-	echo
-fi
-
 echo $RESULT
 exit $RESULT
 

scripts/autobahn-test-server.sh

+#!/bin/bash
+#
+# Requires pip install autobahntestsuite
+#
+# you should run this from ./build, after building with
+# cmake .. -DLWS_WITH_MINIMAL_EXAMPLES=1
+#
+# It will use the minimal echo client and server to run
+# autobahn ws tests as both client and server.
+
+set -u
+
+PARALLEL=8
+N=1
+OS=`uname`
+
+CLIE=bin/lws-minimal-ws-client-echo
+SERV=bin/lws-minimal-ws-server-echo
+
+RESULT=0
+
+which wstest 2>/dev/null
+if [ $? -ne 0 ]; then
+	echo "wstest is not installed"
+	exit 8
+fi
+
+killall wstest 2>/dev/null
+
+#
+# 2.10 / 2.11:      There is no requirement to handle multiple PING / PONG
+#                   in flight in RFC6455.  lws doesn't waste memory on it
+#                   since it is useless.
+#
+# 12.3.1 / 12.3.2
+# 12.4.* / 12.5.*:  Autobahn has been broken for these tests since Aug 2017
+#                   https://github.com/crossbario/autobahn-testsuite/issues/71
+
+
+cat << EOF >fuzzingserver.json
+{
+   "url": "ws://127.0.0.1:9001",
+   "outdir": "./reports/clients",
+   "cases": ["*"],
+   "exclude-cases": [ "2.10", "2.11", "12.3.1", "12.3.2", "12.4.*", "12.5.*"],
+   "exclude-agent-cases": {}
+}
+EOF
+
+cat << EOF >fuzzingclient.json
+{ 
+   "outdir": "./reports/servers",
+   "servers": [
+      {
+         "url": "ws://127.0.0.1:9001"
+      }
+   ],
+   "cases": ["*"],
+   "exclude-cases": ["2.10", "2.11", "12.3.1", "12.3.2", "12.4.*", "12.5.*" ],
+   "exclude-agent-cases": {}
+}
+EOF
+
+echo
+echo "----------------------------------------------"
+echo "-------   tests: autobahn as server"
+echo
+
+$SERV -p 9001 -d7 &
+wstest -m fuzzingclient
+R=$?
+echo "Autobahn client exit $R"
+
+killall lws-minimal-ws-server-echo
+sleep 1s
+
+# repeat the client results
+
+R=`cat /tmp/ji | grep -v '"behavior": "OK"' | grep -v '"behavior": "NON-STRICT"' | grep -v '"behavior": "INFORMATIONAL"' | wc -l`
+echo -n "AUTOBAHN SERVER / LWS CLIENT: Total tests: " `cat /tmp/ji | wc -l` " : "
+if [ "$R" == "0" ] ;then
+	echo "All pass"
+else
+	RESULT=1
+	echo -n "$R FAIL : "
+	cat /tmp/ji | grep -v '"behavior": "OK"' | grep -v '"behavior": "NON-STRICT"' | grep -v '"behavior": "INFORMATIONAL"' | cut -d\" -f2 | tr '\n' ','
+	echo
+fi
+
+# and then the server results
+
+cat reports/servers/index.json | tr '\n' '!' | sed "s|\},\!|\n|g" | tr '!' ' ' | tr -s ' ' > /tmp/jis
+R=`cat /tmp/jis | grep -v '"behavior": "OK"' | grep -v '"behavior": "NON-STRICT"' | grep -v '"behavior": "INFORMATIONAL"' | wc -l`
+
+echo -n "AUTOBAHN CLIENT / LWS SERVER: Total tests: " `cat /tmp/jis | wc -l` " : "
+if [ "$R" == "0" ] ;then
+	echo "All pass"
+else
+	RESULT=$(( $RESULT + 2 ))
+	echo -n "$R FAIL : "
+	cat /tmp/jis | grep -v '"behavior": "OK"' | grep -v '"behavior": "NON-STRICT"' | grep -v '"behavior": "INFORMATIONAL"' | cut -d\" -f2 | tr '\n' ','
+	echo
+fi
+
+echo $RESULT
+exit $RESULT
+

scripts/travis_control.sh

@@ -18,25 +18,33 @@ else
 			../scripts/h2spec.sh &&
 			../scripts/attack.sh &&
 			../scripts/h2load.sh &&
-			../scripts/autobahn-test.sh
+			../scripts/autobahn-test-client.sh
 		else
-			if [ "$LWS_METHOD" = "smp" ] ; then
+			if [ "$LWS_METHOD" = "lwsws2" ] ; then
 				cmake -DLWS_OPENSSL_LIBRARIES="/usr/local/lib/libssl.so;/usr/local/lib/libcrypto.so" \
 				      -DLWS_OPENSSL_INCLUDE_DIRS="/usr/local/include/openssl" $CMAKE_ARGS .. &&
 				cmake --build . &&
-				../scripts/h2load-smp.sh
+				sudo make install &&
+				../scripts/autobahn-test-server.sh
 			else
-				if [ "$LWS_METHOD" = "mbedtls" ] ; then
-					cmake $CMAKE_ARGS .. &&
+				if [ "$LWS_METHOD" = "smp" ] ; then
+					cmake -DLWS_OPENSSL_LIBRARIES="/usr/local/lib/libssl.so;/usr/local/lib/libcrypto.so" \
+					      -DLWS_OPENSSL_INCLUDE_DIRS="/usr/local/include/openssl" $CMAKE_ARGS .. &&
 					cmake --build . &&
-					sudo make install &&
-					../minimal-examples/selftests.sh &&
-					../scripts/h2spec.sh &&
-					../scripts/h2load.sh &&
-					../scripts/attack.sh
+					../scripts/h2load-smp.sh
 				else
-					cmake $CMAKE_ARGS .. &&
-					cmake --build .
+					if [ "$LWS_METHOD" = "mbedtls" ] ; then
+						cmake $CMAKE_ARGS .. &&
+						cmake --build . &&
+						sudo make install &&
+						../minimal-examples/selftests.sh &&
+						../scripts/h2spec.sh &&
+						../scripts/h2load.sh &&
+						../scripts/attack.sh
+					else
+						cmake $CMAKE_ARGS .. &&
+						cmake --build .
+					fi
 				fi
 			fi
 		fi

scripts/travis_install.sh

@@ -6,7 +6,7 @@ if [ "$TRAVIS_OS_NAME" == "linux" ];
 then
 	sudo apt-get update -qq
 
-	if [ "$LWS_METHOD" == "lwsws" ];
+	if [ "$LWS_METHOD" == "lwsws" -o "$LWS_METHOD" == "lwsws2" ];
 	then
 		sudo apt-get install -y -qq realpath libjemalloc1 libev4 libuv-dev
 		sudo apt-get remove python-six
@@ -46,7 +46,7 @@ then
 		sudo apt-get install -y -qq libev-dev;
 	fi
 
-	if [ "$LWS_METHOD" == "libuv" -o "$LWS_METHOD" == "lwsws" ];
+	if [ "$LWS_METHOD" == "libuv" -o "$LWS_METHOD" == "lwsws" -o "$LWS_METHOD" == "lwsws2" ];
 	then
 		sudo apt-get install -y -qq libuv-dev;
 #libuv1 libuv1-dev;
@@ -62,7 +62,7 @@ then
 		brew install libev;
 	fi
 
-	if [ "$LWS_METHOD" == "libuv" -o "$LWS_METHOD" == "lwsws" ];
+	if [ "$LWS_METHOD" == "libuv" -o "$LWS_METHOD" == "lwsws" -o "$LWS_METHOD" == "lwsws2" ];
 	then
 		brew update;
 		brew install libuv;