The Thingsquare Mist regression test scripts

regression-tests/Makefile

@@ -0,0 +1,49 @@
+# Copyright (c) 2012, Thingsquare, www.thingsquare.com.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+# 3. Neither the name of the Institute nor the names of its contributors
+#    may be used to endorse or promote products derived from this software
+#    without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE INSTITUTE AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE INSTITUTE OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+
+TESTS=$(wildcard ??-*)
+SUMMARIES=$(foreach test,$(TESTS),summary-$(test))
+
+CONTIKI=..
+
+run: clean summary
+
+summary: $(SUMMARIES)
+	grep '' $(SUMMARIES) > summary
+
+summary-%:
+	@make -C $* RUNALL=true summary || true
+	@echo -n $* | cat - $*/summary > $@
+	@rm $*/summary
+
+clean:
+	rm -f $(SUMMARIES)
+
+cooja: $(CONTIKI)/tools/cooja/dist/cooja.jar
+$(CONTIKI)/tools/cooja/dist/cooja.jar:
+	(cd $(CONTIKI)/tools/cooja; ant jar)
+

regression-tests/Makefile.compile-test

@@ -0,0 +1,95 @@
+# Copyright (c) 2012, Thingsquare, www.thingsquare.com.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+# 3. Neither the name of the Institute nor the names of its contributors
+#    may be used to endorse or promote products derived from this software
+#    without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE INSTITUTE AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE INSTITUTE OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+
+all: summary
+
+build: examples tools
+
+dirnam = $(word 1, $(subst /, ,${1}))
+target = $(word 2, $(subst /, ,${1}))
+
+# The stuff below is some GNU make magic to automatically make make
+# give each compile test a number, prefixed with a 0 if the number is
+# < 10, to match the way the simulation tests output works.
+nine := x x x x x x x x x
+max = $(subst xx,x,$(join ${1},${2}))
+gt = $(filter-out $(words ${1}),$(words $(call max,${1},${2})))
+addzero = $(if $(call gt,${nine},$(1)),$(words ${1}),0$(words ${1}))
+
+define dooneexample
+@echo Buildling example $(3): $(1) for target $(2)
+@((cd $(EXAMPLESDIR)/$(1); \
+ make TARGET=$(2) clean && make TARGET=$(2)) > \
+      $(3)-$(1)-$(2).report 2>&1 && \
+ (echo $(1) $(2): OK | tee $(3)-$(1)-$(2).summary) || \
+ (echo $(1) $(2): FAIL ಠ.ಠ | tee $(3)-$(1)-$(2).summary ; \
+  tail -10 $(3)-$(1)-$(2).report > $(3)-$(1)-$(2).faillog))
+endef
+
+define doexample
+$(eval i+=x)
+$(call dooneexample,$(call dirnam,${1}),$(call target,${1}),$(call addzero,${i}))
+endef
+#end of GNU make magic
+
+
+examples:
+	$(foreach ex, $(EXAMPLES), $(call doexample, ${ex}))
+
+report: build
+	@echo Examples | cat - ??-*.report > report
+	@echo Tools | cat - $(foreach tool, $(TOOLS), $(tool).report) >> report
+
+examples-summary: build
+	@echo Example summary | cat - $(foreach example, $(EXAMPLES), \
+             $(foreach target, $(EXAMPLESTARGETS), $(example)-$(target).summary)) > \
+               $@
+	@echo Tools summary | cat - $(foreach tool, $(TOOLS), $(tool).summary) >> $@
+
+faillog-summary: build
+	@rm -f $@; touch $@
+	@$(foreach log, $(wildcard *.faillog), grep -H '' $(log) >> $@;)
+
+summary: examples-summary faillog-summary
+	@cat *.summary > $@
+	@ls -1 ??-*.faillog > /dev/null 2>&1; [ $$? = 0 ] && tail -v ??-*.faillog >> $@ || true
+	@rm -f $^
+
+tools:
+	@$(foreach tool, $(TOOLS), \
+           (((cd $(TOOLSDIR)/$(tool); make) > $(tool).report 2>&1) && \
+             (echo $(tool): OK | tee $(tool).summary) || \
+             (echo $(tool): FAIL ಠ_ಠ | tee $(tool).summary ; \
+                tail -10 $(tool).report > $(tool).faillog)) ; )
+
+clean:
+	@rm -f *.summary *.report *.faillog summary report
+	@$(foreach example, $(EXAMPLES), \
+           $(foreach target, $(EXAMPLESTARGETS), \
+             (cd $(EXAMPLESDIR)/$(example); make TARGET=$(target) clean);))
+	@$(foreach tool, $(TOOLS), \
+           (cd $(TOOLSDIR)/$(tool); make clean);)

regression-tests/Makefile.simulation-test

@@ -0,0 +1,76 @@
+# Copyright (c) 2012, Thingsquare, www.thingsquare.com.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+# 3. Neither the name of the Institute nor the names of its contributors
+#    may be used to endorse or promote products derived from this software
+#    without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE INSTITUTE AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE INSTITUTE OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+
+
+TESTS=$(wildcard ??-*.csc)
+TESTLOGS=$(patsubst %.csc,%.testlog,$(TESTS))
+LOGS=$(patsubst %.csc,%.log,$(TESTS))
+FAILLOGS=$(patsubst %.csc,%.faillog,$(TESTS))
+
+CONTIKI=../..
+
+tests: $(TESTLOGS)
+
+report: clean tests
+	@echo | grep -s -e '' - $(LOGS) $(TESTLOGS) $(FAILLOGS) > $@ || true
+
+summary: report
+ifeq ($(TESTS),)
+	@echo No tests > $@
+else
+	@egrep -e ' OK| FAIL' $< > $@
+	@ls -1 ??-*.faillog > /dev/null 2>&1; [ $$? = 0 ] && tail -v ??-*.log ??-*.faillog >> $@ || true
+endif
+
+all: cooja clean tests
+
+ifdef RUNALL
+RUNALL=true
+else
+RUNALL=false
+endif
+
+%.testlog: %.csc cooja
+	@echo -n Running test $(basename $<) ... ""
+	@(java -jar $(CONTIKI)/tools/cooja/dist/cooja.jar \
+              -nogui=$< -contiki=$(CONTIKI) > $(basename $@).log || \
+         (echo " FAIL ಠ_ಠ" | tee -a COOJA.testlog; \
+          mv COOJA.testlog $(basename $<).faillog; \
+          $(RUNALL))) && \
+         (touch COOJA.testlog; \
+	  mv COOJA.testlog $@; \
+	  echo " OK")
+
+clean:
+	@rm -f $(TESTLOGS) $(LOGS) $(FAILLOGS) COOJA.log COOJA.testlog \
+               report summary
+
+
+cooja: $(CONTIKI)/tools/cooja/dist/cooja.jar
+$(CONTIKI)/tools/cooja/dist/cooja.jar:
+	(cd $(CONTIKI)/tools/cooja; ant jar)
+