Merge f10d0d9 into 566ab03

.gitignore

@@ -7,6 +7,7 @@
 /include/
 /templates/
 /configure/*.local
+/.tests-failed
 O.*/
 /QtC-*
 *.orig

appveyor.yml

@@ -77,7 +77,10 @@ build_script:
   - cmd: ci/appveyor-make.bat
 
 test_script:
-  - cmd: ci/appveyor-make.bat runtests
+  - cmd: ci/appveyor-make.bat tapfiles
+
+on_finish:
+  - cmd: ci/appveyor-make.bat -s test-results
 
 #---------------------------------#
 #         notifications           #

configure/CONFIG_BASE

@@ -68,8 +68,6 @@ DBTOMENUH                  = $(PERL) $(TOOLS)/dbdToMenuH.pl
 REGISTERRECORDDEVICEDRIVER = $(PERL) $(TOOLS)/registerRecordDeviceDriver.pl
 CONVERTRELEASE             = $(PERL) $(call FIND_TOOL,convertRelease.pl)
 FULLPATHNAME               = $(PERL) $(TOOLS)/fullPathName.pl
-TAPTOJUNIT                 = $(PERL) $(TOOLS)/tap-to-junit-xml.pl
-PROVE                      = $(PERL) $(TOOLS)/epicsProve.pl
 
 #-------------------------------------------------------
 # tools for installing libraries and products
@@ -83,6 +81,15 @@ INSTALL_LIBRARY = $(INSTALL)
 MKMF                       = $(PERL) $(TOOLS)/mkmf.pl
 REPLACEVAR                 = $(PERL) $(TOOLS)/replaceVAR.pl
 
+#---------------------------------------------------------------
+# Tools for testing
+TAPTOJUNIT                 = $(PERL) $(TOOLS)/tap-to-junit-xml.pl
+PROVE                      = $(PERL) $(TOOLS)/epicsProve.pl
+PROVE.tap                  = $(PROVE) --ext .tap --exec "$(CAT)"
+
+TEST_FAILURE_FILE          = $(TOP)/.tests-failed
+PROVE_FAILURE              = echo $(abspath .)>> $(TEST_FAILURE_FILE)
+
 #---------------------------------------------------------------
 # private versions of lex/yacc from EPICS
 EYACC = $(TOOLS)/antelope$(HOSTEXE)

configure/RULES_BUILD

@@ -105,17 +105,17 @@ PRODTARGETS += $(PRODNAME) $(MUNCHNAME) $(CTDT_SRCS) $(CTDT_OBJS) $(NMS)
 TESTPRODTARGETS += $(TESTPRODNAME) $(TESTMUNCHNAME)
 
 #---------------------------------------------------------------
-#   Test specifications and test result files
+#   Test result files
 #
-ifneq (,$(strip $(TESTS)))
-TARGETS += testspec
-endif
 
-# Enable testing if this host can run tests on the current target
-ifneq (,$(findstring $(T_A),$(EPICS_HOST_ARCH) $(CROSS_COMPILER_RUNTEST_ARCHS)))
+# Enable testing if this host can run tests for the current target
+ifneq (,$(filter $(T_A), $(EPICS_HOST_ARCH) $(CROSS_COMPILER_RUNTEST_ARCHS)))
 RUNTESTS_ENABLED = YES
-TAPFILES += $(TESTSCRIPTS:.t=.tap)
-JUNITFILES += $(TAPFILES:.tap=.xml)
+TESTSCRIPTS.t = $(filter %.t, $(TESTSCRIPTS))
+TAPFILES.t += $(TESTSCRIPTS.t:.t=.tap)
+JUNITFILES.t += $(TESTSCRIPTS.t:.t=.xml)
+TAPFILES += $(TAPFILES.t)
+JUNITFILES += $(JUNITFILES.t)
 endif
 
 #---------------------------------------------------------------
@@ -335,25 +335,22 @@ $(MODNAME): %$(MODEXT): %$(EXE)
 #---------------------------------------------------------------
 # Automated testing
 
-ifneq ($(strip $(TESTSCRIPTS)),)
-runtests:: $(TESTSCRIPTS)
+runtests: run-tap-tests
+run-tap-tests: $(TESTSCRIPTS.t)
+ifneq ($(TESTSCRIPTS.t),)
 ifdef RUNTESTS_ENABLED
-	-$(PERL) -MTest::Harness -e 'runtests @ARGV if @ARGV;' $^
+	$(PROVE) --failures --color $^ || $(PROVE_FAILURE)
 endif
 endif
 
-testspec: $(TESTSCRIPTS)
-	@$(RM) $@
-	@echo OS-class: $(OS_CLASS) > $@
-	@echo Target-arch: $(T_A) >> $@
-	$(if $^, @echo Tests: $^ >> $@)
-	$(if $(TESTFILES), @echo Files: $(TESTFILES) >> $@)
-	$(if $(TESTSPEC_$(OS_CLASS)), @echo "Harness: $(TESTSPEC_$(OS_CLASS))" >> $@)
+tapfiles: $(TAPFILES)
+junitfiles: $(JUNITFILES)
 
-test-results: tapfiles
+test-results: tap-results
+tap-results: $(TAPFILES)
 ifneq ($(strip $(TAPFILES)),)
 ifdef RUNTESTS_ENABLED
-	-$(PROVE) --failures --ext .tap --exec "$(CAT)" --color $(TAPFILES)
+	$(PROVE.tap) --failures --color $^ || $(PROVE_FAILURE)
 endif
 
 CURRENT_TAPFILES := $(wildcard $(TAPFILES))
@@ -368,16 +365,13 @@ ifneq ($(CURRENT_JUNITFILES),)
 	$(RM) $(CURRENT_JUNITFILES)
 endif
 
-tapfiles: $(TESTSCRIPTS) $(TAPFILES)
-junitfiles: $(JUNITFILES)
-
 # A .tap file is the output from running the associated test script
-%.tap: %.t
+$(TAPFILES.t): %.tap: %.t
 ifdef RUNTESTS_ENABLED
 	-$(PERL) $< -tap > $@
 endif
 
-%.xml: %.tap
+$(JUNITFILES.t): %.xml: %.tap
 	$(TAPTOJUNIT) --puretap --output $@ --input $< $*
 
 # If there's a perl test script (.plt) available, use it
@@ -524,8 +518,8 @@ $(INSTALL_TEMPLATES_SUBDIR)/%: %
 .PRECIOUS: $(COMMON_INC)
 
 .PHONY: all host inc build install clean rebuild buildInstall build_clean
-.PHONY: runtests tapfiles clean-tests test-results junitfiles
-.PHONY: checkRelease warnRelease noCheckRelease
+.PHONY: runtests run-tap-tests tapfiles junitfiles test-results tap-results
+.PHONY: clean-tests checkRelease warnRelease noCheckRelease
 
 endif # BASE_RULES_BUILD
 #	EOF RULES_BUILD

configure/RULES_DIRS

@@ -4,7 +4,7 @@
 # Copyright (c) 2002 The Regents of the University of California, as
 #     Operator of Los Alamos National Laboratory.
 # EPICS BASE is distributed subject to a Software License Agreement found
-# in the file LICENSE that is included with this distribution. 
+# in the file LICENSE that is included with this distribution.
 #*************************************************************************
 
 ARCHS += $(BUILD_ARCHS)
@@ -54,7 +54,7 @@ $(foreach dir, $(DIRS), \
 
 define DEP_template2
 $(1)$$(DIVIDER)$(2) : $$(foreach ddir, $$($(1)_DEPEND_DIRS), \
-    $$(addsuffix $$(DIVIDER)$(2),$$(ddir)))
+    $$(addsuffix $$(DIVIDER)$(2),$$(ddir))) | before-$(2)
 endef
 $(foreach action, $(ACTIONS), \
     $(foreach dir, $(DIRS), \
@@ -79,15 +79,22 @@ $(foreach arch, $(ARCHS), \
 
 dirPart = $(join $(dir $@), $(word 1, $(subst $(DIVIDER), ,$(notdir $@))))
 actionArchPart = $(join $(word 2, $(subst $(DIVIDER), ,$(notdir $@))), \
-   $(addprefix $(DIVIDER),$(word 3, $(subst $(DIVIDER), ,$(notdir $@))))) 
-$(DIRS) $(dirActionTargets) $(dirArchTargets) $(dirActionArchTargets) :
+   $(addprefix $(DIVIDER),$(word 3, $(subst $(DIVIDER), ,$(notdir $@)))))
+
+$(DIRS) $(dirActionTargets) $(dirArchTargets) $(dirActionArchTargets):
 	$(MAKE) -C $(dirPart) $(actionArchPart)
 
+# before-action rules are run once prior to recursing through the
+# list of subdirectories and running the action rule in each one.
+# See DEP_template2 above for how that rule ordering is achieved.
+beforeActions = $(addprefix before-,$(ACTIONS))
+$(beforeActions):
+
 $(ARCHS) $(ACTIONS) $(actionArchTargets) :%: \
     $(foreach dir, $(DIRS), $(dir)$(DIVIDER)%)
 
 .PHONY : $(DIRS) all host rebuild
-.PHONY : $(ARCHS) $(ACTIONS)
+.PHONY : $(ARCHS) $(ACTIONS) $(beforeActions)
 .PHONY : $(dirActionTargets) $(dirArchTargets)
 .PHONY : $(dirActionArchTargets)
 .PHONY : $(actionArchTargets)

configure/RULES_TOP

@@ -35,6 +35,11 @@ uninstall$(DIVIDER)%:
 	$(RMDIR) $(addsuffix /$(subst uninstall$(DIVIDER),,$@), \
 	    $(INSTALL_LOCATION_BIN) $(INSTALL_LOCATION_LIB))
 
+before-runtests before-test-results: rm-failure-file
+rm-failure-file:
+	@$(RM) $(TEST_FAILURE_FILE)
+runtests test-results:
+	$(PERL) $(TOOLS)/testFailures.pl $(TEST_FAILURE_FILE)
 
 help:
 	@echo "Usage: gnumake [options] [target] ..."
@@ -69,7 +74,7 @@ help:
 	@echo "     xxxRecord.o"
 
 .PHONY: distclean cvsclean realuninstall archuninstall uninstallDirs
-.PHONY: uninstall help
+.PHONY: uninstall rm-failure-file help
 
 # Include <top>/cfg/TOP_RULES* files from tops defined in RELEASE* files
 #

src/tools/Makefile

@@ -52,6 +52,7 @@ PERL_SCRIPTS += mkmf.pl
 PERL_SCRIPTS += munch.pl
 PERL_SCRIPTS += replaceVAR.pl
 PERL_SCRIPTS += tap-to-junit-xml.pl
+PERL_SCRIPTS += testFailures.pl
 PERL_SCRIPTS += useManifestTool.pl
 
 PERL_SCRIPTS += dbdToMenuH.pl

src/tools/testFailures.pl

@@ -0,0 +1,33 @@
+#!/usr/bin/env perl
+#*************************************************************************
+# EPICS BASE is distributed subject to a Software License Agreement found
+# in the file LICENSE that is included with this distribution.
+#*************************************************************************
+
+# This file may appear trivial, but it exists to let the build system
+# fail the 'make test-results' target with a nice output including a
+# summary of the directories where test failures were reported.
+# Test results are collected from the .tap files fed to epicsProve.pl
+# which returns with an exit status of 0 (success) if all tests passed
+# or 1 (failure) if any of the .tap files contained failed tests.
+# When epicsProve.pl indicates a failure, the directory that it was
+# running in is appended to the file $(TOP)/.tests-failed which this
+# program reads in after all the test directories have been visited.
+# The exit status of this program is 1 (failure) if any tests failed,
+# otherwise 0 (success).
+
+use strict;
+use warnings;
+
+die "Usage: testFailures.pl .tests-failed\n"
+    unless @ARGV == 1;
+
+open FAILURES, '<', shift or
+    exit 0;
+my @failures = <FAILURES>;
+close FAILURES;
+
+print "\nTest failures were reported in:\n",
+    (map {"    $_"} @failures), "\n";
+
+exit 1;