7503 zfs-test should tail ::zfs_dbgmsg on test failure

Reviewed by: John Kennedy <john.kennedy@delphix.com>
Reviewed by: Matt Ahrens <mahrens@delphix.com>
Approved by: Gordon Ross <gordon.w.ross@gmail.com>

usr/src/pkg/manifests/system-test-zfstest.mf

@@ -23,6 +23,7 @@ set name=info.classification \
 set name=variant.arch value=$(ARCH)
 dir path=opt/zfs-tests
 dir path=opt/zfs-tests/bin
+dir path=opt/zfs-tests/callbacks
 dir path=opt/zfs-tests/include
 dir path=opt/zfs-tests/runfiles
 dir path=opt/zfs-tests/tests
@@ -166,6 +167,7 @@ file path=opt/zfs-tests/bin/readmmap mode=0555
 file path=opt/zfs-tests/bin/rename_dir mode=0555
 file path=opt/zfs-tests/bin/rm_lnkcnt_zero_file mode=0555
 file path=opt/zfs-tests/bin/zfstest mode=0555
+file path=opt/zfs-tests/callbacks/zfs_dbgmsg.ksh mode=0555
 file path=opt/zfs-tests/include/commands.cfg mode=0555
 file path=opt/zfs-tests/include/default.cfg mode=0555
 file path=opt/zfs-tests/include/libtest.shlib mode=0555

usr/src/test/test-runner/stf/contrib/include/logapi.shlib

@@ -23,7 +23,7 @@
 # Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
 # Use is subject to license terms.
 #
-# Copyright (c) 2012 by Delphix. All rights reserved.
+# Copyright (c) 2012, 2016 by Delphix. All rights reserved.
 #
 
 . ${STF_TOOLS}/include/stf.shlib
@@ -316,6 +316,22 @@ function log_other
 # Internal functions
 #
 
+# Execute custom callback scripts on test failure
+#
+# callback script paths are stored in TESTFAIL_CALLBACKS, delimited by ':'.
+
+function _execute_testfail_callbacks
+{
+	typeset callback
+
+	print "$TESTFAIL_CALLBACKS:" | while read -d ":" callback; do
+		if [[ -n "$callback" ]] ; then
+			log_note "Performing test-fail callback ($callback)"
+			$callback
+		fi
+	done
+}
+
 # Perform cleanup and exit
 #
 # $1 - stf exit code
@@ -326,6 +342,10 @@ function _endlog
 	typeset logfile="/tmp/log.$$"
 	_recursive_output $logfile
 
+	if [[ $1 == $STF_FAIL ]] ; then
+		_execute_testfail_callbacks
+	fi
+
 	if [[ -n $_CLEANUP ]] ; then
 		typeset cleanup=$_CLEANUP
 		log_onexit ""

usr/src/test/zfs-tests/Makefile

@@ -10,11 +10,11 @@
 #
 
 #
-# Copyright (c) 2012 by Delphix. All rights reserved.
+# Copyright (c) 2012, 2016 by Delphix. All rights reserved.
 #
 
 .PARALLEL: $(SUBDIRS)
 
-SUBDIRS = cmd include runfiles tests doc
+SUBDIRS = cmd include runfiles tests doc callbacks
 
 include $(SRC)/test/Makefile.com

usr/src/test/zfs-tests/callbacks/Makefile

@@ -0,0 +1,36 @@
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2016 by Delphix. All rights reserved.
+#
+
+include $(SRC)/Makefile.master
+
+CALLBACKS = zfs_dbgmsg.ksh
+
+ROOTOPTPKG = $(ROOT)/opt/zfs-tests
+CALLBACKSDIR = $(ROOTOPTPKG)/callbacks
+
+FILES = $(CALLBACKS:%=$(CALLBACKSDIR)/%)
+$(FILES) := FILEMODE = 0555
+
+all: $(CALLBACKS)
+
+install: $(CALLBACKSDIR) $(FILES)
+
+clean lint clobber:
+
+$(CALLBACKSDIR):
+	$(INS.dir)
+
+$(CALLBACKSDIR)/%: %
+	$(INS.file)

usr/src/test/zfs-tests/callbacks/zfs_dbgmsg.ksh

@@ -0,0 +1,29 @@
+#!/usr/bin/ksh -p
+
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2016 by Delphix. All rights reserved.
+#
+
+# $1: number of lines to output (default: 200)
+typeset lines=${1:-200}
+
+echo "================================================================="
+echo " Tailing last $lines lines of zfs_dbgmsg log"
+echo "================================================================="
+
+sudo mdb -ke "::zfs_dbgmsg ! tail -n $lines"
+
+echo "================================================================="
+echo " End of zfs_dbgmsg log"
+echo "================================================================="

usr/src/test/zfs-tests/cmd/scripts/zfstest.ksh

@@ -12,7 +12,7 @@
 #
 
 #
-# Copyright (c) 2012, 2015 by Delphix. All rights reserved.
+# Copyright (c) 2012, 2016 by Delphix. All rights reserved.
 # Copyright 2014, OmniTI Computer Consulting, Inc. All rights reserved.
 # Copyright 2016 Nexenta Systems, Inc.
 #
@@ -23,6 +23,10 @@ export STF_TOOLS="/opt/test-runner/stf"
 runner="/opt/test-runner/bin/run"
 auto_detect=false
 
+if [[ -z "$TESTFAIL_CALLBACKS" ]] ; then
+	export TESTFAIL_CALLBACKS="$STF_SUITE/callbacks/zfs_dbgmsg.ksh"
+fi
+
 function fail
 {
 	echo $1

usr/src/test/zfs-tests/include/default.cfg

@@ -41,23 +41,23 @@ export ZONE_POOL="zonepool"
 export ZONE_CTR="zonectr"
 
 # Test Suite Specific Commands
-export CHG_USR_EXEC="/opt/zfs-tests/bin/chg_usr_exec"
-export DEVNAME2DEVID="/opt/zfs-tests/bin/devname2devid"
-export DIR_RD_UPDATE="/opt/zfs-tests/bin/dir_rd_update"
-export FILE_CHECK="/opt/zfs-tests/bin/file_check"
-export FILE_TRUNC="/opt/zfs-tests/bin/file_trunc"
-export FILE_WRITE="/opt/zfs-tests/bin/file_write"
-export GETHOLES="/opt/zfs-tests/bin/getholes"
-export LARGEST_FILE="/opt/zfs-tests/bin/largest_file"
-export MKBUSY="/opt/zfs-tests/bin/mkbusy"
-export MKFILES="/opt/zfs-tests/bin/mkfiles"
-export MKHOLES="/opt/zfs-tests/bin/mkholes"
-export MKTREE="/opt/zfs-tests/bin/mktree"
-export MMAPWRITE="/opt/zfs-tests/bin/mmapwrite"
-export RANDFREE_FILE="/opt/zfs-tests/bin/randfree_file"
-export READMMAP="/opt/zfs-tests/bin/readmmap"
-export RENAME_DIR="/opt/zfs-tests/bin/rename_dir"
-export RM_LNKCNT_ZERO_FILE="/opt/zfs-tests/bin/rm_lnkcnt_zero_file"
+export CHG_USR_EXEC="$STF_SUITE/bin/chg_usr_exec"
+export DEVNAME2DEVID="$STF_SUITE/bin/devname2devid"
+export DIR_RD_UPDATE="$STF_SUITE/bin/dir_rd_update"
+export FILE_CHECK="$STF_SUITE/bin/file_check"
+export FILE_TRUNC="$STF_SUITE/bin/file_trunc"
+export FILE_WRITE="$STF_SUITE/bin/file_write"
+export GETHOLES="$STF_SUITE/bin/getholes"
+export LARGEST_FILE="$STF_SUITE/bin/largest_file"
+export MKBUSY="$STF_SUITE/bin/mkbusy"
+export MKFILES="$STF_SUITE/bin/mkfiles"
+export MKHOLES="$STF_SUITE/bin/mkholes"
+export MKTREE="$STF_SUITE/bin/mktree"
+export MMAPWRITE="$STF_SUITE/bin/mmapwrite"
+export RANDFREE_FILE="$STF_SUITE/bin/randfree_file"
+export READMMAP="$STF_SUITE/bin/readmmap"
+export RENAME_DIR="$STF_SUITE/bin/rename_dir"
+export RM_LNKCNT_ZERO_FILE="$STF_SUITE/bin/rm_lnkcnt_zero_file"
 
 # ensure we're running in the C locale, since
 # localised messages may result in test failures