Added auto_online_001_pos test as apart of ZED/FMA tests in the ZTS

config/user-commands.m4

@@ -108,6 +108,9 @@ AC_DEFUN([ZFS_AC_CONFIG_USER_COMMANDS_LINUX], [
 	AC_PATH_TOOL(IOSTAT, iostat, "")
 	AC_PATH_TOOL(LOCKFS, lsof, "")
 	AC_PATH_TOOL(LSBLK, lsblk, "")
+	AC_PATH_TOOL(LSMOD, lsmod, "")
+	AC_PATH_TOOL(LSSCSI, lsscsi, "")
+	AC_PATH_TOOL(MODLOAD, modprobe, "")
 	AC_PATH_TOOL(MODUNLOAD, rmmod, "")
 	AC_PATH_TOOL(MPSTAT, mpstat, "")
 	AC_PATH_TOOL(NEWFS, mke2fs, "")

configure.ac

@@ -234,6 +234,7 @@ AC_CONFIG_FILES([
 	tests/zfs-tests/tests/functional/delegate/Makefile
 	tests/zfs-tests/tests/functional/devices/Makefile
 	tests/zfs-tests/tests/functional/exec/Makefile
+	tests/zfs-tests/tests/functional/fault/Makefile
 	tests/zfs-tests/tests/functional/features/async_destroy/Makefile
 	tests/zfs-tests/tests/functional/features/large_dnode/Makefile
 	tests/zfs-tests/tests/functional/features/Makefile

tests/runfiles/linux.run

@@ -398,6 +398,9 @@ tests = ['devices_003_pos']
 [tests/functional/exec]
 tests = ['exec_001_pos']
 
+[tests/functional/fault]
+tests = ['auto_online_001_pos']
+
 [tests/functional/features/async_destroy]
 tests = ['async_destroy_001_pos']
 

tests/zfs-tests/include/commands.cfg.in

@@ -56,12 +56,15 @@ export LOFIADM="@LOFIADM@"
 export LOGNAME="@LOGNAME@"
 export LS="@LS@"
 export LSBLK="@LSBLK@"
+export LSMOD="@LSMOD@"
+export LSSCSI="@LSSCSI@"
 export MD5SUM="@MD5SUM@"
 export MKDIR="@MKDIR@"
 export MKNOD="@MKNOD@"
 export MKTEMP="@MKTEMP@"
 export MNTTAB="@MNTTAB@"
 export MODINFO="@MODINFO@"
+export MODLOAD="@MODLOAD@"
 export MODUNLOAD="@MODUNLOAD@"
 export MOUNT="@MOUNT@"
 export MPSTAT="@MPSTAT@"

tests/zfs-tests/include/default.cfg.in

@@ -46,6 +46,7 @@ export RAIDZ_TEST=${RAIDZ_TEST:-${bindir}/raidz_test}
 export ARC_SUMMARY=${ARC_SUMMARY:-${bindir}/arc_summary.py}
 export ARCSTAT=${ARCSTAT:-${bindir}/arcstat.py}
 export DBUFSTAT=${DBUFSTAT:-${bindir}/dbufstat.py}
+export ZED=${ZED:-${sbindir}/zed}
 
 . $STF_SUITE/include/libtest.shlib
 

tests/zfs-tests/include/libtest.shlib

@@ -1574,22 +1574,133 @@ function reexport_pool
 }
 
 #
-# Verify a given disk is online or offline
+# Verify a given disk or pool state
 #
 # Return 0 is pool/disk matches expected state, 1 otherwise
 #
-function check_state # pool disk state{online,offline}
+function check_state # pool disk state{online,offline,degraded}
 {
 	typeset pool=$1
 	typeset disk=${2#$DEV_DSKDIR/}
 	typeset state=$3
 
-	$ZPOOL status -v $pool | grep "$disk"  \
-	    | grep -i "$state" > /dev/null 2>&1
+	[[ -z $pool ]] || [[ -z $state ]] \
+	    && log_fail "Arguments invalid or missing"
+
+	if [[ -z $disk ]]; then
+		#check pool state only
+		$ZPOOL get -H -o value health $pool \
+		    | grep -i "$state" > /dev/null 2>&1
+	else
+		$ZPOOL status -v $pool | grep "$disk"  \
+		    | grep -i "$state" > /dev/null 2>&1
+	fi
 
 	return $?
 }
 
+#
+# Online or offline a disk on the system
+#
+# First checks state of disk. Test will fail if disk is not properly onlined
+# or offlined. Online is a full rescan of SCSI disks with rescan-scsi-bus.sh
+# tool. Ubuntu and CentoOS distros currently supported - need to add supporting
+# package if distro is different.
+#
+function on_off_disk # disk state{online,offline}
+{
+	typeset disk=$1
+	typeset state=$2
+
+	[[ -z $disk ]] || [[ -z $state ]] &&  \
+	    log_fail "Arguments invalid or missing"
+
+	if is_linux; then
+		if [[ $state == "offline" ]] && ( is_mpath_device $disk ); then
+			dm_name="$($READLINK $DEV_DSKDIR/$disk \
+			    | $NAWK -F / '{print $2}')"
+			slave="$($LS /sys/block/${dm_name}/slaves \
+			    | $NAWK '{print $1}')"
+			while [[ -n $slave ]]; do
+				#check if disk is online
+				$LSSCSI | $EGREP $slave > /dev/null
+				if (($? == 0)); then
+					slave_dir="/sys/block/${dm_name}"
+					slave_dir+="/slaves/${slave}/device"
+					ss="${slave_dir}/state"
+					sd="${slave_dir}/delete"
+					log_must eval "$ECHO "offline" > ${ss}"
+					log_must eval "$ECHO "1" > ${sd}"
+					$LSSCSI | $EGREP $slave > /dev/null
+						if (($? == 0)); then
+							log_fail "Offlining" \
+							    "$disk failed"
+						fi
+				fi
+				slave="$($LS /sys/block/$dm_name/slaves \
+				    2>/dev/null | $NAWK '{print $1}')"
+			done
+		elif [[ $state == "offline" ]] && ( is_real_device $disk ); then
+			#check if disk is online
+			$LSSCSI | $EGREP $disk > /dev/null
+			if (($? == 0)); then
+				dev_state="/sys/block/$disk/device/state"
+				dev_delete="/sys/block/$disk/device/delete"
+				log_must eval "$ECHO "offline" > ${dev_state}"
+				log_must eval "$ECHO "1" > ${dev_delete}"
+				$LSSCSI | $EGREP $disk > /dev/null
+					if (($? == 0)); then
+						log_fail "Offlining $disk" \
+						    "failed"
+					fi
+			else
+				log_note "$disk is already offline"
+			fi
+		elif [[ $state == "online" ]]; then
+			#install rescan-scsi-bus tool for distro
+			$CAT /etc/*release | $EGREP Ubuntu > /dev/null
+			if (($? == 0)); then
+				(dpkg-query --list scsitools 2>&1) > /dev/null
+				if (($? != 0)); then
+					log_must eval "apt-get install -y" \
+					    "scsitools > /dev/null"
+				fi
+			fi
+			$CAT /etc/*release | $EGREP CentOS > /dev/null
+			if (($? == 0)); then
+				(rpm -qa | $EGREP sg3_utils 2>&1) > /dev/null
+				if (($? != 0)); then
+					log_must eval "yum install -y" \
+					    "sg3_utils > /dev/null"
+				fi
+			fi
+			#force a full rescan
+			log_must eval "/usr/bin/rescan-scsi-bus.sh -a" \
+			    "> /dev/null"
+			$SLEEP 5
+			if is_mpath_device $disk; then
+				dm_name="$($READLINK $DEV_DSKDIR/$disk \
+				    | $NAWK -F / '{print $2}')"
+				slave="$($LS /sys/block/$dm_name/slaves \
+				    | $NAWK '{print $1}')"
+				$LSSCSI | $EGREP $slave > /dev/null
+				if (($? != 0)); then
+					log_fail "Onlining $disk failed"
+				fi
+			elif is_real_device $disk; then
+				$LSSCSI | $EGREP $disk > /dev/null
+				if (($? != 0)); then
+					log_fail "Onlining $disk failed"
+				fi
+			else
+				log_fail "$disk is not a real dev"
+			fi
+		else
+			log_fail "$disk failed to $state"
+		fi
+	fi
+}
+
 #
 # Get the mountpoint of snapshot
 # For the snapshot use <mp_filesystem>/.zfs/snapshot/<snap>
@@ -2698,7 +2809,8 @@ function is_real_device #disk
 	[[ -z $disk ]] && log_fail "No argument for disk given."
 
        if is_linux; then
-		$LSBLK $DEV_RDSKDIR/$disk -o TYPE | $EGREP disk > /dev/null 2>&1
+		($LSBLK $DEV_RDSKDIR/$disk -o TYPE | $EGREP disk > /dev/null) \
+		    2>/dev/null
 		return $?
        fi
 }
@@ -2712,7 +2824,8 @@ function is_loop_device #disk
 	[[ -z $disk ]] && log_fail "No argument for disk given."
 
        if is_linux; then
-		$LSBLK $DEV_RDSKDIR/$disk -o TYPE | $EGREP loop > /dev/null 2>&1
+		($LSBLK $DEV_RDSKDIR/$disk -o TYPE | $EGREP loop > /dev/null) \
+		    2>/dev/null
 		return $?
        fi
 }
@@ -2728,7 +2841,8 @@ function is_mpath_device #disk
 	[[ -z $disk ]] && log_fail "No argument for disk given."
 
 	if is_linux; then
-		$LSBLK $DEV_MPATHDIR/$disk -o TYPE | $EGREP mpath > /dev/null 2>&1
+		($LSBLK $DEV_MPATHDIR/$disk -o TYPE | $EGREP mpath >/dev/null) \
+		     2>/dev/null
 		if (($? == 0)); then
 			$READLINK $DEV_MPATHDIR/$disk > /dev/null 2>&1
 			return $?
@@ -2751,11 +2865,13 @@ function set_slice_prefix
 	if is_linux; then
 		while (( i < $DISK_ARRAY_NUM )); do
 			disk="$($ECHO $DISKS | $NAWK '{print $(i + 1)}')"
-			if ( is_mpath_device $disk ) && [[ -z $($ECHO $disk | awk 'substr($1,18,1)\
-			     ~ /^[[:digit:]]+$/') ]] || ( is_real_device $disk ); then
+			if ( is_mpath_device $disk ) && [[ -z $($ECHO $disk \
+			    | awk 'substr($1,18,1) ~ /^[[:digit:]]+$/') ]] || \
+			    ( is_real_device $disk ); then
 				export SLICE_PREFIX=""
 				return 0
-			elif ( is_mpath_device $disk || is_loop_device $disk ); then
+			elif ( is_mpath_device $disk || is_loop_device \
+			    $disk ); then
 				export SLICE_PREFIX="p"
 				return 0
 			else
@@ -2816,6 +2932,34 @@ function get_device_dir #device
 	fi
 }
 
+#
+# Get persistent name for given disk
+#
+function get_persistent_disk_name #device
+{
+	typeset device=$1
+	typeset dev_id
+
+	if is_linux; then
+		if is_real_device $device; then
+			dev_id="$($UDEVADM info -q all -n $DEV_DSKDIR/$device \
+			    | $EGREP disk/by-id | $NAWK '{print $2; exit}' \
+			    | $NAWK -F / '{print $3}')"
+			$ECHO $dev_id
+		elif is_mpath_device $device; then
+			dev_id="$($UDEVADM info -q all -n $DEV_DSKDIR/$device \
+			    | $EGREP disk/by-id/dm-uuid \
+			    | $NAWK '{print $2; exit}' \
+			    | $NAWK -F / '{print $3}')"
+			$ECHO $dev_id
+		else
+			$ECHO $device
+		fi
+	else
+		$ECHO $device
+	fi
+}
+
 #
 # Get the package name
 #

tests/zfs-tests/tests/functional/Makefile.am

@@ -14,6 +14,7 @@ SUBDIRS = \
 	delegate \
 	devices \
 	exec \
+	fault \
 	features \
 	grow_pool \
 	grow_replicas \

tests/zfs-tests/tests/functional/fault/Makefile.am

@@ -0,0 +1,6 @@
+pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/fault
+dist_pkgdata_SCRIPTS = \
+	fault.cfg \
+	setup.ksh \
+	cleanup.ksh \
+	auto_online_001_pos.ksh