OS-62 slow io error detector is needed.

Author: gila

usr/src/cmd/fm/modules/common/Makefile

@@ -21,6 +21,8 @@
 #
 # Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
 #
+# Copyright 2013 Nexenta Systems, Inc.  All rights reserved.
+#
 
 SUBDIRS = cpumem-retire \
 	disk-lights	\
@@ -34,6 +36,7 @@ SUBDIRS = cpumem-retire \
 	ip-transport	\
 	sensor-transport \
 	ses-log-transport \
+	slow-io-de \
 	sw-diag-response \
 	sp-monitor	\
 	syslog-msgs	\

usr/src/cmd/fm/modules/common/slow-io-de/Makefile

@@ -0,0 +1,23 @@
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+
+#
+# Copyright 2013 Nexenta Systems, Inc.  All rights reserved.
+#
+
+MODULE = slow-io-de
+CLASS = common
+SRCS = slow-io-de.c
+
+include ../../Makefile.plugin
+
+LINTFLAGS += -L$(ROOT)/usr/lib/fm
+#LDLIBS += -ltopo
+LDFLAGS += -L$(ROOT)/usr/lib/fm -R/usr/lib/fm

usr/src/cmd/fm/modules/common/slow-io-de/slow-io-de.c

@@ -0,0 +1,108 @@
+/*
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ */
+
+/*
+ * Copyright 2013 Nexenta Systems, Inc.  All rights reserved.
+ */
+
+#include <fm/fmd_api.h>
+
+typedef struct slow_io_stat {
+	fmd_stat_t bad_fmri;
+	fmd_stat_t bad_scheme;
+} slow_io_stat_t;
+
+slow_io_stat_t slow_io_stats  = {
+	{ "bad_FMRI", FMD_TYPE_UINT64,
+		"event FMRI is missing or invalid" },
+	{ "bad_scheme", FMD_TYPE_UINT64,
+		"event does not contain a valid detector"},
+};
+
+static const fmd_prop_t fmd_props [] = {
+	{ "io_N", FMD_TYPE_INT32, "10" },
+	{ "io_T", FMD_TYPE_TIME, "10min"},
+	{ NULL, 0, NULL }
+};
+
+void
+slow_io_close(fmd_hdl_t *hdl, fmd_case_t *c)
+{
+	char *devid = fmd_case_getspecific(hdl, c);
+	fmd_hdl_debug(hdl, "Destroying serd: %s", devid);
+	fmd_serd_destroy(hdl, devid);
+}
+
+void
+slow_io_recv(fmd_hdl_t *hdl, fmd_event_t *event, nvlist_t *nvl,
+	const char *class)
+{
+	nvlist_t *detector = NULL;
+	char *devid = NULL;
+
+	if (nvlist_lookup_nvlist(nvl, "detector", &detector) != 0) {
+		slow_io_stats.bad_scheme.fmds_value.ui64++;
+		return;
+	}
+
+	if (nvlist_lookup_string(detector, "devid", &devid) != 0) {
+		slow_io_stats.bad_fmri.fmds_value.ui64++;
+		return;
+	}
+
+	if (fmd_serd_exists(hdl, devid) == 0) {
+		fmd_serd_create(hdl, devid, fmd_prop_get_int32(hdl, "io_N"),
+		    fmd_prop_get_int64(hdl, "io_T"));
+		(void) fmd_serd_record(hdl, devid, event);
+		return;
+	}
+
+	if (fmd_serd_record(hdl, devid, event) == FMD_B_TRUE) {
+		fmd_case_t *c = fmd_case_open(hdl, NULL);
+		nvlist_t *fault = fmd_nvl_create_fault(hdl,
+		    "fault.io.disk.predictive-failure", 100,
+		    detector, NULL, detector);
+		fmd_case_add_serd(hdl, c, devid);
+		fmd_case_add_suspect(hdl, c, fault);
+		fmd_case_setspecific(hdl, c, devid);
+		fmd_case_solve(hdl, c);
+	}
+}
+
+static const fmd_hdl_ops_t fmd_ops = {
+	slow_io_recv,
+	NULL,
+	slow_io_close,
+	NULL,
+	NULL,
+};
+
+static const fmd_hdl_info_t fmd_info = {
+	"slow-io-de", "0.1", &fmd_ops, fmd_props
+};
+
+void
+_fmd_init(fmd_hdl_t *hdl)
+{
+	if (fmd_hdl_register(hdl, FMD_API_VERSION, &fmd_info) != 0) {
+		fmd_hdl_debug(hdl, "Internal error\n");
+		return;
+	}
+
+	fmd_stat_create(hdl, FMD_STAT_NOALLOC, sizeof (slow_io_stats) /
+	    sizeof (fmd_stat_t), (fmd_stat_t *)&slow_io_stats);
+}
+
+void
+_fmd_fini(fmd_hdl_t *hdl)
+{
+
+}

usr/src/cmd/fm/modules/common/slow-io-de/slow-io-de.conf

@@ -0,0 +1,19 @@
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+
+#
+# Copyright 2013 Nexenta Systems, Inc.  All rights reserved.
+#
+
+#
+# fmd configuration file for the slow-io diagnosis
+#
+
+subscribe ereport.io.disk.slow-io

usr/src/pkg/manifests/service-fault-management.mf

@@ -10,6 +10,7 @@
 # See the License for the specific language governing permissions
 # and limitations under the License.
 #
+#
 # When distributing Covered Code, include this CDDL HEADER in each
 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 # If applicable, add the following below this CDDL HEADER, with the
@@ -22,7 +23,7 @@
 #
 # Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
 #
-
+# Copyright 2013 Nexenta Systems, Inc.  All rights reserved.
 #
 # The default for payload-bearing actions in this package is to appear in the
 # global zone only.  See the include file for greater detail, as well as
@@ -450,6 +451,8 @@ file path=usr/lib/fm/fmd/plugins/sensor-transport.conf
 file path=usr/lib/fm/fmd/plugins/sensor-transport.so mode=0555
 file path=usr/lib/fm/fmd/plugins/ses-log-transport.conf
 file path=usr/lib/fm/fmd/plugins/ses-log-transport.so mode=0555
+file path=usr/lib/fm/fmd/plugins/slow-io-de.conf
+file path=usr/lib/fm/fmd/plugins/slow-io-de.so mode=0555
 file path=usr/lib/fm/fmd/plugins/software-diagnosis.conf \
     variant.opensolaris.zone=__NODEFAULT
 file path=usr/lib/fm/fmd/plugins/software-diagnosis.so mode=0555 \
@@ -470,6 +473,8 @@ file path=usr/lib/fm/fmd/plugins/zfs-monitor.conf
 file path=usr/lib/fm/fmd/plugins/zfs-monitor.so mode=0555
 file path=usr/lib/fm/fmd/plugins/zfs-retire.conf
 file path=usr/lib/fm/fmd/plugins/zfs-retire.so mode=0555
+
+
 #
 # fmri scheme support: all are common to both global and non-global zones
 #

usr/src/uts/common/io/scsi/adapters/mpt_sas/mptsas.c

@@ -3464,6 +3464,8 @@ mptsas_scsi_init_pkt(struct scsi_address *ap, struct scsi_pkt *pkt,
 
 	} else {
 		cmd = PKT2CMD(pkt);
+		pkt->pkt_start = 0;
+		pkt->pkt_stop = 0;
 		new_cmd = NULL;
 	}
 
@@ -4666,6 +4668,9 @@ mptsas_handle_scsi_io_success(mptsas_t *mpt,
 	}
 
 	pkt = CMD2PKT(cmd);
+	ASSERT(pkt->pkt_start != 0);
+	ASSERT(pkt->pkt_stop == 0);
+	pkt->pkt_stop = gethrtime();
 	pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET | STATE_SENT_CMD |
 	    STATE_GOT_STATUS);
 	if (cmd->cmd_flags & CFLAG_DMAVALID) {
@@ -4976,6 +4981,9 @@ mptsas_check_scsi_io_error(mptsas_t *mpt, pMpi2SCSIIOReply_t reply,
 	    scsi_status, ioc_status, scsi_state));
 
 	pkt = CMD2PKT(cmd);
+	ASSERT(pkt->pkt_start != 0);
+	ASSERT(pkt->pkt_stop == 0);
+	pkt->pkt_stop = gethrtime();
 	*(pkt->pkt_scbp) = scsi_status;
 
 	if (loginfo == 0x31170000) {
@@ -7936,7 +7944,7 @@ mptsas_start_cmd(mptsas_t *mpt, mptsas_cmd_t *cmd)
 	    (void *)(uintptr_t)mpt->m_req_frame_dma_addr, (void *)cmd));
 
 	(void) ddi_dma_sync(dma_hdl, 0, 0, DDI_DMA_SYNC_FORDEV);
-
+	pkt->pkt_start = gethrtime();
 	/*
 	 * Build request descriptor and write it to the request desc post reg.
 	 */
@@ -7947,8 +7955,10 @@ mptsas_start_cmd(mptsas_t *mpt, mptsas_cmd_t *cmd)
 	/*
 	 * Start timeout.
 	 */
-	cmd->cmd_active_expiration =
-	    gethrtime() + (hrtime_t)pkt->pkt_time * NANOSEC;
+
+	cmd->cmd_active_expiration = pkt->pkt_start +
+	    (hrtime_t)pkt->pkt_time * (hrtime_t)NANOSEC;
+
 #ifdef MPTSAS_TEST
 	/*
 	 * Force timeouts to happen immediately.

usr/src/uts/common/io/scsi/adapters/scsi_vhci/scsi_vhci.c

@@ -3189,6 +3189,8 @@ vhci_intr(struct scsi_pkt *pkt)
 	tpkt->pkt_state = pkt->pkt_state;
 	tpkt->pkt_statistics = pkt->pkt_statistics;
 	tpkt->pkt_reason = pkt->pkt_reason;
+	tpkt->pkt_start = pkt->pkt_start;
+	tpkt->pkt_stop = pkt->pkt_stop;
 
 	/* Return path_instance information back to the target driver. */
 	if (scsi_pkt_allocated_correctly(tpkt)) {

usr/src/uts/common/io/scsi/targets/sd.c

@@ -21,14 +21,15 @@
 
 /*
  * Copyright (c) 1990, 2010, Oracle and/or its affiliates. All rights reserved.
- */
-/*
+ *
+ *
  * Copyright (c) 2011 Bayard G. Bell.  All rights reserved.
  * Copyright (c) 2012 by Delphix. All rights reserved.
  * Copyright 2013 Nexenta Systems, Inc.  All rights reserved.
- */
-/*
+ *
+ *
  * Copyright 2011 cyril.galibern@opensvc.com
+ *
  */
 
 /*
@@ -241,6 +242,14 @@ int sd_qfull_throttle_enable		= TRUE;
 
 int sd_retry_on_reservation_conflict	= 1;
 int sd_reinstate_resv_delay		= SD_REINSTATE_RESV_DELAY;
+
+/*
+ * Default safe I/O delay threshold of 2s for all devices.
+ * Can be overriden for vendor/device id in sd.conf
+ */
+
+hrtime_t sd_g_slow_io_threshold = 2 * NANOSEC;
+
 _NOTE(SCHEME_PROTECTS_DATA("safe sharing", sd_reinstate_resv_delay))
 
 static int sd_dtype_optical_bind	= -1;
@@ -4231,6 +4240,23 @@ sd_set_properties(struct sd_lun *un, char *name, char *value)
 		    "physical block size set to %d\n", un->un_phy_blocksize);
 	}
 
+	if (strcasecmp(name, "slow-io-threshold") == 0) {
+		if (ddi_strtol(value, &endptr, 0, &val) == 0) {
+			un->un_slow_io_threshold = (hrtime_t)val * NANOSEC;
+		} else {
+			un->un_slow_io_threshold =
+			    (hrtime_t)sd_g_slow_io_threshold;
+			goto value_invalid;
+		}
+		SD_INFO(SD_LOG_ATTACH_DETACH, un, "sd_set_properties: "
+		    "slow IO threshold set to %llu\n",
+		    un->un_slow_io_threshold);
+#ifdef SDDEBUG
+		cmn_err(CE_NOTE, "slow IO set to %llu",
+		    un->un_slow_io_threshold);
+#endif
+	}
+
 	if (strcasecmp(name, "retries-victim") == 0) {
 		if (ddi_strtol(value, &endptr, 0, &val) == 0) {
 			un->un_victim_retry_count = val;
@@ -7534,6 +7560,8 @@ sd_unit_attach(dev_info_t *devi)
 	 */
 	un->un_reserve_release_time = 5;
 
+	un->un_slow_io_threshold = sd_g_slow_io_threshold;
+
 	/*
 	 * Set up the default maximum transfer size. Note that this may
 	 * get updated later in the attach, when setting up default wide
@@ -16825,6 +16853,29 @@ sdrunout(caddr_t arg)
 	return (1);
 }
 
+static void
+sd_slow_io_ereport(struct scsi_pkt *pktp)
+{
+	struct buf *bp;
+	struct sd_lun *un;
+	char *devid;
+
+	ASSERT(pktp != NULL);
+	bp = (struct buf *)pktp->pkt_private;
+	ASSERT(bp != NULL);
+	un = SD_GET_UN(bp);
+	ASSERT(un != NULL);
+
+	devid = DEVI(un->un_sd->sd_dev)->devi_devid_str;
+	scsi_fm_ereport_post(un->un_sd, 0, NULL, "cmd.disk.slow-io",
+	    fm_ena_generate(0, FM_ENA_FMT1), devid, NULL, DDI_NOSLEEP, NULL,
+	    FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0,
+	    "start", DATA_TYPE_UINT64, pktp->pkt_start,
+	    "stop", DATA_TYPE_UINT64, pktp->pkt_stop,
+	    "delta", DATA_TYPE_UINT64, pktp->pkt_stop - pktp->pkt_start,
+	    "threshold", DATA_TYPE_UINT64, un->un_slow_io_threshold,
+	    NULL);
+}
 
 /*
  *    Function: sdintr
@@ -16879,6 +16930,14 @@ sdintr(struct scsi_pkt *pktp)
 	un->un_in_callback++;
 
 	SD_UPDATE_KSTATS(un, kstat_runq_exit, bp);
+	if ((pktp->pkt_stop - pktp->pkt_start) > un->un_slow_io_threshold) {
+		sd_slow_io_ereport(pktp);
+#ifdef	SDDEBUG
+	cmn_err(CE_WARN, "Slow IO detected SD: 0x%p delta in nsec: %llu",
+	    un, pktp->pkt_stop - pktp->pkt_start);
+#endif
+	}
+
 
 #ifdef	SDDEBUG
 	if (bp == un->un_retry_bp) {

usr/src/uts/common/sys/scsi/scsi_pkt.h

@@ -21,6 +21,8 @@
 /*
  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
+ *
+ * Copyright 2013 Nexenta Systems, Inc.  All rights reserved.
  */
 
 #ifndef	_SYS_SCSI_SCSI_PKT_H
@@ -106,6 +108,8 @@ struct scsi_pkt {
 
 	/* stage-temporary: iff scsi_pkt_allocated_correctly() */
 	void	*pkt_stmp;		/* temporary for current pkt stage */
+	hrtime_t pkt_start;
+	hrtime_t pkt_stop;
 
 #ifdef	SCSI_SIZE_CLEAN_VERIFY
 	/*