Permalink
Browse files

OS-62 slow io error detector is needed.

  • Loading branch information...
gila authored and nljmo committed Nov 22, 2013
1 parent 505fdfe commit 94a046f61fca51c315656a6900bceda28a61b375
@@ -21,6 +21,8 @@
#
# Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
#
# Copyright 2013 Nexenta Systems, Inc. All rights reserved.
#
SUBDIRS = cpumem-retire \
disk-lights \
@@ -34,6 +36,7 @@ SUBDIRS = cpumem-retire \
ip-transport \
sensor-transport \
ses-log-transport \
slow-io-de \
sw-diag-response \
sp-monitor \
syslog-msgs \
@@ -0,0 +1,23 @@
#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source. A copy of the CDDL is also available via the Internet at
# http://www.illumos.org/license/CDDL.
#
# Copyright 2013 Nexenta Systems, Inc. All rights reserved.
#
MODULE = slow-io-de
CLASS = common
SRCS = slow-io-de.c
include ../../Makefile.plugin
LINTFLAGS += -L$(ROOT)/usr/lib/fm
#LDLIBS += -ltopo
LDFLAGS += -L$(ROOT)/usr/lib/fm -R/usr/lib/fm
@@ -0,0 +1,108 @@
/*
* This file and its contents are supplied under the terms of the
* Common Development and Distribution License ("CDDL"), version 1.0.
* You may only use this file in accordance with the terms of version
* 1.0 of the CDDL.
*
* A full copy of the text of the CDDL should have accompanied this
* source. A copy of the CDDL is also available via the Internet at
* http://www.illumos.org/license/CDDL.
*/
/*
* Copyright 2013 Nexenta Systems, Inc. All rights reserved.
*/
#include <fm/fmd_api.h>
typedef struct slow_io_stat {
fmd_stat_t bad_fmri;
fmd_stat_t bad_scheme;
} slow_io_stat_t;
slow_io_stat_t slow_io_stats = {
{ "bad_FMRI", FMD_TYPE_UINT64,
"event FMRI is missing or invalid" },
{ "bad_scheme", FMD_TYPE_UINT64,
"event does not contain a valid detector"},
};
static const fmd_prop_t fmd_props [] = {
{ "io_N", FMD_TYPE_INT32, "10" },
{ "io_T", FMD_TYPE_TIME, "10min"},
{ NULL, 0, NULL }
};
void
slow_io_close(fmd_hdl_t *hdl, fmd_case_t *c)
{
char *devid = fmd_case_getspecific(hdl, c);
fmd_hdl_debug(hdl, "Destroying serd: %s", devid);
fmd_serd_destroy(hdl, devid);
}
void
slow_io_recv(fmd_hdl_t *hdl, fmd_event_t *event, nvlist_t *nvl,
const char *class)
{
nvlist_t *detector = NULL;
char *devid = NULL;
if (nvlist_lookup_nvlist(nvl, "detector", &detector) != 0) {
slow_io_stats.bad_scheme.fmds_value.ui64++;
return;
}
if (nvlist_lookup_string(detector, "devid", &devid) != 0) {
slow_io_stats.bad_fmri.fmds_value.ui64++;
return;
}
if (fmd_serd_exists(hdl, devid) == 0) {
fmd_serd_create(hdl, devid, fmd_prop_get_int32(hdl, "io_N"),
fmd_prop_get_int64(hdl, "io_T"));
(void) fmd_serd_record(hdl, devid, event);
return;
}
if (fmd_serd_record(hdl, devid, event) == FMD_B_TRUE) {
fmd_case_t *c = fmd_case_open(hdl, NULL);
nvlist_t *fault = fmd_nvl_create_fault(hdl,
"fault.io.disk.predictive-failure", 100,
detector, NULL, detector);
fmd_case_add_serd(hdl, c, devid);
fmd_case_add_suspect(hdl, c, fault);
fmd_case_setspecific(hdl, c, devid);
fmd_case_solve(hdl, c);
}
}
static const fmd_hdl_ops_t fmd_ops = {
slow_io_recv,
NULL,
slow_io_close,
NULL,
NULL,
};
static const fmd_hdl_info_t fmd_info = {
"slow-io-de", "0.1", &fmd_ops, fmd_props
};
void
_fmd_init(fmd_hdl_t *hdl)
{
if (fmd_hdl_register(hdl, FMD_API_VERSION, &fmd_info) != 0) {
fmd_hdl_debug(hdl, "Internal error\n");
return;
}
fmd_stat_create(hdl, FMD_STAT_NOALLOC, sizeof (slow_io_stats) /
sizeof (fmd_stat_t), (fmd_stat_t *)&slow_io_stats);
}
void
_fmd_fini(fmd_hdl_t *hdl)
{
}
@@ -0,0 +1,19 @@
#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source. A copy of the CDDL is also available via the Internet at
# http://www.illumos.org/license/CDDL.
#
# Copyright 2013 Nexenta Systems, Inc. All rights reserved.
#
#
# fmd configuration file for the slow-io diagnosis
#
subscribe ereport.io.disk.slow-io
@@ -10,6 +10,7 @@
# See the License for the specific language governing permissions
# and limitations under the License.
#
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
@@ -22,7 +23,7 @@
#
# Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
#
# Copyright 2013 Nexenta Systems, Inc. All rights reserved.
#
# The default for payload-bearing actions in this package is to appear in the
# global zone only. See the include file for greater detail, as well as
@@ -450,6 +451,8 @@ file path=usr/lib/fm/fmd/plugins/sensor-transport.conf
file path=usr/lib/fm/fmd/plugins/sensor-transport.so mode=0555
file path=usr/lib/fm/fmd/plugins/ses-log-transport.conf
file path=usr/lib/fm/fmd/plugins/ses-log-transport.so mode=0555
file path=usr/lib/fm/fmd/plugins/slow-io-de.conf
file path=usr/lib/fm/fmd/plugins/slow-io-de.so mode=0555
file path=usr/lib/fm/fmd/plugins/software-diagnosis.conf \
variant.opensolaris.zone=__NODEFAULT
file path=usr/lib/fm/fmd/plugins/software-diagnosis.so mode=0555 \
@@ -470,6 +473,8 @@ file path=usr/lib/fm/fmd/plugins/zfs-monitor.conf
file path=usr/lib/fm/fmd/plugins/zfs-monitor.so mode=0555
file path=usr/lib/fm/fmd/plugins/zfs-retire.conf
file path=usr/lib/fm/fmd/plugins/zfs-retire.so mode=0555
#
# fmri scheme support: all are common to both global and non-global zones
#
@@ -3464,6 +3464,8 @@ mptsas_scsi_init_pkt(struct scsi_address *ap, struct scsi_pkt *pkt,
} else {
cmd = PKT2CMD(pkt);
pkt->pkt_start = 0;
pkt->pkt_stop = 0;
new_cmd = NULL;
}
@@ -4666,6 +4668,9 @@ mptsas_handle_scsi_io_success(mptsas_t *mpt,
}
pkt = CMD2PKT(cmd);
ASSERT(pkt->pkt_start != 0);
ASSERT(pkt->pkt_stop == 0);
pkt->pkt_stop = gethrtime();
pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET | STATE_SENT_CMD |
STATE_GOT_STATUS);
if (cmd->cmd_flags & CFLAG_DMAVALID) {
@@ -4976,6 +4981,9 @@ mptsas_check_scsi_io_error(mptsas_t *mpt, pMpi2SCSIIOReply_t reply,
scsi_status, ioc_status, scsi_state));
pkt = CMD2PKT(cmd);
ASSERT(pkt->pkt_start != 0);
ASSERT(pkt->pkt_stop == 0);
pkt->pkt_stop = gethrtime();
*(pkt->pkt_scbp) = scsi_status;
if (loginfo == 0x31170000) {
@@ -7936,7 +7944,7 @@ mptsas_start_cmd(mptsas_t *mpt, mptsas_cmd_t *cmd)
(void *)(uintptr_t)mpt->m_req_frame_dma_addr, (void *)cmd));
(void) ddi_dma_sync(dma_hdl, 0, 0, DDI_DMA_SYNC_FORDEV);
pkt->pkt_start = gethrtime();
/*
* Build request descriptor and write it to the request desc post reg.
*/
@@ -7947,8 +7955,10 @@ mptsas_start_cmd(mptsas_t *mpt, mptsas_cmd_t *cmd)
/*
* Start timeout.
*/
cmd->cmd_active_expiration =
gethrtime() + (hrtime_t)pkt->pkt_time * NANOSEC;
cmd->cmd_active_expiration = pkt->pkt_start +
(hrtime_t)pkt->pkt_time * (hrtime_t)NANOSEC;
#ifdef MPTSAS_TEST
/*
* Force timeouts to happen immediately.
@@ -3189,6 +3189,8 @@ vhci_intr(struct scsi_pkt *pkt)
tpkt->pkt_state = pkt->pkt_state;
tpkt->pkt_statistics = pkt->pkt_statistics;
tpkt->pkt_reason = pkt->pkt_reason;
tpkt->pkt_start = pkt->pkt_start;
tpkt->pkt_stop = pkt->pkt_stop;
/* Return path_instance information back to the target driver. */
if (scsi_pkt_allocated_correctly(tpkt)) {
@@ -21,14 +21,15 @@
/*
* Copyright (c) 1990, 2010, Oracle and/or its affiliates. All rights reserved.
*/
/*
*
*
* Copyright (c) 2011 Bayard G. Bell. All rights reserved.
* Copyright (c) 2012 by Delphix. All rights reserved.
* Copyright 2013 Nexenta Systems, Inc. All rights reserved.
*/
/*
*
*
* Copyright 2011 cyril.galibern@opensvc.com
*
*/
/*
@@ -241,6 +242,14 @@ int sd_qfull_throttle_enable = TRUE;
int sd_retry_on_reservation_conflict = 1;
int sd_reinstate_resv_delay = SD_REINSTATE_RESV_DELAY;
/*
* Default safe I/O delay threshold of 2s for all devices.
* Can be overriden for vendor/device id in sd.conf
*/
hrtime_t sd_g_slow_io_threshold = 2 * NANOSEC;
_NOTE(SCHEME_PROTECTS_DATA("safe sharing", sd_reinstate_resv_delay))
static int sd_dtype_optical_bind = -1;
@@ -4231,6 +4240,23 @@ sd_set_properties(struct sd_lun *un, char *name, char *value)
"physical block size set to %d\n", un->un_phy_blocksize);
}
if (strcasecmp(name, "slow-io-threshold") == 0) {
if (ddi_strtol(value, &endptr, 0, &val) == 0) {
un->un_slow_io_threshold = (hrtime_t)val * NANOSEC;
} else {
un->un_slow_io_threshold =
(hrtime_t)sd_g_slow_io_threshold;
goto value_invalid;
}
SD_INFO(SD_LOG_ATTACH_DETACH, un, "sd_set_properties: "
"slow IO threshold set to %llu\n",
un->un_slow_io_threshold);
#ifdef SDDEBUG
cmn_err(CE_NOTE, "slow IO set to %llu",
un->un_slow_io_threshold);
#endif
}
if (strcasecmp(name, "retries-victim") == 0) {
if (ddi_strtol(value, &endptr, 0, &val) == 0) {
un->un_victim_retry_count = val;
@@ -7534,6 +7560,8 @@ sd_unit_attach(dev_info_t *devi)
*/
un->un_reserve_release_time = 5;
un->un_slow_io_threshold = sd_g_slow_io_threshold;
/*
* Set up the default maximum transfer size. Note that this may
* get updated later in the attach, when setting up default wide
@@ -16825,6 +16853,29 @@ sdrunout(caddr_t arg)
return (1);
}
static void
sd_slow_io_ereport(struct scsi_pkt *pktp)
{
struct buf *bp;
struct sd_lun *un;
char *devid;
ASSERT(pktp != NULL);
bp = (struct buf *)pktp->pkt_private;
ASSERT(bp != NULL);
un = SD_GET_UN(bp);
ASSERT(un != NULL);
devid = DEVI(un->un_sd->sd_dev)->devi_devid_str;
scsi_fm_ereport_post(un->un_sd, 0, NULL, "cmd.disk.slow-io",
fm_ena_generate(0, FM_ENA_FMT1), devid, NULL, DDI_NOSLEEP, NULL,
FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0,
"start", DATA_TYPE_UINT64, pktp->pkt_start,
"stop", DATA_TYPE_UINT64, pktp->pkt_stop,
"delta", DATA_TYPE_UINT64, pktp->pkt_stop - pktp->pkt_start,
"threshold", DATA_TYPE_UINT64, un->un_slow_io_threshold,
NULL);
}
/*
* Function: sdintr
@@ -16879,6 +16930,14 @@ sdintr(struct scsi_pkt *pktp)
un->un_in_callback++;
SD_UPDATE_KSTATS(un, kstat_runq_exit, bp);
if ((pktp->pkt_stop - pktp->pkt_start) > un->un_slow_io_threshold) {
sd_slow_io_ereport(pktp);
#ifdef SDDEBUG
cmn_err(CE_WARN, "Slow IO detected SD: 0x%p delta in nsec: %llu",
un, pktp->pkt_stop - pktp->pkt_start);
#endif
}
#ifdef SDDEBUG
if (bp == un->un_retry_bp) {
@@ -21,6 +21,8 @@
/*
* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*
* Copyright 2013 Nexenta Systems, Inc. All rights reserved.
*/
#ifndef _SYS_SCSI_SCSI_PKT_H
@@ -106,6 +108,8 @@ struct scsi_pkt {
/* stage-temporary: iff scsi_pkt_allocated_correctly() */
void *pkt_stmp; /* temporary for current pkt stage */
hrtime_t pkt_start;
hrtime_t pkt_stop;
#ifdef SCSI_SIZE_CLEAN_VERIFY
/*
Oops, something went wrong.

0 comments on commit 94a046f

Please sign in to comment.