Permalink
Browse files

MMP interval and fail_intervals in uberblock

When Multihost is enabled, and a pool is imported, uberblock writes
include ub_mmp_delay to allow an importing node to calculate the
duration of an activity test.  This value, however, is not enough
information.

If zfs_multihost_fail_intervals > 0 on the node with the pool imported,
the safe minimum duration of the activity test is well defined, but does
not depend on ub_mmp_delay:

  zfs_multihost_fail_intervals * zfs_multihost_interval

and if zfs_multihost_fail_intervals == 0 on that node, there is no such
well defined safe duration, but the importing host cannot tell whether
mmp_delay is high due to I/O delays, or due to a very large
zfs_multihost_interval setting on the host which last imported the pool.
As a result, it may use a far longer period for the activity test than
is necessary.

This patch renames ub_mmp_sequence to ub_mmp_config and uses it to
record the zfs_multihost_interval and zfs_multihost_fail_intervals
values, as well as the mmp sequence.  This allows a shorter activity
test duration to be calculated by the importing host in most situations.
These values are also added to the multihost_history kstat records.

ZTS tests are added to verify the new functionality.

In addition, it makes a few other improvements:
* It updates the "sequence" part of ub_mmp_config when MMP writes
  in between syncs occur.  This allows an importing host to detect MMP
  on the remote host sooner, when the pool is idle, as it is not limited
  to the granularity of ub_timestamp (1 second).
* It issues writes immediately when zfs_multihost_interval is changed
  so remote hosts see the udpated value as soon as possible.
* It fixes a bug where setting zfs_multihost_fail_intervals = 1 results
  in immediate pool suspension.
* It reports nanoseconds remaining in the activity test via
  /proc/spl/kstat/zfs/<pool>/activity_test (during a tryimport,
  where the test is normally performed, the pool name is $import)
* It fixes a cleanup issue with test mmp_active_import, where ztest is
  not killed for some failure modes.
* In ZTS, when checking whether the activity test occurred, check
  against a duration specified via an argument, so it is clear reading
  the test what is expected.

Signed-off-by: Olaf Faaland <faaland1@llnl.gov>
  • Loading branch information...
ofaaland committed Aug 1, 2018
1 parent 00f198d commit 5a4b7fc87b38d70b0d7e7f44bfa6c3a54dbbfc86
@@ -2553,9 +2553,22 @@ dump_uberblock(uberblock_t *ub, const char *header, const char *footer)

(void) printf("\tmmp_magic = %016llx\n",
(u_longlong_t)ub->ub_mmp_magic);
if (ub->ub_mmp_magic == MMP_MAGIC)
if (MMP_VALID(ub)) {
(void) printf("\tmmp_delay = %0llu\n",
(u_longlong_t)ub->ub_mmp_delay);
if (MMP_SEQ_VALID(ub))
(void) printf("\tmmp_seq = %u\n",
(unsigned int) MMP_SEQ(ub));
if (MMP_FAIL_IVS_VALID(ub))
(void) printf("\tmmp_fail = %u\n",
(unsigned int) MMP_FAIL_IVS(ub));
if (MMP_INTERVAL_VALID(ub))
(void) printf("\tmmp_write = %u\n",
(unsigned int) MMP_INTERVAL(ub));
/* After MMP_* to make summarize_uberblock_mmp cleaner */
(void) printf("\tmmp_valid = %x\n",
(unsigned int) ub->ub_mmp_config & 0xFF);
}

if (dump_opt['u'] >= 4) {
char blkbuf[BP_SPRINTF_LEN];
@@ -31,6 +31,11 @@ extern "C" {
#define MMP_DEFAULT_INTERVAL 1000 /* ms */
#define MMP_DEFAULT_IMPORT_INTERVALS 10
#define MMP_DEFAULT_FAIL_INTERVALS 5
#define MMP_MIN_FAIL_INTERVALS 2 /* min if != 0 */
#define MMP_IMPORT_SAFETY_FACTOR 200 /* pct */
#define MMP_INTERVAL_OK(interval) MAX(interval, MMP_MIN_INTERVAL)
#define MMP_FAIL_INTVS_OK(fails) (fails == 0 ? 0 : MAX(fails, \
MMP_MIN_FAIL_INTERVALS))

typedef struct mmp_thread {
kmutex_t mmp_thread_lock; /* protect thread mgmt fields */
@@ -44,6 +49,11 @@ typedef struct mmp_thread {
zio_t *mmp_zio_root; /* root of mmp write zios */
uint64_t mmp_kstat_id; /* unique id for next MMP write kstat */
int mmp_skip_error; /* reason for last skipped write */
uint32_t mmp_seq; /* intra-second update counter */
uint32_t mmp_fail_intervals; /* processed fail intervals */
uint64_t mmp_interval; /* processed multihost interval (ns) */
hrtime_t mmp_fail_ns; /* max time without successful write ns */
uint64_t mmp_test_ns_remaining; /* activ. test time left (ns) */
} mmp_thread_t;


@@ -884,6 +884,7 @@ typedef struct spa_stats {
spa_history_kstat_t io_history;
spa_history_list_t mmp_history;
spa_history_kstat_t state; /* pool state */
spa_history_kstat_t activity_test; /* mmp test duration */
} spa_stats_t;

typedef enum txg_state {
@@ -918,7 +919,7 @@ extern int spa_mmp_history_set(spa_t *spa, uint64_t mmp_kstat_id, int io_error,
hrtime_t duration);
extern void spa_mmp_history_add(spa_t *spa, uint64_t txg, uint64_t timestamp,
uint64_t mmp_delay, vdev_t *vd, int label, uint64_t mmp_kstat_id,
int error);
int error, uint64_t mmp_interval, uint32_t mmp_fail_intervals);

/* Pool configuration locks */
extern int spa_config_tryenter(spa_t *spa, int locks, void *tag, krw_t rw);
@@ -44,7 +44,36 @@ extern "C" {
*/
#define UBERBLOCK_MAGIC 0x00bab10c /* oo-ba-bloc! */
#define UBERBLOCK_SHIFT 10 /* up to 1K */
#define MMP_MAGIC 0xa11cea11 /* all-see-all */
#define MMP_MAGIC 0xa11cea11 /* all-see-all */

#define MMP_SEQ_VALID_BIT 0x01
#define MMP_FAIL_IVS_VALID_BIT 0x02
#define MMP_INTERVAL_VALID_BIT 0x04

#define MMP_VALID(ubp) (ubp->ub_magic == UBERBLOCK_MAGIC && \
ubp->ub_mmp_magic == MMP_MAGIC)
#define MMP_SEQ_VALID(ubp) (MMP_VALID(ubp) && (ubp->ub_mmp_config & \
MMP_SEQ_VALID_BIT))
#define MMP_FAIL_IVS_VALID(ubp) (MMP_VALID(ubp) && (ubp->ub_mmp_config & \
MMP_FAIL_IVS_VALID_BIT))
#define MMP_INTERVAL_VALID(ubp) (MMP_VALID(ubp) && (ubp->ub_mmp_config & \
MMP_INTERVAL_VALID_BIT))

#define MMP_SEQ(ubp) ((ubp->ub_mmp_config & 0x000000000000FF00) \
>> 8)
#define MMP_FAIL_IVS(ubp) ((ubp->ub_mmp_config & 0x00000000FFFF0000) \
>> 16)
#define MMP_INTERVAL(ubp) ((ubp->ub_mmp_config & 0xFFFFFFFF00000000) \
>> 32)

#define MMP_SEQ_SET(ubp, seq) \
(((uint64_t)(seq & 0xFF) << 8) | MMP_SEQ_VALID_BIT)

#define MMP_FAIL_IVS_SET(ubp, fail) \
(((uint64_t)(fail & 0xFFFF) << 16) | MMP_FAIL_IVS_VALID_BIT)

#define MMP_INTERVAL_SET(ubp, write) \
(((uint64_t)(write & 0xFFFFFFFF) << 32) | MMP_INTERVAL_VALID_BIT)

struct uberblock {
uint64_t ub_magic; /* UBERBLOCK_MAGIC */
@@ -59,8 +88,29 @@ struct uberblock {

/* Maybe missing in uberblocks we read, but always written */
uint64_t ub_mmp_magic; /* MMP_MAGIC */
uint64_t ub_mmp_delay; /* nanosec since last MMP write */
uint64_t ub_mmp_seq; /* reserved for sequence number */
/*
* If ub_mmp_delay == 0 and ub_mmp_magic is valid, MMP is off.
* Otherwise, nanosec since last MMP write.
*/
uint64_t ub_mmp_delay;

/*
* The ub_mmp_config contains the multihost write internal, multihost
* fail intervals, sequence number for sub-second granularity, and valid
* bit mask. This layout is as follows:
*
* 64 56 48 40 32 24 16 8 0
* +-------+-------+-------+-------+-------+-------+-------+-------+
* 0 | Write Interval (ms) | Fail Intervals| Seq | VALID |
* +-------+-------+-------+-------+-------+-------+-------+-------+
*
* VALID Bits:
* - 0x01 - Sequence number exists
* - 0x02 - Fail Intervals
* - 0x04 - Write Interval (ms)
* - 0xf8 - Reserved
*/
uint64_t ub_mmp_config;

/*
* ub_checkpoint_txg indicates two things about the current uberblock:
@@ -1763,20 +1763,14 @@ Default value: \fB0\fR.
.ad
.RS 12n
Used to control the frequency of multihost writes which are performed when the
\fBmultihost\fR pool property is on. This is one factor used to determine
the length of the activity check during import.
\fBmultihost\fR pool property is on. This is one factor used to determine the
length of the activity check during import.
.sp
The multihost write period is \fBzfs_multihost_interval / leaf-vdevs\fR milliseconds.
This means that on average a multihost write will be issued for each leaf vdev every
\fBzfs_multihost_interval\fR milliseconds. In practice, the observed period can
vary with the I/O load and this observed value is the delay which is stored in
the uberblock.
.sp
On import the activity check waits a minimum amount of time determined by
\fBzfs_multihost_interval * zfs_multihost_import_intervals\fR. The activity
check time may be further extended if the value of mmp delay found in the best
uberblock indicates actual multihost updates happened at longer intervals than
\fBzfs_multihost_interval\fR. A minimum value of \fB100ms\fR is enforced.
The multihost write period is \fBzfs_multihost_interval / leaf-vdevs\fR
milliseconds. On average a multihost write will be issued for each leaf vdev
every \fBzfs_multihost_interval\fR milliseconds. In practice, the observed
period can vary with the I/O load and this observed value is the delay which is
stored in the uberblock.
.sp
Default value: \fB1000\fR.
.RE
@@ -1790,8 +1784,17 @@ Default value: \fB1000\fR.
Used to control the duration of the activity test on import. Smaller values of
\fBzfs_multihost_import_intervals\fR will reduce the import time but increase
the risk of failing to detect an active pool. The total activity check time is
never allowed to drop below one second. A value of 0 is ignored and treated as
if it was set to 1
never allowed to drop below one second.
.sp
On import the activity check waits a minimum amount of time determined by
\fBzfs_multihost_interval * zfs_multihost_import_intervals\fR, or the same
product computed on the host which last had the pool imported (whichever is
greater). The activity check time may be further extended if the value of mmp
delay found in the best uberblock indicates actual multihost updates happened
at longer intervals than \fBzfs_multihost_interval\fR. A minimum value of
\fB100ms\fR is enforced.
.sp
A value of 0 is ignored and treated as if it was set to 1.
.sp
Default value: \fB10\fR.
.RE
@@ -1802,17 +1805,20 @@ Default value: \fB10\fR.
\fBzfs_multihost_fail_intervals\fR (uint)
.ad
.RS 12n
Controls the behavior of the pool when multihost write failures are detected.
Controls the behavior of the pool when multihost write failures or delays are
detected.
.sp
When \fBzfs_multihost_fail_intervals = 0\fR then multihost write failures are ignored.
When \fBzfs_multihost_fail_intervals = 0\fR, multihost write failures or delays are ignored.
The failures will still be reported to the ZED which depending on its
configuration may take action such as suspending the pool or offlining a device.

.sp
When \fBzfs_multihost_fail_intervals > 0\fR then sequential multihost write failures
will cause the pool to be suspended. This occurs when
\fBzfs_multihost_fail_intervals * zfs_multihost_interval\fR milliseconds have
passed since the last successful multihost write. This guarantees the activity test
will see multihost writes if the pool is imported.
When \fBzfs_multihost_fail_intervals > 0\fR, the pool will be suspended if
\fBzfs_multihost_fail_intervals * zfs_multihost_interval\fR milliseconds pass
without a successful mmp write. This guarantees the activity test will see
mmp writes if the pool is imported. A value of 1 is ignored and treated as
if it was set to 2. This is necessary to prevent the pool from being suspended due to normal, small I/O latency variations.

.sp
Default value: \fB5\fR.
.RE
Oops, something went wrong.

0 comments on commit 5a4b7fc

Please sign in to comment.