Permalink
Browse files

Remove races from scrub / resilver tests

Currently, several tests in the ZFS Test Suite that attempt to
test scrub and resilver behavior occasionally fail. A big reason
for this is that these tests use a combination of zinject and
zfs_scan_vdev_limit to attempt to slow these operations enough
to verify their test commands. This method works most of the time,
but provides no guarantees and leads to flaky behavior. This patch
adds a new tunable, zfs_scan_disable_progress, that ensures that
scans make no progress, guaranteeing that tests can be run without
racing.

Signed-off-by: Tom Caputi <tcaputi@datto.com>
  • Loading branch information...
tcaputi committed Nov 8, 2018
1 parent d8244d3 commit cd6726a963e099e9f0cf792a7efb703d55650eef
@@ -2139,6 +2139,17 @@ resilvers per leaf device, given in bytes.
Default value: \fB41943040\fR.
.RE
.sp
.ne 2
.na
\fBzfs_scan_disable_progress\fR (int)
.ad
.RS 12n
Set to disable scanning progress. This can be used for both scrubs and resilvers.
.sp
Default value: \fB0\fR.
.RE
.sp
.ne 2
.na
@@ -169,6 +169,7 @@ int zfs_obsolete_min_time_ms = 500; /* min millisecs to obsolete per txg */
int zfs_free_min_time_ms = 1000; /* min millisecs to free per txg */
int zfs_resilver_min_time_ms = 3000; /* min millisecs to resilver per txg */
int zfs_scan_checkpoint_intval = 7200; /* in seconds */
int zfs_scan_disable_progress = 0; /* set to prevent scans from progressing */
int zfs_no_scrub_io = B_FALSE; /* set to disable scrub i/o */
int zfs_no_scrub_prefetch = B_FALSE; /* set to disable scrub prefetch */
enum ddt_class zfs_scrub_ddt_class_max = DDT_CLASS_DUPLICATE;
@@ -3356,6 +3357,25 @@ dsl_scan_sync(dsl_pool_t *dp, dmu_tx_t *tx)
if (spa->spa_syncing_txg < spa->spa_first_txg + SCAN_IMPORT_WAIT_TXGS)
return;
/*
* zfs_scan_disable_progress can be set to disable scan progress.
* We don't want to spin the txg_sync thread, so we add a delay
* here to simulate the time spent doing a scan. This is mostly
* useful for testing and debugging.
*/
if (zfs_scan_disable_progress) {
uint64_t scan_time_ns = gethrtime() - scn->scn_sync_start_time;
while (zfs_scan_disable_progress &&
!txg_sync_waiting(scn->scn_dp) &&
!spa_shutting_down(scn->scn_dp->dp_spa) &&
NSEC2SEC(scan_time_ns) < zfs_txg_timeout) {
delay(hz);
scan_time_ns = gethrtime() - scn->scn_sync_start_time;
}
return;
}
/*
* It is possible to switch from unsorted to sorted at any time,
* but afterwards the scan will remain sorted unless reloaded from
@@ -4070,6 +4090,10 @@ MODULE_PARM_DESC(zfs_free_min_time_ms, "Min millisecs to free per txg");
module_param(zfs_resilver_min_time_ms, int, 0644);
MODULE_PARM_DESC(zfs_resilver_min_time_ms, "Min millisecs to resilver per txg");
module_param(zfs_scan_disable_progress, int, 0644);
MODULE_PARM_DESC(zfs_scan_disable_progress,
"Set to prevent scans from progressing");
module_param(zfs_no_scrub_io, int, 0644);
MODULE_PARM_DESC(zfs_no_scrub_io, "Set to disable scrub I/O");
@@ -42,10 +42,10 @@
# each sync.
# 2. Add data to pool
# 3. Re-import the pool so that data isn't cached
# 4. Use zinject to slow down device I/O
# 4. Use zfs_scan_disable_progress to ensure resilvers don't progress
# 5. Trigger the resilvering
# 6. Use spa freeze to stop writing to the pool.
# 7. Clear zinject events (needed to export the pool)
# 7. Re-enable scan progress
# 8. Export the pool
#
@@ -59,8 +59,7 @@ function custom_cleanup
[[ -n ZFS_TXG_TIMEOUT ]] &&
log_must set_zfs_txg_timeout $ZFS_TXG_TIMEOUT
zinject -c all
log_must set_tunable64 zfs_scan_vdev_limit $ZFS_SCAN_VDEV_LIMIT_DEFAULT
log_must set_tunable32 zfs_scan_disable_progress 0
cleanup
}
@@ -88,11 +87,7 @@ function test_replacing_vdevs
log_must zpool export $TESTPOOL1
log_must cp $CPATHBKP $CPATH
log_must zpool import -c $CPATH -o cachefile=$CPATH $TESTPOOL1
log_must set_tunable64 zfs_scan_vdev_limit $ZFS_SCAN_VDEV_LIMIT_SLOW
typeset device
for device in $zinjectdevices ; do
log_must zinject -d $device -D 50:1 $TESTPOOL1 > /dev/null
done
log_must set_tunable32 zfs_scan_disable_progress 1
log_must zpool replace $TESTPOOL1 $replacevdev $replaceby
# Cachefile: pool in resilvering state
@@ -103,8 +98,7 @@ function test_replacing_vdevs
log_must zpool freeze $TESTPOOL1
# Confirm pool is still replacing
log_must pool_is_replacing $TESTPOOL1
log_must zinject -c all > /dev/null
log_must set_tunable64 zfs_scan_vdev_limit $ZFS_SCAN_VDEV_LIMIT_DEFAULT
log_must set_tunable32 zfs_scan_disable_progress 0
log_must zpool export $TESTPOOL1
( $earlyremove ) && log_must rm $replacevdev
@@ -40,18 +40,12 @@
# deferred
# 4. Manually restart the resilver with all drives
#
# NOTES:
# Artificially limit the scrub speed by setting the zfs_scan_vdev_limit
# low and adding a 50ms zio delay in order to ensure that the resilver
# does not complete early.
#
verify_runnable "global"
function cleanup
{
log_must zinject -c all
log_must set_tunable64 zfs_scan_vdev_limit $ZFS_SCAN_VDEV_LIMIT_DEFAULT
log_must set_tunable32 zfs_scan_disable_progress 0
log_must rm -f $mntpnt/biggerfile1
log_must rm -f $mntpnt/biggerfile2
}
@@ -73,22 +67,19 @@ log_must sync
log_must zpool detach $TESTPOOL $DISK3
# 3. Reattach the drives, causing the second drive's resilver to be deferred
log_must set_tunable64 zfs_scan_vdev_limit $ZFS_SCAN_VDEV_LIMIT_SLOW
log_must set_tunable32 zfs_scan_disable_progress 1
log_must zpool attach $TESTPOOL $DISK1 $DISK2
log_must zinject -d $DISK2 -D50:1 $TESTPOOL
log_must is_pool_resilvering $TESTPOOL true
log_must zpool attach $TESTPOOL $DISK1 $DISK3
log_must zinject -d $DISK3 -D50:1 $TESTPOOL
log_must is_pool_resilvering $TESTPOOL true
# 4. Manually restart the resilver with all drives
log_must zpool resilver $TESTPOOL
log_must zinject -c all
log_must set_tunable64 zfs_scan_vdev_limit $ZFS_SCAN_VDEV_LIMIT_DEFAULT
log_must wait_for_resilver_end $TESTPOOL $MAXTIMEOUT
log_must is_deferred_scan_started $TESTPOOL
log_must set_tunable32 zfs_scan_disable_progress 0
log_must wait_for_resilver_end $TESTPOOL $MAXTIMEOUT
log_must check_state $TESTPOOL "$DISK2" "online"
log_must check_state $TESTPOOL "$DISK3" "online"
@@ -30,5 +30,4 @@
verify_runnable "global"
log_must set_tunable64 zfs_scan_vdev_limit $ZFS_SCAN_VDEV_LIMIT_DEFAULT
destroy_mirrors
@@ -45,18 +45,12 @@
# 5. Resume the paused scrub and verify scrub is again being performed.
# 6. Verify zpool scrub -s succeed when the system is scrubbing.
#
# NOTES:
# Artificially limit the scrub speed by setting the zfs_scan_vdev_limit
# low and adding a 50ms zio delay in order to ensure that the scrub does
# not complete early.
#
verify_runnable "global"
function cleanup
{
log_must zinject -c all
log_must set_tunable64 zfs_scan_vdev_limit $ZFS_SCAN_VDEV_LIMIT_DEFAULT
log_must set_tunable32 zfs_scan_disable_progress 0
log_must rm -f $mntpnt/biggerfile
}
@@ -69,8 +63,7 @@ mntpnt=$(get_prop mountpoint $TESTPOOL/$TESTFS)
log_must file_write -b 1048576 -c 1024 -o create -d 0 -f $mntpnt/biggerfile
log_must sync
log_must zinject -d $DISK1 -D50:1 $TESTPOOL
log_must set_tunable64 zfs_scan_vdev_limit $ZFS_SCAN_VDEV_LIMIT_SLOW
log_must set_tunable32 zfs_scan_disable_progress 1
log_must zpool scrub $TESTPOOL
log_must is_pool_scrubbing $TESTPOOL true
log_must zpool scrub -p $TESTPOOL
@@ -42,23 +42,19 @@
# 2. Kick off a scrub
# 2. Kick off a second scrub and verify it fails
#
# NOTES:
# Artificially limit the scrub speed by setting the zfs_scan_vdev_limit
# low in order to ensure that the scrub does not complete early.
#
verify_runnable "global"
function cleanup
{
log_must set_tunable64 zfs_scan_vdev_limit $ZFS_SCAN_VDEV_LIMIT_DEFAULT
log_must set_tunable32 zfs_scan_disable_progress 0
}
log_onexit cleanup
log_assert "Scrub command fails when there is already a scrub in progress"
log_must set_tunable64 zfs_scan_vdev_limit $ZFS_SCAN_VDEV_LIMIT_SLOW
log_must set_tunable32 zfs_scan_disable_progress 1
log_must zpool scrub $TESTPOOL
log_must is_pool_scrubbing $TESTPOOL true
log_mustnot zpool scrub $TESTPOOL
@@ -43,14 +43,10 @@
# 4. Export/import the pool to ensure the cache is dropped
# 5. Verify scrub failed until the resilver completed
#
# NOTES:
# Artificially limit the scrub speed by setting the zfs_scan_vdev_limit
# low in order to ensure that the scrub does not complete early.
#
function cleanup
{
log_must set_tunable64 zfs_scan_vdev_limit $ZFS_SCAN_VDEV_LIMIT_DEFAULT
log_must set_tunable32 zfs_scan_disable_progress 0
rm -f $mntpnt/extra
}
@@ -61,7 +57,9 @@ log_onexit cleanup
log_assert "Resilver prevent scrub from starting until the resilver completes"
mntpnt=$(get_prop mountpoint $TESTPOOL/$TESTFS)
log_must set_tunable64 zfs_scan_vdev_limit $ZFS_SCAN_VDEV_LIMIT_SLOW
# Temporarily prevent scan progress so our test doesn't race
log_must set_tunable32 zfs_scan_disable_progress 1
while ! is_pool_resilvering $TESTPOOL; do
log_must zpool detach $TESTPOOL $DISK2
@@ -74,6 +72,7 @@ done
log_must is_pool_resilvering $TESTPOOL
log_mustnot zpool scrub $TESTPOOL
log_must set_tunable32 zfs_scan_disable_progress 0
while ! is_pool_resilvered $TESTPOOL; do
sleep 1
done
@@ -40,7 +40,7 @@ verify_runnable "both"
function cleanup
{
log_must zinject -c all
log_must set_tunable32 zfs_scan_disable_progress 0
destroy_pool $TESTPOOL
destroy_pool $TESTPOOL2
rm -f $DEVICE1 $DEVICE2
@@ -67,10 +67,8 @@ function zpool_split #disk_to_be_offline/online
log_must file_write -b 2097152 -c 1024 -o create -d 0 -f $mntpnt/biggerfile
log_must sync
# slow-down resilvering, so it will not finish too early
log_must set_tunable64 zfs_scan_vdev_limit $ZFS_SCAN_VDEV_LIMIT_SLOW
log_must zinject -d $DEVICE1 -D 50:1 $TESTPOOL
log_must zinject -d $DEVICE2 -D 50:1 $TESTPOOL
# temporarily prevent resilvering progress, so it will not finish too early
log_must set_tunable32 zfs_scan_disable_progress 1
log_must zpool online $TESTPOOL $disk
@@ -85,7 +83,7 @@ function zpool_split #disk_to_be_offline/online
log_mustnot zpool split $TESTPOOL $TESTPOOL2
log_must set_tunable64 zfs_scan_vdev_limit $ZFS_SCAN_VDEV_LIMIT_DEFAULT
log_must set_tunable32 zfs_scan_disable_progress 0
}
log_assert "Verify 'zpool split' will fail if resilver in progress for a disk"
@@ -95,15 +93,12 @@ DEVSIZE='3g'
DEVICE1="$TEST_BASE_DIR/device-1"
DEVICE2="$TEST_BASE_DIR/device-2"
ZFS_SCAN_VDEV_LIMIT_SLOW=$((128*1024))
ZFS_SCAN_VDEV_LIMIT_DEFAULT=$(get_tunable zfs_scan_vdev_limit)
log_note "Verify ZFS prevents main pool curruption during 'split'"
log_note "Verify ZFS prevents main pool corruption during 'split'"
zpool_split $DEVICE1
cleanup
log_note "Verify ZFS prevents new pool curruption during 'split'"
log_note "Verify ZFS prevents new pool corruption during 'split'"
zpool_split $DEVICE2
log_pass "'zpool split' failed as expected"

0 comments on commit cd6726a

Please sign in to comment.