Skip to content

Commit

Permalink
Improve performance for zpool trim on linux
Browse files Browse the repository at this point in the history 
On Linux, ZFS uses blkdev_issue_discard in vdev_disk_io_trim to issue
trim command which is synchronous.

This commit updates vdev_disk_io_trim to use __blkdev_issue_discard,
which is asynchronous. Unfortunately there isn't any asynchronous
version for blkdev_issue_secure_erase, so performance of secure trim
will still suffer.

Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-by: Alexander Motin <mav@FreeBSD.org>
Signed-off-by: Umer Saleem <usaleem@ixsystems.com>
Closes #15843
  • Loading branch information
@usaleem-ix @tonyhutter
usaleem-ix authored and tonyhutter committed Feb 6, 2024
1 parent 00d85a9 commit 08fd5cc
Show file tree
Hide file tree
Showing 2 changed files with 87 additions and 19 deletions.
34 changes: 29 additions & 5 deletions config/kernel-blkdev.m4
View file
Original file line number Diff line number Diff line change
Expand Up @@ -524,6 +524,7 @@ AC_DEFUN([ZFS_AC_KERNEL_BLKDEV_BDEVNAME], [

dnl #
dnl # 5.19 API: blkdev_issue_secure_erase()
dnl # 4.7 API: __blkdev_issue_discard(..., BLKDEV_DISCARD_SECURE)
dnl # 3.10 API: blkdev_issue_discard(..., BLKDEV_DISCARD_SECURE)
dnl #
AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV_ISSUE_SECURE_ERASE], [
Expand All @@ -539,6 +540,20 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV_ISSUE_SECURE_ERASE], [
sector, nr_sects, GFP_KERNEL);
])
ZFS_LINUX_TEST_SRC([blkdev_issue_discard_async_flags], [
#include <linux/blkdev.h>
],[
struct block_device *bdev = NULL;
sector_t sector = 0;
sector_t nr_sects = 0;
unsigned long flags = 0;
struct bio *biop = NULL;
int error __attribute__ ((unused));
error = __blkdev_issue_discard(bdev,
sector, nr_sects, GFP_KERNEL, flags, &biop);
])
ZFS_LINUX_TEST_SRC([blkdev_issue_discard_flags], [
#include <linux/blkdev.h>
],[
Expand All @@ -562,13 +577,22 @@ AC_DEFUN([ZFS_AC_KERNEL_BLKDEV_ISSUE_SECURE_ERASE], [
],[
AC_MSG_RESULT(no)
AC_MSG_CHECKING([whether blkdev_issue_discard() is available])
ZFS_LINUX_TEST_RESULT([blkdev_issue_discard_flags], [
AC_MSG_CHECKING([whether __blkdev_issue_discard() is available])
ZFS_LINUX_TEST_RESULT([blkdev_issue_discard_async_flags], [
AC_MSG_RESULT(yes)
AC_DEFINE(HAVE_BLKDEV_ISSUE_DISCARD, 1,
[blkdev_issue_discard() is available])
AC_DEFINE(HAVE_BLKDEV_ISSUE_DISCARD_ASYNC, 1,
[__blkdev_issue_discard() is available])
],[
ZFS_LINUX_TEST_ERROR([blkdev_issue_discard()])
AC_MSG_RESULT(no)
AC_MSG_CHECKING([whether blkdev_issue_discard() is available])
ZFS_LINUX_TEST_RESULT([blkdev_issue_discard_flags], [
AC_MSG_RESULT(yes)
AC_DEFINE(HAVE_BLKDEV_ISSUE_DISCARD, 1,
[blkdev_issue_discard() is available])
],[
ZFS_LINUX_TEST_ERROR([blkdev_issue_discard()])
])
])
])
])
Expand Down
72 changes: 58 additions & 14 deletions module/os/linux/zfs/vdev_disk.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -862,27 +862,66 @@ vdev_disk_io_flush(struct block_device *bdev, zio_t *zio)
return (0);
}

#if defined(HAVE_BLKDEV_ISSUE_SECURE_ERASE) || \
defined(HAVE_BLKDEV_ISSUE_DISCARD_ASYNC)
BIO_END_IO_PROTO(vdev_disk_discard_end_io, bio, error)
{
zio_t *zio = bio->bi_private;
#ifdef HAVE_1ARG_BIO_END_IO_T
zio->io_error = BIO_END_IO_ERROR(bio);
#else
zio->io_error = -error;
#endif
bio_put(bio);
if (zio->io_error)
vdev_disk_error(zio);
zio_interrupt(zio);
}

static int
vdev_disk_io_trim(zio_t *zio)
vdev_issue_discard_trim(zio_t *zio, unsigned long flags)
{
vdev_t *v = zio->io_vd;
vdev_disk_t *vd = v->vdev_tsd;
int ret;
struct bio *bio = NULL;

#if defined(HAVE_BLKDEV_ISSUE_SECURE_ERASE)
if (zio->io_trim_flags & ZIO_TRIM_SECURE) {
return (-blkdev_issue_secure_erase(BDH_BDEV(vd->vd_bdh),
zio->io_offset >> 9, zio->io_size >> 9, GFP_NOFS));
} else {
return (-blkdev_issue_discard(BDH_BDEV(vd->vd_bdh),
zio->io_offset >> 9, zio->io_size >> 9, GFP_NOFS));
#if defined(BLKDEV_DISCARD_SECURE)
ret = - __blkdev_issue_discard(
BDH_BDEV(((vdev_disk_t *)zio->io_vd->vdev_tsd)->vd_bdh),
zio->io_offset >> 9, zio->io_size >> 9, GFP_NOFS, flags, &bio);
#else
(void) flags;
ret = - __blkdev_issue_discard(
BDH_BDEV(((vdev_disk_t *)zio->io_vd->vdev_tsd)->vd_bdh),
zio->io_offset >> 9, zio->io_size >> 9, GFP_NOFS, &bio);
#endif
if (!ret && bio) {
bio->bi_private = zio;
bio->bi_end_io = vdev_disk_discard_end_io;
vdev_submit_bio(bio);
}
#elif defined(HAVE_BLKDEV_ISSUE_DISCARD)
return (ret);
}
#endif

static int
vdev_disk_io_trim(zio_t *zio)
{
unsigned long trim_flags = 0;
#if defined(BLKDEV_DISCARD_SECURE)
if (zio->io_trim_flags & ZIO_TRIM_SECURE)
if (zio->io_trim_flags & ZIO_TRIM_SECURE) {
#if defined(HAVE_BLKDEV_ISSUE_SECURE_ERASE)
return (-blkdev_issue_secure_erase(
BDH_BDEV(((vdev_disk_t *)zio->io_vd->vdev_tsd)->vd_bdh),
zio->io_offset >> 9, zio->io_size >> 9, GFP_NOFS));
#elif defined(BLKDEV_DISCARD_SECURE)
trim_flags |= BLKDEV_DISCARD_SECURE;
#endif
return (-blkdev_issue_discard(BDH_BDEV(vd->vd_bdh),
}
#if defined(HAVE_BLKDEV_ISSUE_SECURE_ERASE) || \
defined(HAVE_BLKDEV_ISSUE_DISCARD_ASYNC)
return (vdev_issue_discard_trim(zio, trim_flags));
#elif defined(HAVE_BLKDEV_ISSUE_DISCARD)
return (-blkdev_issue_discard(
BDH_BDEV(((vdev_disk_t *)zio->io_vd->vdev_tsd)->vd_bdh),
zio->io_offset >> 9, zio->io_size >> 9, GFP_NOFS, trim_flags));
#else
#error "Unsupported kernel"
Expand Down Expand Up @@ -968,7 +1007,12 @@ vdev_disk_io_start(zio_t *zio)
case ZIO_TYPE_TRIM:
zio->io_error = vdev_disk_io_trim(zio);
rw_exit(&vd->vd_lock);
#if defined(HAVE_BLKDEV_ISSUE_SECURE_ERASE)
if (zio->io_trim_flags & ZIO_TRIM_SECURE)
zio_interrupt(zio);
#elif defined(HAVE_BLKDEV_ISSUE_DISCARD)
zio_interrupt(zio);
#endif
return;

default:
Expand Down

0 comments on commit 08fd5cc

Please sign in to comment.