From 672de729a1f93d84e7597652b1125ab5d62421d8 Mon Sep 17 00:00:00 2001 From: Maxim Levitsky Date: Tue, 16 Jul 2019 19:19:01 +0300 Subject: [PATCH 01/17] LUKS: support preallocation preallocation=off and preallocation=metadata both allocate luks header only, and preallocation=falloc/full is passed to underlying file. Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1534951 Signed-off-by: Maxim Levitsky Message-id: 20190716161901.1430-1-mlevitsk@redhat.com Signed-off-by: Max Reitz --- block/crypto.c | 30 +++++++++++++++++++++++++++--- qapi/block-core.json | 6 +++++- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/block/crypto.c b/block/crypto.c index 8237424ae6d2..7eb698774eed 100644 --- a/block/crypto.c +++ b/block/crypto.c @@ -74,6 +74,7 @@ static ssize_t block_crypto_read_func(QCryptoBlock *block, struct BlockCryptoCreateData { BlockBackend *blk; uint64_t size; + PreallocMode prealloc; }; @@ -112,7 +113,7 @@ static ssize_t block_crypto_init_func(QCryptoBlock *block, * available to the guest, so we must take account of that * which will be used by the crypto header */ - return blk_truncate(data->blk, data->size + headerlen, PREALLOC_MODE_OFF, + return blk_truncate(data->blk, data->size + headerlen, data->prealloc, errp); } @@ -251,6 +252,7 @@ static int block_crypto_open_generic(QCryptoBlockFormat format, static int block_crypto_co_create_generic(BlockDriverState *bs, int64_t size, QCryptoBlockCreateOptions *opts, + PreallocMode prealloc, Error **errp) { int ret; @@ -266,9 +268,14 @@ static int block_crypto_co_create_generic(BlockDriverState *bs, goto cleanup; } + if (prealloc == PREALLOC_MODE_METADATA) { + prealloc = PREALLOC_MODE_OFF; + } + data = (struct BlockCryptoCreateData) { .blk = blk, .size = size, + .prealloc = prealloc, }; crypto = qcrypto_block_create(opts, NULL, @@ -500,6 +507,7 @@ block_crypto_co_create_luks(BlockdevCreateOptions *create_options, Error **errp) BlockdevCreateOptionsLUKS *luks_opts; BlockDriverState *bs = NULL; QCryptoBlockCreateOptions create_opts; + PreallocMode preallocation = PREALLOC_MODE_OFF; int ret; assert(create_options->driver == BLOCKDEV_DRIVER_LUKS); @@ -515,8 +523,12 @@ block_crypto_co_create_luks(BlockdevCreateOptions *create_options, Error **errp) .u.luks = *qapi_BlockdevCreateOptionsLUKS_base(luks_opts), }; + if (luks_opts->has_preallocation) { + preallocation = luks_opts->preallocation; + } + ret = block_crypto_co_create_generic(bs, luks_opts->size, &create_opts, - errp); + preallocation, errp); if (ret < 0) { goto fail; } @@ -534,12 +546,24 @@ static int coroutine_fn block_crypto_co_create_opts_luks(const char *filename, QCryptoBlockCreateOptions *create_opts = NULL; BlockDriverState *bs = NULL; QDict *cryptoopts; + PreallocMode prealloc; + char *buf = NULL; int64_t size; int ret; + Error *local_err = NULL; /* Parse options */ size = qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0); + buf = qemu_opt_get_del(opts, BLOCK_OPT_PREALLOC); + prealloc = qapi_enum_parse(&PreallocMode_lookup, buf, + PREALLOC_MODE_OFF, &local_err); + g_free(buf); + if (local_err) { + error_propagate(errp, local_err); + return -EINVAL; + } + cryptoopts = qemu_opts_to_qdict_filtered(opts, NULL, &block_crypto_create_opts_luks, true); @@ -565,7 +589,7 @@ static int coroutine_fn block_crypto_co_create_opts_luks(const char *filename, } /* Create format layer */ - ret = block_crypto_co_create_generic(bs, size, create_opts, errp); + ret = block_crypto_co_create_generic(bs, size, create_opts, prealloc, errp); if (ret < 0) { goto fail; } diff --git a/qapi/block-core.json b/qapi/block-core.json index e9364a4a2938..a5ab38db99bc 100644 --- a/qapi/block-core.json +++ b/qapi/block-core.json @@ -4212,13 +4212,17 @@ # # @file Node to create the image format on # @size Size of the virtual disk in bytes +# @preallocation Preallocation mode for the new image +# (since: 4.2) +# (default: off; allowed values: off, metadata, falloc, full) # # Since: 2.12 ## { 'struct': 'BlockdevCreateOptionsLUKS', 'base': 'QCryptoBlockCreateOptionsLUKS', 'data': { 'file': 'BlockdevRef', - 'size': 'size' } } + 'size': 'size', + '*preallocation': 'PreallocMode' } } ## # @BlockdevCreateOptionsNfs: From 4d7c487eac1652dfe4498fe84f32900ad461d61b Mon Sep 17 00:00:00 2001 From: Max Reitz Date: Wed, 24 Jul 2019 19:12:29 +0200 Subject: [PATCH 02/17] qemu-img: Fix bdrv_has_zero_init() use in convert bdrv_has_zero_init() only has meaning for newly created images or image areas. If qemu-img convert did not create the image itself, it cannot rely on bdrv_has_zero_init()'s result to carry any meaning. Signed-off-by: Max Reitz Message-id: 20190724171239.8764-2-mreitz@redhat.com Reviewed-by: Maxim Levitsky Reviewed-by: Stefano Garzarella Signed-off-by: Max Reitz --- qemu-img.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/qemu-img.c b/qemu-img.c index c920e3564cba..7daa05e51a53 100644 --- a/qemu-img.c +++ b/qemu-img.c @@ -1578,6 +1578,7 @@ typedef struct ImgConvertState { bool has_zero_init; bool compressed; bool unallocated_blocks_are_zero; + bool target_is_new; bool target_has_backing; int64_t target_backing_sectors; /* negative if unknown */ bool wr_in_order; @@ -1975,9 +1976,11 @@ static int convert_do_copy(ImgConvertState *s) int64_t sector_num = 0; /* Check whether we have zero initialisation or can get it efficiently */ - s->has_zero_init = s->min_sparse && !s->target_has_backing - ? bdrv_has_zero_init(blk_bs(s->target)) - : false; + if (s->target_is_new && s->min_sparse && !s->target_has_backing) { + s->has_zero_init = bdrv_has_zero_init(blk_bs(s->target)); + } else { + s->has_zero_init = false; + } if (!s->has_zero_init && !s->target_has_backing && bdrv_can_write_zeroes_with_unmap(blk_bs(s->target))) @@ -2428,6 +2431,8 @@ static int img_convert(int argc, char **argv) } } + s.target_is_new = !skip_create; + flags = s.min_sparse ? (BDRV_O_RDWR | BDRV_O_UNMAP) : BDRV_O_RDWR; ret = bdrv_parse_cache_mode(cache, &flags, &writethrough); if (ret < 0) { From cdf3bc934ad1e5319b03f2c85f381f5ffd2f8ca8 Mon Sep 17 00:00:00 2001 From: Max Reitz Date: Wed, 24 Jul 2019 19:12:30 +0200 Subject: [PATCH 03/17] mirror: Fix bdrv_has_zero_init() use bdrv_has_zero_init() only has meaning for newly created images or image areas. If the mirror job itself did not create the image, it cannot rely on bdrv_has_zero_init()'s result to carry any meaning. This is the case for drive-mirror with mode=existing and always for blockdev-mirror. Note that we only have to zero-initialize the target with sync=full, because other modes actually do not promise that the target will contain the same data as the source after the job -- sync=top only promises to copy anything allocated in the top layer, and sync=none will only copy new I/O. (Which is how mirror has always handled it.) Signed-off-by: Max Reitz Message-id: 20190724171239.8764-3-mreitz@redhat.com Reviewed-by: Maxim Levitsky Signed-off-by: Max Reitz --- block/mirror.c | 11 ++++++++--- blockdev.c | 16 +++++++++++++--- include/block/block_int.h | 2 ++ tests/test-block-iothread.c | 2 +- 4 files changed, 24 insertions(+), 7 deletions(-) diff --git a/block/mirror.c b/block/mirror.c index 2b870683f14c..853e2c751048 100644 --- a/block/mirror.c +++ b/block/mirror.c @@ -51,6 +51,8 @@ typedef struct MirrorBlockJob { Error *replace_blocker; bool is_none_mode; BlockMirrorBackingMode backing_mode; + /* Whether the target image requires explicit zero-initialization */ + bool zero_target; MirrorCopyMode copy_mode; BlockdevOnError on_source_error, on_target_error; bool synced; @@ -767,7 +769,7 @@ static int coroutine_fn mirror_dirty_init(MirrorBlockJob *s) int ret; int64_t count; - if (base == NULL && !bdrv_has_zero_init(target_bs)) { + if (s->zero_target) { if (!bdrv_can_write_zeroes_with_unmap(target_bs)) { bdrv_set_dirty_bitmap(s->dirty_bitmap, 0, s->bdev_length); return 0; @@ -1515,6 +1517,7 @@ static BlockJob *mirror_start_job( const char *replaces, int64_t speed, uint32_t granularity, int64_t buf_size, BlockMirrorBackingMode backing_mode, + bool zero_target, BlockdevOnError on_source_error, BlockdevOnError on_target_error, bool unmap, @@ -1643,6 +1646,7 @@ static BlockJob *mirror_start_job( s->on_target_error = on_target_error; s->is_none_mode = is_none_mode; s->backing_mode = backing_mode; + s->zero_target = zero_target; s->copy_mode = copy_mode; s->base = base; s->granularity = granularity; @@ -1747,6 +1751,7 @@ void mirror_start(const char *job_id, BlockDriverState *bs, int creation_flags, int64_t speed, uint32_t granularity, int64_t buf_size, MirrorSyncMode mode, BlockMirrorBackingMode backing_mode, + bool zero_target, BlockdevOnError on_source_error, BlockdevOnError on_target_error, bool unmap, const char *filter_node_name, @@ -1764,7 +1769,7 @@ void mirror_start(const char *job_id, BlockDriverState *bs, is_none_mode = mode == MIRROR_SYNC_MODE_NONE; base = mode == MIRROR_SYNC_MODE_TOP ? backing_bs(bs) : NULL; mirror_start_job(job_id, bs, creation_flags, target, replaces, - speed, granularity, buf_size, backing_mode, + speed, granularity, buf_size, backing_mode, zero_target, on_source_error, on_target_error, unmap, NULL, NULL, &mirror_job_driver, is_none_mode, base, false, filter_node_name, true, copy_mode, errp); @@ -1791,7 +1796,7 @@ BlockJob *commit_active_start(const char *job_id, BlockDriverState *bs, ret = mirror_start_job( job_id, bs, creation_flags, base, NULL, speed, 0, 0, - MIRROR_LEAVE_BACKING_CHAIN, + MIRROR_LEAVE_BACKING_CHAIN, false, on_error, on_error, true, cb, opaque, &commit_active_job_driver, false, base, auto_complete, filter_node_name, false, MIRROR_COPY_MODE_BACKGROUND, diff --git a/blockdev.c b/blockdev.c index 2e536dde3e9c..fbef6845c846 100644 --- a/blockdev.c +++ b/blockdev.c @@ -3782,6 +3782,7 @@ static void blockdev_mirror_common(const char *job_id, BlockDriverState *bs, bool has_replaces, const char *replaces, enum MirrorSyncMode sync, BlockMirrorBackingMode backing_mode, + bool zero_target, bool has_speed, int64_t speed, bool has_granularity, uint32_t granularity, bool has_buf_size, int64_t buf_size, @@ -3890,7 +3891,7 @@ static void blockdev_mirror_common(const char *job_id, BlockDriverState *bs, */ mirror_start(job_id, bs, target, has_replaces ? replaces : NULL, job_flags, - speed, granularity, buf_size, sync, backing_mode, + speed, granularity, buf_size, sync, backing_mode, zero_target, on_source_error, on_target_error, unmap, filter_node_name, copy_mode, errp); } @@ -3906,6 +3907,7 @@ void qmp_drive_mirror(DriveMirror *arg, Error **errp) int flags; int64_t size; const char *format = arg->format; + bool zero_target; int ret; bs = qmp_get_root_bs(arg->device, errp); @@ -4007,6 +4009,10 @@ void qmp_drive_mirror(DriveMirror *arg, Error **errp) goto out; } + zero_target = (arg->sync == MIRROR_SYNC_MODE_FULL && + (arg->mode == NEW_IMAGE_MODE_EXISTING || + !bdrv_has_zero_init(target_bs))); + ret = bdrv_try_set_aio_context(target_bs, aio_context, errp); if (ret < 0) { bdrv_unref(target_bs); @@ -4015,7 +4021,8 @@ void qmp_drive_mirror(DriveMirror *arg, Error **errp) blockdev_mirror_common(arg->has_job_id ? arg->job_id : NULL, bs, target_bs, arg->has_replaces, arg->replaces, arg->sync, - backing_mode, arg->has_speed, arg->speed, + backing_mode, zero_target, + arg->has_speed, arg->speed, arg->has_granularity, arg->granularity, arg->has_buf_size, arg->buf_size, arg->has_on_source_error, arg->on_source_error, @@ -4055,6 +4062,7 @@ void qmp_blockdev_mirror(bool has_job_id, const char *job_id, AioContext *aio_context; BlockMirrorBackingMode backing_mode = MIRROR_LEAVE_BACKING_CHAIN; Error *local_err = NULL; + bool zero_target; int ret; bs = qmp_get_root_bs(device, errp); @@ -4067,6 +4075,8 @@ void qmp_blockdev_mirror(bool has_job_id, const char *job_id, return; } + zero_target = (sync == MIRROR_SYNC_MODE_FULL); + aio_context = bdrv_get_aio_context(bs); aio_context_acquire(aio_context); @@ -4077,7 +4087,7 @@ void qmp_blockdev_mirror(bool has_job_id, const char *job_id, blockdev_mirror_common(has_job_id ? job_id : NULL, bs, target_bs, has_replaces, replaces, sync, backing_mode, - has_speed, speed, + zero_target, has_speed, speed, has_granularity, granularity, has_buf_size, buf_size, has_on_source_error, on_source_error, diff --git a/include/block/block_int.h b/include/block/block_int.h index aa697f1f694a..8fa011654a54 100644 --- a/include/block/block_int.h +++ b/include/block/block_int.h @@ -1115,6 +1115,7 @@ BlockJob *commit_active_start(const char *job_id, BlockDriverState *bs, * @buf_size: The amount of data that can be in flight at one time. * @mode: Whether to collapse all images in the chain to the target. * @backing_mode: How to establish the target's backing chain after completion. + * @zero_target: Whether the target should be explicitly zero-initialized * @on_source_error: The action to take upon error reading from the source. * @on_target_error: The action to take upon error writing to the target. * @unmap: Whether to unmap target where source sectors only contain zeroes. @@ -1134,6 +1135,7 @@ void mirror_start(const char *job_id, BlockDriverState *bs, int creation_flags, int64_t speed, uint32_t granularity, int64_t buf_size, MirrorSyncMode mode, BlockMirrorBackingMode backing_mode, + bool zero_target, BlockdevOnError on_source_error, BlockdevOnError on_target_error, bool unmap, const char *filter_node_name, diff --git a/tests/test-block-iothread.c b/tests/test-block-iothread.c index e81b5b8dc49b..926577b1f9db 100644 --- a/tests/test-block-iothread.c +++ b/tests/test-block-iothread.c @@ -612,7 +612,7 @@ static void test_propagate_mirror(void) /* Start a mirror job */ mirror_start("job0", src, target, NULL, JOB_DEFAULT, 0, 0, 0, - MIRROR_SYNC_MODE_NONE, MIRROR_OPEN_BACKING_CHAIN, + MIRROR_SYNC_MODE_NONE, MIRROR_OPEN_BACKING_CHAIN, false, BLOCKDEV_ON_ERROR_REPORT, BLOCKDEV_ON_ERROR_REPORT, false, "filter_node", MIRROR_COPY_MODE_BACKGROUND, &error_abort); From ceaca56feee6a1d682c882300855cad44a67ec8e Mon Sep 17 00:00:00 2001 From: Max Reitz Date: Wed, 24 Jul 2019 19:12:31 +0200 Subject: [PATCH 04/17] block: Add bdrv_has_zero_init_truncate() No .bdrv_has_zero_init() implementation returns 1 if growing the file would add non-zero areas (at least with PREALLOC_MODE_OFF), so using it in lieu of this new function was always safe. But on the other hand, it is possible that growing an image that is not zero-initialized would still add a zero-initialized area, like when using nonpreallocating truncation on a preallocated image. For callers that care only about truncation, not about creation with potential preallocation, this new function is useful. Alternatively, we could have added a PreallocMode parameter to bdrv_has_zero_init(). But the only user would have been qemu-img convert, which does not have a plain PreallocMode value right now -- it would have to parse the creation option to obtain it. Therefore, the simpler solution is to let bdrv_has_zero_init() inquire the preallocation status and add the new bdrv_has_zero_init_truncate() that presupposes PREALLOC_MODE_OFF. Signed-off-by: Max Reitz Message-id: 20190724171239.8764-4-mreitz@redhat.com Reviewed-by: Maxim Levitsky Reviewed-by: Stefano Garzarella Signed-off-by: Max Reitz --- block.c | 21 +++++++++++++++++++++ include/block/block.h | 1 + include/block/block_int.h | 7 +++++++ 3 files changed, 29 insertions(+) diff --git a/block.c b/block.c index 3e698e9cabdd..874a29a983fe 100644 --- a/block.c +++ b/block.c @@ -5078,6 +5078,27 @@ int bdrv_has_zero_init(BlockDriverState *bs) return 0; } +int bdrv_has_zero_init_truncate(BlockDriverState *bs) +{ + if (!bs->drv) { + return 0; + } + + if (bs->backing) { + /* Depends on the backing image length, but better safe than sorry */ + return 0; + } + if (bs->drv->bdrv_has_zero_init_truncate) { + return bs->drv->bdrv_has_zero_init_truncate(bs); + } + if (bs->file && bs->drv->is_filter) { + return bdrv_has_zero_init_truncate(bs->file->bs); + } + + /* safe default */ + return 0; +} + bool bdrv_unallocated_blocks_are_zero(BlockDriverState *bs) { BlockDriverInfo bdi; diff --git a/include/block/block.h b/include/block/block.h index 89e40318cfbd..124ad408096e 100644 --- a/include/block/block.h +++ b/include/block/block.h @@ -443,6 +443,7 @@ int bdrv_pdiscard(BdrvChild *child, int64_t offset, int64_t bytes); int bdrv_co_pdiscard(BdrvChild *child, int64_t offset, int64_t bytes); int bdrv_has_zero_init_1(BlockDriverState *bs); int bdrv_has_zero_init(BlockDriverState *bs); +int bdrv_has_zero_init_truncate(BlockDriverState *bs); bool bdrv_unallocated_blocks_are_zero(BlockDriverState *bs); bool bdrv_can_write_zeroes_with_unmap(BlockDriverState *bs); int bdrv_block_status(BlockDriverState *bs, int64_t offset, diff --git a/include/block/block_int.h b/include/block/block_int.h index 8fa011654a54..ceec8c2f5622 100644 --- a/include/block/block_int.h +++ b/include/block/block_int.h @@ -419,9 +419,16 @@ struct BlockDriver { /* * Returns 1 if newly created images are guaranteed to contain only * zeros, 0 otherwise. + * Must return 0 if .bdrv_has_zero_init_truncate() returns 0. */ int (*bdrv_has_zero_init)(BlockDriverState *bs); + /* + * Returns 1 if new areas added by growing the image with + * PREALLOC_MODE_OFF contain only zeros, 0 otherwise. + */ + int (*bdrv_has_zero_init_truncate)(BlockDriverState *bs); + /* Remove fd handlers, timers, and other event loop callbacks so the event * loop is no longer in use. Called with no in-flight requests and in * depth-first traversal order with parents before child nodes. From 1dcaf52760ccaa2e019164c887b8436ac6c5d8ea Mon Sep 17 00:00:00 2001 From: Max Reitz Date: Wed, 24 Jul 2019 19:12:32 +0200 Subject: [PATCH 05/17] block: Implement .bdrv_has_zero_init_truncate() We need to implement .bdrv_has_zero_init_truncate() for every block driver that supports truncation and has a .bdrv_has_zero_init() implementation. Implement it the same way each driver implements .bdrv_has_zero_init(). This is at least not any more unsafe than what we had before. Signed-off-by: Max Reitz Message-id: 20190724171239.8764-5-mreitz@redhat.com Reviewed-by: Maxim Levitsky Reviewed-by: Stefano Garzarella Signed-off-by: Max Reitz --- block/file-posix.c | 1 + block/file-win32.c | 1 + block/gluster.c | 4 ++++ block/nfs.c | 1 + block/qcow2.c | 1 + block/qed.c | 1 + block/raw-format.c | 6 ++++++ block/rbd.c | 1 + block/sheepdog.c | 1 + block/ssh.c | 1 + 10 files changed, 18 insertions(+) diff --git a/block/file-posix.c b/block/file-posix.c index e41e91e075b0..fbeb0068db0a 100644 --- a/block/file-posix.c +++ b/block/file-posix.c @@ -2938,6 +2938,7 @@ BlockDriver bdrv_file = { .bdrv_co_create = raw_co_create, .bdrv_co_create_opts = raw_co_create_opts, .bdrv_has_zero_init = bdrv_has_zero_init_1, + .bdrv_has_zero_init_truncate = bdrv_has_zero_init_1, .bdrv_co_block_status = raw_co_block_status, .bdrv_co_invalidate_cache = raw_co_invalidate_cache, .bdrv_co_pwrite_zeroes = raw_co_pwrite_zeroes, diff --git a/block/file-win32.c b/block/file-win32.c index 6b2d67b239cf..41f55dfece6e 100644 --- a/block/file-win32.c +++ b/block/file-win32.c @@ -635,6 +635,7 @@ BlockDriver bdrv_file = { .bdrv_close = raw_close, .bdrv_co_create_opts = raw_co_create_opts, .bdrv_has_zero_init = bdrv_has_zero_init_1, + .bdrv_has_zero_init_truncate = bdrv_has_zero_init_1, .bdrv_aio_preadv = raw_aio_preadv, .bdrv_aio_pwritev = raw_aio_pwritev, diff --git a/block/gluster.c b/block/gluster.c index f64dc5b01e42..64028b2cba30 100644 --- a/block/gluster.c +++ b/block/gluster.c @@ -1567,6 +1567,7 @@ static BlockDriver bdrv_gluster = { .bdrv_co_writev = qemu_gluster_co_writev, .bdrv_co_flush_to_disk = qemu_gluster_co_flush_to_disk, .bdrv_has_zero_init = qemu_gluster_has_zero_init, + .bdrv_has_zero_init_truncate = qemu_gluster_has_zero_init, #ifdef CONFIG_GLUSTERFS_DISCARD .bdrv_co_pdiscard = qemu_gluster_co_pdiscard, #endif @@ -1598,6 +1599,7 @@ static BlockDriver bdrv_gluster_tcp = { .bdrv_co_writev = qemu_gluster_co_writev, .bdrv_co_flush_to_disk = qemu_gluster_co_flush_to_disk, .bdrv_has_zero_init = qemu_gluster_has_zero_init, + .bdrv_has_zero_init_truncate = qemu_gluster_has_zero_init, #ifdef CONFIG_GLUSTERFS_DISCARD .bdrv_co_pdiscard = qemu_gluster_co_pdiscard, #endif @@ -1629,6 +1631,7 @@ static BlockDriver bdrv_gluster_unix = { .bdrv_co_writev = qemu_gluster_co_writev, .bdrv_co_flush_to_disk = qemu_gluster_co_flush_to_disk, .bdrv_has_zero_init = qemu_gluster_has_zero_init, + .bdrv_has_zero_init_truncate = qemu_gluster_has_zero_init, #ifdef CONFIG_GLUSTERFS_DISCARD .bdrv_co_pdiscard = qemu_gluster_co_pdiscard, #endif @@ -1666,6 +1669,7 @@ static BlockDriver bdrv_gluster_rdma = { .bdrv_co_writev = qemu_gluster_co_writev, .bdrv_co_flush_to_disk = qemu_gluster_co_flush_to_disk, .bdrv_has_zero_init = qemu_gluster_has_zero_init, + .bdrv_has_zero_init_truncate = qemu_gluster_has_zero_init, #ifdef CONFIG_GLUSTERFS_DISCARD .bdrv_co_pdiscard = qemu_gluster_co_pdiscard, #endif diff --git a/block/nfs.c b/block/nfs.c index ed0cce63bb73..0ec50953e464 100644 --- a/block/nfs.c +++ b/block/nfs.c @@ -863,6 +863,7 @@ static BlockDriver bdrv_nfs = { .create_opts = &nfs_create_opts, .bdrv_has_zero_init = nfs_has_zero_init, + .bdrv_has_zero_init_truncate = nfs_has_zero_init, .bdrv_get_allocated_file_size = nfs_get_allocated_file_size, .bdrv_co_truncate = nfs_file_co_truncate, diff --git a/block/qcow2.c b/block/qcow2.c index 59cff1d4cb6d..ea3b42fdac4f 100644 --- a/block/qcow2.c +++ b/block/qcow2.c @@ -5188,6 +5188,7 @@ BlockDriver bdrv_qcow2 = { .bdrv_co_create_opts = qcow2_co_create_opts, .bdrv_co_create = qcow2_co_create, .bdrv_has_zero_init = bdrv_has_zero_init_1, + .bdrv_has_zero_init_truncate = bdrv_has_zero_init_1, .bdrv_co_block_status = qcow2_co_block_status, .bdrv_co_preadv = qcow2_co_preadv, diff --git a/block/qed.c b/block/qed.c index d0dcc5f14d83..0d8fd507aaf4 100644 --- a/block/qed.c +++ b/block/qed.c @@ -1669,6 +1669,7 @@ static BlockDriver bdrv_qed = { .bdrv_co_create = bdrv_qed_co_create, .bdrv_co_create_opts = bdrv_qed_co_create_opts, .bdrv_has_zero_init = bdrv_has_zero_init_1, + .bdrv_has_zero_init_truncate = bdrv_has_zero_init_1, .bdrv_co_block_status = bdrv_qed_co_block_status, .bdrv_co_readv = bdrv_qed_co_readv, .bdrv_co_writev = bdrv_qed_co_writev, diff --git a/block/raw-format.c b/block/raw-format.c index bffd424dd0da..42c28cc29aaf 100644 --- a/block/raw-format.c +++ b/block/raw-format.c @@ -413,6 +413,11 @@ static int raw_has_zero_init(BlockDriverState *bs) return bdrv_has_zero_init(bs->file->bs); } +static int raw_has_zero_init_truncate(BlockDriverState *bs) +{ + return bdrv_has_zero_init_truncate(bs->file->bs); +} + static int coroutine_fn raw_co_create_opts(const char *filename, QemuOpts *opts, Error **errp) { @@ -572,6 +577,7 @@ BlockDriver bdrv_raw = { .bdrv_co_ioctl = &raw_co_ioctl, .create_opts = &raw_create_opts, .bdrv_has_zero_init = &raw_has_zero_init, + .bdrv_has_zero_init_truncate = &raw_has_zero_init_truncate, .strong_runtime_opts = raw_strong_runtime_opts, .mutable_opts = mutable_opts, }; diff --git a/block/rbd.c b/block/rbd.c index 59757b312022..057af43d484c 100644 --- a/block/rbd.c +++ b/block/rbd.c @@ -1288,6 +1288,7 @@ static BlockDriver bdrv_rbd = { .bdrv_co_create = qemu_rbd_co_create, .bdrv_co_create_opts = qemu_rbd_co_create_opts, .bdrv_has_zero_init = bdrv_has_zero_init_1, + .bdrv_has_zero_init_truncate = bdrv_has_zero_init_1, .bdrv_get_info = qemu_rbd_getinfo, .create_opts = &qemu_rbd_create_opts, .bdrv_getlength = qemu_rbd_getlength, diff --git a/block/sheepdog.c b/block/sheepdog.c index 31b0a820c264..773dfc6ab13a 100644 --- a/block/sheepdog.c +++ b/block/sheepdog.c @@ -3229,6 +3229,7 @@ static BlockDriver bdrv_sheepdog = { .bdrv_co_create = sd_co_create, .bdrv_co_create_opts = sd_co_create_opts, .bdrv_has_zero_init = bdrv_has_zero_init_1, + .bdrv_has_zero_init_truncate = bdrv_has_zero_init_1, .bdrv_getlength = sd_getlength, .bdrv_get_allocated_file_size = sd_get_allocated_file_size, .bdrv_co_truncate = sd_co_truncate, diff --git a/block/ssh.c b/block/ssh.c index 501933b85531..84d01e892bc3 100644 --- a/block/ssh.c +++ b/block/ssh.c @@ -1390,6 +1390,7 @@ static BlockDriver bdrv_ssh = { .bdrv_co_create_opts = ssh_co_create_opts, .bdrv_close = ssh_close, .bdrv_has_zero_init = ssh_has_zero_init, + .bdrv_has_zero_init_truncate = ssh_has_zero_init, .bdrv_co_readv = ssh_co_readv, .bdrv_co_writev = ssh_co_writev, .bdrv_getlength = ssh_getlength, From b647d69adc394312dedb56f6b3ce19b9c316931f Mon Sep 17 00:00:00 2001 From: Max Reitz Date: Wed, 24 Jul 2019 19:12:33 +0200 Subject: [PATCH 06/17] block: Use bdrv_has_zero_init_truncate() vhdx and parallels call bdrv_has_zero_init() when they do not really care about an image's post-create state but only about what happens when you grow an image. That is a bit ugly, and also overly safe when growing preallocated images without preallocating the new areas. Let them use bdrv_has_zero_init_truncate() instead. Signed-off-by: Max Reitz Message-id: 20190724171239.8764-6-mreitz@redhat.com Reviewed-by: Maxim Levitsky Reviewed-by: Stefano Garzarella [mreitz: Added commit message] Signed-off-by: Max Reitz --- block/parallels.c | 2 +- block/vhdx.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/block/parallels.c b/block/parallels.c index 00fae125d14b..7cd2714b69f5 100644 --- a/block/parallels.c +++ b/block/parallels.c @@ -835,7 +835,7 @@ static int parallels_open(BlockDriverState *bs, QDict *options, int flags, goto fail_options; } - if (!bdrv_has_zero_init(bs->file->bs)) { + if (!bdrv_has_zero_init_truncate(bs->file->bs)) { s->prealloc_mode = PRL_PREALLOC_MODE_FALLOCATE; } diff --git a/block/vhdx.c b/block/vhdx.c index d6070b6fa812..a02d1c99a76c 100644 --- a/block/vhdx.c +++ b/block/vhdx.c @@ -1282,7 +1282,7 @@ static coroutine_fn int vhdx_co_writev(BlockDriverState *bs, int64_t sector_num, /* Queue another write of zero buffers if the underlying file * does not zero-fill on file extension */ - if (bdrv_has_zero_init(bs->file->bs) == 0) { + if (bdrv_has_zero_init_truncate(bs->file->bs) == 0) { use_zero_buffers = true; /* zero fill the front, if any */ From 38841dcd27a41e2d8916b7cf4f508a5663f453ff Mon Sep 17 00:00:00 2001 From: Max Reitz Date: Wed, 24 Jul 2019 19:12:34 +0200 Subject: [PATCH 07/17] qcow2: Fix .bdrv_has_zero_init() If a qcow2 file is preallocated, it can no longer guarantee that it initially appears as filled with zeroes. So implement .bdrv_has_zero_init() by checking whether the file is preallocated; if so, forward the call to the underlying storage node, except for when it is encrypted: Encrypted preallocated images always return effectively random data, so .bdrv_has_zero_init() must always return 0 for them. .bdrv_has_zero_init_truncate() can remain bdrv_has_zero_init_1(), because it presupposes PREALLOC_MODE_OFF. Reported-by: Stefano Garzarella Signed-off-by: Max Reitz Message-id: 20190724171239.8764-7-mreitz@redhat.com Reviewed-by: Maxim Levitsky Signed-off-by: Max Reitz --- block/qcow2.c | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/block/qcow2.c b/block/qcow2.c index ea3b42fdac4f..7c5a4859f731 100644 --- a/block/qcow2.c +++ b/block/qcow2.c @@ -4632,6 +4632,33 @@ static ImageInfoSpecific *qcow2_get_specific_info(BlockDriverState *bs, return spec_info; } +static int qcow2_has_zero_init(BlockDriverState *bs) +{ + BDRVQcow2State *s = bs->opaque; + bool preallocated; + + if (qemu_in_coroutine()) { + qemu_co_mutex_lock(&s->lock); + } + /* + * Check preallocation status: Preallocated images have all L2 + * tables allocated, nonpreallocated images have none. It is + * therefore enough to check the first one. + */ + preallocated = s->l1_size > 0 && s->l1_table[0] != 0; + if (qemu_in_coroutine()) { + qemu_co_mutex_unlock(&s->lock); + } + + if (!preallocated) { + return 1; + } else if (bs->encrypted) { + return 0; + } else { + return bdrv_has_zero_init(s->data_file->bs); + } +} + static int qcow2_save_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos) { @@ -5187,7 +5214,7 @@ BlockDriver bdrv_qcow2 = { .bdrv_child_perm = bdrv_format_default_perms, .bdrv_co_create_opts = qcow2_co_create_opts, .bdrv_co_create = qcow2_co_create, - .bdrv_has_zero_init = bdrv_has_zero_init_1, + .bdrv_has_zero_init = qcow2_has_zero_init, .bdrv_has_zero_init_truncate = bdrv_has_zero_init_1, .bdrv_co_block_status = qcow2_co_block_status, From 0a28bf2826ca027f0d993388f118b0bdbfca73a9 Mon Sep 17 00:00:00 2001 From: Max Reitz Date: Wed, 24 Jul 2019 19:12:35 +0200 Subject: [PATCH 08/17] vdi: Fix .bdrv_has_zero_init() Static VDI images cannot guarantee to be zero-initialized. If the image has been statically allocated, forward the call to the underlying storage node. Reported-by: Stefano Garzarella Signed-off-by: Max Reitz Reviewed-by: Stefan Weil Acked-by: Stefano Garzarella Tested-by: Stefano Garzarella Message-id: 20190724171239.8764-8-mreitz@redhat.com Reviewed-by: Maxim Levitsky Signed-off-by: Max Reitz --- block/vdi.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/block/vdi.c b/block/vdi.c index b9845a4cbded..0caa3f281d26 100644 --- a/block/vdi.c +++ b/block/vdi.c @@ -988,6 +988,17 @@ static void vdi_close(BlockDriverState *bs) error_free(s->migration_blocker); } +static int vdi_has_zero_init(BlockDriverState *bs) +{ + BDRVVdiState *s = bs->opaque; + + if (s->header.image_type == VDI_TYPE_STATIC) { + return bdrv_has_zero_init(bs->file->bs); + } else { + return 1; + } +} + static QemuOptsList vdi_create_opts = { .name = "vdi-create-opts", .head = QTAILQ_HEAD_INITIALIZER(vdi_create_opts.head), @@ -1028,7 +1039,7 @@ static BlockDriver bdrv_vdi = { .bdrv_child_perm = bdrv_format_default_perms, .bdrv_co_create = vdi_co_create, .bdrv_co_create_opts = vdi_co_create_opts, - .bdrv_has_zero_init = bdrv_has_zero_init_1, + .bdrv_has_zero_init = vdi_has_zero_init, .bdrv_co_block_status = vdi_co_block_status, .bdrv_make_empty = vdi_make_empty, From 9956688a8f05ddafb36b9d50d1ff7fbf67464275 Mon Sep 17 00:00:00 2001 From: Max Reitz Date: Wed, 24 Jul 2019 19:12:36 +0200 Subject: [PATCH 09/17] vhdx: Fix .bdrv_has_zero_init() Fixed VHDX images cannot guarantee to be zero-initialized. If the image has the "fixed" subformat, forward the call to the underlying storage node. Reported-by: Stefano Garzarella Signed-off-by: Max Reitz Message-id: 20190724171239.8764-9-mreitz@redhat.com Reviewed-by: Maxim Levitsky Signed-off-by: Max Reitz --- block/vhdx.c | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/block/vhdx.c b/block/vhdx.c index a02d1c99a76c..6a09d0a55c68 100644 --- a/block/vhdx.c +++ b/block/vhdx.c @@ -2075,6 +2075,30 @@ static int coroutine_fn vhdx_co_check(BlockDriverState *bs, return 0; } +static int vhdx_has_zero_init(BlockDriverState *bs) +{ + BDRVVHDXState *s = bs->opaque; + int state; + + /* + * Check the subformat: Fixed images have all BAT entries present, + * dynamic images have none (right after creation). It is + * therefore enough to check the first BAT entry. + */ + if (!s->bat_entries) { + return 1; + } + + state = s->bat[0] & VHDX_BAT_STATE_BIT_MASK; + if (state == PAYLOAD_BLOCK_FULLY_PRESENT) { + /* Fixed subformat */ + return bdrv_has_zero_init(bs->file->bs); + } + + /* Dynamic subformat */ + return 1; +} + static QemuOptsList vhdx_create_opts = { .name = "vhdx-create-opts", .head = QTAILQ_HEAD_INITIALIZER(vhdx_create_opts.head), @@ -2128,7 +2152,7 @@ static BlockDriver bdrv_vhdx = { .bdrv_co_create_opts = vhdx_co_create_opts, .bdrv_get_info = vhdx_get_info, .bdrv_co_check = vhdx_co_check, - .bdrv_has_zero_init = bdrv_has_zero_init_1, + .bdrv_has_zero_init = vhdx_has_zero_init, .create_opts = &vhdx_create_opts, }; From 5a840549ee68a6c2bbcee4c402793375646bdc92 Mon Sep 17 00:00:00 2001 From: Max Reitz Date: Wed, 24 Jul 2019 19:12:37 +0200 Subject: [PATCH 10/17] iotests: Convert to preallocated encrypted qcow2 Add a test case for converting an empty image (which only returns zeroes when read) to a preallocated encrypted qcow2 image. qcow2_has_zero_init() should return 0 then, thus forcing qemu-img convert to create zero clusters. Signed-off-by: Max Reitz Acked-by: Stefano Garzarella Tested-by: Stefano Garzarella Message-id: 20190724171239.8764-10-mreitz@redhat.com Reviewed-by: Maxim Levitsky Signed-off-by: Max Reitz --- tests/qemu-iotests/188 | 20 +++++++++++++++++++- tests/qemu-iotests/188.out | 4 ++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/tests/qemu-iotests/188 b/tests/qemu-iotests/188 index be7278aa652d..afca44df5427 100755 --- a/tests/qemu-iotests/188 +++ b/tests/qemu-iotests/188 @@ -48,7 +48,7 @@ SECRETALT="secret,id=sec0,data=platypus" _make_test_img --object $SECRET -o "encrypt.format=luks,encrypt.key-secret=sec0,encrypt.iter-time=10" $size -IMGSPEC="driver=$IMGFMT,file.filename=$TEST_IMG,encrypt.key-secret=sec0" +IMGSPEC="driver=$IMGFMT,encrypt.key-secret=sec0,file.filename=$TEST_IMG" QEMU_IO_OPTIONS=$QEMU_IO_OPTIONS_NO_FMT @@ -68,6 +68,24 @@ echo echo "== verify open failure with wrong password ==" $QEMU_IO --object $SECRETALT -c "read -P 0xa 0 $size" --image-opts $IMGSPEC | _filter_qemu_io | _filter_testdir +_cleanup_test_img + +echo +echo "== verify that has_zero_init returns false when preallocating ==" + +# Empty source file +if [ -n "$TEST_IMG_FILE" ]; then + TEST_IMG_FILE="${TEST_IMG_FILE}.orig" _make_test_img $size +else + TEST_IMG="${TEST_IMG}.orig" _make_test_img $size +fi + +$QEMU_IMG convert -O "$IMGFMT" --object $SECRET \ + -o "encrypt.format=luks,encrypt.key-secret=sec0,encrypt.iter-time=10,preallocation=metadata" \ + "${TEST_IMG}.orig" "$TEST_IMG" + +$QEMU_IMG compare --object $SECRET --image-opts "${IMGSPEC}.orig" "$IMGSPEC" + # success, all done echo "*** done" diff --git a/tests/qemu-iotests/188.out b/tests/qemu-iotests/188.out index 97b1402671bf..c568ef370145 100644 --- a/tests/qemu-iotests/188.out +++ b/tests/qemu-iotests/188.out @@ -15,4 +15,8 @@ read 16777216/16777216 bytes at offset 0 == verify open failure with wrong password == qemu-io: can't open: Invalid password, cannot unlock any keyslot + +== verify that has_zero_init returns false when preallocating == +Formatting 'TEST_DIR/t.IMGFMT.orig', fmt=IMGFMT size=16777216 +Images are identical. *** done From c2acc95befc2e03d8395dead80024a49819b4a23 Mon Sep 17 00:00:00 2001 From: Max Reitz Date: Wed, 24 Jul 2019 19:12:38 +0200 Subject: [PATCH 11/17] iotests: Test convert -n to pre-filled image Signed-off-by: Max Reitz Message-id: 20190724171239.8764-11-mreitz@redhat.com Reviewed-by: Maxim Levitsky Signed-off-by: Max Reitz --- tests/qemu-iotests/122 | 17 +++++++++++++++++ tests/qemu-iotests/122.out | 8 ++++++++ 2 files changed, 25 insertions(+) diff --git a/tests/qemu-iotests/122 b/tests/qemu-iotests/122 index 85c3a8d04729..059011ebb1ec 100755 --- a/tests/qemu-iotests/122 +++ b/tests/qemu-iotests/122 @@ -257,6 +257,23 @@ for min_sparse in 4k 8k; do $QEMU_IMG map --output=json "$TEST_IMG".orig | _filter_qemu_img_map done + +echo +echo '=== -n to a non-zero image ===' +echo + +# Keep source zero +_make_test_img 64M + +# Output is not zero, but has bdrv_has_zero_init() == 1 +TEST_IMG="$TEST_IMG".orig _make_test_img 64M +$QEMU_IO -c "write -P 42 0 64k" "$TEST_IMG".orig | _filter_qemu_io + +# Convert with -n, which should not assume that the target is zeroed +$QEMU_IMG convert -O $IMGFMT -n "$TEST_IMG" "$TEST_IMG".orig + +$QEMU_IMG compare "$TEST_IMG" "$TEST_IMG".orig + # success, all done echo '*** done' rm -f $seq.full diff --git a/tests/qemu-iotests/122.out b/tests/qemu-iotests/122.out index c576705284e7..849b6cc2efd3 100644 --- a/tests/qemu-iotests/122.out +++ b/tests/qemu-iotests/122.out @@ -220,4 +220,12 @@ convert -c -S 8k { "start": 9216, "length": 8192, "depth": 0, "zero": true, "data": false}, { "start": 17408, "length": 1024, "depth": 0, "zero": false, "data": true}, { "start": 18432, "length": 67090432, "depth": 0, "zero": true, "data": false}] + +=== -n to a non-zero image === + +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 +Formatting 'TEST_DIR/t.IMGFMT.orig', fmt=IMGFMT size=67108864 +wrote 65536/65536 bytes at offset 0 +64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +Images are identical. *** done From 9463ee1f5f4e0c496f2f1f3ae641b650f4836627 Mon Sep 17 00:00:00 2001 From: Max Reitz Date: Wed, 24 Jul 2019 19:12:39 +0200 Subject: [PATCH 12/17] iotests: Full mirror to existing non-zero image The result of a sync=full mirror should always be the equal to the input. Therefore, existing images should be treated as potentially non-zero and thus should be explicitly initialized to be zero beforehand. Signed-off-by: Max Reitz Message-id: 20190724171239.8764-12-mreitz@redhat.com Signed-off-by: Max Reitz --- tests/qemu-iotests/041 | 62 +++++++++++++++++++++++++++++++++++--- tests/qemu-iotests/041.out | 4 +-- 2 files changed, 60 insertions(+), 6 deletions(-) diff --git a/tests/qemu-iotests/041 b/tests/qemu-iotests/041 index 26bf1701ebd6..8bc8f81db775 100755 --- a/tests/qemu-iotests/041 +++ b/tests/qemu-iotests/041 @@ -741,8 +741,15 @@ class TestUnbackedSource(iotests.QMPTestCase): def setUp(self): qemu_img('create', '-f', iotests.imgfmt, test_img, str(TestUnbackedSource.image_len)) - self.vm = iotests.VM().add_drive(test_img) + self.vm = iotests.VM() self.vm.launch() + result = self.vm.qmp('blockdev-add', node_name='drive0', + driver=iotests.imgfmt, + file={ + 'driver': 'file', + 'filename': test_img, + }) + self.assert_qmp(result, 'return', {}) def tearDown(self): self.vm.shutdown() @@ -751,7 +758,7 @@ class TestUnbackedSource(iotests.QMPTestCase): def test_absolute_paths_full(self): self.assert_no_active_block_jobs() - result = self.vm.qmp('drive-mirror', device='drive0', + result = self.vm.qmp('drive-mirror', job_id='drive0', device='drive0', sync='full', target=target_img, mode='absolute-paths') self.assert_qmp(result, 'return', {}) @@ -760,7 +767,7 @@ class TestUnbackedSource(iotests.QMPTestCase): def test_absolute_paths_top(self): self.assert_no_active_block_jobs() - result = self.vm.qmp('drive-mirror', device='drive0', + result = self.vm.qmp('drive-mirror', job_id='drive0', device='drive0', sync='top', target=target_img, mode='absolute-paths') self.assert_qmp(result, 'return', {}) @@ -769,13 +776,60 @@ class TestUnbackedSource(iotests.QMPTestCase): def test_absolute_paths_none(self): self.assert_no_active_block_jobs() - result = self.vm.qmp('drive-mirror', device='drive0', + result = self.vm.qmp('drive-mirror', job_id='drive0', device='drive0', sync='none', target=target_img, mode='absolute-paths') self.assert_qmp(result, 'return', {}) self.complete_and_wait() self.assert_no_active_block_jobs() + def test_existing_full(self): + qemu_img('create', '-f', iotests.imgfmt, target_img, + str(self.image_len)) + qemu_io('-c', 'write -P 42 0 64k', target_img) + + self.assert_no_active_block_jobs() + result = self.vm.qmp('drive-mirror', job_id='drive0', device='drive0', + sync='full', target=target_img, mode='existing') + self.assert_qmp(result, 'return', {}) + self.complete_and_wait() + self.assert_no_active_block_jobs() + + result = self.vm.qmp('blockdev-del', node_name='drive0') + self.assert_qmp(result, 'return', {}) + + self.assertTrue(iotests.compare_images(test_img, target_img), + 'target image does not match source after mirroring') + + def test_blockdev_full(self): + qemu_img('create', '-f', iotests.imgfmt, target_img, + str(self.image_len)) + qemu_io('-c', 'write -P 42 0 64k', target_img) + + result = self.vm.qmp('blockdev-add', node_name='target', + driver=iotests.imgfmt, + file={ + 'driver': 'file', + 'filename': target_img, + }) + self.assert_qmp(result, 'return', {}) + + self.assert_no_active_block_jobs() + result = self.vm.qmp('blockdev-mirror', job_id='drive0', device='drive0', + sync='full', target='target') + self.assert_qmp(result, 'return', {}) + self.complete_and_wait() + self.assert_no_active_block_jobs() + + result = self.vm.qmp('blockdev-del', node_name='drive0') + self.assert_qmp(result, 'return', {}) + + result = self.vm.qmp('blockdev-del', node_name='target') + self.assert_qmp(result, 'return', {}) + + self.assertTrue(iotests.compare_images(test_img, target_img), + 'target image does not match source after mirroring') + class TestGranularity(iotests.QMPTestCase): image_len = 10 * 1024 * 1024 # MB diff --git a/tests/qemu-iotests/041.out b/tests/qemu-iotests/041.out index e071d0b26119..2c448b4239f7 100644 --- a/tests/qemu-iotests/041.out +++ b/tests/qemu-iotests/041.out @@ -1,5 +1,5 @@ -........................................................................................ +.......................................................................................... ---------------------------------------------------------------------- -Ran 88 tests +Ran 90 tests OK From ad6434dc620958503a0ad2ab92269805c26bb806 Mon Sep 17 00:00:00 2001 From: Max Reitz Date: Thu, 25 Jul 2019 17:55:10 +0200 Subject: [PATCH 13/17] vdi: Make block_status recurse for fixed images Suggested-by: Vladimir Sementsov-Ogievskiy Fixes: 69f47505ee66afaa513305de0c1895a224e52c45 Signed-off-by: Max Reitz Message-id: 20190725155512.9827-2-mreitz@redhat.com Reviewed-by: Vladimir Sementsov-Ogievskiy Reviewed-by: John Snow Signed-off-by: Max Reitz --- block/vdi.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/block/vdi.c b/block/vdi.c index 0caa3f281d26..806ba7f53c7b 100644 --- a/block/vdi.c +++ b/block/vdi.c @@ -542,7 +542,8 @@ static int coroutine_fn vdi_co_block_status(BlockDriverState *bs, *map = s->header.offset_data + (uint64_t)bmap_entry * s->block_size + index_in_block; *file = bs->file->bs; - return BDRV_BLOCK_DATA | BDRV_BLOCK_OFFSET_VALID; + return BDRV_BLOCK_DATA | BDRV_BLOCK_OFFSET_VALID | + (s->header.image_type == VDI_TYPE_STATIC ? BDRV_BLOCK_RECURSE : 0); } static int coroutine_fn From 4dd84ac9a731f19576aec14d0f2f0037e81a5922 Mon Sep 17 00:00:00 2001 From: Max Reitz Date: Thu, 25 Jul 2019 17:55:11 +0200 Subject: [PATCH 14/17] vmdk: Make block_status recurse for flat extents Fixes: 69f47505ee66afaa513305de0c1895a224e52c45 Signed-off-by: Max Reitz Message-id: 20190725155512.9827-3-mreitz@redhat.com Reviewed-by: Vladimir Sementsov-Ogievskiy Reviewed-by: John Snow Signed-off-by: Max Reitz --- block/vmdk.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/block/vmdk.c b/block/vmdk.c index bd36ece125a6..fd78fd0ccf67 100644 --- a/block/vmdk.c +++ b/block/vmdk.c @@ -1692,6 +1692,9 @@ static int coroutine_fn vmdk_co_block_status(BlockDriverState *bs, if (!extent->compressed) { ret |= BDRV_BLOCK_OFFSET_VALID; *map = cluster_offset + index_in_cluster; + if (extent->flat) { + ret |= BDRV_BLOCK_RECURSE; + } } *file = extent->file->bs; break; From fbc8e1b7e49dee9b137c1037dc72d8e6fd4aba3d Mon Sep 17 00:00:00 2001 From: Max Reitz Date: Thu, 25 Jul 2019 17:55:12 +0200 Subject: [PATCH 15/17] vpc: Do not return RAW from block_status vpc is not really a passthrough driver, even when using the fixed subformat (where host and guest offsets are equal). It should handle preallocation like all other drivers do, namely by returning DATA | RECURSE instead of RAW. There is no tangible difference but the fact that bdrv_is_allocated() no longer falls through to the protocol layer. Signed-off-by: Max Reitz Message-id: 20190725155512.9827-4-mreitz@redhat.com Reviewed-by: John Snow Signed-off-by: Max Reitz --- block/vpc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/block/vpc.c b/block/vpc.c index d4776ee8a522..b25aab042544 100644 --- a/block/vpc.c +++ b/block/vpc.c @@ -737,7 +737,7 @@ static int coroutine_fn vpc_co_block_status(BlockDriverState *bs, *pnum = bytes; *map = offset; *file = bs->file->bs; - return BDRV_BLOCK_RAW | BDRV_BLOCK_OFFSET_VALID; + return BDRV_BLOCK_DATA | BDRV_BLOCK_OFFSET_VALID | BDRV_BLOCK_RECURSE; } qemu_co_mutex_lock(&s->lock); From 9c46f4a06d9d57bc9f357cec3e37de94e6068413 Mon Sep 17 00:00:00 2001 From: Max Reitz Date: Fri, 9 Aug 2019 20:52:53 +0200 Subject: [PATCH 16/17] iotests: Fix 141 when run with qed 69f47505ee has changed qcow2 in such a way that the commit job run in test 141 (and 144[1]) returns before it emits the READY event. However, 141 also runs with qed, where the order is still the other way around. Just filter out the {"return": {}} so the test passes for qed again. [1] 144 only runs with qcow2, so it is fine as it is. Suggested-by: Vladimir Sementsov-Ogievskiy Fixes: 69f47505ee66afaa513305de0c1895a224e52c45 Signed-off-by: Max Reitz Message-id: 20190809185253.17535-1-mreitz@redhat.com Reviewed-by: Vladimir Sementsov-Ogievskiy Reviewed-by: John Snow Signed-off-by: Max Reitz --- tests/qemu-iotests/141 | 9 +++++++-- tests/qemu-iotests/141.out | 5 ----- tests/qemu-iotests/common.filter | 5 +++++ 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/qemu-iotests/141 b/tests/qemu-iotests/141 index 2197a82d454e..8c2ae79f2b95 100755 --- a/tests/qemu-iotests/141 +++ b/tests/qemu-iotests/141 @@ -58,16 +58,21 @@ test_blockjob() }}}" \ 'return' + # If "$2" is an event, we may or may not see it before the + # {"return": {}}. Therefore, filter the {"return": {}} out both + # here and in the next command. (Naturally, if we do not see it + # here, we will see it before the next command can be executed, + # so it will appear in the next _send_qemu_cmd's output.) _send_qemu_cmd $QEMU_HANDLE \ "$1" \ "$2" \ - | _filter_img_create + | _filter_img_create | _filter_qmp_empty_return # We want this to return an error because the block job is still running _send_qemu_cmd $QEMU_HANDLE \ "{'execute': 'blockdev-del', 'arguments': {'node-name': 'drv0'}}" \ - 'error' | _filter_generated_node_ids + 'error' | _filter_generated_node_ids | _filter_qmp_empty_return _send_qemu_cmd $QEMU_HANDLE \ "{'execute': 'block-job-cancel', diff --git a/tests/qemu-iotests/141.out b/tests/qemu-iotests/141.out index 4d71d9dcae88..dbd3bdef6c19 100644 --- a/tests/qemu-iotests/141.out +++ b/tests/qemu-iotests/141.out @@ -10,7 +10,6 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576 backing_file=TEST_DIR/m. Formatting 'TEST_DIR/o.IMGFMT', fmt=IMGFMT size=1048576 backing_file=TEST_DIR/t.IMGFMT backing_fmt=IMGFMT {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "created", "id": "job0"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "running", "id": "job0"}} -{"return": {}} {"error": {"class": "GenericError", "desc": "Node drv0 is in use"}} {"return": {}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "aborting", "id": "job0"}} @@ -27,7 +26,6 @@ Formatting 'TEST_DIR/o.IMGFMT', fmt=IMGFMT size=1048576 backing_file=TEST_DIR/t. {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "running", "id": "job0"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "ready", "id": "job0"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "job0", "len": 0, "offset": 0, "speed": 0, "type": "mirror"}} -{"return": {}} {"error": {"class": "GenericError", "desc": "Node 'drv0' is busy: block device is in use by block job: mirror"}} {"return": {}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "waiting", "id": "job0"}} @@ -42,7 +40,6 @@ Formatting 'TEST_DIR/o.IMGFMT', fmt=IMGFMT size=1048576 backing_file=TEST_DIR/t. {"return": {}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "created", "id": "job0"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "running", "id": "job0"}} -{"return": {}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "ready", "id": "job0"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "job0", "len": 0, "offset": 0, "speed": 0, "type": "commit"}} {"error": {"class": "GenericError", "desc": "Node 'drv0' is busy: block device is in use by block job: commit"}} @@ -61,7 +58,6 @@ wrote 1048576/1048576 bytes at offset 0 {"return": {}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "created", "id": "job0"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "running", "id": "job0"}} -{"return": {}} {"error": {"class": "GenericError", "desc": "Node drv0 is in use"}} {"return": {}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "aborting", "id": "job0"}} @@ -77,7 +73,6 @@ wrote 1048576/1048576 bytes at offset 0 {"return": {}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "created", "id": "job0"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "running", "id": "job0"}} -{"return": {}} {"error": {"class": "GenericError", "desc": "Node drv0 is in use"}} {"return": {}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "aborting", "id": "job0"}} diff --git a/tests/qemu-iotests/common.filter b/tests/qemu-iotests/common.filter index 35fddc746f5b..8e9235d6fe0c 100644 --- a/tests/qemu-iotests/common.filter +++ b/tests/qemu-iotests/common.filter @@ -219,5 +219,10 @@ _filter_nbd() -e 's#\(foo\|PORT/\?\|.sock\): Failed to .*$#\1#' } +_filter_qmp_empty_return() +{ + grep -v '{"return": {}}' +} + # make sure this script returns success true From fa27c478102a6b5d1c6b02c005607ad9404b915f Mon Sep 17 00:00:00 2001 From: Max Reitz Date: Thu, 11 Jul 2019 15:29:35 +0200 Subject: [PATCH 17/17] doc: Preallocation does not require writing zeroes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When preallocating an encrypted qcow2 image, it just lets the protocol driver write data and then does not mark the clusters as zero. Therefore, reading this image will yield effectively random data. As such, we have not fulfilled the promise of always writing zeroes when preallocating an image in a while. It seems that nobody has really cared, so change the documentation to conform to qemu's actual behavior. Signed-off-by: Max Reitz Message-id: 20190711132935.13070-1-mreitz@redhat.com Reviewed-by: Eric Blake Reviewed-by: Daniel P. Berrangé Reviewed-by: Maxim Levitsky Signed-off-by: Max Reitz --- docs/qemu-block-drivers.texi | 4 ++-- qapi/block-core.json | 9 +++++---- qemu-img.texi | 4 ++-- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/qemu-block-drivers.texi b/docs/qemu-block-drivers.texi index 91ab0eceae2e..c02547e28c62 100644 --- a/docs/qemu-block-drivers.texi +++ b/docs/qemu-block-drivers.texi @@ -31,8 +31,8 @@ Supported options: @item preallocation Preallocation mode (allowed values: @code{off}, @code{falloc}, @code{full}). @code{falloc} mode preallocates space for image by calling posix_fallocate(). -@code{full} mode preallocates space for image by writing zeros to underlying -storage. +@code{full} mode preallocates space for image by writing data to underlying +storage. This data may or may not be zero, depending on the storage location. @end table @item qcow2 diff --git a/qapi/block-core.json b/qapi/block-core.json index a5ab38db99bc..e6edd641f18a 100644 --- a/qapi/block-core.json +++ b/qapi/block-core.json @@ -5178,10 +5178,11 @@ # @off: no preallocation # @metadata: preallocate only for metadata # @falloc: like @full preallocation but allocate disk space by -# posix_fallocate() rather than writing zeros. -# @full: preallocate all data by writing zeros to device to ensure disk -# space is really available. @full preallocation also sets up -# metadata correctly. +# posix_fallocate() rather than writing data. +# @full: preallocate all data by writing it to the device to ensure +# disk space is really available. This data may or may not be +# zero, depending on the image format and storage. +# @full preallocation also sets up metadata correctly. # # Since: 2.2 ## diff --git a/qemu-img.texi b/qemu-img.texi index c8e9bba51578..b5156d631681 100644 --- a/qemu-img.texi +++ b/qemu-img.texi @@ -666,8 +666,8 @@ Supported options: @item preallocation Preallocation mode (allowed values: @code{off}, @code{falloc}, @code{full}). @code{falloc} mode preallocates space for image by calling posix_fallocate(). -@code{full} mode preallocates space for image by writing zeros to underlying -storage. +@code{full} mode preallocates space for image by writing data to underlying +storage. This data may or may not be zero, depending on the storage location. @end table @item qcow2