From 6caaad46de02c02fb1eeaae64fcdc74f9e0a5eb0 Mon Sep 17 00:00:00 2001 From: Peter Lieven Date: Tue, 10 Sep 2019 17:26:22 +0200 Subject: [PATCH 01/15] block/vhdx: add check for truncated image files qemu is currently not able to detect truncated vhdx image files. Add a basic check if all allocated blocks are reachable at open and report all errors during bdrv_co_check. Signed-off-by: Peter Lieven Signed-off-by: Kevin Wolf --- block/vhdx.c | 120 +++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 103 insertions(+), 17 deletions(-) diff --git a/block/vhdx.c b/block/vhdx.c index 6a09d0a55c68..371f226286ce 100644 --- a/block/vhdx.c +++ b/block/vhdx.c @@ -24,6 +24,7 @@ #include "qemu/option.h" #include "qemu/crc32c.h" #include "qemu/bswap.h" +#include "qemu/error-report.h" #include "vhdx.h" #include "migration/blocker.h" #include "qemu/uuid.h" @@ -235,6 +236,9 @@ static int vhdx_region_check(BDRVVHDXState *s, uint64_t start, uint64_t length) end = start + length; QLIST_FOREACH(r, &s->regions, entries) { if (!((start >= r->end) || (end <= r->start))) { + error_report("VHDX region %" PRIu64 "-%" PRIu64 " overlaps with " + "region %" PRIu64 "-%." PRIu64, start, end, r->start, + r->end); ret = -EINVAL; goto exit; } @@ -877,6 +881,95 @@ static void vhdx_calc_bat_entries(BDRVVHDXState *s) } +static int vhdx_check_bat_entries(BlockDriverState *bs, int *errcnt) +{ + BDRVVHDXState *s = bs->opaque; + int64_t image_file_size = bdrv_getlength(bs->file->bs); + uint64_t payblocks = s->chunk_ratio; + uint64_t i; + int ret = 0; + + if (image_file_size < 0) { + error_report("Could not determinate VHDX image file size."); + return image_file_size; + } + + for (i = 0; i < s->bat_entries; i++) { + if ((s->bat[i] & VHDX_BAT_STATE_BIT_MASK) == + PAYLOAD_BLOCK_FULLY_PRESENT) { + uint64_t offset = s->bat[i] & VHDX_BAT_FILE_OFF_MASK; + /* + * Allow that the last block exists only partially. The VHDX spec + * states that the image file can only grow in blocksize increments, + * but QEMU created images with partial last blocks in the past. + */ + uint32_t block_length = MIN(s->block_size, + bs->total_sectors * BDRV_SECTOR_SIZE - i * s->block_size); + /* + * Check for BAT entry overflow. + */ + if (offset > INT64_MAX - s->block_size) { + error_report("VHDX BAT entry %" PRIu64 " offset overflow.", i); + ret = -EINVAL; + if (!errcnt) { + break; + } + (*errcnt)++; + } + /* + * Check if fully allocated BAT entries do not reside after + * end of the image file. + */ + if (offset >= image_file_size) { + error_report("VHDX BAT entry %" PRIu64 " start offset %" PRIu64 + " points after end of file (%" PRIi64 "). Image" + " has probably been truncated.", + i, offset, image_file_size); + ret = -EINVAL; + if (!errcnt) { + break; + } + (*errcnt)++; + } else if (offset + block_length > image_file_size) { + error_report("VHDX BAT entry %" PRIu64 " end offset %" PRIu64 + " points after end of file (%" PRIi64 "). Image" + " has probably been truncated.", + i, offset + block_length - 1, image_file_size); + ret = -EINVAL; + if (!errcnt) { + break; + } + (*errcnt)++; + } + + /* + * verify populated BAT field file offsets against + * region table and log entries + */ + if (payblocks--) { + /* payload bat entries */ + int ret2; + ret2 = vhdx_region_check(s, offset, s->block_size); + if (ret2 < 0) { + ret = -EINVAL; + if (!errcnt) { + break; + } + (*errcnt)++; + } + } else { + payblocks = s->chunk_ratio; + /* + * Once differencing files are supported, verify sector bitmap + * blocks here + */ + } + } + } + + return ret; +} + static void vhdx_close(BlockDriverState *bs) { BDRVVHDXState *s = bs->opaque; @@ -981,25 +1074,15 @@ static int vhdx_open(BlockDriverState *bs, QDict *options, int flags, goto fail; } - uint64_t payblocks = s->chunk_ratio; - /* endian convert, and verify populated BAT field file offsets against - * region table and log entries */ + /* endian convert populated BAT field entires */ for (i = 0; i < s->bat_entries; i++) { s->bat[i] = le64_to_cpu(s->bat[i]); - if (payblocks--) { - /* payload bat entries */ - if ((s->bat[i] & VHDX_BAT_STATE_BIT_MASK) == - PAYLOAD_BLOCK_FULLY_PRESENT) { - ret = vhdx_region_check(s, s->bat[i] & VHDX_BAT_FILE_OFF_MASK, - s->block_size); - if (ret < 0) { - goto fail; - } - } - } else { - payblocks = s->chunk_ratio; - /* Once differencing files are supported, verify sector bitmap - * blocks here */ + } + + if (!(flags & BDRV_O_CHECK)) { + ret = vhdx_check_bat_entries(bs, NULL); + if (ret < 0) { + goto fail; } } @@ -2072,6 +2155,9 @@ static int coroutine_fn vhdx_co_check(BlockDriverState *bs, if (s->log_replayed_on_open) { result->corruptions_fixed++; } + + vhdx_check_bat_entries(bs, &result->corruptions); + return 0; } From 3c6c4348f23131155846dc53ce19aca309796fda Mon Sep 17 00:00:00 2001 From: Pavel Dovgalyuk Date: Tue, 17 Sep 2019 14:57:51 +0300 Subject: [PATCH 02/15] block: implement bdrv_snapshot_goto for blkreplay This patch enables making snapshots with blkreplay used in block devices. This function is required to make bdrv_snapshot_goto without calling .bdrv_open which is not implemented. Signed-off-by: Pavel Dovgalyuk Acked-by: Kevin Wolf Signed-off-by: Kevin Wolf --- block/blkreplay.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/block/blkreplay.c b/block/blkreplay.c index 2b7931b94014..c96ac8f4bc4d 100644 --- a/block/blkreplay.c +++ b/block/blkreplay.c @@ -126,6 +126,12 @@ static int coroutine_fn blkreplay_co_flush(BlockDriverState *bs) return ret; } +static int blkreplay_snapshot_goto(BlockDriverState *bs, + const char *snapshot_id) +{ + return bdrv_snapshot_goto(bs->file->bs, snapshot_id, NULL); +} + static BlockDriver bdrv_blkreplay = { .format_name = "blkreplay", .instance_size = 0, @@ -140,6 +146,8 @@ static BlockDriver bdrv_blkreplay = { .bdrv_co_pwrite_zeroes = blkreplay_co_pwrite_zeroes, .bdrv_co_pdiscard = blkreplay_co_pdiscard, .bdrv_co_flush = blkreplay_co_flush, + + .bdrv_snapshot_goto = blkreplay_snapshot_goto, }; static void bdrv_blkreplay_init(void) From 258639754461e39d307ce3d340d213e0e5f339e3 Mon Sep 17 00:00:00 2001 From: Pavel Dovgalyuk Date: Tue, 17 Sep 2019 14:57:56 +0300 Subject: [PATCH 03/15] replay: disable default snapshot for record/replay This patch disables setting '-snapshot' option on by default in record/replay mode. This is needed for creating vmstates in record and replay modes. Signed-off-by: Pavel Dovgalyuk Acked-by: Kevin Wolf Signed-off-by: Kevin Wolf --- vl.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/vl.c b/vl.c index 0a295e5d77d6..fce9ce2364e3 100644 --- a/vl.c +++ b/vl.c @@ -1203,7 +1203,7 @@ static void configure_blockdev(BlockdevOptionsQueue *bdo_queue, qapi_free_BlockdevOptions(bdo->bdo); g_free(bdo); } - if (snapshot || replay_mode != REPLAY_MODE_NONE) { + if (snapshot) { qemu_opts_foreach(qemu_find_opts("drive"), drive_enable_snapshot, NULL, NULL); } @@ -3066,7 +3066,13 @@ int main(int argc, char **argv, char **envp) drive_add(IF_PFLASH, -1, optarg, PFLASH_OPTS); break; case QEMU_OPTION_snapshot: - snapshot = 1; + { + Error *blocker = NULL; + snapshot = 1; + error_setg(&blocker, QERR_REPLAY_NOT_SUPPORTED, + "-snapshot"); + replay_add_blocker(blocker); + } break; case QEMU_OPTION_numa: opts = qemu_opts_parse_noisily(qemu_find_opts("numa"), From de499eb67cda5c852605fe365950844412cbe789 Mon Sep 17 00:00:00 2001 From: Pavel Dovgalyuk Date: Tue, 17 Sep 2019 14:58:02 +0300 Subject: [PATCH 04/15] replay: update docs for record/replay with block devices This patch updates the description of the command lines for using record/replay with attached block devices. Signed-off-by: Pavel Dovgalyuk Signed-off-by: Kevin Wolf --- docs/replay.txt | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/docs/replay.txt b/docs/replay.txt index ee6aee9861e7..ce97c3f72fc8 100644 --- a/docs/replay.txt +++ b/docs/replay.txt @@ -27,7 +27,7 @@ Usage of the record/replay: * First, record the execution with the following command line: qemu-system-i386 \ -icount shift=7,rr=record,rrfile=replay.bin \ - -drive file=disk.qcow2,if=none,id=img-direct \ + -drive file=disk.qcow2,if=none,snapshot,id=img-direct \ -drive driver=blkreplay,if=none,image=img-direct,id=img-blkreplay \ -device ide-hd,drive=img-blkreplay \ -netdev user,id=net1 -device rtl8139,netdev=net1 \ @@ -35,7 +35,7 @@ Usage of the record/replay: * After recording, you can replay it by using another command line: qemu-system-i386 \ -icount shift=7,rr=replay,rrfile=replay.bin \ - -drive file=disk.qcow2,if=none,id=img-direct \ + -drive file=disk.qcow2,if=none,snapshot,id=img-direct \ -drive driver=blkreplay,if=none,image=img-direct,id=img-blkreplay \ -device ide-hd,drive=img-blkreplay \ -netdev user,id=net1 -device rtl8139,netdev=net1 \ @@ -223,7 +223,7 @@ Block devices record/replay module intercepts calls of bdrv coroutine functions at the top of block drivers stack. To record and replay block operations the drive must be configured as following: - -drive file=disk.qcow2,if=none,id=img-direct + -drive file=disk.qcow2,if=none,snapshot,id=img-direct -drive driver=blkreplay,if=none,image=img-direct,id=img-blkreplay -device ide-hd,drive=img-blkreplay @@ -252,6 +252,12 @@ This snapshot is created at start of recording and restored at start of replaying. It also can be loaded while replaying to roll back the execution. +'snapshot' flag of the disk image must be removed to save the snapshots +in the overlay (or original image) instead of using the temporary overlay. + -drive file=disk.ovl,if=none,id=img-direct + -drive driver=blkreplay,if=none,image=img-direct,id=img-blkreplay + -device ide-hd,drive=img-blkreplay + Use QEMU monitor to create additional snapshots. 'savevm ' command created the snapshot and 'loadvm ' restores it. To prevent corruption of the original disk image, use overlay files linked to the original images. From c8aa7895eb536e1fc9ef2740e4e32bb9ec6f3afa Mon Sep 17 00:00:00 2001 From: Pavel Dovgalyuk Date: Tue, 17 Sep 2019 14:58:08 +0300 Subject: [PATCH 05/15] replay: don't drain/flush bdrv queue while RR is working In record/replay mode bdrv queue is controlled by replay mechanism. It does not allow saving or loading the snapshots when bdrv queue is not empty. Stopping the VM is not blocked by nonempty queue, but flushing the queue is still impossible there, because it may cause deadlocks in replay mode. This patch disables bdrv_drain_all and bdrv_flush_all in record/replay mode. Stopping the machine when the IO requests are not finished is needed for the debugging. E.g., breakpoint may be set at the specified step, and forcing the IO requests to finish may break the determinism of the execution. Signed-off-by: Pavel Dovgalyuk Acked-by: Kevin Wolf Signed-off-by: Kevin Wolf --- block/io.c | 28 ++++++++++++++++++++++++++++ cpus.c | 2 -- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/block/io.c b/block/io.c index 4f9ee97c2bd5..834841142a25 100644 --- a/block/io.c +++ b/block/io.c @@ -33,6 +33,7 @@ #include "qapi/error.h" #include "qemu/error-report.h" #include "qemu/main-loop.h" +#include "sysemu/replay.h" #define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */ @@ -600,6 +601,15 @@ void bdrv_drain_all_begin(void) return; } + /* + * bdrv queue is managed by record/replay, + * waiting for finishing the I/O requests may + * be infinite + */ + if (replay_events_enabled()) { + return; + } + /* AIO_WAIT_WHILE() with a NULL context can only be called from the main * loop AioContext, so make sure we're in the main context. */ assert(qemu_get_current_aio_context() == qemu_get_aio_context()); @@ -629,6 +639,15 @@ void bdrv_drain_all_end(void) BlockDriverState *bs = NULL; int drained_end_counter = 0; + /* + * bdrv queue is managed by record/replay, + * waiting for finishing the I/O requests may + * be endless + */ + if (replay_events_enabled()) { + return; + } + while ((bs = bdrv_next_all_states(bs))) { AioContext *aio_context = bdrv_get_aio_context(bs); @@ -2124,6 +2143,15 @@ int bdrv_flush_all(void) BlockDriverState *bs = NULL; int result = 0; + /* + * bdrv queue is managed by record/replay, + * creating new flush request for stopping + * the VM may break the determinism + */ + if (replay_events_enabled()) { + return result; + } + for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) { AioContext *aio_context = bdrv_get_aio_context(bs); int ret; diff --git a/cpus.c b/cpus.c index d2c61ff1558a..367f0657c5d9 100644 --- a/cpus.c +++ b/cpus.c @@ -1097,7 +1097,6 @@ static int do_vm_stop(RunState state, bool send_stop) } bdrv_drain_all(); - replay_disable_events(); ret = bdrv_flush_all(); return ret; @@ -2181,7 +2180,6 @@ int vm_prepare_start(void) /* We are sending this now, but the CPUs will be resumed shortly later */ qapi_event_send_resume(); - replay_enable_events(); cpu_enable_ticks(); runstate_set(RUN_STATE_RUNNING); vm_state_notify(1, RUN_STATE_RUNNING); From ae25dccb7df64ec9cbb52d7a0a67308d74372cfd Mon Sep 17 00:00:00 2001 From: Pavel Dovgalyuk Date: Tue, 17 Sep 2019 14:58:13 +0300 Subject: [PATCH 06/15] replay: finish record/replay before closing the disks After recent updates block devices cannot be closed on qemu exit. This happens due to the block request polling when replay is not finished. Therefore now we stop execution recording before closing the block devices. Signed-off-by: Pavel Dovgalyuk Signed-off-by: Kevin Wolf --- replay/replay.c | 2 ++ vl.c | 1 + 2 files changed, 3 insertions(+) diff --git a/replay/replay.c b/replay/replay.c index 713395b33d9a..5cc25bd2f8c3 100644 --- a/replay/replay.c +++ b/replay/replay.c @@ -385,6 +385,8 @@ void replay_finish(void) g_free(replay_snapshot); replay_snapshot = NULL; + replay_mode = REPLAY_MODE_NONE; + replay_finish_events(); } diff --git a/vl.c b/vl.c index fce9ce2364e3..57d4fd726f69 100644 --- a/vl.c +++ b/vl.c @@ -4524,6 +4524,7 @@ int main(int argc, char **argv, char **envp) /* No more vcpu or device emulation activity beyond this point */ vm_shutdown(); + replay_finish(); job_cancel_sync_all(); bdrv_close_all(); From e4ec5ad464e48ab8d978b4dd8aacd05c1c4a87cc Mon Sep 17 00:00:00 2001 From: Pavel Dovgalyuk Date: Tue, 17 Sep 2019 14:58:19 +0300 Subject: [PATCH 07/15] replay: add BH oneshot event for block layer Replay is capable of recording normal BH events, but sometimes there are single use callbacks scheduled with aio_bh_schedule_oneshot function. This patch enables recording and replaying such callbacks. Block layer uses these events for calling the completion function. Replaying these calls makes the execution deterministic. Signed-off-by: Pavel Dovgalyuk Acked-by: Kevin Wolf Signed-off-by: Kevin Wolf --- block/block-backend.c | 9 ++++++--- block/io.c | 4 ++-- block/iscsi.c | 5 +++-- block/nfs.c | 6 ++++-- block/null.c | 4 +++- block/nvme.c | 6 ++++-- block/rbd.c | 5 +++-- block/vxhs.c | 5 +++-- include/sysemu/replay.h | 4 ++++ replay/replay-events.c | 16 ++++++++++++++++ replay/replay-internal.h | 1 + stubs/Makefile.objs | 1 + stubs/replay-user.c | 9 +++++++++ 13 files changed, 59 insertions(+), 16 deletions(-) create mode 100644 stubs/replay-user.c diff --git a/block/block-backend.c b/block/block-backend.c index 1c605d54446c..eb22ff306e90 100644 --- a/block/block-backend.c +++ b/block/block-backend.c @@ -18,6 +18,8 @@ #include "hw/qdev-core.h" #include "sysemu/blockdev.h" #include "sysemu/runstate.h" +#include "sysemu/sysemu.h" +#include "sysemu/replay.h" #include "qapi/error.h" #include "qapi/qapi-events-block.h" #include "qemu/id.h" @@ -1306,7 +1308,8 @@ BlockAIOCB *blk_abort_aio_request(BlockBackend *blk, acb->blk = blk; acb->ret = ret; - aio_bh_schedule_oneshot(blk_get_aio_context(blk), error_callback_bh, acb); + replay_bh_schedule_oneshot_event(blk_get_aio_context(blk), + error_callback_bh, acb); return &acb->common; } @@ -1362,8 +1365,8 @@ static BlockAIOCB *blk_aio_prwv(BlockBackend *blk, int64_t offset, int bytes, acb->has_returned = true; if (acb->rwco.ret != NOT_DONE) { - aio_bh_schedule_oneshot(blk_get_aio_context(blk), - blk_aio_complete_bh, acb); + replay_bh_schedule_oneshot_event(blk_get_aio_context(blk), + blk_aio_complete_bh, acb); } return &acb->common; diff --git a/block/io.c b/block/io.c index 834841142a25..8b6dace05641 100644 --- a/block/io.c +++ b/block/io.c @@ -369,8 +369,8 @@ static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs, if (bs) { bdrv_inc_in_flight(bs); } - aio_bh_schedule_oneshot(bdrv_get_aio_context(bs), - bdrv_co_drain_bh_cb, &data); + replay_bh_schedule_oneshot_event(bdrv_get_aio_context(bs), + bdrv_co_drain_bh_cb, &data); qemu_coroutine_yield(); /* If we are resumed from some other event (such as an aio completion or a diff --git a/block/iscsi.c b/block/iscsi.c index 506bf5f87570..2ced15066ab4 100644 --- a/block/iscsi.c +++ b/block/iscsi.c @@ -40,6 +40,7 @@ #include "qemu/module.h" #include "qemu/option.h" #include "qemu/uuid.h" +#include "sysemu/replay.h" #include "qapi/error.h" #include "qapi/qapi-commands-misc.h" #include "qapi/qmp/qdict.h" @@ -280,8 +281,8 @@ iscsi_co_generic_cb(struct iscsi_context *iscsi, int status, } if (iTask->co) { - aio_bh_schedule_oneshot(iTask->iscsilun->aio_context, - iscsi_co_generic_bh_cb, iTask); + replay_bh_schedule_oneshot_event(iTask->iscsilun->aio_context, + iscsi_co_generic_bh_cb, iTask); } else { iTask->complete = 1; } diff --git a/block/nfs.c b/block/nfs.c index f39acfdb2811..40f23495a029 100644 --- a/block/nfs.c +++ b/block/nfs.c @@ -37,6 +37,8 @@ #include "qemu/option.h" #include "qemu/uri.h" #include "qemu/cutils.h" +#include "sysemu/sysemu.h" +#include "sysemu/replay.h" #include "qapi/qapi-visit-block-core.h" #include "qapi/qmp/qdict.h" #include "qapi/qmp/qstring.h" @@ -257,8 +259,8 @@ nfs_co_generic_cb(int ret, struct nfs_context *nfs, void *data, if (task->ret < 0) { error_report("NFS Error: %s", nfs_get_error(nfs)); } - aio_bh_schedule_oneshot(task->client->aio_context, - nfs_co_generic_bh_cb, task); + replay_bh_schedule_oneshot_event(task->client->aio_context, + nfs_co_generic_bh_cb, task); } static int coroutine_fn nfs_co_preadv(BlockDriverState *bs, uint64_t offset, diff --git a/block/null.c b/block/null.c index 699aa295cb58..15e1d5674643 100644 --- a/block/null.c +++ b/block/null.c @@ -17,6 +17,7 @@ #include "qemu/module.h" #include "qemu/option.h" #include "block/block_int.h" +#include "sysemu/replay.h" #define NULL_OPT_LATENCY "latency-ns" #define NULL_OPT_ZEROES "read-zeroes" @@ -179,7 +180,8 @@ static inline BlockAIOCB *null_aio_common(BlockDriverState *bs, timer_mod_ns(&acb->timer, qemu_clock_get_ns(QEMU_CLOCK_REALTIME) + s->latency_ns); } else { - aio_bh_schedule_oneshot(bdrv_get_aio_context(bs), null_bh_cb, acb); + replay_bh_schedule_oneshot_event(bdrv_get_aio_context(bs), + null_bh_cb, acb); } return &acb->common; } diff --git a/block/nvme.c b/block/nvme.c index 5be3a39b632e..910872ec596e 100644 --- a/block/nvme.c +++ b/block/nvme.c @@ -23,6 +23,7 @@ #include "qemu/option.h" #include "qemu/vfio-helpers.h" #include "block/block_int.h" +#include "sysemu/replay.h" #include "trace.h" #include "block/nvme.h" @@ -351,7 +352,8 @@ static bool nvme_process_completion(BDRVNVMeState *s, NVMeQueuePair *q) smp_mb_release(); *q->cq.doorbell = cpu_to_le32(q->cq.head); if (!qemu_co_queue_empty(&q->free_req_queue)) { - aio_bh_schedule_oneshot(s->aio_context, nvme_free_req_queue_cb, q); + replay_bh_schedule_oneshot_event(s->aio_context, + nvme_free_req_queue_cb, q); } } q->busy = false; @@ -935,7 +937,7 @@ static void nvme_rw_cb(void *opaque, int ret) /* The rw coroutine hasn't yielded, don't try to enter. */ return; } - aio_bh_schedule_oneshot(data->ctx, nvme_rw_cb_bh, data); + replay_bh_schedule_oneshot_event(data->ctx, nvme_rw_cb_bh, data); } static coroutine_fn int nvme_co_prw_aligned(BlockDriverState *bs, diff --git a/block/rbd.c b/block/rbd.c index 057af43d484c..c71e45d7c32b 100644 --- a/block/rbd.c +++ b/block/rbd.c @@ -22,6 +22,7 @@ #include "block/qdict.h" #include "crypto/secret.h" #include "qemu/cutils.h" +#include "sysemu/replay.h" #include "qapi/qmp/qstring.h" #include "qapi/qmp/qdict.h" #include "qapi/qmp/qjson.h" @@ -884,8 +885,8 @@ static void rbd_finish_aiocb(rbd_completion_t c, RADOSCB *rcb) rcb->ret = rbd_aio_get_return_value(c); rbd_aio_release(c); - aio_bh_schedule_oneshot(bdrv_get_aio_context(acb->common.bs), - rbd_finish_bh, rcb); + replay_bh_schedule_oneshot_event(bdrv_get_aio_context(acb->common.bs), + rbd_finish_bh, rcb); } static int rbd_aio_discard_wrapper(rbd_image_t image, diff --git a/block/vxhs.c b/block/vxhs.c index 77fd5eb20d1b..d79fc97df66a 100644 --- a/block/vxhs.c +++ b/block/vxhs.c @@ -22,6 +22,7 @@ #include "qapi/error.h" #include "qemu/uuid.h" #include "crypto/tlscredsx509.h" +#include "sysemu/replay.h" #define VXHS_OPT_FILENAME "filename" #define VXHS_OPT_VDISK_ID "vdisk-id" @@ -105,8 +106,8 @@ static void vxhs_iio_callback(void *ctx, uint32_t opcode, uint32_t error) trace_vxhs_iio_callback(error); } - aio_bh_schedule_oneshot(bdrv_get_aio_context(acb->common.bs), - vxhs_complete_aio_bh, acb); + replay_bh_schedule_oneshot_event(bdrv_get_aio_context(acb->common.bs), + vxhs_complete_aio_bh, acb); break; default: diff --git a/include/sysemu/replay.h b/include/sysemu/replay.h index dfc7a31c6671..8df517298c5e 100644 --- a/include/sysemu/replay.h +++ b/include/sysemu/replay.h @@ -15,6 +15,7 @@ #include "qapi/qapi-types-misc.h" #include "qapi/qapi-types-run-state.h" #include "qapi/qapi-types-ui.h" +#include "block/aio.h" /* replay clock kinds */ enum ReplayClockKind { @@ -140,6 +141,9 @@ void replay_enable_events(void); bool replay_events_enabled(void); /*! Adds bottom half event to the queue */ void replay_bh_schedule_event(QEMUBH *bh); +/* Adds oneshot bottom half event to the queue */ +void replay_bh_schedule_oneshot_event(AioContext *ctx, + QEMUBHFunc *cb, void *opaque); /*! Adds input event to the queue */ void replay_input_event(QemuConsole *src, InputEvent *evt); /*! Adds input sync event to the queue */ diff --git a/replay/replay-events.c b/replay/replay-events.c index 008e80f63665..302b84043af2 100644 --- a/replay/replay-events.c +++ b/replay/replay-events.c @@ -36,6 +36,9 @@ static void replay_run_event(Event *event) case REPLAY_ASYNC_EVENT_BH: aio_bh_call(event->opaque); break; + case REPLAY_ASYNC_EVENT_BH_ONESHOT: + ((QEMUBHFunc *)event->opaque)(event->opaque2); + break; case REPLAY_ASYNC_EVENT_INPUT: qemu_input_event_send_impl(NULL, (InputEvent *)event->opaque); qapi_free_InputEvent((InputEvent *)event->opaque); @@ -131,6 +134,17 @@ void replay_bh_schedule_event(QEMUBH *bh) } } +void replay_bh_schedule_oneshot_event(AioContext *ctx, + QEMUBHFunc *cb, void *opaque) +{ + if (events_enabled) { + uint64_t id = replay_get_current_icount(); + replay_add_event(REPLAY_ASYNC_EVENT_BH_ONESHOT, cb, opaque, id); + } else { + aio_bh_schedule_oneshot(ctx, cb, opaque); + } +} + void replay_add_input_event(struct InputEvent *event) { replay_add_event(REPLAY_ASYNC_EVENT_INPUT, event, NULL, 0); @@ -161,6 +175,7 @@ static void replay_save_event(Event *event, int checkpoint) /* save event-specific data */ switch (event->event_kind) { case REPLAY_ASYNC_EVENT_BH: + case REPLAY_ASYNC_EVENT_BH_ONESHOT: replay_put_qword(event->id); break; case REPLAY_ASYNC_EVENT_INPUT: @@ -216,6 +231,7 @@ static Event *replay_read_event(int checkpoint) /* Events that has not to be in the queue */ switch (replay_state.read_event_kind) { case REPLAY_ASYNC_EVENT_BH: + case REPLAY_ASYNC_EVENT_BH_ONESHOT: if (replay_state.read_event_id == -1) { replay_state.read_event_id = replay_get_qword(); } diff --git a/replay/replay-internal.h b/replay/replay-internal.h index afba9a3e0c66..55fca1ac6b4b 100644 --- a/replay/replay-internal.h +++ b/replay/replay-internal.h @@ -51,6 +51,7 @@ enum ReplayEvents { enum ReplayAsyncEventKind { REPLAY_ASYNC_EVENT_BH, + REPLAY_ASYNC_EVENT_BH_ONESHOT, REPLAY_ASYNC_EVENT_INPUT, REPLAY_ASYNC_EVENT_INPUT_SYNC, REPLAY_ASYNC_EVENT_CHAR_READ, diff --git a/stubs/Makefile.objs b/stubs/Makefile.objs index 9c7393b08c86..4a50e95ec3e9 100644 --- a/stubs/Makefile.objs +++ b/stubs/Makefile.objs @@ -20,6 +20,7 @@ stub-obj-y += monitor.o stub-obj-y += notify-event.o stub-obj-y += qtest.o stub-obj-y += replay.o +stub-obj-y += replay-user.o stub-obj-y += runstate-check.o stub-obj-y += set-fd-handler.o stub-obj-y += sysbus.o diff --git a/stubs/replay-user.c b/stubs/replay-user.c new file mode 100644 index 000000000000..2ad9e272037a --- /dev/null +++ b/stubs/replay-user.c @@ -0,0 +1,9 @@ +#include "qemu/osdep.h" +#include "sysemu/replay.h" +#include "sysemu/sysemu.h" + +void replay_bh_schedule_oneshot_event(AioContext *ctx, + QEMUBHFunc *cb, void *opaque) +{ + aio_bh_schedule_oneshot(ctx, cb, opaque); +} From f2208fdc5ba2f4d0cbc023e2530b5e3e110d71c2 Mon Sep 17 00:00:00 2001 From: Alberto Garcia Date: Mon, 14 Oct 2019 11:15:45 +0300 Subject: [PATCH 08/15] block: Reject misaligned write requests with BDRV_REQ_NO_FALLBACK The BDRV_REQ_NO_FALLBACK flag means that an operation should only be performed if it can be offloaded or otherwise performed efficiently. However a misaligned write request requires a RMW so we should return an error and let the caller decide how to proceed. This hits an assertion since commit c8bb23cbdb if the required alignment is larger than the cluster size: qemu-img create -f qcow2 -o cluster_size=2k img.qcow2 4G qemu-io -c "open -o driver=qcow2,file.align=4k blkdebug::img.qcow2" \ -c 'write 0 512' qemu-io: block/io.c:1127: bdrv_driver_pwritev: Assertion `!(flags & BDRV_REQ_NO_FALLBACK)' failed. Aborted The reason is that when writing to an unallocated cluster we try to skip the copy-on-write part and zeroize it using BDRV_REQ_NO_FALLBACK instead, resulting in a write request that is too small (2KB cluster size vs 4KB required alignment). Signed-off-by: Alberto Garcia Signed-off-by: Kevin Wolf --- block/io.c | 7 +++++ tests/qemu-iotests/268 | 55 ++++++++++++++++++++++++++++++++++++++ tests/qemu-iotests/268.out | 7 +++++ tests/qemu-iotests/group | 1 + 4 files changed, 70 insertions(+) create mode 100755 tests/qemu-iotests/268 create mode 100644 tests/qemu-iotests/268.out diff --git a/block/io.c b/block/io.c index 8b6dace05641..f0b86c1d19ee 100644 --- a/block/io.c +++ b/block/io.c @@ -2090,6 +2090,13 @@ int coroutine_fn bdrv_co_pwritev_part(BdrvChild *child, return ret; } + /* If the request is misaligned then we can't make it efficient */ + if ((flags & BDRV_REQ_NO_FALLBACK) && + !QEMU_IS_ALIGNED(offset | bytes, align)) + { + return -ENOTSUP; + } + bdrv_inc_in_flight(bs); /* * Align write if necessary by performing a read-modify-write cycle. diff --git a/tests/qemu-iotests/268 b/tests/qemu-iotests/268 new file mode 100755 index 000000000000..78c3f4db3a14 --- /dev/null +++ b/tests/qemu-iotests/268 @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# +# Test write request with required alignment larger than the cluster size +# +# Copyright (C) 2019 Igalia, S.L. +# Author: Alberto Garcia +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +# creator +owner=berto@igalia.com + +seq=`basename $0` +echo "QA output created by $seq" + +status=1 # failure is the default! + +_cleanup() +{ + _cleanup_test_img +} +trap "_cleanup; exit \$status" 0 1 2 3 15 + +# get standard environment, filters and checks +. ./common.rc +. ./common.filter + +_supported_fmt qcow2 +_supported_proto file + +echo +echo "== Required alignment larger than cluster size ==" + +CLUSTER_SIZE=2k _make_test_img 1M +# Since commit c8bb23cbdb writing to an unallocated cluster fills the +# empty COW areas with bdrv_write_zeroes(flags=BDRV_REQ_NO_FALLBACK) +$QEMU_IO -c "open -o driver=$IMGFMT,file.align=4k blkdebug::$TEST_IMG" \ + -c "write 0 512" | _filter_qemu_io + +# success, all done +echo "*** done" +rm -f $seq.full +status=0 diff --git a/tests/qemu-iotests/268.out b/tests/qemu-iotests/268.out new file mode 100644 index 000000000000..2ed6c6852952 --- /dev/null +++ b/tests/qemu-iotests/268.out @@ -0,0 +1,7 @@ +QA output created by 268 + +== Required alignment larger than cluster size == +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576 +wrote 512/512 bytes at offset 0 +512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +*** done diff --git a/tests/qemu-iotests/group b/tests/qemu-iotests/group index 5805a79d9e86..4c861f7eedd8 100644 --- a/tests/qemu-iotests/group +++ b/tests/qemu-iotests/group @@ -278,3 +278,4 @@ 265 rw auto quick 266 rw quick 267 rw auto quick snapshot +268 rw auto quick From 48c8d3ce6daad66d86ba972e8930adc54c02db1a Mon Sep 17 00:00:00 2001 From: Max Reitz Date: Fri, 11 Oct 2019 14:18:08 +0200 Subject: [PATCH 09/15] iotests/028: Fix for long $TEST_DIRs For long test image paths, the order of the "Formatting" line and the "(qemu)" prompt after a drive_backup HMP command may be reversed. In fact, the interaction between the prompt and the line may lead to the "Formatting" to being greppable at all after "read"-ing it (if the prompt injects an IFS character into the "Formatting" string). So just wait until we get a prompt. At that point, the block job must have been started, so "info block-jobs" will only return "No active jobs" once it is done. Reported-by: Thomas Huth Signed-off-by: Max Reitz Reviewed-by: John Snow Signed-off-by: Kevin Wolf --- tests/qemu-iotests/028 | 11 ++++++++--- tests/qemu-iotests/028.out | 1 - 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/qemu-iotests/028 b/tests/qemu-iotests/028 index 71301ec6e508..bba1ee59ae56 100755 --- a/tests/qemu-iotests/028 +++ b/tests/qemu-iotests/028 @@ -119,9 +119,14 @@ fi # Silence output since it contains the disk image path and QEMU's readline # character echoing makes it very hard to filter the output. Plus, there # is no telling how many times the command will repeat before succeeding. -_send_qemu_cmd $h "drive_backup disk ${TEST_IMG}.copy" "(qemu)" >/dev/null -_send_qemu_cmd $h "" "Formatting" | _filter_img_create -qemu_cmd_repeat=20 _send_qemu_cmd $h "info block-jobs" "No active jobs" >/dev/null +# (Note that creating the image results in a "Formatting..." message over +# stdout, which is the same channel the monitor uses. We cannot reliably +# wait for it because the monitor output may interact with it in such a +# way that _timed_wait_for cannot read it. However, once the block job is +# done, we know that the "Formatting..." message must have appeared +# already, so the output is still deterministic.) +silent=y _send_qemu_cmd $h "drive_backup disk ${TEST_IMG}.copy" "(qemu)" +silent=y qemu_cmd_repeat=20 _send_qemu_cmd $h "info block-jobs" "No active jobs" _send_qemu_cmd $h "info block-jobs" "No active jobs" _send_qemu_cmd $h 'quit' "" diff --git a/tests/qemu-iotests/028.out b/tests/qemu-iotests/028.out index 7d54aeb003f9..37aed8443628 100644 --- a/tests/qemu-iotests/028.out +++ b/tests/qemu-iotests/028.out @@ -468,7 +468,6 @@ No errors were found on the image. block-backup -Formatting 'TEST_DIR/t.IMGFMT.copy', fmt=IMGFMT size=4294968832 backing_file=TEST_DIR/t.IMGFMT.base backing_fmt=IMGFMT (qemu) info block-jobs No active jobs === IO: pattern 195 From 3e9297f3659701a72110f0f560b4cc22452972f1 Mon Sep 17 00:00:00 2001 From: Kevin Wolf Date: Fri, 11 Oct 2019 19:20:12 +0200 Subject: [PATCH 10/15] vl: Split off user_creatable_print_help() Printing help for --object is something that we not only want in the system emulator, but also in tools that support --object. Move it into a separate function in qom/object_interfaces.c to make the code accessible for tools. Signed-off-by: Kevin Wolf Reviewed-by: Eric Blake --- include/qom/object_interfaces.h | 12 +++++++ qom/object_interfaces.c | 61 +++++++++++++++++++++++++++++++++ vl.c | 52 +--------------------------- 3 files changed, 74 insertions(+), 51 deletions(-) diff --git a/include/qom/object_interfaces.h b/include/qom/object_interfaces.h index 682ba1d9b06e..3e4e1d928b7c 100644 --- a/include/qom/object_interfaces.h +++ b/include/qom/object_interfaces.h @@ -132,6 +132,18 @@ typedef bool (*user_creatable_add_opts_predicate)(const char *type); int user_creatable_add_opts_foreach(void *opaque, QemuOpts *opts, Error **errp); +/** + * user_creatable_print_help: + * @type: the QOM type to be added + * @opts: options to create + * + * Prints help if requested in @opts. + * + * Returns: true if @opts contained a help option and help was printed, false + * if no help option was found. + */ +bool user_creatable_print_help(const char *type, QemuOpts *opts); + /** * user_creatable_del: * @id: the unique ID for the object diff --git a/qom/object_interfaces.c b/qom/object_interfaces.c index cb5809934a4e..46cd6eab5c6b 100644 --- a/qom/object_interfaces.c +++ b/qom/object_interfaces.c @@ -1,8 +1,11 @@ #include "qemu/osdep.h" + +#include "qemu/cutils.h" #include "qapi/error.h" #include "qapi/qmp/qdict.h" #include "qapi/qmp/qerror.h" #include "qom/object_interfaces.h" +#include "qemu/help_option.h" #include "qemu/module.h" #include "qemu/option.h" #include "qapi/opts-visitor.h" @@ -155,6 +158,64 @@ int user_creatable_add_opts_foreach(void *opaque, QemuOpts *opts, Error **errp) return 0; } +bool user_creatable_print_help(const char *type, QemuOpts *opts) +{ + ObjectClass *klass; + + if (is_help_option(type)) { + GSList *l, *list; + + printf("List of user creatable objects:\n"); + list = object_class_get_list_sorted(TYPE_USER_CREATABLE, false); + for (l = list; l != NULL; l = l->next) { + ObjectClass *oc = OBJECT_CLASS(l->data); + printf(" %s\n", object_class_get_name(oc)); + } + g_slist_free(list); + return true; + } + + klass = object_class_by_name(type); + if (klass && qemu_opt_has_help_opt(opts)) { + ObjectPropertyIterator iter; + ObjectProperty *prop; + GPtrArray *array = g_ptr_array_new(); + int i; + + object_class_property_iter_init(&iter, klass); + while ((prop = object_property_iter_next(&iter))) { + GString *str; + + if (!prop->set) { + continue; + } + + str = g_string_new(NULL); + g_string_append_printf(str, " %s=<%s>", prop->name, prop->type); + if (prop->description) { + if (str->len < 24) { + g_string_append_printf(str, "%*s", 24 - (int)str->len, ""); + } + g_string_append_printf(str, " - %s", prop->description); + } + g_ptr_array_add(array, g_string_free(str, false)); + } + g_ptr_array_sort(array, (GCompareFunc)qemu_pstrcmp0); + if (array->len > 0) { + printf("%s options:\n", type); + } else { + printf("There are no options for %s.\n", type); + } + for (i = 0; i < array->len; i++) { + printf("%s\n", (char *)array->pdata[i]); + } + g_ptr_array_set_free_func(array, g_free); + g_ptr_array_free(array, true); + return true; + } + + return false; +} void user_creatable_del(const char *id, Error **errp) { diff --git a/vl.c b/vl.c index 57d4fd726f69..4489cfb2bb6f 100644 --- a/vl.c +++ b/vl.c @@ -2649,57 +2649,7 @@ static int machine_set_property(void *opaque, */ static bool object_create_initial(const char *type, QemuOpts *opts) { - ObjectClass *klass; - - if (is_help_option(type)) { - GSList *l, *list; - - printf("List of user creatable objects:\n"); - list = object_class_get_list_sorted(TYPE_USER_CREATABLE, false); - for (l = list; l != NULL; l = l->next) { - ObjectClass *oc = OBJECT_CLASS(l->data); - printf(" %s\n", object_class_get_name(oc)); - } - g_slist_free(list); - exit(0); - } - - klass = object_class_by_name(type); - if (klass && qemu_opt_has_help_opt(opts)) { - ObjectPropertyIterator iter; - ObjectProperty *prop; - GPtrArray *array = g_ptr_array_new(); - int i; - - object_class_property_iter_init(&iter, klass); - while ((prop = object_property_iter_next(&iter))) { - GString *str; - - if (!prop->set) { - continue; - } - - str = g_string_new(NULL); - g_string_append_printf(str, " %s=<%s>", prop->name, prop->type); - if (prop->description) { - if (str->len < 24) { - g_string_append_printf(str, "%*s", 24 - (int)str->len, ""); - } - g_string_append_printf(str, " - %s", prop->description); - } - g_ptr_array_add(array, g_string_free(str, false)); - } - g_ptr_array_sort(array, (GCompareFunc)qemu_pstrcmp0); - if (array->len > 0) { - printf("%s options:\n", type); - } else { - printf("There are no options for %s.\n", type); - } - for (i = 0; i < array->len; i++) { - printf("%s\n", (char *)array->pdata[i]); - } - g_ptr_array_set_free_func(array, g_free); - g_ptr_array_free(array, true); + if (user_creatable_print_help(type, opts)) { exit(0); } From 4fa1f0dc052968b41d2bdffd9cb9fcab8137cdaf Mon Sep 17 00:00:00 2001 From: Kevin Wolf Date: Fri, 11 Oct 2019 21:49:17 +0200 Subject: [PATCH 11/15] qemu-io: Support help options for --object Instead of parsing help options as normal object properties and returning an error, provide the same help functionality as the system emulator in qemu-io, too. Signed-off-by: Kevin Wolf Reviewed-by: Eric Blake --- qemu-io.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/qemu-io.c b/qemu-io.c index f64eca694039..91e327659274 100644 --- a/qemu-io.c +++ b/qemu-io.c @@ -475,6 +475,13 @@ static QemuOptsList qemu_object_opts = { }, }; +static bool qemu_io_object_print_help(const char *type, QemuOpts *opts) +{ + if (user_creatable_print_help(type, opts)) { + exit(0); + } + return true; +} static QemuOptsList file_opts = { .name = "file", @@ -622,7 +629,7 @@ int main(int argc, char **argv) qemu_opts_foreach(&qemu_object_opts, user_creatable_add_opts_foreach, - NULL, &error_fatal); + qemu_io_object_print_help, &error_fatal); if (!trace_init_backends()) { exit(1); From c6e5cdfd4b06918a6bcefa2039af7253dbd5e3c0 Mon Sep 17 00:00:00 2001 From: Kevin Wolf Date: Fri, 11 Oct 2019 21:49:17 +0200 Subject: [PATCH 12/15] qemu-img: Support help options for --object Instead of parsing help options as normal object properties and returning an error, provide the same help functionality as the system emulator in qemu-img, too. Signed-off-by: Kevin Wolf Reviewed-by: Eric Blake --- qemu-img.c | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/qemu-img.c b/qemu-img.c index 384c6f38bc62..8b03ef81714f 100644 --- a/qemu-img.c +++ b/qemu-img.c @@ -214,6 +214,14 @@ static QemuOptsList qemu_object_opts = { }, }; +static bool qemu_img_object_print_help(const char *type, QemuOpts *opts) +{ + if (user_creatable_print_help(type, opts)) { + exit(0); + } + return true; +} + static QemuOptsList qemu_source_opts = { .name = "source", .implied_opt_name = "file", @@ -516,7 +524,7 @@ static int img_create(int argc, char **argv) if (qemu_opts_foreach(&qemu_object_opts, user_creatable_add_opts_foreach, - NULL, &error_fatal)) { + qemu_img_object_print_help, &error_fatal)) { goto fail; } @@ -766,7 +774,7 @@ static int img_check(int argc, char **argv) if (qemu_opts_foreach(&qemu_object_opts, user_creatable_add_opts_foreach, - NULL, &error_fatal)) { + qemu_img_object_print_help, &error_fatal)) { return 1; } @@ -979,7 +987,7 @@ static int img_commit(int argc, char **argv) if (qemu_opts_foreach(&qemu_object_opts, user_creatable_add_opts_foreach, - NULL, &error_fatal)) { + qemu_img_object_print_help, &error_fatal)) { return 1; } @@ -1362,7 +1370,7 @@ static int img_compare(int argc, char **argv) if (qemu_opts_foreach(&qemu_object_opts, user_creatable_add_opts_foreach, - NULL, &error_fatal)) { + qemu_img_object_print_help, &error_fatal)) { ret = 2; goto out4; } @@ -2210,7 +2218,7 @@ static int img_convert(int argc, char **argv) if (qemu_opts_foreach(&qemu_object_opts, user_creatable_add_opts_foreach, - NULL, &error_fatal)) { + qemu_img_object_print_help, &error_fatal)) { goto fail_getopt; } @@ -2776,7 +2784,7 @@ static int img_info(int argc, char **argv) if (qemu_opts_foreach(&qemu_object_opts, user_creatable_add_opts_foreach, - NULL, &error_fatal)) { + qemu_img_object_print_help, &error_fatal)) { return 1; } @@ -3002,7 +3010,7 @@ static int img_map(int argc, char **argv) if (qemu_opts_foreach(&qemu_object_opts, user_creatable_add_opts_foreach, - NULL, &error_fatal)) { + qemu_img_object_print_help, &error_fatal)) { return 1; } @@ -3154,7 +3162,7 @@ static int img_snapshot(int argc, char **argv) if (qemu_opts_foreach(&qemu_object_opts, user_creatable_add_opts_foreach, - NULL, &error_fatal)) { + qemu_img_object_print_help, &error_fatal)) { return 1; } @@ -3321,7 +3329,7 @@ static int img_rebase(int argc, char **argv) if (qemu_opts_foreach(&qemu_object_opts, user_creatable_add_opts_foreach, - NULL, &error_fatal)) { + qemu_img_object_print_help, &error_fatal)) { return 1; } @@ -3742,7 +3750,7 @@ static int img_resize(int argc, char **argv) if (qemu_opts_foreach(&qemu_object_opts, user_creatable_add_opts_foreach, - NULL, &error_fatal)) { + qemu_img_object_print_help, &error_fatal)) { return 1; } @@ -3986,7 +3994,7 @@ static int img_amend(int argc, char **argv) if (qemu_opts_foreach(&qemu_object_opts, user_creatable_add_opts_foreach, - NULL, &error_fatal)) { + qemu_img_object_print_help, &error_fatal)) { ret = -1; goto out_no_progress; } @@ -4630,7 +4638,7 @@ static int img_dd(int argc, char **argv) if (qemu_opts_foreach(&qemu_object_opts, user_creatable_add_opts_foreach, - NULL, &error_fatal)) { + qemu_img_object_print_help, &error_fatal)) { ret = -1; goto out; } @@ -4907,7 +4915,7 @@ static int img_measure(int argc, char **argv) if (qemu_opts_foreach(&qemu_object_opts, user_creatable_add_opts_foreach, - NULL, &error_fatal)) { + qemu_img_object_print_help, &error_fatal)) { goto out; } From 495bf893b0db2621726815b5625a6a788d3958aa Mon Sep 17 00:00:00 2001 From: Kevin Wolf Date: Fri, 11 Oct 2019 21:49:17 +0200 Subject: [PATCH 13/15] qemu-nbd: Support help options for --object Instead of parsing help options as normal object properties and returning an error, provide the same help functionality as the system emulator in qemu-nbd, too. Signed-off-by: Kevin Wolf Reviewed-by: Eric Blake --- qemu-nbd.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/qemu-nbd.c b/qemu-nbd.c index 9032b6de2ace..caacf0ed7379 100644 --- a/qemu-nbd.c +++ b/qemu-nbd.c @@ -507,6 +507,13 @@ static QemuOptsList qemu_object_opts = { }, }; +static bool qemu_nbd_object_print_help(const char *type, QemuOpts *opts) +{ + if (user_creatable_print_help(type, opts)) { + exit(0); + } + return true; +} static QCryptoTLSCreds *nbd_get_tls_creds(const char *id, bool list, @@ -902,7 +909,7 @@ int main(int argc, char **argv) qemu_opts_foreach(&qemu_object_opts, user_creatable_add_opts_foreach, - NULL, &error_fatal); + qemu_nbd_object_print_help, &error_fatal); if (!trace_init_backends()) { exit(1); From d1b9d19f99586b33795e20a79f645186ccbc070f Mon Sep 17 00:00:00 2001 From: Max Reitz Date: Thu, 10 Oct 2019 12:08:57 +0200 Subject: [PATCH 14/15] qcow2: Limit total allocation range to INT_MAX MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the COW areas are included, the size of an allocation can exceed INT_MAX. This is kind of limited by handle_alloc() in that it already caps avail_bytes at INT_MAX, but the number of clusters still reflects the original length. This can have all sorts of effects, ranging from the storage layer write call failing to image corruption. (If there were no image corruption, then I suppose there would be data loss because the .cow_end area is forced to be empty, even though there might be something we need to COW.) Fix all of it by limiting nb_clusters so the equivalent number of bytes will not exceed INT_MAX. Cc: qemu-stable@nongnu.org Signed-off-by: Max Reitz Reviewed-by: Eric Blake Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Kevin Wolf --- block/qcow2-cluster.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/block/qcow2-cluster.c b/block/qcow2-cluster.c index 8d5fa1539c47..8982b7b762ee 100644 --- a/block/qcow2-cluster.c +++ b/block/qcow2-cluster.c @@ -1330,6 +1330,9 @@ static int handle_alloc(BlockDriverState *bs, uint64_t guest_offset, nb_clusters = MIN(nb_clusters, s->l2_slice_size - l2_index); assert(nb_clusters <= INT_MAX); + /* Limit total allocation byte count to INT_MAX */ + nb_clusters = MIN(nb_clusters, INT_MAX >> s->cluster_bits); + /* Find L2 entry for the first involved cluster */ ret = get_cluster_table(bs, guest_offset, &l2_slice, &l2_index); if (ret < 0) { @@ -1412,7 +1415,7 @@ static int handle_alloc(BlockDriverState *bs, uint64_t guest_offset, * request actually writes to (excluding COW at the end) */ uint64_t requested_bytes = *bytes + offset_into_cluster(s, guest_offset); - int avail_bytes = MIN(INT_MAX, nb_clusters << s->cluster_bits); + int avail_bytes = nb_clusters << s->cluster_bits; int nb_bytes = MIN(requested_bytes, avail_bytes); QCowL2Meta *old_m = *m; From a1406a9262a087d9ec9627b88da13c4590b61dae Mon Sep 17 00:00:00 2001 From: Max Reitz Date: Thu, 10 Oct 2019 12:08:58 +0200 Subject: [PATCH 15/15] iotests: Test large write request to qcow2 file Without HEAD^, the following happens when you attempt a large write request to a qcow2 file such that the number of bytes covered by all clusters involved in a single allocation will exceed INT_MAX: (A) handle_alloc_space() decides to fill the whole area with zeroes and fails because bdrv_co_pwrite_zeroes() fails (the request is too large). (B) If handle_alloc_space() does not do anything, but merge_cow() decides that the requests can be merged, it will create a too long IOV that later cannot be written. (C) Otherwise, all parts will be written separately, so those requests will work. In either B or C, though, qcow2_alloc_cluster_link_l2() will have an overflow: We use an int (i) to iterate over nb_clusters, and then calculate the L2 entry based on "i << s->cluster_bits" -- which will overflow if the range covers more than INT_MAX bytes. This then leads to image corruption because the L2 entry will be wrong (it will be recognized as a compressed cluster). Even if that were not the case, the .cow_end area would be empty (because handle_alloc() will cap avail_bytes and nb_bytes at INT_MAX, so their difference (which is the .cow_end size) will be 0). So this test checks that on such large requests, the image will not be corrupted. Unfortunately, we cannot check whether COW will be handled correctly, because that data is discarded when it is written to null-co (but we have to use null-co, because writing 2 GB of data in a test is not quite reasonable). Signed-off-by: Max Reitz Reviewed-by: Eric Blake Signed-off-by: Kevin Wolf --- tests/qemu-iotests/270 | 83 ++++++++++++++++++++++++++++++++++++++ tests/qemu-iotests/270.out | 9 +++++ tests/qemu-iotests/group | 1 + 3 files changed, 93 insertions(+) create mode 100755 tests/qemu-iotests/270 create mode 100644 tests/qemu-iotests/270.out diff --git a/tests/qemu-iotests/270 b/tests/qemu-iotests/270 new file mode 100755 index 000000000000..b9a12b908c3e --- /dev/null +++ b/tests/qemu-iotests/270 @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# +# Test large write to a qcow2 image +# +# Copyright (C) 2019 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +seq=$(basename "$0") +echo "QA output created by $seq" + +status=1 # failure is the default! + +_cleanup() +{ + _cleanup_test_img +} +trap "_cleanup; exit \$status" 0 1 2 3 15 + +# get standard environment, filters and checks +. ./common.rc +. ./common.filter + +# This is a qcow2 regression test +_supported_fmt qcow2 +_supported_proto file +_supported_os Linux + +# We use our own external data file and our own cluster size, and we +# require v3 images +_unsupported_imgopts data_file cluster_size 'compat=0.10' + + +# We need a backing file so that handle_alloc_space() will not do +# anything. (If it were to do anything, it would simply fail its +# write-zeroes request because the request range is too large.) +TEST_IMG="$TEST_IMG.base" _make_test_img 4G +$QEMU_IO -c 'write 0 512' "$TEST_IMG.base" | _filter_qemu_io + +# (Use .orig because _cleanup_test_img will remove that file) +# We need a large cluster size, see below for why (above the $QEMU_IO +# invocation) +_make_test_img -o cluster_size=2M,data_file="$TEST_IMG.orig" \ + -b "$TEST_IMG.base" 4G + +# We want a null-co as the data file, because it allows us to quickly +# "write" 2G of data without using any space. +# (qemu-img create does not like it, though, because null-co does not +# support image creation.) +$QEMU_IMG amend -o data_file="json:{'driver':'null-co',,'size':'4294967296'}" \ + "$TEST_IMG" + +# This gives us a range of: +# 2^31 - 512 + 768 - 1 = 2^31 + 255 > 2^31 +# until the beginning of the end COW block. (The total allocation +# size depends on the cluster size, but all that is important is that +# it exceeds INT_MAX.) +# +# 2^31 - 512 is the maximum request size. We want this to result in a +# single allocation, and because the qcow2 driver splits allocations +# on L2 boundaries, we need large L2 tables; hence the cluster size of +# 2 MB. (Anything from 256 kB should work, though, because then one L2 +# table covers 8 GB.) +$QEMU_IO -c "write 768 $((2 ** 31 - 512))" "$TEST_IMG" | _filter_qemu_io + +_check_test_img + +# success, all done +echo "*** done" +rm -f $seq.full +status=0 diff --git a/tests/qemu-iotests/270.out b/tests/qemu-iotests/270.out new file mode 100644 index 000000000000..c7be1110142f --- /dev/null +++ b/tests/qemu-iotests/270.out @@ -0,0 +1,9 @@ +QA output created by 270 +Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=4294967296 +wrote 512/512 bytes at offset 0 +512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 backing_file=TEST_DIR/t.IMGFMT.base data_file=TEST_DIR/t.IMGFMT.orig +wrote 2147483136/2147483136 bytes at offset 768 +2 GiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +No errors were found on the image. +*** done diff --git a/tests/qemu-iotests/group b/tests/qemu-iotests/group index 4c861f7eedd8..7dac79a78336 100644 --- a/tests/qemu-iotests/group +++ b/tests/qemu-iotests/group @@ -279,3 +279,4 @@ 266 rw quick 267 rw auto quick snapshot 268 rw auto quick +270 rw backing quick