Skip to content

Commit

Permalink
Merge remote-tracking branch 'remotes/maxreitz/tags/pull-block-2019-0…
Browse files Browse the repository at this point in the history 
…9-03' into staging

Block patches:
- qemu-io now accepts a file to read a write pattern from
- Ensure that raw files have their first block allocated so we can probe
  the O_DIRECT alignment if necessary
- Various fixes

# gpg: Signature made Tue 03 Sep 2019 13:58:57 BST
# gpg:                using RSA key 91BEB60A30DB3E8857D11829F407DB0061D5CF40
# gpg:                issuer "mreitz@redhat.com"
# gpg: Good signature from "Max Reitz <mreitz@redhat.com>" [full]
# Primary key fingerprint: 91BE B60A 30DB 3E88 57D1  1829 F407 DB00 61D5 CF40

* remotes/maxreitz/tags/pull-block-2019-09-03:
  iotests: Unify cache mode quoting
  tests/check-block: Skip iotests when sanitizers are enabled
  iotests: Check for enabled drivers before testing them
  iotests: Add -display none to the qemu options
  file-posix: fix request_alignment typo
  iotests: Disable 126 for flat vmdk subformats
  iotests: Disable 110 for vmdk.twoGbMaxExtentSparse
  iotests: Disable broken streamOptimized tests
  vmdk: Reject invalid compressed writes
  iotests: Keep testing broken relative extent paths
  vmdk: Use bdrv_dirname() for relative extent paths
  iotests: Fix _filter_img_create()
  iotests: Test allocate_first_block() with O_DIRECT
  block: posix: Always allocate the first block
  block: fix permission update in bdrv_replace_node
  qemu-io: add pattern file for write command

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
  • Loading branch information
@pm215
pm215 committed Sep 4, 2019
2 parents d371479 + 755c5fe commit 6b422e5
Show file tree
Hide file tree
Showing 51 changed files with 394 additions and 90 deletions.
5 changes: 2 additions & 3 deletions block.c
View file
Expand Up @@ -4165,7 +4165,6 @@ void bdrv_replace_node(BlockDriverState *from, BlockDriverState *to,
{
BdrvChild *c, *next;
GSList *list = NULL, *p;
uint64_t old_perm, old_shared;
uint64_t perm = 0, shared = BLK_PERM_ALL;
int ret;

Expand Down Expand Up @@ -4211,8 +4210,8 @@ void bdrv_replace_node(BlockDriverState *from, BlockDriverState *to,
bdrv_unref(from);
}

bdrv_get_cumulative_perm(to, &old_perm, &old_shared);
bdrv_set_perm(to, old_perm | perm, old_shared | shared);
bdrv_get_cumulative_perm(to, &perm, &shared);
bdrv_set_perm(to, perm, shared);

out:
g_slist_free(list);
Expand Down
53 changes: 52 additions & 1 deletion block/file-posix.c
View file
Expand Up @@ -380,7 +380,7 @@ static void raw_probe_alignment(BlockDriverState *bs, int fd, Error **errp)
for (i = 0; i < ARRAY_SIZE(alignments); i++) {
align = alignments[i];
if (raw_is_io_aligned(fd, buf + align, max_align)) {
/* Fallback to request_aligment. */
/* Fallback to request_alignment. */
s->buf_align = (align != 1) ? align : bs->bl.request_alignment;
break;
}
Expand Down Expand Up @@ -1749,6 +1749,43 @@ static int handle_aiocb_discard(void *opaque)
return ret;
}

/*
* Help alignment probing by allocating the first block.
*
* When reading with direct I/O from unallocated area on Gluster backed by XFS,
* reading succeeds regardless of request length. In this case we fallback to
* safe alignment which is not optimal. Allocating the first block avoids this
* fallback.
*
* fd may be opened with O_DIRECT, but we don't know the buffer alignment or
* request alignment, so we use safe values.
*
* Returns: 0 on success, -errno on failure. Since this is an optimization,
* caller may ignore failures.
*/
static int allocate_first_block(int fd, size_t max_size)
{
size_t write_size = (max_size < MAX_BLOCKSIZE)
? BDRV_SECTOR_SIZE
: MAX_BLOCKSIZE;
size_t max_align = MAX(MAX_BLOCKSIZE, getpagesize());
void *buf;
ssize_t n;
int ret;

buf = qemu_memalign(max_align, write_size);
memset(buf, 0, write_size);

do {
n = pwrite(fd, buf, write_size, 0);
} while (n == -1 && errno == EINTR);

ret = (n == -1) ? -errno : 0;

qemu_vfree(buf);
return ret;
}

static int handle_aiocb_truncate(void *opaque)
{
RawPosixAIOData *aiocb = opaque;
Expand Down Expand Up @@ -1788,6 +1825,17 @@ static int handle_aiocb_truncate(void *opaque)
/* posix_fallocate() doesn't set errno. */
error_setg_errno(errp, -result,
"Could not preallocate new data");
} else if (current_length == 0) {
/*
* posix_fallocate() uses fallocate() if the filesystem
* supports it, or fallback to manually writing zeroes. If
* fallocate() was used, unaligned reads from the fallocated
* area in raw_probe_alignment() will succeed, hence we need to
* allocate the first block.
*
* Optimize future alignment probing; ignore failures.
*/
allocate_first_block(fd, offset);
}
} else {
result = 0;
Expand Down Expand Up @@ -1849,6 +1897,9 @@ static int handle_aiocb_truncate(void *opaque)
if (ftruncate(fd, offset) != 0) {
result = -errno;
error_setg_errno(errp, -result, "Could not resize file");
} else if (current_length == 0 && offset > current_length) {
/* Optimize future alignment probing; ignore failures. */
allocate_first_block(fd, offset);
}
return result;
default:
Expand Down
64 changes: 44 additions & 20 deletions block/vmdk.c
View file
Expand Up @@ -1076,8 +1076,7 @@ static const char *next_line(const char *s)
}

static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
const char *desc_file_path, QDict *options,
Error **errp)
QDict *options, Error **errp)
{
int ret;
int matches;
Expand All @@ -1087,6 +1086,7 @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
const char *p, *np;
int64_t sectors = 0;
int64_t flat_offset;
char *desc_file_dir = NULL;
char *extent_path;
BdrvChild *extent_file;
BDRVVmdkState *s = bs->opaque;
Expand Down Expand Up @@ -1130,16 +1130,23 @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
continue;
}

if (!path_is_absolute(fname) && !path_has_protocol(fname) &&
!desc_file_path[0])
{
bdrv_refresh_filename(bs->file->bs);
error_setg(errp, "Cannot use relative extent paths with VMDK "
"descriptor file '%s'", bs->file->bs->filename);
return -EINVAL;
}
if (path_is_absolute(fname)) {
extent_path = g_strdup(fname);
} else {
if (!desc_file_dir) {
desc_file_dir = bdrv_dirname(bs->file->bs, errp);
if (!desc_file_dir) {
bdrv_refresh_filename(bs->file->bs);
error_prepend(errp, "Cannot use relative paths with VMDK "
"descriptor file '%s': ",
bs->file->bs->filename);
ret = -EINVAL;
goto out;
}
}

extent_path = path_combine(desc_file_path, fname);
extent_path = g_strconcat(desc_file_dir, fname, NULL);
}

ret = snprintf(extent_opt_prefix, 32, "extents.%d", s->num_extents);
assert(ret < 32);
Expand All @@ -1149,7 +1156,8 @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
g_free(extent_path);
if (local_err) {
error_propagate(errp, local_err);
return -EINVAL;
ret = -EINVAL;
goto out;
}

/* save to extents array */
Expand All @@ -1160,7 +1168,7 @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
0, 0, 0, 0, 0, &extent, errp);
if (ret < 0) {
bdrv_unref_child(bs, extent_file);
return ret;
goto out;
}
extent->flat_start_offset = flat_offset << 9;
} else if (!strcmp(type, "SPARSE") || !strcmp(type, "VMFSSPARSE")) {
Expand All @@ -1175,24 +1183,27 @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
g_free(buf);
if (ret) {
bdrv_unref_child(bs, extent_file);
return ret;
goto out;
}
extent = &s->extents[s->num_extents - 1];
} else if (!strcmp(type, "SESPARSE")) {
ret = vmdk_open_se_sparse(bs, extent_file, bs->open_flags, errp);
if (ret) {
bdrv_unref_child(bs, extent_file);
return ret;
goto out;
}
extent = &s->extents[s->num_extents - 1];
} else {
error_setg(errp, "Unsupported extent type '%s'", type);
bdrv_unref_child(bs, extent_file);
return -ENOTSUP;
ret = -ENOTSUP;
goto out;
}
extent->type = g_strdup(type);
}
return 0;

ret = 0;
goto out;

invalid:
np = next_line(p);
Expand All @@ -1201,7 +1212,11 @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
np--;
}
error_setg(errp, "Invalid extent line: %.*s", (int)(np - p), p);
return -EINVAL;
ret = -EINVAL;

out:
g_free(desc_file_dir);
return ret;
}

static int vmdk_open_desc_file(BlockDriverState *bs, int flags, char *buf,
Expand All @@ -1228,8 +1243,7 @@ static int vmdk_open_desc_file(BlockDriverState *bs, int flags, char *buf,
}
s->create_type = g_strdup(ct);
s->desc_offset = 0;
ret = vmdk_parse_extents(buf, bs, bs->file->bs->exact_filename, options,
errp);
ret = vmdk_parse_extents(buf, bs, options, errp);
exit:
return ret;
}
Expand Down Expand Up @@ -1720,6 +1734,16 @@ static int vmdk_write_extent(VmdkExtent *extent, int64_t cluster_offset,
if (extent->compressed) {
void *compressed_data;

/* Only whole clusters */
if (offset_in_cluster ||
n_bytes > (extent->cluster_sectors * SECTOR_SIZE) ||
(n_bytes < (extent->cluster_sectors * SECTOR_SIZE) &&
offset + n_bytes != extent->end_sector * SECTOR_SIZE))
{
ret = -EINVAL;
goto out;
}

if (!extent->has_marker) {
ret = -EINVAL;
goto out;
Expand Down
99 changes: 93 additions & 6 deletions qemu-io-cmds.c
View file
Expand Up @@ -350,6 +350,79 @@ static void qemu_io_free(void *p)
qemu_vfree(p);
}

/*
* qemu_io_alloc_from_file()
*
* Allocates the buffer and populates it with the content of the given file
* up to @len bytes. If the file length is less than @len, then the buffer
* is populated with the file content cyclically.
*
* @blk - the block backend where the buffer content is going to be written to
* @len - the buffer length
* @file_name - the file to read the content from
*
* Returns: the buffer pointer on success
* NULL on error
*/
static void *qemu_io_alloc_from_file(BlockBackend *blk, size_t len,
const char *file_name)
{
char *buf, *buf_origin;
FILE *f = fopen(file_name, "r");
int pattern_len;

if (!f) {
perror(file_name);
return NULL;
}

if (qemuio_misalign) {
len += MISALIGN_OFFSET;
}

buf_origin = buf = blk_blockalign(blk, len);

if (qemuio_misalign) {
buf_origin += MISALIGN_OFFSET;
buf += MISALIGN_OFFSET;
len -= MISALIGN_OFFSET;
}

pattern_len = fread(buf_origin, 1, len, f);

if (ferror(f)) {
perror(file_name);
goto error;
}

if (pattern_len == 0) {
fprintf(stderr, "%s: file is empty\n", file_name);
goto error;
}

fclose(f);

if (len > pattern_len) {
len -= pattern_len;
buf += pattern_len;

while (len > 0) {
size_t len_to_copy = MIN(pattern_len, len);

memcpy(buf, buf_origin, len_to_copy);

len -= len_to_copy;
buf += len_to_copy;
}
}

return buf_origin;

error:
qemu_io_free(buf_origin);
return NULL;
}

static void dump_buffer(const void *buffer, int64_t offset, int64_t len)
{
uint64_t i;
Expand Down Expand Up @@ -948,6 +1021,7 @@ static void write_help(void)
" -n, -- with -z, don't allow slow fallback\n"
" -p, -- ignored for backwards compatibility\n"
" -P, -- use different pattern to fill file\n"
" -s, -- use a pattern file to fill the write buffer\n"
" -C, -- report statistics in a machine parsable format\n"
" -q, -- quiet mode, do not show I/O statistics\n"
" -u, -- with -z, allow unmapping\n"
Expand All @@ -964,7 +1038,7 @@ static const cmdinfo_t write_cmd = {
.perm = BLK_PERM_WRITE,
.argmin = 2,
.argmax = -1,
.args = "[-bcCfnquz] [-P pattern] off len",
.args = "[-bcCfnquz] [-P pattern | -s source_file] off len",
.oneline = "writes a number of bytes at a specified offset",
.help = write_help,
};
Expand All @@ -973,7 +1047,7 @@ static int write_f(BlockBackend *blk, int argc, char **argv)
{
struct timespec t1, t2;
bool Cflag = false, qflag = false, bflag = false;
bool Pflag = false, zflag = false, cflag = false;
bool Pflag = false, zflag = false, cflag = false, sflag = false;
int flags = 0;
int c, cnt, ret;
char *buf = NULL;
Expand All @@ -982,8 +1056,9 @@ static int write_f(BlockBackend *blk, int argc, char **argv)
/* Some compilers get confused and warn if this is not initialized. */
int64_t total = 0;
int pattern = 0xcd;
const char *file_name = NULL;

while ((c = getopt(argc, argv, "bcCfnpP:quz")) != -1) {
while ((c = getopt(argc, argv, "bcCfnpP:qs:uz")) != -1) {
switch (c) {
case 'b':
bflag = true;
Expand Down Expand Up @@ -1013,6 +1088,10 @@ static int write_f(BlockBackend *blk, int argc, char **argv)
case 'q':
qflag = true;
break;
case 's':
sflag = true;
file_name = optarg;
break;
case 'u':
flags |= BDRV_REQ_MAY_UNMAP;
break;
Expand Down Expand Up @@ -1050,8 +1129,9 @@ static int write_f(BlockBackend *blk, int argc, char **argv)
return -EINVAL;
}

if (zflag && Pflag) {
printf("-z and -P cannot be specified at the same time\n");
if (zflag + Pflag + sflag > 1) {
printf("Only one of -z, -P, and -s "
"can be specified at the same time\n");
return -EINVAL;
}

Expand Down Expand Up @@ -1087,7 +1167,14 @@ static int write_f(BlockBackend *blk, int argc, char **argv)
}

if (!zflag) {
buf = qemu_io_alloc(blk, count, pattern);
if (sflag) {
buf = qemu_io_alloc_from_file(blk, count, file_name);
if (!buf) {
return -EINVAL;
}
} else {
buf = qemu_io_alloc(blk, count, pattern);
}
}

clock_gettime(CLOCK_MONOTONIC, &t1);
Expand Down

0 comments on commit 6b422e5

Please sign in to comment.