Permalink
Browse files

Implement Redacted Send/Receive

  • Loading branch information...
pcd1193182 authored and sdimitro committed Oct 3, 2017
1 parent cc99f27 commit 53e49f33d1cd8990ccb4a9e9d1121ebdcd1d585a
Showing with 15,241 additions and 4,467 deletions.
  1. +249 −31 cmd/zdb/zdb.c
  2. +158 −25 cmd/zfs/zfs_main.c
  3. +16 −0 cmd/zstreamdump/zstreamdump.c
  4. +3 −0 configure.ac
  5. +6 −3 include/libzfs.h
  6. +12 −1 include/libzfs_core.h
  7. +56 −19 include/spl/sys/debug.h
  8. +3 −0 include/sys/Makefile.am
  9. +4 −2 include/sys/bqueue.h
  10. +4 −0 include/sys/dbuf.h
  11. +3 −0 include/sys/dmu.h
  12. +8 −33 include/sys/dmu_impl.h
  13. +87 −0 include/sys/dmu_recv.h
  14. +58 −0 include/sys/dmu_redact.h
  15. +26 −39 include/sys/dmu_send.h
  16. +15 −1 include/sys/dmu_traverse.h
  17. +81 −4 include/sys/dsl_bookmark.h
  18. +38 −12 include/sys/dsl_dataset.h
  19. +2 −1 include/sys/dsl_destroy.h
  20. +23 −20 include/sys/fs/zfs.h
  21. +25 −15 include/sys/nvpair.h
  22. +51 −0 include/sys/objlist.h
  23. +24 −2 include/sys/spa.h
  24. +2 −1 include/sys/zfs_context.h
  25. +10 −4 include/sys/zfs_ioctl.h
  26. +3 −0 include/sys/zfs_vfsops.h
  27. +15 −2 include/zfeature_common.h
  28. +49 −6 lib/libzfs/libzfs_dataset.c
  29. +7 −4 lib/libzfs/libzfs_iter.c
  30. +14 −6 lib/libzfs/libzfs_mount.c
  31. +746 −149 lib/libzfs/libzfs_sendrecv.c
  32. +2 −0 lib/libzfs/libzfs_util.c
  33. +119 −9 lib/libzfs_core/libzfs_core.c
  34. +3 −0 lib/libzpool/Makefile.am
  35. +1 −0 lib/libzpool/kernel.c
  36. +40 −1 man/man5/zpool-features.5
  37. +132 −8 man/man8/zfs.8
  38. +80 −1 module/nvpair/fnvpair.c
  39. +79 −25 module/zcommon/zfeature_common.c
  40. +1 −0 module/zcommon/zfs_namecheck.c
  41. +16 −6 module/zcommon/zfs_prop.c
  42. +3 −0 module/zfs/Makefile.in
  43. +2 −0 module/zfs/arc.c
  44. +3 −2 module/zfs/bptree.c
  45. +58 −15 module/zfs/bqueue.c
  46. +100 −7 module/zfs/dbuf.c
  47. +15 −1 module/zfs/dmu.c
  48. +3 −2 module/zfs/dmu_diff.c
  49. +2 −1 module/zfs/dmu_object.c
  50. +11 −10 module/zfs/dmu_objset.c
  51. +3,008 −0 module/zfs/dmu_recv.c
  52. +1,105 −0 module/zfs/dmu_redact.c
  53. +2,073 −3,565 module/zfs/dmu_send.c
  54. +15 −13 module/zfs/dmu_traverse.c
  55. +4 −2 module/zfs/dnode_sync.c
  56. +1,135 −88 module/zfs/dsl_bookmark.c
  57. +5 −3 module/zfs/dsl_crypt.c
  58. +489 −141 module/zfs/dsl_dataset.c
  59. +37 −31 module/zfs/dsl_deadlist.c
  60. +113 −71 module/zfs/dsl_destroy.c
  61. +1 −2 module/zfs/dsl_pool.c
  62. +12 −2 module/zfs/dsl_scan.c
  63. +84 −0 module/zfs/objlist.c
  64. +3 −3 module/zfs/refcount.c
  65. +2 −1 module/zfs/spa.c
  66. +311 −71 module/zfs/zfs_ioctl.c
  67. +101 −1 module/zfs/zfs_vfsops.c
  68. +3 −0 module/zfs/zio.c
  69. +9 −0 tests/runfiles/linux.run
  70. +3 −1 tests/zfs-tests/cmd/Makefile.am
  71. +6 −0 tests/zfs-tests/cmd/get_diff/Makefile.am
  72. +109 −0 tests/zfs-tests/cmd/get_diff/get_diff.c
  73. +7 −0 tests/zfs-tests/cmd/zfs_dd/Makefile.am
  74. +1,966 −0 tests/zfs-tests/cmd/zfs_dd/zfs_dd.c
  75. +4 −1 tests/zfs-tests/include/commands.cfg
  76. +1 −0 tests/zfs-tests/tests/functional/Makefile.am
  77. +77 −0 tests/zfs-tests/tests/functional/cli_root/zfs_clone/zfs_clone_rm_nested.ksh
  78. +3 −1 tests/zfs-tests/tests/functional/cli_root/zfs_send/zfs_send_006_pos.ksh
  79. +25 −0 tests/zfs-tests/tests/functional/redacted_send/Makefile.am
  80. +33 −0 tests/zfs-tests/tests/functional/redacted_send/cleanup.ksh
  81. +86 −0 tests/zfs-tests/tests/functional/redacted_send/redacted.cfg
  82. +271 −0 tests/zfs-tests/tests/functional/redacted_send/redacted.kshlib
  83. +72 −0 tests/zfs-tests/tests/functional/redacted_send/redacted_compressed.ksh
  84. +162 −0 tests/zfs-tests/tests/functional/redacted_send/redacted_contents.ksh
  85. +102 −0 tests/zfs-tests/tests/functional/redacted_send/redacted_deleted.ksh
  86. +71 −0 tests/zfs-tests/tests/functional/redacted_send/redacted_disabled_feature.ksh
  87. +103 −0 tests/zfs-tests/tests/functional/redacted_send/redacted_embedded.ksh
  88. +122 −0 tests/zfs-tests/tests/functional/redacted_send/redacted_holes.ksh
  89. +152 −0 tests/zfs-tests/tests/functional/redacted_send/redacted_incrementals.ksh
  90. +63 −0 tests/zfs-tests/tests/functional/redacted_send/redacted_largeblocks.ksh
  91. +68 −0 tests/zfs-tests/tests/functional/redacted_send/redacted_many_clones.ksh
  92. +77 −0 tests/zfs-tests/tests/functional/redacted_send/redacted_mixed_recsize.ksh
  93. +109 −0 tests/zfs-tests/tests/functional/redacted_send/redacted_mounts.ksh
  94. +80 −0 tests/zfs-tests/tests/functional/redacted_send/redacted_negative.ksh
  95. +87 −0 tests/zfs-tests/tests/functional/redacted_send/redacted_origin.ksh
  96. +77 −0 tests/zfs-tests/tests/functional/redacted_send/redacted_props.ksh
  97. +88 −0 tests/zfs-tests/tests/functional/redacted_send/redacted_resume.ksh
  98. +64 −0 tests/zfs-tests/tests/functional/redacted_send/redacted_size.ksh
  99. +104 −0 tests/zfs-tests/tests/functional/redacted_send/redacted_volume.ksh
  100. +36 −0 tests/zfs-tests/tests/functional/redacted_send/setup.ksh
  101. +2 −2 tests/zfs-tests/tests/functional/rsend/rsend.kshlib
  102. +33 −0 tests/zfs-tests/tests/functional/rsend/rsend_016_neg.ksh
View

Large diffs are not rendered by default.

Oops, something went wrong.
View
@@ -32,6 +32,7 @@
#include <assert.h>
#include <ctype.h>
#include <sys/debug.h>
#include <errno.h>
#include <getopt.h>
#include <libgen.h>
@@ -116,6 +117,7 @@ static int zfs_do_load_key(int argc, char **argv);
static int zfs_do_unload_key(int argc, char **argv);
static int zfs_do_change_key(int argc, char **argv);
static int zfs_do_project(int argc, char **argv);
static int zfs_do_redact(int argc, char **argv);
/*
* Enable a reasonable set of defaults for libumem debugging on DEBUG builds.
@@ -170,6 +172,7 @@ typedef enum {
HELP_LOAD_KEY,
HELP_UNLOAD_KEY,
HELP_CHANGE_KEY,
HELP_REDACT,
} zfs_help_t;
typedef struct zfs_command {
@@ -232,6 +235,7 @@ static zfs_command_t command_table[] = {
{ "load-key", zfs_do_load_key, HELP_LOAD_KEY },
{ "unload-key", zfs_do_unload_key, HELP_UNLOAD_KEY },
{ "change-key", zfs_do_change_key, HELP_CHANGE_KEY },
{ "redact", zfs_do_redact, HELP_REDACT },
};
#define NCOMMAND (sizeof (command_table) / sizeof (command_table[0]))
@@ -273,7 +277,7 @@ get_usage(zfs_help_t idx)
"[filesystem|volume|snapshot] ...\n"));
case HELP_MOUNT:
return (gettext("\tmount\n"
"\tmount [-lvO] [-o opts] <-a | filesystem>\n"));
"\tmount [-flvO] [-o opts] <-a | filesystem>\n"));
case HELP_PROMOTE:
return (gettext("\tpromote <clone-filesystem>\n"));
case HELP_RECEIVE:
@@ -296,6 +300,9 @@ get_usage(zfs_help_t idx)
"<snapshot>\n"
"\tsend [-nvPLecw] [-i snapshot|bookmark] "
"<filesystem|volume|snapshot>\n"
"[-i bookmark] <snapshot> <bookmark_name>\n"
"\tsend [-DnPpvLecr] [-i bookmark|snapshot] "
"--redact <bookmark> <snapshot>\n"
"\tsend [-nvPe] -t <receive_resume_token>\n"));
case HELP_SET:
return (gettext("\tset <property=value> ... "
@@ -378,6 +385,9 @@ get_usage(zfs_help_t idx)
"\t [-o keylocation=<value>] [-o pbkfd2iters=<value>]\n"
"\t <filesystem|volume>\n"
"\tchange-key -i [-l] <filesystem|volume>\n"));
case HELP_REDACT:
return (gettext("\tredact <snapshot> <bookmark> "
"<redaction_snapshot> ..."));
}
abort();
@@ -535,6 +545,8 @@ usage(boolean_t requested)
(void) fprintf(fp, "YES NO <size> | none\n");
(void) fprintf(fp, "\t%-15s ", "written@<snap>");
(void) fprintf(fp, " NO NO <size>\n");
(void) fprintf(fp, "\t%-15s ", "written#<bookmark>");
(void) fprintf(fp, " NO NO <size>\n");
(void) fprintf(fp, gettext("\nSizes are specified in bytes "
"with standard units such as K, M, G, etc.\n"));
@@ -1493,6 +1505,13 @@ zfs_do_destroy(int argc, char **argv)
return (-1);
}
/*
* Unfortunately, zfs_bookmark() doesn't honor the
* casesensitivity setting. However, we can't simply
* remove this check, because lzc_destroy_bookmarks()
* ignores non-existent bookmarks, so this is necessary
* to get a proper error message.
*/
if (!zfs_bookmark_exists(argv[0])) {
(void) fprintf(stderr, gettext("bookmark '%s' "
"does not exist.\n"), argv[0]);
@@ -3557,6 +3576,73 @@ zfs_do_promote(int argc, char **argv)
return (ret);
}
static int
zfs_do_redact(int argc, char **argv)
{
char *snap = NULL;
char *bookname = NULL;
char **rsnaps = NULL;
int numrsnaps = 0;
argv++;
argc--;
if (argc < 3) {
(void) fprintf(stderr, gettext("too few arguments"));
usage(B_FALSE);
}
snap = argv[0];
bookname = argv[1];
rsnaps = argv + 2;
numrsnaps = argc - 2;
nvlist_t *rsnapnv = fnvlist_alloc();
for (int i = 0; i < numrsnaps; i++) {
fnvlist_add_boolean(rsnapnv, rsnaps[i]);
}
int err = lzc_redact(snap, bookname, rsnapnv);
fnvlist_free(rsnapnv);
switch (err) {
case 0:
break;
case ENOENT:
(void) fprintf(stderr,
gettext("provided snapshot %s does not exist"), snap);
break;
case EEXIST:
(void) fprintf(stderr, gettext("specified redaction bookmark "
"(%s) provided already exists"), bookname);
break;
case ENAMETOOLONG:
(void) fprintf(stderr, gettext("provided bookmark name cannot "
"be used, final name would be too long"));
break;
case E2BIG:
(void) fprintf(stderr, gettext("too many redaction snapshots "
"specified"));
break;
case EINVAL:
(void) fprintf(stderr, gettext("redaction snapshot must be "
"descendent of snapshot being redacted"));
break;
case EALREADY:
(void) fprintf(stderr, gettext("attempted to redact redacted "
"dataset or with respect to redacted dataset"));
break;
case ENOTSUP:
(void) fprintf(stderr, gettext("redaction bookmarks feature "
"not enabled"));
break;
default:
(void) fprintf(stderr, gettext("internal error: %s"),
strerror(errno));
}
return (err);
}
/*
* zfs rollback [-rRf] <snapshot>
*
@@ -3941,6 +4027,9 @@ zfs_do_snapshot(int argc, char **argv)
return (-1);
}
#define REDACT_OPT 1024
/*
* Send a backup stream to stdout.
*/
@@ -3955,10 +4044,11 @@ zfs_do_send(int argc, char **argv)
sendflags_t flags = { 0 };
int c, err;
nvlist_t *dbgnv = NULL;
boolean_t extraverbose = B_FALSE;
char *redactbook = NULL;
struct option long_options[] = {
{"replicate", no_argument, NULL, 'R'},
{"redact-bookmark", required_argument, NULL, REDACT_OPT},
{"props", no_argument, NULL, 'p'},
{"parsable", no_argument, NULL, 'P'},
{"dedup", no_argument, NULL, 'D'},
@@ -3991,6 +4081,9 @@ zfs_do_send(int argc, char **argv)
case 'R':
flags.replicate = B_TRUE;
break;
case REDACT_OPT:
redactbook = optarg;
break;
case 'p':
flags.props = B_TRUE;
break;
@@ -3999,12 +4092,9 @@ zfs_do_send(int argc, char **argv)
break;
case 'P':
flags.parsable = B_TRUE;
flags.verbose = B_TRUE;
break;
case 'v':
if (flags.verbose)
extraverbose = B_TRUE;
flags.verbose = B_TRUE;
flags.verbosity++;
flags.progress = B_TRUE;
break;
case 'D':
@@ -4072,19 +4162,21 @@ zfs_do_send(int argc, char **argv)
}
}
if (flags.parsable && flags.verbosity == 0)
flags.verbosity = 1;
argc -= optind;
argv += optind;
if (resume_token != NULL) {
if (fromname != NULL || flags.replicate || flags.props ||
flags.backup || flags.dedup) {
flags.backup || flags.dedup || redactbook != NULL) {
(void) fprintf(stderr,
gettext("invalid flags combined with -t\n"));
usage(B_FALSE);
}
if (argc != 0) {
(void) fprintf(stderr, gettext("no additional "
"arguments are permitted with -t\n"));
if (argc > 0) {
(void) fprintf(stderr, gettext("too many arguments\n"));
usage(B_FALSE);
}
} else {
@@ -4112,43 +4204,70 @@ zfs_do_send(int argc, char **argv)
}
/*
* Special case sending a filesystem, or from a bookmark.
* For everything except -R and -I, use the new, cleaner code path.
*/
if (strchr(argv[0], '@') == NULL ||
(fromname && strchr(fromname, '#') != NULL)) {
if (!(flags.replicate || flags.doall)) {
char frombuf[ZFS_MAX_DATASET_NAME_LEN];
if (flags.replicate || flags.doall || flags.props ||
flags.backup || flags.dedup ||
(strchr(argv[0], '@') == NULL &&
(flags.dryrun || flags.verbose || flags.progress))) {
(void) fprintf(stderr, gettext("Error: "
"Unsupported flag with filesystem or bookmark.\n"));
return (1);
if (redactbook != NULL) {
if (strchr(argv[0], '@') == NULL) {
(void) fprintf(stderr, gettext("Error: Cannot "
"do a redacted send to a filesystem.\n"));
return (1);
}
}
zhp = zfs_open(g_zfs, argv[0], ZFS_TYPE_DATASET);
if (zhp == NULL)
return (1);
if (fromname != NULL && (strchr(fromname, '#') == NULL &&
strchr(fromname, '@') == NULL)) {
/*
* Neither bookmark or snapshot was specified. Print a
* warning, and assume snapshot.
*/
(void) fprintf(stderr, "Warning: incremental source "
"didn't specify type, assuming snapshot. Use '@' "
"or '#' prefix to avoid ambiguity.\n");
(void) snprintf(frombuf, sizeof (frombuf), "@%s",
fromname);
fromname = frombuf;
}
if (fromname != NULL &&
(fromname[0] == '#' || fromname[0] == '@')) {
/*
* Incremental source name begins with # or @.
* Default to same fs as target.
*/
char tmpbuf[ZFS_MAX_DATASET_NAME_LEN];
(void) strlcpy(tmpbuf, fromname, sizeof (tmpbuf));
(void) strlcpy(frombuf, argv[0], sizeof (frombuf));
cp = strchr(frombuf, '@');
if (cp != NULL)
*cp = '\0';
(void) strlcat(frombuf, fromname, sizeof (frombuf));
(void) strlcat(frombuf, tmpbuf, sizeof (frombuf));
fromname = frombuf;
}
err = zfs_send_one(zhp, fromname, STDOUT_FILENO, flags);
err = zfs_send_one(zhp, fromname, STDOUT_FILENO, &flags,
redactbook);
zfs_close(zhp);
return (err != 0);
}
if (fromname != NULL && strchr(fromname, '#')) {
(void) fprintf(stderr,
gettext("Error: multiple snapshots cannot be "
"sent from a bookmark.\n"));
return (1);
}
if (redactbook != NULL) {
(void) fprintf(stderr, gettext("Error: multiple snapshots "
"cannot be sent redacted.\n"));
return (1);
}
cp = strchr(argv[0], '@');
*cp = '\0';
toname = cp + 1;
@@ -4192,9 +4311,9 @@ zfs_do_send(int argc, char **argv)
flags.doall = B_TRUE;
err = zfs_send(zhp, fromname, toname, &flags, STDOUT_FILENO, NULL, 0,
extraverbose ? &dbgnv : NULL);
flags.verbosity >= 3 ? &dbgnv : NULL);
if (extraverbose && dbgnv != NULL) {
if (flags.verbosity >= 3 && dbgnv != NULL) {
/*
* dump_nvlist prints to stdout, but that's been
* redirected to a file. Make it print to stderr
@@ -6279,6 +6398,17 @@ share_mount_one(zfs_handle_t *zhp, int op, int flags, char *protocol,
return (1);
}
if (zfs_prop_get_int(zhp, ZFS_PROP_REDACTED) && !(flags & MS_FORCE)) {
if (!explicit)
return (0);
(void) fprintf(stderr, gettext("cannot %s '%s': "
"Dataset is not complete, was created by receiving "
"a redacted zfs send stream.\n"), cmdname,
zfs_get_name(zhp));
return (1);
}
/*
* At this point, we have verified that the mountpoint and/or
* shareopts are appropriate for auto management. If the
@@ -6414,7 +6544,7 @@ share_mount(int op, int argc, char **argv)
int flags = 0;
/* check options */
while ((c = getopt(argc, argv, op == OP_MOUNT ? ":alvo:O" : "al"))
while ((c = getopt(argc, argv, op == OP_MOUNT ? ":alvo:Of" : "al"))
!= -1) {
switch (c) {
case 'a':
@@ -6442,6 +6572,9 @@ share_mount(int op, int argc, char **argv)
case 'O':
flags |= MS_OVERLAY;
break;
case 'f':
flags |= MS_FORCE;
break;
case ':':
(void) fprintf(stderr, gettext("missing argument for "
"'%c' option\n"), optopt);
@@ -236,6 +236,7 @@ main(int argc, char *argv[])
struct drr_spill *drrs = &thedrr.drr_u.drr_spill;
struct drr_write_embedded *drrwe = &thedrr.drr_u.drr_write_embedded;
struct drr_object_range *drror = &thedrr.drr_u.drr_object_range;
struct drr_redact *drrr = &thedrr.drr_u.drr_redact;
struct drr_checksum *drrc = &thedrr.drr_u.drr_checksum;
char c;
boolean_t verbose = B_FALSE;
@@ -705,6 +706,21 @@ main(int argc, char *argv[])
mac);
}
break;
case DRR_REDACT:
if (do_byteswap) {
drrr->drr_object = BSWAP_64(drrr->drr_object);
drrr->drr_offset = BSWAP_64(drrr->drr_offset);
drrr->drr_length = BSWAP_64(drrr->drr_length);
drrr->drr_toguid = BSWAP_64(drrr->drr_toguid);
}
if (verbose) {
(void) printf("REDACT object = %llu offset = "
"%llu length = %llu\n",
(u_longlong_t)drrr->drr_object,
(u_longlong_t)drrr->drr_offset,
(u_longlong_t)drrr->drr_length);
}
break;
case DRR_NUMTYPES:
/* should never be reached */
exit(1);
Oops, something went wrong.

0 comments on commit 53e49f3

Please sign in to comment.