Merge pull request #10487 from yangdongsheng/rbd_export

rbd: introduce v2 format for rbd export/import

Reviewed-by: Jason Dillaman <dillaman@redhat.com>

doc/dev/rbd-diff.rst

@@ -60,6 +60,83 @@ Zero data
 - le64: length
 
 
+Final Record
+~~~~~~~~~~~~
+
+End
+---
+
+- u8: 'e'
+
+
+Header
+~~~~~~
+
+"rbd diff v2\\n"
+
+Metadata records
+~~~~~~~~~~~~~~~~
+
+Every record has a one byte "tag" that identifies the record type,
+followed by length of data, and then some other data.
+
+Metadata records come in the first part of the image.  Order is not
+important, as long as all the metadata records come before the data
+records.
+
+In v2, we have the following metadata in each section:
+(1 Bytes) tag.
+(8 Bytes) length.
+(n Bytes) data.
+
+In this way, we can skip the unrecognized tag.
+
+From snap
+---------
+
+- u8: 'f'
+- le64: length of appending data (4 + length)
+- le32: snap name length
+- snap name
+
+To snap
+-------
+
+- u8: 't'
+- le64: length of appending data (4 + length)
+- le32: snap name length
+- snap name
+
+Size
+----
+
+- u8: 's'
+- le64: length of appending data (8)
+- le64: (ending) image size
+
+Data Records
+~~~~~~~~~~~~
+
+These records come in the second part of the sequence.
+
+Updated data
+------------
+
+- u8: 'w'
+- le64: length of appending data (8 + 8 + length)
+- le64: offset
+- le64: length
+- length bytes of actual data
+
+Zero data
+---------
+
+- u8: 'z'
+- le64: length of appending data (8 + 8)
+- le64: offset
+- le64: length
+
+
 Final Record
 ~~~~~~~~~~~~
 

doc/dev/rbd-export.rst

@@ -0,0 +1,84 @@
+RBD Export & Import
+===================
+
+This is a file format of an RBD image or snapshot. It's a sparse format
+for the full image. There are three recording sections in the file.
+
+(1) Header.
+(2) Metadata.
+(3) Diffs.
+
+Header
+~~~~~~
+
+"rbd image v2\\n"
+
+Metadata records
+~~~~~~~~~~~~~~~~
+
+Every record has a one byte "tag" that identifies the record type,
+followed by length of data, and then some other data.
+
+Metadata records come in the first part of the image.  Order is not
+important, as long as all the metadata records come before the data
+records.
+
+In v2, we have the following metadata in each section:
+(1 Bytes) tag.
+(8 Bytes) length.
+(n Bytes) data.
+
+In this way, we can skip the unrecognized tag.
+
+Image order
+-----------
+
+- u8: 'O'
+- le64: length of appending data (8)
+- le64: image order
+
+Image format
+------------
+
+- u8: 'F'
+- le64: length of appending data (8)
+- le64: image format
+
+Image Features
+--------------
+
+- u8: 'T'
+- le64: length of appending data (8)
+- le64: image features
+
+Image Stripe unit
+-----------------
+
+- u8: 'U'
+- le64: length of appending data (8)
+- le64: image striping unit
+
+Image Stripe count
+------------------
+
+- u8: 'C'
+- le64: length of appending data (8)
+- le64: image striping count
+
+Final Record
+~~~~~~~~~~~~
+
+End
+---
+
+- u8: 'E'
+
+
+Diffs records
+~~~~~~~~~~~~~~~~~
+Record the all snapshots and the HEAD in this section. 
+
+- le64: number of diffs
+- Diffs ...
+
+Detail please refer to rbd-diff.rst

doc/man/8/rbd.rst

@@ -224,10 +224,12 @@ Commands
   Deletes an rbd image (including all data blocks). If the image has
   snapshots, this fails and nothing is deleted.
 
-:command:`export` (*image-spec* | *snap-spec*) [*dest-path*]
+:command:`export` [--export-format *format (1 or 2)*] (*image-spec* | *snap-spec*) [*dest-path*]
   Exports image to dest path (use - for stdout).
+  The --export-format accepts '1' or '2' currently. Format 2 allow us to export not only the content
+  of image, but also the snapshots and other properties, such as image_order, features.
 
-:command:`import` [--image-format *format-id*] [--object-size *size-in-B/K/M*] [--stripe-unit *size-in-B/K/M* --stripe-count *num*] [--image-feature *feature-name*]... [--image-shared] *src-path* [*image-spec*]
+:command:`import` [--export-format *format (1 or 2)*] [--image-format *format-id*] [--object-size *size-in-B/K/M*] [--stripe-unit *size-in-B/K/M* --stripe-count *num*] [--image-feature *feature-name*]... [--image-shared] *src-path* [*image-spec*]
   Creates a new image and imports its data from path (use - for
   stdin).  The import operation will try to create sparse rbd images 
   if possible.  For import from stdin, the sparsification unit is
@@ -236,6 +238,9 @@ Commands
   The --stripe-unit and --stripe-count arguments are optional, but must be
   used together.
 
+  The --export-format accepts '1' or '2' currently. Format 2 allow us to import not only the content
+  of image, but also the snapshots and other properties, such as image_order, features.
+
 :command:`export-diff` [--from-snap *snap-name*] [--whole-object] (*image-spec* | *snap-spec*) *dest-path*
   Exports an incremental diff for an image to dest path (use - for stdout).  If
   an initial snapshot is specified, only changes since that snapshot are included; otherwise,

qa/workunits/rbd/import_export.sh

@@ -74,6 +74,70 @@ cmp ${TMPDIR}/img ${TMPDIR}/img3
 
 rm ${TMPDIR}/img ${TMPDIR}/img2 ${TMPDIR}/img3
 
+# try with --export-format for snapshots
+dd if=/bin/dd of=${TMPDIR}/img bs=1k count=10 seek=100
+rbd import $RBD_CREATE_ARGS ${TMPDIR}/img testimg
+rbd snap create testimg@snap
+rbd export --export-format 2 testimg ${TMPDIR}/img_v2
+rbd import --export-format 2 ${TMPDIR}/img_v2 testimg_import
+rbd info testimg_import
+rbd info testimg_import@snap
+
+# compare the contents between testimg and testimg_import
+rbd export testimg_import ${TMPDIR}/img_import
+compare_files_and_ondisk_sizes ${TMPDIR}/img ${TMPDIR}/img_import
+
+rbd export testimg@snap ${TMPDIR}/img_snap
+rbd export testimg_import@snap ${TMPDIR}/img_snap_import
+compare_files_and_ondisk_sizes ${TMPDIR}/img_snap ${TMPDIR}/img_snap_import
+
+rm ${TMPDIR}/img_v2
+rm ${TMPDIR}/img_import
+rm ${TMPDIR}/img_snap
+rm ${TMPDIR}/img_snap_import
+
+rbd snap rm testimg_import@snap
+rbd remove testimg_import
+rbd snap rm testimg@snap
+rbd rm testimg
+
+# order
+rbd import --order 20 ${TMPDIR}/img testimg
+rbd export --export-format 2 testimg ${TMPDIR}/img_v2
+rbd import --export-format 2 ${TMPDIR}/img_v2 testimg_import
+rbd info testimg_import|grep order|awk '{print $2}'|grep 20
+
+rm ${TMPDIR}/img_v2
+
+rbd remove testimg_import
+rbd remove testimg
+
+# features
+rbd import --image-feature layering ${TMPDIR}/img testimg
+FEATURES_BEFORE=`rbd info testimg|grep features`
+rbd export --export-format 2 testimg ${TMPDIR}/img_v2
+rbd import --export-format 2 ${TMPDIR}/img_v2 testimg_import
+FEATURES_AFTER=`rbd info testimg_import|grep features`
+if [ "$FEATURES_BEFORE" != "$FEATURES_AFTER" ]; then
+  false
+fi
+
+rm ${TMPDIR}/img_v2
+
+rbd remove testimg_import
+rbd remove testimg
+
+# stripe
+rbd import --stripe-count 1000 --stripe-unit 4096 ${TMPDIR}/img testimg
+rbd export --export-format 2 testimg ${TMPDIR}/img_v2
+rbd import --export-format 2 ${TMPDIR}/img_v2 testimg_import
+rbd info testimg_import|grep "stripe unit"|awk '{print $3}'|grep 4096
+rbd info testimg_import|grep "stripe count"|awk '{print $3}'|grep 1000
+
+rm ${TMPDIR}/img_v2
+
+rbd remove testimg_import
+rbd remove testimg
 
 tiered=0
 if ceph osd dump | grep ^pool | grep "'rbd'" | grep tier; then

src/test/cli/rbd/help.t

@@ -320,6 +320,7 @@
   rbd help export
   usage: rbd export [--pool <pool>] [--image <image>] [--snap <snap>] 
                     [--path <path>] [--no-progress] 
+                    [--export-format <export-format>] 
                     <source-image-or-snap-spec> <path-name> 
 
   Export image to file.
@@ -336,6 +337,7 @@
     --snap arg                   source snapshot name
     --path arg                   export file (or '-' for stdout)
     --no-progress                disable progress output
+    --export-format arg          format of image file
 
   rbd help export-diff
   usage: rbd export-diff [--pool <pool>] [--image <image>] [--snap <snap>] 
@@ -585,7 +587,8 @@
                     [--journal-splay-width <journal-splay-width>] 
                     [--journal-object-size <journal-object-size>] 
                     [--journal-pool <journal-pool>] [--no-progress] 
-                    [--pool <pool>] [--image <image>] 
+                    [--export-format <export-format>] [--pool <pool>] 
+                    [--image <image>] 
                     <path-name> <dest-image-spec> 
 
   Import image from file.
@@ -616,6 +619,7 @@
     --journal-object-size arg size of journal objects
     --journal-pool arg        pool for journal objects
     --no-progress             disable progress output
+    --export-format arg       format of image file
     -p [ --pool ] arg         pool name (deprecated)
     --image arg               image name (deprecated)
 

src/tools/rbd/ArgumentTypes.cc

@@ -331,6 +331,11 @@ void add_no_error_option(boost::program_options::options_description *opt) {
     (NO_ERROR.c_str(), po::bool_switch(), "continue after error");
 }
 
+void add_export_format_option(boost::program_options::options_description *opt) {
+  opt->add_options()
+    ("export-format", po::value<ExportFormat>(), "format of image file");
+}
+
 std::string get_short_features_help(bool append_suffix) {
   std::ostringstream oss;
   bool first_feature = true;
@@ -492,5 +497,19 @@ void validate(boost::any& v, const std::vector<std::string>& values,
   throw po::validation_error(po::validation_error::invalid_option_value);
 }
 
+void validate(boost::any& v, const std::vector<std::string>& values,
+              ExportFormat *target_type, int) {
+  po::validators::check_first_occurrence(v);
+  const std::string &s = po::validators::get_single_string(values);
+
+  std::string parse_error;
+  uint64_t format = strict_sistrtoll(s.c_str(), &parse_error);
+  if (!parse_error.empty() || (format != 1 && format != 2)) {
+    throw po::validation_error(po::validation_error::invalid_option_value);
+  }
+
+  v = boost::any(format);
+}
+
 } // namespace argument_types
 } // namespace rbd

src/tools/rbd/ArgumentTypes.h

@@ -114,6 +114,13 @@ struct Format : public TypedValue<std::string> {
 
 struct JournalObjectSize {};
 
+struct ExportFormat {};
+
+void validate(boost::any& v, const std::vector<std::string>& values,
+              ExportFormat *target_type, int);
+
+void add_export_format_option(boost::program_options::options_description *opt);
+
 std::string get_name_prefix(ArgumentModifier modifier);
 std::string get_description_prefix(ArgumentModifier modifier);
 

src/tools/rbd/CMakeLists.txt

@@ -13,13 +13,11 @@ set(rbd_srcs
   action/Diff.cc
   action/DiskUsage.cc
   action/Export.cc
-  action/ExportDiff.cc
   action/Feature.cc
   action/Flatten.cc
   action/Group.cc
   action/ImageMeta.cc
   action/Import.cc
-  action/ImportDiff.cc
   action/Info.cc
   action/Journal.cc
   action/Kernel.cc

src/tools/rbd/Utils.h

@@ -16,6 +16,23 @@ namespace utils {
 
 static const std::string RBD_DIFF_BANNER ("rbd diff v1\n");
 
+static const std::string RBD_IMAGE_BANNER_V2 ("rbd image v2\n");
+static const std::string RBD_IMAGE_DIFFS_BANNER_V2 ("rbd image diffss v2\n");
+static const std::string RBD_DIFF_BANNER_V2 ("rbd diff v2\n");
+
+#define RBD_DIFF_FROM_SNAP	'f'
+#define RBD_DIFF_TO_SNAP	't'
+#define RBD_DIFF_IMAGE_SIZE	's'
+#define RBD_DIFF_WRITE		'w'
+#define RBD_DIFF_ZERO		'z'
+#define RBD_DIFF_END		'e'
+
+#define RBD_EXPORT_IMAGE_ORDER		'O'
+#define RBD_EXPORT_IMAGE_FEATURES	'T'
+#define RBD_EXPORT_IMAGE_STRIPE_UNIT	'U'
+#define RBD_EXPORT_IMAGE_STRIPE_COUNT	'C'
+#define RBD_EXPORT_IMAGE_END		'E'
+
 enum SnapshotPresence {
   SNAPSHOT_PRESENCE_NONE,
   SNAPSHOT_PRESENCE_PERMITTED,