v2v: Add -o rhv-upload output mode (RHBZ#1557273).

PROBLEMS:
 - -of qcow2 does not work, with multiple problems
    * needs to set NBD size to something larger than virtual size
 - Not tested against imageio which supports zero/trim/flush.

This adds a new output mode to virt-v2v.  virt-v2v -o rhv-upload
streams images directly to an oVirt or RHV >= 4 Data Domain using the
oVirt SDK v4.  It is more efficient than -o rhv because it does not
need to go via the Export Storage Domain, and is possible for humans
to use unlike -o vdsm.

The implementation uses the Python SDK (‘ovirtsdk4’ module).  An
nbdkit Python 3 plugin translates NBD calls from qemu into HTTPS
requests to oVirt via the SDK.

(cherry picked from commit 719a2725714cb3d0ae06cb6aa01e2849ec167bec)

.gitignore

@@ -614,6 +614,9 @@ Makefile.in
 /utils/qemu-speed-test/qemu-speed-test
 /v2v/.depend
 /v2v/oUnit-*
+/v2v/output_rhv_upload_createvm_source.ml
+/v2v/output_rhv_upload_plugin_source.ml
+/v2v/output_rhv_upload_precheck_source.ml
 /v2v/real-*.d/
 /v2v/real-*.img
 /v2v/real-*.xml

v2v/Makefile.am

@@ -20,6 +20,10 @@ include $(top_srcdir)/subdir-rules.mk
 EXTRA_DIST = \
 	$(SOURCES_MLI) $(SOURCES_ML) $(SOURCES_C) \
 	copy_to_local.ml \
+	embed-code.sh \
+	rhv-upload-createvm.py \
+	rhv-upload-plugin.py \
+	rhv-upload-precheck.py \
 	v2v_unit_tests.ml \
 	virt-v2v.pod \
 	virt-v2v-copy-to-local.pod
@@ -53,6 +57,10 @@ SOURCES_MLI = \
 	output_null.mli \
 	output_qemu.mli \
 	output_rhv.mli \
+	output_rhv_upload.mli \
+	output_rhv_upload_createvm_source.mli \
+	output_rhv_upload_plugin_source.mli \
+	output_rhv_upload_precheck_source.mli \
 	output_vdsm.mli \
 	OVF.mli \
 	parse_ovf_from_ova.mli \
@@ -105,6 +113,10 @@ SOURCES_ML = \
 	output_local.ml \
 	output_qemu.ml \
 	output_rhv.ml \
+	output_rhv_upload_createvm_source.ml \
+	output_rhv_upload_plugin_source.ml \
+	output_rhv_upload_precheck_source.ml \
+	output_rhv_upload.ml \
 	output_vdsm.ml \
 	inspect_source.ml \
 	target_bus_assignment.ml \
@@ -115,6 +127,15 @@ SOURCES_C = \
 	libvirt_utils-c.c \
 	utils-c.c
 
+# These files are generated and contain rhv-upload-*.py embedded as an
+# OCaml string.
+output_rhv_upload_createvm_source.ml: rhv-upload-createvm.py
+	./embed.sh code $^ $@
+output_rhv_upload_plugin_source.ml: rhv-upload-plugin.py
+	./embed.sh code $^ $@
+output_rhv_upload_precheck_source.ml: rhv-upload-precheck.py
+	./embed.sh code $^ $@
+
 if HAVE_OCAML
 
 bin_PROGRAMS = virt-v2v virt-v2v-copy-to-local
@@ -264,6 +285,7 @@ TESTS_ENVIRONMENT = $(top_builddir)/run --test
 
 TESTS = \
 	test-v2v-docs.sh \
+	test-v2v-python-syntax.sh \
 	test-v2v-i-ova-bad-sha1.sh \
 	test-v2v-i-ova-bad-sha256.sh \
 	test-v2v-i-ova-formats.sh \
@@ -276,6 +298,7 @@ TESTS = \
 	test-v2v-i-ova-two-disks.sh \
 	test-v2v-i-vmx.sh \
 	test-v2v-it-vddk-io-query.sh \
+	test-v2v-o-rhv-upload-oo-query.sh \
 	test-v2v-o-vdsm-oo-query.sh \
 	test-v2v-bad-networks-and-bridges.sh
 
@@ -441,6 +464,7 @@ EXTRA_DIST += \
 	test-v2v-o-null.sh \
 	test-v2v-o-qemu.sh \
 	test-v2v-o-rhv.sh \
+	test-v2v-o-rhv-upload-oo-query.sh \
 	test-v2v-o-vdsm-oo-query.sh \
 	test-v2v-o-vdsm-options.sh \
 	test-v2v-oa-option.sh \
@@ -449,6 +473,7 @@ EXTRA_DIST += \
 	test-v2v-print-source.expected \
 	test-v2v-print-source.sh \
 	test-v2v-print-source.xml \
+	test-v2v-python-syntax.sh \
 	test-v2v-conversion-of.sh \
 	test-v2v-sound.sh \
 	test-v2v-sound.xml \

v2v/cmdline.ml

@@ -134,6 +134,8 @@ let parse_cmdline () =
     | "disk" | "local" -> output_mode := `Local
     | "null" -> output_mode := `Null
     | "ovirt" | "rhv" | "rhev" -> output_mode := `RHV
+    | "ovirt-upload" | "ovirt_upload" | "rhv-upload" | "rhv_upload" ->
+       output_mode := `RHV_Upload
     | "qemu" -> output_mode := `QEmu
     | "vdsm" -> output_mode := `VDSM
     | s ->
@@ -383,6 +385,16 @@ read the man page virt-v2v(1).
     | `Null -> no_options (); `Null
     | `RHV -> no_options (); `RHV
     | `QEmu -> no_options (); `QEmu
+    | `RHV_Upload ->
+       if is_query then (
+         Output_rhv_upload.print_output_options ();
+         exit 0
+       )
+       else (
+         let rhv_options =
+           Output_rhv_upload.parse_output_options output_options in
+         `RHV_Upload rhv_options
+       )
     | `VDSM ->
        if is_query then (
          Output_vdsm.print_output_options ();
@@ -572,6 +584,30 @@ read the man page virt-v2v(1).
       Output_rhv.output_rhv os output_alloc,
       output_format, output_alloc
 
+    | `RHV_Upload rhv_options ->
+      let output_conn =
+        match output_conn with
+        | None ->
+           error (f_"-o rhv-upload: use ‘-oc’ to point to the oVirt or RHV server REST API URL, which is usually https://servername/ovirt-engine/api")
+        | Some oc -> oc in
+      (* In theory we could make the password optional in future. *)
+      let output_password =
+        match output_password with
+        | None ->
+           error (f_"-o rhv-upload: output password file was not specified, use ‘-op’ to point to a file which contains the password used to connect to the oVirt or RHV server")
+        | Some op -> op in
+      let os =
+        match output_storage with
+        | None ->
+           error (f_"-o rhv-upload: output storage was not specified, use ‘-os’");
+        | Some os -> os in
+      if qemu_boot then
+        error_option_cannot_be_used_in_output_mode "rhv-upload" "--qemu-boot";
+      Output_rhv_upload.output_rhv_upload output_alloc output_conn
+                                          output_password os
+                                          rhv_options,
+      output_format, output_alloc
+
     | `VDSM vdsm_options ->
       if output_password <> None then
         error_option_cannot_be_used_in_output_mode "vdsm" "-op";

v2v/embed.sh

@@ -0,0 +1,45 @@
+#!/bin/bash -
+# Embed code or other content into an OCaml file.
+# Copyright (C) 2018 Red Hat Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+# Embed code or other content into an OCaml file.
+#
+# It is embedded into a string.  As OCaml string literals have virtually
+# no restrictions on length or content we only have to escape double
+# quotes for backslash characters.
+
+if [ $# -ne 3 ]; then
+    echo "embed.sh identifier input output"
+    exit 1
+fi
+
+ident="$1"
+input="$2"
+output="$3"
+
+rm -f "$output" "$output"-t
+
+exec >"$output"-t
+
+echo "(* Generated by embed.sh from $input *)"
+echo
+echo let "$ident" = '"'
+sed -e 's/\(["\]\)/\\\1/g' < "$input"
+echo '"'
+
+chmod -w "$output"-t
+mv "$output"-t "$output"