Support waiting until VM is network-accessible.

With the new flag ``downburst create --wait``, we add a final step
into cloud-init user-data, that ejects the ``cidata`` CD-ROM. Outside
the vm, we then poll for the CD-ROM tray position, waiting for it to
be open.

When using a reactive DNS updating mechanism (such as
https://github.com/ceph/propernoun ) that updates DNS after seeing the
VM creation and the DHCP lease, the DNS name won't exist until after
the VM has requested a DHCP lease. This combined with caching DNS
resolvers that enforce a multi-minute lower bound on DNS
negative-cache entries means that if you ask for the IP address of the
new VM too early, you won't see the correct answer for several
minutes. This is frustrating.

With ``--wait``, downburst waits for the cloud-init run to complete.
And we know that the DHCP lease completes before the cloud-init run
starts. This does not completely eliminate the above issue (there's
still a race if the DNS updating is significantly delayed), but in
practice ``--wait`` is enough to make things work.

doc/examples/eject-cdrom.user.yaml

@@ -0,0 +1,11 @@
+#cloud-config-archive
+
+# eject the cdrom (containing the cloud-init metadata)
+# as a signal that we've reached full functionality;
+# you can poll this via libvirt with
+#
+#  virsh qemu-monitor-command DOMAIN --cmd '{"execute": "query-block"}'
+#
+- |
+  #!/bin/sh
+  exec eject /dev/cdrom

downburst/create.py

@@ -9,6 +9,7 @@
 from . import exc
 from . import meta
 from . import template
+from . import wait
 
 
 log = logging.getLogger(__name__)
@@ -39,6 +40,16 @@ def create(args):
         name=args.name,
         extra_user=args.user_data,
         )
+
+    if args.wait:
+        user_data.append("""\
+#!/bin/sh
+# eject the cdrom (containing the cloud-init metadata)
+# as a signal that we've reached full functionality;
+# this is used by ``downburst create --wait``
+exec eject /dev/cdrom
+""")
+
     capacity = meta_data.get('downburst', {}).get('disk-size', '10G')
     capacity = dehumanize.parse(capacity)
 
@@ -71,6 +82,10 @@ def create(args):
     dom = conn.defineXML(etree.tostring(domainxml))
     dom.create()
 
+    if args.wait:
+        log.debug('Waiting for vm to be initialized...')
+        wait.wait_for_cdrom_eject(dom)
+
 
 def make(parser):
     """
@@ -88,6 +103,11 @@ def make(parser):
         action='append',
         help='extra meta-data, must contain a yaml mapping',
         )
+    parser.add_argument(
+        '--wait',
+        action='store_true',
+        help='wait for VM to initialize',
+        )
     parser.add_argument(
         'name',
         metavar='NAME',

downburst/wait.py

@@ -0,0 +1,48 @@
+import json
+import libvirt_qemu
+import time
+
+from . import exc
+
+
+def is_cdrom_tray_open(domain):
+    """
+    Returns True if even one CD-ROM tray is open.
+    """
+    res = libvirt_qemu.qemuMonitorCommand(
+        domain,
+        json.dumps(
+            {'execute': 'query-block'},
+            ),
+        # TODO should force this to be qmp, but python-libvirt 0.9.8
+        # doesn't seem to be able to do that
+        libvirt_qemu.VIR_DOMAIN_QEMU_MONITOR_COMMAND_DEFAULT,
+        )
+    res = json.loads(res)
+    if 'error' in res:
+        raise exc.DownburstError(
+            'Cannot query QEmu for block device state',
+            res['error'].get('desc'),
+            )
+
+    cdroms = [dev for dev in res['return'] if 'tray_open' in dev]
+    if not cdroms:
+        raise exc.DownburstError(
+            'VM must have at least one CD-ROM to check tray status',
+            res['error'].get('desc'),
+            )
+
+    for dev in cdroms:
+        if dev['tray_open']:
+            return True
+
+    return False
+
+
+def wait_for_cdrom_eject(domain):
+    cd_ejected = False
+    while not cd_ejected:
+        cd_ejected = is_cdrom_tray_open(domain)
+
+        if not cd_ejected:
+            time.sleep(1)