Poll remote servers until live.

It takes time between the power on signal is sent to the remote servers
and they are available over ssh for rho. These changes make it so
servers are polled until they respond or a timeout limit is reached.

Closes #28

camayoc/tests/remote/conftest.py

@@ -12,6 +12,29 @@
 )
 
 
+def wait_until_live(servers, timeout=60):
+    """Wait for servers to be live.
+
+    For each server in the "servers" list, attempt to make a connection via
+    ping. Keep trying until a connection is made for all servers or the timeout
+    limit is reached.
+    """
+    from camayoc import cli
+
+    system = cli.System(hostname='localhost', transport='local')
+    client = cli.Client(system, response_handler=cli.echo_handler)
+
+    live = 0
+    while live < len(servers) and timeout > 0:
+        for server in servers:
+            ping = client.run(('ping', '-c 10', server))
+            if ping.returncode == 0:
+                live += 1
+        sleep(10)
+        timeout -= 10
+    return
+
+
 @pytest.fixture(scope='session', autouse=True)
 def manage_systems(request):
     """Fixture that manages remote systems.
@@ -100,15 +123,17 @@ def manage_systems(request):
         vm_host = fldr.childEntity[VCENTER_CLUSTER].host[VCENTER_HOST]
         # the host has a property "vm" which is a list of VMs
         vms = vm_host.vm
+        vc_hosts = []
         for host in cfg['rho']['hosts']:
             if ('provider' in host.keys()) and (
                     host['provider'] == 'vcenter'):
+                vc_hosts.append(host['ip'])
                 for vm in vms:
                     if vm.name == host['hostname']:
                         if vm.runtime.powerState == 'poweredOff':
                             vm.PowerOnVM_Task()
         # need to wait for machines to boot and start sshd
-        sleep(30)
+        wait_until_live(vc_hosts)
 
     def shutdown_systems():
         """Turn off hosts at the end of a session.