From a50cdafe0cb40b9eef789b05729a80365eaf2bc3 Mon Sep 17 00:00:00 2001 From: Ruben Schuller Date: Tue, 4 Apr 2017 09:08:23 +0200 Subject: [PATCH 1/3] bugfixes and more debug logging --- idbiaas/idbiaas.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/idbiaas/idbiaas.py b/idbiaas/idbiaas.py index fb2bddf..9f2cf34 100755 --- a/idbiaas/idbiaas.py +++ b/idbiaas/idbiaas.py @@ -110,11 +110,14 @@ def machines(self): idb_machines = [] for host in self.hosts: + logging.debug("LibvirtZone: retrieving nodes from %s", host.uri()) + driver = libcloud.compute.providers.get_driver( libcloud.compute.types.Provider.LIBVIRT)(uri=host.uri()) nodes = driver.list_nodes() for node in nodes: + logging.debug("LibvirtZone: got node %s", node) idb_machines.append(IDBMachine( node.name, driver.ex_get_hypervisor_hostname(), node.extra['vcpu_count'], node.extra['used_memory'])) @@ -133,6 +136,7 @@ def __init__(self, token, version): self.version = version def machines(self): + logging.debug("DigitalOceanZone: retrieving nodes") idb_machines = [] driver = libcloud.compute.providers.get_driver( @@ -140,6 +144,7 @@ def machines(self): nodes = driver.list_nodes() for node in nodes: + logging.debug("DigitalOceanZone: got node %s", node) idb_machines.append(IDBMachine(node.name, "", node.extra["vcpus"], node.extra["memory"])) return idb_machines @@ -204,7 +209,7 @@ def submit_machines(self, machines): logging.info("Sending machines in zone %s to IDB API at %s", self.__class__.__name__, self.url) - for machines_chunk in self.grouper(machines, 10): + for machines_chunk in self.grouper(machines, 2): json_machines = self.json_machines(machines_chunk) req = requests.Request("PUT", self.url + "/machines", headers={ @@ -222,7 +227,7 @@ def submit_machines(self, machines): s.verify = self.verify res = s.send(prepared) - logging.info("{}\n{}".format(res.status_code, res.text)) + logging.info("%s\n%s", res.status_code, res.text.encode('utf-8')) class IDBIaas(object): @@ -258,7 +263,7 @@ def run_zones(cls, zones): logging.info("Found machines in zone %s:", zone.__class__.__name__) for machine in machines: logging.info(machine.fqdn) - zone.submit_machines(machines) + zone.idb.submit_machines(machines) def run(self): zones = IDBIaas.zones_from_dict(self.config) From 251a8e1f24e9f6365b693ac5b15188633108b7cc Mon Sep 17 00:00:00 2001 From: Ruben Schuller Date: Tue, 4 Apr 2017 09:14:23 +0200 Subject: [PATCH 2/3] update version in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b13b4ba..c7f595b 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name="idbiaas", - version="0.0.1", + version="0.0.2", description="IDB IAAS adapter", packages=["idbiaas"], author="bytemine GmbH", From 46c1362577e53e7f15ca40fa7ab6a4467a311e17 Mon Sep 17 00:00:00 2001 From: Ruben Schuller Date: Tue, 30 May 2017 10:47:16 +0200 Subject: [PATCH 3/3] catch exception when ssh aborts on host key change, skip None values in json generation --- idbiaas/idbiaas.py | 38 ++++++++++++++++++++++---------------- setup.py | 2 +- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/idbiaas/idbiaas.py b/idbiaas/idbiaas.py index 9f2cf34..1ee9520 100755 --- a/idbiaas/idbiaas.py +++ b/idbiaas/idbiaas.py @@ -112,15 +112,18 @@ def machines(self): for host in self.hosts: logging.debug("LibvirtZone: retrieving nodes from %s", host.uri()) - driver = libcloud.compute.providers.get_driver( - libcloud.compute.types.Provider.LIBVIRT)(uri=host.uri()) - - nodes = driver.list_nodes() - for node in nodes: - logging.debug("LibvirtZone: got node %s", node) - idb_machines.append(IDBMachine( - node.name, driver.ex_get_hypervisor_hostname(), - node.extra['vcpu_count'], node.extra['used_memory'])) + try: + driver = libcloud.compute.providers.get_driver( + libcloud.compute.types.Provider.LIBVIRT)(uri=host.uri()) + + nodes = driver.list_nodes() + for node in nodes: + logging.debug("LibvirtZone: got node %s", node) + idb_machines.append(IDBMachine( + node.name, driver.ex_get_hypervisor_hostname(), + node.extra['vcpu_count'], node.extra['used_memory'])) + except Exception as e: + logging.debug("LibvirtZone: %s, continuing with next host", e) return idb_machines @@ -139,13 +142,16 @@ def machines(self): logging.debug("DigitalOceanZone: retrieving nodes") idb_machines = [] - driver = libcloud.compute.providers.get_driver( - libcloud.compute.types.Provider.DIGITAL_OCEAN)(self.token, api_version=self.version) + try: + driver = libcloud.compute.providers.get_driver( + libcloud.compute.types.Provider.DIGITAL_OCEAN)(self.token, api_version=self.version) - nodes = driver.list_nodes() - for node in nodes: - logging.debug("DigitalOceanZone: got node %s", node) - idb_machines.append(IDBMachine(node.name, "", node.extra["vcpus"], node.extra["memory"])) + nodes = driver.list_nodes() + for node in nodes: + logging.debug("DigitalOceanZone: got node %s", node) + idb_machines.append(IDBMachine(node.name, "", node.extra["vcpus"], node.extra["memory"])) + except Exception as e: + logging.debug("DigitalOceanZone: %s, continuing with next host", e) return idb_machines @@ -201,7 +207,7 @@ def grouper(self, iterable, n, fillvalue=None): def json_machines(self, machines): """Converts machines list to IDB compatible json.""" - return json.dumps({"create_machine": self.create, "machines": [x.dict() for x in machines]}) + return json.dumps({"create_machine": self.create, "machines": [x.dict() for x in machines if x != None]}) def submit_machines(self, machines): """Submit machines to the IDB.""" diff --git a/setup.py b/setup.py index c7f595b..23326e9 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name="idbiaas", - version="0.0.2", + version="0.0.3", description="IDB IAAS adapter", packages=["idbiaas"], author="bytemine GmbH",