Merge pull request #218 from googlegenomics/preemptible_rollup

Update cluster.update() to try to set the node's preferred_ip

elasticluster/cluster.py

@@ -770,6 +770,14 @@ def update(self):
         for node in self.get_all_nodes():
             try:
                 node.update_ips()
+
+                # If we previously did not have a preferred_ip or the
+                # preferred_ip is not in the current list, then try to connect
+                # to one of the node ips and update the preferred_ip.
+                if node.ips and \
+                   not (node.preferred_ip and \
+                        node.preferred_ip in node.ips):
+                  node.connect()
             except InstanceError as ex:
                 log.warning("Ignoring error updating information on node %s: %s",
                           node, str(ex))

elasticluster/providers/gce.py

@@ -361,7 +361,15 @@ def stop_instance(self, instance_id):
                                         instance=instance_id, zone=self._zone)
             response = self._execute_request(request)
             self._check_response(response)
-        except (HttpError, CloudProviderError) as e:
+        except HttpError as e:
+            # If the instance does not exist, we can a 404 - just log it, and
+            # return without exception so the caller can remove the reference.
+            if e.resp.status == 404:
+              log.warning("Instance to stop `%s` was not found" % instance_id)
+            else:
+              raise InstanceError("Could not stop instance `%s`: `%s`"
+                                  % (instance_id, e))
+        except CloudProviderError as e:
             raise InstanceError("Could not stop instance `%s`: `%s`"
                                 % (instance_id, e))
 
@@ -408,9 +416,9 @@ def get_ips(self, instance_id):
 
             # If the instance is in status TERMINATED, then there will be
             # no IP addresses.
-            if response and response['status'] == 'TERMINATED':
+            if response and response['status'] in ('STOPPING', 'TERMINATED'):
               log.info("node '%s' state is '%s'; no IP address(es)" %
-                       (instance_id, "TERMINATED"))
+                       (instance_id, response['status']))
               return [None]
 
             if response and "networkInterfaces" in response:

elasticluster/subcommands.py

@@ -633,6 +633,8 @@ def execute(self):
                                         storage_path=self.params.storage,
                                         include_config_dirs=True)
         cluster_name = self.params.cluster
+
+        print("Updating cluster `%s`..." % cluster_name)
         try:
             cluster = configurator.load_cluster(cluster_name)
             cluster.update()