Add multiple zones support

Closes #27.

README.md

@@ -242,6 +242,26 @@ gcutil deletedisk boot-myesnode1 boot-myesnode2  \
        --zone=europe-west1-a
 ```
 
+Using zones
+-----------
+
+`cloud.gce.zone` helps to retrieve instances running in a given zone. It should be one of the 
+[GCE supported zones](https://developers.google.com/compute/docs/zones#available).
+
+The GCE discovery can support multi zones although you need to be aware of network latency between zones. 
+To enable discovery across more than one zone, just enter add your zone list to `cloud.gce.zone` setting:
+
+```yaml
+  cloud:
+      gce:
+          project_id: <your-google-project-id>
+          zone: <your-zone1>, <your-zone2>
+  discovery:
+          type: gce
+```
+
+
+
 Filtering by tags
 -----------------
 

src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java

@@ -28,6 +28,9 @@
 import com.google.api.services.compute.model.Instance;
 import com.google.api.services.compute.model.InstanceList;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.base.Function;
+import org.elasticsearch.common.collect.Iterables;
+import org.elasticsearch.common.collect.Lists;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -36,8 +39,8 @@
 import org.elasticsearch.discovery.DiscoveryException;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Collection;
+import java.util.List;
 
 /**
  *
@@ -46,7 +49,7 @@ public class GceComputeServiceImpl extends AbstractLifecycleComponent<GceCompute
     implements GceComputeService {
 
     private final String project;
-    private final String zone;
+    private final List<String> zoneList;
 
     // Forcing Google Token API URL as set in GCE SDK to
     //      http://metadata/computeMetadata/v1/instance/service-accounts/default/token
@@ -55,18 +58,34 @@ public class GceComputeServiceImpl extends AbstractLifecycleComponent<GceCompute
 
     @Override
     public Collection<Instance> instances() {
-        try {
-            logger.debug("get instances for project [{}], zone [{}]", project, zone);
 
-            Compute.Instances.List list = client().instances().list(project, zone);
-            InstanceList instanceList = list.execute();
+            logger.debug("get instances for project [{}], zoneList [{}]", project, zoneList);
 
-            return instanceList.getItems();
-        } catch (IOException e) {
-            logger.warn("disabling GCE discovery. Can not get list of nodes: {}", e.getMessage());
-            logger.debug("Full exception:", e);
-            return new ArrayList<Instance>();
-        }
+            List<List<Instance>> instanceListByZone = Lists.transform(zoneList, new Function<String, List<Instance>>() {
+                @Override
+                public List<Instance> apply(String zoneId) {
+                    try {
+                        Compute.Instances.List list = client().instances().list(project, zoneId);
+                        InstanceList instanceList = list.execute();
+
+                        return instanceList.getItems();
+                    } catch (IOException e) {
+                        logger.warn("Problem fetching instance list for zone {}", zoneId);
+                        logger.debug("Full exception:", e);
+
+                        return Lists.newArrayList();
+                    }
+                }
+            });
+
+            //Collapse instances from all zones into one neat list
+            List<Instance> instanceList = Lists.newArrayList(Iterables.concat(instanceListByZone));
+
+            if (instanceList.size() == 0) {
+                logger.warn("disabling GCE discovery. Can not get list of nodes");
+            }
+
+            return instanceList;
     }
 
     private Compute client;
@@ -85,7 +104,10 @@ public GceComputeServiceImpl(Settings settings, SettingsFilter settingsFilter) {
         settingsFilter.addFilter(new GceSettingsFilter());
 
         this.project = componentSettings.get(Fields.PROJECT, settings.get("cloud.gce." + Fields.PROJECT));
-        this.zone = componentSettings.get(Fields.ZONE, settings.get("cloud.gce." + Fields.ZONE));
+
+        String[] zoneList = componentSettings.getAsArray(Fields.ZONE, settings.getAsArray("cloud.gce." + Fields.ZONE));
+        this.zoneList = Lists.newArrayList(zoneList);
+
     }
 
     public synchronized Compute client() {

src/test/java/org/elasticsearch/discovery/gce/GceComputeEngineTest.java

@@ -25,7 +25,9 @@
 import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.discovery.gce.mock.GceComputeServiceTwoNodesDifferentTagsMock;
+import org.elasticsearch.discovery.gce.mock.GceComputeServiceTwoNodesOneZoneMock;
 import org.elasticsearch.discovery.gce.mock.GceComputeServiceTwoNodesSameTagsMock;
+import org.elasticsearch.discovery.gce.mock.GceComputeServiceTwoNodesTwoZonesMock;
 import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.test.ElasticsearchIntegrationTest;
 import org.junit.Ignore;
@@ -179,4 +181,35 @@ public void nodes_with_same_tags_and_two_tags_set() {
         // We expect having 2 nodes as part of the cluster, let's test that
         checkNumberOfNodes(2);
     }
+
+    @Test @Ignore
+    public void multiple_zones_and_two_nodes_in_same_zone() {
+        startNode(1,
+                GceComputeServiceTwoNodesOneZoneMock.class,
+                ImmutableSettings.settingsBuilder().put("cloud.gce.zones", Lists.newArrayList("us-central1-a", "us-central1-b",
+                        "us-central1-f", "europe-west1-a", "europe-west1-b")).build());
+        startNode(2,
+                GceComputeServiceTwoNodesOneZoneMock.class,
+                ImmutableSettings.settingsBuilder().put("cloud.gce.zones", Lists.newArrayList("us-central1-a", "us-central1-b",
+                        "us-central1-f", "europe-west1-a", "europe-west1-b")).build());
+
+        // We expect having 2 nodes as part of the cluster, let's test that
+        checkNumberOfNodes(2);
+    }
+
+    @Test @Ignore
+    public void multiple_zones_and_two_nodes_in_different_zones() {
+        startNode(1,
+                GceComputeServiceTwoNodesTwoZonesMock.class,
+                ImmutableSettings.settingsBuilder().put("cloud.gce.zones", Lists.newArrayList("us-central1-a", "us-central1-b",
+                        "us-central1-f", "europe-west1-a", "europe-west1-b")).build());
+        startNode(2,
+                GceComputeServiceTwoNodesTwoZonesMock.class,
+                ImmutableSettings.settingsBuilder().put("cloud.gce.zones", Lists.newArrayList("us-central1-a", "us-central1-b",
+                        "us-central1-f", "europe-west1-a", "europe-west1-b")).build());
+
+        // We expect having 2 nodes as part of the cluster, let's test that
+        checkNumberOfNodes(2);
+    }
+
 }

src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceAbstractMock.java

@@ -25,6 +25,7 @@
 import com.google.api.services.compute.model.Tags;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.cloud.gce.GceComputeService;
+import org.elasticsearch.common.collect.Lists;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.discovery.gce.GceComputeEngineTest;
@@ -33,35 +34,48 @@
 import java.util.Collection;
 import java.util.List;
 
+import static com.carrotsearch.randomizedtesting.RandomizedTest.randomInt;
+
 /**
  *
  */
 public abstract class GceComputeServiceAbstractMock extends AbstractLifecycleComponent<GceComputeServiceAbstractMock>
     implements GceComputeService {
 
     protected abstract List<ArrayList<String>> getTags();
+    protected abstract List<String> getZones();
+    private final List<String> zoneList;
 
     protected GceComputeServiceAbstractMock(Settings settings) {
         super(settings);
-        logger.debug("starting GCE Api Mock with {} nodes:", getTags().size());
-        for (List<String> tags : getTags()) {
-            logger.debug(" - {}", tags);
+
+        int numNodes = getTags().size() > getZones().size() ? getTags().size() : getZones().size();
+
+        logger.debug("starting GCE Api Mock with {} nodes:", numNodes);
+        for (int i = 0; i < numNodes; i++) {
+            List<String> tags = getTags().size() > i ? getTags().get(i) : null;
+            String zone = getZones().size() > i ? getZones().get(i) : null;
+            logger.debug(" - node #{}: tags [{}], zone [{}]", i, tags, zone);
         }
+
+        String[] zoneList = componentSettings.getAsArray(Fields.ZONE, settings.getAsArray("cloud.gce." + Fields.ZONE));
+        this.zoneList = Lists.newArrayList(zoneList);
     }
 
     private Collection<Instance> instances = null;
 
     private void computeInstances() {
         instances = new ArrayList<Instance>();
-
         int nodeNumber = 0;
         // For each instance (item of tags)
         for (List<String> tags : getTags()) {
-            logger.info(" ----> GCE Mock API: Adding node {}", nodeNumber);
+            String zone = zoneList.isEmpty() ? "dummy" : zoneList.get(randomInt(zoneList.size()-1));
+            logger.info(" ----> GCE Mock API: Adding node [{}] in zone [{}]", nodeNumber, zone);
             Instance instance = new Instance();
             instance.setName("Mock Node " + tags);
             instance.setMachineType("Mock Type machine");
             instance.setStatus("STARTED");
+            instance.setZone(zone);
             Tags instanceTags = new Tags();
             instanceTags.setItems(tags);
             instance.setTags(instanceTags);

src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesDifferentTagsMock.java

@@ -39,6 +39,11 @@ protected List<ArrayList<String>> getTags() {
         return tags;
     }
 
+    @Override
+    protected List<String> getZones() {
+        return Lists.newArrayList();
+    }
+
     @Inject
     protected GceComputeServiceTwoNodesDifferentTagsMock(Settings settings) {
         super(settings);

src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesOneZoneMock.java

@@ -0,0 +1,49 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.discovery.gce.mock;
+
+import org.elasticsearch.common.collect.Lists;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ *
+ */
+public class GceComputeServiceTwoNodesOneZoneMock extends GceComputeServiceAbstractMock {
+    private static List<String> zones = Lists.newArrayList("us-central1-a","us-central1-a");
+
+    @Override
+    protected List<ArrayList<String>> getTags() {
+        return Lists.newArrayList();
+    }
+
+    @Override
+    protected List<String> getZones() {
+        return zones;
+    }
+
+    @Inject
+    protected GceComputeServiceTwoNodesOneZoneMock(Settings settings) {
+        super(settings);
+    }
+}

src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesSameTagsMock.java

@@ -40,6 +40,11 @@ protected List<ArrayList<String>> getTags() {
         return tags;
     }
 
+    @Override
+    protected List<String> getZones() {
+        return Lists.newArrayList();
+    }
+
     @Inject
     protected GceComputeServiceTwoNodesSameTagsMock(Settings settings) {
         super(settings);

src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesTwoZonesMock.java

@@ -0,0 +1,49 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.discovery.gce.mock;
+
+import org.elasticsearch.common.collect.Lists;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ *
+ */
+public class GceComputeServiceTwoNodesTwoZonesMock extends GceComputeServiceAbstractMock {
+    private static List<String> zones = Lists.newArrayList("us-central1-a","europe-west1-a");
+
+    @Override
+    protected List<ArrayList<String>> getTags() {
+        return Lists.newArrayList();
+    }
+
+    @Override
+    protected List<String> getZones() {
+        return zones;
+    }
+
+    @Inject
+    protected GceComputeServiceTwoNodesTwoZonesMock(Settings settings) {
+        super(settings);
+    }
+}