XContentMapValues.filter now works with nested arrays

The filter method of XContentMapValues actually filtered out nested
arrays/lists completely due to a bug in the filter method, which threw
away all data inside of such an array.

Closes #2944
This bug was a follow up problem, because of the filtering of nested arrays
in case source exclusion was configured.

src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java

@@ -206,6 +206,9 @@ private static void filter(List<Object> from, List<Object> to, String[] includes
             } else if (o instanceof List) {
                 List<Object> innerInto = new ArrayList<Object>();
                 filter((List<Object>) o, innerInto, includes, excludes, sb);
+                if (!innerInto.isEmpty()) {
+                    to.add(innerInto);
+                }
             } else {
                 to.add(o);
             }

src/test/java/org/elasticsearch/test/integration/search/geo/GeoShapeIntegrationTests.java

@@ -27,17 +27,21 @@
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
 
+import com.spatial4j.core.shape.Shape;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.geo.GeoJSONShapeSerializer;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.test.integration.AbstractSharedClusterTest;
 import org.testng.annotations.BeforeTest;
 import org.testng.annotations.Test;
 
-import com.spatial4j.core.shape.Shape;
+import java.util.List;
+import java.util.Map;
 
 public class GeoShapeIntegrationTests extends AbstractSharedClusterTest {
 
@@ -190,4 +194,48 @@ public void testIndexedShapeReference() throws Exception {
         assertThat(searchResponse.getHits().hits().length, equalTo(1));
         assertThat(searchResponse.getHits().getAt(0).id(), equalTo("1"));
     }
+
+    @Test // Issue 2944
+    public void testThatShapeIsReturnedEvenWhenExclusionsAreSet() throws Exception {
+        client().admin().indices().prepareDelete().execute().actionGet();
+
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
+                .startObject("properties").startObject("location")
+                .field("type", "geo_shape")
+                .endObject().endObject()
+                .startObject("_source")
+                    .startArray("excludes").value("nonExistingField").endArray()
+                .endObject()
+                .endObject().endObject()
+                .string();
+        client().admin().indices().prepareCreate("test").addMapping("type1", mapping).execute().actionGet();
+        client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().execute().actionGet();
+
+        client().prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject()
+                .field("name", "Document 1")
+                .startObject("location")
+                .field("type", "envelope")
+                .startArray("coordinates").startArray().value(-45.0).value(45).endArray().startArray().value(45).value(-45).endArray().endArray()
+                .endObject()
+                .endObject()).execute().actionGet();
+
+        client().admin().indices().prepareRefresh("test").execute().actionGet();
+
+        SearchResponse searchResponse = client().prepareSearch("test").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet();
+        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
+
+        Map<String, Object> indexedMap = searchResponse.getHits().getAt(0).sourceAsMap();
+        assertThat(indexedMap.get("location"), instanceOf(Map.class));
+        Map<String, Object> locationMap = (Map<String, Object>) indexedMap.get("location");
+        assertThat(locationMap.get("coordinates"), instanceOf(List.class));
+        List<List<Number>> coordinates = (List<List<Number>>) locationMap.get("coordinates");
+        assertThat(coordinates.size(), equalTo(2));
+        assertThat(coordinates.get(0).size(), equalTo(2));
+        assertThat(coordinates.get(0).get(0).doubleValue(), equalTo(-45.0));
+        assertThat(coordinates.get(0).get(1).doubleValue(), equalTo(45.0));
+        assertThat(coordinates.get(1).size(), equalTo(2));
+        assertThat(coordinates.get(1).get(0).doubleValue(), equalTo(45.0));
+        assertThat(coordinates.get(1).get(1).doubleValue(), equalTo(-45.0));
+        assertThat(locationMap.size(), equalTo(2));
+    }
 }

src/test/java/org/elasticsearch/test/unit/common/xcontent/support/XContentMapValuesTests.java

@@ -20,8 +20,10 @@
 package org.elasticsearch.test.unit.common.xcontent.support;
 
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.testng.annotations.Test;
@@ -195,4 +197,18 @@ public void testExtractRawValue() throws Exception {
         map = XContentFactory.xContent(XContentType.JSON).createParser(builder.string()).mapAndClose();
         assertThat(XContentMapValues.extractRawValues("path1.xxx.path2.yyy.test", map).get(0).toString(), equalTo("value"));
     }
+
+    @Test
+    public void testThatFilteringWithNestedArrayAndExclusionWorks() throws Exception {
+        XContentBuilder builder = XContentFactory.jsonBuilder().startObject()
+                .startArray("coordinates")
+                    .startArray().value("foo").endArray()
+                .endArray()
+            .endObject();
+
+        Tuple<XContentType, Map<String, Object>> mapTuple = XContentHelper.convertToMap(builder.bytes(), true);
+        Map<String, Object> filteredSource = XContentMapValues.filter(mapTuple.v2(), Strings.EMPTY_ARRAY, new String[]{"nonExistingField"});
+
+        assertThat(mapTuple.v2(), equalTo(filteredSource));
+    }
 }