Add exclusion filters support to filter_path

This commit adds the support for exclusion filter to the response filtering (filter_path) feature. It changes the XContentBuilder APIs so that it now accepts two types of filters: inclusive and exclusive. Filters are no more String arrays but sets of String instead.

core/src/main/java/org/elasticsearch/common/xcontent/XContent.java

@@ -19,12 +19,15 @@
 
 package org.elasticsearch.common.xcontent;
 
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.Reader;
+import java.util.Collections;
+import java.util.Set;
 
 /**
  * A generic abstraction on top of handling content, inspired by JSON and pull parsing.
@@ -42,27 +45,20 @@ public interface XContent {
      * Creates a new generator using the provided output stream.
      */
     default XContentGenerator createGenerator(OutputStream os) throws IOException {
-        return createGenerator(os, null, true);
+        return createGenerator(os, Collections.emptySet(), Collections.emptySet());
     }
 
     /**
-     * Creates a new generator using the provided output stream and some
-     * inclusive filters. Same as createGenerator(os, filters, true).
-     */
-    default XContentGenerator createGenerator(OutputStream os, String[] filters) throws IOException {
-        return createGenerator(os, filters, true);
-    }
-
-    /**
-     * Creates a new generator using the provided output stream and some
-     * filters.
+     * Creates a new generator using the provided output stream and some inclusive and/or exclusive filters. When both exclusive and
+     * inclusive filters are provided, the underlying generator will first use exclusion filters to remove fields and then will check the
+     * remaining fields against the inclusive filters.
      *
-     * @param inclusive
-     *            If true only paths matching a filter will be included in
-     *            output. If false no path matching a filter will be included in
-     *            output
+     * @param os       the output stream
+     * @param includes the inclusive filters: only fields and objects that match the inclusive filters will be written to the output.
+     * @param excludes the exclusive filters: only fields and objects that don't match the exclusive filters will be written to the output.
      */
-    XContentGenerator createGenerator(OutputStream os, String[] filters, boolean inclusive) throws IOException;
+    XContentGenerator createGenerator(OutputStream os, Set<String> includes, Set<String> excludes) throws IOException;
+
     /**
      * Creates a parser over the provided string content.
      */

core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java

@@ -19,21 +19,8 @@
 
 package org.elasticsearch.common.xcontent;
 
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.math.BigDecimal;
-import java.math.RoundingMode;
-import java.nio.file.Path;
-import java.util.Calendar;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Locale;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.io.BytesStream;
@@ -47,6 +34,21 @@
 import org.joda.time.format.DateTimeFormatter;
 import org.joda.time.format.ISODateTimeFormat;
 
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.math.BigDecimal;
+import java.math.RoundingMode;
+import java.nio.file.Path;
+import java.util.Calendar;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
 /**
  * A utility to build XContent (ie json).
  */
@@ -58,12 +60,8 @@ public static XContentBuilder builder(XContent xContent) throws IOException {
         return new XContentBuilder(xContent, new BytesStreamOutput());
     }
 
-    public static XContentBuilder builder(XContent xContent, String[] filters) throws IOException {
-        return new XContentBuilder(xContent, new BytesStreamOutput(), filters);
-    }
-
-    public static XContentBuilder builder(XContent xContent, String[] filters, boolean inclusive) throws IOException {
-        return new XContentBuilder(xContent, new BytesStreamOutput(), filters, inclusive);
+    public static XContentBuilder builder(XContent xContent, Set<String> includes, Set<String> excludes) throws IOException {
+        return new XContentBuilder(xContent, new BytesStreamOutput(), includes, excludes);
     }
 
     private XContentGenerator generator;
@@ -77,7 +75,7 @@ public static XContentBuilder builder(XContent xContent, String[] filters, boole
      * to call {@link #close()} when the builder is done with.
      */
     public XContentBuilder(XContent xContent, OutputStream bos) throws IOException {
-        this(xContent, bos, null);
+        this(xContent, bos, Collections.emptySet(), Collections.emptySet());
     }
 
     /**
@@ -86,20 +84,24 @@ public XContentBuilder(XContent xContent, OutputStream bos) throws IOException {
      * filter will be written to the output stream. Make sure to call
      * {@link #close()} when the builder is done with.
      */
-    public XContentBuilder(XContent xContent, OutputStream bos, String[] filters) throws IOException {
-        this(xContent, bos, filters, true);
+    public XContentBuilder(XContent xContent, OutputStream bos, Set<String> includes) throws IOException {
+        this(xContent, bos, includes, Collections.emptySet());
     }
 
     /**
-     * Constructs a new builder using the provided xcontent, an OutputStream and
-     * some filters. If {@code filters} are specified and {@code inclusive} is
-     * true, only those values matching a filter will be written to the output
-     * stream. If {@code inclusive} is false, those matching will be excluded.
+     * Creates a new builder using the provided XContent, output stream and some inclusive and/or exclusive filters. When both exclusive and
+     * inclusive filters are provided, the underlying builder will first use exclusion filters to remove fields and then will check the
+     * remaining fields against the inclusive filters.
+     * <p>
      * Make sure to call {@link #close()} when the builder is done with.
+     *
+     * @param os       the output stream
+     * @param includes the inclusive filters: only fields and objects that match the inclusive filters will be written to the output.
+     * @param excludes the exclusive filters: only fields and objects that don't match the exclusive filters will be written to the output.
      */
-    public XContentBuilder(XContent xContent, OutputStream bos, String[] filters, boolean inclusive) throws IOException {
-        this.bos = bos;
-        this.generator = xContent.createGenerator(bos, filters, inclusive);
+    public XContentBuilder(XContent xContent, OutputStream os, Set<String> includes, Set<String> excludes) throws IOException {
+        this.bos = os;
+        this.generator = xContent.createGenerator(bos, includes, excludes);
     }
 
     public XContentType contentType() {

core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java

@@ -35,6 +35,7 @@
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.Reader;
+import java.util.Set;
 
 /**
  * A CBOR based content implementation using Jackson.
@@ -70,8 +71,8 @@ public byte streamSeparator() {
     }
 
     @Override
-    public XContentGenerator createGenerator(OutputStream os, String[] filters, boolean inclusive) throws IOException {
-        return new CborXContentGenerator(cborFactory.createGenerator(os, JsonEncoding.UTF8), os, filters, inclusive);
+    public XContentGenerator createGenerator(OutputStream os, Set<String> includes, Set<String> excludes) throws IOException {
+        return new CborXContentGenerator(cborFactory.createGenerator(os, JsonEncoding.UTF8), os, includes, excludes);
     }
 
     @Override

core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentGenerator.java

@@ -20,23 +20,22 @@
 package org.elasticsearch.common.xcontent.cbor;
 
 import com.fasterxml.jackson.core.JsonGenerator;
-
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContentGenerator;
 
 import java.io.OutputStream;
+import java.util.Collections;
+import java.util.Set;
 
-/**
- *
- */
 public class CborXContentGenerator extends JsonXContentGenerator {
 
-    public CborXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, String... filters) {
-        this(jsonGenerator, os, filters, true);
+    public CborXContentGenerator(JsonGenerator jsonGenerator, OutputStream os) {
+        this(jsonGenerator, os, Collections.emptySet(), Collections.emptySet());
     }
 
-    public CborXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, String[] filters, boolean inclusive) {
-        super(jsonGenerator, os, filters, inclusive);
+    public CborXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, Set<String> includes, Set<String> excludes) {
+        super(jsonGenerator, os, includes, excludes);
     }
 
     @Override

core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java

@@ -35,6 +35,7 @@
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.Reader;
+import java.util.Set;
 
 /**
  * A JSON based content implementation using Jackson.
@@ -92,8 +93,8 @@ public byte streamSeparator() {
     }
 
     @Override
-    public XContentGenerator createGenerator(OutputStream os, String[] filters, boolean inclusive) throws IOException {
-        return new JsonXContentGenerator(jsonFactory.createGenerator(os, JsonEncoding.UTF8), os, filters, inclusive);
+    public XContentGenerator createGenerator(OutputStream os, Set<String> includes, Set<String> excludes) throws IOException {
+        return new JsonXContentGenerator(jsonFactory.createGenerator(os, JsonEncoding.UTF8), os, includes, excludes);
     }
 
     @Override

core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java

@@ -27,10 +27,10 @@
 import com.fasterxml.jackson.core.json.JsonWriteContext;
 import com.fasterxml.jackson.core.util.DefaultIndenter;
 import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;
+import com.fasterxml.jackson.core.util.JsonGeneratorDelegate;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentGenerator;
@@ -43,6 +43,9 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.util.Collections;
+import java.util.Objects;
+import java.util.Set;
 
 /**
  *
@@ -72,23 +75,38 @@ public class JsonXContentGenerator implements XContentGenerator {
     private static final DefaultPrettyPrinter.Indenter INDENTER = new DefaultIndenter("  ", LF.getValue());
     private boolean prettyPrint = false;
 
-    public JsonXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, String[] filters, boolean inclusive) {
+    public JsonXContentGenerator(JsonGenerator jsonGenerator, OutputStream os) {
+        this(jsonGenerator, os, Collections.emptySet(), Collections.emptySet());
+    }
+
+    public JsonXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, Set<String> includes, Set<String> excludes) {
+        Objects.requireNonNull(includes, "Including filters must not be null");
+        Objects.requireNonNull(excludes, "Excluding filters must not be null");
+        this.os = os;
         if (jsonGenerator instanceof GeneratorBase) {
             this.base = (GeneratorBase) jsonGenerator;
         } else {
             this.base = null;
         }
 
-        if (CollectionUtils.isEmpty(filters)) {
-            this.generator = jsonGenerator;
-            this.filter = null;
-        } else {
-            this.filter = new FilteringGeneratorDelegate(jsonGenerator,
-                    new FilterPathBasedFilter(filters, inclusive), true, true);
-            this.generator = this.filter;
+        JsonGenerator generator = jsonGenerator;
+
+        boolean hasExcludes = excludes.isEmpty() == false;
+        if (hasExcludes) {
+            generator = new FilteringGeneratorDelegate(generator, new FilterPathBasedFilter(excludes, false), true, true);
         }
 
-        this.os = os;
+        boolean hasIncludes = includes.isEmpty() == false;
+        if (hasIncludes) {
+            generator = new FilteringGeneratorDelegate(generator, new FilterPathBasedFilter(includes, true), true, true);
+        }
+
+        if (hasExcludes || hasIncludes) {
+            this.filter = (FilteringGeneratorDelegate) generator;
+        } else {
+            this.filter = null;
+        }
+        this.generator = generator;
     }
 
     @Override
@@ -122,33 +140,45 @@ public void writeEndArray() throws IOException {
         generator.writeEndArray();
     }
 
-    protected boolean isFiltered() {
+    private boolean isFiltered() {
         return filter != null;
     }
 
-    protected boolean inRoot() {
+    private JsonGenerator getLowLevelGenerator() {
         if (isFiltered()) {
-            JsonStreamContext context = filter.getFilterContext();
-            return ((context != null) && (context.inRoot() && context.getCurrentName() == null));
+            JsonGenerator delegate = filter.getDelegate();
+            if (delegate instanceof JsonGeneratorDelegate) {
+                // In case of combined inclusion and exclusion filters, we have one and only one another delegating level
+                delegate = ((JsonGeneratorDelegate) delegate).getDelegate();
+                assert delegate instanceof JsonGeneratorDelegate == false;
+            }
+            return delegate;
         }
-        return false;
+        return generator;
+    }
+
+    private boolean inRoot() {
+        JsonStreamContext context = generator.getOutputContext();
+        return ((context != null) && (context.inRoot() && context.getCurrentName() == null));
     }
 
     @Override
     public void writeStartObject() throws IOException {
-        if (isFiltered() && inRoot()) {
-            // Bypass generator to always write the root start object
-            filter.getDelegate().writeStartObject();
+        if (inRoot()) {
+            // Use the low level generator to write the startObject so that the root
+            // start object is always written even if a filtered generator is used
+            getLowLevelGenerator().writeStartObject();
             return;
         }
         generator.writeStartObject();
     }
 
     @Override
     public void writeEndObject() throws IOException {
-        if (isFiltered() && inRoot()) {
-            // Bypass generator to always write the root end object
-            filter.getDelegate().writeEndObject();
+        if (inRoot()) {
+            // Use the low level generator to write the startObject so that the root
+            // start object is always written even if a filtered generator is used
+            getLowLevelGenerator().writeEndObject();
             return;
         }
         generator.writeEndObject();
@@ -390,7 +420,8 @@ public void close() throws IOException {
         }
         if (writeLineFeedAtEnd) {
             flush();
-            generator.writeRaw(LF);
+            // Bypass generator to always write the line feed
+            getLowLevelGenerator().writeRaw(LF);
         }
         generator.close();
     }

core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java

@@ -35,6 +35,7 @@
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.Reader;
+import java.util.Set;
 
 /**
  * A Smile based content implementation using Jackson.
@@ -71,8 +72,8 @@ public byte streamSeparator() {
     }
 
     @Override
-    public XContentGenerator createGenerator(OutputStream os, String[] filters, boolean inclusive) throws IOException {
-        return new SmileXContentGenerator(smileFactory.createGenerator(os, JsonEncoding.UTF8), os, filters, inclusive);
+    public XContentGenerator createGenerator(OutputStream os, Set<String> includes, Set<String> excludes) throws IOException {
+        return new SmileXContentGenerator(smileFactory.createGenerator(os, JsonEncoding.UTF8), os, includes, excludes);
     }
 
     @Override