Apply exclusions in the automaton directly.

elastic · Apr 8, 2015 · a7dea5f · a7dea5f
1 parent da688b2
commit a7dea5f
Show file tree

Hide file tree

Showing 7 changed files with 98 additions and 52 deletions.
diff --git a/...arch/search/aggregations/bucket/significant/GlobalOrdinalsSignificantTermsAggregator.java b/...arch/search/aggregations/bucket/significant/GlobalOrdinalsSignificantTermsAggregator.java
@@ -48,7 +48,7 @@ public class GlobalOrdinalsSignificantTermsAggregator extends GlobalOrdinalsStri
 
     public GlobalOrdinalsSignificantTermsAggregator(String name, AggregatorFactories factories, ValuesSource.Bytes.WithOrdinals.FieldData valuesSource,
                                                     BucketCountThresholds bucketCountThresholds,
-                                                    IncludeExclude includeExclude, AggregationContext aggregationContext, Aggregator parent,
+                                                    IncludeExclude.OrdinalsFilter includeExclude, AggregationContext aggregationContext, Aggregator parent,
                                                     SignificantTermsAggregatorFactory termsAggFactory, Map<String, Object> metaData) throws IOException {
 
         super(name, factories, valuesSource, null, bucketCountThresholds, includeExclude, aggregationContext, parent, SubAggCollectionMode.DEPTH_FIRST, false, metaData);
@@ -145,7 +145,7 @@ public static class WithHash extends GlobalOrdinalsSignificantTermsAggregator {
 
         private final LongHash bucketOrds;
 
-        public WithHash(String name, AggregatorFactories factories, ValuesSource.Bytes.WithOrdinals.FieldData valuesSource, BucketCountThresholds bucketCountThresholds, IncludeExclude includeExclude, AggregationContext aggregationContext, Aggregator parent, SignificantTermsAggregatorFactory termsAggFactory, Map<String, Object> metaData) throws IOException {
+        public WithHash(String name, AggregatorFactories factories, ValuesSource.Bytes.WithOrdinals.FieldData valuesSource, BucketCountThresholds bucketCountThresholds, IncludeExclude.OrdinalsFilter includeExclude, AggregationContext aggregationContext, Aggregator parent, SignificantTermsAggregatorFactory termsAggFactory, Map<String, Object> metaData) throws IOException {
             super(name, factories, valuesSource, bucketCountThresholds, includeExclude, aggregationContext, parent, termsAggFactory, metaData);
             bucketOrds = new LongHash(1, aggregationContext.bigArrays());
         }

diff --git a/...lasticsearch/search/aggregations/bucket/significant/SignificantStringTermsAggregator.java b/...lasticsearch/search/aggregations/bucket/significant/SignificantStringTermsAggregator.java
@@ -47,7 +47,7 @@ public class SignificantStringTermsAggregator extends StringTermsAggregator {
 
     public SignificantStringTermsAggregator(String name, AggregatorFactories factories, ValuesSource valuesSource,
             BucketCountThresholds bucketCountThresholds,
-            IncludeExclude includeExclude, AggregationContext aggregationContext, Aggregator parent,
+            IncludeExclude.StringFilter includeExclude, AggregationContext aggregationContext, Aggregator parent,
             SignificantTermsAggregatorFactory termsAggFactory, Map<String, Object> metaData) throws IOException {
 
         super(name, factories, valuesSource, null, bucketCountThresholds, includeExclude, aggregationContext, parent, SubAggCollectionMode.DEPTH_FIRST, false, metaData);

diff --git a/...asticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java b/...asticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java
@@ -65,7 +65,8 @@ public enum ExecutionMode {
             Aggregator create(String name, AggregatorFactories factories, ValuesSource valuesSource,
                               TermsAggregator.BucketCountThresholds bucketCountThresholds, IncludeExclude includeExclude,
                               AggregationContext aggregationContext, Aggregator parent, SignificantTermsAggregatorFactory termsAggregatorFactory, Map<String, Object> metaData) throws IOException {
-                return new SignificantStringTermsAggregator(name, factories, valuesSource, bucketCountThresholds, includeExclude, aggregationContext, parent, termsAggregatorFactory, metaData);
+                final IncludeExclude.StringFilter filter = includeExclude == null ? null : includeExclude.convertToStringFilter();
+                return new SignificantStringTermsAggregator(name, factories, valuesSource, bucketCountThresholds, filter, aggregationContext, parent, termsAggregatorFactory, metaData);
             }
 
         },
@@ -77,7 +78,8 @@ Aggregator create(String name, AggregatorFactories factories, ValuesSource value
                               AggregationContext aggregationContext, Aggregator parent, SignificantTermsAggregatorFactory termsAggregatorFactory, Map<String, Object> metaData) throws IOException {
                 ValuesSource.Bytes.WithOrdinals valueSourceWithOrdinals = (ValuesSource.Bytes.WithOrdinals) valuesSource;
                 IndexSearcher indexSearcher = aggregationContext.searchContext().searcher();
-                return new GlobalOrdinalsSignificantTermsAggregator(name, factories, (ValuesSource.Bytes.WithOrdinals.FieldData) valuesSource, bucketCountThresholds, includeExclude, aggregationContext, parent, termsAggregatorFactory, metaData);
+                final IncludeExclude.OrdinalsFilter filter = includeExclude == null ? null : includeExclude.convertToOrdinalsFilter();
+                return new GlobalOrdinalsSignificantTermsAggregator(name, factories, (ValuesSource.Bytes.WithOrdinals.FieldData) valuesSource, bucketCountThresholds, filter, aggregationContext, parent, termsAggregatorFactory, metaData);
             }
 
         },
@@ -87,7 +89,8 @@ Aggregator create(String name, AggregatorFactories factories, ValuesSource value
             Aggregator create(String name, AggregatorFactories factories, ValuesSource valuesSource,
                               TermsAggregator.BucketCountThresholds bucketCountThresholds, IncludeExclude includeExclude,
                               AggregationContext aggregationContext, Aggregator parent, SignificantTermsAggregatorFactory termsAggregatorFactory, Map<String, Object> metaData) throws IOException {
-                return new GlobalOrdinalsSignificantTermsAggregator.WithHash(name, factories, (ValuesSource.Bytes.WithOrdinals.FieldData) valuesSource, bucketCountThresholds, includeExclude, aggregationContext, parent, termsAggregatorFactory, metaData);
+                final IncludeExclude.OrdinalsFilter filter = includeExclude == null ? null : includeExclude.convertToOrdinalsFilter();
+                return new GlobalOrdinalsSignificantTermsAggregator.WithHash(name, factories, (ValuesSource.Bytes.WithOrdinals.FieldData) valuesSource, bucketCountThresholds, filter, aggregationContext, parent, termsAggregatorFactory, metaData);
             }
         };
 

diff --git a/...g/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java b/...g/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java
@@ -57,7 +57,7 @@
 public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggregator {
 
     protected final ValuesSource.Bytes.WithOrdinals.FieldData valuesSource;
-    protected final IncludeExclude includeExclude;
+    protected final IncludeExclude.OrdinalsFilter includeExclude;
 
     // TODO: cache the acceptedglobalValues per aggregation definition.
     // We can't cache this yet in ValuesSource, since ValuesSource is reused per field for aggs during the execution.
@@ -71,7 +71,7 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr
 
     public GlobalOrdinalsStringTermsAggregator(String name, AggregatorFactories factories, ValuesSource.Bytes.WithOrdinals.FieldData valuesSource,
                                                Terms.Order order, BucketCountThresholds bucketCountThresholds,
-                                               IncludeExclude includeExclude, AggregationContext aggregationContext, Aggregator parent, SubAggCollectionMode collectionMode, boolean showTermDocCountError, Map<String, Object> metaData) throws IOException {
+                                               IncludeExclude.OrdinalsFilter includeExclude, AggregationContext aggregationContext, Aggregator parent, SubAggCollectionMode collectionMode, boolean showTermDocCountError, Map<String, Object> metaData) throws IOException {
         super(name, factories, aggregationContext, parent, order, bucketCountThresholds, collectionMode, showTermDocCountError, metaData);
         this.valuesSource = valuesSource;
         this.includeExclude = includeExclude;
@@ -260,7 +260,7 @@ public static class WithHash extends GlobalOrdinalsStringTermsAggregator {
         private final LongHash bucketOrds;
 
         public WithHash(String name, AggregatorFactories factories, ValuesSource.Bytes.WithOrdinals.FieldData valuesSource,
-                        Terms.Order order, BucketCountThresholds bucketCountThresholds, IncludeExclude includeExclude, AggregationContext aggregationContext,
+                        Terms.Order order, BucketCountThresholds bucketCountThresholds, IncludeExclude.OrdinalsFilter includeExclude, AggregationContext aggregationContext,
                         Aggregator parent, SubAggCollectionMode collectionMode, boolean showTermDocCountError, Map<String, Object> metaData) throws IOException {
             super(name, factories, valuesSource, order, bucketCountThresholds, includeExclude, aggregationContext, parent, collectionMode, showTermDocCountError, metaData);
             bucketOrds = new LongHash(1, aggregationContext.bigArrays());

diff --git a/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsAggregator.java b/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsAggregator.java
@@ -45,11 +45,11 @@ public class StringTermsAggregator extends AbstractStringTermsAggregator {
 
     private final ValuesSource valuesSource;
     protected final BytesRefHash bucketOrds;
-    private final IncludeExclude includeExclude;
+    private final IncludeExclude.StringFilter includeExclude;
 
     public StringTermsAggregator(String name, AggregatorFactories factories, ValuesSource valuesSource,
             Terms.Order order, BucketCountThresholds bucketCountThresholds,
-            IncludeExclude includeExclude, AggregationContext aggregationContext, Aggregator parent, SubAggCollectionMode collectionMode, boolean showTermDocCountError, Map<String, Object> metaData) throws IOException {
+            IncludeExclude.StringFilter includeExclude, AggregationContext aggregationContext, Aggregator parent, SubAggCollectionMode collectionMode, boolean showTermDocCountError, Map<String, Object> metaData) throws IOException {
 
         super(name, factories, aggregationContext, parent, order, bucketCountThresholds, collectionMode, showTermDocCountError, metaData);
         this.valuesSource = valuesSource;

diff --git a/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java b/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java
@@ -50,7 +50,8 @@ public enum ExecutionMode {
             Aggregator create(String name, AggregatorFactories factories, ValuesSource valuesSource,
                               Terms.Order order, TermsAggregator.BucketCountThresholds bucketCountThresholds, IncludeExclude includeExclude,
                               AggregationContext aggregationContext, Aggregator parent, SubAggCollectionMode subAggCollectMode, boolean showTermDocCountError, Map<String, Object> metaData) throws IOException {
-                return new StringTermsAggregator(name, factories, valuesSource, order, bucketCountThresholds, includeExclude, aggregationContext, parent, subAggCollectMode, showTermDocCountError, metaData);
+                final IncludeExclude.StringFilter filter = includeExclude == null ? null : includeExclude.convertToStringFilter();
+                return new StringTermsAggregator(name, factories, valuesSource, order, bucketCountThresholds, filter, aggregationContext, parent, subAggCollectMode, showTermDocCountError, metaData);
             }
 
             @Override
@@ -65,7 +66,8 @@ boolean needsGlobalOrdinals() {
             Aggregator create(String name, AggregatorFactories factories, ValuesSource valuesSource,
                               Terms.Order order, TermsAggregator.BucketCountThresholds bucketCountThresholds, IncludeExclude includeExclude,
                               AggregationContext aggregationContext, Aggregator parent, SubAggCollectionMode subAggCollectMode, boolean showTermDocCountError, Map<String, Object> metaData) throws IOException {
-                return new GlobalOrdinalsStringTermsAggregator(name, factories, (ValuesSource.Bytes.WithOrdinals.FieldData) valuesSource, order, bucketCountThresholds, includeExclude, aggregationContext, parent, subAggCollectMode, showTermDocCountError, metaData);
+                final IncludeExclude.OrdinalsFilter filter = includeExclude == null ? null : includeExclude.convertToOrdinalsFilter();
+                return new GlobalOrdinalsStringTermsAggregator(name, factories, (ValuesSource.Bytes.WithOrdinals.FieldData) valuesSource, order, bucketCountThresholds, filter, aggregationContext, parent, subAggCollectMode, showTermDocCountError, metaData);
             }
 
             @Override
@@ -80,7 +82,8 @@ boolean needsGlobalOrdinals() {
             Aggregator create(String name, AggregatorFactories factories, ValuesSource valuesSource,
                               Terms.Order order, TermsAggregator.BucketCountThresholds bucketCountThresholds, IncludeExclude includeExclude,
                               AggregationContext aggregationContext, Aggregator parent, SubAggCollectionMode subAggCollectMode, boolean showTermDocCountError, Map<String, Object> metaData) throws IOException {
-                return new GlobalOrdinalsStringTermsAggregator.WithHash(name, factories, (ValuesSource.Bytes.WithOrdinals.FieldData) valuesSource, order, bucketCountThresholds, includeExclude, aggregationContext, parent, subAggCollectMode, showTermDocCountError, metaData);
+                final IncludeExclude.OrdinalsFilter filter = includeExclude == null ? null : includeExclude.convertToOrdinalsFilter();
+                return new GlobalOrdinalsStringTermsAggregator.WithHash(name, factories, (ValuesSource.Bytes.WithOrdinals.FieldData) valuesSource, order, bucketCountThresholds, filter, aggregationContext, parent, subAggCollectMode, showTermDocCountError, metaData);
             }
 
             @Override

diff --git a/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/support/IncludeExclude.java b/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/support/IncludeExclude.java
@@ -36,7 +36,6 @@
 import org.apache.lucene.util.automaton.RegExp;
 import org.elasticsearch.ElasticsearchIllegalArgumentException;
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 
@@ -81,48 +80,75 @@ private void addReject(long val) {
         }
     }
 
-    private final CompiledAutomaton compiled;
-    private final ByteRunAutomaton automaton;
+    public static class StringFilter {
 
-    private IncludeExclude(Automaton include, Automaton exclude) {
-        if (include == null && exclude == null) {
-            throw new IllegalArgumentException();
+        private final ByteRunAutomaton runAutomaton;
+
+        private StringFilter(Automaton automaton) {
+            this.runAutomaton = new ByteRunAutomaton(automaton);
         }
-        Automaton a = null;
-        if (include == null) {
-            a = Automata.makeAnyString();
-        } else {
-            a = include;
+
+        /**
+         * Returns whether the given value is accepted based on the {@code include} & {@code exclude} patterns.
+         */
+        public boolean accept(BytesRef value) {
+            return runAutomaton.run(value.bytes, value.offset, value.length);
         }
-        if (exclude != null) {
-            a = Operations.minus(a, exclude, Operations.DEFAULT_MAX_DETERMINIZED_STATES);
+    }
+
+    public static class OrdinalsFilter {
+
+        private final CompiledAutomaton compiled;
+
+        private OrdinalsFilter(Automaton automaton) {
+            this.compiled = new CompiledAutomaton(automaton);
+        }
+
+        /**
+         * Computes which global ordinals are accepted by this IncludeExclude instance.
+         */
+        public LongBitSet acceptedGlobalOrdinals(RandomAccessOrds globalOrdinals, ValuesSource.Bytes.WithOrdinals valueSource) throws IOException {
+            LongBitSet acceptedGlobalOrdinals = new LongBitSet(globalOrdinals.getValueCount());
+            TermsEnum globalTermsEnum;
+            Terms globalTerms = new DocValuesTerms(globalOrdinals);
+            globalTermsEnum = compiled.getTermsEnum(globalTerms);
+            for (BytesRef term = globalTermsEnum.next(); term != null; term = globalTermsEnum.next()) {
+                acceptedGlobalOrdinals.set(globalTermsEnum.ord());
+            }
+            return acceptedGlobalOrdinals;
         }
-        a = Operations.determinize(a, Operations.DEFAULT_MAX_DETERMINIZED_STATES);
-        compiled = new CompiledAutomaton(a);
-        automaton = new ByteRunAutomaton(a);
+
     }
 
+    private final RegExp include, exclude;
+    private final SortedSet<BytesRef> includeValues, excludeValues;
+
     /**
      * @param include   The regular expression pattern for the terms to be included
      * @param exclude   The regular expression pattern for the terms to be excluded
      */
     public IncludeExclude(RegExp include, RegExp exclude) {
-        this(include == null ? null : include.toAutomaton(), exclude == null ? null : exclude.toAutomaton());
+        if (include == null && exclude == null) {
+            throw new IllegalArgumentException();
+        }
+        this.include = include;
+        this.exclude = exclude;
+        this.includeValues = null;
+        this.excludeValues = null;
     }
 
     /**
      * @param includeValues   The terms to be included
      * @param excludeValues   The terms to be excluded
      */
     public IncludeExclude(SortedSet<BytesRef> includeValues, SortedSet<BytesRef> excludeValues) {
-        this(includeValues == null ? null : Automata.makeStringUnion(includeValues), excludeValues == null ? null : Automata.makeStringUnion(excludeValues));
-    }
-
-    /**
-     * Returns whether the given value is accepted based on the {@code include} & {@code exclude} patterns.
-     */
-    public boolean accept(BytesRef value) {
-        return automaton.run(value.bytes, value.offset, value.length);
+        if (includeValues == null && excludeValues == null) {
+            throw new IllegalArgumentException();
+        }
+        this.include = null;
+        this.exclude = null;
+        this.includeValues = includeValues;
+        this.excludeValues = excludeValues;
     }
 
     /**
@@ -183,19 +209,7 @@ public boolean hasPayloads() {
 
     }
 
-    /**
-     * Computes which global ordinals are accepted by this IncludeExclude instance.
-     */
-    public LongBitSet acceptedGlobalOrdinals(RandomAccessOrds globalOrdinals, ValuesSource.Bytes.WithOrdinals valueSource) throws IOException {
-        LongBitSet acceptedGlobalOrdinals = new LongBitSet(globalOrdinals.getValueCount());
-        TermsEnum globalTermsEnum;
-        Terms globalTerms = new DocValuesTerms(globalOrdinals);
-        globalTermsEnum = compiled.getTermsEnum(globalTerms);
-        for (BytesRef term = globalTermsEnum.next(); term != null; term = globalTermsEnum.next()) {
-            acceptedGlobalOrdinals.set(globalTermsEnum.ord());
-        }
-        return acceptedGlobalOrdinals;
-    }
+
 
     public static class Parser {
 
@@ -292,6 +306,31 @@ public boolean isRegexBased() {
         return include != null || exclude != null;
     }
 
+    private Automaton toAutomaton() {
+        Automaton a = null;
+        if (include != null) {
+            a = include.toAutomaton();
+        } else if (includeValues != null) {
+            a = Automata.makeStringUnion(includeValues);
+        } else {
+            a = Automata.makeAnyString();
+        }
+        if (exclude != null) {
+            a = Operations.minus(a, exclude.toAutomaton(), Operations.DEFAULT_MAX_DETERMINIZED_STATES);
+        } else if (excludeValues != null) {
+            a = Operations.minus(a, Automata.makeStringUnion(excludeValues), Operations.DEFAULT_MAX_DETERMINIZED_STATES);
+        }
+        return a;
+    }
+
+    public StringFilter convertToStringFilter() {
+        return new StringFilter(toAutomaton());
+    }
+
+    public OrdinalsFilter convertToOrdinalsFilter() {
+        return new OrdinalsFilter(toAutomaton());
+    }
+
     public LongFilter convertToLongFilter() {
         int numValids = includeValues == null ? 0 : includeValues.size();
         int numInvalids = excludeValues == null ? 0 : excludeValues.size();
@@ -308,6 +347,7 @@ public LongFilter convertToLongFilter() {
         }
         return result;
     }
+
     public LongFilter convertToDoubleFilter() {
         int numValids = includeValues == null ? 0 : includeValues.size();
         int numInvalids = excludeValues == null ? 0 : excludeValues.size();