Core: Added query/filter wrapper that builds the actual query to be e…

…xecuted on the last possible moment to aid with index aliases and percolator queries using `now` date expression.

Percolator queries and index alias filters are parsed once and reused as long as they exist on a node. If they contain time based range filters with a `now` expression then the alias filters and percolator queries are going to be incorrect from the moment these are constructed (depending on the date rounding).

 If a range filter or range query is constructed as part of adding a percolator query or a index alias filter then these get wrapped in special query or filter wrappers that defer the resolution of now at last possible moment as apposed during parse time. In the case of the range filter a special Resolvable Filter makes sure that `now` is resolved when the DocIdSet is pulled and in the case of the range query `now` is resolved at query rewrite time. Both occur at the time the range filter or query is used as apposed when the query or filter is constructed during parse time.

Closes #8474
Closes #8534

src/main/java/org/elasticsearch/common/lucene/search/NoCacheQuery.java

@@ -0,0 +1,36 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.lucene.search;
+
+import org.apache.lucene.search.Query;
+
+/**
+ * Queries are never cached directly, but a query can be wrapped in a filter that may end being cached.
+ * Filters that wrap this query either directly or indirectly will never be cached.
+ */
+public abstract class NoCacheQuery extends Query {
+
+    @Override
+    public final String toString(String s) {
+        return "no_cache(" + innerToString(s) + ")";
+    }
+
+    public abstract String innerToString(String s);
+}

src/main/java/org/elasticsearch/common/lucene/search/ResolvableFilter.java

@@ -0,0 +1,51 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.lucene.search;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.DocIdSet;
+import org.apache.lucene.search.Filter;
+import org.apache.lucene.util.Bits;
+
+import java.io.IOException;
+
+/**
+ * A filter implementation that resolves details at the last possible moment between filter parsing and execution.
+ * For example a date filter based on 'now'.
+ */
+public abstract class ResolvableFilter extends Filter {
+
+    /**
+     * @return The actual filter instance to be executed containing the latest details.
+     */
+    public abstract Filter resolve();
+
+
+
+    @Override
+    public DocIdSet getDocIdSet(LeafReaderContext context, Bits acceptDocs) throws IOException {
+        Filter resolvedFilter = resolve();
+        if (resolvedFilter != null) {
+            return resolvedFilter.getDocIdSet(context, acceptDocs);
+        } else {
+            return null;
+        }
+    }
+}

src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java

@@ -22,13 +22,15 @@
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.search.Filter;
 import org.apache.lucene.search.NumericRangeFilter;
 import org.apache.lucene.search.NumericRangeQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefBuilder;
 import org.apache.lucene.util.NumericUtils;
+import org.apache.lucene.util.ToStringUtils;
 import org.elasticsearch.ElasticsearchIllegalArgumentException;
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.Nullable;
@@ -38,6 +40,8 @@
 import org.elasticsearch.common.joda.FormatDateTimeFormatter;
 import org.elasticsearch.common.joda.Joda;
 import org.elasticsearch.common.lucene.search.NoCacheFilter;
+import org.elasticsearch.common.lucene.search.NoCacheQuery;
+import org.elasticsearch.common.lucene.search.ResolvableFilter;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.common.util.LocaleUtils;
@@ -53,6 +57,7 @@
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.search.NumericRangeFieldDataFilter;
 import org.elasticsearch.index.similarity.SimilarityProvider;
+import org.elasticsearch.search.internal.SearchContext;
 import org.joda.time.DateTimeZone;
 
 import java.io.IOException;
@@ -300,39 +305,35 @@ public Query fuzzyQuery(String value, Fuzziness fuzziness, int prefixLength, int
 
     @Override
     public Query termQuery(Object value, @Nullable QueryParseContext context) {
-        long lValue = parseToMilliseconds(value, context);
+        long lValue = parseToMilliseconds(value);
         return NumericRangeQuery.newLongRange(names.indexName(), precisionStep,
                 lValue, lValue, true, true);
     }
 
-    public long parseToMilliseconds(Object value, @Nullable QueryParseContext context) {
-        return parseToMilliseconds(value, context, false);
+    public long parseToMilliseconds(Object value) {
+        return parseToMilliseconds(value, false, null, dateMathParser);
     }
 
-    public long parseToMilliseconds(Object value, @Nullable QueryParseContext context, boolean includeUpper) {
-        return parseToMilliseconds(value, context, includeUpper, null, dateMathParser);
-    }
-
-    public long parseToMilliseconds(Object value, @Nullable QueryParseContext context, boolean includeUpper, @Nullable DateTimeZone zone, @Nullable DateMathParser forcedDateParser) {
+    public long parseToMilliseconds(Object value, boolean includeUpper, @Nullable DateTimeZone zone, @Nullable DateMathParser forcedDateParser) {
         if (value instanceof Number) {
             return ((Number) value).longValue();
         }
-        return parseToMilliseconds(convertToString(value), context, includeUpper, zone, forcedDateParser);
+        return parseToMilliseconds(convertToString(value), includeUpper, zone, forcedDateParser);
     }
 
-    public long parseToMilliseconds(String value, @Nullable QueryParseContext context, boolean includeUpper, @Nullable DateTimeZone zone, @Nullable DateMathParser forcedDateParser) {
-        long now = context == null ? System.currentTimeMillis() : context.nowInMillis();
+    public long parseToMilliseconds(String value, boolean includeUpper, @Nullable DateTimeZone zone, @Nullable DateMathParser forcedDateParser) {
+        SearchContext sc = SearchContext.current();
+        long now = sc == null ? System.currentTimeMillis() : sc.nowInMillis();
         DateMathParser dateParser = dateMathParser;
         if (forcedDateParser != null) {
             dateParser = forcedDateParser;
         }
-        long time = includeUpper && roundCeil ? dateParser.parseRoundCeil(value, now, zone) : dateParser.parse(value, now, zone);
-        return time;
+        return includeUpper && roundCeil ? dateParser.parseRoundCeil(value, now, zone) : dateParser.parse(value, now, zone);
     }
 
     @Override
     public Filter termFilter(Object value, @Nullable QueryParseContext context) {
-        final long lValue = parseToMilliseconds(value, context);
+        final long lValue = parseToMilliseconds(value);
         return NumericRangeFilter.newLongRange(names.indexName(), precisionStep,
                 lValue, lValue, true, true);
     }
@@ -343,9 +344,18 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower
     }
 
     public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable DateTimeZone timeZone, @Nullable DateMathParser forcedDateParser, @Nullable QueryParseContext context) {
+        // If the current search context is null we're parsing percolator query or a index alias filter.
+        if (SearchContext.current() == null) {
+            return new LateParsingQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, forcedDateParser);
+        } else {
+            return innerRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, forcedDateParser);
+        }
+    }
+
+    private Query innerRangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable DateTimeZone timeZone, @Nullable DateMathParser forcedDateParser) {
         return NumericRangeQuery.newLongRange(names.indexName(), precisionStep,
-                lowerTerm == null ? null : parseToMilliseconds(lowerTerm, context, false, timeZone, forcedDateParser == null ? dateMathParser : forcedDateParser),
-                upperTerm == null ? null : parseToMilliseconds(upperTerm, context, includeUpper, timeZone, forcedDateParser == null ? dateMathParser : forcedDateParser),
+                lowerTerm == null ? null : parseToMilliseconds(lowerTerm, false, timeZone, forcedDateParser == null ? dateMathParser : forcedDateParser),
+                upperTerm == null ? null : parseToMilliseconds(upperTerm, includeUpper, timeZone, forcedDateParser == null ? dateMathParser : forcedDateParser),
                 includeLower, includeUpper);
     }
 
@@ -370,6 +380,16 @@ public Filter rangeFilter(QueryParseContext parseContext, Object lowerTerm, Obje
      * - the String to parse does not have already a timezone defined (ie. `2014-01-01T00:00:00+03:00`)
      */
     public Filter rangeFilter(QueryParseContext parseContext, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable DateTimeZone timeZone, @Nullable DateMathParser forcedDateParser, @Nullable QueryParseContext context, @Nullable Boolean explicitCaching) {
+        IndexNumericFieldData fieldData = parseContext != null ? (IndexNumericFieldData) parseContext.getForField(this) : null;
+        // If the current search context is null we're parsing percolator query or a index alias filter.
+        if (SearchContext.current() == null) {
+            return new LateParsingFilter(fieldData, lowerTerm, upperTerm, includeLower, includeUpper, timeZone, forcedDateParser, explicitCaching);
+        } else {
+            return innerRangeFilter(fieldData, lowerTerm, upperTerm, includeLower, includeUpper, timeZone, forcedDateParser, explicitCaching);
+        }
+    }
+
+    private Filter innerRangeFilter(IndexNumericFieldData fieldData, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable DateTimeZone timeZone, @Nullable DateMathParser forcedDateParser, @Nullable Boolean explicitCaching) {
         boolean cache;
         boolean cacheable = true;
         Long lowerVal = null;
@@ -380,7 +400,7 @@ public Filter rangeFilter(QueryParseContext parseContext, Object lowerTerm, Obje
             } else {
                 String value = convertToString(lowerTerm);
                 cacheable = !hasDateExpressionWithNoRounding(value);
-                lowerVal = parseToMilliseconds(value, context, false, timeZone, forcedDateParser);
+                lowerVal = parseToMilliseconds(value, false, timeZone, forcedDateParser);
             }
         }
         if (upperTerm != null) {
@@ -389,7 +409,7 @@ public Filter rangeFilter(QueryParseContext parseContext, Object lowerTerm, Obje
             } else {
                 String value = convertToString(upperTerm);
                 cacheable = cacheable && !hasDateExpressionWithNoRounding(value);
-                upperVal = parseToMilliseconds(value, context, includeUpper, timeZone, forcedDateParser);
+                upperVal = parseToMilliseconds(value, includeUpper, timeZone, forcedDateParser);
             }
         }
 
@@ -404,12 +424,10 @@ public Filter rangeFilter(QueryParseContext parseContext, Object lowerTerm, Obje
         }
 
         Filter filter;
-        if (parseContext != null) {
-            filter =  NumericRangeFieldDataFilter.newLongRange(
-                (IndexNumericFieldData) parseContext.getForField(this), lowerVal,upperVal, includeLower, includeUpper
-            );
+        if (fieldData != null) {
+            filter = NumericRangeFieldDataFilter.newLongRange(fieldData, lowerVal,upperVal, includeLower, includeUpper);
         } else {
-            filter =  NumericRangeFilter.newLongRange(
+            filter = NumericRangeFilter.newLongRange(
                     names.indexName(), precisionStep, lowerVal, upperVal, includeLower, includeUpper
             );
         }
@@ -597,4 +615,70 @@ private long parseStringValue(String value) {
             }
         }
     }
+
+    private final class LateParsingFilter extends ResolvableFilter {
+
+        final IndexNumericFieldData fieldData;
+        final Object lowerTerm;
+        final Object upperTerm;
+        final boolean includeLower;
+        final boolean includeUpper;
+        final DateTimeZone timeZone;
+        final DateMathParser forcedDateParser;
+        final Boolean explicitCaching;
+
+        public LateParsingFilter(IndexNumericFieldData fieldData, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, DateTimeZone timeZone, DateMathParser forcedDateParser, Boolean explicitCaching) {
+            this.fieldData = fieldData;
+            this.lowerTerm = lowerTerm;
+            this.upperTerm = upperTerm;
+            this.includeLower = includeLower;
+            this.includeUpper = includeUpper;
+            this.timeZone = timeZone;
+            this.forcedDateParser = forcedDateParser;
+            this.explicitCaching = explicitCaching;
+        }
+
+        @Override
+        public Filter resolve() {
+            return innerRangeFilter(fieldData, lowerTerm, upperTerm, includeLower, includeUpper, timeZone, forcedDateParser, explicitCaching);
+        }
+    }
+
+    public final class LateParsingQuery extends NoCacheQuery {
+
+        final Object lowerTerm;
+        final Object upperTerm;
+        final boolean includeLower;
+        final boolean includeUpper;
+        final DateTimeZone timeZone;
+        final DateMathParser forcedDateParser;
+
+        public LateParsingQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, DateTimeZone timeZone, DateMathParser forcedDateParser) {
+            this.lowerTerm = lowerTerm;
+            this.upperTerm = upperTerm;
+            this.includeLower = includeLower;
+            this.includeUpper = includeUpper;
+            this.timeZone = timeZone;
+            this.forcedDateParser = forcedDateParser;
+        }
+
+        @Override
+        public Query rewrite(IndexReader reader) throws IOException {
+            Query query = innerRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, forcedDateParser);
+            return query.rewrite(reader);
+        }
+
+        @Override
+        public String innerToString(String s) {
+            final StringBuilder sb = new StringBuilder();
+            return sb.append(names.indexName()).append(':')
+                    .append(includeLower ? '[' : '{')
+                    .append((lowerTerm == null) ? "*" : lowerTerm.toString())
+                    .append(" TO ")
+                    .append((upperTerm == null) ? "*" : upperTerm.toString())
+                    .append(includeUpper ? ']' : '}')
+                    .append(ToStringUtils.boost(getBoost()))
+                    .toString();
+        }
+    }
 }

src/main/java/org/elasticsearch/index/query/QueryParseContext.java

@@ -21,17 +21,22 @@
 
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
+import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.queryparser.classic.MapperQueryParser;
 import org.apache.lucene.queryparser.classic.QueryParserSettings;
+import org.apache.lucene.search.DocIdSet;
 import org.apache.lucene.search.Filter;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.join.BitDocIdSetFilter;
 import org.apache.lucene.search.similarities.Similarity;
+import org.apache.lucene.util.Bits;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.lucene.search.NoCacheFilter;
+import org.elasticsearch.common.lucene.search.NoCacheQuery;
 import org.elasticsearch.common.lucene.search.Queries;
+import org.elasticsearch.common.lucene.search.ResolvableFilter;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.analysis.AnalysisService;
@@ -187,17 +192,37 @@ public BitDocIdSetFilter bitsetFilter(Filter filter) {
         return indexQueryParser.bitsetFilterCache.getBitDocIdSetFilter(filter);
     }
 
-    public Filter cacheFilter(Filter filter, @Nullable CacheKeyFilter.Key cacheKey) {
+    public Filter cacheFilter(Filter filter, @Nullable final CacheKeyFilter.Key cacheKey) {
         if (filter == null) {
             return null;
         }
         if (this.disableFilterCaching || this.propagateNoCache || filter instanceof NoCacheFilter) {
             return filter;
         }
-        if (cacheKey != null) {
-            filter = new CacheKeyFilter.Wrapper(filter, cacheKey);
+        if (filter instanceof ResolvableFilter) {
+            final ResolvableFilter resolvableFilter = (ResolvableFilter) filter;
+            // We need to wrap it another filter, because this method is invoked at query parse time, which
+            // may not be during search execution time. (for example index alias filter and percolator)
+            return new Filter() {
+                @Override
+                public DocIdSet getDocIdSet(LeafReaderContext atomicReaderContext, Bits bits) throws IOException {
+                    Filter filter = resolvableFilter.resolve();
+                    if (filter == null) {
+                        return null;
+                    }
+                    if (cacheKey != null) {
+                        filter = new CacheKeyFilter.Wrapper(filter, cacheKey);
+                    }
+                    filter = indexQueryParser.indexCache.filter().cache(filter);
+                    return filter.getDocIdSet(atomicReaderContext, bits);
+                }
+            };
+        } else {
+            if (cacheKey != null) {
+                filter = new CacheKeyFilter.Wrapper(filter, cacheKey);
+            }
+            return indexQueryParser.indexCache.filter().cache(filter);
         }
-        return indexQueryParser.indexCache.filter().cache(filter);
     }
 
     public <IFD extends IndexFieldData<?>> IFD getForField(FieldMapper<?> mapper) {
@@ -249,6 +274,9 @@ public Query parseInnerQuery() throws IOException, QueryParsingException {
             // if we are at END_OBJECT, move to the next one...
             parser.nextToken();
         }
+        if (result instanceof NoCacheQuery) {
+            propagateNoCache = true;
+        }
         if (CustomQueryWrappingFilter.shouldUseCustomQueryWrappingFilter(result)) {
             requireCustomQueryWrappingFilter = true;
             // If later on, either directly or indirectly this query gets wrapped in a query filter it must never

src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionParser.java

@@ -265,7 +265,7 @@ private AbstractDistanceScoreFunction parseDateVariable(String fieldName, XConte
         }
         long origin = SearchContext.current().nowInMillis();
         if (originString != null) {
-            origin = dateFieldMapper.parseToMilliseconds(originString, parseContext);
+            origin = dateFieldMapper.parseToMilliseconds(originString);
         }
 
         if (scaleString == null) {