Upgrade to lucene-5.2.0-snapshot-1673124.

pom.xml

@@ -31,8 +31,8 @@
     </parent>
 
     <properties>
-        <lucene.version>5.1.0</lucene.version>
-        <lucene.maven.version>5.1.0-snapshot-1671894</lucene.maven.version>
+        <lucene.version>5.2.0</lucene.version>
+        <lucene.maven.version>5.2.0-snapshot-1673124</lucene.maven.version>
         <tests.jvms>auto</tests.jvms>
         <tests.shuffle>true</tests.shuffle>
         <tests.output>onerror</tests.output>
@@ -66,7 +66,7 @@
         <repository>
             <id>lucene-snapshots</id>
             <name>Lucene Snapshots</name>
-            <url>https://download.elastic.co/lucenesnapshots/1671894</url>
+            <url>https://download.elastic.co/lucenesnapshots/1673124</url>
         </repository>
     </repositories>
 

src/main/java/org/apache/lucene/search/postingshighlight/XPostingsHighlighter.java

@@ -28,6 +28,7 @@
 import org.apache.lucene.util.UnicodeUtil;
 
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.text.BreakIterator;
 import java.util.*;
 
@@ -426,7 +427,7 @@ protected Map<Integer,Object> highlightField(String field, String contents[], Br
                 throw new IllegalArgumentException("field '" + field + "' was indexed without offsets, cannot highlight");
             }
             if (leaf != lastLeaf) {
-                termsEnum = t.iterator(null);
+                termsEnum = t.iterator();
                 postings = new PostingsEnum[terms.length];
             }
             Passage passages[] = highlightDoc(field, terms, content.length(), bi, doc - subContext.docBase, termsEnum, postings, maxPassages);
@@ -745,7 +746,8 @@ public LimitedStoredFieldVisitor(String fields[], char valueSeparators[], int ma
         }
 
         @Override
-        public void stringField(FieldInfo fieldInfo, String value) throws IOException {
+        public void stringField(FieldInfo fieldInfo, byte[] bytes) throws IOException {
+            String value = new String(bytes, StandardCharsets.UTF_8);
             assert currentField >= 0;
             StringBuilder builder = builders[currentField];
             if (builder.length() > 0 && builder.length() < maxLength) {

src/main/java/org/elasticsearch/Version.java

@@ -236,7 +236,7 @@ public class Version {
     public static final int V_1_6_0_ID = 1060099;
     public static final Version V_1_6_0 = new Version(V_1_6_0_ID, true, org.apache.lucene.util.Version.LUCENE_4_10_4);
     public static final int V_2_0_0_ID = 2000099;
-    public static final Version V_2_0_0 = new Version(V_2_0_0_ID, true, org.apache.lucene.util.Version.LUCENE_5_1_0);
+    public static final Version V_2_0_0 = new Version(V_2_0_0_ID, true, org.apache.lucene.util.Version.LUCENE_5_2_0);
 
     public static final Version CURRENT = V_2_0_0;
 

src/main/java/org/elasticsearch/action/termvectors/TermVectorsFields.java

@@ -212,7 +212,7 @@ private void reset() throws IOException {
         }
 
         @Override
-        public TermsEnum iterator(TermsEnum reuse) throws IOException {
+        public TermsEnum iterator() throws IOException {
             // reset before asking for an iterator
             reset();
             // convert bytes ref for the terms to actual data

src/main/java/org/elasticsearch/action/termvectors/TermVectorsResponse.java

@@ -201,7 +201,7 @@ private void buildField(XContentBuilder builder, final CharsRefBuilder spare, Fi
         // write field statistics
         buildFieldStatistics(builder, curTerms);
         builder.startObject(FieldStrings.TERMS);
-        TermsEnum termIter = curTerms.iterator(null);
+        TermsEnum termIter = curTerms.iterator();
         for (int i = 0; i < curTerms.size(); i++) {
             buildTerm(builder, spare, curTerms, termIter);
         }

src/main/java/org/elasticsearch/action/termvectors/TermVectorsWriter.java

@@ -52,10 +52,8 @@ final class TermVectorsWriter {
 
     void setFields(Fields termVectorsByField, Set<String> selectedFields, EnumSet<Flag> flags, Fields topLevelFields, @Nullable AggregatedDfs dfs) throws IOException {
         int numFieldsWritten = 0;
-        TermsEnum iterator = null;
         PostingsEnum docsAndPosEnum = null;
         PostingsEnum docsEnum = null;
-        TermsEnum topLevelIterator = null;
         for (String field : termVectorsByField) {
             if ((selectedFields != null) && (!selectedFields.contains(field))) {
                 continue;
@@ -69,7 +67,7 @@ void setFields(Fields termVectorsByField, Set<String> selectedFields, EnumSet<Fl
                 topLevelTerms = fieldTermVector;
             }
 
-            topLevelIterator = topLevelTerms.iterator(topLevelIterator);
+            TermsEnum topLevelIterator = topLevelTerms.iterator();
             boolean positions = flags.contains(Flag.Positions) && fieldTermVector.hasPositions();
             boolean offsets = flags.contains(Flag.Offsets) && fieldTermVector.hasOffsets();
             boolean payloads = flags.contains(Flag.Payloads) && fieldTermVector.hasPayloads();
@@ -81,7 +79,7 @@ void setFields(Fields termVectorsByField, Set<String> selectedFields, EnumSet<Fl
                     writeFieldStatistics(topLevelTerms);
                 }
             }
-            iterator = fieldTermVector.iterator(iterator);
+            TermsEnum iterator = fieldTermVector.iterator();
             final boolean useDocsAndPos = positions || offsets || payloads;
             while (iterator.next() != null) { // iterate all terms of the
                 // current field

src/main/java/org/elasticsearch/action/termvectors/dfs/DfsOnlyRequest.java

@@ -53,13 +53,12 @@ public DfsOnlyRequest(Fields termVectorsFields, String[] indices, String[] types
 
         // build a search request with a query of all the terms
         final BoolQueryBuilder boolBuilder = boolQuery();
-        TermsEnum iterator = null;
         for (String fieldName : termVectorsFields) {
             if ((selectedFields != null) && (!selectedFields.contains(fieldName))) {
                 continue;
             }
             Terms terms = termVectorsFields.terms(fieldName);
-            iterator = terms.iterator(iterator);
+            TermsEnum iterator = terms.iterator();
             while (iterator.next() != null) {
                 String text = iterator.term().utf8ToString();
                 boolBuilder.should(QueryBuilders.termQuery(fieldName, text));

src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java

@@ -25,12 +25,12 @@
 import org.apache.lucene.search.ComplexExplanation;
 import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Weight;
 import org.apache.lucene.search.similarities.Similarity;
 import org.apache.lucene.search.similarities.Similarity.SimScorer;
 import org.apache.lucene.search.spans.SpanScorer;
 import org.apache.lucene.search.spans.SpanTermQuery;
 import org.apache.lucene.search.spans.SpanWeight;
+import org.apache.lucene.search.spans.Spans;
 import org.apache.lucene.search.spans.TermSpans;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
@@ -51,7 +51,7 @@ public AllTermQuery(Term term) {
     }
 
     @Override
-    public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException {
+    public SpanWeight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException {
         // TODO: needsScores
         // we should be able to just return a regular SpanTermWeight, at most here if needsScores == false?
         return new AllTermWeight(this, searcher);
@@ -68,40 +68,56 @@ public AllTermSpanScorer scorer(LeafReaderContext context, Bits acceptDocs) thro
             if (this.stats == null) {
                 return null;
             }
+            TermSpans spans = (TermSpans) query.getSpans(context, acceptDocs, termContexts);
+            if (spans == null) {
+                return null;
+            }
             SimScorer sloppySimScorer = similarity.simScorer(stats, context);
-            return new AllTermSpanScorer((TermSpans) query.getSpans(context, acceptDocs, termContexts), this, sloppySimScorer);
+            return new AllTermSpanScorer(spans, this, sloppySimScorer);
         }
 
         protected class AllTermSpanScorer extends SpanScorer {
             protected PostingsEnum positions;
             protected float payloadScore;
             protected int payloadsSeen;
 
-            public AllTermSpanScorer(TermSpans spans, Weight weight, Similarity.SimScorer docScorer) throws IOException {
+            public AllTermSpanScorer(TermSpans spans, SpanWeight weight, Similarity.SimScorer docScorer) throws IOException {
                 super(spans, weight, docScorer);
                 positions = spans.getPostings();
             }
 
             @Override
-            protected boolean setFreqCurrentDoc() throws IOException {
-                if (!more) {
-                    return false;
-                }
-                doc = spans.doc();
+            protected void setFreqCurrentDoc() throws IOException {
                 freq = 0.0f;
                 numMatches = 0;
                 payloadScore = 0;
                 payloadsSeen = 0;
-                do {
-                    int matchLength = spans.end() - spans.start();
 
-                    freq += docScorer.computeSlopFactor(matchLength);
+                assert spans.startPosition() == -1 : "incorrect initial start position, spans="+spans;
+                assert spans.endPosition() == -1 : "incorrect initial end position, spans="+spans;
+                int prevStartPos = -1;
+                int prevEndPos = -1;
+
+                int startPos = spans.nextStartPosition();
+                assert startPos != Spans.NO_MORE_POSITIONS : "initial startPos NO_MORE_POSITIONS, spans="+spans;
+                do {
+                    assert startPos >= prevStartPos;
+                    int endPos = spans.endPosition();
+                    assert endPos != Spans.NO_MORE_POSITIONS;
+                    // This assertion can fail for Or spans on the same term:
+                    // assert (startPos != prevStartPos) || (endPos > prevEndPos) : "non increased endPos="+endPos;
+                    assert (startPos != prevStartPos) || (endPos >= prevEndPos) : "decreased endPos="+endPos;
                     numMatches++;
+                    int matchLength = endPos - startPos;
+                    freq += docScorer.computeSlopFactor(matchLength);
                     processPayload();
+                    prevStartPos = startPos;
+                    prevEndPos = endPos;
+                    startPos = spans.nextStartPosition();
+                } while (startPos != Spans.NO_MORE_POSITIONS);
 
-                    more = spans.next();// this moves positions to the next match
-                } while (more && (doc == spans.doc()));
-                return true;
+                assert spans.startPosition() == Spans.NO_MORE_POSITIONS : "incorrect final start position, spans="+spans;
+                assert spans.endPosition() == Spans.NO_MORE_POSITIONS : "incorrect final end position, spans="+spans;
             }
 
             protected void processPayload() throws IOException {
@@ -120,7 +136,7 @@ protected void processPayload() throws IOException {
              * @throws IOException
              */
             @Override
-            public float score() throws IOException {
+            public float scoreCurrentDoc() throws IOException {
                 return getSpanScore() * getPayloadScore();
             }
 
@@ -134,7 +150,7 @@ public float score() throws IOException {
              * @see #score()
              */
             protected float getSpanScore() throws IOException {
-                return super.score();
+                return super.scoreCurrentDoc();
             }
 
             /**

src/main/java/org/elasticsearch/common/lucene/index/FilterableTermsEnum.java

@@ -84,7 +84,7 @@ public FilterableTermsEnum(IndexReader reader, String field, int docsEnumFlag, @
             if (terms == null) {
                 continue;
             }
-            TermsEnum termsEnum = terms.iterator(null);
+            TermsEnum termsEnum = terms.iterator();
             if (termsEnum == null) {
                 continue;
             }

src/main/java/org/elasticsearch/common/lucene/search/MoreLikeThisQuery.java

@@ -202,7 +202,7 @@ private void handleSkipTerms(XMoreLikeThis mlt, String[] ignoreText, Fields[] ig
             for (Fields fields : ignoreFields) {
                 for (String fieldName : fields) {
                     Terms terms = fields.terms(fieldName);
-                    final TermsEnum termsEnum = terms.iterator(null);
+                    final TermsEnum termsEnum = terms.iterator();
                     BytesRef text;
                     while ((text = termsEnum.next()) != null) {
                         skipTerms.add(new Term(fieldName, text.utf8ToString()));

src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java

@@ -155,15 +155,14 @@ public Query rewrite(IndexReader reader) throws IOException {
     private void getPrefixTerms(ObjectOpenHashSet<Term> terms, final Term prefix, final IndexReader reader) throws IOException {
         // SlowCompositeReaderWrapper could be used... but this would merge all terms from each segment into one terms
         // instance, which is very expensive. Therefore I think it is better to iterate over each leaf individually.
-        TermsEnum termsEnum = null;
         List<LeafReaderContext> leaves = reader.leaves();
         for (LeafReaderContext leaf : leaves) {
             Terms _terms = leaf.reader().terms(field);
             if (_terms == null) {
                 continue;
             }
 
-            termsEnum = _terms.iterator(termsEnum);
+            TermsEnum termsEnum = _terms.iterator();
             TermsEnum.SeekStatus seekStatus = termsEnum.seekCeil(prefix.bytes());
             if (TermsEnum.SeekStatus.END == seekStatus) {
                 continue;

src/main/java/org/elasticsearch/common/lucene/search/XMoreLikeThis.java

@@ -842,7 +842,7 @@ private void addTermFrequencies(Map<String, Int> termFreqMap, Terms vector) thro
      * @param fieldName Optional field name of the terms for skip terms
      */
     private void addTermFrequencies(Map<String, Int> termFreqMap, Terms vector, @Nullable String fieldName) throws IOException {
-        final TermsEnum termsEnum = vector.iterator(null);
+        final TermsEnum termsEnum = vector.iterator();
         final CharsRefBuilder spare = new CharsRefBuilder();
         BytesRef text;
         while((text = termsEnum.next()) != null) {

src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDAndVersionLookup.java

@@ -82,7 +82,7 @@ public PerThreadIDAndVersionLookup(IndexReader r) throws IOException {
                 if (terms != null) {
                     readerContexts[numSegs] = readerContext;
                     hasPayloads[numSegs] = terms.hasPayloads();
-                    termsEnums[numSegs] = terms.iterator(null);
+                    termsEnums[numSegs] = terms.iterator();
                     assert termsEnums[numSegs] != null;
                     liveDocs[numSegs] = readerContext.reader().getLiveDocs();
                     hasDeletions |= readerContext.reader().hasDeletions();

src/main/java/org/elasticsearch/index/codec/postingsformat/BloomFilterPostingsFormat.java

@@ -239,40 +239,23 @@ public BloomFilter getFilter() {
         }
 
         @Override
-        public TermsEnum iterator(TermsEnum reuse) throws IOException {
-            TermsEnum result;
-            if ((reuse != null) && (reuse instanceof BloomFilteredTermsEnum)) {
-                // recycle the existing BloomFilteredTermsEnum by asking the delegate
-                // to recycle its contained TermsEnum
-                BloomFilteredTermsEnum bfte = (BloomFilteredTermsEnum) reuse;
-                if (bfte.filter == filter) {
-                    bfte.reset(this.in);
-                    return bfte;
-                }
-                reuse = bfte.reuse;
-            }
-            // We have been handed something we cannot reuse (either null, wrong
-            // class or wrong filter) so allocate a new object
-            result = new BloomFilteredTermsEnum(this.in, reuse, filter);
-            return result;
+        public TermsEnum iterator() throws IOException {
+            return new BloomFilteredTermsEnum(this.in, filter);
         }
     }
 
     static final class BloomFilteredTermsEnum extends TermsEnum {
 
         private Terms delegateTerms;
         private TermsEnum delegateTermsEnum;
-        private TermsEnum reuse;
         private BloomFilter filter;
 
-        public BloomFilteredTermsEnum(Terms other, TermsEnum reuse, BloomFilter filter) {
+        public BloomFilteredTermsEnum(Terms other, BloomFilter filter) {
             this.delegateTerms = other;
-            this.reuse = reuse;
             this.filter = filter;
         }
 
         void reset(Terms others) {
-            reuse = this.delegateTermsEnum;
             this.delegateTermsEnum = null;
             this.delegateTerms = others;
         }
@@ -283,7 +266,7 @@ private TermsEnum getDelegate() throws IOException {
                  * this can be a relatively heavy operation depending on the 
                  * delegate postings format and they underlying directory
                  * (clone IndexInput) */
-                delegateTermsEnum = delegateTerms.iterator(reuse);
+                delegateTermsEnum = delegateTerms.iterator();
             }
             return delegateTermsEnum;
         }
@@ -385,7 +368,7 @@ public void write(Fields fields) throws IOException {
                     continue;
                 }
                 FieldInfo fieldInfo = state.fieldInfos.fieldInfo(field);
-                TermsEnum termsEnum = terms.iterator(null);
+                TermsEnum termsEnum = terms.iterator();
 
                 BloomFilter bloomFilter = null;
 

src/main/java/org/elasticsearch/index/engine/EngineSearcherFactory.java

@@ -39,7 +39,7 @@ public EngineSearcherFactory(EngineConfig engineConfig) {
     }
 
     @Override
-    public IndexSearcher newSearcher(IndexReader reader) throws IOException {
+    public IndexSearcher newSearcher(IndexReader reader, IndexReader previousReader) throws IOException {
         IndexSearcher searcher = new IndexSearcher(reader);
         searcher.setSimilarity(engineConfig.getSimilarity());
         return searcher;

src/main/java/org/elasticsearch/index/engine/InternalEngine.java

@@ -1030,7 +1030,7 @@ class SearchFactory extends EngineSearcherFactory {
         }
 
         @Override
-        public IndexSearcher newSearcher(IndexReader reader) throws IOException {
+        public IndexSearcher newSearcher(IndexReader reader, IndexReader previousReader) throws IOException {
             IndexSearcher searcher = new IndexSearcher(reader);
             searcher.setSimilarity(engineConfig.getSimilarity());
             if (warmer != null) {

src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java

@@ -81,7 +81,7 @@ public IndexOrdinalsFieldData localGlobalDirect(IndexReader indexReader) throws
     }
 
     protected TermsEnum filter(Terms terms, LeafReader reader) throws IOException {
-        TermsEnum iterator = terms.iterator(null);
+        TermsEnum iterator = terms.iterator();
         if (iterator == null) {
             return null;
         }

src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayIndexFieldData.java

@@ -108,7 +108,7 @@ public AtomicNumericFieldData loadDirect(LeafReaderContext context) throws Excep
         final float acceptableTransientOverheadRatio = fieldDataType.getSettings().getAsFloat("acceptable_transient_overhead_ratio", OrdinalsBuilder.DEFAULT_ACCEPTABLE_OVERHEAD_RATIO);
         boolean success = false;
         try (OrdinalsBuilder builder = new OrdinalsBuilder(reader.maxDoc(), acceptableTransientOverheadRatio)) {
-            final BytesRefIterator iter = builder.buildFromTerms(getNumericType().wrapTermsEnum(terms.iterator(null)));
+            final BytesRefIterator iter = builder.buildFromTerms(getNumericType().wrapTermsEnum(terms.iterator()));
             BytesRef term;
             long numTerms = 0;
             while ((term = iter.next()) != null) {

src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayIndexFieldData.java

@@ -106,7 +106,7 @@ public AtomicNumericFieldData loadDirect(LeafReaderContext context) throws Excep
         final float acceptableTransientOverheadRatio = fieldDataType.getSettings().getAsFloat("acceptable_transient_overhead_ratio", OrdinalsBuilder.DEFAULT_ACCEPTABLE_OVERHEAD_RATIO);
         boolean success = false;
         try (OrdinalsBuilder builder = new OrdinalsBuilder(reader.maxDoc(), acceptableTransientOverheadRatio)) {
-            BytesRefIterator iter = builder.buildFromTerms(getNumericType().wrapTermsEnum(terms.iterator(null)));
+            BytesRefIterator iter = builder.buildFromTerms(getNumericType().wrapTermsEnum(terms.iterator()));
             BytesRef term;
             long numTerms = 0;
             while ((term = iter.next()) != null) {