From 8cbcbb484329ea914606cd944ac7d15494d89e55 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 26 Mar 2018 08:34:11 +0100 Subject: [PATCH 01/45] WIP --- .../apache/lucene/search/OffsetCollector.java | 49 +++++++++++++++++++ .../apache/lucene/search/OffsetValues.java | 26 ++++++++++ .../apache/lucene/search/OffsetsIterator.java | 28 +++++++++++ .../java/org/apache/lucene/search/Weight.java | 15 ++++++ 4 files changed, 118 insertions(+) create mode 100644 lucene/core/src/java/org/apache/lucene/search/OffsetCollector.java create mode 100644 lucene/core/src/java/org/apache/lucene/search/OffsetValues.java create mode 100644 lucene/core/src/java/org/apache/lucene/search/OffsetsIterator.java diff --git a/lucene/core/src/java/org/apache/lucene/search/OffsetCollector.java b/lucene/core/src/java/org/apache/lucene/search/OffsetCollector.java new file mode 100644 index 000000000000..49b6ef530df8 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/search/OffsetCollector.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.search; + +import java.util.ArrayList; + +import org.apache.lucene.analysis.Analyzer; + +public class OffsetCollector { + + private final Analyzer analyzer; + private final String field; + + private final List automata = new ArrayList<>(); + + public OffsetCollector(Analyzer analyzer, String field) { + this.analyzer = analyzer; + this.field = field; + } + + public boolean hasAutomata() { + return automata.size() > 0; + } + + public void registerAutomaton(AutomatonOffsetIterator it) { + automata.add(it); + } + + public void notifyAutomata(String source) { + for (AutomatonOffsetIterator it : automata) { + it.setSource(source); + } + } +} diff --git a/lucene/core/src/java/org/apache/lucene/search/OffsetValues.java b/lucene/core/src/java/org/apache/lucene/search/OffsetValues.java new file mode 100644 index 000000000000..ec69f99d96e1 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/search/OffsetValues.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.search; + +import org.apache.lucene.analysis.Analyzer; + +public abstract class OffsetValues { + + public abstract OffsetsIterator advanceTo(int doc, String source); + +} diff --git a/lucene/core/src/java/org/apache/lucene/search/OffsetsIterator.java b/lucene/core/src/java/org/apache/lucene/search/OffsetsIterator.java new file mode 100644 index 000000000000..a372afb9f04b --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/search/OffsetsIterator.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.search; + +public interface OffsetsIterator { + + int startOffset(); + + int endOffset(); + + boolean nextOffset(); + +} diff --git a/lucene/core/src/java/org/apache/lucene/search/Weight.java b/lucene/core/src/java/org/apache/lucene/search/Weight.java index 7853ccf2465b..dd3c3891adb9 100644 --- a/lucene/core/src/java/org/apache/lucene/search/Weight.java +++ b/lucene/core/src/java/org/apache/lucene/search/Weight.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.Set; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.index.IndexReaderContext; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; @@ -69,6 +70,20 @@ protected Weight(Query query) { */ public abstract void extractTerms(Set terms); + public OffsetValues offsets(LeafReaderContext context, Analyzer analyzer, String field) { + OffsetCollector collector = new OffsetCollector(analyzer, field); + OffsetValues ov = offsets(context, collector); + return new OffsetValues() { + @Override + public OffsetsIterator advanceTo(int doc, String source) { + collector.setSource(source); + return ov.advanceTo(doc, source); + } + }; + } + + protected abstract OffsetValues offsets(LeafReaderContext context, OffsetCollector collector); + /** * An explanation of the score computation for the named document. * From f3512d5556832855e69ade9d1a9aea6a15982027 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Tue, 27 Mar 2018 20:21:06 +0100 Subject: [PATCH 02/45] Add Weight.matches() --- .../apache/lucene/document/FeatureQuery.java | 6 + .../lucene/document/RangeFieldQuery.java | 6 + .../SortedNumericDocValuesRangeQuery.java | 6 + .../SortedSetDocValuesRangeQuery.java | 6 + .../apache/lucene/search/BooleanWeight.java | 26 +++ .../lucene/search/ConstantScoreQuery.java | 5 + .../search/DisjunctionMatchesIterator.java | 133 ++++++++++++ .../lucene/search/DisjunctionMaxQuery.java | 15 ++ .../search/DocValuesFieldExistsQuery.java | 5 + .../lucene/search/DocValuesRewriteMethod.java | 23 +- .../apache/lucene/search/FilterWeight.java | 4 + .../lucene/search/IndexOrDocValuesQuery.java | 5 + .../apache/lucene/search/LRUQueryCache.java | 5 + .../lucene/search/MatchAllDocsQuery.java | 6 + .../lucene/search/MatchNoDocsQuery.java | 5 + ...setsIterator.java => MatchesIterator.java} | 14 +- .../lucene/search/MultiPhraseQuery.java | 5 + .../MultiTermQueryConstantScoreWrapper.java | 9 + .../lucene/search/NormsFieldExistsQuery.java | 5 + .../apache/lucene/search/OffsetValues.java | 26 --- .../org/apache/lucene/search/PhraseQuery.java | 5 + .../apache/lucene/search/PointInSetQuery.java | 5 + .../apache/lucene/search/PointRangeQuery.java | 5 + .../apache/lucene/search/SynonymQuery.java | 7 + .../apache/lucene/search/TermInSetQuery.java | 5 + ...ollector.java => TermMatchesIterator.java} | 48 +++-- .../org/apache/lucene/search/TermQuery.java | 17 ++ .../java/org/apache/lucene/search/Weight.java | 14 +- .../lucene/search/spans/SpanWeight.java | 58 +++++ .../lucene/search/JustCompileSearch.java | 5 + .../lucene/search/TestBooleanScorer.java | 5 + .../lucene/search/TestLRUQueryCache.java | 25 +++ .../lucene/search/TestMatchesIterator.java | 203 ++++++++++++++++++ .../lucene/search/TestQueryRescorer.java | 5 + .../apache/lucene/search/TestScorerPerf.java | 5 + .../apache/lucene/search/TestSortRandom.java | 5 + .../TestUsageTrackingFilterCachingPolicy.java | 5 + .../lucene/search/BlockScoreQueryWrapper.java | 5 + 38 files changed, 677 insertions(+), 65 deletions(-) create mode 100644 lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java rename lucene/core/src/java/org/apache/lucene/search/{OffsetsIterator.java => MatchesIterator.java} (78%) delete mode 100644 lucene/core/src/java/org/apache/lucene/search/OffsetValues.java rename lucene/core/src/java/org/apache/lucene/search/{OffsetCollector.java => TermMatchesIterator.java} (53%) create mode 100644 lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java diff --git a/lucene/core/src/java/org/apache/lucene/document/FeatureQuery.java b/lucene/core/src/java/org/apache/lucene/document/FeatureQuery.java index 841b2ad298b0..a7af04e93f5f 100644 --- a/lucene/core/src/java/org/apache/lucene/document/FeatureQuery.java +++ b/lucene/core/src/java/org/apache/lucene/document/FeatureQuery.java @@ -30,6 +30,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -81,6 +82,11 @@ public boolean isCacheable(LeafReaderContext ctx) { @Override public void extractTerms(Set terms) {} + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { String desc = "weight(" + getQuery() + " in " + doc + ") [" + function + "]"; diff --git a/lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java b/lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java index a24b7cdfae58..7d1a6c3dee57 100644 --- a/lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java +++ b/lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java @@ -30,6 +30,7 @@ import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.ScorerSupplier; @@ -349,6 +350,11 @@ public long cost() { } } + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { ScorerSupplier scorerSupplier = scorerSupplier(context); diff --git a/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java b/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java index 246b50f3dab6..c9137a3cd5cd 100644 --- a/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java +++ b/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java @@ -29,6 +29,7 @@ import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -101,6 +102,11 @@ public boolean isCacheable(LeafReaderContext ctx) { return DocValues.isCacheable(ctx, field); } + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { SortedNumericDocValues values = getValues(context.reader(), field); diff --git a/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java b/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java index de7c11b1cc9a..a21d9514385d 100644 --- a/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java +++ b/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java @@ -29,6 +29,7 @@ import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -106,6 +107,11 @@ public Query rewrite(IndexReader reader) throws IOException { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { SortedSetDocValues values = getValues(context.reader(), field); diff --git a/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java b/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java index fffdd09093f1..844a9f600a25 100644 --- a/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java +++ b/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java @@ -31,6 +31,7 @@ import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.util.Bits; +import org.apache.lucene.util.PriorityQueue; /** * Expert: the Weight for BooleanQuery, used to @@ -119,6 +120,31 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio } } + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + if (query.getClauses(Occur.SHOULD).size() != weights.size() || query.getMinimumNumberShouldMatch() > 0) { + // check that we actually match the doc in this case + Scorer scorer = scorer(context); + if (scorer == null) { + return null; + } + if (scorer.iterator().advance(doc) != doc) { + return null; + } + } + List mis = new ArrayList<>(); + for (Weight w : weights) { + MatchesIterator mi = w.matches(context, doc, field); + if (mi != null) { + mis.add(mi); + } + } + if (mis.size() == 0) { + return null; + } + return new DisjunctionMatchesIterator(mis); + } + static BulkScorer disableScoring(final BulkScorer scorer) { return new BulkScorer() { diff --git a/lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java b/lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java index 464cde6a45f9..725cad1ff9ff 100644 --- a/lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java @@ -158,6 +158,11 @@ public long cost() { }; } + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return innerWeight.matches(context, doc, field); + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { ScorerSupplier scorerSupplier = scorerSupplier(context); diff --git a/lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java b/lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java new file mode 100644 index 000000000000..648da6e42ab7 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.search; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.PostingsEnum; +import org.apache.lucene.index.PrefixCodedTerms; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.spans.SpanWeight; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefIterator; +import org.apache.lucene.util.PriorityQueue; + +class DisjunctionMatchesIterator implements MatchesIterator { + + public static DisjunctionMatchesIterator fromTerms(LeafReaderContext context, int doc, String field, List terms) throws IOException { + List mis = new ArrayList<>(); + Terms t = context.reader().terms(field); + if (t == null) + return null; + TermsEnum te = t.iterator(); + PostingsEnum reuse = null; + for (Term term : terms) { + if (te.seekExact(term.bytes())) { + PostingsEnum pe = te.postings(reuse, PostingsEnum.OFFSETS); + if (pe.advance(doc) == doc) { + mis.add(new TermMatchesIterator(pe)); + } + else { + reuse = pe; + } + } + } + if (mis.size() == 0) + return null; + return new DisjunctionMatchesIterator(mis); + } + + public static DisjunctionMatchesIterator fromTermsEnum(LeafReaderContext context, int doc, String field, BytesRefIterator terms) throws IOException { + List mis = new ArrayList<>(); + Terms t = context.reader().terms(field); + if (t == null) + return null; + TermsEnum te = t.iterator(); + PostingsEnum reuse = null; + for (BytesRef term = terms.next(); term != null; term = terms.next()) { + if (te.seekExact(term)) { + PostingsEnum pe = te.postings(reuse, PostingsEnum.OFFSETS); + if (pe.advance(doc) == doc) { + mis.add(new TermMatchesIterator(pe)); + } + else { + reuse = pe; + } + } + } + if (mis.size() == 0) + return null; + return new DisjunctionMatchesIterator(mis); + } + + private final PriorityQueue queue; + + private boolean started = false; + + public DisjunctionMatchesIterator(List matches) throws IOException { + queue = new PriorityQueue(matches.size()){ + @Override + protected boolean lessThan(MatchesIterator a, MatchesIterator b) { + return a.startPosition() < b.startPosition() || + (a.startPosition() == b.startPosition() && a.endPosition() < b.endPosition()); + } + }; + for (MatchesIterator mi : matches) { + mi.next(); + queue.add(mi); + } + } + + @Override + public boolean next() throws IOException { + if (started == false) { + return started = true; + } + MatchesIterator mi = queue.pop(); + if (mi != null && mi.next()) { + queue.add(mi); + return true; + } + return queue.size() > 0; + } + + @Override + public int startPosition() { + return queue.top().startPosition(); + } + + @Override + public int endPosition() { + return queue.top().endPosition(); + } + + @Override + public int startOffset() throws IOException { + return queue.top().startOffset(); + } + + @Override + public int endOffset() throws IOException { + return queue.top().endOffset(); + } +} diff --git a/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java b/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java index 1e67cb150465..9f2cc3bd18f7 100644 --- a/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java @@ -118,6 +118,21 @@ public void extractTerms(Set terms) { } } + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + List mis = new ArrayList<>(); + for (Weight weight : weights) { + MatchesIterator mi = weight.matches(context, doc, field); + if (mi != null) { + mis.add(mi); + } + } + if (mis.size() == 0) { + return null; + } + return new DisjunctionMatchesIterator(mis); + } + /** Create the scorer used to score our associated DisjunctionMaxQuery */ @Override public Scorer scorer(LeafReaderContext context) throws IOException { diff --git a/lucene/core/src/java/org/apache/lucene/search/DocValuesFieldExistsQuery.java b/lucene/core/src/java/org/apache/lucene/search/DocValuesFieldExistsQuery.java index 009f11cf116f..8fc0542b57f6 100644 --- a/lucene/core/src/java/org/apache/lucene/search/DocValuesFieldExistsQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/DocValuesFieldExistsQuery.java @@ -64,6 +64,11 @@ public String toString(String field) { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { FieldInfos fieldInfos = context.reader().getFieldInfos(); diff --git a/lucene/core/src/java/org/apache/lucene/search/DocValuesRewriteMethod.java b/lucene/core/src/java/org/apache/lucene/search/DocValuesRewriteMethod.java index 5d591983fab0..33b64387ea15 100644 --- a/lucene/core/src/java/org/apache/lucene/search/DocValuesRewriteMethod.java +++ b/lucene/core/src/java/org/apache/lucene/search/DocValuesRewriteMethod.java @@ -74,11 +74,19 @@ public final int hashCode() { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { + @Override - public Scorer scorer(LeafReaderContext context) throws IOException { + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + if (query.field.equals(field) == false) { + return null; + } final SortedSetDocValues fcsi = DocValues.getSortedSet(context.reader(), query.field); - TermsEnum termsEnum = query.getTermsEnum(new Terms() { - + return DisjunctionMatchesIterator.fromTermsEnum(context, doc, field, getTermsEnum(fcsi)); + } + + private TermsEnum getTermsEnum(SortedSetDocValues fcsi) throws IOException { + return query.getTermsEnum(new Terms() { + @Override public TermsEnum iterator() throws IOException { return fcsi.termsEnum(); @@ -118,13 +126,18 @@ public boolean hasOffsets() { public boolean hasPositions() { return false; } - + @Override public boolean hasPayloads() { return false; } }); - + } + + @Override + public Scorer scorer(LeafReaderContext context) throws IOException { + final SortedSetDocValues fcsi = DocValues.getSortedSet(context.reader(), query.field); + TermsEnum termsEnum = getTermsEnum(fcsi); assert termsEnum != null; if (termsEnum.next() == null) { // no matching terms diff --git a/lucene/core/src/java/org/apache/lucene/search/FilterWeight.java b/lucene/core/src/java/org/apache/lucene/search/FilterWeight.java index 925c9534f898..3e409823432d 100644 --- a/lucene/core/src/java/org/apache/lucene/search/FilterWeight.java +++ b/lucene/core/src/java/org/apache/lucene/search/FilterWeight.java @@ -75,4 +75,8 @@ public Scorer scorer(LeafReaderContext context) throws IOException { return in.scorer(context); } + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return in.matches(context, doc, field); + } } diff --git a/lucene/core/src/java/org/apache/lucene/search/IndexOrDocValuesQuery.java b/lucene/core/src/java/org/apache/lucene/search/IndexOrDocValuesQuery.java index f89924d16054..8024e1a97bb7 100644 --- a/lucene/core/src/java/org/apache/lucene/search/IndexOrDocValuesQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/IndexOrDocValuesQuery.java @@ -119,6 +119,11 @@ public void extractTerms(Set terms) { indexWeight.extractTerms(terms); } + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return indexWeight.matches(context, doc, field); + } + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { // We need to check a single doc, so the dv query should perform better diff --git a/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java b/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java index 72239e876cbe..35234f696740 100644 --- a/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java +++ b/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java @@ -678,6 +678,11 @@ public void extractTerms(Set terms) { in.extractTerms(terms); } + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return in.matches(context, doc, field); + } + private boolean cacheEntryHasReasonableWorstCaseSize(int maxDoc) { // The worst-case (dense) is a bit set which needs one bit per document final long worstCaseRamUsage = maxDoc / 8; diff --git a/lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java b/lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java index 89b299734144..bfeaff62e7a0 100644 --- a/lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java @@ -35,6 +35,12 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo public String toString() { return "weight(" + MatchAllDocsQuery.this + ")"; } + + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { return new ConstantScoreScorer(this, score(), DocIdSetIterator.all(context.reader().maxDoc())); diff --git a/lucene/core/src/java/org/apache/lucene/search/MatchNoDocsQuery.java b/lucene/core/src/java/org/apache/lucene/search/MatchNoDocsQuery.java index 525a18395434..b457eaebf97b 100644 --- a/lucene/core/src/java/org/apache/lucene/search/MatchNoDocsQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/MatchNoDocsQuery.java @@ -48,6 +48,11 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo public void extractTerms(Set terms) { } + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { return Explanation.noMatch(reason); diff --git a/lucene/core/src/java/org/apache/lucene/search/OffsetsIterator.java b/lucene/core/src/java/org/apache/lucene/search/MatchesIterator.java similarity index 78% rename from lucene/core/src/java/org/apache/lucene/search/OffsetsIterator.java rename to lucene/core/src/java/org/apache/lucene/search/MatchesIterator.java index a372afb9f04b..d1bc7c54fc1a 100644 --- a/lucene/core/src/java/org/apache/lucene/search/OffsetsIterator.java +++ b/lucene/core/src/java/org/apache/lucene/search/MatchesIterator.java @@ -17,12 +17,18 @@ package org.apache.lucene.search; -public interface OffsetsIterator { +import java.io.IOException; - int startOffset(); +public interface MatchesIterator { - int endOffset(); + boolean next() throws IOException; - boolean nextOffset(); + int startPosition(); + + int endPosition(); + + int startOffset() throws IOException; + + int endOffset() throws IOException; } diff --git a/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java b/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java index 65d6631e9a7c..9b6f57c5496c 100644 --- a/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java @@ -235,6 +235,11 @@ public void extractTerms(Set terms) { } } + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; // nocommit + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { assert termArrays.length != 0; diff --git a/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryConstantScoreWrapper.java b/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryConstantScoreWrapper.java index 3a46b96411cf..ec6e3fdd568b 100644 --- a/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryConstantScoreWrapper.java +++ b/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryConstantScoreWrapper.java @@ -202,6 +202,15 @@ public BulkScorer bulkScorer(LeafReaderContext context) throws IOException { } } + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + final Terms terms = context.reader().terms(query.field); + if (terms == null) { + return null; + } + return DisjunctionMatchesIterator.fromTermsEnum(context, doc, field, query.getTermsEnum(terms)); + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { final WeightOrDocIdSet weightOrBitSet = rewrite(context); diff --git a/lucene/core/src/java/org/apache/lucene/search/NormsFieldExistsQuery.java b/lucene/core/src/java/org/apache/lucene/search/NormsFieldExistsQuery.java index 74218b40b0c3..45e7bb181e29 100644 --- a/lucene/core/src/java/org/apache/lucene/search/NormsFieldExistsQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/NormsFieldExistsQuery.java @@ -64,6 +64,11 @@ public String toString(String field) { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { FieldInfos fieldInfos = context.reader().getFieldInfos(); diff --git a/lucene/core/src/java/org/apache/lucene/search/OffsetValues.java b/lucene/core/src/java/org/apache/lucene/search/OffsetValues.java deleted file mode 100644 index ec69f99d96e1..000000000000 --- a/lucene/core/src/java/org/apache/lucene/search/OffsetValues.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.lucene.search; - -import org.apache.lucene.analysis.Analyzer; - -public abstract class OffsetValues { - - public abstract OffsetsIterator advanceTo(int doc, String source); - -} diff --git a/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java index ff1538820d61..82859200a4ee 100644 --- a/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java @@ -392,6 +392,11 @@ public void extractTerms(Set queryTerms) { Collections.addAll(queryTerms, terms); } + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; // nocommit + } + @Override public String toString() { return "weight(" + PhraseQuery.this + ")"; } diff --git a/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java b/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java index 689d64a50d74..a15b57ec2deb 100644 --- a/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java @@ -113,6 +113,11 @@ public final Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, fl return new ConstantScoreWeight(this, boost) { + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { LeafReader reader = context.reader(); diff --git a/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java b/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java index 7e48383b4720..452089a3143e 100644 --- a/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java @@ -313,6 +313,11 @@ public long cost() { } } + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { ScorerSupplier scorerSupplier = scorerSupplier(context); diff --git a/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java b/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java index 2a7c450805d9..6c73444db8d0 100644 --- a/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java @@ -29,9 +29,11 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermStates; import org.apache.lucene.index.TermState; +import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.util.BytesRef; @@ -159,6 +161,11 @@ public void extractTerms(Set terms) { } } + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return DisjunctionMatchesIterator.fromTerms(context, doc, field, Arrays.asList(SynonymQuery.this.terms)); + } + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { Scorer scorer = scorer(context); diff --git a/lucene/core/src/java/org/apache/lucene/search/TermInSetQuery.java b/lucene/core/src/java/org/apache/lucene/search/TermInSetQuery.java index a8bf5b0679c1..778297184adc 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TermInSetQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/TermInSetQuery.java @@ -220,6 +220,11 @@ public void extractTerms(Set terms) { // order to protect highlighters } + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return DisjunctionMatchesIterator.fromTermsEnum(context, doc, field, termData.iterator()); + } + /** * On the given leaf context, try to either rewrite to a disjunction if * there are few matching terms, or build a bitset containing matching docs. diff --git a/lucene/core/src/java/org/apache/lucene/search/OffsetCollector.java b/lucene/core/src/java/org/apache/lucene/search/TermMatchesIterator.java similarity index 53% rename from lucene/core/src/java/org/apache/lucene/search/OffsetCollector.java rename to lucene/core/src/java/org/apache/lucene/search/TermMatchesIterator.java index 49b6ef530df8..e3c695fb04bf 100644 --- a/lucene/core/src/java/org/apache/lucene/search/OffsetCollector.java +++ b/lucene/core/src/java/org/apache/lucene/search/TermMatchesIterator.java @@ -17,33 +17,47 @@ package org.apache.lucene.search; -import java.util.ArrayList; +import java.io.IOException; -import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.index.PostingsEnum; -public class OffsetCollector { +public class TermMatchesIterator implements MatchesIterator { - private final Analyzer analyzer; - private final String field; + private int upto; + private int pos; + private final PostingsEnum pe; - private final List automata = new ArrayList<>(); + public TermMatchesIterator(PostingsEnum pe) throws IOException { + this.pe = pe; + this.upto = pe.freq(); + } - public OffsetCollector(Analyzer analyzer, String field) { - this.analyzer = analyzer; - this.field = field; + @Override + public boolean next() throws IOException { + if (upto-- > 0) { + pos = pe.nextPosition(); + return true; + } + return false; } - public boolean hasAutomata() { - return automata.size() > 0; + @Override + public int startPosition() { + return pos; } - public void registerAutomaton(AutomatonOffsetIterator it) { - automata.add(it); + @Override + public int endPosition() { + return pos; } - public void notifyAutomata(String source) { - for (AutomatonOffsetIterator it : automata) { - it.setSource(source); - } + @Override + public int startOffset() throws IOException { + return pe.startOffset(); + } + + @Override + public int endOffset() throws IOException { + return pe.endOffset(); } } diff --git a/lucene/core/src/java/org/apache/lucene/search/TermQuery.java b/lucene/core/src/java/org/apache/lucene/search/TermQuery.java index f1f44154f554..77abec06d36f 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TermQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/TermQuery.java @@ -25,6 +25,7 @@ import org.apache.lucene.index.IndexReaderContext; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermStates; @@ -80,6 +81,22 @@ public void extractTerms(Set terms) { terms.add(getTerm()); } + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + if (term.field().equals(field) == false) { + return null; + } + TermsEnum te = getTermsEnum(context); + if (te == null) { + return null; + } + PostingsEnum pe = te.postings(null, PostingsEnum.OFFSETS); + if (pe.advance(doc) != doc) { + return null; + } + return new TermMatchesIterator(pe); + } + @Override public String toString() { return "weight(" + TermQuery.this + ")"; diff --git a/lucene/core/src/java/org/apache/lucene/search/Weight.java b/lucene/core/src/java/org/apache/lucene/search/Weight.java index dd3c3891adb9..784f520cd1af 100644 --- a/lucene/core/src/java/org/apache/lucene/search/Weight.java +++ b/lucene/core/src/java/org/apache/lucene/search/Weight.java @@ -70,19 +70,7 @@ protected Weight(Query query) { */ public abstract void extractTerms(Set terms); - public OffsetValues offsets(LeafReaderContext context, Analyzer analyzer, String field) { - OffsetCollector collector = new OffsetCollector(analyzer, field); - OffsetValues ov = offsets(context, collector); - return new OffsetValues() { - @Override - public OffsetsIterator advanceTo(int doc, String source) { - collector.setSource(source); - return ov.advanceTo(doc, source); - } - }; - } - - protected abstract OffsetValues offsets(LeafReaderContext context, OffsetCollector collector); + public abstract MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException; /** * An explanation of the score computation for the named document. diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java b/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java index 25b58fdc39a0..26c69c97257f 100644 --- a/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java +++ b/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java @@ -29,6 +29,7 @@ import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.LeafSimScorer; +import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.TermStatistics; import org.apache.lucene.search.Weight; import org.apache.lucene.search.similarities.Similarity; @@ -161,4 +162,61 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio return Explanation.noMatch("no matching term"); } + + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + if (this.field.equals(field) == false) + return null; + Spans spans = getSpans(context, Postings.OFFSETS); + if (spans.advance(doc) != doc) + return null; + return new MatchesIterator() { + + SpanCollector offsetCollector = new SpanCollector() { + @Override + public void collectLeaf(PostingsEnum postings, int position, Term term) throws IOException { + startOffset = Math.min(startOffset, postings.startOffset()); + endOffset = Math.max(endOffset, postings.endOffset()); + } + + @Override + public void reset() { + startOffset = Integer.MAX_VALUE; + endOffset = Integer.MIN_VALUE; + } + }; + + int startOffset = -1; + int endOffset = -1; + + @Override + public boolean next() throws IOException { + int next = spans.nextStartPosition(); + if (next == Spans.NO_MORE_POSITIONS) + return false; + spans.collect(offsetCollector); + return true; + } + + @Override + public int startPosition() { + return spans.startPosition(); + } + + @Override + public int endPosition() { + return spans.endPosition(); + } + + @Override + public int startOffset() { + return startOffset; + } + + @Override + public int endOffset() { + return endOffset; + } + }; + } } diff --git a/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java b/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java index 1657f9b9ced1..4e08a44ea5e4 100644 --- a/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java +++ b/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java @@ -247,6 +247,11 @@ public void extractTerms(Set terms) { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Explanation explain(LeafReaderContext context, int doc) { throw new UnsupportedOperationException(UNSUPPORTED_MSG); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java b/lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java index 8a8379be3432..903622cdd556 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java @@ -84,6 +84,11 @@ public void extractTerms(Set terms) { throw new UnsupportedOperationException(); } + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Explanation explain(LeafReaderContext context, int doc) { throw new UnsupportedOperationException(); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java b/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java index 9cbf29ce2cf3..0b8eb839a6e4 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java @@ -349,6 +349,11 @@ private static class DummyQuery extends Query { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { return null; @@ -943,6 +948,11 @@ private static class BadQuery extends Query { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { return null; @@ -1292,6 +1302,11 @@ public void extractTerms(Set terms) { } + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { return null; @@ -1364,6 +1379,11 @@ private static class DummyQuery2 extends Query { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { return scorerSupplier(context).get(Long.MAX_VALUE); @@ -1464,6 +1484,11 @@ public int hashCode() { public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, 1) { + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { scorerCreatedCount.incrementAndGet(); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java b/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java new file mode 100644 index 000000000000..8a520f5ea1be --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.search; + +import java.io.IOException; + +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.index.ReaderUtil; +import org.apache.lucene.index.Term; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.LuceneTestCase; + +public class TestMatchesIterator extends LuceneTestCase { + + protected IndexSearcher searcher; + protected Directory directory; + protected IndexReader reader; + + public static final String FIELD_WITH_OFFSETS = "field_offsets"; + public static final String FIELD_NO_OFFSETS = "field_no_offsets"; + + public static final FieldType OFFSETS = new FieldType(TextField.TYPE_STORED); + static { + OFFSETS.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS); + } + + @Override + public void tearDown() throws Exception { + reader.close(); + directory.close(); + super.tearDown(); + } + + @Override + public void setUp() throws Exception { + super.setUp(); + directory = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), directory, + newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy())); + for (int i = 0; i < docFields.length; i++) { + Document doc = new Document(); + doc.add(newField(FIELD_WITH_OFFSETS, docFields[i], OFFSETS)); + doc.add(newField(FIELD_NO_OFFSETS, docFields[i], TextField.TYPE_STORED)); + doc.add(new NumericDocValuesField("id", i)); + writer.addDocument(doc); + } + writer.forceMerge(1); + reader = writer.getReader(); + writer.close(); + searcher = newSearcher(getOnlyLeafReader(reader)); + } + + protected String[] docFields = { + "w1 w2 w3 w4 w5", + "w1 w3 w2 w3 zz", + "w1 xx w2 yy w4", + "w1 w2 w1 w4 w2 w3" + }; + + void checkMatches(Query q, String field, int[][] expected) throws IOException { + Weight w = searcher.createNormalizedWeight(q, ScoreMode.COMPLETE); + for (int i = 0; i < expected.length; i++) { + LeafReaderContext ctx = searcher.leafContexts.get(ReaderUtil.subIndex(expected[i][0], searcher.leafContexts)); + int doc = expected[i][0] - ctx.docBase; + MatchesIterator it = w.matches(ctx, doc, field); + if (it == null) { + assertEquals(expected[i].length, 1); + continue; + } + int pos = 1; + while (it.next()) { + System.out.println(expected[i][pos] + "->" + expected[i][pos + 1] + "[" + expected[i][pos + 2] + "->" + expected[i][pos + 3] + "]"); + assertEquals(expected[i][pos], it.startPosition()); + assertEquals(expected[i][pos + 1], it.endPosition()); + assertEquals(expected[i][pos + 2], it.startOffset()); + assertEquals(expected[i][pos + 3], it.endOffset()); + pos += 4; + } + assertEquals(expected[i].length, pos); + } + } + + public void testTermQuery() throws IOException { + Query q = new TermQuery(new Term(FIELD_WITH_OFFSETS, "w1")); + checkMatches(q, FIELD_WITH_OFFSETS, new int[][]{ + { 0, 0, 0, 0, 2 }, + { 1, 0, 0, 0, 2 }, + { 2, 0, 0, 0, 2 }, + { 3, 0, 0, 0, 2, 2, 2, 6, 8 } + }); + } + + public void testTermQueryNoStoredOffsets() throws IOException { + Query q = new TermQuery(new Term(FIELD_NO_OFFSETS, "w1")); + checkMatches(q, FIELD_NO_OFFSETS, new int[][]{ + { 0, 0, 0, -1, -1 }, + { 1, 0, 0, -1, -1 }, + { 2, 0, 0, -1, -1 }, + { 3, 0, 0, -1, -1, 2, 2, -1, -1 } + }); + } + + public void testDisjunction() throws IOException { + Query q = new BooleanQuery.Builder() + .add(new TermQuery(new Term(FIELD_WITH_OFFSETS, "w1")), BooleanClause.Occur.SHOULD) + .add(new TermQuery(new Term(FIELD_WITH_OFFSETS, "w3")), BooleanClause.Occur.SHOULD) + .build(); + checkMatches(q, FIELD_WITH_OFFSETS, new int[][]{ + { 0, 0, 0, 0, 2, 2, 2, 6, 8 }, + { 1, 0, 0, 0, 2, 1, 1, 3, 5, 3, 3, 9, 11 }, + { 2, 0, 0, 0, 2 }, + { 3, 0, 0, 0, 2, 2, 2, 6, 8, 5, 5, 15, 17 } + }); + } + + public void testReqOpt() throws IOException { + Query q = new BooleanQuery.Builder() + .add(new TermQuery(new Term(FIELD_WITH_OFFSETS, "w1")), BooleanClause.Occur.SHOULD) + .add(new TermQuery(new Term(FIELD_WITH_OFFSETS, "w3")), BooleanClause.Occur.MUST) + .build(); + checkMatches(q, FIELD_WITH_OFFSETS, new int[][]{ + { 0, 0, 0, 0, 2, 2, 2, 6, 8 }, + { 1, 0, 0, 0, 2, 1, 1, 3, 5, 3, 3, 9, 11 }, + { 2 }, + { 3, 0, 0, 0, 2, 2, 2, 6, 8, 5, 5, 15, 17 } + }); + } + + public void testMinShouldMatch() throws IOException { + Query q = new BooleanQuery.Builder() + .add(new TermQuery(new Term(FIELD_WITH_OFFSETS, "w3")), BooleanClause.Occur.SHOULD) + .add(new BooleanQuery.Builder() + .add(new TermQuery(new Term(FIELD_WITH_OFFSETS, "w1")), BooleanClause.Occur.SHOULD) + .add(new TermQuery(new Term(FIELD_WITH_OFFSETS, "w4")), BooleanClause.Occur.SHOULD) + .add(new TermQuery(new Term(FIELD_WITH_OFFSETS, "xx")), BooleanClause.Occur.SHOULD) + .setMinimumNumberShouldMatch(2) + .build(), BooleanClause.Occur.SHOULD) + .build(); + checkMatches(q, FIELD_WITH_OFFSETS, new int[][]{ + { 0, 0, 0, 0, 2, 2, 2, 6, 8, 3, 3, 9, 11 }, + { 1, 1, 1, 3, 5, 3, 3, 9, 11 }, + { 2, 0, 0, 0, 2, 1, 1, 3, 5, 4, 4, 12, 14 }, + { 3, 0, 0, 0, 2, 2, 2, 6, 8, 3, 3, 9, 11, 5, 5, 15, 17 } + }); + } + + public void testExclusion() throws IOException { + Query q = new BooleanQuery.Builder() + .add(new TermQuery(new Term(FIELD_WITH_OFFSETS, "w3")), BooleanClause.Occur.SHOULD) + .add(new TermQuery(new Term(FIELD_WITH_OFFSETS, "zz")), BooleanClause.Occur.MUST_NOT) + .build(); + checkMatches(q, FIELD_WITH_OFFSETS, new int[][]{ + { 0, 2, 2, 6, 8 }, + { 1 }, + { 2 }, + { 3, 5, 5, 15, 17 } + }); + } + + public void testConjunction() throws IOException { + Query q = new BooleanQuery.Builder() + .add(new TermQuery(new Term(FIELD_WITH_OFFSETS, "w3")), BooleanClause.Occur.MUST) + .add(new TermQuery(new Term(FIELD_WITH_OFFSETS, "w4")), BooleanClause.Occur.MUST) + .build(); + checkMatches(q, FIELD_WITH_OFFSETS, new int[][]{ + { 0, 2, 2, 6, 8, 3, 3, 9, 11 }, + { 1 }, + { 2 }, + { 3, 3, 3, 9, 11, 5, 5, 15, 17 } + }); + } + + protected String[] doc1Fields = { + "w1 w2 w3 w4 w5", + "w1 w3 w2 w3 zz", + "w1 xx w2 yy w4", + "w1 w2 w1 w4 w2 w3" + }; + +} diff --git a/lucene/core/src/test/org/apache/lucene/search/TestQueryRescorer.java b/lucene/core/src/test/org/apache/lucene/search/TestQueryRescorer.java index d1f307d063ec..1e9f8f455f01 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestQueryRescorer.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestQueryRescorer.java @@ -426,6 +426,11 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo public void extractTerms(Set terms) { } + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Scorer scorer(final LeafReaderContext context) throws IOException { diff --git a/lucene/core/src/test/org/apache/lucene/search/TestScorerPerf.java b/lucene/core/src/test/org/apache/lucene/search/TestScorerPerf.java index 59a246cb6647..680ff615c0e1 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestScorerPerf.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestScorerPerf.java @@ -151,6 +151,11 @@ private static class BitSetQuery extends Query { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { return new ConstantScoreScorer(this, score(), new BitSetIterator(docs, docs.approximateCardinality())); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java b/lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java index 05b016c31c35..391c2a0abc59 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java @@ -231,6 +231,11 @@ public RandomQuery(long seed, float density, List docValues) { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { Random random = new Random(context.docBase ^ seed); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestUsageTrackingFilterCachingPolicy.java b/lucene/core/src/test/org/apache/lucene/search/TestUsageTrackingFilterCachingPolicy.java index 187accfdc524..d01eb951b9c8 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestUsageTrackingFilterCachingPolicy.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestUsageTrackingFilterCachingPolicy.java @@ -120,6 +120,11 @@ public int hashCode() { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(DummyQuery.this, boost) { + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { return new ConstantScoreScorer(this, score(), DocIdSetIterator.all(1)); diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/BlockScoreQueryWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/search/BlockScoreQueryWrapper.java index 3b9a740a448f..aea4b1f5ae6f 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/search/BlockScoreQueryWrapper.java +++ b/lucene/test-framework/src/java/org/apache/lucene/search/BlockScoreQueryWrapper.java @@ -204,6 +204,11 @@ public void extractTerms(Set terms) { inWeight.extractTerms(terms); } + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { return inWeight.explain(context, doc); From 06fef88448c3f30f66584b2e30f0ad968ab994da Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 28 Mar 2018 11:57:12 +0100 Subject: [PATCH 03/45] Javadocs, assertions --- .../apache/lucene/search/MatchesIterator.java | 27 ++++++++ .../java/org/apache/lucene/search/Weight.java | 14 ++++ .../lucene/search/TestMatchesIterator.java | 2 +- .../search/AssertingMatchesIterator.java | 69 +++++++++++++++++++ .../apache/lucene/search/AssertingWeight.java | 8 +++ 5 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 lucene/test-framework/src/java/org/apache/lucene/search/AssertingMatchesIterator.java diff --git a/lucene/core/src/java/org/apache/lucene/search/MatchesIterator.java b/lucene/core/src/java/org/apache/lucene/search/MatchesIterator.java index d1bc7c54fc1a..8cc10f8dfcad 100644 --- a/lucene/core/src/java/org/apache/lucene/search/MatchesIterator.java +++ b/lucene/core/src/java/org/apache/lucene/search/MatchesIterator.java @@ -19,16 +19,43 @@ import java.io.IOException; +import org.apache.lucene.index.LeafReaderContext; + +/** + * An iterator over match positions (and optionally offsets) for a single document and field + * + * To iterate over the matches, call {@link #next()} until it returns {@code false}, retrieving + * positions and/or offsets after each call. You should not call the position or offset methods + * before {@link #next()} has been called, or after {@link #next()} has returned {@code false}. + * + * @see Weight#matches(LeafReaderContext, int, String) + */ public interface MatchesIterator { + /** + * Advance the iterator to the next match position + * @return {@code true} if matches have not been exhausted + */ boolean next() throws IOException; + /** + * The start position of the current match + */ int startPosition(); + /** + * The end position of the current match + */ int endPosition(); + /** + * The starting offset of the current match, or {@code -1} if offsets are not available + */ int startOffset() throws IOException; + /** + * The ending offset of the current match, or {@code -1} if offsets are not available + */ int endOffset() throws IOException; } diff --git a/lucene/core/src/java/org/apache/lucene/search/Weight.java b/lucene/core/src/java/org/apache/lucene/search/Weight.java index 784f520cd1af..1487d1089f7b 100644 --- a/lucene/core/src/java/org/apache/lucene/search/Weight.java +++ b/lucene/core/src/java/org/apache/lucene/search/Weight.java @@ -70,6 +70,20 @@ protected Weight(Query query) { */ public abstract void extractTerms(Set terms); + /** + * Returns an iterator over the match positions in a field for the named document + * + * If there are no matches, returns {@code null} + * + * If the field was indexed with offsets, then the {@link MatchesIterator} will + * expose those offsets, otherwise only positions will be returned and + * {@link MatchesIterator#startOffset()} and {@link MatchesIterator#endOffset()} will + * always return {@code -1}. + * + * @param context the reader's context to create the {@link MatchesIterator} for + * @param doc the document's id relative to the given context's reader + * @param field the field to return match positions for + */ public abstract MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException; /** diff --git a/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java b/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java index 8a520f5ea1be..ec3aa4699671 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java @@ -92,7 +92,7 @@ void checkMatches(Query q, String field, int[][] expected) throws IOException { } int pos = 1; while (it.next()) { - System.out.println(expected[i][pos] + "->" + expected[i][pos + 1] + "[" + expected[i][pos + 2] + "->" + expected[i][pos + 3] + "]"); + // System.out.println(expected[i][pos] + "->" + expected[i][pos + 1] + "[" + expected[i][pos + 2] + "->" + expected[i][pos + 3] + "]"); assertEquals(expected[i][pos], it.startPosition()); assertEquals(expected[i][pos + 1], it.endPosition()); assertEquals(expected[i][pos + 2], it.startOffset()); diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/AssertingMatchesIterator.java b/lucene/test-framework/src/java/org/apache/lucene/search/AssertingMatchesIterator.java new file mode 100644 index 000000000000..8eb73d6e0923 --- /dev/null +++ b/lucene/test-framework/src/java/org/apache/lucene/search/AssertingMatchesIterator.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.search; + +import java.io.IOException; + +public class AssertingMatchesIterator implements MatchesIterator { + + private final MatchesIterator in; + private State state = State.UNPOSITIONED; + + private enum State { UNPOSITIONED, ITERATING, EXHAUSTED } + + public AssertingMatchesIterator(MatchesIterator in) { + this.in = in; + } + + @Override + public boolean next() throws IOException { + assert state != State.EXHAUSTED : state; + boolean more = in.next(); + if (more == false) { + state = State.EXHAUSTED; + } + else { + state = State.ITERATING; + } + return more; + } + + @Override + public int startPosition() { + assert state == State.ITERATING : state; + return in.startPosition(); + } + + @Override + public int endPosition() { + assert state == State.ITERATING : state; + return in.endPosition(); + } + + @Override + public int startOffset() throws IOException { + assert state == State.ITERATING : state; + return in.startOffset(); + } + + @Override + public int endOffset() throws IOException { + assert state == State.ITERATING : state; + return in.endOffset(); + } +} diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/AssertingWeight.java b/lucene/test-framework/src/java/org/apache/lucene/search/AssertingWeight.java index 8e3a29fb0023..aa8792b6c39b 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/search/AssertingWeight.java +++ b/lucene/test-framework/src/java/org/apache/lucene/search/AssertingWeight.java @@ -31,6 +31,14 @@ class AssertingWeight extends FilterWeight { this.scoreMode = scoreMode; } + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + MatchesIterator matches = in.matches(context, doc, field); + if (matches == null) + return null; + return new AssertingMatchesIterator(matches); + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { if (random.nextBoolean()) { From 8cd20120a0bf6b8442b8984b8f68e33cd42a6af3 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 28 Mar 2018 14:12:57 +0100 Subject: [PATCH 04/45] feedback --- .../java/org/apache/lucene/search/BooleanWeight.java | 9 ++++++++- .../lucene/search/DisjunctionMatchesIterator.java | 11 +++++++---- .../org/apache/lucene/search/MatchesIterator.java | 2 ++ 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java b/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java index 844a9f600a25..0040fde3bf62 100644 --- a/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java +++ b/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java @@ -133,7 +133,14 @@ public MatchesIterator matches(LeafReaderContext context, int doc, String field) } } List mis = new ArrayList<>(); - for (Weight w : weights) { + Iterator wIt = weights.iterator(); + Iterator cIt = query.clauses().iterator(); + while (wIt.hasNext()) { + Weight w = wIt.next(); + BooleanClause bc = cIt.next(); + if (bc.getOccur() == Occur.MUST_NOT) { + continue; + } MatchesIterator mi = w.matches(context, doc, field); if (mi != null) { mis.add(mi); diff --git a/lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java b/lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java index 648da6e42ab7..5fbea724d9fe 100644 --- a/lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java +++ b/lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java @@ -69,6 +69,7 @@ public static DisjunctionMatchesIterator fromTermsEnum(LeafReaderContext context PostingsEnum pe = te.postings(reuse, PostingsEnum.OFFSETS); if (pe.advance(doc) == doc) { mis.add(new TermMatchesIterator(pe)); + reuse = null; } else { reuse = pe; @@ -103,12 +104,14 @@ public boolean next() throws IOException { if (started == false) { return started = true; } - MatchesIterator mi = queue.pop(); - if (mi != null && mi.next()) { - queue.add(mi); + if (queue.top().next() == false) { + queue.pop(); + } + if (queue.size() > 0) { + queue.updateTop(); return true; } - return queue.size() > 0; + return false; } @Override diff --git a/lucene/core/src/java/org/apache/lucene/search/MatchesIterator.java b/lucene/core/src/java/org/apache/lucene/search/MatchesIterator.java index 8cc10f8dfcad..f77f985a813f 100644 --- a/lucene/core/src/java/org/apache/lucene/search/MatchesIterator.java +++ b/lucene/core/src/java/org/apache/lucene/search/MatchesIterator.java @@ -28,6 +28,8 @@ * positions and/or offsets after each call. You should not call the position or offset methods * before {@link #next()} has been called, or after {@link #next()} has returned {@code false}. * + * Matches are ordered by start position, and then by end position. Match intervals map overlap. + * * @see Weight#matches(LeafReaderContext, int, String) */ public interface MatchesIterator { From 49ade102103ceb9512f51f449b39cb60ea1d1647 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 28 Mar 2018 14:18:52 +0100 Subject: [PATCH 05/45] Wildcard tests --- .../lucene/search/TestMatchesIterator.java | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java b/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java index ec3aa4699671..419080ac4ea6 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java @@ -92,7 +92,7 @@ void checkMatches(Query q, String field, int[][] expected) throws IOException { } int pos = 1; while (it.next()) { - // System.out.println(expected[i][pos] + "->" + expected[i][pos + 1] + "[" + expected[i][pos + 2] + "->" + expected[i][pos + 3] + "]"); + System.out.println(expected[i][pos] + "->" + expected[i][pos + 1] + "[" + expected[i][pos + 2] + "->" + expected[i][pos + 3] + "]"); assertEquals(expected[i][pos], it.startPosition()); assertEquals(expected[i][pos + 1], it.endPosition()); assertEquals(expected[i][pos + 2], it.startOffset()); @@ -193,6 +193,24 @@ public void testConjunction() throws IOException { }); } + public void testWildcards() throws IOException { + Query q = new PrefixQuery(new Term(FIELD_WITH_OFFSETS, "x")); + checkMatches(q, FIELD_WITH_OFFSETS, new int[][]{ + { 0 }, + { 1 }, + { 2, 1, 1, 3, 5 }, + { 0 } + }); + + Query rq = new RegexpQuery(new Term(FIELD_WITH_OFFSETS, "w[1-2]")); + checkMatches(rq, FIELD_WITH_OFFSETS, new int[][]{ + { 0, 0, 0, 0, 2, 1, 1, 3, 5 }, + { 1, 0, 0, 0, 2, 2, 2, 6, 8 }, + { 2, 0, 0, 0, 2, 2, 2, 6, 8 }, + { 3, 0, 0, 0, 2, 1, 1, 3, 5, 2, 2, 6, 8, 4, 4, 12, 14 } + }); + } + protected String[] doc1Fields = { "w1 w2 w3 w4 w5", "w1 w3 w2 w3 zz", From fd9df3b1f6a110613935b8f76ae92690835eb84d Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 28 Mar 2018 14:40:44 +0100 Subject: [PATCH 06/45] All modules compile --- .../lucene/facet/DrillSidewaysQuery.java | 6 ++++++ .../lucene/facet/range/DoubleRange.java | 6 ++++++ .../apache/lucene/facet/range/LongRange.java | 6 ++++++ .../lucene/facet/TestDrillSideways.java | 6 ++++++ .../search/join/GlobalOrdinalsQuery.java | 6 ++++++ .../join/ParentChildrenBlockJoinQuery.java | 6 ++++++ .../join/PointInSetIncludingScoreQuery.java | 6 ++++++ .../search/join/TermsIncludingScoreQuery.java | 6 ++++++ .../lucene/search/join/TestJoinUtil.java | 5 +++++ .../TestDiversifiedTopDocsCollector.java | 7 ++++++- .../queries/function/FunctionMatchQuery.java | 6 ++++++ .../queries/function/FunctionQuery.java | 6 ++++++ .../queries/function/FunctionRangeQuery.java | 6 ++++++ .../queries/function/FunctionScoreQuery.java | 6 ++++++ .../document/LatLonDocValuesBoxQuery.java | 6 ++++++ .../LatLonDocValuesDistanceQuery.java | 6 ++++++ .../document/LatLonPointDistanceQuery.java | 6 ++++++ .../document/LatLonPointInPolygonQuery.java | 6 ++++++ .../apache/lucene/search/CoveringQuery.java | 19 +++++++++++++++++++ .../lucene/search/DocValuesNumbersQuery.java | 5 +++++ .../lucene/search/DocValuesTermsQuery.java | 5 +++++ .../lucene/search/TermAutomatonQuery.java | 5 +++++ .../lucene/search/TestTermAutomatonQuery.java | 5 +++++ .../composite/CompositeVerifyQuery.java | 6 ++++++ .../composite/IntersectsRPTVerifyQuery.java | 7 +++++++ .../prefix/AbstractPrefixTreeQuery.java | 6 ++++++ .../serialized/SerializedDVStrategy.java | 7 +++++++ .../spatial/vector/PointVectorStrategy.java | 7 +++++++ .../spatial3d/PointInGeo3DShapeQuery.java | 6 ++++++ .../suggest/document/CompletionWeight.java | 6 ++++++ .../org/apache/solr/ltr/LTRScoringQuery.java | 6 ++++++ .../org/apache/solr/ltr/feature/Feature.java | 6 ++++++ .../org/apache/solr/query/SolrRangeQuery.java | 6 ++++++ .../org/apache/solr/schema/LatLonType.java | 6 ++++++ .../java/org/apache/solr/search/Filter.java | 6 ++++++ .../solr/search/GraphTermsQParserPlugin.java | 11 +++++++++++ .../apache/solr/search/JoinQParserPlugin.java | 6 ++++++ .../solr/search/SolrConstantScoreQuery.java | 6 ++++++ .../apache/solr/search/join/GraphQuery.java | 8 +++++++- .../solr/update/DeleteByQueryWrapper.java | 6 ++++++ .../uninverting/TestFieldCacheSortRandom.java | 6 ++++++ 41 files changed, 263 insertions(+), 2 deletions(-) diff --git a/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysQuery.java b/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysQuery.java index 130e1d43cea9..f090e0817699 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysQuery.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysQuery.java @@ -32,6 +32,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -91,6 +92,11 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo @Override public void extractTerms(Set terms) {} + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { return baseWeight.explain(context, doc); diff --git a/lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRange.java b/lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRange.java index 56910f215cb5..901547821e01 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRange.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRange.java @@ -27,6 +27,7 @@ import org.apache.lucene.search.DoubleValues; import org.apache.lucene.search.DoubleValuesSource; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -143,6 +144,11 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo : searcher.createWeight(fastMatchQuery, ScoreMode.COMPLETE_NO_SCORES, 1f); return new ConstantScoreWeight(this, boost) { + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { final int maxDoc = context.reader().maxDoc(); diff --git a/lucene/facet/src/java/org/apache/lucene/facet/range/LongRange.java b/lucene/facet/src/java/org/apache/lucene/facet/range/LongRange.java index 88b569d4f87f..74d6c1e168b7 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/range/LongRange.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/range/LongRange.java @@ -27,6 +27,7 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.LongValues; import org.apache.lucene.search.LongValuesSource; +import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -135,6 +136,11 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo : searcher.createWeight(fastMatchQuery, ScoreMode.COMPLETE_NO_SCORES, 1f); return new ConstantScoreWeight(this, boost) { + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { final int maxDoc = context.reader().maxDoc(); diff --git a/lucene/facet/src/test/org/apache/lucene/facet/TestDrillSideways.java b/lucene/facet/src/test/org/apache/lucene/facet/TestDrillSideways.java index c632f3ae51cb..52b0e8b7a78d 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/TestDrillSideways.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/TestDrillSideways.java @@ -51,6 +51,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.ScoreMode; @@ -723,6 +724,11 @@ public void testRandom() throws Exception { public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { DocIdSetIterator approximation = DocIdSetIterator.all(context.reader().maxDoc()); diff --git a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java index 8247f81352a7..6d1c68cb1a26 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java +++ b/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java @@ -28,6 +28,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.TwoPhaseIterator; @@ -110,6 +111,11 @@ final class W extends ConstantScoreWeight { @Override public void extractTerms(Set terms) {} + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { SortedDocValues values = DocValues.getSorted(context.reader(), joinField); diff --git a/lucene/join/src/java/org/apache/lucene/search/join/ParentChildrenBlockJoinQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/ParentChildrenBlockJoinQuery.java index 7ce1c294592f..752ec34e6a5a 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/ParentChildrenBlockJoinQuery.java +++ b/lucene/join/src/java/org/apache/lucene/search/join/ParentChildrenBlockJoinQuery.java @@ -27,6 +27,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; @@ -104,6 +105,11 @@ public void extractTerms(Set terms) { childWeight.extractTerms(terms); } + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { return Explanation.noMatch("Not implemented, use ToParentBlockJoinQuery explain why a document matched"); diff --git a/lucene/join/src/java/org/apache/lucene/search/join/PointInSetIncludingScoreQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/PointInSetIncludingScoreQuery.java index 02b7e86d1688..2eaebdd5e4aa 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/PointInSetIncludingScoreQuery.java +++ b/lucene/join/src/java/org/apache/lucene/search/join/PointInSetIncludingScoreQuery.java @@ -40,6 +40,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.PointInSetQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; @@ -127,6 +128,11 @@ public final Weight createWeight(IndexSearcher searcher, org.apache.lucene.searc public void extractTerms(Set terms) { } + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { Scorer scorer = scorer(context); diff --git a/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java index 43ddd528393f..efb49d135af3 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java +++ b/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java @@ -29,6 +29,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; @@ -102,6 +103,11 @@ public Weight createWeight(IndexSearcher searcher, org.apache.lucene.search.Scor @Override public void extractTerms(Set terms) {} + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; // TODO do matches make sense here? + } + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { Terms terms = context.reader().terms(toField); diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java b/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java index a0f86af925b1..76c63c97b53a 100644 --- a/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java +++ b/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java @@ -524,6 +524,11 @@ public Weight createWeight(IndexSearcher searcher, org.apache.lucene.search.Scor public void extractTerms(Set terms) { } + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { return null; diff --git a/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java b/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java index db17a90464e7..7aa3d1cacf71 100644 --- a/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java +++ b/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java @@ -525,7 +525,12 @@ public int docID() { public void extractTerms(Set terms) { inner.extractTerms(terms); } - + + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { Scorer s = scorer(context); diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionMatchQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionMatchQuery.java index 7a837342f762..60937fba1a98 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionMatchQuery.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionMatchQuery.java @@ -28,6 +28,7 @@ import org.apache.lucene.search.DoubleValues; import org.apache.lucene.search.DoubleValuesSource; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -65,6 +66,11 @@ public String toString(String field) { public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { DoubleValuesSource vs = source.rewrite(searcher); return new ConstantScoreWeight(this, boost) { + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { DoubleValues values = vs.getValues(context, null); diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java index f996306a72d1..ec93772e7a31 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java @@ -26,6 +26,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -70,6 +71,11 @@ public FunctionWeight(IndexSearcher searcher, float boost) throws IOException { @Override public void extractTerms(Set terms) {} + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { return new AllScorer(context, this, boost); diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionRangeQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionRangeQuery.java index 2d55bae7693d..4f8593638379 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionRangeQuery.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionRangeQuery.java @@ -25,6 +25,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Weight; @@ -134,6 +135,11 @@ public void extractTerms(Set terms) { //none } + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { FunctionValues functionValues = valueSource.getValues(vsContext, context); diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionScoreQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionScoreQuery.java index 0d39e8b25892..a190f7f2f7d2 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionScoreQuery.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionScoreQuery.java @@ -29,6 +29,7 @@ import org.apache.lucene.search.Explanation; import org.apache.lucene.search.FilterScorer; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -148,6 +149,11 @@ public void extractTerms(Set terms) { this.inner.extractTerms(terms); } + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return inner.matches(context, doc, field); + } + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { Explanation scoreExplanation = inner.explain(context, doc); diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesBoxQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesBoxQuery.java index 31037f9c36e4..0013fbdf7b79 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesBoxQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesBoxQuery.java @@ -26,6 +26,7 @@ import org.apache.lucene.search.ConstantScoreScorer; import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -98,6 +99,11 @@ public int hashCode() { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; // TODO does it make sense to return matches here? + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { final SortedNumericDocValues values = context.reader().getSortedNumericDocValues(field); diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesDistanceQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesDistanceQuery.java index df350e6db95c..bde3895b403a 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesDistanceQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesDistanceQuery.java @@ -26,6 +26,7 @@ import org.apache.lucene.search.ConstantScoreScorer; import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -98,6 +99,11 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo private final GeoEncodingUtils.DistancePredicate distancePredicate = GeoEncodingUtils.createDistancePredicate(latitude, longitude, radiusMeters); + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; // TODO does it make sense to try and return matches? + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { final SortedNumericDocValues values = context.reader().getSortedNumericDocValues(field); diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java index a72d458ddc9d..c78aee7e4a7c 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java @@ -31,6 +31,7 @@ import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -109,6 +110,11 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo final GeoEncodingUtils.DistancePredicate distancePredicate = GeoEncodingUtils.createDistancePredicate(latitude, longitude, radiusMeters); + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; // TODO does it makes sense to try and return matches here? + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { ScorerSupplier scorerSupplier = scorerSupplier(context); diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java index 1b2e3a6b6d17..d0025154d882 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java @@ -32,6 +32,7 @@ import org.apache.lucene.search.ConstantScoreScorer; import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -98,6 +99,11 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo return new ConstantScoreWeight(this, boost) { + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; // TODO does it make sense to try and return matches here? + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { LeafReader reader = context.reader(); diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/CoveringQuery.java b/lucene/sandbox/src/java/org/apache/lucene/search/CoveringQuery.java index 8f821922c6d8..cdc9b376803a 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/search/CoveringQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/search/CoveringQuery.java @@ -136,6 +136,25 @@ public void extractTerms(Set terms) { } } + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + Scorer scorer = scorer(context); + if (scorer.iterator().advance(doc) != doc) { + return null; + } + List mis = new ArrayList<>(); + for (Weight w : weights) { + MatchesIterator mi = w.matches(context, doc, field); + if (mi != null) { + mis.add(mi); + } + } + if (mis.size() == 0) { + return null; + } + return new DisjunctionMatchesIterator(mis); + } + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { LongValues minMatchValues = minimumNumberMatch.getValues(context, null); diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesNumbersQuery.java b/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesNumbersQuery.java index e72e99241b60..4a1b5df63632 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesNumbersQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesNumbersQuery.java @@ -99,6 +99,11 @@ public String toString(String defaultField) { public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; // TODO maybe do something like return the matching docvalue index? + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { final SortedNumericDocValues values = DocValues.getSortedNumeric(context.reader(), field); diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesTermsQuery.java b/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesTermsQuery.java index 0e615b4ecbeb..841b16b1b68c 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesTermsQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesTermsQuery.java @@ -168,6 +168,11 @@ public String toString(String defaultField) { public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; // TODO does it make sense to return matches here? + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { final SortedSetDocValues values = DocValues.getSortedSet(context.reader(), field); diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java b/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java index 42a5f74216bb..877060b16516 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java @@ -371,6 +371,11 @@ public void extractTerms(Set terms) { } } + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; // TODO + } + @Override public String toString() { return "weight(" + TermAutomatonQuery.this + ")"; diff --git a/lucene/sandbox/src/test/org/apache/lucene/search/TestTermAutomatonQuery.java b/lucene/sandbox/src/test/org/apache/lucene/search/TestTermAutomatonQuery.java index d6bf90dfcbe9..70752acfdbad 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/search/TestTermAutomatonQuery.java +++ b/lucene/sandbox/src/test/org/apache/lucene/search/TestTermAutomatonQuery.java @@ -584,6 +584,11 @@ public RandomQuery(long seed, float density) { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { int maxDoc = context.reader().maxDoc(); diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/CompositeVerifyQuery.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/CompositeVerifyQuery.java index 2bfa4d53e70a..33f88200ea39 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/CompositeVerifyQuery.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/CompositeVerifyQuery.java @@ -23,6 +23,7 @@ import org.apache.lucene.search.ConstantScoreScorer; import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -86,6 +87,11 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo return new ConstantScoreWeight(this, boost) { + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; // TODO is there a way of reporting matches that makes sense? + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/IntersectsRPTVerifyQuery.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/IntersectsRPTVerifyQuery.java index e6324dae28e6..2fc8bac3750b 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/IntersectsRPTVerifyQuery.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/IntersectsRPTVerifyQuery.java @@ -24,6 +24,7 @@ import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -84,6 +85,12 @@ public int hashCode() { public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { + + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; // TODO is there a way of reporting matches that makes sense? + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { // Compute approx & exact diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/AbstractPrefixTreeQuery.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/AbstractPrefixTreeQuery.java index 75b3c2b80bc0..eabe65efe5a5 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/AbstractPrefixTreeQuery.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/AbstractPrefixTreeQuery.java @@ -28,6 +28,7 @@ import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -79,6 +80,11 @@ public int hashCode() { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; // TODO is there a way of reporting matches that makes sense? + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { DocIdSet docSet = getDocIdSet(context); diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java index cd94bf429bf2..2dda25383070 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java @@ -33,6 +33,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.DoubleValuesSource; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -136,6 +137,12 @@ public PredicateValueSourceQuery(ShapeValuesPredicate predicateValueSource) { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { + + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; // TODO is there a way of reporting matches that makes sense? + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { DocIdSetIterator approximation = DocIdSetIterator.all(context.reader().maxDoc()); diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java index c7904df72160..bff19c56cbc1 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java @@ -36,6 +36,7 @@ import org.apache.lucene.search.DoubleValues; import org.apache.lucene.search.DoubleValuesSource; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -269,6 +270,12 @@ public Query rewrite(IndexReader reader) throws IOException { public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { Weight w = inner.createWeight(searcher, scoreMode, 1f); return new ConstantScoreWeight(this, boost) { + + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return w.matches(context, doc, field); + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { Scorer in = w.scorer(context); diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInGeo3DShapeQuery.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInGeo3DShapeQuery.java index 83a471fab55e..04be8ccf4113 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInGeo3DShapeQuery.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInGeo3DShapeQuery.java @@ -24,6 +24,7 @@ import org.apache.lucene.search.ConstantScoreScorer; import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -68,6 +69,11 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo return new ConstantScoreWeight(this, boost) { + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; // TODO is there a way of reporting matches that makes sense here? + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { LeafReader reader = context.reader(); diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionWeight.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionWeight.java index 1f97d559c4da..225c057ad702 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionWeight.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionWeight.java @@ -25,6 +25,7 @@ import org.apache.lucene.index.Terms; import org.apache.lucene.search.BulkScorer; import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; import org.apache.lucene.search.suggest.BitsProducer; @@ -147,6 +148,11 @@ public void extractTerms(Set terms) { // no-op } + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { //TODO diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/LTRScoringQuery.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/LTRScoringQuery.java index c143d81f2273..6674885f688d 100644 --- a/solr/contrib/ltr/src/java/org/apache/solr/ltr/LTRScoringQuery.java +++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/LTRScoringQuery.java @@ -39,6 +39,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -450,6 +451,11 @@ public void extractTerms(Set terms) { } } + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + protected void reset() { for (int i = 0; i < extractedFeatureWeights.length;++i){ int featId = extractedFeatureWeights[i].getIndex(); diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/Feature.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/Feature.java index 066d28125017..9bf4606350e2 100644 --- a/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/Feature.java +++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/Feature.java @@ -26,6 +26,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; @@ -228,6 +229,11 @@ public float getDefaultValue() { public abstract FeatureScorer scorer(LeafReaderContext context) throws IOException; + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public boolean isCacheable(LeafReaderContext ctx) { return false; diff --git a/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java b/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java index 1b81c7f5ec45..8b7038472c5e 100644 --- a/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java +++ b/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java @@ -38,6 +38,7 @@ import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -485,6 +486,11 @@ public BulkScorer bulkScorer(LeafReaderContext context) throws IOException { } } + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { final SegState weightOrBitSet = getSegState(context); diff --git a/solr/core/src/java/org/apache/solr/schema/LatLonType.java b/solr/core/src/java/org/apache/solr/schema/LatLonType.java index 9f9dcd18dfb8..f095fdbe06d4 100644 --- a/solr/core/src/java/org/apache/solr/schema/LatLonType.java +++ b/solr/core/src/java/org/apache/solr/schema/LatLonType.java @@ -33,6 +33,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -338,6 +339,11 @@ public boolean isCacheable(LeafReaderContext ctx) { return false; } + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { return ((SpatialScorer)scorer(context)).explain(super.explain(context, doc), doc); diff --git a/solr/core/src/java/org/apache/solr/search/Filter.java b/solr/core/src/java/org/apache/solr/search/Filter.java index 3af83e2cfdfe..7f53d0589b04 100644 --- a/solr/core/src/java/org/apache/solr/search/Filter.java +++ b/solr/core/src/java/org/apache/solr/search/Filter.java @@ -26,6 +26,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -95,6 +96,11 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo @Override public void extractTerms(Set terms) {} + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { final Scorer scorer = scorer(context); diff --git a/solr/core/src/java/org/apache/solr/search/GraphTermsQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/GraphTermsQParserPlugin.java index d1f7ff2e4707..085a835d6bc1 100644 --- a/solr/core/src/java/org/apache/solr/search/GraphTermsQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/GraphTermsQParserPlugin.java @@ -49,6 +49,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -275,6 +276,11 @@ public void extractTerms(Set terms) { // order to protect highlighters } + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + private WeightOrDocIdSet rewrite(LeafReaderContext context) throws IOException { final LeafReader reader = context.reader(); Terms terms = reader.terms(field); @@ -613,6 +619,11 @@ public final Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, fl return new ConstantScoreWeight(this, boost) { Filter filter; + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { if (filter == null) { diff --git a/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java index ebf0ebcee782..661df487040f 100644 --- a/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java @@ -35,6 +35,7 @@ import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -239,6 +240,11 @@ public void close() { Filter filter; + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { if (filter == null) { diff --git a/solr/core/src/java/org/apache/solr/search/SolrConstantScoreQuery.java b/solr/core/src/java/org/apache/solr/search/SolrConstantScoreQuery.java index 1f81e1e61d60..4cbf730221fc 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrConstantScoreQuery.java +++ b/solr/core/src/java/org/apache/solr/search/SolrConstantScoreQuery.java @@ -27,6 +27,7 @@ import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -92,6 +93,11 @@ public ConstantWeight(IndexSearcher searcher, float boost) throws IOException { ((SolrFilter)filter).createWeight(context, searcher); } + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { DocIdSet docIdSet = filter instanceof SolrFilter ? ((SolrFilter)filter).getDocIdSet(this.context, context, null) : filter.getDocIdSet(context, null); diff --git a/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java b/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java index 4402a2667479..8777bc690543 100644 --- a/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java +++ b/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java @@ -33,6 +33,7 @@ import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -286,7 +287,12 @@ public boolean isCacheable(LeafReaderContext ctx) { public void extractTerms(Set terms) { // NoOp for now , not used.. / supported } - + + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + } private static class GraphScorer extends Scorer { diff --git a/solr/core/src/java/org/apache/solr/update/DeleteByQueryWrapper.java b/solr/core/src/java/org/apache/solr/update/DeleteByQueryWrapper.java index 1c87a39b2a1c..a8b9a3576185 100644 --- a/solr/core/src/java/org/apache/solr/update/DeleteByQueryWrapper.java +++ b/solr/core/src/java/org/apache/solr/update/DeleteByQueryWrapper.java @@ -26,6 +26,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -77,6 +78,11 @@ public void extractTerms(Set terms) { throw new UnsupportedOperationException(); } + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { throw new UnsupportedOperationException(); } diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSortRandom.java b/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSortRandom.java index 58069cd060bb..927cf8ff4b9c 100644 --- a/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSortRandom.java +++ b/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSortRandom.java @@ -43,6 +43,7 @@ import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -271,6 +272,11 @@ public RandomQuery(long seed, float density, List docValues) { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { + @Override + public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + return null; + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { Random random = new Random(seed ^ context.docBase); From b6bc20bcc9ef22d4f177723c28c7a466b49a7893 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 28 Mar 2018 14:53:21 +0100 Subject: [PATCH 07/45] Simplify DMI a bit, add test for SynonymQuery --- .../search/DisjunctionMatchesIterator.java | 31 +++++++------------ .../lucene/search/TestMatchesIterator.java | 12 ++++++- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java b/lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java index 5fbea724d9fe..c2af20f6bfcf 100644 --- a/lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java +++ b/lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java @@ -35,26 +35,19 @@ class DisjunctionMatchesIterator implements MatchesIterator { public static DisjunctionMatchesIterator fromTerms(LeafReaderContext context, int doc, String field, List terms) throws IOException { - List mis = new ArrayList<>(); - Terms t = context.reader().terms(field); - if (t == null) - return null; - TermsEnum te = t.iterator(); - PostingsEnum reuse = null; - for (Term term : terms) { - if (te.seekExact(term.bytes())) { - PostingsEnum pe = te.postings(reuse, PostingsEnum.OFFSETS); - if (pe.advance(doc) == doc) { - mis.add(new TermMatchesIterator(pe)); - } - else { - reuse = pe; - } + return fromTermsEnum(context, doc, field, asBytesRefIterator(terms)); + } + + private static BytesRefIterator asBytesRefIterator(List terms) { + return new BytesRefIterator() { + int i = 0; + @Override + public BytesRef next() { + if (i >= terms.size()) + return null; + return terms.get(i++).bytes(); } - } - if (mis.size() == 0) - return null; - return new DisjunctionMatchesIterator(mis); + }; } public static DisjunctionMatchesIterator fromTermsEnum(LeafReaderContext context, int doc, String field, BytesRefIterator terms) throws IOException { diff --git a/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java b/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java index 419080ac4ea6..82919b1a691b 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java @@ -92,7 +92,7 @@ void checkMatches(Query q, String field, int[][] expected) throws IOException { } int pos = 1; while (it.next()) { - System.out.println(expected[i][pos] + "->" + expected[i][pos + 1] + "[" + expected[i][pos + 2] + "->" + expected[i][pos + 3] + "]"); + //System.out.println(expected[i][pos] + "->" + expected[i][pos + 1] + "[" + expected[i][pos + 2] + "->" + expected[i][pos + 3] + "]"); assertEquals(expected[i][pos], it.startPosition()); assertEquals(expected[i][pos + 1], it.endPosition()); assertEquals(expected[i][pos + 2], it.startOffset()); @@ -211,6 +211,16 @@ public void testWildcards() throws IOException { }); } + public void testSynonymQuery() throws IOException { + Query q = new SynonymQuery(new Term(FIELD_WITH_OFFSETS, "w1"), new Term(FIELD_WITH_OFFSETS, "w2")); + checkMatches(q, FIELD_WITH_OFFSETS, new int[][]{ + { 0, 0, 0, 0, 2, 1, 1, 3, 5 }, + { 1, 0, 0, 0, 2, 2, 2, 6, 8 }, + { 2, 0, 0, 0, 2, 2, 2, 6, 8 }, + { 3, 0, 0, 0, 2, 1, 1, 3, 5, 2, 2, 6, 8, 4, 4, 12, 14 } + }); + } + protected String[] doc1Fields = { "w1 w2 w3 w4 w5", "w1 w3 w2 w3 zz", From d91eb9ea5fe55546a3a675b1e24401a3a248114c Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 28 Mar 2018 15:00:36 +0100 Subject: [PATCH 08/45] PhraseQuery nocommit -> TODO --- .../src/java/org/apache/lucene/search/MultiPhraseQuery.java | 2 +- lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java b/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java index 9b6f57c5496c..b99f37f0f9d9 100644 --- a/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java @@ -237,7 +237,7 @@ public void extractTerms(Set terms) { @Override public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; // nocommit + return null; // TODO - see PhraseQuery } @Override diff --git a/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java index 82859200a4ee..dcc57d34820d 100644 --- a/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java @@ -394,7 +394,7 @@ public void extractTerms(Set queryTerms) { @Override public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; // nocommit + return null; // TODO - should be able to refactor Exact/SloppyPhraseScorer into something that reports positions too? } @Override From 8402732d93c32354c0103ee0dcd218ff2ece9d6e Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Thu, 29 Mar 2018 09:28:13 +0100 Subject: [PATCH 09/45] WIP --- .../org/apache/lucene/search/ScoreDoc.java | 2 +- .../search/matchhighlight/HighlightDoc.java | 32 +++++++ .../matchhighlight/MatchHighlighter.java | 92 +++++++++++++++++++ .../matchhighlight/SnippetGenerator.java | 21 +++++ .../search/matchhighlight/TopHighlights.java | 21 +++++ 5 files changed, 167 insertions(+), 1 deletion(-) create mode 100644 lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/HighlightDoc.java create mode 100644 lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchHighlighter.java create mode 100644 lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SnippetGenerator.java create mode 100644 lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/TopHighlights.java diff --git a/lucene/core/src/java/org/apache/lucene/search/ScoreDoc.java b/lucene/core/src/java/org/apache/lucene/search/ScoreDoc.java index eb95e2980538..26e006db3a2d 100644 --- a/lucene/core/src/java/org/apache/lucene/search/ScoreDoc.java +++ b/lucene/core/src/java/org/apache/lucene/search/ScoreDoc.java @@ -19,7 +19,7 @@ /** Holds one hit in {@link TopDocs}. */ -public class ScoreDoc { +public class ScoreDoc { /** The score of this document for the query. */ public float score; diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/HighlightDoc.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/HighlightDoc.java new file mode 100644 index 000000000000..b0dbc5e47569 --- /dev/null +++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/HighlightDoc.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.search.matchhighlight; + +import org.apache.lucene.document.Document; + +public class HighlightDoc { + + public final int doc; + + public final Document fields; + + public HighlightDoc(int doc, Document fields) { + this.doc = doc; + this.fields = fields; + } +} diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchHighlighter.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchHighlighter.java new file mode 100644 index 000000000000..86f1b8e691cf --- /dev/null +++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchHighlighter.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.search.matchhighlight; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.ReaderUtil; +import org.apache.lucene.index.StoredFieldVisitor; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchesIterator; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.Weight; + +public class MatchHighlighter { + + private final IndexSearcher searcher; + private final Analyzer analyzer; + + public MatchHighlighter(IndexSearcher searcher, Analyzer analyzer) { + this.searcher = searcher; + this.analyzer = analyzer; + } + + public TopHighlights highlight(Query query, TopDocs docs, SnippetGenerator generator) throws IOException { + List> highlights = new ArrayList<>(); + Weight weight = searcher.createNormalizedWeight(query, ScoreMode.COMPLETE); + for (ScoreDoc doc : docs.scoreDocs) { + int contextOrd = ReaderUtil.subIndex(doc.doc, searcher.getIndexReader().leaves()); + LeafReaderContext ctx = searcher.getIndexReader().leaves().get(contextOrd); + HighlightingFieldVisitor visitor = new HighlightingFieldVisitor<>(weight, ctx, doc.doc - ctx.docBase, generator); + ctx.reader().document(doc.doc, visitor); + highlights.add(visitor.getHighights()); + } + return new TopHighlights<>(highlights); + } + + private class HighlightingFieldVisitor extends StoredFieldVisitor { + + final LeafReaderContext context; + final int doc; + final SnippetGenerator generator; + final Weight weight; + + final HighlightDoc highlights; + + private HighlightingFieldVisitor(Weight weight, LeafReaderContext context, int doc, SnippetGenerator generator) { + this.context = context; + this.doc = doc; + this.generator = generator; + this.weight = weight; + this.highlights = new HighlightDoc<>(doc, fields); + } + + @Override + public Status needsField(FieldInfo fieldInfo) throws IOException { + return generator.needsField(fieldInfo.name); + } + + @Override + public void stringField(FieldInfo fieldInfo, byte[] value) throws IOException { + String field = fieldInfo.name; + MatchesIterator matches = weight.matches(context, doc, field); + if (matches != null) { + this.highlights.highlights.put(field, generator.getSnippets(field, value, matches)); + } + } + } + +} diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SnippetGenerator.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SnippetGenerator.java new file mode 100644 index 000000000000..c1e15be49839 --- /dev/null +++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SnippetGenerator.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.search.matchhighlight; + +public class SnippetGenerator { +} diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/TopHighlights.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/TopHighlights.java new file mode 100644 index 000000000000..5329508676cf --- /dev/null +++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/TopHighlights.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.search.matchhighlight; + +public class TopHighlights { +} From f58745e846c9fef676784f69c542bc2252352a66 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Thu, 29 Mar 2018 11:02:29 +0100 Subject: [PATCH 10/45] Add Matches object to store MatchesIterators for multiple fields --- .../apache/lucene/document/FeatureQuery.java | 9 ++- .../lucene/document/RangeFieldQuery.java | 9 ++- .../SortedNumericDocValuesRangeQuery.java | 9 ++- .../SortedSetDocValuesRangeQuery.java | 9 ++- .../apache/lucene/search/BooleanWeight.java | 41 +++++----- .../lucene/search/ConstantScoreQuery.java | 4 +- .../search/DisjunctionMatchesIterator.java | 11 ++- .../lucene/search/DisjunctionMaxQuery.java | 11 +-- .../search/DocValuesFieldExistsQuery.java | 8 +- .../lucene/search/DocValuesRewriteMethod.java | 7 +- .../apache/lucene/search/FilterWeight.java | 4 +- .../lucene/search/IndexOrDocValuesQuery.java | 5 +- .../apache/lucene/search/LRUQueryCache.java | 4 +- .../lucene/search/MatchAllDocsQuery.java | 4 +- .../lucene/search/MatchNoDocsQuery.java | 2 +- .../org/apache/lucene/search/Matches.java | 80 +++++++++++++++++++ .../apache/lucene/search/MatchesIterator.java | 29 ++++++- .../lucene/search/MultiPhraseQuery.java | 8 +- .../MultiTermQueryConstantScoreWrapper.java | 4 +- .../lucene/search/NormsFieldExistsQuery.java | 8 +- .../org/apache/lucene/search/PhraseQuery.java | 8 +- .../apache/lucene/search/PointInSetQuery.java | 8 +- .../apache/lucene/search/PointRangeQuery.java | 8 +- .../apache/lucene/search/SynonymQuery.java | 5 +- .../apache/lucene/search/TermInSetQuery.java | 4 +- .../org/apache/lucene/search/TermQuery.java | 7 +- .../java/org/apache/lucene/search/Weight.java | 10 +-- .../lucene/search/spans/SpanWeight.java | 10 ++- .../lucene/search/JustCompileSearch.java | 2 +- .../lucene/search/TestBooleanScorer.java | 2 +- .../lucene/search/TestLRUQueryCache.java | 10 +-- .../lucene/search/TestMatchesIterator.java | 51 +++++++++--- .../lucene/search/TestQueryRescorer.java | 2 +- .../apache/lucene/search/TestScorerPerf.java | 2 +- .../apache/lucene/search/TestSortRandom.java | 2 +- .../TestUsageTrackingFilterCachingPolicy.java | 2 +- .../lucene/search/AssertingMatches.java | 43 ++++++++++ .../apache/lucene/search/AssertingWeight.java | 6 +- .../lucene/search/BlockScoreQueryWrapper.java | 4 +- 39 files changed, 337 insertions(+), 115 deletions(-) create mode 100644 lucene/core/src/java/org/apache/lucene/search/Matches.java create mode 100644 lucene/test-framework/src/java/org/apache/lucene/search/AssertingMatches.java diff --git a/lucene/core/src/java/org/apache/lucene/document/FeatureQuery.java b/lucene/core/src/java/org/apache/lucene/document/FeatureQuery.java index a7af04e93f5f..387b365fbfa7 100644 --- a/lucene/core/src/java/org/apache/lucene/document/FeatureQuery.java +++ b/lucene/core/src/java/org/apache/lucene/document/FeatureQuery.java @@ -30,6 +30,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; @@ -83,8 +84,12 @@ public boolean isCacheable(LeafReaderContext ctx) { public void extractTerms(Set terms) {} @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.fromField(fieldName, MatchesIterator.EMPTY); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java b/lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java index 7d1a6c3dee57..4d0e109de899 100644 --- a/lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java +++ b/lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java @@ -30,6 +30,7 @@ import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; @@ -351,8 +352,12 @@ public long cost() { } @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.fromField(field, MatchesIterator.EMPTY); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java b/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java index c9137a3cd5cd..3ba9fd41bb30 100644 --- a/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java +++ b/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java @@ -29,6 +29,7 @@ import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; @@ -103,8 +104,12 @@ public boolean isCacheable(LeafReaderContext ctx) { } @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.fromField(field, MatchesIterator.EMPTY); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java b/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java index a21d9514385d..d23d4ca722a5 100644 --- a/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java +++ b/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java @@ -29,6 +29,7 @@ import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; @@ -108,8 +109,12 @@ public Query rewrite(IndexReader reader) throws IOException { public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.fromField(field, MatchesIterator.EMPTY); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java b/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java index 0040fde3bf62..dd320e78d0af 100644 --- a/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java +++ b/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java @@ -121,35 +121,38 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio } @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - if (query.getClauses(Occur.SHOULD).size() != weights.size() || query.getMinimumNumberShouldMatch() > 0) { - // check that we actually match the doc in this case - Scorer scorer = scorer(context); - if (scorer == null) { - return null; - } - if (scorer.iterator().advance(doc) != doc) { - return null; - } - } - List mis = new ArrayList<>(); + public Matches matches(LeafReaderContext context, int doc) throws IOException { + final int minShouldMatch = query.getMinimumNumberShouldMatch(); + List matches = new ArrayList<>(); + int shouldMatchCount = 0; Iterator wIt = weights.iterator(); Iterator cIt = query.clauses().iterator(); while (wIt.hasNext()) { Weight w = wIt.next(); BooleanClause bc = cIt.next(); - if (bc.getOccur() == Occur.MUST_NOT) { - continue; + Matches m = w.matches(context, doc); + if (bc.isProhibited()) { + if (m != null) { + return null; + } + } + if (bc.isRequired()) { + if (m == null) { + return null; + } + matches.add(m); } - MatchesIterator mi = w.matches(context, doc, field); - if (mi != null) { - mis.add(mi); + if (bc.getOccur() == Occur.SHOULD) { + if (m != null) { + matches.add(m); + shouldMatchCount++; + } } } - if (mis.size() == 0) { + if (shouldMatchCount < minShouldMatch) { return null; } - return new DisjunctionMatchesIterator(mis); + return Matches.fromSubMatches(matches); } static BulkScorer disableScoring(final BulkScorer scorer) { diff --git a/lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java b/lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java index 725cad1ff9ff..d696c215f1bf 100644 --- a/lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java @@ -159,8 +159,8 @@ public long cost() { } @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return innerWeight.matches(context, doc, field); + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return innerWeight.matches(context, doc); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java b/lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java index c2af20f6bfcf..6a506c6fc202 100644 --- a/lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java +++ b/lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java @@ -74,11 +74,19 @@ public static DisjunctionMatchesIterator fromTermsEnum(LeafReaderContext context return new DisjunctionMatchesIterator(mis); } + public static MatchesIterator fromSubIterators(List mis) throws IOException { + if (mis.size() == 0) + return null; + if (mis.size() == 1) + return mis.get(0); + return new DisjunctionMatchesIterator(mis); + } + private final PriorityQueue queue; private boolean started = false; - public DisjunctionMatchesIterator(List matches) throws IOException { + private DisjunctionMatchesIterator(List matches) throws IOException { queue = new PriorityQueue(matches.size()){ @Override protected boolean lessThan(MatchesIterator a, MatchesIterator b) { @@ -126,4 +134,5 @@ public int startOffset() throws IOException { public int endOffset() throws IOException { return queue.top().endOffset(); } + } diff --git a/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java b/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java index 9f2cc3bd18f7..5b99c3ce24b4 100644 --- a/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java @@ -119,18 +119,15 @@ public void extractTerms(Set terms) { } @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - List mis = new ArrayList<>(); + public Matches matches(LeafReaderContext context, int doc) throws IOException { + List mis = new ArrayList<>(); for (Weight weight : weights) { - MatchesIterator mi = weight.matches(context, doc, field); + Matches mi = weight.matches(context, doc); if (mi != null) { mis.add(mi); } } - if (mis.size() == 0) { - return null; - } - return new DisjunctionMatchesIterator(mis); + return Matches.fromSubMatches(mis); } /** Create the scorer used to score our associated DisjunctionMaxQuery */ diff --git a/lucene/core/src/java/org/apache/lucene/search/DocValuesFieldExistsQuery.java b/lucene/core/src/java/org/apache/lucene/search/DocValuesFieldExistsQuery.java index 8fc0542b57f6..1febbff301b4 100644 --- a/lucene/core/src/java/org/apache/lucene/search/DocValuesFieldExistsQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/DocValuesFieldExistsQuery.java @@ -65,8 +65,12 @@ public String toString(String field) { public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.fromField(field, MatchesIterator.EMPTY); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/search/DocValuesRewriteMethod.java b/lucene/core/src/java/org/apache/lucene/search/DocValuesRewriteMethod.java index 33b64387ea15..c745ced475c4 100644 --- a/lucene/core/src/java/org/apache/lucene/search/DocValuesRewriteMethod.java +++ b/lucene/core/src/java/org/apache/lucene/search/DocValuesRewriteMethod.java @@ -76,12 +76,9 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo return new ConstantScoreWeight(this, boost) { @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - if (query.field.equals(field) == false) { - return null; - } + public Matches matches(LeafReaderContext context, int doc) throws IOException { final SortedSetDocValues fcsi = DocValues.getSortedSet(context.reader(), query.field); - return DisjunctionMatchesIterator.fromTermsEnum(context, doc, field, getTermsEnum(fcsi)); + return Matches.fromField(query.field, DisjunctionMatchesIterator.fromTermsEnum(context, doc, query.field, getTermsEnum(fcsi))); } private TermsEnum getTermsEnum(SortedSetDocValues fcsi) throws IOException { diff --git a/lucene/core/src/java/org/apache/lucene/search/FilterWeight.java b/lucene/core/src/java/org/apache/lucene/search/FilterWeight.java index 3e409823432d..8a2b57b41ea6 100644 --- a/lucene/core/src/java/org/apache/lucene/search/FilterWeight.java +++ b/lucene/core/src/java/org/apache/lucene/search/FilterWeight.java @@ -76,7 +76,7 @@ public Scorer scorer(LeafReaderContext context) throws IOException { } @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return in.matches(context, doc, field); + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return in.matches(context, doc); } } diff --git a/lucene/core/src/java/org/apache/lucene/search/IndexOrDocValuesQuery.java b/lucene/core/src/java/org/apache/lucene/search/IndexOrDocValuesQuery.java index 8024e1a97bb7..1a236db673ba 100644 --- a/lucene/core/src/java/org/apache/lucene/search/IndexOrDocValuesQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/IndexOrDocValuesQuery.java @@ -120,8 +120,9 @@ public void extractTerms(Set terms) { } @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return indexWeight.matches(context, doc, field); + public Matches matches(LeafReaderContext context, int doc) throws IOException { + // TODO we can probably make use of the doc values here to return something more useful + return indexWeight.matches(context, doc); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java b/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java index 35234f696740..9391afdbf962 100644 --- a/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java +++ b/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java @@ -679,8 +679,8 @@ public void extractTerms(Set terms) { } @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return in.matches(context, doc, field); + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return in.matches(context, doc); } private boolean cacheEntryHasReasonableWorstCaseSize(int maxDoc) { diff --git a/lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java b/lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java index bfeaff62e7a0..2214c902ce21 100644 --- a/lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java @@ -37,8 +37,8 @@ public String toString() { } @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.fromField("*", MatchesIterator.EMPTY); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/search/MatchNoDocsQuery.java b/lucene/core/src/java/org/apache/lucene/search/MatchNoDocsQuery.java index b457eaebf97b..2527539dd3a4 100644 --- a/lucene/core/src/java/org/apache/lucene/search/MatchNoDocsQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/MatchNoDocsQuery.java @@ -49,7 +49,7 @@ public void extractTerms(Set terms) { } @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + public Matches matches(LeafReaderContext context, int doc) throws IOException { return null; } diff --git a/lucene/core/src/java/org/apache/lucene/search/Matches.java b/lucene/core/src/java/org/apache/lucene/search/Matches.java new file mode 100644 index 000000000000..c5a54bcc97af --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/search/Matches.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.search; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class Matches { + + private final Map matches; + + public static Matches fromField(String field, MatchesIterator it) { + if (it == null) { + return null; + } + return new Matches(field, it); + } + + public static Matches fromSubMatches(List subMatches) throws IOException { + if (subMatches == null || subMatches.size() == 0) { + return null; + } + if (subMatches.size() == 1) { + return subMatches.get(0); + } + Map matches = new HashMap<>(); + Set allFields = new HashSet<>(); + for (Matches m : subMatches) { + allFields.addAll(m.getMatchFields()); + } + for (String field : allFields) { + List mis = new ArrayList<>(); + for (Matches m : subMatches) { + MatchesIterator mi = m.getMatches(field); + if (mi != null) { + mis.add(mi); + } + } + matches.put(field, DisjunctionMatchesIterator.fromSubIterators(mis)); + } + return new Matches(matches); + } + + protected Matches(Map matches) { + this.matches = matches; + } + + private Matches(String field, MatchesIterator iterator) { + this.matches = new HashMap<>(); + this.matches.put(field, iterator); + } + + public MatchesIterator getMatches(String field) { + return matches.get(field); + } + + public Set getMatchFields() { + return matches.keySet(); + } +} diff --git a/lucene/core/src/java/org/apache/lucene/search/MatchesIterator.java b/lucene/core/src/java/org/apache/lucene/search/MatchesIterator.java index f77f985a813f..7d70f16d91fc 100644 --- a/lucene/core/src/java/org/apache/lucene/search/MatchesIterator.java +++ b/lucene/core/src/java/org/apache/lucene/search/MatchesIterator.java @@ -30,7 +30,7 @@ * * Matches are ordered by start position, and then by end position. Match intervals map overlap. * - * @see Weight#matches(LeafReaderContext, int, String) + * @see Weight#matches(LeafReaderContext, int) */ public interface MatchesIterator { @@ -60,4 +60,31 @@ public interface MatchesIterator { */ int endOffset() throws IOException; + final MatchesIterator EMPTY = new MatchesIterator() { + @Override + public boolean next() throws IOException { + return false; + } + + @Override + public int startPosition() { + throw new UnsupportedOperationException(); + } + + @Override + public int endPosition() { + throw new UnsupportedOperationException(); + } + + @Override + public int startOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endOffset() throws IOException { + throw new UnsupportedOperationException(); + } + }; + } diff --git a/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java b/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java index b99f37f0f9d9..5ec597d2b834 100644 --- a/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java @@ -236,8 +236,12 @@ public void extractTerms(Set terms) { } @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; // TODO - see PhraseQuery + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.fromField(field, MatchesIterator.EMPTY); // TODO } @Override diff --git a/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryConstantScoreWrapper.java b/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryConstantScoreWrapper.java index ec6e3fdd568b..51214e53f675 100644 --- a/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryConstantScoreWrapper.java +++ b/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryConstantScoreWrapper.java @@ -203,12 +203,12 @@ public BulkScorer bulkScorer(LeafReaderContext context) throws IOException { } @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + public Matches matches(LeafReaderContext context, int doc) throws IOException { final Terms terms = context.reader().terms(query.field); if (terms == null) { return null; } - return DisjunctionMatchesIterator.fromTermsEnum(context, doc, field, query.getTermsEnum(terms)); + return Matches.fromField(query.field, DisjunctionMatchesIterator.fromTermsEnum(context, doc, query.field, query.getTermsEnum(terms))); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/search/NormsFieldExistsQuery.java b/lucene/core/src/java/org/apache/lucene/search/NormsFieldExistsQuery.java index 45e7bb181e29..85951b7e4469 100644 --- a/lucene/core/src/java/org/apache/lucene/search/NormsFieldExistsQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/NormsFieldExistsQuery.java @@ -65,8 +65,12 @@ public String toString(String field) { public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.fromField(field, MatchesIterator.EMPTY); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java index dcc57d34820d..e213bda5fbca 100644 --- a/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java @@ -393,8 +393,12 @@ public void extractTerms(Set queryTerms) { } @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; // TODO - should be able to refactor Exact/SloppyPhraseScorer into something that reports positions too? + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.fromField(field, MatchesIterator.EMPTY); // TODO } @Override diff --git a/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java b/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java index a15b57ec2deb..16cfbd1dd6ab 100644 --- a/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java @@ -114,8 +114,12 @@ public final Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, fl return new ConstantScoreWeight(this, boost) { @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.fromField(field, MatchesIterator.EMPTY); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java b/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java index 452089a3143e..f61146d56bdf 100644 --- a/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java @@ -314,8 +314,12 @@ public long cost() { } @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.fromField(field, MatchesIterator.EMPTY); // TODO can we return values here somehow? } @Override diff --git a/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java b/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java index 6c73444db8d0..7dd3067514a2 100644 --- a/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java @@ -162,8 +162,9 @@ public void extractTerms(Set terms) { } @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return DisjunctionMatchesIterator.fromTerms(context, doc, field, Arrays.asList(SynonymQuery.this.terms)); + public Matches matches(LeafReaderContext context, int doc) throws IOException { + String field = terms[0].field(); + return Matches.fromField(field, DisjunctionMatchesIterator.fromTerms(context, doc, field, Arrays.asList(SynonymQuery.this.terms))); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/search/TermInSetQuery.java b/lucene/core/src/java/org/apache/lucene/search/TermInSetQuery.java index 778297184adc..be106f554641 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TermInSetQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/TermInSetQuery.java @@ -221,8 +221,8 @@ public void extractTerms(Set terms) { } @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return DisjunctionMatchesIterator.fromTermsEnum(context, doc, field, termData.iterator()); + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.fromField(field, DisjunctionMatchesIterator.fromTermsEnum(context, doc, field, termData.iterator())); } /** diff --git a/lucene/core/src/java/org/apache/lucene/search/TermQuery.java b/lucene/core/src/java/org/apache/lucene/search/TermQuery.java index 77abec06d36f..beb7946ae6b9 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TermQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/TermQuery.java @@ -82,10 +82,7 @@ public void extractTerms(Set terms) { } @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - if (term.field().equals(field) == false) { - return null; - } + public Matches matches(LeafReaderContext context, int doc) throws IOException { TermsEnum te = getTermsEnum(context); if (te == null) { return null; @@ -94,7 +91,7 @@ public MatchesIterator matches(LeafReaderContext context, int doc, String field) if (pe.advance(doc) != doc) { return null; } - return new TermMatchesIterator(pe); + return Matches.fromField(term.field(), new TermMatchesIterator(pe)); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/search/Weight.java b/lucene/core/src/java/org/apache/lucene/search/Weight.java index 1487d1089f7b..8c8e5b0c7aa6 100644 --- a/lucene/core/src/java/org/apache/lucene/search/Weight.java +++ b/lucene/core/src/java/org/apache/lucene/search/Weight.java @@ -71,20 +71,14 @@ protected Weight(Query query) { public abstract void extractTerms(Set terms); /** - * Returns an iterator over the match positions in a field for the named document + * Returns the {@link Matches} for a specific document * * If there are no matches, returns {@code null} * - * If the field was indexed with offsets, then the {@link MatchesIterator} will - * expose those offsets, otherwise only positions will be returned and - * {@link MatchesIterator#startOffset()} and {@link MatchesIterator#endOffset()} will - * always return {@code -1}. - * * @param context the reader's context to create the {@link MatchesIterator} for * @param doc the document's id relative to the given context's reader - * @param field the field to return match positions for */ - public abstract MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException; + public abstract Matches matches(LeafReaderContext context, int doc) throws IOException; /** * An explanation of the score computation for the named document. diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java b/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java index 26c69c97257f..323a053d9170 100644 --- a/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java +++ b/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java @@ -29,6 +29,7 @@ import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.LeafSimScorer; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.TermStatistics; import org.apache.lucene.search.Weight; @@ -164,13 +165,12 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio } @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - if (this.field.equals(field) == false) - return null; + public Matches matches(LeafReaderContext context, int doc) throws IOException { Spans spans = getSpans(context, Postings.OFFSETS); if (spans.advance(doc) != doc) return null; - return new MatchesIterator() { + + MatchesIterator mi = new MatchesIterator() { SpanCollector offsetCollector = new SpanCollector() { @Override @@ -218,5 +218,7 @@ public int endOffset() { return endOffset; } }; + + return Matches.fromField(field, mi); } } diff --git a/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java b/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java index 4e08a44ea5e4..909e93290f04 100644 --- a/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java +++ b/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java @@ -248,7 +248,7 @@ public void extractTerms(Set terms) { } @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + public Matches matches(LeafReaderContext context, int doc) throws IOException { return null; } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java b/lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java index 903622cdd556..c76b1607c046 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java @@ -85,7 +85,7 @@ public void extractTerms(Set terms) { } @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + public Matches matches(LeafReaderContext context, int doc) throws IOException { return null; } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java b/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java index 0b8eb839a6e4..5b052f6fed8d 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java @@ -350,7 +350,7 @@ private static class DummyQuery extends Query { public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + public Matches matches(LeafReaderContext context, int doc) throws IOException { return null; } @@ -949,7 +949,7 @@ private static class BadQuery extends Query { public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + public Matches matches(LeafReaderContext context, int doc) throws IOException { return null; } @@ -1303,7 +1303,7 @@ public void extractTerms(Set terms) { } @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + public Matches matches(LeafReaderContext context, int doc) throws IOException { return null; } @@ -1380,7 +1380,7 @@ private static class DummyQuery2 extends Query { public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + public Matches matches(LeafReaderContext context, int doc) throws IOException { return null; } @@ -1485,7 +1485,7 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo return new ConstantScoreWeight(this, 1) { @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + public Matches matches(LeafReaderContext context, int doc) throws IOException { return null; } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java b/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java index 82919b1a691b..2ab42fc55769 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java @@ -18,6 +18,7 @@ package org.apache.lucene.search; import java.io.IOException; +import java.util.Set; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; @@ -65,6 +66,7 @@ public void setUp() throws Exception { doc.add(newField(FIELD_WITH_OFFSETS, docFields[i], OFFSETS)); doc.add(newField(FIELD_NO_OFFSETS, docFields[i], TextField.TYPE_STORED)); doc.add(new NumericDocValuesField("id", i)); + doc.add(newField("id", Integer.toString(i), TextField.TYPE_STORED)); writer.addDocument(doc); } writer.forceMerge(1); @@ -85,22 +87,27 @@ void checkMatches(Query q, String field, int[][] expected) throws IOException { for (int i = 0; i < expected.length; i++) { LeafReaderContext ctx = searcher.leafContexts.get(ReaderUtil.subIndex(expected[i][0], searcher.leafContexts)); int doc = expected[i][0] - ctx.docBase; - MatchesIterator it = w.matches(ctx, doc, field); - if (it == null) { + Matches matches = w.matches(ctx, doc); + if (matches == null) { assertEquals(expected[i].length, 1); continue; } - int pos = 1; - while (it.next()) { - //System.out.println(expected[i][pos] + "->" + expected[i][pos + 1] + "[" + expected[i][pos + 2] + "->" + expected[i][pos + 3] + "]"); - assertEquals(expected[i][pos], it.startPosition()); - assertEquals(expected[i][pos + 1], it.endPosition()); - assertEquals(expected[i][pos + 2], it.startOffset()); - assertEquals(expected[i][pos + 3], it.endOffset()); - pos += 4; - } - assertEquals(expected[i].length, pos); + MatchesIterator it = matches.getMatches(field); + checkFieldMatches(it, expected[i]); + } + } + + void checkFieldMatches(MatchesIterator it, int[] expected) throws IOException { + int pos = 1; + while (it.next()) { + //System.out.println(expected[i][pos] + "->" + expected[i][pos + 1] + "[" + expected[i][pos + 2] + "->" + expected[i][pos + 3] + "]"); + assertEquals(expected[pos], it.startPosition()); + assertEquals(expected[pos + 1], it.endPosition()); + assertEquals(expected[pos + 2], it.startOffset()); + assertEquals(expected[pos + 3], it.endOffset()); + pos += 4; } + assertEquals(expected.length, pos); } public void testTermQuery() throws IOException { @@ -221,6 +228,26 @@ public void testSynonymQuery() throws IOException { }); } + public void testMultipleFields() throws IOException { + Query q = new BooleanQuery.Builder() + .add(new TermQuery(new Term("id", "1")), BooleanClause.Occur.SHOULD) + .add(new TermQuery(new Term(FIELD_WITH_OFFSETS, "w3")), BooleanClause.Occur.MUST) + .build(); + Weight w = searcher.createNormalizedWeight(q, ScoreMode.COMPLETE); + + LeafReaderContext ctx = searcher.leafContexts.get(ReaderUtil.subIndex(1, searcher.leafContexts)); + Matches m = w.matches(ctx, 1 - ctx.docBase); + assertNotNull(m); + checkFieldMatches(m.getMatches("id"), new int[]{ -1, 0, 0, -1, -1 }); + checkFieldMatches(m.getMatches(FIELD_WITH_OFFSETS), new int[]{ -1, 1, 1, 3, 5, 3, 3, 9, 11 }); + assertNull(m.getMatches("bogus")); + + Set fields = m.getMatchFields(); + assertEquals(2, fields.size()); + assertTrue(fields.contains(FIELD_WITH_OFFSETS)); + assertTrue(fields.contains("id")); + } + protected String[] doc1Fields = { "w1 w2 w3 w4 w5", "w1 w3 w2 w3 zz", diff --git a/lucene/core/src/test/org/apache/lucene/search/TestQueryRescorer.java b/lucene/core/src/test/org/apache/lucene/search/TestQueryRescorer.java index 1e9f8f455f01..26f4d15a6906 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestQueryRescorer.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestQueryRescorer.java @@ -427,7 +427,7 @@ public void extractTerms(Set terms) { } @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + public Matches matches(LeafReaderContext context, int doc) throws IOException { return null; } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestScorerPerf.java b/lucene/core/src/test/org/apache/lucene/search/TestScorerPerf.java index 680ff615c0e1..8469e4954bf4 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestScorerPerf.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestScorerPerf.java @@ -152,7 +152,7 @@ private static class BitSetQuery extends Query { public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + public Matches matches(LeafReaderContext context, int doc) throws IOException { return null; } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java b/lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java index 391c2a0abc59..d97662ab857b 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java @@ -232,7 +232,7 @@ public RandomQuery(long seed, float density, List docValues) { public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + public Matches matches(LeafReaderContext context, int doc) throws IOException { return null; } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestUsageTrackingFilterCachingPolicy.java b/lucene/core/src/test/org/apache/lucene/search/TestUsageTrackingFilterCachingPolicy.java index d01eb951b9c8..242051cffc44 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestUsageTrackingFilterCachingPolicy.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestUsageTrackingFilterCachingPolicy.java @@ -121,7 +121,7 @@ public int hashCode() { public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(DummyQuery.this, boost) { @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + public Matches matches(LeafReaderContext context, int doc) throws IOException { return null; } diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/AssertingMatches.java b/lucene/test-framework/src/java/org/apache/lucene/search/AssertingMatches.java new file mode 100644 index 000000000000..75f75647bdf1 --- /dev/null +++ b/lucene/test-framework/src/java/org/apache/lucene/search/AssertingMatches.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.search; + +import java.util.Set; + +public class AssertingMatches extends Matches { + + private final Matches in; + + public AssertingMatches(Matches matches) { + super(null); + this.in = matches; + } + + @Override + public MatchesIterator getMatches(String field) { + MatchesIterator mi = in.getMatches(field); + if (mi == null) + return null; + return new AssertingMatchesIterator(mi); + } + + @Override + public Set getMatchFields() { + return in.getMatchFields(); + } +} diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/AssertingWeight.java b/lucene/test-framework/src/java/org/apache/lucene/search/AssertingWeight.java index aa8792b6c39b..55fda238ce73 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/search/AssertingWeight.java +++ b/lucene/test-framework/src/java/org/apache/lucene/search/AssertingWeight.java @@ -32,11 +32,11 @@ class AssertingWeight extends FilterWeight { } @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - MatchesIterator matches = in.matches(context, doc, field); + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Matches matches = in.matches(context, doc); if (matches == null) return null; - return new AssertingMatchesIterator(matches); + return new AssertingMatches(matches); } @Override diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/BlockScoreQueryWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/search/BlockScoreQueryWrapper.java index aea4b1f5ae6f..dfe721b681ac 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/search/BlockScoreQueryWrapper.java +++ b/lucene/test-framework/src/java/org/apache/lucene/search/BlockScoreQueryWrapper.java @@ -205,8 +205,8 @@ public void extractTerms(Set terms) { } @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return inWeight.matches(context, doc); } @Override From 5efb7168d037806710e4c6bc61332b38a8345770 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Thu, 29 Mar 2018 11:36:24 +0100 Subject: [PATCH 11/45] Get all modules compiling --- .../lucene/facet/DrillSidewaysQuery.java | 3 ++- .../lucene/facet/range/DoubleRange.java | 3 ++- .../apache/lucene/facet/range/LongRange.java | 3 ++- .../lucene/facet/TestDrillSideways.java | 22 ++---------------- .../search/join/GlobalOrdinalsQuery.java | 9 ++++++-- .../join/ParentChildrenBlockJoinQuery.java | 9 ++++++-- .../join/PointInSetIncludingScoreQuery.java | 9 ++++++-- .../search/join/TermsIncludingScoreQuery.java | 9 ++++++-- .../lucene/search/join/TestJoinUtil.java | 2 +- .../TestDiversifiedTopDocsCollector.java | 8 +++++-- .../queries/function/FunctionMatchQuery.java | 10 ++++++-- .../queries/function/FunctionQuery.java | 9 ++++++-- .../queries/function/FunctionRangeQuery.java | 10 ++++++-- .../queries/function/FunctionScoreQuery.java | 5 ++-- .../document/LatLonDocValuesBoxQuery.java | 10 ++++++-- .../LatLonDocValuesDistanceQuery.java | 9 ++++++-- .../document/LatLonPointDistanceQuery.java | 9 ++++++-- .../document/LatLonPointInPolygonQuery.java | 9 ++++++-- .../apache/lucene/search/CoveringQuery.java | 23 +++++++++++-------- .../lucene/search/DocValuesNumbersQuery.java | 8 +++++-- .../lucene/search/DocValuesTermsQuery.java | 8 +++++-- .../lucene/search/TermAutomatonQuery.java | 8 +++++-- .../lucene/search/TestTermAutomatonQuery.java | 8 +++++-- .../composite/CompositeVerifyQuery.java | 16 +++++++++++-- .../composite/IntersectsRPTVerifyQuery.java | 11 +++++++-- .../prefix/AbstractPrefixTreeQuery.java | 9 ++++++-- .../serialized/SerializedDVStrategy.java | 15 ++++++++---- .../spatial/vector/PointVectorStrategy.java | 16 ++++++++++--- .../spatial3d/PointInGeo3DShapeQuery.java | 9 ++++++-- .../suggest/document/CompletionWeight.java | 3 ++- .../org/apache/solr/ltr/LTRScoringQuery.java | 9 ++++++-- .../org/apache/solr/ltr/feature/Feature.java | 9 ++++++-- .../org/apache/solr/query/SolrRangeQuery.java | 9 ++++++-- .../org/apache/solr/schema/LatLonType.java | 9 ++++++-- .../java/org/apache/solr/search/Filter.java | 9 ++++++-- .../solr/search/GraphTermsQParserPlugin.java | 17 ++++++++++---- .../apache/solr/search/JoinQParserPlugin.java | 9 ++++++-- .../solr/search/SolrConstantScoreQuery.java | 9 ++++++-- .../apache/solr/search/join/GraphQuery.java | 9 ++++++-- .../solr/update/DeleteByQueryWrapper.java | 9 ++++++-- .../uninverting/TestFieldCacheSortRandom.java | 3 ++- 41 files changed, 276 insertions(+), 107 deletions(-) diff --git a/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysQuery.java b/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysQuery.java index f090e0817699..77d9b4e69df6 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysQuery.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysQuery.java @@ -32,6 +32,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; @@ -93,7 +94,7 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo public void extractTerms(Set terms) {} @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + public Matches matches(LeafReaderContext context, int doc) throws IOException { return null; } diff --git a/lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRange.java b/lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRange.java index 901547821e01..9c2eb4889e8d 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRange.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRange.java @@ -27,6 +27,7 @@ import org.apache.lucene.search.DoubleValues; import org.apache.lucene.search.DoubleValuesSource; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; @@ -145,7 +146,7 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo return new ConstantScoreWeight(this, boost) { @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + public Matches matches(LeafReaderContext context, int doc) throws IOException { return null; } diff --git a/lucene/facet/src/java/org/apache/lucene/facet/range/LongRange.java b/lucene/facet/src/java/org/apache/lucene/facet/range/LongRange.java index 74d6c1e168b7..931f799d0da2 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/range/LongRange.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/range/LongRange.java @@ -27,6 +27,7 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.LongValues; import org.apache.lucene.search.LongValuesSource; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; @@ -137,7 +138,7 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo return new ConstantScoreWeight(this, boost) { @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + public Matches matches(LeafReaderContext context, int doc) throws IOException { return null; } diff --git a/lucene/facet/src/test/org/apache/lucene/facet/TestDrillSideways.java b/lucene/facet/src/test/org/apache/lucene/facet/TestDrillSideways.java index 52b0e8b7a78d..653999ea5020 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/TestDrillSideways.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/TestDrillSideways.java @@ -43,26 +43,8 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; -import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.*; import org.apache.lucene.search.BooleanClause.Occur; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.ConstantScoreScorer; -import org.apache.lucene.search.ConstantScoreWeight; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.MatchAllDocsQuery; -import org.apache.lucene.search.MatchesIterator; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.ScoreMode; -import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.SimpleCollector; -import org.apache.lucene.search.Sort; -import org.apache.lucene.search.SortField; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.TopDocs; -import org.apache.lucene.search.TwoPhaseIterator; -import org.apache.lucene.search.Weight; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.IOUtils; @@ -725,7 +707,7 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo return new ConstantScoreWeight(this, boost) { @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + public Matches matches(LeafReaderContext context, int doc) throws IOException { return null; } diff --git a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java index 6d1c68cb1a26..19b5cc269e38 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java +++ b/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java @@ -28,6 +28,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; @@ -112,8 +113,12 @@ final class W extends ConstantScoreWeight { public void extractTerms(Set terms) {} @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.fromField(joinField, MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/lucene/join/src/java/org/apache/lucene/search/join/ParentChildrenBlockJoinQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/ParentChildrenBlockJoinQuery.java index 752ec34e6a5a..88410c8508b9 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/ParentChildrenBlockJoinQuery.java +++ b/lucene/join/src/java/org/apache/lucene/search/join/ParentChildrenBlockJoinQuery.java @@ -27,6 +27,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; @@ -106,8 +107,12 @@ public void extractTerms(Set terms) { } @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.fromField("*", MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/lucene/join/src/java/org/apache/lucene/search/join/PointInSetIncludingScoreQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/PointInSetIncludingScoreQuery.java index 2eaebdd5e4aa..e58b56d98bf6 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/PointInSetIncludingScoreQuery.java +++ b/lucene/join/src/java/org/apache/lucene/search/join/PointInSetIncludingScoreQuery.java @@ -40,6 +40,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.PointInSetQuery; import org.apache.lucene.search.Query; @@ -129,8 +130,12 @@ public void extractTerms(Set terms) { } @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.fromField(field, MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java index efb49d135af3..cda6df5ebf88 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java +++ b/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java @@ -29,6 +29,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; @@ -104,8 +105,12 @@ public Weight createWeight(IndexSearcher searcher, org.apache.lucene.search.Scor public void extractTerms(Set terms) {} @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; // TODO do matches make sense here? + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.fromField(toField, MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java b/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java index 76c63c97b53a..895569726586 100644 --- a/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java +++ b/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java @@ -525,7 +525,7 @@ public void extractTerms(Set terms) { } @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + public Matches matches(LeafReaderContext context, int doc) throws IOException { return null; } diff --git a/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java b/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java index 7aa3d1cacf71..58d5feeee9bf 100644 --- a/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java +++ b/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java @@ -527,8 +527,12 @@ public void extractTerms(Set terms) { } @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.fromField("*", MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionMatchQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionMatchQuery.java index 60937fba1a98..c40e8ea0bbd5 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionMatchQuery.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionMatchQuery.java @@ -28,6 +28,7 @@ import org.apache.lucene.search.DoubleValues; import org.apache.lucene.search.DoubleValuesSource; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; @@ -66,9 +67,14 @@ public String toString(String field) { public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { DoubleValuesSource vs = source.rewrite(searcher); return new ConstantScoreWeight(this, boost) { + @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.fromField(source.toString(), MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java index ec93772e7a31..63689def6e55 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java @@ -26,6 +26,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; @@ -72,8 +73,12 @@ public FunctionWeight(IndexSearcher searcher, float boost) throws IOException { public void extractTerms(Set terms) {} @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.fromField(func.toString(), MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionRangeQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionRangeQuery.java index 4f8593638379..da40738f7e71 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionRangeQuery.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionRangeQuery.java @@ -25,9 +25,11 @@ import org.apache.lucene.index.Term; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; /** @@ -136,8 +138,12 @@ public void extractTerms(Set terms) { } @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.fromField(valueSource.toString(), MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionScoreQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionScoreQuery.java index a190f7f2f7d2..78c0e29542f6 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionScoreQuery.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionScoreQuery.java @@ -29,6 +29,7 @@ import org.apache.lucene.search.Explanation; import org.apache.lucene.search.FilterScorer; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; @@ -150,8 +151,8 @@ public void extractTerms(Set terms) { } @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return inner.matches(context, doc, field); + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return inner.matches(context, doc); } @Override diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesBoxQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesBoxQuery.java index 0013fbdf7b79..8db81cc4b3d3 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesBoxQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesBoxQuery.java @@ -26,6 +26,7 @@ import org.apache.lucene.search.ConstantScoreScorer; import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; @@ -99,9 +100,14 @@ public int hashCode() { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { + @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; // TODO does it make sense to return matches here? + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.fromField(field, MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesDistanceQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesDistanceQuery.java index bde3895b403a..677d56c5df59 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesDistanceQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesDistanceQuery.java @@ -26,6 +26,7 @@ import org.apache.lucene.search.ConstantScoreScorer; import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; @@ -100,8 +101,12 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo private final GeoEncodingUtils.DistancePredicate distancePredicate = GeoEncodingUtils.createDistancePredicate(latitude, longitude, radiusMeters); @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; // TODO does it make sense to try and return matches? + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.fromField(field, MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java index c78aee7e4a7c..d591135268a7 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java @@ -31,6 +31,7 @@ import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; @@ -111,8 +112,12 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo final GeoEncodingUtils.DistancePredicate distancePredicate = GeoEncodingUtils.createDistancePredicate(latitude, longitude, radiusMeters); @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; // TODO does it makes sense to try and return matches here? + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.fromField(field, MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java index d0025154d882..d5fb64ddbdfa 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java @@ -32,6 +32,7 @@ import org.apache.lucene.search.ConstantScoreScorer; import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; @@ -100,8 +101,12 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo return new ConstantScoreWeight(this, boost) { @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; // TODO does it make sense to try and return matches here? + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.fromField(field, MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/CoveringQuery.java b/lucene/sandbox/src/java/org/apache/lucene/search/CoveringQuery.java index cdc9b376803a..8d6836b6105a 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/search/CoveringQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/search/CoveringQuery.java @@ -137,22 +137,25 @@ public void extractTerms(Set terms) { } @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - Scorer scorer = scorer(context); - if (scorer.iterator().advance(doc) != doc) { + public Matches matches(LeafReaderContext context, int doc) throws IOException { + LongValues minMatchValues = minimumNumberMatch.getValues(context, null); + if (minMatchValues.advanceExact(doc) == false) { return null; } - List mis = new ArrayList<>(); - for (Weight w : weights) { - MatchesIterator mi = w.matches(context, doc, field); - if (mi != null) { - mis.add(mi); + final long minimumNumberMatch = Math.max(1, minMatchValues.longValue()); + long matchCount = 0; + List subMatches = new ArrayList<>(); + for (Weight weight : weights) { + Matches matches = weight.matches(context, doc); + if (matches != null) { + matchCount++; + subMatches.add(matches); } } - if (mis.size() == 0) { + if (matchCount < minimumNumberMatch) { return null; } - return new DisjunctionMatchesIterator(mis); + return Matches.fromSubMatches(subMatches); } @Override diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesNumbersQuery.java b/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesNumbersQuery.java index 4a1b5df63632..6efb2c06e26b 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesNumbersQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesNumbersQuery.java @@ -100,8 +100,12 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo return new ConstantScoreWeight(this, boost) { @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; // TODO maybe do something like return the matching docvalue index? + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.fromField(field, MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesTermsQuery.java b/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesTermsQuery.java index 841b16b1b68c..0b620937113a 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesTermsQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesTermsQuery.java @@ -169,8 +169,12 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo return new ConstantScoreWeight(this, boost) { @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; // TODO does it make sense to return matches here? + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.fromField(field, MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java b/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java index 877060b16516..0bf6a2a0ec11 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java @@ -372,8 +372,12 @@ public void extractTerms(Set terms) { } @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; // TODO + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.fromField(field, MatchesIterator.EMPTY); // TODO } @Override diff --git a/lucene/sandbox/src/test/org/apache/lucene/search/TestTermAutomatonQuery.java b/lucene/sandbox/src/test/org/apache/lucene/search/TestTermAutomatonQuery.java index 70752acfdbad..ceccf5e1c5ff 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/search/TestTermAutomatonQuery.java +++ b/lucene/sandbox/src/test/org/apache/lucene/search/TestTermAutomatonQuery.java @@ -585,8 +585,12 @@ public RandomQuery(long seed, float density) { public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.fromField("*", MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/CompositeVerifyQuery.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/CompositeVerifyQuery.java index 33f88200ea39..0ba3a5083263 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/CompositeVerifyQuery.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/CompositeVerifyQuery.java @@ -17,12 +17,15 @@ package org.apache.lucene.spatial.composite; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.ConstantScoreScorer; import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; @@ -88,8 +91,17 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo return new ConstantScoreWeight(this, boost) { @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; // TODO is there a way of reporting matches that makes sense? + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + Matches innerMatches = indexQueryWeight.matches(context, doc); + List subMatches = new ArrayList<>(); + for (String field : innerMatches.getMatchFields()) { + subMatches.add(Matches.fromField(field, MatchesIterator.EMPTY)); + } + return Matches.fromSubMatches(subMatches); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/IntersectsRPTVerifyQuery.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/IntersectsRPTVerifyQuery.java index 2fc8bac3750b..09c3ed750f03 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/IntersectsRPTVerifyQuery.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/IntersectsRPTVerifyQuery.java @@ -24,6 +24,7 @@ import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; @@ -49,12 +50,14 @@ public class IntersectsRPTVerifyQuery extends Query { private final IntersectsDifferentiatingQuery intersectsDiffQuery; private final ShapeValuesPredicate predicateValueSource; + private final String field; public IntersectsRPTVerifyQuery(Shape queryShape, String fieldName, SpatialPrefixTree grid, int detailLevel, int prefixGridScanLevel, ShapeValuesPredicate predicateValueSource) { this.predicateValueSource = predicateValueSource; this.intersectsDiffQuery = new IntersectsDifferentiatingQuery(queryShape, fieldName, grid, detailLevel, prefixGridScanLevel); + this.field = fieldName; } @Override @@ -87,8 +90,12 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo return new ConstantScoreWeight(this, boost) { @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; // TODO is there a way of reporting matches that makes sense? + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.fromField(field, MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/AbstractPrefixTreeQuery.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/AbstractPrefixTreeQuery.java index eabe65efe5a5..b9d6a9c1f082 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/AbstractPrefixTreeQuery.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/AbstractPrefixTreeQuery.java @@ -28,6 +28,7 @@ import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; @@ -81,8 +82,12 @@ public int hashCode() { public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; // TODO is there a way of reporting matches that makes sense? + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.fromField(fieldName, MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java index 2dda25383070..8b67e6f4cebd 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java @@ -33,6 +33,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.DoubleValuesSource; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; @@ -114,7 +115,7 @@ public DoubleValuesSource makeDistanceValueSource(Point queryPoint, double multi public Query makeQuery(SpatialArgs args) { ShapeValuesSource shapeValueSource = makeShapeValueSource(); ShapeValuesPredicate predicateValueSource = new ShapeValuesPredicate(shapeValueSource, args.getOperation(), args.getShape()); - return new PredicateValueSourceQuery(predicateValueSource); + return new PredicateValueSourceQuery(getFieldName(), predicateValueSource); } /** @@ -129,9 +130,11 @@ public ShapeValuesSource makeShapeValueSource() { */ static class PredicateValueSourceQuery extends Query { private final ShapeValuesPredicate predicateValueSource; + private final String field; - public PredicateValueSourceQuery(ShapeValuesPredicate predicateValueSource) { + public PredicateValueSourceQuery(String field, ShapeValuesPredicate predicateValueSource) { this.predicateValueSource = predicateValueSource; + this.field = field; } @Override @@ -139,8 +142,12 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo return new ConstantScoreWeight(this, boost) { @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; // TODO is there a way of reporting matches that makes sense? + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.fromField(field, MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java index bff19c56cbc1..efbdce155350 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java @@ -17,6 +17,8 @@ package org.apache.lucene.spatial.vector; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import java.util.Objects; import org.apache.lucene.document.DoubleDocValuesField; @@ -36,6 +38,7 @@ import org.apache.lucene.search.DoubleValues; import org.apache.lucene.search.DoubleValuesSource; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; @@ -246,7 +249,7 @@ private Query rangeQuery(String fieldName, Double min, Double max) { throw new UnsupportedOperationException("An index is required for this operation."); } - private static class DistanceRangeQuery extends Query { + private class DistanceRangeQuery extends Query { final Query inner; final DoubleValuesSource distanceSource; @@ -272,8 +275,15 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo return new ConstantScoreWeight(this, boost) { @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return w.matches(context, doc, field); + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + List subFieldMatches = new ArrayList<>(); + subFieldMatches.add(Matches.fromField(fieldNameX, MatchesIterator.EMPTY)); + subFieldMatches.add(Matches.fromField(fieldNameY, MatchesIterator.EMPTY)); + return Matches.fromSubMatches(subFieldMatches); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInGeo3DShapeQuery.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInGeo3DShapeQuery.java index 04be8ccf4113..898547d9d425 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInGeo3DShapeQuery.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInGeo3DShapeQuery.java @@ -24,6 +24,7 @@ import org.apache.lucene.search.ConstantScoreScorer; import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; @@ -70,8 +71,12 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo return new ConstantScoreWeight(this, boost) { @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; // TODO is there a way of reporting matches that makes sense here? + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.fromField(field, MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionWeight.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionWeight.java index 225c057ad702..c9b4d84f9f22 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionWeight.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionWeight.java @@ -25,6 +25,7 @@ import org.apache.lucene.index.Terms; import org.apache.lucene.search.BulkScorer; import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; @@ -149,7 +150,7 @@ public void extractTerms(Set terms) { } @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + public Matches matches(LeafReaderContext context, int doc) throws IOException { return null; } diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/LTRScoringQuery.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/LTRScoringQuery.java index 6674885f688d..7ab611448a47 100644 --- a/solr/contrib/ltr/src/java/org/apache/solr/ltr/LTRScoringQuery.java +++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/LTRScoringQuery.java @@ -39,6 +39,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; @@ -452,8 +453,12 @@ public void extractTerms(Set terms) { } @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.fromField("*", MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? } protected void reset() { diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/Feature.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/Feature.java index 9bf4606350e2..67fa36e47d71 100644 --- a/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/Feature.java +++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/Feature.java @@ -26,6 +26,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; @@ -230,8 +231,12 @@ public abstract FeatureScorer scorer(LeafReaderContext context) throws IOException; @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.fromField(name, MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java b/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java index 8b7038472c5e..8c74172098b6 100644 --- a/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java +++ b/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java @@ -38,6 +38,7 @@ import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; @@ -487,8 +488,12 @@ public BulkScorer bulkScorer(LeafReaderContext context) throws IOException { } @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.fromField(field, MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/solr/core/src/java/org/apache/solr/schema/LatLonType.java b/solr/core/src/java/org/apache/solr/schema/LatLonType.java index f095fdbe06d4..07f3daf0479a 100644 --- a/solr/core/src/java/org/apache/solr/schema/LatLonType.java +++ b/solr/core/src/java/org/apache/solr/schema/LatLonType.java @@ -33,6 +33,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; @@ -340,8 +341,12 @@ public boolean isCacheable(LeafReaderContext ctx) { } @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.fromField("*", MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/solr/core/src/java/org/apache/solr/search/Filter.java b/solr/core/src/java/org/apache/solr/search/Filter.java index 7f53d0589b04..a2794eb266d6 100644 --- a/solr/core/src/java/org/apache/solr/search/Filter.java +++ b/solr/core/src/java/org/apache/solr/search/Filter.java @@ -26,6 +26,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; @@ -97,8 +98,12 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo public void extractTerms(Set terms) {} @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.fromField("*", MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/solr/core/src/java/org/apache/solr/search/GraphTermsQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/GraphTermsQParserPlugin.java index 085a835d6bc1..5d984b8a7c9b 100644 --- a/solr/core/src/java/org/apache/solr/search/GraphTermsQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/GraphTermsQParserPlugin.java @@ -49,6 +49,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; @@ -277,8 +278,12 @@ public void extractTerms(Set terms) { } @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.fromField(field, MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? } private WeightOrDocIdSet rewrite(LeafReaderContext context) throws IOException { @@ -620,8 +625,12 @@ public final Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, fl Filter filter; @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.fromField(field, MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java index 661df487040f..a82810a2762c 100644 --- a/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java @@ -35,6 +35,7 @@ import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; @@ -241,8 +242,12 @@ public void close() { @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.fromField("*", MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/solr/core/src/java/org/apache/solr/search/SolrConstantScoreQuery.java b/solr/core/src/java/org/apache/solr/search/SolrConstantScoreQuery.java index 4cbf730221fc..e8a3f2be8394 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrConstantScoreQuery.java +++ b/solr/core/src/java/org/apache/solr/search/SolrConstantScoreQuery.java @@ -27,6 +27,7 @@ import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; @@ -94,8 +95,12 @@ public ConstantWeight(IndexSearcher searcher, float boost) throws IOException { } @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.fromField("*", MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java b/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java index 8777bc690543..9726a39c7745 100644 --- a/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java +++ b/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java @@ -33,6 +33,7 @@ import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; @@ -289,8 +290,12 @@ public void extractTerms(Set terms) { } @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.fromField(collectSchemaField.getName(), MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? } } diff --git a/solr/core/src/java/org/apache/solr/update/DeleteByQueryWrapper.java b/solr/core/src/java/org/apache/solr/update/DeleteByQueryWrapper.java index a8b9a3576185..aba026546328 100644 --- a/solr/core/src/java/org/apache/solr/update/DeleteByQueryWrapper.java +++ b/solr/core/src/java/org/apache/solr/update/DeleteByQueryWrapper.java @@ -26,6 +26,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; @@ -79,8 +80,12 @@ public void extractTerms(Set terms) { } @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { - return null; + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.fromField("*", MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSortRandom.java b/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSortRandom.java index 927cf8ff4b9c..8dd4801521ff 100644 --- a/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSortRandom.java +++ b/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSortRandom.java @@ -43,6 +43,7 @@ import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; @@ -273,7 +274,7 @@ public RandomQuery(long seed, float density, List docValues) { public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { @Override - public MatchesIterator matches(LeafReaderContext context, int doc, String field) throws IOException { + public Matches matches(LeafReaderContext context, int doc) throws IOException { return null; } From 5eb0ac538e195b3924626b0611f36b60d5a4d0dc Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Thu, 29 Mar 2018 11:43:53 +0100 Subject: [PATCH 12/45] javadocs --- .../org/apache/lucene/search/Matches.java | 24 +++++++++++++++++++ .../apache/lucene/search/MatchesIterator.java | 2 +- .../java/org/apache/lucene/search/Weight.java | 7 +++--- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/search/Matches.java b/lucene/core/src/java/org/apache/lucene/search/Matches.java index c5a54bcc97af..9d2d4d450883 100644 --- a/lucene/core/src/java/org/apache/lucene/search/Matches.java +++ b/lucene/core/src/java/org/apache/lucene/search/Matches.java @@ -25,10 +25,21 @@ import java.util.Map; import java.util.Set; +/** + * Reports the positions and optionally offsets of all matching terms in a query + * for a single document + * + * To find all fields that have matches, call {@link #getMatchFields()} + * + * To obtain a {@link MatchesIterator} for a particular field, call {@link #getMatches(String)} + */ public class Matches { private final Map matches; + /** + * Create a simple {@link Matches} for a single field + */ public static Matches fromField(String field, MatchesIterator it) { if (it == null) { return null; @@ -36,6 +47,9 @@ public static Matches fromField(String field, MatchesIterator it) { return new Matches(field, it); } + /** + * Amalgamate a collection of {@link Matches} into a single object + */ public static Matches fromSubMatches(List subMatches) throws IOException { if (subMatches == null || subMatches.size() == 0) { return null; @@ -61,6 +75,9 @@ public static Matches fromSubMatches(List subMatches) throws IOExceptio return new Matches(matches); } + /** + * Create a {@link Matches} from a map of fields to iterators + */ protected Matches(Map matches) { this.matches = matches; } @@ -70,10 +87,17 @@ private Matches(String field, MatchesIterator iterator) { this.matches.put(field, iterator); } + /** + * Returns a {@link MatchesIterator} over the matches for a single field, + * or {@code null} if there are no matches in that field + */ public MatchesIterator getMatches(String field) { return matches.get(field); } + /** + * Returns the fields with matches for this document + */ public Set getMatchFields() { return matches.keySet(); } diff --git a/lucene/core/src/java/org/apache/lucene/search/MatchesIterator.java b/lucene/core/src/java/org/apache/lucene/search/MatchesIterator.java index 7d70f16d91fc..854aa176632e 100644 --- a/lucene/core/src/java/org/apache/lucene/search/MatchesIterator.java +++ b/lucene/core/src/java/org/apache/lucene/search/MatchesIterator.java @@ -28,7 +28,7 @@ * positions and/or offsets after each call. You should not call the position or offset methods * before {@link #next()} has been called, or after {@link #next()} has returned {@code false}. * - * Matches are ordered by start position, and then by end position. Match intervals map overlap. + * Matches are ordered by start position, and then by end position. Match intervals may overlap. * * @see Weight#matches(LeafReaderContext, int) */ diff --git a/lucene/core/src/java/org/apache/lucene/search/Weight.java b/lucene/core/src/java/org/apache/lucene/search/Weight.java index 8c8e5b0c7aa6..b4ae7a552cdf 100644 --- a/lucene/core/src/java/org/apache/lucene/search/Weight.java +++ b/lucene/core/src/java/org/apache/lucene/search/Weight.java @@ -71,11 +71,10 @@ protected Weight(Query query) { public abstract void extractTerms(Set terms); /** - * Returns the {@link Matches} for a specific document + * Returns {@link Matches} for a specific document, or {@code null} if the document + * does not match the parent query * - * If there are no matches, returns {@code null} - * - * @param context the reader's context to create the {@link MatchesIterator} for + * @param context the reader's context to create the {@link Matches} for * @param doc the document's id relative to the given context's reader */ public abstract Matches matches(LeafReaderContext context, int doc) throws IOException; From 992eacaaaf2906790da7dfbef38ab2457890c7e3 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Thu, 29 Mar 2018 11:58:32 +0100 Subject: [PATCH 13/45] clean up --- .../apache/lucene/document/FeatureQuery.java | 6 +-- .../lucene/document/RangeFieldQuery.java | 6 +-- .../SortedNumericDocValuesRangeQuery.java | 6 +-- .../SortedSetDocValuesRangeQuery.java | 6 +-- .../search/DocValuesFieldExistsQuery.java | 6 +-- .../lucene/search/MatchAllDocsQuery.java | 2 +- .../org/apache/lucene/search/Matches.java | 48 +++++++++++++++++++ .../apache/lucene/search/MatchesIterator.java | 27 ----------- .../lucene/search/MultiPhraseQuery.java | 6 +-- .../lucene/search/NormsFieldExistsQuery.java | 6 +-- .../org/apache/lucene/search/PhraseQuery.java | 6 +-- .../apache/lucene/search/PointInSetQuery.java | 6 +-- .../apache/lucene/search/PointRangeQuery.java | 6 +-- .../search/join/GlobalOrdinalsQuery.java | 6 +-- .../join/ParentChildrenBlockJoinQuery.java | 6 +-- .../join/PointInSetIncludingScoreQuery.java | 6 +-- .../search/join/TermsIncludingScoreQuery.java | 6 +-- .../TestDiversifiedTopDocsCollector.java | 6 +-- .../queries/function/FunctionMatchQuery.java | 6 +-- .../queries/function/FunctionQuery.java | 6 +-- .../queries/function/FunctionRangeQuery.java | 6 +-- .../document/LatLonDocValuesBoxQuery.java | 6 +-- .../LatLonDocValuesDistanceQuery.java | 6 +-- .../document/LatLonPointDistanceQuery.java | 6 +-- .../document/LatLonPointInPolygonQuery.java | 6 +-- .../lucene/search/DocValuesNumbersQuery.java | 6 +-- .../lucene/search/DocValuesTermsQuery.java | 6 +-- .../lucene/search/TermAutomatonQuery.java | 6 +-- .../lucene/search/TestTermAutomatonQuery.java | 6 +-- .../composite/CompositeVerifyQuery.java | 2 +- .../composite/IntersectsRPTVerifyQuery.java | 6 +-- .../prefix/AbstractPrefixTreeQuery.java | 6 +-- .../serialized/SerializedDVStrategy.java | 6 +-- .../spatial/vector/PointVectorStrategy.java | 10 +--- .../spatial3d/PointInGeo3DShapeQuery.java | 6 +-- .../org/apache/solr/ltr/LTRScoringQuery.java | 6 +-- .../org/apache/solr/ltr/feature/Feature.java | 6 +-- .../org/apache/solr/query/SolrRangeQuery.java | 6 +-- .../org/apache/solr/schema/LatLonType.java | 6 +-- .../java/org/apache/solr/search/Filter.java | 6 +-- .../solr/search/GraphTermsQParserPlugin.java | 12 +---- .../apache/solr/search/JoinQParserPlugin.java | 6 +-- .../solr/search/SolrConstantScoreQuery.java | 6 +-- .../apache/solr/search/join/GraphQuery.java | 6 +-- .../solr/update/DeleteByQueryWrapper.java | 6 +-- 45 files changed, 93 insertions(+), 242 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/document/FeatureQuery.java b/lucene/core/src/java/org/apache/lucene/document/FeatureQuery.java index 387b365fbfa7..64d83fcef497 100644 --- a/lucene/core/src/java/org/apache/lucene/document/FeatureQuery.java +++ b/lucene/core/src/java/org/apache/lucene/document/FeatureQuery.java @@ -85,11 +85,7 @@ public void extractTerms(Set terms) {} @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer == null || scorer.iterator().advance(doc) != doc) { - return null; - } - return Matches.fromField(fieldName, MatchesIterator.EMPTY); + return Matches.emptyMatches(context, doc, this, fieldName); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java b/lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java index 4d0e109de899..dc51429d1952 100644 --- a/lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java +++ b/lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java @@ -353,11 +353,7 @@ public long cost() { @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer == null || scorer.iterator().advance(doc) != doc) { - return null; - } - return Matches.fromField(field, MatchesIterator.EMPTY); + return Matches.emptyMatches(context, doc, this, field); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java b/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java index 3ba9fd41bb30..b2a898c3d91b 100644 --- a/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java +++ b/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java @@ -105,11 +105,7 @@ public boolean isCacheable(LeafReaderContext ctx) { @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer == null || scorer.iterator().advance(doc) != doc) { - return null; - } - return Matches.fromField(field, MatchesIterator.EMPTY); + return Matches.emptyMatches(context, doc, this, field); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java b/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java index d23d4ca722a5..e20d26ca6fd7 100644 --- a/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java +++ b/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java @@ -110,11 +110,7 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo return new ConstantScoreWeight(this, boost) { @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer == null || scorer.iterator().advance(doc) != doc) { - return null; - } - return Matches.fromField(field, MatchesIterator.EMPTY); + return Matches.emptyMatches(context, doc, this, field); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/search/DocValuesFieldExistsQuery.java b/lucene/core/src/java/org/apache/lucene/search/DocValuesFieldExistsQuery.java index 1febbff301b4..22cb5aab781d 100644 --- a/lucene/core/src/java/org/apache/lucene/search/DocValuesFieldExistsQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/DocValuesFieldExistsQuery.java @@ -66,11 +66,7 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo return new ConstantScoreWeight(this, boost) { @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer == null || scorer.iterator().advance(doc) != doc) { - return null; - } - return Matches.fromField(field, MatchesIterator.EMPTY); + return Matches.emptyMatches(context, doc, this, field); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java b/lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java index 2214c902ce21..a9945c66d812 100644 --- a/lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java @@ -38,7 +38,7 @@ public String toString() { @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.fromField("*", MatchesIterator.EMPTY); + return Matches.emptyMatches(context, doc, this, "*"); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/search/Matches.java b/lucene/core/src/java/org/apache/lucene/search/Matches.java index 9d2d4d450883..2c5cb5a05abf 100644 --- a/lucene/core/src/java/org/apache/lucene/search/Matches.java +++ b/lucene/core/src/java/org/apache/lucene/search/Matches.java @@ -25,6 +25,8 @@ import java.util.Map; import java.util.Set; +import org.apache.lucene.index.LeafReaderContext; + /** * Reports the positions and optionally offsets of all matching terms in a query * for a single document @@ -47,6 +49,25 @@ public static Matches fromField(String field, MatchesIterator it) { return new Matches(field, it); } + /** + * Create an empty {@link Matches} for a Weight + * + * If the Weight's parent query does not match this document, returns {@code null}, + * otherwise returns a {@link Matches} document with an empty iterator on the given + * fields + */ + public static Matches emptyMatches(LeafReaderContext context, int doc, Weight weight, String... fields) throws IOException { + Scorer scorer = weight.scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + List matches = new ArrayList<>(); + for (String field : fields) { + matches.add(Matches.fromField(field, EMPTY)); + } + return Matches.fromSubMatches(matches); + } + /** * Amalgamate a collection of {@link Matches} into a single object */ @@ -101,4 +122,31 @@ public MatchesIterator getMatches(String field) { public Set getMatchFields() { return matches.keySet(); } + + private static final MatchesIterator EMPTY = new MatchesIterator() { + @Override + public boolean next() throws IOException { + return false; + } + + @Override + public int startPosition() { + throw new UnsupportedOperationException(); + } + + @Override + public int endPosition() { + throw new UnsupportedOperationException(); + } + + @Override + public int startOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endOffset() throws IOException { + throw new UnsupportedOperationException(); + } + }; } diff --git a/lucene/core/src/java/org/apache/lucene/search/MatchesIterator.java b/lucene/core/src/java/org/apache/lucene/search/MatchesIterator.java index 854aa176632e..1868a0a384fe 100644 --- a/lucene/core/src/java/org/apache/lucene/search/MatchesIterator.java +++ b/lucene/core/src/java/org/apache/lucene/search/MatchesIterator.java @@ -60,31 +60,4 @@ public interface MatchesIterator { */ int endOffset() throws IOException; - final MatchesIterator EMPTY = new MatchesIterator() { - @Override - public boolean next() throws IOException { - return false; - } - - @Override - public int startPosition() { - throw new UnsupportedOperationException(); - } - - @Override - public int endPosition() { - throw new UnsupportedOperationException(); - } - - @Override - public int startOffset() throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public int endOffset() throws IOException { - throw new UnsupportedOperationException(); - } - }; - } diff --git a/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java b/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java index 5ec597d2b834..31599fa33fb2 100644 --- a/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java @@ -237,11 +237,7 @@ public void extractTerms(Set terms) { @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer == null || scorer.iterator().advance(doc) != doc) { - return null; - } - return Matches.fromField(field, MatchesIterator.EMPTY); // TODO + return Matches.emptyMatches(context, doc, this, field); // TODO } @Override diff --git a/lucene/core/src/java/org/apache/lucene/search/NormsFieldExistsQuery.java b/lucene/core/src/java/org/apache/lucene/search/NormsFieldExistsQuery.java index 85951b7e4469..5ff326d2f4c8 100644 --- a/lucene/core/src/java/org/apache/lucene/search/NormsFieldExistsQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/NormsFieldExistsQuery.java @@ -66,11 +66,7 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo return new ConstantScoreWeight(this, boost) { @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer == null || scorer.iterator().advance(doc) != doc) { - return null; - } - return Matches.fromField(field, MatchesIterator.EMPTY); + return Matches.emptyMatches(context, doc, this, field); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java index e213bda5fbca..5c333e782a8f 100644 --- a/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java @@ -394,11 +394,7 @@ public void extractTerms(Set queryTerms) { @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer == null || scorer.iterator().advance(doc) != doc) { - return null; - } - return Matches.fromField(field, MatchesIterator.EMPTY); // TODO + return Matches.emptyMatches(context, doc, this, field); // TODO } @Override diff --git a/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java b/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java index 16cfbd1dd6ab..7c3814b1c855 100644 --- a/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java @@ -115,11 +115,7 @@ public final Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, fl @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer == null || scorer.iterator().advance(doc) != doc) { - return null; - } - return Matches.fromField(field, MatchesIterator.EMPTY); + return Matches.emptyMatches(context, doc, this, field); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java b/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java index f61146d56bdf..2f789e41ea84 100644 --- a/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java @@ -315,11 +315,7 @@ public long cost() { @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer == null || scorer.iterator().advance(doc) != doc) { - return null; - } - return Matches.fromField(field, MatchesIterator.EMPTY); // TODO can we return values here somehow? + return Matches.emptyMatches(context, doc, this, field); // TODO can we return values here somehow? } @Override diff --git a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java index 19b5cc269e38..a19ed10351ec 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java +++ b/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java @@ -114,11 +114,7 @@ public void extractTerms(Set terms) {} @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer == null || scorer.iterator().advance(doc) != doc) { - return null; - } - return Matches.fromField(joinField, MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? + return Matches.emptyMatches(context, doc, this, joinField); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/lucene/join/src/java/org/apache/lucene/search/join/ParentChildrenBlockJoinQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/ParentChildrenBlockJoinQuery.java index 88410c8508b9..4abf7d9d4ef7 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/ParentChildrenBlockJoinQuery.java +++ b/lucene/join/src/java/org/apache/lucene/search/join/ParentChildrenBlockJoinQuery.java @@ -108,11 +108,7 @@ public void extractTerms(Set terms) { @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer == null || scorer.iterator().advance(doc) != doc) { - return null; - } - return Matches.fromField("*", MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? + return Matches.emptyMatches(context, doc, this, "*"); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/lucene/join/src/java/org/apache/lucene/search/join/PointInSetIncludingScoreQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/PointInSetIncludingScoreQuery.java index e58b56d98bf6..d175d51ab195 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/PointInSetIncludingScoreQuery.java +++ b/lucene/join/src/java/org/apache/lucene/search/join/PointInSetIncludingScoreQuery.java @@ -131,11 +131,7 @@ public void extractTerms(Set terms) { @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer == null || scorer.iterator().advance(doc) != doc) { - return null; - } - return Matches.fromField(field, MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? + return Matches.emptyMatches(context, doc, this, field); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java index cda6df5ebf88..ab5895461aac 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java +++ b/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java @@ -106,11 +106,7 @@ public void extractTerms(Set terms) {} @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer == null || scorer.iterator().advance(doc) != doc) { - return null; - } - return Matches.fromField(toField, MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? + return Matches.emptyMatches(context, doc, this, toField); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java b/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java index 58d5feeee9bf..b22a203819c9 100644 --- a/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java +++ b/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java @@ -528,11 +528,7 @@ public void extractTerms(Set terms) { @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer == null || scorer.iterator().advance(doc) != doc) { - return null; - } - return Matches.fromField("*", MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? + return Matches.emptyMatches(context, doc, this, "*"); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionMatchQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionMatchQuery.java index c40e8ea0bbd5..96bf29dea0c2 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionMatchQuery.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionMatchQuery.java @@ -70,11 +70,7 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer.iterator().advance(doc) != doc) { - return null; - } - return Matches.fromField(source.toString(), MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? + return Matches.emptyMatches(context, doc, this, source.toString()); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java index 63689def6e55..6af7ba3721d6 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java @@ -74,11 +74,7 @@ public void extractTerms(Set terms) {} @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer == null || scorer.iterator().advance(doc) != doc) { - return null; - } - return Matches.fromField(func.toString(), MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? + return Matches.emptyMatches(context, doc, this, func.toString()); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionRangeQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionRangeQuery.java index da40738f7e71..97a50b3c5170 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionRangeQuery.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionRangeQuery.java @@ -139,11 +139,7 @@ public void extractTerms(Set terms) { @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer == null || scorer.iterator().advance(doc) != doc) { - return null; - } - return Matches.fromField(valueSource.toString(), MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? + return Matches.emptyMatches(context, doc, this, valueSource.toString()); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesBoxQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesBoxQuery.java index 8db81cc4b3d3..5fc5b0357122 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesBoxQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesBoxQuery.java @@ -103,11 +103,7 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer == null || scorer.iterator().advance(doc) != doc) { - return null; - } - return Matches.fromField(field, MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? + return Matches.emptyMatches(context, doc, this, field); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesDistanceQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesDistanceQuery.java index 677d56c5df59..5c167dba2a12 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesDistanceQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesDistanceQuery.java @@ -102,11 +102,7 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer == null || scorer.iterator().advance(doc) != doc) { - return null; - } - return Matches.fromField(field, MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? + return Matches.emptyMatches(context, doc, this, field); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java index d591135268a7..81cdeb4bb94b 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java @@ -113,11 +113,7 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer == null || scorer.iterator().advance(doc) != doc) { - return null; - } - return Matches.fromField(field, MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? + return Matches.emptyMatches(context, doc, this, field); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java index d5fb64ddbdfa..e0148ef96ea9 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java @@ -102,11 +102,7 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer == null || scorer.iterator().advance(doc) != doc) { - return null; - } - return Matches.fromField(field, MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? + return Matches.emptyMatches(context, doc, this, field); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesNumbersQuery.java b/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesNumbersQuery.java index 6efb2c06e26b..4f651cdc433f 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesNumbersQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesNumbersQuery.java @@ -101,11 +101,7 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer == null || scorer.iterator().advance(doc) != doc) { - return null; - } - return Matches.fromField(field, MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? + return Matches.emptyMatches(context, doc, this, field); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesTermsQuery.java b/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesTermsQuery.java index 0b620937113a..33fca99af1d1 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesTermsQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesTermsQuery.java @@ -170,11 +170,7 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer == null || scorer.iterator().advance(doc) != doc) { - return null; - } - return Matches.fromField(field, MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? + return Matches.emptyMatches(context, doc, this, field); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java b/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java index 0bf6a2a0ec11..b25c5e1bc792 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java @@ -373,11 +373,7 @@ public void extractTerms(Set terms) { @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer == null || scorer.iterator().advance(doc) != doc) { - return null; - } - return Matches.fromField(field, MatchesIterator.EMPTY); // TODO + return Matches.emptyMatches(context, doc, this, field); // TODO } @Override diff --git a/lucene/sandbox/src/test/org/apache/lucene/search/TestTermAutomatonQuery.java b/lucene/sandbox/src/test/org/apache/lucene/search/TestTermAutomatonQuery.java index ceccf5e1c5ff..7178d2ec337e 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/search/TestTermAutomatonQuery.java +++ b/lucene/sandbox/src/test/org/apache/lucene/search/TestTermAutomatonQuery.java @@ -586,11 +586,7 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo return new ConstantScoreWeight(this, boost) { @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer.iterator().advance(doc) != doc) { - return null; - } - return Matches.fromField("*", MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? + return Matches.emptyMatches(context, doc, this, "*"); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/CompositeVerifyQuery.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/CompositeVerifyQuery.java index 0ba3a5083263..0de3b1c695d7 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/CompositeVerifyQuery.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/CompositeVerifyQuery.java @@ -99,7 +99,7 @@ public Matches matches(LeafReaderContext context, int doc) throws IOException { Matches innerMatches = indexQueryWeight.matches(context, doc); List subMatches = new ArrayList<>(); for (String field : innerMatches.getMatchFields()) { - subMatches.add(Matches.fromField(field, MatchesIterator.EMPTY)); + subMatches.add(Matches.emptyMatches(context, doc, this, field)); } return Matches.fromSubMatches(subMatches); // TODO is there a way of reporting matches that makes sense here? } diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/IntersectsRPTVerifyQuery.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/IntersectsRPTVerifyQuery.java index 09c3ed750f03..3df9bb7281f7 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/IntersectsRPTVerifyQuery.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/IntersectsRPTVerifyQuery.java @@ -91,11 +91,7 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer == null || scorer.iterator().advance(doc) != doc) { - return null; - } - return Matches.fromField(field, MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? + return Matches.emptyMatches(context, doc, this, field); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/AbstractPrefixTreeQuery.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/AbstractPrefixTreeQuery.java index b9d6a9c1f082..e1161021d5eb 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/AbstractPrefixTreeQuery.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/AbstractPrefixTreeQuery.java @@ -83,11 +83,7 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo return new ConstantScoreWeight(this, boost) { @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer == null || scorer.iterator().advance(doc) != doc) { - return null; - } - return Matches.fromField(fieldName, MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? + return Matches.emptyMatches(context, doc, this, fieldName); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java index 8b67e6f4cebd..78476acdd310 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java @@ -143,11 +143,7 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer.iterator().advance(doc) != doc) { - return null; - } - return Matches.fromField(field, MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? + return Matches.emptyMatches(context, doc, this, field); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java index efbdce155350..ee1cc9dc170e 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java @@ -276,14 +276,8 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer == null || scorer.iterator().advance(doc) != doc) { - return null; - } - List subFieldMatches = new ArrayList<>(); - subFieldMatches.add(Matches.fromField(fieldNameX, MatchesIterator.EMPTY)); - subFieldMatches.add(Matches.fromField(fieldNameY, MatchesIterator.EMPTY)); - return Matches.fromSubMatches(subFieldMatches); // TODO is there a way of reporting matches that makes sense here? + return Matches.emptyMatches(context, doc, this, fieldNameX, fieldNameY); + // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInGeo3DShapeQuery.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInGeo3DShapeQuery.java index 898547d9d425..69afa02c1b1a 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInGeo3DShapeQuery.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInGeo3DShapeQuery.java @@ -72,11 +72,7 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer == null || scorer.iterator().advance(doc) != doc) { - return null; - } - return Matches.fromField(field, MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? + return Matches.emptyMatches(context, doc, this, field); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/LTRScoringQuery.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/LTRScoringQuery.java index 7ab611448a47..4645b7aad4c3 100644 --- a/solr/contrib/ltr/src/java/org/apache/solr/ltr/LTRScoringQuery.java +++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/LTRScoringQuery.java @@ -454,11 +454,7 @@ public void extractTerms(Set terms) { @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer == null || scorer.iterator().advance(doc) != doc) { - return null; - } - return Matches.fromField("*", MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? + return Matches.emptyMatches(context, doc, this, "*"); // TODO is there a way of reporting matches that makes sense here? } protected void reset() { diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/Feature.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/Feature.java index 67fa36e47d71..78304dc74b49 100644 --- a/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/Feature.java +++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/Feature.java @@ -232,11 +232,7 @@ public abstract FeatureScorer scorer(LeafReaderContext context) @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer == null || scorer.iterator().advance(doc) != doc) { - return null; - } - return Matches.fromField(name, MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? + return Matches.emptyMatches(context, doc, this, name); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java b/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java index 8c74172098b6..bf97f2acf058 100644 --- a/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java +++ b/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java @@ -489,11 +489,7 @@ public BulkScorer bulkScorer(LeafReaderContext context) throws IOException { @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer == null || scorer.iterator().advance(doc) != doc) { - return null; - } - return Matches.fromField(field, MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? + return Matches.emptyMatches(context, doc, this, field); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/solr/core/src/java/org/apache/solr/schema/LatLonType.java b/solr/core/src/java/org/apache/solr/schema/LatLonType.java index 07f3daf0479a..3152d5a0a19d 100644 --- a/solr/core/src/java/org/apache/solr/schema/LatLonType.java +++ b/solr/core/src/java/org/apache/solr/schema/LatLonType.java @@ -342,11 +342,7 @@ public boolean isCacheable(LeafReaderContext ctx) { @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer == null || scorer.iterator().advance(doc) != doc) { - return null; - } - return Matches.fromField("*", MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? + return Matches.emptyMatches(context, doc, this, "*"); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/solr/core/src/java/org/apache/solr/search/Filter.java b/solr/core/src/java/org/apache/solr/search/Filter.java index a2794eb266d6..8868adb5417f 100644 --- a/solr/core/src/java/org/apache/solr/search/Filter.java +++ b/solr/core/src/java/org/apache/solr/search/Filter.java @@ -99,11 +99,7 @@ public void extractTerms(Set terms) {} @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer == null || scorer.iterator().advance(doc) != doc) { - return null; - } - return Matches.fromField("*", MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? + return Matches.emptyMatches(context, doc, this, "*"); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/solr/core/src/java/org/apache/solr/search/GraphTermsQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/GraphTermsQParserPlugin.java index 5d984b8a7c9b..e5667752662b 100644 --- a/solr/core/src/java/org/apache/solr/search/GraphTermsQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/GraphTermsQParserPlugin.java @@ -279,11 +279,7 @@ public void extractTerms(Set terms) { @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer == null || scorer.iterator().advance(doc) != doc) { - return null; - } - return Matches.fromField(field, MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? + return Matches.emptyMatches(context, doc, this, field); // TODO is there a way of reporting matches that makes sense here? } private WeightOrDocIdSet rewrite(LeafReaderContext context) throws IOException { @@ -626,11 +622,7 @@ public final Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, fl @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer == null || scorer.iterator().advance(doc) != doc) { - return null; - } - return Matches.fromField(field, MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? + return Matches.emptyMatches(context, doc, this, field); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java index a82810a2762c..096999647893 100644 --- a/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java @@ -243,11 +243,7 @@ public void close() { @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer == null || scorer.iterator().advance(doc) != doc) { - return null; - } - return Matches.fromField("*", MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? + return Matches.emptyMatches(context, doc, this, "*"); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/solr/core/src/java/org/apache/solr/search/SolrConstantScoreQuery.java b/solr/core/src/java/org/apache/solr/search/SolrConstantScoreQuery.java index e8a3f2be8394..f4031b505f3c 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrConstantScoreQuery.java +++ b/solr/core/src/java/org/apache/solr/search/SolrConstantScoreQuery.java @@ -96,11 +96,7 @@ public ConstantWeight(IndexSearcher searcher, float boost) throws IOException { @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer == null || scorer.iterator().advance(doc) != doc) { - return null; - } - return Matches.fromField("*", MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? + return Matches.emptyMatches(context, doc, this, "*"); // TODO is there a way of reporting matches that makes sense here? } @Override diff --git a/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java b/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java index 9726a39c7745..4c81f787b725 100644 --- a/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java +++ b/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java @@ -291,11 +291,7 @@ public void extractTerms(Set terms) { @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer == null || scorer.iterator().advance(doc) != doc) { - return null; - } - return Matches.fromField(collectSchemaField.getName(), MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? + return Matches.emptyMatches(context, doc, this, collectSchemaField.getName()); // TODO is there a way of reporting matches that makes sense here? } } diff --git a/solr/core/src/java/org/apache/solr/update/DeleteByQueryWrapper.java b/solr/core/src/java/org/apache/solr/update/DeleteByQueryWrapper.java index aba026546328..9a9c05fdc848 100644 --- a/solr/core/src/java/org/apache/solr/update/DeleteByQueryWrapper.java +++ b/solr/core/src/java/org/apache/solr/update/DeleteByQueryWrapper.java @@ -81,11 +81,7 @@ public void extractTerms(Set terms) { @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer == null || scorer.iterator().advance(doc) != doc) { - return null; - } - return Matches.fromField("*", MatchesIterator.EMPTY); // TODO is there a way of reporting matches that makes sense here? + return Matches.emptyMatches(context, doc, this, "*"); // TODO is there a way of reporting matches that makes sense here? } @Override From bdeb470f17ac7538ddd4246f0c027e9a327f66a0 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 2 Apr 2018 17:19:20 +0100 Subject: [PATCH 14/45] Further WIP --- .../search/matchhighlight/HighlightDoc.java | 2 +- .../matchhighlight/MatchHighlighter.java | 49 ++++++------- ...etGenerator.java => SnippetCollector.java} | 16 ++++- .../matchhighlight/SourceAwareMatches.java | 50 +++++++++++++ .../SourceAwareMatchesIterator.java | 71 +++++++++++++++++++ .../search/matchhighlight/TopHighlights.java | 8 ++- 6 files changed, 166 insertions(+), 30 deletions(-) rename lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/{SnippetGenerator.java => SnippetCollector.java} (71%) create mode 100644 lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SourceAwareMatches.java create mode 100644 lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SourceAwareMatchesIterator.java diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/HighlightDoc.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/HighlightDoc.java index b0dbc5e47569..cf3c6c237ddc 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/HighlightDoc.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/HighlightDoc.java @@ -19,7 +19,7 @@ import org.apache.lucene.document.Document; -public class HighlightDoc { +public class HighlightDoc { public final int doc; diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchHighlighter.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchHighlighter.java index 86f1b8e691cf..16d485b00192 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchHighlighter.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchHighlighter.java @@ -18,16 +18,16 @@ package org.apache.lucene.search.matchhighlight; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; +import java.util.function.Supplier; import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.document.Document; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.index.StoredFieldVisitor; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.MatchesIterator; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.ScoreMode; @@ -44,48 +44,43 @@ public MatchHighlighter(IndexSearcher searcher, Analyzer analyzer) { this.analyzer = analyzer; } - public TopHighlights highlight(Query query, TopDocs docs, SnippetGenerator generator) throws IOException { - List> highlights = new ArrayList<>(); - Weight weight = searcher.createNormalizedWeight(query, ScoreMode.COMPLETE); + public TopHighlights highlight(Query query, TopDocs docs, Supplier collectorSupplier) throws IOException { + HighlightDoc[] highlights = new HighlightDoc[docs.scoreDocs.length]; + Weight weight = searcher.createNormalizedWeight(query, ScoreMode.COMPLETE_NO_SCORES); + int i = 0; for (ScoreDoc doc : docs.scoreDocs) { int contextOrd = ReaderUtil.subIndex(doc.doc, searcher.getIndexReader().leaves()); LeafReaderContext ctx = searcher.getIndexReader().leaves().get(contextOrd); - HighlightingFieldVisitor visitor = new HighlightingFieldVisitor<>(weight, ctx, doc.doc - ctx.docBase, generator); + Matches matches = weight.matches(ctx, doc.doc - ctx.docBase); + HighlightingFieldVisitor visitor = new HighlightingFieldVisitor(new SourceAwareMatches(matches, analyzer), collectorSupplier.get()); ctx.reader().document(doc.doc, visitor); - highlights.add(visitor.getHighights()); + highlights[i++] = new HighlightDoc(doc.doc, visitor.getHighlights()); } - return new TopHighlights<>(highlights); + return new TopHighlights(highlights); } - private class HighlightingFieldVisitor extends StoredFieldVisitor { + private class HighlightingFieldVisitor extends StoredFieldVisitor { - final LeafReaderContext context; - final int doc; - final SnippetGenerator generator; - final Weight weight; + final SourceAwareMatches matches; + final SnippetCollector collector; - final HighlightDoc highlights; + private HighlightingFieldVisitor(SourceAwareMatches matches, SnippetCollector collector) { + this.matches = matches; + this.collector = collector; + } - private HighlightingFieldVisitor(Weight weight, LeafReaderContext context, int doc, SnippetGenerator generator) { - this.context = context; - this.doc = doc; - this.generator = generator; - this.weight = weight; - this.highlights = new HighlightDoc<>(doc, fields); + Document getHighlights() { + return collector.getHighlights(); } @Override public Status needsField(FieldInfo fieldInfo) throws IOException { - return generator.needsField(fieldInfo.name); + return collector.needsField(fieldInfo.name) ? Status.YES : Status.NO; } @Override public void stringField(FieldInfo fieldInfo, byte[] value) throws IOException { - String field = fieldInfo.name; - MatchesIterator matches = weight.matches(context, doc, field); - if (matches != null) { - this.highlights.highlights.put(field, generator.getSnippets(field, value, matches)); - } + collector.collectSnippets(matches, fieldInfo.name, value); } } diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SnippetGenerator.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SnippetCollector.java similarity index 71% rename from lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SnippetGenerator.java rename to lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SnippetCollector.java index c1e15be49839..d4464e8afa13 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SnippetGenerator.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SnippetCollector.java @@ -17,5 +17,19 @@ package org.apache.lucene.search.matchhighlight; -public class SnippetGenerator { +import org.apache.lucene.document.Document; +import org.apache.lucene.index.StoredFieldVisitor; + +public class SnippetCollector { + public Document getHighlights() { + return null; + } + + public boolean needsField(String name) { + return false; + } + + public void collectSnippets(SourceAwareMatches matches, String name, byte[] value) { + + } } diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SourceAwareMatches.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SourceAwareMatches.java new file mode 100644 index 000000000000..dff8f5775298 --- /dev/null +++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SourceAwareMatches.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.search.matchhighlight; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.search.Matches; +import org.apache.lucene.search.MatchesIterator; + +class SourceAwareMatches { + + private final Matches in; + private final Analyzer analyzer; + + private final Map iterators = new HashMap<>(); + + SourceAwareMatches(Matches in, Analyzer analyzer) { + this.in = in; + this.analyzer = analyzer; + } + + MatchesIterator getMatches(FieldInfo fi, byte[] value) { + SourceAwareMatchesIterator it = iterators.computeIfAbsent(fi, fieldinfo -> { + if (fieldinfo.getIndexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) + return SourceAwareMatchesIterator.wrapOffsets(in.getMatches(fieldinfo.name)); + return SourceAwareMatchesIterator.fromTokenStream(in.getMatches(fieldinfo.name), analyzer); + }); + it.addSource(value); + return it; + } +} diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SourceAwareMatchesIterator.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SourceAwareMatchesIterator.java new file mode 100644 index 000000000000..2354dc12d7c2 --- /dev/null +++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SourceAwareMatchesIterator.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.search.matchhighlight; + +import java.io.IOException; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.search.MatchesIterator; + +public interface SourceAwareMatchesIterator extends MatchesIterator { + + void addSource(byte[] source); + + static SourceAwareMatchesIterator wrapOffsets(MatchesIterator in) { + return new SourceAwareMatchesIterator() { + @Override + public void addSource(byte[] source) { + + } + + @Override + public boolean next() throws IOException { + return in.next(); + } + + @Override + public int startPosition() { + return in.startPosition(); + } + + @Override + public int endPosition() { + return in.endPosition(); + } + + @Override + public int startOffset() throws IOException { + return in.startOffset(); + } + + @Override + public int endOffset() throws IOException { + return in.endOffset(); + } + }; + } + + static SourceAwareMatchesIterator fromTokenStream(MatchesIterator in, Analyzer analyzer) { + // build a TokenStream in setSource(), adding count * analyzer.getPositionIncrementGap + // add an OffsetsAttribute + // when in.next() is called, advance the ts until we're at the same position + // return the values from the offset attribute + return null; + } + +} diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/TopHighlights.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/TopHighlights.java index 5329508676cf..5b7e3276d213 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/TopHighlights.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/TopHighlights.java @@ -17,5 +17,11 @@ package org.apache.lucene.search.matchhighlight; -public class TopHighlights { +public class TopHighlights { + + final HighlightDoc[] docs; + + public TopHighlights(HighlightDoc[] docs) { + this.docs = docs; + } } From 3bf1d993ae11b23ee313736e2b0989af40a4fde1 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 2 Apr 2018 17:21:55 +0100 Subject: [PATCH 15/45] IndexOrDocValuesQuery should use the dvQuery for matches --- .../java/org/apache/lucene/search/IndexOrDocValuesQuery.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/search/IndexOrDocValuesQuery.java b/lucene/core/src/java/org/apache/lucene/search/IndexOrDocValuesQuery.java index 1a236db673ba..d69421ec4d6d 100644 --- a/lucene/core/src/java/org/apache/lucene/search/IndexOrDocValuesQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/IndexOrDocValuesQuery.java @@ -121,8 +121,8 @@ public void extractTerms(Set terms) { @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - // TODO we can probably make use of the doc values here to return something more useful - return indexWeight.matches(context, doc); + // We need to check a single doc, so the dv query should perform better + return dvWeight.matches(context, doc); } @Override From f1e690104c949e4d725f8eacc51ed76a82a46c11 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 2 Apr 2018 18:02:53 +0100 Subject: [PATCH 16/45] Add term() to iterator; remove Span iterators for now --- .../search/DisjunctionMatchesIterator.java | 7 ++- .../org/apache/lucene/search/Matches.java | 9 +++ .../apache/lucene/search/MatchesIterator.java | 6 ++ .../lucene/search/TermMatchesIterator.java | 11 +++- .../org/apache/lucene/search/TermQuery.java | 2 +- .../lucene/search/spans/SpanWeight.java | 56 +------------------ .../search/AssertingMatchesIterator.java | 8 +++ 7 files changed, 42 insertions(+), 57 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java b/lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java index 6a506c6fc202..17ab54ef8328 100644 --- a/lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java +++ b/lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java @@ -61,7 +61,7 @@ public static DisjunctionMatchesIterator fromTermsEnum(LeafReaderContext context if (te.seekExact(term)) { PostingsEnum pe = te.postings(reuse, PostingsEnum.OFFSETS); if (pe.advance(doc) == doc) { - mis.add(new TermMatchesIterator(pe)); + mis.add(new TermMatchesIterator(term, pe)); reuse = null; } else { @@ -135,4 +135,9 @@ public int endOffset() throws IOException { return queue.top().endOffset(); } + @Override + public BytesRef term() { + return queue.top().term(); + } + } diff --git a/lucene/core/src/java/org/apache/lucene/search/Matches.java b/lucene/core/src/java/org/apache/lucene/search/Matches.java index 2c5cb5a05abf..780ec8ebab5f 100644 --- a/lucene/core/src/java/org/apache/lucene/search/Matches.java +++ b/lucene/core/src/java/org/apache/lucene/search/Matches.java @@ -26,6 +26,7 @@ import java.util.Set; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.util.BytesRef; /** * Reports the positions and optionally offsets of all matching terms in a query @@ -123,7 +124,10 @@ public Set getMatchFields() { return matches.keySet(); } + private static final BytesRef EMPTY_BYTES = new BytesRef(); + private static final MatchesIterator EMPTY = new MatchesIterator() { + @Override public boolean next() throws IOException { return false; @@ -148,5 +152,10 @@ public int startOffset() throws IOException { public int endOffset() throws IOException { throw new UnsupportedOperationException(); } + + @Override + public BytesRef term() { + return EMPTY_BYTES; + } }; } diff --git a/lucene/core/src/java/org/apache/lucene/search/MatchesIterator.java b/lucene/core/src/java/org/apache/lucene/search/MatchesIterator.java index 1868a0a384fe..abca34eb3ccb 100644 --- a/lucene/core/src/java/org/apache/lucene/search/MatchesIterator.java +++ b/lucene/core/src/java/org/apache/lucene/search/MatchesIterator.java @@ -20,6 +20,7 @@ import java.io.IOException; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.util.BytesRef; /** * An iterator over match positions (and optionally offsets) for a single document and field @@ -60,4 +61,9 @@ public interface MatchesIterator { */ int endOffset() throws IOException; + /** + * The underlying term of the current match + */ + BytesRef term(); + } diff --git a/lucene/core/src/java/org/apache/lucene/search/TermMatchesIterator.java b/lucene/core/src/java/org/apache/lucene/search/TermMatchesIterator.java index e3c695fb04bf..03dba14b3a5d 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TermMatchesIterator.java +++ b/lucene/core/src/java/org/apache/lucene/search/TermMatchesIterator.java @@ -20,16 +20,20 @@ import java.io.IOException; import org.apache.lucene.index.PostingsEnum; +import org.apache.lucene.index.Term; +import org.apache.lucene.util.BytesRef; public class TermMatchesIterator implements MatchesIterator { private int upto; private int pos; private final PostingsEnum pe; + private final BytesRef term; - public TermMatchesIterator(PostingsEnum pe) throws IOException { + public TermMatchesIterator(BytesRef term, PostingsEnum pe) throws IOException { this.pe = pe; this.upto = pe.freq(); + this.term = term; } @Override @@ -60,4 +64,9 @@ public int startOffset() throws IOException { public int endOffset() throws IOException { return pe.endOffset(); } + + @Override + public BytesRef term() { + return term; + } } diff --git a/lucene/core/src/java/org/apache/lucene/search/TermQuery.java b/lucene/core/src/java/org/apache/lucene/search/TermQuery.java index beb7946ae6b9..0928aca9f7e9 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TermQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/TermQuery.java @@ -91,7 +91,7 @@ public Matches matches(LeafReaderContext context, int doc) throws IOException { if (pe.advance(doc) != doc) { return null; } - return Matches.fromField(term.field(), new TermMatchesIterator(pe)); + return Matches.fromField(term.field(), new TermMatchesIterator(term.bytes(), pe)); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java b/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java index 323a053d9170..07c6a353c0cb 100644 --- a/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java +++ b/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java @@ -166,59 +166,7 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - Spans spans = getSpans(context, Postings.OFFSETS); - if (spans.advance(doc) != doc) - return null; - - MatchesIterator mi = new MatchesIterator() { - - SpanCollector offsetCollector = new SpanCollector() { - @Override - public void collectLeaf(PostingsEnum postings, int position, Term term) throws IOException { - startOffset = Math.min(startOffset, postings.startOffset()); - endOffset = Math.max(endOffset, postings.endOffset()); - } - - @Override - public void reset() { - startOffset = Integer.MAX_VALUE; - endOffset = Integer.MIN_VALUE; - } - }; - - int startOffset = -1; - int endOffset = -1; - - @Override - public boolean next() throws IOException { - int next = spans.nextStartPosition(); - if (next == Spans.NO_MORE_POSITIONS) - return false; - spans.collect(offsetCollector); - return true; - } - - @Override - public int startPosition() { - return spans.startPosition(); - } - - @Override - public int endPosition() { - return spans.endPosition(); - } - - @Override - public int startOffset() { - return startOffset; - } - - @Override - public int endOffset() { - return endOffset; - } - }; - - return Matches.fromField(field, mi); + // TODO: Composite matches + return Matches.emptyMatches(context, doc, this, field); } } diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/AssertingMatchesIterator.java b/lucene/test-framework/src/java/org/apache/lucene/search/AssertingMatchesIterator.java index 8eb73d6e0923..7617a33c32c1 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/search/AssertingMatchesIterator.java +++ b/lucene/test-framework/src/java/org/apache/lucene/search/AssertingMatchesIterator.java @@ -19,6 +19,8 @@ import java.io.IOException; +import org.apache.lucene.util.BytesRef; + public class AssertingMatchesIterator implements MatchesIterator { private final MatchesIterator in; @@ -66,4 +68,10 @@ public int endOffset() throws IOException { assert state == State.ITERATING : state; return in.endOffset(); } + + @Override + public BytesRef term() { + assert state == State.ITERATING : state; + return in.term(); + } } From d48c33114c0b183895c0a42190e40289379e7ed9 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 2 Apr 2018 18:24:10 +0100 Subject: [PATCH 17/45] Add some tests for MatchesIterator.term() --- .../search/DisjunctionMatchesIterator.java | 3 +- .../lucene/search/TestMatchesIterator.java | 43 ++++++++++++++++++- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java b/lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java index 17ab54ef8328..8e039a158195 100644 --- a/lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java +++ b/lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java @@ -61,7 +61,8 @@ public static DisjunctionMatchesIterator fromTermsEnum(LeafReaderContext context if (te.seekExact(term)) { PostingsEnum pe = te.postings(reuse, PostingsEnum.OFFSETS); if (pe.advance(doc) == doc) { - mis.add(new TermMatchesIterator(term, pe)); + // TODO do we want to use the copied term here, or instead create a label that associates all of the TMIs with a single term? + mis.add(new TermMatchesIterator(BytesRef.deepCopyOf(term), pe)); reuse = null; } else { diff --git a/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java b/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java index 2ab42fc55769..9b74b590c3d2 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java @@ -83,7 +83,7 @@ public void setUp() throws Exception { }; void checkMatches(Query q, String field, int[][] expected) throws IOException { - Weight w = searcher.createNormalizedWeight(q, ScoreMode.COMPLETE); + Weight w = searcher.createNormalizedWeight(q, ScoreMode.COMPLETE_NO_SCORES); for (int i = 0; i < expected.length; i++) { LeafReaderContext ctx = searcher.leafContexts.get(ReaderUtil.subIndex(expected[i][0], searcher.leafContexts)); int doc = expected[i][0] - ctx.docBase; @@ -110,6 +110,26 @@ void checkFieldMatches(MatchesIterator it, int[] expected) throws IOException { assertEquals(expected.length, pos); } + void checkTerms(Query q, String field, String[][] expected) throws IOException { + Weight w = searcher.createNormalizedWeight(q, ScoreMode.COMPLETE_NO_SCORES); + for (int i = 0; i < expected.length; i++) { + LeafReaderContext ctx = searcher.leafContexts.get(ReaderUtil.subIndex(i, searcher.leafContexts)); + int doc = i - ctx.docBase; + Matches matches = w.matches(ctx, doc); + if (matches == null) { + assertEquals(expected[i].length, 0); + continue; + } + MatchesIterator it = matches.getMatches(field); + int pos = 0; + while (it.next()) { + assertEquals(expected[i][pos], it.term().utf8ToString()); + pos += 1; + } + assertEquals(expected[i].length, pos); + } + } + public void testTermQuery() throws IOException { Query q = new TermQuery(new Term(FIELD_WITH_OFFSETS, "w1")); checkMatches(q, FIELD_WITH_OFFSETS, new int[][]{ @@ -128,6 +148,12 @@ public void testTermQueryNoStoredOffsets() throws IOException { { 2, 0, 0, -1, -1 }, { 3, 0, 0, -1, -1, 2, 2, -1, -1 } }); + checkTerms(q, FIELD_NO_OFFSETS, new String[][]{ + { "w1" }, + { "w1" }, + { "w1" }, + { "w1", "w1" }, + }); } public void testDisjunction() throws IOException { @@ -141,6 +167,12 @@ public void testDisjunction() throws IOException { { 2, 0, 0, 0, 2 }, { 3, 0, 0, 0, 2, 2, 2, 6, 8, 5, 5, 15, 17 } }); + checkTerms(q, FIELD_WITH_OFFSETS, new String[][]{ + { "w1", "w3" }, + { "w1", "w3", "w3" }, + { "w1" }, + { "w1", "w1", "w3" } + }); } public void testReqOpt() throws IOException { @@ -172,6 +204,12 @@ public void testMinShouldMatch() throws IOException { { 2, 0, 0, 0, 2, 1, 1, 3, 5, 4, 4, 12, 14 }, { 3, 0, 0, 0, 2, 2, 2, 6, 8, 3, 3, 9, 11, 5, 5, 15, 17 } }); + checkTerms(q, FIELD_WITH_OFFSETS, new String[][]{ + { "w1", "w3", "w4" }, + { "w3", "w3" }, + { "w1", "xx", "w4" }, + { "w1", "w1", "w4", "w3" } + }); } public void testExclusion() throws IOException { @@ -208,6 +246,9 @@ public void testWildcards() throws IOException { { 2, 1, 1, 3, 5 }, { 0 } }); + checkTerms(q, FIELD_WITH_OFFSETS, new String[][]{ + {}, {}, { "xx" }, {} + }); Query rq = new RegexpQuery(new Term(FIELD_WITH_OFFSETS, "w[1-2]")); checkMatches(rq, FIELD_WITH_OFFSETS, new int[][]{ From 330398e53b08c4d0c902629c41cdad38f33484d8 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 4 Apr 2018 11:42:08 +0100 Subject: [PATCH 18/45] javadocs, precommit --- .../apache/lucene/document/FeatureQuery.java | 1 - .../lucene/document/RangeFieldQuery.java | 1 - .../SortedNumericDocValuesRangeQuery.java | 1 - .../SortedSetDocValuesRangeQuery.java | 1 - .../apache/lucene/search/BooleanWeight.java | 1 - .../search/DisjunctionMatchesIterator.java | 22 +++++++++++++++---- .../apache/lucene/search/SynonymQuery.java | 4 +--- .../lucene/search/TermMatchesIterator.java | 11 +++++++--- .../java/org/apache/lucene/search/Weight.java | 1 - .../lucene/search/spans/SpanWeight.java | 1 - .../lucene/facet/DrillSidewaysQuery.java | 1 - .../lucene/facet/range/DoubleRange.java | 1 - .../apache/lucene/facet/range/LongRange.java | 1 - .../search/join/GlobalOrdinalsQuery.java | 1 - .../join/ParentChildrenBlockJoinQuery.java | 1 - .../join/PointInSetIncludingScoreQuery.java | 1 - .../search/join/TermsIncludingScoreQuery.java | 1 - .../queries/function/FunctionMatchQuery.java | 1 - .../queries/function/FunctionQuery.java | 1 - .../queries/function/FunctionRangeQuery.java | 2 -- .../queries/function/FunctionScoreQuery.java | 1 - .../document/LatLonDocValuesBoxQuery.java | 1 - .../LatLonDocValuesDistanceQuery.java | 1 - .../document/LatLonPointDistanceQuery.java | 1 - .../document/LatLonPointInPolygonQuery.java | 1 - .../composite/CompositeVerifyQuery.java | 1 - .../composite/IntersectsRPTVerifyQuery.java | 1 - .../prefix/AbstractPrefixTreeQuery.java | 1 - .../serialized/SerializedDVStrategy.java | 1 - .../spatial/vector/PointVectorStrategy.java | 3 --- .../spatial3d/PointInGeo3DShapeQuery.java | 1 - .../suggest/document/CompletionWeight.java | 1 - .../lucene/search/AssertingMatches.java | 4 ++-- .../search/AssertingMatchesIterator.java | 4 ++-- .../org/apache/solr/ltr/LTRScoringQuery.java | 1 - .../org/apache/solr/ltr/feature/Feature.java | 1 - .../org/apache/solr/query/SolrRangeQuery.java | 3 +-- .../org/apache/solr/schema/LatLonType.java | 2 +- .../java/org/apache/solr/search/Filter.java | 1 - .../solr/search/GraphTermsQParserPlugin.java | 3 +-- .../apache/solr/search/JoinQParserPlugin.java | 1 - .../solr/search/SolrConstantScoreQuery.java | 1 - .../apache/solr/search/join/GraphQuery.java | 1 - .../solr/update/DeleteByQueryWrapper.java | 1 - .../uninverting/TestFieldCacheSortRandom.java | 1 - 45 files changed, 34 insertions(+), 59 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/document/FeatureQuery.java b/lucene/core/src/java/org/apache/lucene/document/FeatureQuery.java index 64d83fcef497..d74fd06a0da8 100644 --- a/lucene/core/src/java/org/apache/lucene/document/FeatureQuery.java +++ b/lucene/core/src/java/org/apache/lucene/document/FeatureQuery.java @@ -31,7 +31,6 @@ import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Matches; -import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; diff --git a/lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java b/lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java index dc51429d1952..7e9bd72fa0e7 100644 --- a/lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java +++ b/lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java @@ -31,7 +31,6 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Matches; -import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.ScorerSupplier; diff --git a/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java b/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java index b2a898c3d91b..430edb4aed4c 100644 --- a/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java +++ b/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java @@ -30,7 +30,6 @@ import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Matches; -import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; diff --git a/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java b/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java index e20d26ca6fd7..eba1e729f57a 100644 --- a/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java +++ b/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java @@ -30,7 +30,6 @@ import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Matches; -import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; diff --git a/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java b/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java index dd320e78d0af..1fddd7c3370d 100644 --- a/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java +++ b/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java @@ -31,7 +31,6 @@ import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.util.Bits; -import org.apache.lucene.util.PriorityQueue; /** * Expert: the Weight for BooleanQuery, used to diff --git a/lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java b/lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java index 8e039a158195..db064c411797 100644 --- a/lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java +++ b/lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java @@ -23,17 +23,26 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.PostingsEnum; -import org.apache.lucene.index.PrefixCodedTerms; import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.search.spans.SpanWeight; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefIterator; import org.apache.lucene.util.PriorityQueue; -class DisjunctionMatchesIterator implements MatchesIterator { +/** + * A {@link MatchesIterator} that combines matches from a set of sub-iterators + * + * Matches are sorted by their start positions, and then by their end positions, so that + * prefixes sort first. Matches may overlap. + */ +public class DisjunctionMatchesIterator implements MatchesIterator { + /** + * Create a {@link DisjunctionMatchesIterator} over a list of terms + * + * Only terms that have at least one match in the given document will be included + */ public static DisjunctionMatchesIterator fromTerms(LeafReaderContext context, int doc, String field, List terms) throws IOException { return fromTermsEnum(context, doc, field, asBytesRefIterator(terms)); } @@ -50,6 +59,11 @@ public BytesRef next() { }; } + /** + * Create a {@link DisjunctionMatchesIterator} over a list of terms extracted from a {@link BytesRefIterator} + * + * Only terms that have at least one match in the given document will be included + */ public static DisjunctionMatchesIterator fromTermsEnum(LeafReaderContext context, int doc, String field, BytesRefIterator terms) throws IOException { List mis = new ArrayList<>(); Terms t = context.reader().terms(field); @@ -75,7 +89,7 @@ public static DisjunctionMatchesIterator fromTermsEnum(LeafReaderContext context return new DisjunctionMatchesIterator(mis); } - public static MatchesIterator fromSubIterators(List mis) throws IOException { + static MatchesIterator fromSubIterators(List mis) throws IOException { if (mis.size() == 0) return null; if (mis.size() == 1) diff --git a/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java b/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java index 7dd3067514a2..00daec5d7bcf 100644 --- a/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java @@ -29,11 +29,9 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermStates; import org.apache.lucene.index.TermState; -import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermStates; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.util.BytesRef; diff --git a/lucene/core/src/java/org/apache/lucene/search/TermMatchesIterator.java b/lucene/core/src/java/org/apache/lucene/search/TermMatchesIterator.java index 03dba14b3a5d..051699651dbe 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TermMatchesIterator.java +++ b/lucene/core/src/java/org/apache/lucene/search/TermMatchesIterator.java @@ -20,17 +20,22 @@ import java.io.IOException; import org.apache.lucene.index.PostingsEnum; -import org.apache.lucene.index.Term; import org.apache.lucene.util.BytesRef; -public class TermMatchesIterator implements MatchesIterator { +/** + * A {@link MatchesIterator} over a single term's postings list + */ +class TermMatchesIterator implements MatchesIterator { private int upto; private int pos; private final PostingsEnum pe; private final BytesRef term; - public TermMatchesIterator(BytesRef term, PostingsEnum pe) throws IOException { + /** + * Create a new {@link TermMatchesIterator} for the given term and postings list + */ + TermMatchesIterator(BytesRef term, PostingsEnum pe) throws IOException { this.pe = pe; this.upto = pe.freq(); this.term = term; diff --git a/lucene/core/src/java/org/apache/lucene/search/Weight.java b/lucene/core/src/java/org/apache/lucene/search/Weight.java index b4ae7a552cdf..6587511d1cb4 100644 --- a/lucene/core/src/java/org/apache/lucene/search/Weight.java +++ b/lucene/core/src/java/org/apache/lucene/search/Weight.java @@ -20,7 +20,6 @@ import java.io.IOException; import java.util.Set; -import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.index.IndexReaderContext; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java b/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java index 07c6a353c0cb..f93f9a1e40de 100644 --- a/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java +++ b/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java @@ -30,7 +30,6 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.LeafSimScorer; import org.apache.lucene.search.Matches; -import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.TermStatistics; import org.apache.lucene.search.Weight; import org.apache.lucene.search.similarities.Similarity; diff --git a/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysQuery.java b/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysQuery.java index 77d9b4e69df6..6383b7fd560b 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysQuery.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysQuery.java @@ -33,7 +33,6 @@ import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Matches; -import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; diff --git a/lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRange.java b/lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRange.java index 9c2eb4889e8d..f92ccf207953 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRange.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRange.java @@ -28,7 +28,6 @@ import org.apache.lucene.search.DoubleValuesSource; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Matches; -import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; diff --git a/lucene/facet/src/java/org/apache/lucene/facet/range/LongRange.java b/lucene/facet/src/java/org/apache/lucene/facet/range/LongRange.java index 931f799d0da2..8d53ef16ed68 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/range/LongRange.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/range/LongRange.java @@ -28,7 +28,6 @@ import org.apache.lucene.search.LongValues; import org.apache.lucene.search.LongValuesSource; import org.apache.lucene.search.Matches; -import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; diff --git a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java index a19ed10351ec..5917affb1065 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java +++ b/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java @@ -29,7 +29,6 @@ import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Matches; -import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.TwoPhaseIterator; diff --git a/lucene/join/src/java/org/apache/lucene/search/join/ParentChildrenBlockJoinQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/ParentChildrenBlockJoinQuery.java index 4abf7d9d4ef7..16f94d243367 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/ParentChildrenBlockJoinQuery.java +++ b/lucene/join/src/java/org/apache/lucene/search/join/ParentChildrenBlockJoinQuery.java @@ -28,7 +28,6 @@ import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Matches; -import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; diff --git a/lucene/join/src/java/org/apache/lucene/search/join/PointInSetIncludingScoreQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/PointInSetIncludingScoreQuery.java index d175d51ab195..ec2d66dd1adb 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/PointInSetIncludingScoreQuery.java +++ b/lucene/join/src/java/org/apache/lucene/search/join/PointInSetIncludingScoreQuery.java @@ -41,7 +41,6 @@ import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Matches; -import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.PointInSetQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; diff --git a/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java index ab5895461aac..24a205c9f9f6 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java +++ b/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java @@ -30,7 +30,6 @@ import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Matches; -import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionMatchQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionMatchQuery.java index 96bf29dea0c2..e721db623339 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionMatchQuery.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionMatchQuery.java @@ -29,7 +29,6 @@ import org.apache.lucene.search.DoubleValuesSource; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Matches; -import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java index 6af7ba3721d6..78c9da0055a5 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java @@ -27,7 +27,6 @@ import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Matches; -import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionRangeQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionRangeQuery.java index 97a50b3c5170..72d12e0bf50f 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionRangeQuery.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionRangeQuery.java @@ -26,10 +26,8 @@ import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Matches; -import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; -import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; /** diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionScoreQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionScoreQuery.java index 78c0e29542f6..de0e6d409f7f 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionScoreQuery.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionScoreQuery.java @@ -30,7 +30,6 @@ import org.apache.lucene.search.FilterScorer; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Matches; -import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesBoxQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesBoxQuery.java index 5fc5b0357122..e1d7c1778730 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesBoxQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesBoxQuery.java @@ -27,7 +27,6 @@ import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Matches; -import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesDistanceQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesDistanceQuery.java index 5c167dba2a12..b5cd2e6b88d8 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesDistanceQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesDistanceQuery.java @@ -27,7 +27,6 @@ import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Matches; -import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java index 81cdeb4bb94b..1c9e3787177f 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java @@ -32,7 +32,6 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Matches; -import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java index e0148ef96ea9..10643f246e79 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java @@ -33,7 +33,6 @@ import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Matches; -import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/CompositeVerifyQuery.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/CompositeVerifyQuery.java index 0de3b1c695d7..44a6485eb473 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/CompositeVerifyQuery.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/CompositeVerifyQuery.java @@ -26,7 +26,6 @@ import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Matches; -import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/IntersectsRPTVerifyQuery.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/IntersectsRPTVerifyQuery.java index 3df9bb7281f7..4d004980b109 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/IntersectsRPTVerifyQuery.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/IntersectsRPTVerifyQuery.java @@ -25,7 +25,6 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Matches; -import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/AbstractPrefixTreeQuery.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/AbstractPrefixTreeQuery.java index e1161021d5eb..5b81c4082c03 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/AbstractPrefixTreeQuery.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/AbstractPrefixTreeQuery.java @@ -29,7 +29,6 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Matches; -import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java index 78476acdd310..92abf9f6b82d 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java @@ -34,7 +34,6 @@ import org.apache.lucene.search.DoubleValuesSource; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Matches; -import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java index ee1cc9dc170e..36b8e6bf21b7 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java @@ -17,8 +17,6 @@ package org.apache.lucene.spatial.vector; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; import java.util.Objects; import org.apache.lucene.document.DoubleDocValuesField; @@ -39,7 +37,6 @@ import org.apache.lucene.search.DoubleValuesSource; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Matches; -import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInGeo3DShapeQuery.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInGeo3DShapeQuery.java index 69afa02c1b1a..d8e0644ebd12 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInGeo3DShapeQuery.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInGeo3DShapeQuery.java @@ -25,7 +25,6 @@ import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Matches; -import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionWeight.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionWeight.java index c9b4d84f9f22..b3bfd2edaa22 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionWeight.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionWeight.java @@ -26,7 +26,6 @@ import org.apache.lucene.search.BulkScorer; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.Matches; -import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; import org.apache.lucene.search.suggest.BitsProducer; diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/AssertingMatches.java b/lucene/test-framework/src/java/org/apache/lucene/search/AssertingMatches.java index 75f75647bdf1..50f49c758ff6 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/search/AssertingMatches.java +++ b/lucene/test-framework/src/java/org/apache/lucene/search/AssertingMatches.java @@ -19,11 +19,11 @@ import java.util.Set; -public class AssertingMatches extends Matches { +class AssertingMatches extends Matches { private final Matches in; - public AssertingMatches(Matches matches) { + AssertingMatches(Matches matches) { super(null); this.in = matches; } diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/AssertingMatchesIterator.java b/lucene/test-framework/src/java/org/apache/lucene/search/AssertingMatchesIterator.java index 7617a33c32c1..52fb184250c6 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/search/AssertingMatchesIterator.java +++ b/lucene/test-framework/src/java/org/apache/lucene/search/AssertingMatchesIterator.java @@ -21,14 +21,14 @@ import org.apache.lucene.util.BytesRef; -public class AssertingMatchesIterator implements MatchesIterator { +class AssertingMatchesIterator implements MatchesIterator { private final MatchesIterator in; private State state = State.UNPOSITIONED; private enum State { UNPOSITIONED, ITERATING, EXHAUSTED } - public AssertingMatchesIterator(MatchesIterator in) { + AssertingMatchesIterator(MatchesIterator in) { this.in = in; } diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/LTRScoringQuery.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/LTRScoringQuery.java index 4645b7aad4c3..779c8eb9aad2 100644 --- a/solr/contrib/ltr/src/java/org/apache/solr/ltr/LTRScoringQuery.java +++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/LTRScoringQuery.java @@ -40,7 +40,6 @@ import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Matches; -import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/Feature.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/Feature.java index 78304dc74b49..12232986aa78 100644 --- a/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/Feature.java +++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/Feature.java @@ -27,7 +27,6 @@ import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Matches; -import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; diff --git a/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java b/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java index bf97f2acf058..8676743505aa 100644 --- a/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java +++ b/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java @@ -25,8 +25,8 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermStates; import org.apache.lucene.index.TermState; +import org.apache.lucene.index.TermStates; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.BooleanClause; @@ -39,7 +39,6 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Matches; -import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; diff --git a/solr/core/src/java/org/apache/solr/schema/LatLonType.java b/solr/core/src/java/org/apache/solr/schema/LatLonType.java index 3152d5a0a19d..336c06e495ba 100644 --- a/solr/core/src/java/org/apache/solr/schema/LatLonType.java +++ b/solr/core/src/java/org/apache/solr/schema/LatLonType.java @@ -15,6 +15,7 @@ * limitations under the License. */ package org.apache.solr.schema; + import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -34,7 +35,6 @@ import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Matches; -import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; diff --git a/solr/core/src/java/org/apache/solr/search/Filter.java b/solr/core/src/java/org/apache/solr/search/Filter.java index 8868adb5417f..0520d059ecd0 100644 --- a/solr/core/src/java/org/apache/solr/search/Filter.java +++ b/solr/core/src/java/org/apache/solr/search/Filter.java @@ -27,7 +27,6 @@ import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Matches; -import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; diff --git a/solr/core/src/java/org/apache/solr/search/GraphTermsQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/GraphTermsQParserPlugin.java index e5667752662b..57e22c7b2496 100644 --- a/solr/core/src/java/org/apache/solr/search/GraphTermsQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/GraphTermsQParserPlugin.java @@ -37,8 +37,8 @@ import org.apache.lucene.index.PrefixCodedTerms; import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermStates; import org.apache.lucene.index.TermState; +import org.apache.lucene.index.TermStates; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.BulkScorer; @@ -50,7 +50,6 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Matches; -import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; diff --git a/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java index 096999647893..5dd3f2f0b898 100644 --- a/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java @@ -36,7 +36,6 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Matches; -import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; diff --git a/solr/core/src/java/org/apache/solr/search/SolrConstantScoreQuery.java b/solr/core/src/java/org/apache/solr/search/SolrConstantScoreQuery.java index f4031b505f3c..cf5da3c87ad6 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrConstantScoreQuery.java +++ b/solr/core/src/java/org/apache/solr/search/SolrConstantScoreQuery.java @@ -28,7 +28,6 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Matches; -import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; diff --git a/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java b/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java index 4c81f787b725..a9d2716d3ffc 100644 --- a/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java +++ b/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java @@ -34,7 +34,6 @@ import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Matches; -import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; diff --git a/solr/core/src/java/org/apache/solr/update/DeleteByQueryWrapper.java b/solr/core/src/java/org/apache/solr/update/DeleteByQueryWrapper.java index 9a9c05fdc848..3262711b3456 100644 --- a/solr/core/src/java/org/apache/solr/update/DeleteByQueryWrapper.java +++ b/solr/core/src/java/org/apache/solr/update/DeleteByQueryWrapper.java @@ -27,7 +27,6 @@ import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Matches; -import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSortRandom.java b/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSortRandom.java index 8dd4801521ff..42be74a8118f 100644 --- a/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSortRandom.java +++ b/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSortRandom.java @@ -44,7 +44,6 @@ import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Matches; -import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; From d2e102a30da3b65b8bcf041389c983ec7e917ef6 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 26 Mar 2018 08:34:11 +0100 Subject: [PATCH 19/45] LUCENE-8229: Add Weight.matches() method --- .../apache/lucene/document/FeatureQuery.java | 6 + .../lucene/document/RangeFieldQuery.java | 6 + .../SortedNumericDocValuesRangeQuery.java | 6 + .../SortedSetDocValuesRangeQuery.java | 6 + .../apache/lucene/search/BooleanWeight.java | 35 ++ .../lucene/search/ConstantScoreQuery.java | 5 + .../search/DisjunctionMatchesIterator.java | 158 +++++++++ .../lucene/search/DisjunctionMaxQuery.java | 12 + .../search/DocValuesFieldExistsQuery.java | 5 + .../lucene/search/DocValuesRewriteMethod.java | 20 +- .../apache/lucene/search/FilterWeight.java | 4 + .../lucene/search/IndexOrDocValuesQuery.java | 6 + .../apache/lucene/search/LRUQueryCache.java | 5 + .../lucene/search/MatchAllDocsQuery.java | 6 + .../lucene/search/MatchNoDocsQuery.java | 5 + .../org/apache/lucene/search/Matches.java | 161 ++++++++++ .../apache/lucene/search/MatchesIterator.java | 69 ++++ .../lucene/search/MultiPhraseQuery.java | 5 + .../MultiTermQueryConstantScoreWrapper.java | 9 + .../lucene/search/NormsFieldExistsQuery.java | 5 + .../org/apache/lucene/search/PhraseQuery.java | 5 + .../apache/lucene/search/PointInSetQuery.java | 5 + .../apache/lucene/search/PointRangeQuery.java | 5 + .../apache/lucene/search/SynonymQuery.java | 8 +- .../apache/lucene/search/TermInSetQuery.java | 5 + .../lucene/search/TermMatchesIterator.java | 77 +++++ .../org/apache/lucene/search/TermQuery.java | 14 + .../java/org/apache/lucene/search/Weight.java | 9 + .../lucene/search/spans/SpanWeight.java | 7 + .../lucene/search/JustCompileSearch.java | 5 + .../lucene/search/TestBooleanScorer.java | 5 + .../lucene/search/TestLRUQueryCache.java | 25 ++ .../lucene/search/TestMatchesIterator.java | 299 ++++++++++++++++++ .../lucene/search/TestQueryRescorer.java | 5 + .../apache/lucene/search/TestScorerPerf.java | 5 + .../apache/lucene/search/TestSortRandom.java | 5 + .../TestUsageTrackingFilterCachingPolicy.java | 5 + .../lucene/facet/DrillSidewaysQuery.java | 6 + .../lucene/facet/range/DoubleRange.java | 6 + .../apache/lucene/facet/range/LongRange.java | 6 + .../lucene/facet/TestDrillSideways.java | 24 +- .../search/join/GlobalOrdinalsQuery.java | 6 + .../join/ParentChildrenBlockJoinQuery.java | 6 + .../join/PointInSetIncludingScoreQuery.java | 6 + .../search/join/TermsIncludingScoreQuery.java | 6 + .../lucene/search/join/TestJoinUtil.java | 5 + .../TestDiversifiedTopDocsCollector.java | 7 +- .../queries/function/FunctionMatchQuery.java | 7 + .../queries/function/FunctionQuery.java | 6 + .../queries/function/FunctionRangeQuery.java | 6 + .../queries/function/FunctionScoreQuery.java | 6 + .../document/LatLonDocValuesBoxQuery.java | 7 + .../LatLonDocValuesDistanceQuery.java | 6 + .../document/LatLonPointDistanceQuery.java | 6 + .../document/LatLonPointInPolygonQuery.java | 6 + .../apache/lucene/search/CoveringQuery.java | 22 ++ .../lucene/search/DocValuesNumbersQuery.java | 5 + .../lucene/search/DocValuesTermsQuery.java | 5 + .../lucene/search/TermAutomatonQuery.java | 5 + .../lucene/search/TestTermAutomatonQuery.java | 5 + .../composite/CompositeVerifyQuery.java | 17 + .../composite/IntersectsRPTVerifyQuery.java | 9 + .../prefix/AbstractPrefixTreeQuery.java | 6 + .../serialized/SerializedDVStrategy.java | 13 +- .../spatial/vector/PointVectorStrategy.java | 10 +- .../spatial3d/PointInGeo3DShapeQuery.java | 6 + .../suggest/document/CompletionWeight.java | 6 + .../lucene/search/AssertingMatches.java | 43 +++ .../search/AssertingMatchesIterator.java | 77 +++++ .../apache/lucene/search/AssertingWeight.java | 8 + .../lucene/search/BlockScoreQueryWrapper.java | 5 + .../org/apache/solr/ltr/LTRScoringQuery.java | 6 + .../org/apache/solr/ltr/feature/Feature.java | 6 + .../org/apache/solr/query/SolrRangeQuery.java | 8 +- .../org/apache/solr/schema/LatLonType.java | 7 + .../java/org/apache/solr/search/Filter.java | 6 + .../solr/search/GraphTermsQParserPlugin.java | 13 +- .../apache/solr/search/JoinQParserPlugin.java | 6 + .../solr/search/SolrConstantScoreQuery.java | 6 + .../apache/solr/search/join/GraphQuery.java | 8 +- .../solr/update/DeleteByQueryWrapper.java | 6 + .../uninverting/TestFieldCacheSortRandom.java | 6 + 82 files changed, 1440 insertions(+), 31 deletions(-) create mode 100644 lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java create mode 100644 lucene/core/src/java/org/apache/lucene/search/Matches.java create mode 100644 lucene/core/src/java/org/apache/lucene/search/MatchesIterator.java create mode 100644 lucene/core/src/java/org/apache/lucene/search/TermMatchesIterator.java create mode 100644 lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java create mode 100644 lucene/test-framework/src/java/org/apache/lucene/search/AssertingMatches.java create mode 100644 lucene/test-framework/src/java/org/apache/lucene/search/AssertingMatchesIterator.java diff --git a/lucene/core/src/java/org/apache/lucene/document/FeatureQuery.java b/lucene/core/src/java/org/apache/lucene/document/FeatureQuery.java index 841b2ad298b0..d74fd06a0da8 100644 --- a/lucene/core/src/java/org/apache/lucene/document/FeatureQuery.java +++ b/lucene/core/src/java/org/apache/lucene/document/FeatureQuery.java @@ -30,6 +30,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -81,6 +82,11 @@ public boolean isCacheable(LeafReaderContext ctx) { @Override public void extractTerms(Set terms) {} + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, fieldName); + } + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { String desc = "weight(" + getQuery() + " in " + doc + ") [" + function + "]"; diff --git a/lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java b/lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java index a24b7cdfae58..7e9bd72fa0e7 100644 --- a/lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java +++ b/lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java @@ -30,6 +30,7 @@ import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.ScorerSupplier; @@ -349,6 +350,11 @@ public long cost() { } } + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, field); + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { ScorerSupplier scorerSupplier = scorerSupplier(context); diff --git a/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java b/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java index 246b50f3dab6..430edb4aed4c 100644 --- a/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java +++ b/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java @@ -29,6 +29,7 @@ import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -101,6 +102,11 @@ public boolean isCacheable(LeafReaderContext ctx) { return DocValues.isCacheable(ctx, field); } + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, field); + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { SortedNumericDocValues values = getValues(context.reader(), field); diff --git a/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java b/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java index de7c11b1cc9a..eba1e729f57a 100644 --- a/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java +++ b/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java @@ -29,6 +29,7 @@ import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -106,6 +107,11 @@ public Query rewrite(IndexReader reader) throws IOException { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, field); + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { SortedSetDocValues values = getValues(context.reader(), field); diff --git a/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java b/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java index fffdd09093f1..1fddd7c3370d 100644 --- a/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java +++ b/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java @@ -119,6 +119,41 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio } } + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + final int minShouldMatch = query.getMinimumNumberShouldMatch(); + List matches = new ArrayList<>(); + int shouldMatchCount = 0; + Iterator wIt = weights.iterator(); + Iterator cIt = query.clauses().iterator(); + while (wIt.hasNext()) { + Weight w = wIt.next(); + BooleanClause bc = cIt.next(); + Matches m = w.matches(context, doc); + if (bc.isProhibited()) { + if (m != null) { + return null; + } + } + if (bc.isRequired()) { + if (m == null) { + return null; + } + matches.add(m); + } + if (bc.getOccur() == Occur.SHOULD) { + if (m != null) { + matches.add(m); + shouldMatchCount++; + } + } + } + if (shouldMatchCount < minShouldMatch) { + return null; + } + return Matches.fromSubMatches(matches); + } + static BulkScorer disableScoring(final BulkScorer scorer) { return new BulkScorer() { diff --git a/lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java b/lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java index 464cde6a45f9..d696c215f1bf 100644 --- a/lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java @@ -158,6 +158,11 @@ public long cost() { }; } + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return innerWeight.matches(context, doc); + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { ScorerSupplier scorerSupplier = scorerSupplier(context); diff --git a/lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java b/lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java new file mode 100644 index 000000000000..db064c411797 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.search; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.PostingsEnum; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefIterator; +import org.apache.lucene.util.PriorityQueue; + +/** + * A {@link MatchesIterator} that combines matches from a set of sub-iterators + * + * Matches are sorted by their start positions, and then by their end positions, so that + * prefixes sort first. Matches may overlap. + */ +public class DisjunctionMatchesIterator implements MatchesIterator { + + /** + * Create a {@link DisjunctionMatchesIterator} over a list of terms + * + * Only terms that have at least one match in the given document will be included + */ + public static DisjunctionMatchesIterator fromTerms(LeafReaderContext context, int doc, String field, List terms) throws IOException { + return fromTermsEnum(context, doc, field, asBytesRefIterator(terms)); + } + + private static BytesRefIterator asBytesRefIterator(List terms) { + return new BytesRefIterator() { + int i = 0; + @Override + public BytesRef next() { + if (i >= terms.size()) + return null; + return terms.get(i++).bytes(); + } + }; + } + + /** + * Create a {@link DisjunctionMatchesIterator} over a list of terms extracted from a {@link BytesRefIterator} + * + * Only terms that have at least one match in the given document will be included + */ + public static DisjunctionMatchesIterator fromTermsEnum(LeafReaderContext context, int doc, String field, BytesRefIterator terms) throws IOException { + List mis = new ArrayList<>(); + Terms t = context.reader().terms(field); + if (t == null) + return null; + TermsEnum te = t.iterator(); + PostingsEnum reuse = null; + for (BytesRef term = terms.next(); term != null; term = terms.next()) { + if (te.seekExact(term)) { + PostingsEnum pe = te.postings(reuse, PostingsEnum.OFFSETS); + if (pe.advance(doc) == doc) { + // TODO do we want to use the copied term here, or instead create a label that associates all of the TMIs with a single term? + mis.add(new TermMatchesIterator(BytesRef.deepCopyOf(term), pe)); + reuse = null; + } + else { + reuse = pe; + } + } + } + if (mis.size() == 0) + return null; + return new DisjunctionMatchesIterator(mis); + } + + static MatchesIterator fromSubIterators(List mis) throws IOException { + if (mis.size() == 0) + return null; + if (mis.size() == 1) + return mis.get(0); + return new DisjunctionMatchesIterator(mis); + } + + private final PriorityQueue queue; + + private boolean started = false; + + private DisjunctionMatchesIterator(List matches) throws IOException { + queue = new PriorityQueue(matches.size()){ + @Override + protected boolean lessThan(MatchesIterator a, MatchesIterator b) { + return a.startPosition() < b.startPosition() || + (a.startPosition() == b.startPosition() && a.endPosition() < b.endPosition()); + } + }; + for (MatchesIterator mi : matches) { + mi.next(); + queue.add(mi); + } + } + + @Override + public boolean next() throws IOException { + if (started == false) { + return started = true; + } + if (queue.top().next() == false) { + queue.pop(); + } + if (queue.size() > 0) { + queue.updateTop(); + return true; + } + return false; + } + + @Override + public int startPosition() { + return queue.top().startPosition(); + } + + @Override + public int endPosition() { + return queue.top().endPosition(); + } + + @Override + public int startOffset() throws IOException { + return queue.top().startOffset(); + } + + @Override + public int endOffset() throws IOException { + return queue.top().endOffset(); + } + + @Override + public BytesRef term() { + return queue.top().term(); + } + +} diff --git a/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java b/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java index 1e67cb150465..5b99c3ce24b4 100644 --- a/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java @@ -118,6 +118,18 @@ public void extractTerms(Set terms) { } } + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + List mis = new ArrayList<>(); + for (Weight weight : weights) { + Matches mi = weight.matches(context, doc); + if (mi != null) { + mis.add(mi); + } + } + return Matches.fromSubMatches(mis); + } + /** Create the scorer used to score our associated DisjunctionMaxQuery */ @Override public Scorer scorer(LeafReaderContext context) throws IOException { diff --git a/lucene/core/src/java/org/apache/lucene/search/DocValuesFieldExistsQuery.java b/lucene/core/src/java/org/apache/lucene/search/DocValuesFieldExistsQuery.java index 009f11cf116f..22cb5aab781d 100644 --- a/lucene/core/src/java/org/apache/lucene/search/DocValuesFieldExistsQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/DocValuesFieldExistsQuery.java @@ -64,6 +64,11 @@ public String toString(String field) { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, field); + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { FieldInfos fieldInfos = context.reader().getFieldInfos(); diff --git a/lucene/core/src/java/org/apache/lucene/search/DocValuesRewriteMethod.java b/lucene/core/src/java/org/apache/lucene/search/DocValuesRewriteMethod.java index 5d591983fab0..c745ced475c4 100644 --- a/lucene/core/src/java/org/apache/lucene/search/DocValuesRewriteMethod.java +++ b/lucene/core/src/java/org/apache/lucene/search/DocValuesRewriteMethod.java @@ -74,11 +74,16 @@ public final int hashCode() { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { + @Override - public Scorer scorer(LeafReaderContext context) throws IOException { + public Matches matches(LeafReaderContext context, int doc) throws IOException { final SortedSetDocValues fcsi = DocValues.getSortedSet(context.reader(), query.field); - TermsEnum termsEnum = query.getTermsEnum(new Terms() { - + return Matches.fromField(query.field, DisjunctionMatchesIterator.fromTermsEnum(context, doc, query.field, getTermsEnum(fcsi))); + } + + private TermsEnum getTermsEnum(SortedSetDocValues fcsi) throws IOException { + return query.getTermsEnum(new Terms() { + @Override public TermsEnum iterator() throws IOException { return fcsi.termsEnum(); @@ -118,13 +123,18 @@ public boolean hasOffsets() { public boolean hasPositions() { return false; } - + @Override public boolean hasPayloads() { return false; } }); - + } + + @Override + public Scorer scorer(LeafReaderContext context) throws IOException { + final SortedSetDocValues fcsi = DocValues.getSortedSet(context.reader(), query.field); + TermsEnum termsEnum = getTermsEnum(fcsi); assert termsEnum != null; if (termsEnum.next() == null) { // no matching terms diff --git a/lucene/core/src/java/org/apache/lucene/search/FilterWeight.java b/lucene/core/src/java/org/apache/lucene/search/FilterWeight.java index 925c9534f898..8a2b57b41ea6 100644 --- a/lucene/core/src/java/org/apache/lucene/search/FilterWeight.java +++ b/lucene/core/src/java/org/apache/lucene/search/FilterWeight.java @@ -75,4 +75,8 @@ public Scorer scorer(LeafReaderContext context) throws IOException { return in.scorer(context); } + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return in.matches(context, doc); + } } diff --git a/lucene/core/src/java/org/apache/lucene/search/IndexOrDocValuesQuery.java b/lucene/core/src/java/org/apache/lucene/search/IndexOrDocValuesQuery.java index f89924d16054..d69421ec4d6d 100644 --- a/lucene/core/src/java/org/apache/lucene/search/IndexOrDocValuesQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/IndexOrDocValuesQuery.java @@ -119,6 +119,12 @@ public void extractTerms(Set terms) { indexWeight.extractTerms(terms); } + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + // We need to check a single doc, so the dv query should perform better + return dvWeight.matches(context, doc); + } + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { // We need to check a single doc, so the dv query should perform better diff --git a/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java b/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java index 72239e876cbe..9391afdbf962 100644 --- a/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java +++ b/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java @@ -678,6 +678,11 @@ public void extractTerms(Set terms) { in.extractTerms(terms); } + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return in.matches(context, doc); + } + private boolean cacheEntryHasReasonableWorstCaseSize(int maxDoc) { // The worst-case (dense) is a bit set which needs one bit per document final long worstCaseRamUsage = maxDoc / 8; diff --git a/lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java b/lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java index 89b299734144..a9945c66d812 100644 --- a/lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java @@ -35,6 +35,12 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo public String toString() { return "weight(" + MatchAllDocsQuery.this + ")"; } + + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, "*"); + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { return new ConstantScoreScorer(this, score(), DocIdSetIterator.all(context.reader().maxDoc())); diff --git a/lucene/core/src/java/org/apache/lucene/search/MatchNoDocsQuery.java b/lucene/core/src/java/org/apache/lucene/search/MatchNoDocsQuery.java index 525a18395434..2527539dd3a4 100644 --- a/lucene/core/src/java/org/apache/lucene/search/MatchNoDocsQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/MatchNoDocsQuery.java @@ -48,6 +48,11 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo public void extractTerms(Set terms) { } + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return null; + } + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { return Explanation.noMatch(reason); diff --git a/lucene/core/src/java/org/apache/lucene/search/Matches.java b/lucene/core/src/java/org/apache/lucene/search/Matches.java new file mode 100644 index 000000000000..780ec8ebab5f --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/search/Matches.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.search; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.util.BytesRef; + +/** + * Reports the positions and optionally offsets of all matching terms in a query + * for a single document + * + * To find all fields that have matches, call {@link #getMatchFields()} + * + * To obtain a {@link MatchesIterator} for a particular field, call {@link #getMatches(String)} + */ +public class Matches { + + private final Map matches; + + /** + * Create a simple {@link Matches} for a single field + */ + public static Matches fromField(String field, MatchesIterator it) { + if (it == null) { + return null; + } + return new Matches(field, it); + } + + /** + * Create an empty {@link Matches} for a Weight + * + * If the Weight's parent query does not match this document, returns {@code null}, + * otherwise returns a {@link Matches} document with an empty iterator on the given + * fields + */ + public static Matches emptyMatches(LeafReaderContext context, int doc, Weight weight, String... fields) throws IOException { + Scorer scorer = weight.scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + List matches = new ArrayList<>(); + for (String field : fields) { + matches.add(Matches.fromField(field, EMPTY)); + } + return Matches.fromSubMatches(matches); + } + + /** + * Amalgamate a collection of {@link Matches} into a single object + */ + public static Matches fromSubMatches(List subMatches) throws IOException { + if (subMatches == null || subMatches.size() == 0) { + return null; + } + if (subMatches.size() == 1) { + return subMatches.get(0); + } + Map matches = new HashMap<>(); + Set allFields = new HashSet<>(); + for (Matches m : subMatches) { + allFields.addAll(m.getMatchFields()); + } + for (String field : allFields) { + List mis = new ArrayList<>(); + for (Matches m : subMatches) { + MatchesIterator mi = m.getMatches(field); + if (mi != null) { + mis.add(mi); + } + } + matches.put(field, DisjunctionMatchesIterator.fromSubIterators(mis)); + } + return new Matches(matches); + } + + /** + * Create a {@link Matches} from a map of fields to iterators + */ + protected Matches(Map matches) { + this.matches = matches; + } + + private Matches(String field, MatchesIterator iterator) { + this.matches = new HashMap<>(); + this.matches.put(field, iterator); + } + + /** + * Returns a {@link MatchesIterator} over the matches for a single field, + * or {@code null} if there are no matches in that field + */ + public MatchesIterator getMatches(String field) { + return matches.get(field); + } + + /** + * Returns the fields with matches for this document + */ + public Set getMatchFields() { + return matches.keySet(); + } + + private static final BytesRef EMPTY_BYTES = new BytesRef(); + + private static final MatchesIterator EMPTY = new MatchesIterator() { + + @Override + public boolean next() throws IOException { + return false; + } + + @Override + public int startPosition() { + throw new UnsupportedOperationException(); + } + + @Override + public int endPosition() { + throw new UnsupportedOperationException(); + } + + @Override + public int startOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public BytesRef term() { + return EMPTY_BYTES; + } + }; +} diff --git a/lucene/core/src/java/org/apache/lucene/search/MatchesIterator.java b/lucene/core/src/java/org/apache/lucene/search/MatchesIterator.java new file mode 100644 index 000000000000..abca34eb3ccb --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/search/MatchesIterator.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.search; + +import java.io.IOException; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.util.BytesRef; + +/** + * An iterator over match positions (and optionally offsets) for a single document and field + * + * To iterate over the matches, call {@link #next()} until it returns {@code false}, retrieving + * positions and/or offsets after each call. You should not call the position or offset methods + * before {@link #next()} has been called, or after {@link #next()} has returned {@code false}. + * + * Matches are ordered by start position, and then by end position. Match intervals may overlap. + * + * @see Weight#matches(LeafReaderContext, int) + */ +public interface MatchesIterator { + + /** + * Advance the iterator to the next match position + * @return {@code true} if matches have not been exhausted + */ + boolean next() throws IOException; + + /** + * The start position of the current match + */ + int startPosition(); + + /** + * The end position of the current match + */ + int endPosition(); + + /** + * The starting offset of the current match, or {@code -1} if offsets are not available + */ + int startOffset() throws IOException; + + /** + * The ending offset of the current match, or {@code -1} if offsets are not available + */ + int endOffset() throws IOException; + + /** + * The underlying term of the current match + */ + BytesRef term(); + +} diff --git a/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java b/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java index 65d6631e9a7c..31599fa33fb2 100644 --- a/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java @@ -235,6 +235,11 @@ public void extractTerms(Set terms) { } } + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, field); // TODO + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { assert termArrays.length != 0; diff --git a/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryConstantScoreWrapper.java b/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryConstantScoreWrapper.java index 3a46b96411cf..51214e53f675 100644 --- a/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryConstantScoreWrapper.java +++ b/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryConstantScoreWrapper.java @@ -202,6 +202,15 @@ public BulkScorer bulkScorer(LeafReaderContext context) throws IOException { } } + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + final Terms terms = context.reader().terms(query.field); + if (terms == null) { + return null; + } + return Matches.fromField(query.field, DisjunctionMatchesIterator.fromTermsEnum(context, doc, query.field, query.getTermsEnum(terms))); + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { final WeightOrDocIdSet weightOrBitSet = rewrite(context); diff --git a/lucene/core/src/java/org/apache/lucene/search/NormsFieldExistsQuery.java b/lucene/core/src/java/org/apache/lucene/search/NormsFieldExistsQuery.java index 74218b40b0c3..5ff326d2f4c8 100644 --- a/lucene/core/src/java/org/apache/lucene/search/NormsFieldExistsQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/NormsFieldExistsQuery.java @@ -64,6 +64,11 @@ public String toString(String field) { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, field); + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { FieldInfos fieldInfos = context.reader().getFieldInfos(); diff --git a/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java index ff1538820d61..5c333e782a8f 100644 --- a/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java @@ -392,6 +392,11 @@ public void extractTerms(Set queryTerms) { Collections.addAll(queryTerms, terms); } + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, field); // TODO + } + @Override public String toString() { return "weight(" + PhraseQuery.this + ")"; } diff --git a/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java b/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java index 689d64a50d74..7c3814b1c855 100644 --- a/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java @@ -113,6 +113,11 @@ public final Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, fl return new ConstantScoreWeight(this, boost) { + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, field); + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { LeafReader reader = context.reader(); diff --git a/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java b/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java index 7e48383b4720..2f789e41ea84 100644 --- a/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java @@ -313,6 +313,11 @@ public long cost() { } } + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, field); // TODO can we return values here somehow? + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { ScorerSupplier scorerSupplier = scorerSupplier(context); diff --git a/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java b/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java index 2a7c450805d9..00daec5d7bcf 100644 --- a/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java @@ -30,8 +30,8 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermStates; import org.apache.lucene.index.TermState; +import org.apache.lucene.index.TermStates; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.util.BytesRef; @@ -159,6 +159,12 @@ public void extractTerms(Set terms) { } } + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + String field = terms[0].field(); + return Matches.fromField(field, DisjunctionMatchesIterator.fromTerms(context, doc, field, Arrays.asList(SynonymQuery.this.terms))); + } + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { Scorer scorer = scorer(context); diff --git a/lucene/core/src/java/org/apache/lucene/search/TermInSetQuery.java b/lucene/core/src/java/org/apache/lucene/search/TermInSetQuery.java index a8bf5b0679c1..be106f554641 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TermInSetQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/TermInSetQuery.java @@ -220,6 +220,11 @@ public void extractTerms(Set terms) { // order to protect highlighters } + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.fromField(field, DisjunctionMatchesIterator.fromTermsEnum(context, doc, field, termData.iterator())); + } + /** * On the given leaf context, try to either rewrite to a disjunction if * there are few matching terms, or build a bitset containing matching docs. diff --git a/lucene/core/src/java/org/apache/lucene/search/TermMatchesIterator.java b/lucene/core/src/java/org/apache/lucene/search/TermMatchesIterator.java new file mode 100644 index 000000000000..051699651dbe --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/search/TermMatchesIterator.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.search; + +import java.io.IOException; + +import org.apache.lucene.index.PostingsEnum; +import org.apache.lucene.util.BytesRef; + +/** + * A {@link MatchesIterator} over a single term's postings list + */ +class TermMatchesIterator implements MatchesIterator { + + private int upto; + private int pos; + private final PostingsEnum pe; + private final BytesRef term; + + /** + * Create a new {@link TermMatchesIterator} for the given term and postings list + */ + TermMatchesIterator(BytesRef term, PostingsEnum pe) throws IOException { + this.pe = pe; + this.upto = pe.freq(); + this.term = term; + } + + @Override + public boolean next() throws IOException { + if (upto-- > 0) { + pos = pe.nextPosition(); + return true; + } + return false; + } + + @Override + public int startPosition() { + return pos; + } + + @Override + public int endPosition() { + return pos; + } + + @Override + public int startOffset() throws IOException { + return pe.startOffset(); + } + + @Override + public int endOffset() throws IOException { + return pe.endOffset(); + } + + @Override + public BytesRef term() { + return term; + } +} diff --git a/lucene/core/src/java/org/apache/lucene/search/TermQuery.java b/lucene/core/src/java/org/apache/lucene/search/TermQuery.java index f1f44154f554..0928aca9f7e9 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TermQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/TermQuery.java @@ -25,6 +25,7 @@ import org.apache.lucene.index.IndexReaderContext; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermStates; @@ -80,6 +81,19 @@ public void extractTerms(Set terms) { terms.add(getTerm()); } + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + TermsEnum te = getTermsEnum(context); + if (te == null) { + return null; + } + PostingsEnum pe = te.postings(null, PostingsEnum.OFFSETS); + if (pe.advance(doc) != doc) { + return null; + } + return Matches.fromField(term.field(), new TermMatchesIterator(term.bytes(), pe)); + } + @Override public String toString() { return "weight(" + TermQuery.this + ")"; diff --git a/lucene/core/src/java/org/apache/lucene/search/Weight.java b/lucene/core/src/java/org/apache/lucene/search/Weight.java index 7853ccf2465b..6587511d1cb4 100644 --- a/lucene/core/src/java/org/apache/lucene/search/Weight.java +++ b/lucene/core/src/java/org/apache/lucene/search/Weight.java @@ -69,6 +69,15 @@ protected Weight(Query query) { */ public abstract void extractTerms(Set terms); + /** + * Returns {@link Matches} for a specific document, or {@code null} if the document + * does not match the parent query + * + * @param context the reader's context to create the {@link Matches} for + * @param doc the document's id relative to the given context's reader + */ + public abstract Matches matches(LeafReaderContext context, int doc) throws IOException; + /** * An explanation of the score computation for the named document. * diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java b/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java index 25b58fdc39a0..f93f9a1e40de 100644 --- a/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java +++ b/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java @@ -29,6 +29,7 @@ import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.LeafSimScorer; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.TermStatistics; import org.apache.lucene.search.Weight; import org.apache.lucene.search.similarities.Similarity; @@ -161,4 +162,10 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio return Explanation.noMatch("no matching term"); } + + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + // TODO: Composite matches + return Matches.emptyMatches(context, doc, this, field); + } } diff --git a/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java b/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java index 1657f9b9ced1..909e93290f04 100644 --- a/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java +++ b/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java @@ -247,6 +247,11 @@ public void extractTerms(Set terms) { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return null; + } + @Override public Explanation explain(LeafReaderContext context, int doc) { throw new UnsupportedOperationException(UNSUPPORTED_MSG); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java b/lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java index 8a8379be3432..c76b1607c046 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java @@ -84,6 +84,11 @@ public void extractTerms(Set terms) { throw new UnsupportedOperationException(); } + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return null; + } + @Override public Explanation explain(LeafReaderContext context, int doc) { throw new UnsupportedOperationException(); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java b/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java index 9cbf29ce2cf3..5b052f6fed8d 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java @@ -349,6 +349,11 @@ private static class DummyQuery extends Query { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return null; + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { return null; @@ -943,6 +948,11 @@ private static class BadQuery extends Query { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return null; + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { return null; @@ -1292,6 +1302,11 @@ public void extractTerms(Set terms) { } + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return null; + } + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { return null; @@ -1364,6 +1379,11 @@ private static class DummyQuery2 extends Query { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return null; + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { return scorerSupplier(context).get(Long.MAX_VALUE); @@ -1464,6 +1484,11 @@ public int hashCode() { public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, 1) { + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return null; + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { scorerCreatedCount.incrementAndGet(); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java b/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java new file mode 100644 index 000000000000..9b74b590c3d2 --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java @@ -0,0 +1,299 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.search; + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.index.ReaderUtil; +import org.apache.lucene.index.Term; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.LuceneTestCase; + +public class TestMatchesIterator extends LuceneTestCase { + + protected IndexSearcher searcher; + protected Directory directory; + protected IndexReader reader; + + public static final String FIELD_WITH_OFFSETS = "field_offsets"; + public static final String FIELD_NO_OFFSETS = "field_no_offsets"; + + public static final FieldType OFFSETS = new FieldType(TextField.TYPE_STORED); + static { + OFFSETS.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS); + } + + @Override + public void tearDown() throws Exception { + reader.close(); + directory.close(); + super.tearDown(); + } + + @Override + public void setUp() throws Exception { + super.setUp(); + directory = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), directory, + newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy())); + for (int i = 0; i < docFields.length; i++) { + Document doc = new Document(); + doc.add(newField(FIELD_WITH_OFFSETS, docFields[i], OFFSETS)); + doc.add(newField(FIELD_NO_OFFSETS, docFields[i], TextField.TYPE_STORED)); + doc.add(new NumericDocValuesField("id", i)); + doc.add(newField("id", Integer.toString(i), TextField.TYPE_STORED)); + writer.addDocument(doc); + } + writer.forceMerge(1); + reader = writer.getReader(); + writer.close(); + searcher = newSearcher(getOnlyLeafReader(reader)); + } + + protected String[] docFields = { + "w1 w2 w3 w4 w5", + "w1 w3 w2 w3 zz", + "w1 xx w2 yy w4", + "w1 w2 w1 w4 w2 w3" + }; + + void checkMatches(Query q, String field, int[][] expected) throws IOException { + Weight w = searcher.createNormalizedWeight(q, ScoreMode.COMPLETE_NO_SCORES); + for (int i = 0; i < expected.length; i++) { + LeafReaderContext ctx = searcher.leafContexts.get(ReaderUtil.subIndex(expected[i][0], searcher.leafContexts)); + int doc = expected[i][0] - ctx.docBase; + Matches matches = w.matches(ctx, doc); + if (matches == null) { + assertEquals(expected[i].length, 1); + continue; + } + MatchesIterator it = matches.getMatches(field); + checkFieldMatches(it, expected[i]); + } + } + + void checkFieldMatches(MatchesIterator it, int[] expected) throws IOException { + int pos = 1; + while (it.next()) { + //System.out.println(expected[i][pos] + "->" + expected[i][pos + 1] + "[" + expected[i][pos + 2] + "->" + expected[i][pos + 3] + "]"); + assertEquals(expected[pos], it.startPosition()); + assertEquals(expected[pos + 1], it.endPosition()); + assertEquals(expected[pos + 2], it.startOffset()); + assertEquals(expected[pos + 3], it.endOffset()); + pos += 4; + } + assertEquals(expected.length, pos); + } + + void checkTerms(Query q, String field, String[][] expected) throws IOException { + Weight w = searcher.createNormalizedWeight(q, ScoreMode.COMPLETE_NO_SCORES); + for (int i = 0; i < expected.length; i++) { + LeafReaderContext ctx = searcher.leafContexts.get(ReaderUtil.subIndex(i, searcher.leafContexts)); + int doc = i - ctx.docBase; + Matches matches = w.matches(ctx, doc); + if (matches == null) { + assertEquals(expected[i].length, 0); + continue; + } + MatchesIterator it = matches.getMatches(field); + int pos = 0; + while (it.next()) { + assertEquals(expected[i][pos], it.term().utf8ToString()); + pos += 1; + } + assertEquals(expected[i].length, pos); + } + } + + public void testTermQuery() throws IOException { + Query q = new TermQuery(new Term(FIELD_WITH_OFFSETS, "w1")); + checkMatches(q, FIELD_WITH_OFFSETS, new int[][]{ + { 0, 0, 0, 0, 2 }, + { 1, 0, 0, 0, 2 }, + { 2, 0, 0, 0, 2 }, + { 3, 0, 0, 0, 2, 2, 2, 6, 8 } + }); + } + + public void testTermQueryNoStoredOffsets() throws IOException { + Query q = new TermQuery(new Term(FIELD_NO_OFFSETS, "w1")); + checkMatches(q, FIELD_NO_OFFSETS, new int[][]{ + { 0, 0, 0, -1, -1 }, + { 1, 0, 0, -1, -1 }, + { 2, 0, 0, -1, -1 }, + { 3, 0, 0, -1, -1, 2, 2, -1, -1 } + }); + checkTerms(q, FIELD_NO_OFFSETS, new String[][]{ + { "w1" }, + { "w1" }, + { "w1" }, + { "w1", "w1" }, + }); + } + + public void testDisjunction() throws IOException { + Query q = new BooleanQuery.Builder() + .add(new TermQuery(new Term(FIELD_WITH_OFFSETS, "w1")), BooleanClause.Occur.SHOULD) + .add(new TermQuery(new Term(FIELD_WITH_OFFSETS, "w3")), BooleanClause.Occur.SHOULD) + .build(); + checkMatches(q, FIELD_WITH_OFFSETS, new int[][]{ + { 0, 0, 0, 0, 2, 2, 2, 6, 8 }, + { 1, 0, 0, 0, 2, 1, 1, 3, 5, 3, 3, 9, 11 }, + { 2, 0, 0, 0, 2 }, + { 3, 0, 0, 0, 2, 2, 2, 6, 8, 5, 5, 15, 17 } + }); + checkTerms(q, FIELD_WITH_OFFSETS, new String[][]{ + { "w1", "w3" }, + { "w1", "w3", "w3" }, + { "w1" }, + { "w1", "w1", "w3" } + }); + } + + public void testReqOpt() throws IOException { + Query q = new BooleanQuery.Builder() + .add(new TermQuery(new Term(FIELD_WITH_OFFSETS, "w1")), BooleanClause.Occur.SHOULD) + .add(new TermQuery(new Term(FIELD_WITH_OFFSETS, "w3")), BooleanClause.Occur.MUST) + .build(); + checkMatches(q, FIELD_WITH_OFFSETS, new int[][]{ + { 0, 0, 0, 0, 2, 2, 2, 6, 8 }, + { 1, 0, 0, 0, 2, 1, 1, 3, 5, 3, 3, 9, 11 }, + { 2 }, + { 3, 0, 0, 0, 2, 2, 2, 6, 8, 5, 5, 15, 17 } + }); + } + + public void testMinShouldMatch() throws IOException { + Query q = new BooleanQuery.Builder() + .add(new TermQuery(new Term(FIELD_WITH_OFFSETS, "w3")), BooleanClause.Occur.SHOULD) + .add(new BooleanQuery.Builder() + .add(new TermQuery(new Term(FIELD_WITH_OFFSETS, "w1")), BooleanClause.Occur.SHOULD) + .add(new TermQuery(new Term(FIELD_WITH_OFFSETS, "w4")), BooleanClause.Occur.SHOULD) + .add(new TermQuery(new Term(FIELD_WITH_OFFSETS, "xx")), BooleanClause.Occur.SHOULD) + .setMinimumNumberShouldMatch(2) + .build(), BooleanClause.Occur.SHOULD) + .build(); + checkMatches(q, FIELD_WITH_OFFSETS, new int[][]{ + { 0, 0, 0, 0, 2, 2, 2, 6, 8, 3, 3, 9, 11 }, + { 1, 1, 1, 3, 5, 3, 3, 9, 11 }, + { 2, 0, 0, 0, 2, 1, 1, 3, 5, 4, 4, 12, 14 }, + { 3, 0, 0, 0, 2, 2, 2, 6, 8, 3, 3, 9, 11, 5, 5, 15, 17 } + }); + checkTerms(q, FIELD_WITH_OFFSETS, new String[][]{ + { "w1", "w3", "w4" }, + { "w3", "w3" }, + { "w1", "xx", "w4" }, + { "w1", "w1", "w4", "w3" } + }); + } + + public void testExclusion() throws IOException { + Query q = new BooleanQuery.Builder() + .add(new TermQuery(new Term(FIELD_WITH_OFFSETS, "w3")), BooleanClause.Occur.SHOULD) + .add(new TermQuery(new Term(FIELD_WITH_OFFSETS, "zz")), BooleanClause.Occur.MUST_NOT) + .build(); + checkMatches(q, FIELD_WITH_OFFSETS, new int[][]{ + { 0, 2, 2, 6, 8 }, + { 1 }, + { 2 }, + { 3, 5, 5, 15, 17 } + }); + } + + public void testConjunction() throws IOException { + Query q = new BooleanQuery.Builder() + .add(new TermQuery(new Term(FIELD_WITH_OFFSETS, "w3")), BooleanClause.Occur.MUST) + .add(new TermQuery(new Term(FIELD_WITH_OFFSETS, "w4")), BooleanClause.Occur.MUST) + .build(); + checkMatches(q, FIELD_WITH_OFFSETS, new int[][]{ + { 0, 2, 2, 6, 8, 3, 3, 9, 11 }, + { 1 }, + { 2 }, + { 3, 3, 3, 9, 11, 5, 5, 15, 17 } + }); + } + + public void testWildcards() throws IOException { + Query q = new PrefixQuery(new Term(FIELD_WITH_OFFSETS, "x")); + checkMatches(q, FIELD_WITH_OFFSETS, new int[][]{ + { 0 }, + { 1 }, + { 2, 1, 1, 3, 5 }, + { 0 } + }); + checkTerms(q, FIELD_WITH_OFFSETS, new String[][]{ + {}, {}, { "xx" }, {} + }); + + Query rq = new RegexpQuery(new Term(FIELD_WITH_OFFSETS, "w[1-2]")); + checkMatches(rq, FIELD_WITH_OFFSETS, new int[][]{ + { 0, 0, 0, 0, 2, 1, 1, 3, 5 }, + { 1, 0, 0, 0, 2, 2, 2, 6, 8 }, + { 2, 0, 0, 0, 2, 2, 2, 6, 8 }, + { 3, 0, 0, 0, 2, 1, 1, 3, 5, 2, 2, 6, 8, 4, 4, 12, 14 } + }); + } + + public void testSynonymQuery() throws IOException { + Query q = new SynonymQuery(new Term(FIELD_WITH_OFFSETS, "w1"), new Term(FIELD_WITH_OFFSETS, "w2")); + checkMatches(q, FIELD_WITH_OFFSETS, new int[][]{ + { 0, 0, 0, 0, 2, 1, 1, 3, 5 }, + { 1, 0, 0, 0, 2, 2, 2, 6, 8 }, + { 2, 0, 0, 0, 2, 2, 2, 6, 8 }, + { 3, 0, 0, 0, 2, 1, 1, 3, 5, 2, 2, 6, 8, 4, 4, 12, 14 } + }); + } + + public void testMultipleFields() throws IOException { + Query q = new BooleanQuery.Builder() + .add(new TermQuery(new Term("id", "1")), BooleanClause.Occur.SHOULD) + .add(new TermQuery(new Term(FIELD_WITH_OFFSETS, "w3")), BooleanClause.Occur.MUST) + .build(); + Weight w = searcher.createNormalizedWeight(q, ScoreMode.COMPLETE); + + LeafReaderContext ctx = searcher.leafContexts.get(ReaderUtil.subIndex(1, searcher.leafContexts)); + Matches m = w.matches(ctx, 1 - ctx.docBase); + assertNotNull(m); + checkFieldMatches(m.getMatches("id"), new int[]{ -1, 0, 0, -1, -1 }); + checkFieldMatches(m.getMatches(FIELD_WITH_OFFSETS), new int[]{ -1, 1, 1, 3, 5, 3, 3, 9, 11 }); + assertNull(m.getMatches("bogus")); + + Set fields = m.getMatchFields(); + assertEquals(2, fields.size()); + assertTrue(fields.contains(FIELD_WITH_OFFSETS)); + assertTrue(fields.contains("id")); + } + + protected String[] doc1Fields = { + "w1 w2 w3 w4 w5", + "w1 w3 w2 w3 zz", + "w1 xx w2 yy w4", + "w1 w2 w1 w4 w2 w3" + }; + +} diff --git a/lucene/core/src/test/org/apache/lucene/search/TestQueryRescorer.java b/lucene/core/src/test/org/apache/lucene/search/TestQueryRescorer.java index d1f307d063ec..26f4d15a6906 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestQueryRescorer.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestQueryRescorer.java @@ -426,6 +426,11 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo public void extractTerms(Set terms) { } + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return null; + } + @Override public Scorer scorer(final LeafReaderContext context) throws IOException { diff --git a/lucene/core/src/test/org/apache/lucene/search/TestScorerPerf.java b/lucene/core/src/test/org/apache/lucene/search/TestScorerPerf.java index 59a246cb6647..8469e4954bf4 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestScorerPerf.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestScorerPerf.java @@ -151,6 +151,11 @@ private static class BitSetQuery extends Query { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return null; + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { return new ConstantScoreScorer(this, score(), new BitSetIterator(docs, docs.approximateCardinality())); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java b/lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java index 05b016c31c35..d97662ab857b 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java @@ -231,6 +231,11 @@ public RandomQuery(long seed, float density, List docValues) { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return null; + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { Random random = new Random(context.docBase ^ seed); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestUsageTrackingFilterCachingPolicy.java b/lucene/core/src/test/org/apache/lucene/search/TestUsageTrackingFilterCachingPolicy.java index 187accfdc524..242051cffc44 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestUsageTrackingFilterCachingPolicy.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestUsageTrackingFilterCachingPolicy.java @@ -120,6 +120,11 @@ public int hashCode() { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(DummyQuery.this, boost) { + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return null; + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { return new ConstantScoreScorer(this, score(), DocIdSetIterator.all(1)); diff --git a/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysQuery.java b/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysQuery.java index 130e1d43cea9..6383b7fd560b 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysQuery.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysQuery.java @@ -32,6 +32,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -91,6 +92,11 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo @Override public void extractTerms(Set terms) {} + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return null; + } + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { return baseWeight.explain(context, doc); diff --git a/lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRange.java b/lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRange.java index 56910f215cb5..f92ccf207953 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRange.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRange.java @@ -27,6 +27,7 @@ import org.apache.lucene.search.DoubleValues; import org.apache.lucene.search.DoubleValuesSource; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -143,6 +144,11 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo : searcher.createWeight(fastMatchQuery, ScoreMode.COMPLETE_NO_SCORES, 1f); return new ConstantScoreWeight(this, boost) { + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return null; + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { final int maxDoc = context.reader().maxDoc(); diff --git a/lucene/facet/src/java/org/apache/lucene/facet/range/LongRange.java b/lucene/facet/src/java/org/apache/lucene/facet/range/LongRange.java index 88b569d4f87f..8d53ef16ed68 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/range/LongRange.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/range/LongRange.java @@ -27,6 +27,7 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.LongValues; import org.apache.lucene.search.LongValuesSource; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -135,6 +136,11 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo : searcher.createWeight(fastMatchQuery, ScoreMode.COMPLETE_NO_SCORES, 1f); return new ConstantScoreWeight(this, boost) { + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return null; + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { final int maxDoc = context.reader().maxDoc(); diff --git a/lucene/facet/src/test/org/apache/lucene/facet/TestDrillSideways.java b/lucene/facet/src/test/org/apache/lucene/facet/TestDrillSideways.java index c632f3ae51cb..653999ea5020 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/TestDrillSideways.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/TestDrillSideways.java @@ -43,25 +43,8 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; -import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.*; import org.apache.lucene.search.BooleanClause.Occur; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.ConstantScoreScorer; -import org.apache.lucene.search.ConstantScoreWeight; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.MatchAllDocsQuery; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.ScoreMode; -import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.SimpleCollector; -import org.apache.lucene.search.Sort; -import org.apache.lucene.search.SortField; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.TopDocs; -import org.apache.lucene.search.TwoPhaseIterator; -import org.apache.lucene.search.Weight; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.IOUtils; @@ -723,6 +706,11 @@ public void testRandom() throws Exception { public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return null; + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { DocIdSetIterator approximation = DocIdSetIterator.all(context.reader().maxDoc()); diff --git a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java index 8247f81352a7..5917affb1065 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java +++ b/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java @@ -28,6 +28,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.TwoPhaseIterator; @@ -110,6 +111,11 @@ final class W extends ConstantScoreWeight { @Override public void extractTerms(Set terms) {} + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, joinField); // TODO is there a way of reporting matches that makes sense here? + } + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { SortedDocValues values = DocValues.getSorted(context.reader(), joinField); diff --git a/lucene/join/src/java/org/apache/lucene/search/join/ParentChildrenBlockJoinQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/ParentChildrenBlockJoinQuery.java index 7ce1c294592f..16f94d243367 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/ParentChildrenBlockJoinQuery.java +++ b/lucene/join/src/java/org/apache/lucene/search/join/ParentChildrenBlockJoinQuery.java @@ -27,6 +27,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; @@ -104,6 +105,11 @@ public void extractTerms(Set terms) { childWeight.extractTerms(terms); } + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, "*"); // TODO is there a way of reporting matches that makes sense here? + } + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { return Explanation.noMatch("Not implemented, use ToParentBlockJoinQuery explain why a document matched"); diff --git a/lucene/join/src/java/org/apache/lucene/search/join/PointInSetIncludingScoreQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/PointInSetIncludingScoreQuery.java index 02b7e86d1688..ec2d66dd1adb 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/PointInSetIncludingScoreQuery.java +++ b/lucene/join/src/java/org/apache/lucene/search/join/PointInSetIncludingScoreQuery.java @@ -40,6 +40,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.PointInSetQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; @@ -127,6 +128,11 @@ public final Weight createWeight(IndexSearcher searcher, org.apache.lucene.searc public void extractTerms(Set terms) { } + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, field); // TODO is there a way of reporting matches that makes sense here? + } + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { Scorer scorer = scorer(context); diff --git a/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java index 43ddd528393f..24a205c9f9f6 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java +++ b/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java @@ -29,6 +29,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; @@ -102,6 +103,11 @@ public Weight createWeight(IndexSearcher searcher, org.apache.lucene.search.Scor @Override public void extractTerms(Set terms) {} + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, toField); // TODO is there a way of reporting matches that makes sense here? + } + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { Terms terms = context.reader().terms(toField); diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java b/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java index a0f86af925b1..895569726586 100644 --- a/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java +++ b/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java @@ -524,6 +524,11 @@ public Weight createWeight(IndexSearcher searcher, org.apache.lucene.search.Scor public void extractTerms(Set terms) { } + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return null; + } + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { return null; diff --git a/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java b/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java index db17a90464e7..b22a203819c9 100644 --- a/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java +++ b/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java @@ -525,7 +525,12 @@ public int docID() { public void extractTerms(Set terms) { inner.extractTerms(terms); } - + + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, "*"); // TODO is there a way of reporting matches that makes sense here? + } + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { Scorer s = scorer(context); diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionMatchQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionMatchQuery.java index 7a837342f762..e721db623339 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionMatchQuery.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionMatchQuery.java @@ -28,6 +28,7 @@ import org.apache.lucene.search.DoubleValues; import org.apache.lucene.search.DoubleValuesSource; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -65,6 +66,12 @@ public String toString(String field) { public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { DoubleValuesSource vs = source.rewrite(searcher); return new ConstantScoreWeight(this, boost) { + + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, source.toString()); // TODO is there a way of reporting matches that makes sense here? + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { DoubleValues values = vs.getValues(context, null); diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java index f996306a72d1..78c9da0055a5 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java @@ -26,6 +26,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -70,6 +71,11 @@ public FunctionWeight(IndexSearcher searcher, float boost) throws IOException { @Override public void extractTerms(Set terms) {} + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, func.toString()); // TODO is there a way of reporting matches that makes sense here? + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { return new AllScorer(context, this, boost); diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionRangeQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionRangeQuery.java index 2d55bae7693d..72d12e0bf50f 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionRangeQuery.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionRangeQuery.java @@ -25,6 +25,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Weight; @@ -134,6 +135,11 @@ public void extractTerms(Set terms) { //none } + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, valueSource.toString()); // TODO is there a way of reporting matches that makes sense here? + } + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { FunctionValues functionValues = valueSource.getValues(vsContext, context); diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionScoreQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionScoreQuery.java index 0d39e8b25892..de0e6d409f7f 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionScoreQuery.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionScoreQuery.java @@ -29,6 +29,7 @@ import org.apache.lucene.search.Explanation; import org.apache.lucene.search.FilterScorer; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -148,6 +149,11 @@ public void extractTerms(Set terms) { this.inner.extractTerms(terms); } + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return inner.matches(context, doc); + } + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { Explanation scoreExplanation = inner.explain(context, doc); diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesBoxQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesBoxQuery.java index 31037f9c36e4..e1d7c1778730 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesBoxQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesBoxQuery.java @@ -26,6 +26,7 @@ import org.apache.lucene.search.ConstantScoreScorer; import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -98,6 +99,12 @@ public int hashCode() { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { + + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, field); // TODO is there a way of reporting matches that makes sense here? + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { final SortedNumericDocValues values = context.reader().getSortedNumericDocValues(field); diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesDistanceQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesDistanceQuery.java index df350e6db95c..b5cd2e6b88d8 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesDistanceQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesDistanceQuery.java @@ -26,6 +26,7 @@ import org.apache.lucene.search.ConstantScoreScorer; import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -98,6 +99,11 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo private final GeoEncodingUtils.DistancePredicate distancePredicate = GeoEncodingUtils.createDistancePredicate(latitude, longitude, radiusMeters); + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, field); // TODO is there a way of reporting matches that makes sense here? + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { final SortedNumericDocValues values = context.reader().getSortedNumericDocValues(field); diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java index a72d458ddc9d..1c9e3787177f 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java @@ -31,6 +31,7 @@ import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -109,6 +110,11 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo final GeoEncodingUtils.DistancePredicate distancePredicate = GeoEncodingUtils.createDistancePredicate(latitude, longitude, radiusMeters); + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, field); // TODO is there a way of reporting matches that makes sense here? + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { ScorerSupplier scorerSupplier = scorerSupplier(context); diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java index 1b2e3a6b6d17..10643f246e79 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java @@ -32,6 +32,7 @@ import org.apache.lucene.search.ConstantScoreScorer; import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -98,6 +99,11 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo return new ConstantScoreWeight(this, boost) { + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, field); // TODO is there a way of reporting matches that makes sense here? + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { LeafReader reader = context.reader(); diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/CoveringQuery.java b/lucene/sandbox/src/java/org/apache/lucene/search/CoveringQuery.java index 8f821922c6d8..8d6836b6105a 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/search/CoveringQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/search/CoveringQuery.java @@ -136,6 +136,28 @@ public void extractTerms(Set terms) { } } + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + LongValues minMatchValues = minimumNumberMatch.getValues(context, null); + if (minMatchValues.advanceExact(doc) == false) { + return null; + } + final long minimumNumberMatch = Math.max(1, minMatchValues.longValue()); + long matchCount = 0; + List subMatches = new ArrayList<>(); + for (Weight weight : weights) { + Matches matches = weight.matches(context, doc); + if (matches != null) { + matchCount++; + subMatches.add(matches); + } + } + if (matchCount < minimumNumberMatch) { + return null; + } + return Matches.fromSubMatches(subMatches); + } + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { LongValues minMatchValues = minimumNumberMatch.getValues(context, null); diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesNumbersQuery.java b/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesNumbersQuery.java index e72e99241b60..4f651cdc433f 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesNumbersQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesNumbersQuery.java @@ -99,6 +99,11 @@ public String toString(String defaultField) { public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, field); // TODO is there a way of reporting matches that makes sense here? + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { final SortedNumericDocValues values = DocValues.getSortedNumeric(context.reader(), field); diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesTermsQuery.java b/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesTermsQuery.java index 0e615b4ecbeb..33fca99af1d1 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesTermsQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesTermsQuery.java @@ -168,6 +168,11 @@ public String toString(String defaultField) { public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, field); // TODO is there a way of reporting matches that makes sense here? + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { final SortedSetDocValues values = DocValues.getSortedSet(context.reader(), field); diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java b/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java index 42a5f74216bb..b25c5e1bc792 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java @@ -371,6 +371,11 @@ public void extractTerms(Set terms) { } } + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, field); // TODO + } + @Override public String toString() { return "weight(" + TermAutomatonQuery.this + ")"; diff --git a/lucene/sandbox/src/test/org/apache/lucene/search/TestTermAutomatonQuery.java b/lucene/sandbox/src/test/org/apache/lucene/search/TestTermAutomatonQuery.java index d6bf90dfcbe9..7178d2ec337e 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/search/TestTermAutomatonQuery.java +++ b/lucene/sandbox/src/test/org/apache/lucene/search/TestTermAutomatonQuery.java @@ -584,6 +584,11 @@ public RandomQuery(long seed, float density) { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, "*"); // TODO is there a way of reporting matches that makes sense here? + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { int maxDoc = context.reader().maxDoc(); diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/CompositeVerifyQuery.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/CompositeVerifyQuery.java index 2bfa4d53e70a..44a6485eb473 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/CompositeVerifyQuery.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/CompositeVerifyQuery.java @@ -17,12 +17,15 @@ package org.apache.lucene.spatial.composite; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.ConstantScoreScorer; import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -86,6 +89,20 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo return new ConstantScoreWeight(this, boost) { + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + Matches innerMatches = indexQueryWeight.matches(context, doc); + List subMatches = new ArrayList<>(); + for (String field : innerMatches.getMatchFields()) { + subMatches.add(Matches.emptyMatches(context, doc, this, field)); + } + return Matches.fromSubMatches(subMatches); // TODO is there a way of reporting matches that makes sense here? + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/IntersectsRPTVerifyQuery.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/IntersectsRPTVerifyQuery.java index e6324dae28e6..4d004980b109 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/IntersectsRPTVerifyQuery.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/IntersectsRPTVerifyQuery.java @@ -24,6 +24,7 @@ import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -48,12 +49,14 @@ public class IntersectsRPTVerifyQuery extends Query { private final IntersectsDifferentiatingQuery intersectsDiffQuery; private final ShapeValuesPredicate predicateValueSource; + private final String field; public IntersectsRPTVerifyQuery(Shape queryShape, String fieldName, SpatialPrefixTree grid, int detailLevel, int prefixGridScanLevel, ShapeValuesPredicate predicateValueSource) { this.predicateValueSource = predicateValueSource; this.intersectsDiffQuery = new IntersectsDifferentiatingQuery(queryShape, fieldName, grid, detailLevel, prefixGridScanLevel); + this.field = fieldName; } @Override @@ -84,6 +87,12 @@ public int hashCode() { public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { + + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, field); // TODO is there a way of reporting matches that makes sense here? + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { // Compute approx & exact diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/AbstractPrefixTreeQuery.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/AbstractPrefixTreeQuery.java index 75b3c2b80bc0..5b81c4082c03 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/AbstractPrefixTreeQuery.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/AbstractPrefixTreeQuery.java @@ -28,6 +28,7 @@ import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -79,6 +80,11 @@ public int hashCode() { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, fieldName); // TODO is there a way of reporting matches that makes sense here? + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { DocIdSet docSet = getDocIdSet(context); diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java index cd94bf429bf2..92abf9f6b82d 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java @@ -33,6 +33,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.DoubleValuesSource; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -113,7 +114,7 @@ public DoubleValuesSource makeDistanceValueSource(Point queryPoint, double multi public Query makeQuery(SpatialArgs args) { ShapeValuesSource shapeValueSource = makeShapeValueSource(); ShapeValuesPredicate predicateValueSource = new ShapeValuesPredicate(shapeValueSource, args.getOperation(), args.getShape()); - return new PredicateValueSourceQuery(predicateValueSource); + return new PredicateValueSourceQuery(getFieldName(), predicateValueSource); } /** @@ -128,14 +129,22 @@ public ShapeValuesSource makeShapeValueSource() { */ static class PredicateValueSourceQuery extends Query { private final ShapeValuesPredicate predicateValueSource; + private final String field; - public PredicateValueSourceQuery(ShapeValuesPredicate predicateValueSource) { + public PredicateValueSourceQuery(String field, ShapeValuesPredicate predicateValueSource) { this.predicateValueSource = predicateValueSource; + this.field = field; } @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { + + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, field); // TODO is there a way of reporting matches that makes sense here? + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { DocIdSetIterator approximation = DocIdSetIterator.all(context.reader().maxDoc()); diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java index c7904df72160..36b8e6bf21b7 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java @@ -36,6 +36,7 @@ import org.apache.lucene.search.DoubleValues; import org.apache.lucene.search.DoubleValuesSource; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -245,7 +246,7 @@ private Query rangeQuery(String fieldName, Double min, Double max) { throw new UnsupportedOperationException("An index is required for this operation."); } - private static class DistanceRangeQuery extends Query { + private class DistanceRangeQuery extends Query { final Query inner; final DoubleValuesSource distanceSource; @@ -269,6 +270,13 @@ public Query rewrite(IndexReader reader) throws IOException { public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { Weight w = inner.createWeight(searcher, scoreMode, 1f); return new ConstantScoreWeight(this, boost) { + + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, fieldNameX, fieldNameY); + // TODO is there a way of reporting matches that makes sense here? + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { Scorer in = w.scorer(context); diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInGeo3DShapeQuery.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInGeo3DShapeQuery.java index 83a471fab55e..d8e0644ebd12 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInGeo3DShapeQuery.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInGeo3DShapeQuery.java @@ -24,6 +24,7 @@ import org.apache.lucene.search.ConstantScoreScorer; import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -68,6 +69,11 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo return new ConstantScoreWeight(this, boost) { + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, field); // TODO is there a way of reporting matches that makes sense here? + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { LeafReader reader = context.reader(); diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionWeight.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionWeight.java index 1f97d559c4da..b3bfd2edaa22 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionWeight.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionWeight.java @@ -25,6 +25,7 @@ import org.apache.lucene.index.Terms; import org.apache.lucene.search.BulkScorer; import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; import org.apache.lucene.search.suggest.BitsProducer; @@ -147,6 +148,11 @@ public void extractTerms(Set terms) { // no-op } + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return null; + } + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { //TODO diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/AssertingMatches.java b/lucene/test-framework/src/java/org/apache/lucene/search/AssertingMatches.java new file mode 100644 index 000000000000..50f49c758ff6 --- /dev/null +++ b/lucene/test-framework/src/java/org/apache/lucene/search/AssertingMatches.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.search; + +import java.util.Set; + +class AssertingMatches extends Matches { + + private final Matches in; + + AssertingMatches(Matches matches) { + super(null); + this.in = matches; + } + + @Override + public MatchesIterator getMatches(String field) { + MatchesIterator mi = in.getMatches(field); + if (mi == null) + return null; + return new AssertingMatchesIterator(mi); + } + + @Override + public Set getMatchFields() { + return in.getMatchFields(); + } +} diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/AssertingMatchesIterator.java b/lucene/test-framework/src/java/org/apache/lucene/search/AssertingMatchesIterator.java new file mode 100644 index 000000000000..52fb184250c6 --- /dev/null +++ b/lucene/test-framework/src/java/org/apache/lucene/search/AssertingMatchesIterator.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.search; + +import java.io.IOException; + +import org.apache.lucene.util.BytesRef; + +class AssertingMatchesIterator implements MatchesIterator { + + private final MatchesIterator in; + private State state = State.UNPOSITIONED; + + private enum State { UNPOSITIONED, ITERATING, EXHAUSTED } + + AssertingMatchesIterator(MatchesIterator in) { + this.in = in; + } + + @Override + public boolean next() throws IOException { + assert state != State.EXHAUSTED : state; + boolean more = in.next(); + if (more == false) { + state = State.EXHAUSTED; + } + else { + state = State.ITERATING; + } + return more; + } + + @Override + public int startPosition() { + assert state == State.ITERATING : state; + return in.startPosition(); + } + + @Override + public int endPosition() { + assert state == State.ITERATING : state; + return in.endPosition(); + } + + @Override + public int startOffset() throws IOException { + assert state == State.ITERATING : state; + return in.startOffset(); + } + + @Override + public int endOffset() throws IOException { + assert state == State.ITERATING : state; + return in.endOffset(); + } + + @Override + public BytesRef term() { + assert state == State.ITERATING : state; + return in.term(); + } +} diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/AssertingWeight.java b/lucene/test-framework/src/java/org/apache/lucene/search/AssertingWeight.java index 8e3a29fb0023..55fda238ce73 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/search/AssertingWeight.java +++ b/lucene/test-framework/src/java/org/apache/lucene/search/AssertingWeight.java @@ -31,6 +31,14 @@ class AssertingWeight extends FilterWeight { this.scoreMode = scoreMode; } + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Matches matches = in.matches(context, doc); + if (matches == null) + return null; + return new AssertingMatches(matches); + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { if (random.nextBoolean()) { diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/BlockScoreQueryWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/search/BlockScoreQueryWrapper.java index 3b9a740a448f..dfe721b681ac 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/search/BlockScoreQueryWrapper.java +++ b/lucene/test-framework/src/java/org/apache/lucene/search/BlockScoreQueryWrapper.java @@ -204,6 +204,11 @@ public void extractTerms(Set terms) { inWeight.extractTerms(terms); } + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return inWeight.matches(context, doc); + } + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { return inWeight.explain(context, doc); diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/LTRScoringQuery.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/LTRScoringQuery.java index c143d81f2273..779c8eb9aad2 100644 --- a/solr/contrib/ltr/src/java/org/apache/solr/ltr/LTRScoringQuery.java +++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/LTRScoringQuery.java @@ -39,6 +39,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -450,6 +451,11 @@ public void extractTerms(Set terms) { } } + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, "*"); // TODO is there a way of reporting matches that makes sense here? + } + protected void reset() { for (int i = 0; i < extractedFeatureWeights.length;++i){ int featId = extractedFeatureWeights[i].getIndex(); diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/Feature.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/Feature.java index 066d28125017..12232986aa78 100644 --- a/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/Feature.java +++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/Feature.java @@ -26,6 +26,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; @@ -228,6 +229,11 @@ public float getDefaultValue() { public abstract FeatureScorer scorer(LeafReaderContext context) throws IOException; + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, name); // TODO is there a way of reporting matches that makes sense here? + } + @Override public boolean isCacheable(LeafReaderContext ctx) { return false; diff --git a/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java b/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java index 1b81c7f5ec45..8676743505aa 100644 --- a/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java +++ b/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java @@ -25,8 +25,8 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermStates; import org.apache.lucene.index.TermState; +import org.apache.lucene.index.TermStates; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.BooleanClause; @@ -38,6 +38,7 @@ import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -485,6 +486,11 @@ public BulkScorer bulkScorer(LeafReaderContext context) throws IOException { } } + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, field); // TODO is there a way of reporting matches that makes sense here? + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { final SegState weightOrBitSet = getSegState(context); diff --git a/solr/core/src/java/org/apache/solr/schema/LatLonType.java b/solr/core/src/java/org/apache/solr/schema/LatLonType.java index 9f9dcd18dfb8..336c06e495ba 100644 --- a/solr/core/src/java/org/apache/solr/schema/LatLonType.java +++ b/solr/core/src/java/org/apache/solr/schema/LatLonType.java @@ -15,6 +15,7 @@ * limitations under the License. */ package org.apache.solr.schema; + import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -33,6 +34,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -338,6 +340,11 @@ public boolean isCacheable(LeafReaderContext ctx) { return false; } + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, "*"); // TODO is there a way of reporting matches that makes sense here? + } + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { return ((SpatialScorer)scorer(context)).explain(super.explain(context, doc), doc); diff --git a/solr/core/src/java/org/apache/solr/search/Filter.java b/solr/core/src/java/org/apache/solr/search/Filter.java index 3af83e2cfdfe..0520d059ecd0 100644 --- a/solr/core/src/java/org/apache/solr/search/Filter.java +++ b/solr/core/src/java/org/apache/solr/search/Filter.java @@ -26,6 +26,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -95,6 +96,11 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo @Override public void extractTerms(Set terms) {} + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, "*"); // TODO is there a way of reporting matches that makes sense here? + } + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { final Scorer scorer = scorer(context); diff --git a/solr/core/src/java/org/apache/solr/search/GraphTermsQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/GraphTermsQParserPlugin.java index d1f7ff2e4707..57e22c7b2496 100644 --- a/solr/core/src/java/org/apache/solr/search/GraphTermsQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/GraphTermsQParserPlugin.java @@ -37,8 +37,8 @@ import org.apache.lucene.index.PrefixCodedTerms; import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermStates; import org.apache.lucene.index.TermState; +import org.apache.lucene.index.TermStates; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.BulkScorer; @@ -49,6 +49,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -275,6 +276,11 @@ public void extractTerms(Set terms) { // order to protect highlighters } + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, field); // TODO is there a way of reporting matches that makes sense here? + } + private WeightOrDocIdSet rewrite(LeafReaderContext context) throws IOException { final LeafReader reader = context.reader(); Terms terms = reader.terms(field); @@ -613,6 +619,11 @@ public final Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, fl return new ConstantScoreWeight(this, boost) { Filter filter; + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, field); // TODO is there a way of reporting matches that makes sense here? + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { if (filter == null) { diff --git a/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java index ebf0ebcee782..5dd3f2f0b898 100644 --- a/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java @@ -35,6 +35,7 @@ import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -239,6 +240,11 @@ public void close() { Filter filter; + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, "*"); // TODO is there a way of reporting matches that makes sense here? + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { if (filter == null) { diff --git a/solr/core/src/java/org/apache/solr/search/SolrConstantScoreQuery.java b/solr/core/src/java/org/apache/solr/search/SolrConstantScoreQuery.java index 1f81e1e61d60..cf5da3c87ad6 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrConstantScoreQuery.java +++ b/solr/core/src/java/org/apache/solr/search/SolrConstantScoreQuery.java @@ -27,6 +27,7 @@ import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -92,6 +93,11 @@ public ConstantWeight(IndexSearcher searcher, float boost) throws IOException { ((SolrFilter)filter).createWeight(context, searcher); } + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, "*"); // TODO is there a way of reporting matches that makes sense here? + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { DocIdSet docIdSet = filter instanceof SolrFilter ? ((SolrFilter)filter).getDocIdSet(this.context, context, null) : filter.getDocIdSet(context, null); diff --git a/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java b/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java index 4402a2667479..a9d2716d3ffc 100644 --- a/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java +++ b/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java @@ -33,6 +33,7 @@ import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -286,7 +287,12 @@ public boolean isCacheable(LeafReaderContext ctx) { public void extractTerms(Set terms) { // NoOp for now , not used.. / supported } - + + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, collectSchemaField.getName()); // TODO is there a way of reporting matches that makes sense here? + } + } private static class GraphScorer extends Scorer { diff --git a/solr/core/src/java/org/apache/solr/update/DeleteByQueryWrapper.java b/solr/core/src/java/org/apache/solr/update/DeleteByQueryWrapper.java index 1c87a39b2a1c..3262711b3456 100644 --- a/solr/core/src/java/org/apache/solr/update/DeleteByQueryWrapper.java +++ b/solr/core/src/java/org/apache/solr/update/DeleteByQueryWrapper.java @@ -26,6 +26,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -77,6 +78,11 @@ public void extractTerms(Set terms) { throw new UnsupportedOperationException(); } + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return Matches.emptyMatches(context, doc, this, "*"); // TODO is there a way of reporting matches that makes sense here? + } + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { throw new UnsupportedOperationException(); } diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSortRandom.java b/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSortRandom.java index 58069cd060bb..42be74a8118f 100644 --- a/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSortRandom.java +++ b/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSortRandom.java @@ -43,6 +43,7 @@ import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -271,6 +272,11 @@ public RandomQuery(long seed, float density, List docValues) { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return null; + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { Random random = new Random(seed ^ context.docBase); From 7714fb582f21921c56f67a61070f983b41175ddd Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Thu, 5 Apr 2018 14:29:29 +0100 Subject: [PATCH 20/45] more wippyness --- .../matchhighlight/SnippetCollector.java | 15 +-- .../SourceAwareMatchesIterator.java | 104 ++++++++++++++++-- 2 files changed, 101 insertions(+), 18 deletions(-) diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SnippetCollector.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SnippetCollector.java index d4464e8afa13..b6f9c1e0febe 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SnippetCollector.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SnippetCollector.java @@ -18,18 +18,13 @@ package org.apache.lucene.search.matchhighlight; import org.apache.lucene.document.Document; -import org.apache.lucene.index.StoredFieldVisitor; -public class SnippetCollector { - public Document getHighlights() { - return null; - } +public interface SnippetCollector { - public boolean needsField(String name) { - return false; - } + Document getHighlights(); - public void collectSnippets(SourceAwareMatches matches, String name, byte[] value) { + boolean needsField(String name); + + void collectSnippets(SourceAwareMatches matches, String name, byte[] value); - } } diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SourceAwareMatchesIterator.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SourceAwareMatchesIterator.java index c46820d3f003..1466c63886f2 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SourceAwareMatchesIterator.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SourceAwareMatchesIterator.java @@ -17,21 +17,32 @@ package org.apache.lucene.search.matchhighlight; +import java.io.ByteArrayInputStream; +import java.io.Closeable; import java.io.IOException; +import java.io.InputStreamReader; import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; +import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.util.BytesRef; -public interface SourceAwareMatchesIterator extends MatchesIterator { +public interface SourceAwareMatchesIterator extends MatchesIterator, Closeable { void addSource(byte[] source); static SourceAwareMatchesIterator wrapOffsets(MatchesIterator in) { return new SourceAwareMatchesIterator() { @Override - public void addSource(byte[] source) { + public void close() throws IOException { + // no-op + } + @Override + public void addSource(byte[] source) { + // no-op - offsets already provided } @Override @@ -66,12 +77,89 @@ public BytesRef term() { }; } - static SourceAwareMatchesIterator fromTokenStream(MatchesIterator in, Analyzer analyzer) { - // build a TokenStream in setSource(), adding count * analyzer.getPositionIncrementGap - // add an OffsetsAttribute - // when in.next() is called, advance the ts until we're at the same position - // return the values from the offset attribute - return null; + static SourceAwareMatchesIterator fromTokenStream(MatchesIterator in, String field, Analyzer analyzer) { + return new SourceAwareMatchesIterator() { + + int sourceCount = -1; + int tsPosition = 0; + TokenStream ts; + OffsetAttribute offsetAttribute; + PositionIncrementAttribute posIncAttribute; + + int startPosition, endPosition, startOffset, endOffset; + + @Override + public void addSource(byte[] source) { + sourceCount++; + if (sourceCount > 0) { + tsPosition += analyzer.getPositionIncrementGap(field); + } + ts = analyzer.tokenStream(field, new InputStreamReader(new ByteArrayInputStream(source))); + offsetAttribute = ts.getAttribute(OffsetAttribute.class); + posIncAttribute = ts.getAttribute(PositionIncrementAttribute.class); + } + + @Override + public boolean next() throws IOException { + boolean next = in.next(); + if (next == false) { + return false; + } + startPosition = in.startPosition(); + if (advanceTokenStream(startPosition) == false) { + return false; + } + startOffset = offsetAttribute.startOffset(); + endPosition = in.endPosition(); + if (advanceTokenStream(endPosition) == false) { + return false; + } + endOffset = offsetAttribute.endOffset(); + return true; + } + + private boolean advanceTokenStream(int targetPos) throws IOException { + while (tsPosition < targetPos) { + if (ts.incrementToken() == false) { + return false; + } + tsPosition += posIncAttribute.getPositionIncrement(); + } + return true; + } + + @Override + public int startPosition() { + return startPosition; + } + + @Override + public int endPosition() { + return endPosition; + } + + @Override + public int startOffset() throws IOException { + return startOffset; + } + + @Override + public int endOffset() throws IOException { + return endOffset; + } + + @Override + public BytesRef term() { + return in.term(); + } + + @Override + public void close() throws IOException { + if (ts != null) { + ts.close(); + } + } + }; } } From af9634f4d3b6ecca70fd8fea1f479fa6064fdc88 Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Wed, 4 Apr 2018 13:33:42 +0200 Subject: [PATCH 21/45] LUCENE-8236: Filter duplicated points when creating GeoPath shapes to avoid creation of bogus planes. --- lucene/CHANGES.txt | 3 +++ .../lucene/spatial3d/geom/GeoPathFactory.java | 22 ++++++++++++++++-- .../lucene/spatial3d/geom/GeoPathTest.java | 23 +++++++++++++++++++ 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 1f8398014332..95d8738cf8b8 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -120,6 +120,9 @@ Bug Fixes * LUCENE-8234: Fixed bug in how spatial relationship is computed for GeoStandardCircle when it covers the whole world. (Ignacio Vera) +* LUCENE-8236: Filter duplicated points when creating GeoPath shapes to + avoid creation of bogus planes. (Ignacio Vera) + Other * LUCENE-8228: removed obsolete IndexDeletionPolicy clone() requirements from diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPathFactory.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPathFactory.java index 2ca132f1244a..6389f57b70a3 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPathFactory.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPathFactory.java @@ -16,6 +16,9 @@ */ package org.apache.lucene.spatial3d.geom; +import java.util.ArrayList; +import java.util.List; + /** * Class which constructs a GeoPath representing an arbitrary path. * @@ -34,9 +37,24 @@ private GeoPathFactory() { */ public static GeoPath makeGeoPath(final PlanetModel planetModel, final double maxCutoffAngle, final GeoPoint[] pathPoints) { if (maxCutoffAngle < Vector.MINIMUM_ANGULAR_RESOLUTION) { - return new GeoDegeneratePath(planetModel, pathPoints); + return new GeoDegeneratePath(planetModel, filterPoints(pathPoints)); + } + return new GeoStandardPath(planetModel, maxCutoffAngle, filterPoints(pathPoints)); + } + + /** Filter duplicate points. + * @param pathPoints with the arras of points. + * @return the filtered array. + */ + private static GeoPoint[] filterPoints(final GeoPoint[] pathPoints) { + final List noIdenticalPoints = new ArrayList<>(pathPoints.length); + for (int i = 0; i < pathPoints.length - 1; i++) { + if (!pathPoints[i].isNumericallyIdentical(pathPoints[i + 1])) { + noIdenticalPoints.add(pathPoints[i]); + } } - return new GeoStandardPath(planetModel, maxCutoffAngle, pathPoints); + noIdenticalPoints.add(pathPoints[pathPoints.length - 1]); + return noIdenticalPoints.toArray(new GeoPoint[noIdenticalPoints.size()]); } } diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPathTest.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPathTest.java index f6d14f2b6a00..93f90f4a67f5 100755 --- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPathTest.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPathTest.java @@ -379,4 +379,27 @@ public void testInterpolation2() { } + @Test + public void testIdenticalPoints() { + PlanetModel planetModel = PlanetModel.WGS84; + GeoPoint point1 = new GeoPoint(planetModel, 1.5707963267948963, -2.4818290647609542E-148); + GeoPoint point2 = new GeoPoint(planetModel, 1.570796326794895, -3.5E-323); + GeoPoint point3 = new GeoPoint(planetModel,4.4E-323, -3.1415926535897896); + GeoPath path = GeoPathFactory.makeGeoPath(planetModel, 0, new GeoPoint[] {point1, point2, point3}); + GeoPoint point = new GeoPoint(planetModel, -1.5707963267948952,2.369064805649877E-284); + //If not filtered the point is wrongly in set + assertFalse(path.isWithin(point)); + //If not filtered it throws error + path = GeoPathFactory.makeGeoPath(planetModel, 1e-6, new GeoPoint[] {point1, point2, point3}); + assertFalse(path.isWithin(point)); + + GeoPoint point4 = new GeoPoint(planetModel, 1.5, 0); + GeoPoint point5 = new GeoPoint(planetModel, 1.5, 0); + GeoPoint point6 = new GeoPoint(planetModel,4.4E-323, -3.1415926535897896); + //If not filtered creates a degenerated Vector + path = GeoPathFactory.makeGeoPath(planetModel, 0, new GeoPoint[] {point4, point5, point6}); + path = GeoPathFactory.makeGeoPath(planetModel, 0.5, new GeoPoint[] {point4, point5, point6}); + + } + } From 80bbc35fb0bc02a03b591b725251ce4d1696ba25 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Wed, 4 Apr 2018 13:44:17 +0200 Subject: [PATCH 22/45] LUCENE-8233: Add support for soft deletes to IndexWriter This change adds support for soft deletes as a fully supported feature by the index writer. Soft deletes are accounted for inside the index writer and therefor also by merge policies. This change also adds a SoftDeletesRetentionMergePolicy that allows users to selectively carry over soft_deleted document across merges for renention policies. The merge policy selects documents that should be kept around in the merged segment based on a user provided query. --- lucene/CHANGES.txt | 6 + .../lucene/index/BufferedUpdatesStream.java | 12 +- .../lucene/index/FrozenBufferedUpdates.java | 2 +- .../org/apache/lucene/index/IndexWriter.java | 51 +-- .../lucene/index/IndexWriterConfig.java | 29 ++ .../lucene/index/LiveIndexWriterConfig.java | 12 + .../org/apache/lucene/index/MergePolicy.java | 8 + .../lucene/index/MergePolicyWrapper.java | 4 + .../apache/lucene/index/NoMergePolicy.java | 7 +- .../apache/lucene/index/PendingDeletes.java | 97 ++++-- .../lucene/index/PendingSoftDeletes.java | 157 +++++++++ .../org/apache/lucene/index/ReaderUtil.java | 2 - .../lucene/index/ReadersAndUpdates.java | 19 +- .../SoftDeletesRetentionMergePolicy.java | 163 +++++++++ .../lucene/index/StandardDirectoryReader.java | 2 +- .../search/DocValuesFieldExistsQuery.java | 49 +-- .../src/java/org/apache/lucene/util/Bits.java | 10 +- .../apache/lucene/index/TestIndexWriter.java | 189 ++++------- .../lucene/index/TestIndexWriterConfig.java | 1 + .../index/TestIndexWriterOnDiskFull.java | 11 +- .../index/TestIndexingSequenceNumbers.java | 6 +- .../apache/lucene/index/TestMultiFields.java | 11 +- .../lucene/index/TestPendingDeletes.java | 10 +- .../lucene/index/TestPendingSoftDeletes.java | 232 +++++++++++++ .../TestSoftDeletesRetentionMergePolicy.java | 312 ++++++++++++++++++ .../apache/lucene/index/TestStressNRT.java | 7 +- .../TestIDVersionPostingsFormat.java | 28 +- .../asserting/AssertingLiveDocsFormat.java | 2 +- .../lucene/index/RandomIndexWriter.java | 79 ++--- 29 files changed, 1225 insertions(+), 293 deletions(-) create mode 100644 lucene/core/src/java/org/apache/lucene/index/PendingSoftDeletes.java create mode 100644 lucene/core/src/java/org/apache/lucene/index/SoftDeletesRetentionMergePolicy.java create mode 100644 lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java create mode 100644 lucene/core/src/test/org/apache/lucene/index/TestSoftDeletesRetentionMergePolicy.java diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 95d8738cf8b8..84e242d5e454 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -115,6 +115,12 @@ New Features searches based on minimum-interval semantics. (Alan Woodward, Adrien Grand, Jim Ferenczi, Simon Willnauer) +* LUCENE-8233: Add support for soft deletes to IndexWriter delete accounting. + Soft deletes are accounted for inside the index writer and therefor also + by merge policies. A SoftDeletesRetentionMergePolicy is added that allows + to selectively carry over soft_deleted document across merges for retention + policies (Simon Willnauer, Mike McCandless, Robert Muir) + Bug Fixes * LUCENE-8234: Fixed bug in how spatial relationship is computed for diff --git a/lucene/core/src/java/org/apache/lucene/index/BufferedUpdatesStream.java b/lucene/core/src/java/org/apache/lucene/index/BufferedUpdatesStream.java index 63001d4d0f5e..78fe9509620b 100644 --- a/lucene/core/src/java/org/apache/lucene/index/BufferedUpdatesStream.java +++ b/lucene/core/src/java/org/apache/lucene/index/BufferedUpdatesStream.java @@ -27,7 +27,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; -import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.IOContext; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.BytesRef; @@ -63,7 +62,6 @@ class BufferedUpdatesStream implements Accountable { private final AtomicLong bytesUsed = new AtomicLong(); private final AtomicInteger numTerms = new AtomicInteger(); private final IndexWriter writer; - private boolean closed; public BufferedUpdatesStream(IndexWriter writer) { this.writer = writer; @@ -122,12 +120,6 @@ public long ramBytesUsed() { return bytesUsed.get(); } - private synchronized void ensureOpen() { - if (closed) { - throw new AlreadyClosedException("already closed"); - } - } - public static class ApplyDeletesResult { // True if any actual deletes took place: @@ -300,8 +292,6 @@ public String toString() { /** Opens SegmentReader and inits SegmentState for each segment. */ public SegmentState[] openSegmentStates(IndexWriter.ReaderPool pool, List infos, Set alreadySeenSegments, long delGen) throws IOException { - ensureOpen(); - List segStates = new ArrayList<>(); try { for (SegmentCommitInfo info : infos) { @@ -334,7 +324,7 @@ public ApplyDeletesResult closeSegmentStates(IndexWriter.ReaderPool pool, Segmen totDelCount += segState.rld.getPendingDeleteCount() - segState.startDelCount; int fullDelCount = segState.rld.info.getDelCount() + segState.rld.getPendingDeleteCount(); assert fullDelCount <= segState.rld.info.info.maxDoc() : fullDelCount + " > " + segState.rld.info.info.maxDoc(); - if (segState.rld.isFullyDeleted()) { + if (segState.rld.isFullyDeleted() && writer.getConfig().mergePolicy.keepFullyDeletedSegment(segState.reader) == false) { if (allDeleted == null) { allDeleted = new ArrayList<>(); } diff --git a/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedUpdates.java b/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedUpdates.java index 1636319bfc1f..f7d16c47d702 100644 --- a/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedUpdates.java +++ b/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedUpdates.java @@ -412,7 +412,7 @@ private void finishApply(IndexWriter writer, BufferedUpdatesStream.SegmentState[ writer.checkpoint(); } - if (writer.keepFullyDeletedSegments == false && result.allDeleted != null) { + if (result.allDeleted != null) { if (infoStream.isEnabled("IW")) { infoStream.message("IW", "drop 100% deleted segments: " + writer.segString(result.allDeleted)); } diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java index 2e141667a20b..43051769cd69 100644 --- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java +++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java @@ -842,7 +842,7 @@ public synchronized ReadersAndUpdates get(SegmentCommitInfo info, boolean create if (create == false) { return null; } - rld = new ReadersAndUpdates(segmentInfos.getIndexCreatedVersionMajor(), info, null, new PendingDeletes(null, info)); + rld = new ReadersAndUpdates(segmentInfos.getIndexCreatedVersionMajor(), info, newPendingDeletes(info)); // Steal initial reference: readerMap.put(info, rld); } else { @@ -884,6 +884,7 @@ public int numDeletedDocs(SegmentCommitInfo info) { if (rld != null) { delCount += rld.getPendingDeleteCount(); } + assert delCount <= info.info.maxDoc(): "delCount: " + delCount + " maxDoc: " + info.info.maxDoc(); return delCount; } @@ -1151,7 +1152,7 @@ public IndexWriter(Directory d, IndexWriterConfig conf) throws IOException { LeafReaderContext leaf = leaves.get(i); SegmentReader segReader = (SegmentReader) leaf.reader(); SegmentReader newReader = new SegmentReader(segmentInfos.info(i), segReader, segReader.getLiveDocs(), segReader.numDocs()); - readerPool.readerMap.put(newReader.getSegmentInfo(), new ReadersAndUpdates(segmentInfos.getIndexCreatedVersionMajor(), newReader, new PendingDeletes(newReader, newReader.getSegmentInfo()))); + readerPool.readerMap.put(newReader.getSegmentInfo(), new ReadersAndUpdates(segmentInfos.getIndexCreatedVersionMajor(), newReader, newPendingDeletes(newReader, newReader.getSegmentInfo()))); } // We always assume we are carrying over incoming changes when opening from reader: @@ -1641,7 +1642,7 @@ public synchronized long tryDeleteDocument(IndexReader readerIn, int docID) thro if (rld != null) { synchronized(bufferedUpdatesStream) { if (rld.delete(docID)) { - if (rld.isFullyDeleted()) { + if (isFullyDeleted(rld)) { dropDeletedSegment(rld.info); checkpoint(); } @@ -4003,21 +4004,21 @@ synchronized private boolean commitMerge(MergePolicy.OneMerge merge, MergeState final boolean allDeleted = merge.segments.size() == 0 || merge.info.info.maxDoc() == 0 || - (mergedUpdates != null && mergedUpdates.isFullyDeleted()); + (mergedUpdates != null && isFullyDeleted(mergedUpdates)); if (infoStream.isEnabled("IW")) { if (allDeleted) { - infoStream.message("IW", "merged segment " + merge.info + " is 100% deleted" + (keepFullyDeletedSegments ? "" : "; skipping insert")); + infoStream.message("IW", "merged segment " + merge.info + " is 100% deleted; skipping insert"); } } - final boolean dropSegment = allDeleted && !keepFullyDeletedSegments; + final boolean dropSegment = allDeleted; // If we merged no segments then we better be dropping // the new segment: assert merge.segments.size() > 0 || dropSegment; - assert merge.info.info.maxDoc() != 0 || keepFullyDeletedSegments || dropSegment; + assert merge.info.info.maxDoc() != 0 || dropSegment; if (mergedUpdates != null) { boolean success = false; @@ -4716,19 +4717,6 @@ private synchronized void doWait() { } } - boolean keepFullyDeletedSegments; - - /** Only for testing. - * - * @lucene.internal */ - void setKeepFullyDeletedSegments(boolean v) { - keepFullyDeletedSegments = v; - } - - boolean getKeepFullyDeletedSegments() { - return keepFullyDeletedSegments; - } - // called only from assert private boolean filesExist(SegmentInfos toSync) throws IOException { @@ -5207,4 +5195,27 @@ private long adjustPendingNumDocs(long numDocs) { assert count >= 0 : "pendingNumDocs is negative: " + count; return count; } + + private PendingDeletes newPendingDeletes(SegmentCommitInfo info) { + String softDeletesField = config.getSoftDeletesField(); + return softDeletesField == null ? new PendingDeletes(info) : new PendingSoftDeletes(softDeletesField, info); + } + + private PendingDeletes newPendingDeletes(SegmentReader reader, SegmentCommitInfo info) { + String softDeletesField = config.getSoftDeletesField(); + return softDeletesField == null ? new PendingDeletes(reader, info) : new PendingSoftDeletes(softDeletesField, reader, info); + } + + final boolean isFullyDeleted(ReadersAndUpdates readersAndUpdates) throws IOException { + if (readersAndUpdates.isFullyDeleted()) { + SegmentReader reader = readersAndUpdates.getReader(IOContext.READ); + try { + return config.mergePolicy.keepFullyDeletedSegment(reader) == false; + } finally { + readersAndUpdates.release(reader); + } + } + return false; + } + } diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java index 997a6862522b..d657d52102a3 100644 --- a/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java +++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java @@ -25,6 +25,7 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.codecs.Codec; +import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexWriter.IndexReaderWarmer; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; @@ -484,5 +485,33 @@ public String toString() { public IndexWriterConfig setCheckPendingFlushUpdate(boolean checkPendingFlushOnUpdate) { return (IndexWriterConfig) super.setCheckPendingFlushUpdate(checkPendingFlushOnUpdate); } + + /** + * Sets the soft deletes field. A soft delete field in lucene is a doc-values field that marks a document as soft-deleted if a + * document has at least one value in that field. If a document is marked as soft-deleted the document is treated as + * if it has been hard-deleted through the IndexWriter API ({@link IndexWriter#deleteDocuments(Term...)}. + * Merges will reclaim soft-deleted as well as hard-deleted documents and index readers obtained from the IndexWriter + * will reflect all deleted documents in it's live docs. If soft-deletes are used documents must be indexed via + * {@link IndexWriter#softUpdateDocument(Term, Iterable, Field...)}. Deletes are applied via + * {@link IndexWriter#updateDocValues(Term, Field...)}. + * + * Soft deletes allow to retain documents across merges if the merge policy modifies the live docs of a merge reader. + * {@link SoftDeletesRetentionMergePolicy} for instance allows to specify an arbitrary query to mark all documents + * that should survive the merge. This can be used to for example keep all document modifications for a certain time + * interval or the last N operations if some kind of sequence ID is available in the index. + * + * Currently there is no API support to un-delete a soft-deleted document. In oder to un-delete the document must be + * re-indexed using {@link IndexWriter#softUpdateDocument(Term, Iterable, Field...)}. + * + * The default value for this is null which disables soft-deletes. If soft-deletes are enabled documents + * can still be hard-deleted. Hard-deleted documents will won't considered as soft-deleted even if they have + * a value in the soft-deletes field. + * + * @see #getSoftDeletesField() + */ + public IndexWriterConfig setSoftDeletesField(String softDeletesField) { + this.softDeletesField = softDeletesField; + return this; + } } diff --git a/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java b/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java index af8ff1531f89..016e88066144 100644 --- a/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java +++ b/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java @@ -106,6 +106,9 @@ public class LiveIndexWriterConfig { /** if an indexing thread should check for pending flushes on update in order to help out on a full flush*/ protected volatile boolean checkPendingFlushOnUpdate = true; + /** soft deletes field */ + protected String softDeletesField = null; + // used by IndexWriterConfig LiveIndexWriterConfig(Analyzer analyzer) { this.analyzer = analyzer; @@ -452,6 +455,14 @@ public LiveIndexWriterConfig setCheckPendingFlushUpdate(boolean checkPendingFlus return this; } + /** + * Returns the soft deletes field or null if soft-deletes are disabled. + * See {@link IndexWriterConfig#setSoftDeletesField(String)} for details. + */ + public String getSoftDeletesField() { + return softDeletesField; + } + @Override public String toString() { StringBuilder sb = new StringBuilder(); @@ -475,6 +486,7 @@ public String toString() { sb.append("commitOnClose=").append(getCommitOnClose()).append("\n"); sb.append("indexSort=").append(getIndexSort()).append("\n"); sb.append("checkPendingFlushOnUpdate=").append(isCheckPendingFlushOnUpdate()).append("\n"); + sb.append("softDeletesField=").append(getSoftDeletesField()).append("\n"); return sb.toString(); } } diff --git a/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java b/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java index d9a0ab83ee81..c0d9748b6fde 100644 --- a/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java +++ b/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java @@ -604,4 +604,12 @@ public void setMaxCFSSegmentSizeMB(double v) { v *= 1024 * 1024; this.maxCFSSegmentSize = v > Long.MAX_VALUE ? Long.MAX_VALUE : (long) v; } + + /** + * Returns true if the segment represented by the given CodecReader should be keep even if it's fully deleted. + * This is useful for testing of for instance if the merge policy implements retention policies for soft deletes. + */ + public boolean keepFullyDeletedSegment(CodecReader reader) throws IOException { + return false; + } } diff --git a/lucene/core/src/java/org/apache/lucene/index/MergePolicyWrapper.java b/lucene/core/src/java/org/apache/lucene/index/MergePolicyWrapper.java index c51cd00d7c6a..606f3c2bd457 100644 --- a/lucene/core/src/java/org/apache/lucene/index/MergePolicyWrapper.java +++ b/lucene/core/src/java/org/apache/lucene/index/MergePolicyWrapper.java @@ -86,4 +86,8 @@ public String toString() { return getClass().getSimpleName() + "(" + in + ")"; } + @Override + public boolean keepFullyDeletedSegment(CodecReader reader) throws IOException { + return in.keepFullyDeletedSegment(reader); + } } diff --git a/lucene/core/src/java/org/apache/lucene/index/NoMergePolicy.java b/lucene/core/src/java/org/apache/lucene/index/NoMergePolicy.java index ec309b8a2975..4387f250513d 100644 --- a/lucene/core/src/java/org/apache/lucene/index/NoMergePolicy.java +++ b/lucene/core/src/java/org/apache/lucene/index/NoMergePolicy.java @@ -67,7 +67,12 @@ public void setMaxCFSSegmentSizeMB(double v) { public void setNoCFSRatio(double noCFSRatio) { super.setNoCFSRatio(noCFSRatio); } - + + @Override + public boolean keepFullyDeletedSegment(CodecReader reader) throws IOException { + return super.keepFullyDeletedSegment(reader); + } + @Override public String toString() { return "NoMergePolicy"; diff --git a/lucene/core/src/java/org/apache/lucene/index/PendingDeletes.java b/lucene/core/src/java/org/apache/lucene/index/PendingDeletes.java index 74043f3f44a2..bce704cb6fc5 100644 --- a/lucene/core/src/java/org/apache/lucene/index/PendingDeletes.java +++ b/lucene/core/src/java/org/apache/lucene/index/PendingDeletes.java @@ -18,6 +18,7 @@ package org.apache.lucene.index; import java.io.IOException; +import java.util.List; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.LiveDocsFormat; @@ -31,57 +32,71 @@ /** * This class handles accounting and applying pending deletes for live segment readers */ -final class PendingDeletes { - private final SegmentCommitInfo info; +class PendingDeletes { + protected final SegmentCommitInfo info; // True if the current liveDocs is referenced by an // external NRT reader: - private boolean liveDocsShared; + protected boolean liveDocsShared; // Holds the current shared (readable and writable) // liveDocs. This is null when there are no deleted // docs, and it's copy-on-write (cloned whenever we need // to change it but it's been shared to an external NRT // reader). private Bits liveDocs; - private int pendingDeleteCount; + protected int pendingDeleteCount; + private boolean liveDocsInitialized; PendingDeletes(SegmentReader reader, SegmentCommitInfo info) { + this(info, reader.getLiveDocs(), true); + pendingDeleteCount = reader.numDeletedDocs() - info.getDelCount(); + } + + PendingDeletes(SegmentCommitInfo info) { + this(info, null, false); + } + + private PendingDeletes(SegmentCommitInfo info, Bits liveDocs, boolean liveDocsInitialized) { this.info = info; liveDocsShared = true; - liveDocs = reader != null ? reader.getLiveDocs() : null; - if (reader != null) { - pendingDeleteCount = reader.numDeletedDocs() - info.getDelCount(); - } else { - pendingDeleteCount = 0; - } + this.liveDocs = liveDocs; + pendingDeleteCount = 0; + this.liveDocsInitialized = liveDocsInitialized; } - /** - * Marks a document as deleted in this segment and return true if a document got actually deleted or - * if the document was already deleted. - */ - boolean delete(int docID) throws IOException { - assert info.info.maxDoc() > 0; + protected MutableBits getMutableBits() throws IOException { if (liveDocsShared) { // Copy on write: this means we've cloned a // SegmentReader sharing the current liveDocs // instance; must now make a private clone so we can // change it: LiveDocsFormat liveDocsFormat = info.info.getCodec().liveDocsFormat(); + MutableBits mutableBits; if (liveDocs == null) { - liveDocs = liveDocsFormat.newLiveDocs(info.info.maxDoc()); + mutableBits = liveDocsFormat.newLiveDocs(info.info.maxDoc()); } else { - liveDocs = liveDocsFormat.newLiveDocs(liveDocs); + mutableBits = liveDocsFormat.newLiveDocs(liveDocs); } + liveDocs = mutableBits; liveDocsShared = false; } + return (MutableBits) liveDocs; + } - assert liveDocs != null; - assert docID >= 0 && docID < liveDocs.length() : "out of bounds: docid=" + docID + " liveDocsLength=" + liveDocs.length() + " seg=" + info.info.name + " maxDoc=" + info.info.maxDoc(); + + /** + * Marks a document as deleted in this segment and return true if a document got actually deleted or + * if the document was already deleted. + */ + boolean delete(int docID) throws IOException { + assert info.info.maxDoc() > 0; + MutableBits mutableBits = getMutableBits(); + assert mutableBits != null; + assert docID >= 0 && docID < mutableBits.length() : "out of bounds: docid=" + docID + " liveDocsLength=" + mutableBits.length() + " seg=" + info.info.name + " maxDoc=" + info.info.maxDoc(); assert !liveDocsShared; - final boolean didDelete = liveDocs.get(docID); + final boolean didDelete = mutableBits.get(docID); if (didDelete) { - ((MutableBits) liveDocs).clear(docID); + mutableBits.clear(docID); pendingDeleteCount++; } return didDelete; @@ -114,10 +129,32 @@ int numPendingDeletes() { /** * Called once a new reader is opened for this segment ie. when deletes or updates are applied. */ - void onNewReader(SegmentReader reader, SegmentCommitInfo info) { - if (liveDocs == null) { - liveDocs = reader.getLiveDocs(); + void onNewReader(SegmentReader reader, SegmentCommitInfo info) throws IOException { + if (liveDocsInitialized == false) { + if (reader.hasDeletions()) { + // we only initialize this once either in the ctor or here + // if we use the live docs from a reader it has to be in a situation where we don't + // have any existing live docs + assert pendingDeleteCount == 0 : "pendingDeleteCount: " + pendingDeleteCount; + liveDocs = reader.getLiveDocs(); + assert liveDocs == null || assertCheckLiveDocs(liveDocs, info.info.maxDoc(), info.getDelCount()); + liveDocsShared = true; + + } + liveDocsInitialized = true; + } + } + + private boolean assertCheckLiveDocs(Bits bits, int expectedLength, int expectedDeleteCount) { + assert bits.length() == expectedLength; + int deletedCount = 0; + for (int i = 0; i < bits.length(); i++) { + if (bits.get(i) == false) { + deletedCount++; + } } + assert deletedCount == expectedDeleteCount : "deleted: " + deletedCount + " != expected: " + expectedDeleteCount; + return true; } /** @@ -188,6 +225,14 @@ boolean writeLiveDocs(Directory dir) throws IOException { * Returns true iff the segment represented by this {@link PendingDeletes} is fully deleted */ boolean isFullyDeleted() { - return info.getDelCount() + pendingDeleteCount == info.info.maxDoc(); + return info.getDelCount() + numPendingDeletes() == info.info.maxDoc(); + } + + /** + * Called before the given DocValuesFieldUpdates are applied + * @param info the field to apply + * @param fieldUpdates the field updates + */ + void onDocValuesUpdate(FieldInfo info, List fieldUpdates) throws IOException { } } diff --git a/lucene/core/src/java/org/apache/lucene/index/PendingSoftDeletes.java b/lucene/core/src/java/org/apache/lucene/index/PendingSoftDeletes.java new file mode 100644 index 000000000000..1f6c2ef25876 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/index/PendingSoftDeletes.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.index; + +import java.io.IOException; +import java.util.List; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.DocValuesFieldExistsQuery; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.MutableBits; + +final class PendingSoftDeletes extends PendingDeletes { + + private final String field; + private long dvGeneration = -2; + private final PendingDeletes hardDeletes; + + PendingSoftDeletes(String field, SegmentCommitInfo info) { + super(info); + this.field = field; + hardDeletes = new PendingDeletes(info); + } + + PendingSoftDeletes(String field, SegmentReader reader, SegmentCommitInfo info) { + super(reader, info); + this.field = field; + hardDeletes = new PendingDeletes(reader, info); + } + + @Override + boolean delete(int docID) throws IOException { + MutableBits mutableBits = getMutableBits(); // we need to fetch this first it might be a shared instance with hardDeletes + if (hardDeletes.delete(docID)) { + if (mutableBits.get(docID)) { // delete it here too! + mutableBits.clear(docID); + assert hardDeletes.delete(docID) == false; + } else { + // if it was deleted subtract the delCount + pendingDeleteCount--; + } + return true; + } + return false; + } + + @Override + int numPendingDeletes() { + return super.numPendingDeletes() + hardDeletes.numPendingDeletes(); + } + + @Override + void onNewReader(SegmentReader reader, SegmentCommitInfo info) throws IOException { + super.onNewReader(reader, info); + hardDeletes.onNewReader(reader, info); + if (dvGeneration != info.getDocValuesGen()) { // only re-calculate this if we haven't seen this generation + final DocIdSetIterator iterator = DocValuesFieldExistsQuery.getDocValuesDocIdSetIterator(field, reader); + if (iterator == null) { // nothing is deleted we don't have a soft deletes field in this segment + this.pendingDeleteCount = 0; + } else { + assert info.info.maxDoc() > 0 : "maxDoc is 0"; + applyUpdates(iterator); + } + dvGeneration = info.getDocValuesGen(); + } + assert numPendingDeletes() + info.getDelCount() <= info.info.maxDoc() : + numPendingDeletes() + " + " + info.getDelCount() + " > " + info.info.maxDoc(); + } + + @Override + boolean writeLiveDocs(Directory dir) throws IOException { + // delegate the write to the hard deletes - it will only write if somebody used it. + return hardDeletes.writeLiveDocs(dir); + } + + @Override + void reset() { + dvGeneration = -2; + super.reset(); + hardDeletes.reset(); + } + + private void applyUpdates(DocIdSetIterator iterator) throws IOException { + final MutableBits mutableBits = getMutableBits(); + int newDeletes = 0; + int docID; + while ((docID = iterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { + if (mutableBits.get(docID)) { // doc is live - clear it + mutableBits.clear(docID); + newDeletes++; + // now that we know we deleted it and we fully control the hard deletes we can do correct accounting + // below. + } + } + pendingDeleteCount += newDeletes; + } + + @Override + void onDocValuesUpdate(FieldInfo info, List updatesToApply) throws IOException { + if (field.equals(info.name)) { + assert dvGeneration < info.getDocValuesGen() : "we have seen this generation update already: " + dvGeneration + " vs. " + info.getDocValuesGen(); + DocValuesFieldUpdates.Iterator[] subs = new DocValuesFieldUpdates.Iterator[updatesToApply.size()]; + for(int i=0; i retentionQuerySupplier; + /** + * Creates a new {@link SoftDeletesRetentionMergePolicy} + * @param field the soft deletes field + * @param retentionQuerySupplier a query supplier for the retention query + * @param in the wrapped MergePolicy + */ + public SoftDeletesRetentionMergePolicy(String field, Supplier retentionQuerySupplier, MergePolicy in) { + super(in, toWrap -> new MergePolicy.OneMerge(toWrap.segments) { + @Override + public CodecReader wrapForMerge(CodecReader reader) throws IOException { + CodecReader wrapped = toWrap.wrapForMerge(reader); + Bits liveDocs = reader.getLiveDocs(); + if (liveDocs == null) { // no deletes - just keep going + return wrapped; + } + return applyRetentionQuery(field, retentionQuerySupplier.get(), wrapped); + } + }); + Objects.requireNonNull(field, "field must not be null"); + Objects.requireNonNull(retentionQuerySupplier, "retentionQuerySupplier must not be null"); + this.field = field; + this.retentionQuerySupplier = retentionQuerySupplier; + } + + @Override + public boolean keepFullyDeletedSegment(CodecReader reader) throws IOException { + Scorer scorer = getScorer(field, retentionQuerySupplier.get(), wrapLiveDocs(reader, null, reader.maxDoc())); + if (scorer != null) { + DocIdSetIterator iterator = scorer.iterator(); + boolean atLeastOneHit = iterator.nextDoc() != DocIdSetIterator.NO_MORE_DOCS; + return atLeastOneHit; + } + return super.keepFullyDeletedSegment(reader) ; + } + + // pkg private for testing + static CodecReader applyRetentionQuery(String softDeleteField, Query retentionQuery, CodecReader reader) throws IOException { + Bits liveDocs = reader.getLiveDocs(); + if (liveDocs == null) { // no deletes - just keep going + return reader; + } + CodecReader wrappedReader = wrapLiveDocs(reader, new Bits() { // only search deleted + @Override + public boolean get(int index) { + return liveDocs.get(index) == false; + } + + @Override + public int length() { + return liveDocs.length(); + } + }, reader.maxDoc() - reader.numDocs()); + Scorer scorer = getScorer(softDeleteField, retentionQuery, wrappedReader); + if (scorer != null) { + FixedBitSet mutableBits; + if (liveDocs instanceof FixedBitSet) { + mutableBits = ((FixedBitSet) liveDocs).clone(); + } else { // mainly if we have asserting codec + mutableBits = new FixedBitSet(liveDocs.length()); + for (int i = 0; i < liveDocs.length(); i++) { + if (liveDocs.get(i)) { + mutableBits.set(i); + } + } + } + DocIdSetIterator iterator = scorer.iterator(); + int numExtraLiveDocs = 0; + while (iterator.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { + if (mutableBits.getAndSet(iterator.docID()) == false) { + // if we bring one back to live we need to account for it + numExtraLiveDocs++; + } + } + assert reader.numDocs() + numExtraLiveDocs <= reader.maxDoc() : "numDocs: " + reader.numDocs() + " numExtraLiveDocs: " + numExtraLiveDocs + " maxDoc: " + reader.maxDoc(); + return wrapLiveDocs(reader, mutableBits, reader.numDocs() + numExtraLiveDocs); + } else { + return reader; + } + } + + private static Scorer getScorer(String softDeleteField, Query retentionQuery, CodecReader reader) throws IOException { + BooleanQuery.Builder builder = new BooleanQuery.Builder(); + builder.add(new DocValuesFieldExistsQuery(softDeleteField), BooleanClause.Occur.FILTER); + builder.add(retentionQuery, BooleanClause.Occur.FILTER); + IndexSearcher s = new IndexSearcher(reader); + s.setQueryCache(null); + Weight weight = s.createWeight(builder.build(), ScoreMode.COMPLETE_NO_SCORES, 1.0f); + return weight.scorer(reader.getContext()); + } + + /** + * Returns a codec reader with the given live docs + */ + private static CodecReader wrapLiveDocs(CodecReader reader, Bits liveDocs, int numDocs) { + return new FilterCodecReader(reader) { + @Override + public CacheHelper getCoreCacheHelper() { + return reader.getCoreCacheHelper(); + } + + @Override + public CacheHelper getReaderCacheHelper() { + return null; // we are altering live docs + } + + @Override + public Bits getLiveDocs() { + return liveDocs; + } + + @Override + public int numDocs() { + return numDocs; + } + }; + }} diff --git a/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java b/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java index f95ca82194c6..23fbb0473bf2 100644 --- a/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java +++ b/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java @@ -103,7 +103,7 @@ static DirectoryReader open(IndexWriter writer, SegmentInfos infos, boolean appl final ReadersAndUpdates rld = writer.readerPool.get(info, true); try { final SegmentReader reader = rld.getReadOnlyClone(IOContext.READ); - if (reader.numDocs() > 0 || writer.getKeepFullyDeletedSegments()) { + if (reader.numDocs() > 0 || writer.getConfig().mergePolicy.keepFullyDeletedSegment(reader)) { // Steal the ref: readers.add(reader); infosUpto++; diff --git a/lucene/core/src/java/org/apache/lucene/search/DocValuesFieldExistsQuery.java b/lucene/core/src/java/org/apache/lucene/search/DocValuesFieldExistsQuery.java index 22cb5aab781d..8f18cbb9ab6e 100644 --- a/lucene/core/src/java/org/apache/lucene/search/DocValuesFieldExistsQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/DocValuesFieldExistsQuery.java @@ -21,9 +21,7 @@ import java.util.Objects; import org.apache.lucene.index.DocValues; -import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; @@ -62,7 +60,7 @@ public String toString(String field) { } @Override - public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { + public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) { return new ConstantScoreWeight(this, boost) { @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { @@ -71,17 +69,33 @@ public Matches matches(LeafReaderContext context, int doc) throws IOException { @Override public Scorer scorer(LeafReaderContext context) throws IOException { - FieldInfos fieldInfos = context.reader().getFieldInfos(); - FieldInfo fieldInfo = fieldInfos.fieldInfo(field); - if (fieldInfo == null) { + DocIdSetIterator iterator = getDocValuesDocIdSetIterator(field, context.reader()); + if (iterator == null) { return null; } - DocValuesType dvType = fieldInfo.getDocValuesType(); - LeafReader reader = context.reader(); - DocIdSetIterator iterator; - switch(dvType) { + return new ConstantScoreScorer(this, score(), iterator); + } + + @Override + public boolean isCacheable(LeafReaderContext ctx) { + return DocValues.isCacheable(ctx, field); + } + + }; + } + + /** + * Returns a {@link DocIdSetIterator} from the given field or null if the field doesn't exist + * in the reader or if the reader has no doc values for the field. + */ + public static DocIdSetIterator getDocValuesDocIdSetIterator(String field, LeafReader reader) throws IOException { + FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(field); + final DocIdSetIterator iterator; + if (fieldInfo != null) { + switch (fieldInfo.getDocValuesType()) { case NONE: - return null; + iterator = null; + break; case NUMERIC: iterator = reader.getNumericDocValues(field); break; @@ -99,16 +113,9 @@ public Scorer scorer(LeafReaderContext context) throws IOException { break; default: throw new AssertionError(); - } - - return new ConstantScoreScorer(this, score(), iterator); } - - @Override - public boolean isCacheable(LeafReaderContext ctx) { - return DocValues.isCacheable(ctx, field); - } - - }; + return iterator; + } + return null; } } diff --git a/lucene/core/src/java/org/apache/lucene/util/Bits.java b/lucene/core/src/java/org/apache/lucene/util/Bits.java index 29935e737b87..1f9a7aa0770c 100644 --- a/lucene/core/src/java/org/apache/lucene/util/Bits.java +++ b/lucene/core/src/java/org/apache/lucene/util/Bits.java @@ -30,17 +30,17 @@ public interface Bits { * by this interface, just don't do it! * @return true if the bit is set, false otherwise. */ - public boolean get(int index); + boolean get(int index); /** Returns the number of bits in this set */ - public int length(); + int length(); - public static final Bits[] EMPTY_ARRAY = new Bits[0]; + Bits[] EMPTY_ARRAY = new Bits[0]; /** * Bits impl of the specified length with all bits set. */ - public static class MatchAllBits implements Bits { + class MatchAllBits implements Bits { final int len; public MatchAllBits(int len) { @@ -61,7 +61,7 @@ public int length() { /** * Bits impl of the specified length with no bits set. */ - public static class MatchNoBits implements Bits { + class MatchNoBits implements Bits { final int len; public MatchNoBits(int len) { diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java index a95a8e3bf15f..e45716de7bd7 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java @@ -22,7 +22,6 @@ import java.io.IOException; import java.io.PrintStream; import java.io.StringReader; -import java.io.UncheckedIOException; import java.net.URI; import java.nio.file.FileSystem; import java.nio.file.Files; @@ -88,7 +87,6 @@ import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.store.SimpleFSDirectory; import org.apache.lucene.store.SimpleFSLockFactory; -import org.apache.lucene.util.BitSet; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Constants; @@ -2223,14 +2221,21 @@ public void testHasUncommittedChanges() throws IOException { public void testMergeAllDeleted() throws IOException { Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random())); + AtomicBoolean keepFullyDeletedSegments = new AtomicBoolean(); + iwc.setMergePolicy(new MergePolicyWrapper(iwc.getMergePolicy()) { + @Override + public boolean keepFullyDeletedSegment(CodecReader reader) throws IOException { + return keepFullyDeletedSegments.get(); + } + }); final SetOnce iwRef = new SetOnce<>(); IndexWriter evilWriter = RandomIndexWriter.mockIndexWriter(random(), dir, iwc, new RandomIndexWriter.TestPoint() { @Override public void apply(String message) { if ("startCommitMerge".equals(message)) { - iwRef.get().setKeepFullyDeletedSegments(false); + keepFullyDeletedSegments.set(false); } else if ("startMergeInit".equals(message)) { - iwRef.get().setKeepFullyDeletedSegments(true); + keepFullyDeletedSegments.set(true); } } }); @@ -2958,94 +2963,10 @@ private static void waitForDocsInBuffers(IndexWriter w, int buffersWithDocs) { } } } - private static Bits getSoftDeletesLiveDocs(LeafReader reader, String field) { - try { - NumericDocValues softDelete = reader.getNumericDocValues(field); - if (softDelete != null) { - BitSet bitSet = BitSet.of(softDelete, reader.maxDoc()); - Bits inLiveDocs = reader.getLiveDocs() == null ? new Bits.MatchAllBits(reader.maxDoc()) : reader.getLiveDocs(); - Bits newliveDocs = new Bits() { - @Override - public boolean get(int index) { - return inLiveDocs.get(index) && bitSet.get(index) == false; - } - - @Override - public int length() { - return inLiveDocs.length(); - } - }; - return newliveDocs; - - } else { - return reader.getLiveDocs(); - } - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - private static DirectoryReader wrapSoftDeletes(DirectoryReader reader, String field) throws IOException { - return new FilterDirectoryReader(reader, new FilterDirectoryReader.SubReaderWrapper() { - @Override - public LeafReader wrap(LeafReader reader) { - Bits softDeletesLiveDocs = getSoftDeletesLiveDocs(reader, field); - int numDocs = getNumDocs(reader, softDeletesLiveDocs); - return new FilterLeafReader(reader) { - - @Override - public Bits getLiveDocs() { - return softDeletesLiveDocs; - } - - @Override - public CacheHelper getReaderCacheHelper() { - return in.getReaderCacheHelper(); - } - - @Override - public CacheHelper getCoreCacheHelper() { - return in.getCoreCacheHelper(); - } - - @Override - public int numDocs() { - return numDocs; - } - }; - } - }) { - @Override - protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) throws IOException { - return wrapSoftDeletes(in, field); - } - - @Override - public CacheHelper getReaderCacheHelper() { - return in.getReaderCacheHelper(); - } - }; - } - - private static int getNumDocs(LeafReader reader, Bits softDeletesLiveDocs) { - int numDocs; - if (softDeletesLiveDocs == reader.getLiveDocs()) { - numDocs = reader.numDocs(); - } else { - int tmp = 0; - for (int i = 0; i < softDeletesLiveDocs.length(); i++) { - if (softDeletesLiveDocs.get(i) ) { - tmp++; - } - } - numDocs = tmp; - } - return numDocs; - } public void testSoftUpdateDocuments() throws IOException { Directory dir = newDirectory(); - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig()); + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig().setSoftDeletesField("soft_delete")); expectThrows(IllegalArgumentException.class, () -> { writer.softUpdateDocument(null, new Document(), new NumericDocValuesField("soft_delete", 1)); }); @@ -3071,7 +2992,7 @@ public void testSoftUpdateDocuments() throws IOException { doc.add(new StringField("version", "2", Field.Store.YES)); Field field = new NumericDocValuesField("soft_delete", 1); writer.softUpdateDocument(new Term("id", "1"), doc, field); - DirectoryReader reader = wrapSoftDeletes(DirectoryReader.open(writer), "soft_delete"); + DirectoryReader reader = DirectoryReader.open(writer); assertEquals(2, reader.docFreq(new Term("id", "1"))); IndexSearcher searcher = new IndexSearcher(reader); TopDocs topDocs = searcher.search(new TermQuery(new Term("id", "1")), 10); @@ -3112,43 +3033,53 @@ public void testSoftUpdateDocuments() throws IOException { } public void testSoftUpdatesConcurrently() throws IOException, InterruptedException { + softUpdatesConcurrently(false); + } + + public void testSoftUpdatesConcurrentlyMixedDeletes() throws IOException, InterruptedException { + softUpdatesConcurrently(true); + } + + public void softUpdatesConcurrently(boolean mixDeletes) throws IOException, InterruptedException { Directory dir = newDirectory(); IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); + indexWriterConfig.setSoftDeletesField("soft_delete"); AtomicBoolean mergeAwaySoftDeletes = new AtomicBoolean(random().nextBoolean()); - indexWriterConfig.setMergePolicy(new OneMergeWrappingMergePolicy(indexWriterConfig.getMergePolicy(), towrap -> - new MergePolicy.OneMerge(towrap.segments) { - @Override - public CodecReader wrapForMerge(CodecReader reader) throws IOException { - if (mergeAwaySoftDeletes.get() == false) { - return towrap.wrapForMerge(reader); - } - Bits softDeletesLiveDocs = getSoftDeletesLiveDocs(reader, "soft_delete"); - int numDocs = getNumDocs(reader, softDeletesLiveDocs); - CodecReader wrapped = towrap.wrapForMerge(reader); - return new FilterCodecReader(wrapped) { + if (mixDeletes == false) { + indexWriterConfig.setMergePolicy(new OneMergeWrappingMergePolicy(indexWriterConfig.getMergePolicy(), towrap -> + new MergePolicy.OneMerge(towrap.segments) { @Override - public CacheHelper getCoreCacheHelper() { - return in.getCoreCacheHelper(); - } + public CodecReader wrapForMerge(CodecReader reader) throws IOException { + if (mergeAwaySoftDeletes.get()) { + return towrap.wrapForMerge(reader); + } else { + CodecReader wrapped = towrap.wrapForMerge(reader); + return new FilterCodecReader(wrapped) { + @Override + public CacheHelper getCoreCacheHelper() { + return in.getCoreCacheHelper(); + } - @Override - public CacheHelper getReaderCacheHelper() { - return in.getReaderCacheHelper(); - } + @Override + public CacheHelper getReaderCacheHelper() { + return in.getReaderCacheHelper(); + } - @Override - public Bits getLiveDocs() { - return softDeletesLiveDocs; - } + @Override + public Bits getLiveDocs() { + return null; // everything is live + } - @Override - public int numDocs() { - return numDocs; + @Override + public int numDocs() { + return maxDoc(); + } + }; + } } - }; - } - } - )); + } + )); + } IndexWriter writer = new IndexWriter(dir, indexWriterConfig); Thread[] threads = new Thread[2 + random().nextInt(3)]; CountDownLatch startLatch = new CountDownLatch(1); @@ -3165,13 +3096,21 @@ public int numDocs() { if (updateSeveralDocs) { Document doc = new Document(); doc.add(new StringField("id", id, Field.Store.YES)); - writer.softUpdateDocuments(new Term("id", id), Arrays.asList(doc, doc), - new NumericDocValuesField("soft_delete", 1)); + if (mixDeletes && random().nextBoolean()) { + writer.updateDocuments(new Term("id", id), Arrays.asList(doc, doc)); + } else { + writer.softUpdateDocuments(new Term("id", id), Arrays.asList(doc, doc), + new NumericDocValuesField("soft_delete", 1)); + } } else { Document doc = new Document(); doc.add(new StringField("id", id, Field.Store.YES)); - writer.softUpdateDocument(new Term("id", id), doc, - new NumericDocValuesField("soft_delete", 1)); + if (mixDeletes && random().nextBoolean()) { + writer.updateDocument(new Term("id", id), doc); + } else { + writer.softUpdateDocument(new Term("id", id), doc, + new NumericDocValuesField("soft_delete", 1)); + } } ids.add(id); } @@ -3187,7 +3126,7 @@ public int numDocs() { for (int i = 0; i < threads.length; i++) { threads[i].join(); } - DirectoryReader reader = wrapSoftDeletes(DirectoryReader.open(writer), "soft_delete"); + DirectoryReader reader = DirectoryReader.open(writer); IndexSearcher searcher = new IndexSearcher(reader); for (String id : ids) { TopDocs topDocs = searcher.search(new TermQuery(new Term("id", id)), 10); @@ -3217,8 +3156,6 @@ public int numDocs() { assertEquals(1, reader.docFreq(new Term("id", id))); } } - IOUtils.close(reader, writer, dir); } - } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterConfig.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterConfig.java index 063045ef2b4a..7238869328c7 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterConfig.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterConfig.java @@ -100,6 +100,7 @@ public void testDefaults() throws Exception { getters.add("getInfoStream"); getters.add("getUseCompoundFile"); getters.add("isCheckPendingFlushOnUpdate"); + getters.add("getSoftDeletesField"); for (Method m : IndexWriterConfig.class.getDeclaredMethods()) { if (m.getDeclaringClass() == IndexWriterConfig.class && m.getName().startsWith("get")) { diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java index be862ef06055..d9e73a1fe848 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java @@ -501,11 +501,14 @@ public void testCorruptionAfterDiskFullDuringMerge() throws IOException { newIndexWriterConfig(new MockAnalyzer(random())) .setMergeScheduler(new SerialMergeScheduler()) .setReaderPooling(true) - .setMergePolicy(newLogMergePolicy(2)) + .setMergePolicy(new MergePolicyWrapper(newLogMergePolicy(2)) { + @Override + public boolean keepFullyDeletedSegment(CodecReader reader) throws IOException { + // we can do this because we add/delete/add (and dont merge to "nothing") + return true; + } + }) ); - // we can do this because we add/delete/add (and dont merge to "nothing") - w.setKeepFullyDeletedSegments(true); - Document doc = new Document(); doc.add(newTextField("f", "doctor who", Field.Store.NO)); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexingSequenceNumbers.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexingSequenceNumbers.java index 52f806a51d7f..44ea74dbd816 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexingSequenceNumbers.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexingSequenceNumbers.java @@ -97,9 +97,7 @@ public void run() { if (random().nextBoolean()) { seqNos[threadID] = w.updateDocument(id, doc); } else { - List docs = new ArrayList<>(); - docs.add(doc); - seqNos[threadID] = w.updateDocuments(id, docs); + seqNos[threadID] = w.updateDocuments(id, Arrays.asList(doc)); } } } catch (Exception e) { @@ -128,7 +126,7 @@ public void run() { DirectoryReader r = w.getReader(); IndexSearcher s = newSearcher(r); TopDocs hits = s.search(new TermQuery(id), 1); - assertEquals(1, hits.totalHits); + assertEquals("maxDoc: " + r.maxDoc(), 1, hits.totalHits); Document doc = r.document(hits.scoreDocs[0].doc); assertEquals(maxThread, doc.getField("thread").numericValue().intValue()); r.close(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestMultiFields.java b/lucene/core/src/test/org/apache/lucene/index/TestMultiFields.java index 27f2f1a3699e..6e0d64393ac6 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestMultiFields.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestMultiFields.java @@ -49,10 +49,13 @@ public void testRandom() throws Exception { Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())) - .setMergePolicy(NoMergePolicy.INSTANCE)); - // we can do this because we use NoMergePolicy (and dont merge to "nothing") - w.setKeepFullyDeletedSegments(true); - + .setMergePolicy(new MergePolicyWrapper(NoMergePolicy.INSTANCE) { + @Override + public boolean keepFullyDeletedSegment(CodecReader reader) { + // we can do this because we use NoMergePolicy (and dont merge to "nothing") + return true; + } + })); Map> docs = new HashMap<>(); Set deleted = new HashSet<>(); List terms = new ArrayList<>(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPendingDeletes.java b/lucene/core/src/test/org/apache/lucene/index/TestPendingDeletes.java index 39f5680a74ff..e150e0689609 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestPendingDeletes.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestPendingDeletes.java @@ -32,12 +32,16 @@ public class TestPendingDeletes extends LuceneTestCase { + protected PendingDeletes newPendingDeletes(SegmentCommitInfo commitInfo) { + return new PendingDeletes(commitInfo); + } + public void testDeleteDoc() throws IOException { RAMDirectory dir = new RAMDirectory(); SegmentInfo si = new SegmentInfo(dir, Version.LATEST, Version.LATEST, "test", 10, false, Codec.getDefault(), Collections.emptyMap(), StringHelper.randomId(), new HashMap<>(), null); SegmentCommitInfo commitInfo = new SegmentCommitInfo(si, 0, 0, 0, 0); - PendingDeletes deletes = new PendingDeletes(null, commitInfo); + PendingDeletes deletes = newPendingDeletes(commitInfo); assertNull(deletes.getLiveDocs()); int docToDelete = TestUtil.nextInt(random(), 0, 7); assertTrue(deletes.delete(docToDelete)); @@ -73,7 +77,7 @@ public void testWriteLiveDocs() throws IOException { SegmentInfo si = new SegmentInfo(dir, Version.LATEST, Version.LATEST, "test", 6, false, Codec.getDefault(), Collections.emptyMap(), StringHelper.randomId(), new HashMap<>(), null); SegmentCommitInfo commitInfo = new SegmentCommitInfo(si, 0, 0, 0, 0); - PendingDeletes deletes = new PendingDeletes(null, commitInfo); + PendingDeletes deletes = newPendingDeletes(commitInfo); assertFalse(deletes.writeLiveDocs(dir)); assertEquals(0, dir.listAll().length); boolean secondDocDeletes = random().nextBoolean(); @@ -130,7 +134,7 @@ public void testIsFullyDeleted() throws IOException { SegmentInfo si = new SegmentInfo(dir, Version.LATEST, Version.LATEST, "test", 3, false, Codec.getDefault(), Collections.emptyMap(), StringHelper.randomId(), new HashMap<>(), null); SegmentCommitInfo commitInfo = new SegmentCommitInfo(si, 0, 0, 0, 0); - PendingDeletes deletes = new PendingDeletes(null, commitInfo); + PendingDeletes deletes = newPendingDeletes(commitInfo); for (int i = 0; i < 3; i++) { assertTrue(deletes.delete(i)); if (random().nextBoolean()) { diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java b/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java new file mode 100644 index 000000000000..c428a4b2599b --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java @@ -0,0 +1,232 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.index; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; + +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.StringHelper; +import org.apache.lucene.util.Version; + +public class TestPendingSoftDeletes extends TestPendingDeletes { + + @Override + protected PendingSoftDeletes newPendingDeletes(SegmentCommitInfo commitInfo) { + return new PendingSoftDeletes("_soft_deletes", commitInfo); + } + + public void testDeleteSoft() throws IOException { + Directory dir = newDirectory(); + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig()); // no soft delete field hier + Document doc = new Document(); + doc.add(new StringField("id", "1", Field.Store.YES)); + writer.softUpdateDocument(new Term("id", "1"), doc, + new NumericDocValuesField("_soft_deletes", 1)); + doc = new Document(); + doc.add(new StringField("id", "2", Field.Store.YES)); + writer.softUpdateDocument(new Term("id", "2"), doc, + new NumericDocValuesField("_soft_deletes", 1)); + doc = new Document(); + doc.add(new StringField("id", "2", Field.Store.YES)); + writer.softUpdateDocument(new Term("id", "2"), doc, + new NumericDocValuesField("_soft_deletes", 1)); + writer.commit(); + DirectoryReader reader = writer.getReader(); + assertEquals(1, reader.leaves().size()); + SegmentReader segmentReader = (SegmentReader) reader.leaves().get(0).reader(); + SegmentCommitInfo segmentInfo = segmentReader.getSegmentInfo(); + PendingSoftDeletes pendingSoftDeletes = newPendingDeletes(segmentInfo); + pendingSoftDeletes.onNewReader(segmentReader, segmentInfo); + assertEquals(1, pendingSoftDeletes.numPendingDeletes()); + assertTrue(pendingSoftDeletes.getLiveDocs().get(0)); + assertFalse(pendingSoftDeletes.getLiveDocs().get(1)); + assertTrue(pendingSoftDeletes.getLiveDocs().get(2)); + // pass reader again + Bits liveDocs = pendingSoftDeletes.getLiveDocs(); + pendingSoftDeletes.liveDocsShared(); + pendingSoftDeletes.onNewReader(segmentReader, segmentInfo); + assertEquals(1, pendingSoftDeletes.numPendingDeletes()); + assertSame(liveDocs, pendingSoftDeletes.getLiveDocs()); + + // now apply a hard delete + writer.deleteDocuments(new Term("id", "1")); + writer.commit(); + IOUtils.close(reader); + reader = DirectoryReader.open(dir); + assertEquals(1, reader.leaves().size()); + segmentReader = (SegmentReader) reader.leaves().get(0).reader(); + segmentInfo = segmentReader.getSegmentInfo(); + pendingSoftDeletes = newPendingDeletes(segmentInfo); + pendingSoftDeletes.onNewReader(segmentReader, segmentInfo); + assertEquals(1, pendingSoftDeletes.numPendingDeletes()); + assertFalse(pendingSoftDeletes.getLiveDocs().get(0)); + assertFalse(pendingSoftDeletes.getLiveDocs().get(1)); + assertTrue(pendingSoftDeletes.getLiveDocs().get(2)); + IOUtils.close(reader, writer, dir); + } + + public void testApplyUpdates() throws IOException { + RAMDirectory dir = new RAMDirectory(); + SegmentInfo si = new SegmentInfo(dir, Version.LATEST, Version.LATEST, "test", 10, false, Codec.getDefault(), + Collections.emptyMap(), StringHelper.randomId(), new HashMap<>(), null); + SegmentCommitInfo commitInfo = new SegmentCommitInfo(si, 0, 0, 0, 0); + PendingSoftDeletes deletes = newPendingDeletes(commitInfo); + FieldInfo fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, 0, Collections.emptyMap(), 0, 0); + List docsDeleted = Arrays.asList(1, 3, 7, 8, DocIdSetIterator.NO_MORE_DOCS); + List updates = Arrays.asList(singleUpdate(docsDeleted, 10)); + deletes.onDocValuesUpdate(fieldInfo, updates); + assertEquals(4, deletes.numPendingDeletes()); + assertTrue(deletes.getLiveDocs().get(0)); + assertFalse(deletes.getLiveDocs().get(1)); + assertTrue(deletes.getLiveDocs().get(2)); + assertFalse(deletes.getLiveDocs().get(3)); + assertTrue(deletes.getLiveDocs().get(4)); + assertTrue(deletes.getLiveDocs().get(5)); + assertTrue(deletes.getLiveDocs().get(6)); + assertFalse(deletes.getLiveDocs().get(7)); + assertFalse(deletes.getLiveDocs().get(8)); + assertTrue(deletes.getLiveDocs().get(9)); + + docsDeleted = Arrays.asList(1, 2, DocIdSetIterator.NO_MORE_DOCS); + updates = Arrays.asList(singleUpdate(docsDeleted, 10)); + fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, 1, Collections.emptyMap(), 0, 0); + deletes.onDocValuesUpdate(fieldInfo, updates); + assertEquals(5, deletes.numPendingDeletes()); + assertTrue(deletes.getLiveDocs().get(0)); + assertFalse(deletes.getLiveDocs().get(1)); + assertFalse(deletes.getLiveDocs().get(2)); + assertFalse(deletes.getLiveDocs().get(3)); + assertTrue(deletes.getLiveDocs().get(4)); + assertTrue(deletes.getLiveDocs().get(5)); + assertTrue(deletes.getLiveDocs().get(6)); + assertFalse(deletes.getLiveDocs().get(7)); + assertFalse(deletes.getLiveDocs().get(8)); + assertTrue(deletes.getLiveDocs().get(9)); + } + + public void testUpdateAppliedOnlyOnce() throws IOException { + Directory dir = newDirectory(); + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig()); // no soft delete field hier + Document doc = new Document(); + doc.add(new StringField("id", "1", Field.Store.YES)); + writer.softUpdateDocument(new Term("id", "1"), doc, + new NumericDocValuesField("_soft_deletes", 1)); + doc = new Document(); + doc.add(new StringField("id", "2", Field.Store.YES)); + writer.softUpdateDocument(new Term("id", "2"), doc, + new NumericDocValuesField("_soft_deletes", 1)); + doc = new Document(); + doc.add(new StringField("id", "2", Field.Store.YES)); + writer.softUpdateDocument(new Term("id", "2"), doc, + new NumericDocValuesField("_soft_deletes", 1)); + writer.commit(); + DirectoryReader reader = writer.getReader(); + assertEquals(1, reader.leaves().size()); + SegmentReader segmentReader = (SegmentReader) reader.leaves().get(0).reader(); + SegmentCommitInfo segmentInfo = segmentReader.getSegmentInfo(); + SegmentInfo si = new SegmentInfo(dir, Version.LATEST, Version.LATEST, "test", 3, false, Codec.getDefault(), + Collections.emptyMap(), StringHelper.randomId(), new HashMap<>(), null); + PendingSoftDeletes deletes = newPendingDeletes(segmentInfo); + FieldInfo fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, segmentInfo.getDocValuesGen(), Collections.emptyMap(), 0, 0); + List docsDeleted = Arrays.asList(1, DocIdSetIterator.NO_MORE_DOCS); + List updates = Arrays.asList(singleUpdate(docsDeleted, 3)); + deletes.onDocValuesUpdate(fieldInfo, updates); + assertEquals(1, deletes.numPendingDeletes()); + assertTrue(deletes.getLiveDocs().get(0)); + assertFalse(deletes.getLiveDocs().get(1)); + assertTrue(deletes.getLiveDocs().get(2)); + deletes.liveDocsShared(); + Bits liveDocs = deletes.getLiveDocs(); + deletes.onNewReader(segmentReader, segmentInfo); + // no changes we don't apply updates twice + assertSame(liveDocs, deletes.getLiveDocs()); + assertTrue(deletes.getLiveDocs().get(0)); + assertFalse(deletes.getLiveDocs().get(1)); + assertTrue(deletes.getLiveDocs().get(2)); + assertEquals(1, deletes.numPendingDeletes()); + IOUtils.close(reader, writer, dir); + } + + private DocValuesFieldUpdates singleUpdate(List docsDeleted, int maxDoc) { + return new DocValuesFieldUpdates(maxDoc, 0, "_soft_deletes", DocValuesType.NUMERIC) { + @Override + public void add(int doc, Object value) { + } + + @Override + public Iterator iterator() { + return new Iterator() { + java.util.Iterator iter = docsDeleted.iterator(); + int doc = -1; + + @Override + int nextDoc() { + return doc = iter.next(); + } + + @Override + int doc() { + return doc; + } + + @Override + Object value() { + return 1; + } + + @Override + long delGen() { + return 0; + } + }; + } + + @Override + public void finish() { + } + + @Override + public boolean any() { + return true; + } + + @Override + public long ramBytesUsed() { + return 0; + } + + @Override + public int size() { + return 1; + } + }; + } +} diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSoftDeletesRetentionMergePolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestSoftDeletesRetentionMergePolicy.java new file mode 100644 index 000000000000..3f4f4053885c --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/index/TestSoftDeletesRetentionMergePolicy.java @@ -0,0 +1,312 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.index; + +import java.io.IOException; +import java.time.Duration; +import java.time.Instant; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.IntPoint; +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.search.DocValuesFieldExistsQuery; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.LuceneTestCase; + +public class TestSoftDeletesRetentionMergePolicy extends LuceneTestCase { + + public void testKeepFullyDeletedSegments() throws IOException { + Directory dir = newDirectory(); + IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); + IndexWriter writer = new IndexWriter(dir, indexWriterConfig); + + Document doc = new Document(); + doc.add(new StringField("id", "1", Field.Store.YES)); + doc.add(new NumericDocValuesField("soft_delete", 1)); + writer.addDocument(doc); + DirectoryReader reader = writer.getReader(); + assertEquals(1, reader.leaves().size()); + SegmentReader segmentReader = (SegmentReader) reader.leaves().get(0).reader(); + MergePolicy policy = new SoftDeletesRetentionMergePolicy("soft_delete", + () -> new DocValuesFieldExistsQuery("keep_around"), NoMergePolicy.INSTANCE); + assertFalse(policy.keepFullyDeletedSegment(segmentReader)); + reader.close(); + + doc = new Document(); + doc.add(new StringField("id", "1", Field.Store.YES)); + doc.add(new NumericDocValuesField("keep_around", 1)); + doc.add(new NumericDocValuesField("soft_delete", 1)); + writer.addDocument(doc); + + reader = writer.getReader(); + assertEquals(2, reader.leaves().size()); + segmentReader = (SegmentReader) reader.leaves().get(0).reader(); + assertFalse(policy.keepFullyDeletedSegment(segmentReader)); + + segmentReader = (SegmentReader) reader.leaves().get(1).reader(); + assertTrue(policy.keepFullyDeletedSegment(segmentReader)); + + IOUtils.close(reader, writer, dir); + } + + public void testFieldBasedRetention() throws IOException { + Directory dir = newDirectory(); + IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); + Instant now = Instant.now(); + Instant time24HoursAgo = now.minus(Duration.ofDays(1)); + String softDeletesField = "soft_delete"; + Supplier docsOfLast24Hours = () -> LongPoint.newRangeQuery("creation_date", time24HoursAgo.toEpochMilli(), now.toEpochMilli()); + indexWriterConfig.setMergePolicy(new SoftDeletesRetentionMergePolicy(softDeletesField, docsOfLast24Hours, + new LogDocMergePolicy())); + indexWriterConfig.setSoftDeletesField(softDeletesField); + IndexWriter writer = new IndexWriter(dir, indexWriterConfig); + + long time28HoursAgo = now.minus(Duration.ofHours(28)).toEpochMilli(); + Document doc = new Document(); + doc.add(new StringField("id", "1", Field.Store.YES)); + doc.add(new StringField("version", "1", Field.Store.YES)); + doc.add(new LongPoint("creation_date", time28HoursAgo)); + writer.addDocument(doc); + + writer.flush(); + long time26HoursAgo = now.minus(Duration.ofHours(26)).toEpochMilli(); + doc = new Document(); + doc.add(new StringField("id", "1", Field.Store.YES)); + doc.add(new StringField("version", "2", Field.Store.YES)); + doc.add(new LongPoint("creation_date", time26HoursAgo)); + writer.softUpdateDocument(new Term("id", "1"), doc, new NumericDocValuesField("soft_delete", 1)); + + if (random().nextBoolean()) { + writer.flush(); + } + long time23HoursAgo = now.minus(Duration.ofHours(23)).toEpochMilli(); + doc = new Document(); + doc.add(new StringField("id", "1", Field.Store.YES)); + doc.add(new StringField("version", "3", Field.Store.YES)); + doc.add(new LongPoint("creation_date", time23HoursAgo)); + writer.softUpdateDocument(new Term("id", "1"), doc, new NumericDocValuesField("soft_delete", 1)); + + if (random().nextBoolean()) { + writer.flush(); + } + long time12HoursAgo = now.minus(Duration.ofHours(12)).toEpochMilli(); + doc = new Document(); + doc.add(new StringField("id", "1", Field.Store.YES)); + doc.add(new StringField("version", "4", Field.Store.YES)); + doc.add(new LongPoint("creation_date", time12HoursAgo)); + writer.softUpdateDocument(new Term("id", "1"), doc, new NumericDocValuesField("soft_delete", 1)); + + if (random().nextBoolean()) { + writer.flush(); + } + doc = new Document(); + doc.add(new StringField("id", "1", Field.Store.YES)); + doc.add(new StringField("version", "5", Field.Store.YES)); + doc.add(new LongPoint("creation_date", now.toEpochMilli())); + writer.softUpdateDocument(new Term("id", "1"), doc, new NumericDocValuesField("soft_delete", 1)); + + if (random().nextBoolean()) { + writer.flush(); + } + writer.forceMerge(1); + DirectoryReader reader = writer.getReader(); + assertEquals(1, reader.numDocs()); + assertEquals(3, reader.maxDoc()); + Set versions = new HashSet<>(); + versions.add(reader.document(0, Collections.singleton("version")).get("version")); + versions.add(reader.document(1, Collections.singleton("version")).get("version")); + versions.add(reader.document(2, Collections.singleton("version")).get("version")); + assertTrue(versions.contains("5")); + assertTrue(versions.contains("4")); + assertTrue(versions.contains("3")); + IOUtils.close(reader, writer, dir); + } + + public void testKeepAllDocsAcrossMerges() throws IOException { + Directory dir = newDirectory(); + IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); + indexWriterConfig.setMergePolicy(new SoftDeletesRetentionMergePolicy("soft_delete", + () -> new MatchAllDocsQuery(), + indexWriterConfig.getMergePolicy())); + indexWriterConfig.setSoftDeletesField("soft_delete"); + IndexWriter writer = new IndexWriter(dir, indexWriterConfig); + + Document doc = new Document(); + doc.add(new StringField("id", "1", Field.Store.YES)); + writer.softUpdateDocument(new Term("id", "1"), doc, + new NumericDocValuesField("soft_delete", 1)); + + writer.commit(); + doc = new Document(); + doc.add(new StringField("id", "1", Field.Store.YES)); + writer.softUpdateDocument(new Term("id", "1"), doc, + new NumericDocValuesField("soft_delete", 1)); + + writer.commit(); + doc = new Document(); + doc.add(new StringField("id", "1", Field.Store.YES)); + doc.add(new NumericDocValuesField("soft_delete", 1)); // already deleted + writer.softUpdateDocument(new Term("id", "1"), doc, + new NumericDocValuesField("soft_delete", 1)); + writer.commit(); + DirectoryReader reader = writer.getReader(); + assertEquals(0, reader.numDocs()); + assertEquals(3, reader.maxDoc()); + assertEquals(0, writer.numDocs()); + assertEquals(3, writer.maxDoc()); + assertEquals(3, reader.leaves().size()); + reader.close(); + writer.forceMerge(1); + reader = writer.getReader(); + assertEquals(0, reader.numDocs()); + assertEquals(3, reader.maxDoc()); + assertEquals(0, writer.numDocs()); + assertEquals(3, writer.maxDoc()); + assertEquals(1, reader.leaves().size()); + IOUtils.close(reader, writer, dir); + } + + /** + * tests soft deletes that carry over deleted documents on merge for history rentention. + */ + public void testSoftDeleteWithRetention() throws IOException, InterruptedException { + AtomicInteger seqIds = new AtomicInteger(0); + Directory dir = newDirectory(); + IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); + indexWriterConfig.setMergePolicy(new SoftDeletesRetentionMergePolicy("soft_delete", + () -> IntPoint.newRangeQuery("seq_id", seqIds.intValue() - 50, Integer.MAX_VALUE), + indexWriterConfig.getMergePolicy())); + indexWriterConfig.setSoftDeletesField("soft_delete"); + IndexWriter writer = new IndexWriter(dir, indexWriterConfig); + Thread[] threads = new Thread[2 + random().nextInt(3)]; + CountDownLatch startLatch = new CountDownLatch(1); + CountDownLatch started = new CountDownLatch(threads.length); + boolean updateSeveralDocs = random().nextBoolean(); + Set ids = Collections.synchronizedSet(new HashSet<>()); + for (int i = 0; i < threads.length; i++) { + threads[i] = new Thread(() -> { + try { + started.countDown(); + startLatch.await(); + for (int d = 0; d < 100; d++) { + String id = String.valueOf(random().nextInt(10)); + int seqId = seqIds.incrementAndGet(); + if (updateSeveralDocs) { + Document doc = new Document(); + doc.add(new StringField("id", id, Field.Store.YES)); + doc.add(new IntPoint("seq_id", seqId)); + writer.softUpdateDocuments(new Term("id", id), Arrays.asList(doc, doc), + new NumericDocValuesField("soft_delete", 1)); + } else { + Document doc = new Document(); + doc.add(new StringField("id", id, Field.Store.YES)); + doc.add(new IntPoint("seq_id", seqId)); + writer.softUpdateDocument(new Term("id", id), doc, + new NumericDocValuesField("soft_delete", 1)); + } + ids.add(id); + } + } catch (IOException | InterruptedException e) { + throw new AssertionError(e); + } + }); + threads[i].start(); + } + started.await(); + startLatch.countDown(); + + for (int i = 0; i < threads.length; i++) { + threads[i].join(); + } + DirectoryReader reader = DirectoryReader.open(writer); + IndexSearcher searcher = new IndexSearcher(reader); + for (String id : ids) { + TopDocs topDocs = searcher.search(new TermQuery(new Term("id", id)), 10); + if (updateSeveralDocs) { + assertEquals(2, topDocs.totalHits); + assertEquals(Math.abs(topDocs.scoreDocs[0].doc - topDocs.scoreDocs[1].doc), 1); + } else { + assertEquals(1, topDocs.totalHits); + } + } + writer.addDocument(new Document()); // add a dummy doc to trigger a segment here + writer.flush(); + writer.forceMerge(1); + DirectoryReader oldReader = reader; + reader = DirectoryReader.openIfChanged(reader, writer); + if (reader != null) { + oldReader.close(); + assertNotSame(oldReader, reader); + } else { + reader = oldReader; + } + assertEquals(1, reader.leaves().size()); + LeafReaderContext leafReaderContext = reader.leaves().get(0); + LeafReader leafReader = leafReaderContext.reader(); + searcher = new IndexSearcher(new FilterLeafReader(leafReader) { + @Override + public CacheHelper getCoreCacheHelper() { + return leafReader.getCoreCacheHelper(); + } + + @Override + public CacheHelper getReaderCacheHelper() { + return leafReader.getReaderCacheHelper(); + } + + @Override + public Bits getLiveDocs() { + return null; + } + + @Override + public int numDocs() { + return maxDoc(); + } + }); + TopDocs seq_id = searcher.search(IntPoint.newRangeQuery("seq_id", seqIds.intValue() - 50, Integer.MAX_VALUE), 10); + assertTrue(seq_id.totalHits + " hits", seq_id.totalHits >= 50); + searcher = new IndexSearcher(reader); + for (String id : ids) { + if (updateSeveralDocs) { + assertEquals(2, searcher.search(new TermQuery(new Term("id", id)), 10).totalHits); + } else { + assertEquals(1, searcher.search(new TermQuery(new Term("id", id)), 10).totalHits); + } + } + IOUtils.close(reader, writer, dir); + } + +} diff --git a/lucene/core/src/test/org/apache/lucene/index/TestStressNRT.java b/lucene/core/src/test/org/apache/lucene/index/TestStressNRT.java index b08a85d6a7c1..e6c91b87c515 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestStressNRT.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestStressNRT.java @@ -73,6 +73,7 @@ public void test() throws Exception { final int ndocs = atLeast(50); final int nWriteThreads = TestUtil.nextInt(random(), 1, TEST_NIGHTLY ? 10 : 5); final int maxConcurrentCommits = TestUtil.nextInt(random(), 1, TEST_NIGHTLY ? 10 : 5); // number of committers at a time... needed if we want to avoid commit errors due to exceeding the max + final boolean useSoftDeletes = random().nextInt(10) < 3; final boolean tombstones = random().nextBoolean(); @@ -106,10 +107,10 @@ public void test() throws Exception { Directory dir = newMaybeVirusCheckingDirectory(); - final RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig(new MockAnalyzer(random()))); + final RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig(new MockAnalyzer(random())), useSoftDeletes); writer.setDoRandomForceMergeAssert(false); writer.commit(); - reader = DirectoryReader.open(dir); + reader = useSoftDeletes ? writer.getReader() : DirectoryReader.open(dir); for (int i=0; i idValues = new HashMap(); @@ -359,7 +359,7 @@ public void testMoreThanOneDocPerIDOneSegment() throws Exception { Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random())); iwc.setCodec(TestUtil.alwaysPostingsFormat(new IDVersionPostingsFormat())); - RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); + RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc, false); Document doc = new Document(); doc.add(makeIDField("id", 17)); w.addDocument(doc); @@ -415,7 +415,7 @@ public void testMoreThanOneDocPerIDWithUpdates() throws Exception { Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random())); iwc.setCodec(TestUtil.alwaysPostingsFormat(new IDVersionPostingsFormat())); - RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); + RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc, false); Document doc = new Document(); doc.add(makeIDField("id", 17)); w.addDocument(doc); @@ -432,7 +432,7 @@ public void testMoreThanOneDocPerIDWithDeletes() throws Exception { Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random())); iwc.setCodec(TestUtil.alwaysPostingsFormat(new IDVersionPostingsFormat())); - RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); + RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc, false); Document doc = new Document(); doc.add(makeIDField("id", 17)); w.addDocument(doc); @@ -460,7 +460,7 @@ public TokenStreamComponents createComponents(String fieldName) { }; IndexWriterConfig iwc = newIndexWriterConfig(a); iwc.setCodec(TestUtil.alwaysPostingsFormat(new IDVersionPostingsFormat())); - RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); + RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc, false); Document doc = new Document(); doc.add(newTextField("id", "id", Field.Store.NO)); expectThrows(IllegalArgumentException.class, () -> { @@ -476,7 +476,7 @@ public void testMissingPositions() throws Exception { Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random())); iwc.setCodec(TestUtil.alwaysPostingsFormat(new IDVersionPostingsFormat())); - RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); + RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc, false); Document doc = new Document(); doc.add(newStringField("id", "id", Field.Store.NO)); expectThrows(IllegalArgumentException.class, () -> { @@ -493,7 +493,7 @@ public void testInvalidPayload() throws Exception { Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random())); iwc.setCodec(TestUtil.alwaysPostingsFormat(new IDVersionPostingsFormat())); - RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); + RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc, false); Document doc = new Document(); doc.add(new StringAndPayloadField("id", "id", new BytesRef("foo"))); expectThrows(IllegalArgumentException.class, () -> { @@ -509,7 +509,7 @@ public void testMoreThanOneDocPerIDWithDeletesAcrossSegments() throws IOExceptio Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random())); iwc.setCodec(TestUtil.alwaysPostingsFormat(new IDVersionPostingsFormat())); - RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); + RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc, false); Document doc = new Document(); doc.add(makeIDField("id", 17)); w.addDocument(doc); @@ -529,7 +529,7 @@ public void testCannotIndexTermVectors() throws Exception { Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random())); iwc.setCodec(TestUtil.alwaysPostingsFormat(new IDVersionPostingsFormat())); - RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); + RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc, false); Document doc = new Document(); FieldType ft = new FieldType(StringAndPayloadField.TYPE); @@ -555,7 +555,7 @@ public void testMoreThanOnceInSingleDoc() throws IOException { Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random())); iwc.setCodec(TestUtil.alwaysPostingsFormat(new IDVersionPostingsFormat())); - RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); + RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc, false); Document doc = new Document(); doc.add(makeIDField("id", 17)); doc.add(makeIDField("id", 17)); @@ -572,7 +572,7 @@ public void testInvalidVersions() throws IOException { Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random())); iwc.setCodec(TestUtil.alwaysPostingsFormat(new IDVersionPostingsFormat())); - RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); + RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc, false); Document doc = new Document(); // -1 doc.add(new StringAndPayloadField("id", "id", new BytesRef(new byte[] {(byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff}))); @@ -590,7 +590,7 @@ public void testInvalidVersions2() throws IOException { Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random())); iwc.setCodec(TestUtil.alwaysPostingsFormat(new IDVersionPostingsFormat())); - RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); + RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc, false); Document doc = new Document(); // Long.MAX_VALUE: doc.add(new StringAndPayloadField("id", "id", new BytesRef(new byte[] {(byte)0x7f, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff}))); @@ -610,7 +610,7 @@ public void testGlobalVersions() throws Exception { Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random())); iwc.setCodec(TestUtil.alwaysPostingsFormat(new IDVersionPostingsFormat())); - final RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); + final RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc, false); IDSource idsSource = getRandomIDs(); int numIDs = atLeast(100); diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingLiveDocsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingLiveDocsFormat.java index f4abb54e8037..e02164baabbb 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingLiveDocsFormat.java +++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingLiveDocsFormat.java @@ -82,7 +82,7 @@ private void check(Bits bits, int expectedLength, int expectedDeleteCount) { deletedCount++; } } - assert deletedCount == expectedDeleteCount; + assert deletedCount == expectedDeleteCount : "deleted: " + deletedCount + " != expected: " + expectedDeleteCount; } @Override diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java b/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java index aa4da54c6c6e..b82df6802976 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java @@ -18,12 +18,14 @@ import java.io.Closeable; import java.io.IOException; +import java.util.Arrays; import java.util.Iterator; import java.util.Random; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Field; +import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; @@ -41,13 +43,14 @@ public class RandomIndexWriter implements Closeable { - public IndexWriter w; + public final IndexWriter w; private final Random r; int docCount; int flushAt; private double flushAtFactor = 1.0; private boolean getReaderCalled; private final Analyzer analyzer; // only if WE created it (then we close it) + private final double softDeletesRatio; /** Returns an indexwriter that randomly mixes up thread scheduling (by yielding at test points) */ public static IndexWriter mockIndexWriter(Directory dir, IndexWriterConfig conf, Random r) throws IOException { @@ -94,7 +97,7 @@ public static IndexWriter mockIndexWriter(Random r, Directory dir, IndexWriterCo /** create a RandomIndexWriter with a random config: Uses MockAnalyzer */ public RandomIndexWriter(Random r, Directory dir) throws IOException { - this(r, dir, LuceneTestCase.newIndexWriterConfig(r, new MockAnalyzer(r)), true); + this(r, dir, LuceneTestCase.newIndexWriterConfig(r, new MockAnalyzer(r)), true, r.nextBoolean()); } /** create a RandomIndexWriter with a random config */ @@ -104,12 +107,23 @@ public RandomIndexWriter(Random r, Directory dir, Analyzer a) throws IOException /** create a RandomIndexWriter with the provided config */ public RandomIndexWriter(Random r, Directory dir, IndexWriterConfig c) throws IOException { - this(r, dir, c, false); + this(r, dir, c, false, r.nextBoolean()); + } + + /** create a RandomIndexWriter with the provided config */ + public RandomIndexWriter(Random r, Directory dir, IndexWriterConfig c, boolean useSoftDeletes) throws IOException { + this(r, dir, c, false, useSoftDeletes); } - private RandomIndexWriter(Random r, Directory dir, IndexWriterConfig c, boolean closeAnalyzer) throws IOException { + private RandomIndexWriter(Random r, Directory dir, IndexWriterConfig c, boolean closeAnalyzer, boolean useSoftDeletes) throws IOException { // TODO: this should be solved in a different way; Random should not be shared (!). this.r = new Random(r.nextLong()); + if (useSoftDeletes) { + c.setSoftDeletesField("___soft_deletes"); + softDeletesRatio = 1.d / (double)1 + r.nextInt(10); + } else { + softDeletesRatio = 0d; + } w = mockIndexWriter(dir, c, r); flushAt = TestUtil.nextInt(r, 10, 1000); if (closeAnalyzer) { @@ -218,49 +232,39 @@ public long addDocuments(Iterable> public long updateDocuments(Term delTerm, Iterable> docs) throws IOException { LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig()); - long seqNo = w.updateDocuments(delTerm, docs); + long seqNo; + if (useSoftDeletes()) { + seqNo = w.softUpdateDocuments(delTerm, docs, new NumericDocValuesField(w.getConfig().getSoftDeletesField(), 1)); + } else { + seqNo = w.updateDocuments(delTerm, docs); + } maybeFlushOrCommit(); return seqNo; } + private boolean useSoftDeletes() { + return r.nextDouble() < softDeletesRatio; + } + /** * Updates a document. * @see IndexWriter#updateDocument(Term, Iterable) */ public long updateDocument(Term t, final Iterable doc) throws IOException { LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig()); - long seqNo; - if (r.nextInt(5) == 3) { - seqNo = w.updateDocuments(t, new Iterable>() { - - @Override - public Iterator> iterator() { - return new Iterator>() { - boolean done; - - @Override - public boolean hasNext() { - return !done; - } - - @Override - public void remove() { - throw new UnsupportedOperationException(); - } - - @Override - public Iterable next() { - if (done) { - throw new IllegalStateException(); - } - done = true; - return doc; - } - }; - } - }); + final long seqNo; + if (useSoftDeletes()) { + if (r.nextInt(5) == 3) { + seqNo = w.softUpdateDocuments(t, Arrays.asList(doc), new NumericDocValuesField(w.getConfig().getSoftDeletesField(), 1)); + } else { + seqNo = w.softUpdateDocument(t, doc, new NumericDocValuesField(w.getConfig().getSoftDeletesField(), 1)); + } } else { - seqNo = w.updateDocument(t, doc); + if (r.nextInt(5) == 3) { + seqNo = w.updateDocuments(t, Arrays.asList(doc)); + } else { + seqNo = w.updateDocument(t, doc); + } } maybeFlushOrCommit(); @@ -377,7 +381,8 @@ public DirectoryReader getReader(boolean applyDeletions, boolean writeAllDeletes if (r.nextInt(20) == 2) { doRandomForceMerge(); } - if (!applyDeletions || r.nextBoolean()) { + if (!applyDeletions || r.nextBoolean() || w.getConfig().getSoftDeletesField() != null) { + // if we have soft deletes we can't open from a directory if (LuceneTestCase.VERBOSE) { System.out.println("RIW.getReader: use NRT reader"); } From 858d246dec64cf4d1f8c3e5328dc33a542c62e12 Mon Sep 17 00:00:00 2001 From: Andrzej Bialecki Date: Wed, 4 Apr 2018 17:13:24 +0200 Subject: [PATCH 23/45] SOLR-12095: Missed a few calls to init(). --- .../solr/cloud/autoscaling/sim/TestNodeAddedTrigger.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestNodeAddedTrigger.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestNodeAddedTrigger.java index fd816caf8cf7..04e8c1d09f2d 100644 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestNodeAddedTrigger.java +++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestNodeAddedTrigger.java @@ -80,6 +80,7 @@ public void testTrigger() throws Exception { try (NodeAddedTrigger trigger = new NodeAddedTrigger("node_added_trigger")) { trigger.configure(cluster.getLoader(), cluster, props); + trigger.init(); trigger.setProcessor(noFirstRunProcessor); trigger.run(); @@ -250,6 +251,7 @@ public void testRestoreState() throws Exception { // and assert that the new trigger still fires NodeAddedTrigger trigger = new NodeAddedTrigger("node_added_trigger"); trigger.configure(cluster.getLoader(), cluster, props); + trigger.init(); trigger.setProcessor(noFirstRunProcessor); trigger.run(); @@ -259,6 +261,7 @@ public void testRestoreState() throws Exception { try (NodeAddedTrigger newTrigger = new NodeAddedTrigger("some_different_name")) { newTrigger.configure(cluster.getLoader(), cluster, props); + trigger.init(); try { newTrigger.restoreState(trigger); fail("Trigger should only be able to restore state from an old trigger of the same name"); @@ -269,6 +272,7 @@ public void testRestoreState() throws Exception { try (NodeAddedTrigger newTrigger = new NodeAddedTrigger("node_added_trigger")) { newTrigger.configure(cluster.getLoader(), cluster, props); + newTrigger.init(); AtomicBoolean fired = new AtomicBoolean(false); AtomicReference eventRef = new AtomicReference<>(); newTrigger.setProcessor(event -> { From 894faad1b0f71d573aeb4550dd1b8b48de2bbd1d Mon Sep 17 00:00:00 2001 From: Steve Rowe Date: Wed, 4 Apr 2018 11:18:10 -0400 Subject: [PATCH 24/45] SOLR-11929: UpdateLog metrics are not initialized on core reload --- solr/CHANGES.txt | 4 +++- solr/core/src/java/org/apache/solr/update/UpdateLog.java | 7 +++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 091ce9410bdb..6a41c8dec267 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -107,7 +107,9 @@ Bug Fixes (Susheel Kumar, Aibao Luo, Nikkolay Martinov via Erick Erickson) * SOLR-12172: Fixed race condition that could cause an invalid set of collection properties to be kept in - memory when multiple collection property changes are done in a short period of time. (Tomás Fernández Löbbe) + memory when multiple collection property changes are done in a short period of time. (Tomás Fernández Löbbe) + +* SOLR-11929: UpdateLog metrics are not initialized on core reload. (ab, Steve Rowe) Optimizations ---------------------- diff --git a/solr/core/src/java/org/apache/solr/update/UpdateLog.java b/solr/core/src/java/org/apache/solr/update/UpdateLog.java index 4ca4bf269c00..fbdf616f711d 100644 --- a/solr/core/src/java/org/apache/solr/update/UpdateLog.java +++ b/solr/core/src/java/org/apache/solr/update/UpdateLog.java @@ -347,13 +347,12 @@ public void init(UpdateHandler uhandler, SolrCore core) { this.uhandler = uhandler; if (dataDir.equals(lastDataDir)) { + versionInfo.reload(); + core.getCoreMetricManager().registerMetricProducer(SolrInfoBean.Category.TLOG.toString(), this); + if (debug) { log.debug("UpdateHandler init: tlogDir=" + tlogDir + ", next id=" + id, " this is a reopen... nothing else to do."); } - - versionInfo.reload(); - - // on a normal reopen, we currently shouldn't have to do anything return; } lastDataDir = dataDir; From 53d790d89add469d61ea32f3ae2e66e7f611c68b Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 4 Apr 2018 17:59:48 +0100 Subject: [PATCH 25/45] Add 7.3.0 release to DOAP files --- dev-tools/doap/lucene.rdf | 7 +++++++ dev-tools/doap/solr.rdf | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/dev-tools/doap/lucene.rdf b/dev-tools/doap/lucene.rdf index c33666d4c685..d5f889394f21 100644 --- a/dev-tools/doap/lucene.rdf +++ b/dev-tools/doap/lucene.rdf @@ -67,6 +67,13 @@ + + + lucene-7.3.0 + 2018-04-04 + 7.3.0 + + lucene-7.2.1 diff --git a/dev-tools/doap/solr.rdf b/dev-tools/doap/solr.rdf index 09bdb6bd2e6c..fb230895221b 100644 --- a/dev-tools/doap/solr.rdf +++ b/dev-tools/doap/solr.rdf @@ -67,6 +67,13 @@ + + + solr-7.3.0 + 2018-04-04 + 7.3.0 + + solr-7.2.1 From 045689f43898a758f62ea707c49936f6bed55d7a Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Thu, 5 Apr 2018 19:46:30 +0100 Subject: [PATCH 26/45] Empty matches don't need a field --- .../apache/lucene/document/FeatureQuery.java | 5 -- .../lucene/document/RangeFieldQuery.java | 5 -- .../SortedNumericDocValuesRangeQuery.java | 6 -- .../SortedSetDocValuesRangeQuery.java | 6 -- .../search/DisjunctionMatchesIterator.java | 7 +- .../search/DocValuesFieldExistsQuery.java | 5 -- .../lucene/search/MatchAllDocsQuery.java | 5 -- .../org/apache/lucene/search/Matches.java | 72 ++++--------------- .../lucene/search/MultiPhraseQuery.java | 5 -- .../lucene/search/NormsFieldExistsQuery.java | 5 -- .../org/apache/lucene/search/PhraseQuery.java | 5 -- .../apache/lucene/search/PointInSetQuery.java | 6 -- .../apache/lucene/search/PointRangeQuery.java | 5 -- .../java/org/apache/lucene/search/Weight.java | 8 ++- .../lucene/search/spans/SpanWeight.java | 6 -- .../search/join/GlobalOrdinalsQuery.java | 6 -- .../join/ParentChildrenBlockJoinQuery.java | 6 -- .../join/PointInSetIncludingScoreQuery.java | 6 -- .../search/join/TermsIncludingScoreQuery.java | 6 -- .../TestDiversifiedTopDocsCollector.java | 5 -- .../queries/function/FunctionMatchQuery.java | 7 -- .../queries/function/FunctionQuery.java | 6 -- .../queries/function/FunctionRangeQuery.java | 6 -- .../document/LatLonDocValuesBoxQuery.java | 7 -- .../LatLonDocValuesDistanceQuery.java | 6 -- .../document/LatLonPointDistanceQuery.java | 5 -- .../document/LatLonPointInPolygonQuery.java | 7 -- .../lucene/search/DocValuesNumbersQuery.java | 6 -- .../lucene/search/DocValuesTermsQuery.java | 6 -- .../lucene/search/TermAutomatonQuery.java | 5 -- .../lucene/search/TestTermAutomatonQuery.java | 5 -- .../composite/CompositeVerifyQuery.java | 18 ----- .../composite/IntersectsRPTVerifyQuery.java | 7 -- .../prefix/AbstractPrefixTreeQuery.java | 6 -- .../serialized/SerializedDVStrategy.java | 7 -- .../spatial/vector/PointVectorStrategy.java | 8 --- .../spatial3d/PointInGeo3DShapeQuery.java | 6 -- .../org/apache/solr/ltr/LTRScoringQuery.java | 6 -- .../org/apache/solr/ltr/feature/Feature.java | 6 -- .../org/apache/solr/query/SolrRangeQuery.java | 6 -- .../org/apache/solr/schema/LatLonType.java | 6 -- .../java/org/apache/solr/search/Filter.java | 6 -- .../solr/search/GraphTermsQParserPlugin.java | 10 --- .../apache/solr/search/JoinQParserPlugin.java | 7 -- .../solr/search/SolrConstantScoreQuery.java | 6 -- .../apache/solr/search/join/GraphQuery.java | 6 -- .../solr/update/DeleteByQueryWrapper.java | 6 -- 47 files changed, 24 insertions(+), 339 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/document/FeatureQuery.java b/lucene/core/src/java/org/apache/lucene/document/FeatureQuery.java index d74fd06a0da8..453daccd2714 100644 --- a/lucene/core/src/java/org/apache/lucene/document/FeatureQuery.java +++ b/lucene/core/src/java/org/apache/lucene/document/FeatureQuery.java @@ -82,11 +82,6 @@ public boolean isCacheable(LeafReaderContext ctx) { @Override public void extractTerms(Set terms) {} - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, fieldName); - } - @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { String desc = "weight(" + getQuery() + " in " + doc + ") [" + function + "]"; diff --git a/lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java b/lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java index 7e9bd72fa0e7..8b75fdf1c1ec 100644 --- a/lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java +++ b/lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java @@ -350,11 +350,6 @@ public long cost() { } } - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, field); - } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { ScorerSupplier scorerSupplier = scorerSupplier(context); diff --git a/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java b/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java index 430edb4aed4c..246b50f3dab6 100644 --- a/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java +++ b/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java @@ -29,7 +29,6 @@ import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -102,11 +101,6 @@ public boolean isCacheable(LeafReaderContext ctx) { return DocValues.isCacheable(ctx, field); } - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, field); - } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { SortedNumericDocValues values = getValues(context.reader(), field); diff --git a/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java b/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java index eba1e729f57a..de7c11b1cc9a 100644 --- a/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java +++ b/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java @@ -29,7 +29,6 @@ import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -107,11 +106,6 @@ public Query rewrite(IndexReader reader) throws IOException { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, field); - } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { SortedSetDocValues values = getValues(context.reader(), field); diff --git a/lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java b/lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java index db064c411797..f00a8563fa8b 100644 --- a/lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java +++ b/lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java @@ -36,7 +36,7 @@ * Matches are sorted by their start positions, and then by their end positions, so that * prefixes sort first. Matches may overlap. */ -public class DisjunctionMatchesIterator implements MatchesIterator { +public final class DisjunctionMatchesIterator implements MatchesIterator { /** * Create a {@link DisjunctionMatchesIterator} over a list of terms @@ -110,8 +110,9 @@ protected boolean lessThan(MatchesIterator a, MatchesIterator b) { } }; for (MatchesIterator mi : matches) { - mi.next(); - queue.add(mi); + if (mi.next()) { + queue.add(mi); + } } } diff --git a/lucene/core/src/java/org/apache/lucene/search/DocValuesFieldExistsQuery.java b/lucene/core/src/java/org/apache/lucene/search/DocValuesFieldExistsQuery.java index 8f18cbb9ab6e..54c851292828 100644 --- a/lucene/core/src/java/org/apache/lucene/search/DocValuesFieldExistsQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/DocValuesFieldExistsQuery.java @@ -62,11 +62,6 @@ public String toString(String field) { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) { return new ConstantScoreWeight(this, boost) { - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, field); - } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { DocIdSetIterator iterator = getDocValuesDocIdSetIterator(field, context.reader()); diff --git a/lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java b/lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java index a9945c66d812..ce1679746410 100644 --- a/lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java @@ -36,11 +36,6 @@ public String toString() { return "weight(" + MatchAllDocsQuery.this + ")"; } - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, "*"); - } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { return new ConstantScoreScorer(this, score(), DocIdSetIterator.all(context.reader().maxDoc())); diff --git a/lucene/core/src/java/org/apache/lucene/search/Matches.java b/lucene/core/src/java/org/apache/lucene/search/Matches.java index 780ec8ebab5f..83fae478f2cb 100644 --- a/lucene/core/src/java/org/apache/lucene/search/Matches.java +++ b/lucene/core/src/java/org/apache/lucene/search/Matches.java @@ -19,14 +19,13 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; - -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.util.BytesRef; +import java.util.stream.Collectors; /** * Reports the positions and optionally offsets of all matching terms in a query @@ -38,6 +37,8 @@ */ public class Matches { + public static final Matches MATCH_WITH_NO_TERMS = new Matches(Collections.emptyMap()); + private final Map matches; /** @@ -50,25 +51,6 @@ public static Matches fromField(String field, MatchesIterator it) { return new Matches(field, it); } - /** - * Create an empty {@link Matches} for a Weight - * - * If the Weight's parent query does not match this document, returns {@code null}, - * otherwise returns a {@link Matches} document with an empty iterator on the given - * fields - */ - public static Matches emptyMatches(LeafReaderContext context, int doc, Weight weight, String... fields) throws IOException { - Scorer scorer = weight.scorer(context); - if (scorer == null || scorer.iterator().advance(doc) != doc) { - return null; - } - List matches = new ArrayList<>(); - for (String field : fields) { - matches.add(Matches.fromField(field, EMPTY)); - } - return Matches.fromSubMatches(matches); - } - /** * Amalgamate a collection of {@link Matches} into a single object */ @@ -76,6 +58,10 @@ public static Matches fromSubMatches(List subMatches) throws IOExceptio if (subMatches == null || subMatches.size() == 0) { return null; } + subMatches = subMatches.stream().filter(m -> m != MATCH_WITH_NO_TERMS).collect(Collectors.toList()); + if (subMatches.size() == 0) { + return MATCH_WITH_NO_TERMS; + } if (subMatches.size() == 1) { return subMatches.get(0); } @@ -105,13 +91,15 @@ protected Matches(Map matches) { } private Matches(String field, MatchesIterator iterator) { - this.matches = new HashMap<>(); - this.matches.put(field, iterator); + this.matches = Collections.singletonMap(field, iterator); } /** * Returns a {@link MatchesIterator} over the matches for a single field, - * or {@code null} if there are no matches in that field + * or {@code null} if there are no matches in that field. + * + * This method always returns the same iterator, so clients should only + * call it once per field */ public MatchesIterator getMatches(String field) { return matches.get(field); @@ -124,38 +112,4 @@ public Set getMatchFields() { return matches.keySet(); } - private static final BytesRef EMPTY_BYTES = new BytesRef(); - - private static final MatchesIterator EMPTY = new MatchesIterator() { - - @Override - public boolean next() throws IOException { - return false; - } - - @Override - public int startPosition() { - throw new UnsupportedOperationException(); - } - - @Override - public int endPosition() { - throw new UnsupportedOperationException(); - } - - @Override - public int startOffset() throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public int endOffset() throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public BytesRef term() { - return EMPTY_BYTES; - } - }; } diff --git a/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java b/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java index 31599fa33fb2..65d6631e9a7c 100644 --- a/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java @@ -235,11 +235,6 @@ public void extractTerms(Set terms) { } } - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, field); // TODO - } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { assert termArrays.length != 0; diff --git a/lucene/core/src/java/org/apache/lucene/search/NormsFieldExistsQuery.java b/lucene/core/src/java/org/apache/lucene/search/NormsFieldExistsQuery.java index 5ff326d2f4c8..74218b40b0c3 100644 --- a/lucene/core/src/java/org/apache/lucene/search/NormsFieldExistsQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/NormsFieldExistsQuery.java @@ -64,11 +64,6 @@ public String toString(String field) { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, field); - } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { FieldInfos fieldInfos = context.reader().getFieldInfos(); diff --git a/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java index 5c333e782a8f..ff1538820d61 100644 --- a/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java @@ -392,11 +392,6 @@ public void extractTerms(Set queryTerms) { Collections.addAll(queryTerms, terms); } - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, field); // TODO - } - @Override public String toString() { return "weight(" + PhraseQuery.this + ")"; } diff --git a/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java b/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java index 7c3814b1c855..a534715d1e6b 100644 --- a/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java @@ -112,12 +112,6 @@ public final Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, fl // This is an inverted structure and should be used in the first pass: return new ConstantScoreWeight(this, boost) { - - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, field); - } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { LeafReader reader = context.reader(); diff --git a/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java b/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java index 2f789e41ea84..7e48383b4720 100644 --- a/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java @@ -313,11 +313,6 @@ public long cost() { } } - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, field); // TODO can we return values here somehow? - } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { ScorerSupplier scorerSupplier = scorerSupplier(context); diff --git a/lucene/core/src/java/org/apache/lucene/search/Weight.java b/lucene/core/src/java/org/apache/lucene/search/Weight.java index 6587511d1cb4..196de6a08f27 100644 --- a/lucene/core/src/java/org/apache/lucene/search/Weight.java +++ b/lucene/core/src/java/org/apache/lucene/search/Weight.java @@ -76,7 +76,13 @@ protected Weight(Query query) { * @param context the reader's context to create the {@link Matches} for * @param doc the document's id relative to the given context's reader */ - public abstract Matches matches(LeafReaderContext context, int doc) throws IOException; + public Matches matches(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.MATCH_WITH_NO_TERMS; + } /** * An explanation of the score computation for the named document. diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java b/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java index f93f9a1e40de..244f9addd7db 100644 --- a/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java +++ b/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java @@ -29,7 +29,6 @@ import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.LeafSimScorer; -import org.apache.lucene.search.Matches; import org.apache.lucene.search.TermStatistics; import org.apache.lucene.search.Weight; import org.apache.lucene.search.similarities.Similarity; @@ -163,9 +162,4 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio return Explanation.noMatch("no matching term"); } - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - // TODO: Composite matches - return Matches.emptyMatches(context, doc, this, field); - } } diff --git a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java index 5917affb1065..8247f81352a7 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java +++ b/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java @@ -28,7 +28,6 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.TwoPhaseIterator; @@ -111,11 +110,6 @@ final class W extends ConstantScoreWeight { @Override public void extractTerms(Set terms) {} - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, joinField); // TODO is there a way of reporting matches that makes sense here? - } - @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { SortedDocValues values = DocValues.getSorted(context.reader(), joinField); diff --git a/lucene/join/src/java/org/apache/lucene/search/join/ParentChildrenBlockJoinQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/ParentChildrenBlockJoinQuery.java index 16f94d243367..7ce1c294592f 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/ParentChildrenBlockJoinQuery.java +++ b/lucene/join/src/java/org/apache/lucene/search/join/ParentChildrenBlockJoinQuery.java @@ -27,7 +27,6 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; @@ -105,11 +104,6 @@ public void extractTerms(Set terms) { childWeight.extractTerms(terms); } - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, "*"); // TODO is there a way of reporting matches that makes sense here? - } - @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { return Explanation.noMatch("Not implemented, use ToParentBlockJoinQuery explain why a document matched"); diff --git a/lucene/join/src/java/org/apache/lucene/search/join/PointInSetIncludingScoreQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/PointInSetIncludingScoreQuery.java index ec2d66dd1adb..02b7e86d1688 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/PointInSetIncludingScoreQuery.java +++ b/lucene/join/src/java/org/apache/lucene/search/join/PointInSetIncludingScoreQuery.java @@ -40,7 +40,6 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Matches; import org.apache.lucene.search.PointInSetQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; @@ -128,11 +127,6 @@ public final Weight createWeight(IndexSearcher searcher, org.apache.lucene.searc public void extractTerms(Set terms) { } - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, field); // TODO is there a way of reporting matches that makes sense here? - } - @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { Scorer scorer = scorer(context); diff --git a/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java index 24a205c9f9f6..43ddd528393f 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java +++ b/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java @@ -29,7 +29,6 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; @@ -103,11 +102,6 @@ public Weight createWeight(IndexSearcher searcher, org.apache.lucene.search.Scor @Override public void extractTerms(Set terms) {} - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, toField); // TODO is there a way of reporting matches that makes sense here? - } - @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { Terms terms = context.reader().terms(toField); diff --git a/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java b/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java index b22a203819c9..a25ff0d1c3a9 100644 --- a/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java +++ b/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java @@ -526,11 +526,6 @@ public void extractTerms(Set terms) { inner.extractTerms(terms); } - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, "*"); // TODO is there a way of reporting matches that makes sense here? - } - @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { Scorer s = scorer(context); diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionMatchQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionMatchQuery.java index e721db623339..7a837342f762 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionMatchQuery.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionMatchQuery.java @@ -28,7 +28,6 @@ import org.apache.lucene.search.DoubleValues; import org.apache.lucene.search.DoubleValuesSource; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -66,12 +65,6 @@ public String toString(String field) { public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { DoubleValuesSource vs = source.rewrite(searcher); return new ConstantScoreWeight(this, boost) { - - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, source.toString()); // TODO is there a way of reporting matches that makes sense here? - } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { DoubleValues values = vs.getValues(context, null); diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java index 78c9da0055a5..f996306a72d1 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java @@ -26,7 +26,6 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -71,11 +70,6 @@ public FunctionWeight(IndexSearcher searcher, float boost) throws IOException { @Override public void extractTerms(Set terms) {} - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, func.toString()); // TODO is there a way of reporting matches that makes sense here? - } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { return new AllScorer(context, this, boost); diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionRangeQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionRangeQuery.java index 72d12e0bf50f..2d55bae7693d 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionRangeQuery.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionRangeQuery.java @@ -25,7 +25,6 @@ import org.apache.lucene.index.Term; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Weight; @@ -135,11 +134,6 @@ public void extractTerms(Set terms) { //none } - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, valueSource.toString()); // TODO is there a way of reporting matches that makes sense here? - } - @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { FunctionValues functionValues = valueSource.getValues(vsContext, context); diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesBoxQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesBoxQuery.java index e1d7c1778730..31037f9c36e4 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesBoxQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesBoxQuery.java @@ -26,7 +26,6 @@ import org.apache.lucene.search.ConstantScoreScorer; import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -99,12 +98,6 @@ public int hashCode() { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { - - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, field); // TODO is there a way of reporting matches that makes sense here? - } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { final SortedNumericDocValues values = context.reader().getSortedNumericDocValues(field); diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesDistanceQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesDistanceQuery.java index b5cd2e6b88d8..df350e6db95c 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesDistanceQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonDocValuesDistanceQuery.java @@ -26,7 +26,6 @@ import org.apache.lucene.search.ConstantScoreScorer; import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -99,11 +98,6 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo private final GeoEncodingUtils.DistancePredicate distancePredicate = GeoEncodingUtils.createDistancePredicate(latitude, longitude, radiusMeters); - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, field); // TODO is there a way of reporting matches that makes sense here? - } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { final SortedNumericDocValues values = context.reader().getSortedNumericDocValues(field); diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java index 1c9e3787177f..82c119ab501f 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java @@ -110,11 +110,6 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo final GeoEncodingUtils.DistancePredicate distancePredicate = GeoEncodingUtils.createDistancePredicate(latitude, longitude, radiusMeters); - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, field); // TODO is there a way of reporting matches that makes sense here? - } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { ScorerSupplier scorerSupplier = scorerSupplier(context); diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java index 10643f246e79..5248ce60bf18 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java @@ -32,7 +32,6 @@ import org.apache.lucene.search.ConstantScoreScorer; import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -98,12 +97,6 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo final GeoEncodingUtils.PolygonPredicate polygonPredicate = GeoEncodingUtils.createPolygonPredicate(polygons, tree); return new ConstantScoreWeight(this, boost) { - - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, field); // TODO is there a way of reporting matches that makes sense here? - } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { LeafReader reader = context.reader(); diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesNumbersQuery.java b/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesNumbersQuery.java index 4f651cdc433f..db0d2befa69a 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesNumbersQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesNumbersQuery.java @@ -98,12 +98,6 @@ public String toString(String defaultField) { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { - - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, field); // TODO is there a way of reporting matches that makes sense here? - } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { final SortedNumericDocValues values = DocValues.getSortedNumeric(context.reader(), field); diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesTermsQuery.java b/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesTermsQuery.java index 33fca99af1d1..4d955c037c9c 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesTermsQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesTermsQuery.java @@ -167,12 +167,6 @@ public String toString(String defaultField) { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { - - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, field); // TODO is there a way of reporting matches that makes sense here? - } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { final SortedSetDocValues values = DocValues.getSortedSet(context.reader(), field); diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java b/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java index b25c5e1bc792..42a5f74216bb 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java @@ -371,11 +371,6 @@ public void extractTerms(Set terms) { } } - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, field); // TODO - } - @Override public String toString() { return "weight(" + TermAutomatonQuery.this + ")"; diff --git a/lucene/sandbox/src/test/org/apache/lucene/search/TestTermAutomatonQuery.java b/lucene/sandbox/src/test/org/apache/lucene/search/TestTermAutomatonQuery.java index 7178d2ec337e..d6bf90dfcbe9 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/search/TestTermAutomatonQuery.java +++ b/lucene/sandbox/src/test/org/apache/lucene/search/TestTermAutomatonQuery.java @@ -584,11 +584,6 @@ public RandomQuery(long seed, float density) { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, "*"); // TODO is there a way of reporting matches that makes sense here? - } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { int maxDoc = context.reader().maxDoc(); diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/CompositeVerifyQuery.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/CompositeVerifyQuery.java index 44a6485eb473..224c24e275e4 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/CompositeVerifyQuery.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/CompositeVerifyQuery.java @@ -17,15 +17,12 @@ package org.apache.lucene.spatial.composite; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.ConstantScoreScorer; import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -88,21 +85,6 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo final Weight indexQueryWeight = indexQuery.createWeight(searcher, ScoreMode.COMPLETE_NO_SCORES, boost);//scores aren't unsupported return new ConstantScoreWeight(this, boost) { - - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer == null || scorer.iterator().advance(doc) != doc) { - return null; - } - Matches innerMatches = indexQueryWeight.matches(context, doc); - List subMatches = new ArrayList<>(); - for (String field : innerMatches.getMatchFields()) { - subMatches.add(Matches.emptyMatches(context, doc, this, field)); - } - return Matches.fromSubMatches(subMatches); // TODO is there a way of reporting matches that makes sense here? - } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/IntersectsRPTVerifyQuery.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/IntersectsRPTVerifyQuery.java index 4d004980b109..6671044be1ee 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/IntersectsRPTVerifyQuery.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/IntersectsRPTVerifyQuery.java @@ -24,7 +24,6 @@ import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -87,12 +86,6 @@ public int hashCode() { public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { - - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, field); // TODO is there a way of reporting matches that makes sense here? - } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { // Compute approx & exact diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/AbstractPrefixTreeQuery.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/AbstractPrefixTreeQuery.java index 5b81c4082c03..75b3c2b80bc0 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/AbstractPrefixTreeQuery.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/AbstractPrefixTreeQuery.java @@ -28,7 +28,6 @@ import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -80,11 +79,6 @@ public int hashCode() { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, fieldName); // TODO is there a way of reporting matches that makes sense here? - } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { DocIdSet docSet = getDocIdSet(context); diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java index 92abf9f6b82d..74a50c5080d3 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java @@ -33,7 +33,6 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.DoubleValuesSource; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -139,12 +138,6 @@ public PredicateValueSourceQuery(String field, ShapeValuesPredicate predicateVal @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { - - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, field); // TODO is there a way of reporting matches that makes sense here? - } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { DocIdSetIterator approximation = DocIdSetIterator.all(context.reader().maxDoc()); diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java index 36b8e6bf21b7..6ab550d6026b 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java @@ -36,7 +36,6 @@ import org.apache.lucene.search.DoubleValues; import org.apache.lucene.search.DoubleValuesSource; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -270,13 +269,6 @@ public Query rewrite(IndexReader reader) throws IOException { public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { Weight w = inner.createWeight(searcher, scoreMode, 1f); return new ConstantScoreWeight(this, boost) { - - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, fieldNameX, fieldNameY); - // TODO is there a way of reporting matches that makes sense here? - } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { Scorer in = w.scorer(context); diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInGeo3DShapeQuery.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInGeo3DShapeQuery.java index d8e0644ebd12..83a471fab55e 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInGeo3DShapeQuery.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInGeo3DShapeQuery.java @@ -24,7 +24,6 @@ import org.apache.lucene.search.ConstantScoreScorer; import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -69,11 +68,6 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo return new ConstantScoreWeight(this, boost) { - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, field); // TODO is there a way of reporting matches that makes sense here? - } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { LeafReader reader = context.reader(); diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/LTRScoringQuery.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/LTRScoringQuery.java index 779c8eb9aad2..c143d81f2273 100644 --- a/solr/contrib/ltr/src/java/org/apache/solr/ltr/LTRScoringQuery.java +++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/LTRScoringQuery.java @@ -39,7 +39,6 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -451,11 +450,6 @@ public void extractTerms(Set terms) { } } - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, "*"); // TODO is there a way of reporting matches that makes sense here? - } - protected void reset() { for (int i = 0; i < extractedFeatureWeights.length;++i){ int featId = extractedFeatureWeights[i].getIndex(); diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/Feature.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/Feature.java index 12232986aa78..066d28125017 100644 --- a/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/Feature.java +++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/Feature.java @@ -26,7 +26,6 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; @@ -229,11 +228,6 @@ public float getDefaultValue() { public abstract FeatureScorer scorer(LeafReaderContext context) throws IOException; - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, name); // TODO is there a way of reporting matches that makes sense here? - } - @Override public boolean isCacheable(LeafReaderContext ctx) { return false; diff --git a/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java b/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java index 8676743505aa..fb1b6f3e0668 100644 --- a/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java +++ b/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java @@ -38,7 +38,6 @@ import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -486,11 +485,6 @@ public BulkScorer bulkScorer(LeafReaderContext context) throws IOException { } } - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, field); // TODO is there a way of reporting matches that makes sense here? - } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { final SegState weightOrBitSet = getSegState(context); diff --git a/solr/core/src/java/org/apache/solr/schema/LatLonType.java b/solr/core/src/java/org/apache/solr/schema/LatLonType.java index 336c06e495ba..15f8a6ec3b56 100644 --- a/solr/core/src/java/org/apache/solr/schema/LatLonType.java +++ b/solr/core/src/java/org/apache/solr/schema/LatLonType.java @@ -34,7 +34,6 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -340,11 +339,6 @@ public boolean isCacheable(LeafReaderContext ctx) { return false; } - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, "*"); // TODO is there a way of reporting matches that makes sense here? - } - @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { return ((SpatialScorer)scorer(context)).explain(super.explain(context, doc), doc); diff --git a/solr/core/src/java/org/apache/solr/search/Filter.java b/solr/core/src/java/org/apache/solr/search/Filter.java index 0520d059ecd0..3af83e2cfdfe 100644 --- a/solr/core/src/java/org/apache/solr/search/Filter.java +++ b/solr/core/src/java/org/apache/solr/search/Filter.java @@ -26,7 +26,6 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -96,11 +95,6 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo @Override public void extractTerms(Set terms) {} - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, "*"); // TODO is there a way of reporting matches that makes sense here? - } - @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { final Scorer scorer = scorer(context); diff --git a/solr/core/src/java/org/apache/solr/search/GraphTermsQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/GraphTermsQParserPlugin.java index 57e22c7b2496..700a044b387f 100644 --- a/solr/core/src/java/org/apache/solr/search/GraphTermsQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/GraphTermsQParserPlugin.java @@ -276,11 +276,6 @@ public void extractTerms(Set terms) { // order to protect highlighters } - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, field); // TODO is there a way of reporting matches that makes sense here? - } - private WeightOrDocIdSet rewrite(LeafReaderContext context) throws IOException { final LeafReader reader = context.reader(); Terms terms = reader.terms(field); @@ -619,11 +614,6 @@ public final Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, fl return new ConstantScoreWeight(this, boost) { Filter filter; - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, field); // TODO is there a way of reporting matches that makes sense here? - } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { if (filter == null) { diff --git a/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java index 5dd3f2f0b898..1f5827cd4b8e 100644 --- a/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java @@ -35,7 +35,6 @@ import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -239,12 +238,6 @@ public void close() { DocSet resultSet; Filter filter; - - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, "*"); // TODO is there a way of reporting matches that makes sense here? - } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { if (filter == null) { diff --git a/solr/core/src/java/org/apache/solr/search/SolrConstantScoreQuery.java b/solr/core/src/java/org/apache/solr/search/SolrConstantScoreQuery.java index cf5da3c87ad6..1f81e1e61d60 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrConstantScoreQuery.java +++ b/solr/core/src/java/org/apache/solr/search/SolrConstantScoreQuery.java @@ -27,7 +27,6 @@ import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -93,11 +92,6 @@ public ConstantWeight(IndexSearcher searcher, float boost) throws IOException { ((SolrFilter)filter).createWeight(context, searcher); } - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, "*"); // TODO is there a way of reporting matches that makes sense here? - } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { DocIdSet docIdSet = filter instanceof SolrFilter ? ((SolrFilter)filter).getDocIdSet(this.context, context, null) : filter.getDocIdSet(context, null); diff --git a/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java b/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java index a9d2716d3ffc..695f616c3f95 100644 --- a/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java +++ b/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java @@ -33,7 +33,6 @@ import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -288,11 +287,6 @@ public void extractTerms(Set terms) { // NoOp for now , not used.. / supported } - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, collectSchemaField.getName()); // TODO is there a way of reporting matches that makes sense here? - } - } private static class GraphScorer extends Scorer { diff --git a/solr/core/src/java/org/apache/solr/update/DeleteByQueryWrapper.java b/solr/core/src/java/org/apache/solr/update/DeleteByQueryWrapper.java index 3262711b3456..1c87a39b2a1c 100644 --- a/solr/core/src/java/org/apache/solr/update/DeleteByQueryWrapper.java +++ b/solr/core/src/java/org/apache/solr/update/DeleteByQueryWrapper.java @@ -26,7 +26,6 @@ import org.apache.lucene.index.Term; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -78,11 +77,6 @@ public void extractTerms(Set terms) { throw new UnsupportedOperationException(); } - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, "*"); // TODO is there a way of reporting matches that makes sense here? - } - @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { throw new UnsupportedOperationException(); } From 7a1d82d03c84801909df44ba21265441d253b8ae Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Thu, 5 Apr 2018 20:05:38 +0100 Subject: [PATCH 27/45] merge conflicts --- .../apache/lucene/document/FeatureQuery.java | 1 - .../lucene/document/RangeFieldQuery.java | 1 - .../lucene/search/MatchAllDocsQuery.java | 1 - .../lucene/search/MatchNoDocsQuery.java | 5 ---- .../apache/lucene/search/PointInSetQuery.java | 1 + .../lucene/search/spans/SpanWeight.java | 1 - .../lucene/search/JustCompileSearch.java | 5 ---- .../lucene/search/TestBooleanScorer.java | 5 ---- .../lucene/search/TestLRUQueryCache.java | 25 ------------------- .../lucene/search/TestQueryRescorer.java | 5 ---- .../apache/lucene/search/TestScorerPerf.java | 5 ---- .../apache/lucene/search/TestSortRandom.java | 5 ---- .../TestUsageTrackingFilterCachingPolicy.java | 5 ---- .../lucene/facet/DrillSidewaysQuery.java | 6 ----- .../lucene/facet/range/DoubleRange.java | 6 ----- .../apache/lucene/facet/range/LongRange.java | 6 ----- .../lucene/facet/TestDrillSideways.java | 24 +++++++++++++----- .../lucene/search/join/TestJoinUtil.java | 5 ---- .../TestDiversifiedTopDocsCollector.java | 2 +- .../document/LatLonPointDistanceQuery.java | 1 - .../document/LatLonPointInPolygonQuery.java | 1 + .../lucene/search/DocValuesNumbersQuery.java | 1 + .../lucene/search/DocValuesTermsQuery.java | 1 + .../search/intervals/IntervalQuery.java | 6 ----- .../composite/CompositeVerifyQuery.java | 1 + .../composite/IntersectsRPTVerifyQuery.java | 2 -- .../serialized/SerializedDVStrategy.java | 6 ++--- .../spatial/vector/PointVectorStrategy.java | 2 +- .../suggest/document/CompletionWeight.java | 6 ----- .../lucene/search/BlockScoreQueryWrapper.java | 5 ---- .../org/apache/solr/query/SolrRangeQuery.java | 2 +- .../org/apache/solr/schema/LatLonType.java | 1 - .../solr/search/GraphTermsQParserPlugin.java | 3 +-- .../apache/solr/search/JoinQParserPlugin.java | 1 + .../apache/solr/search/join/GraphQuery.java | 2 +- .../uninverting/TestFieldCacheSortRandom.java | 6 ----- 36 files changed, 31 insertions(+), 130 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/document/FeatureQuery.java b/lucene/core/src/java/org/apache/lucene/document/FeatureQuery.java index 453daccd2714..841b2ad298b0 100644 --- a/lucene/core/src/java/org/apache/lucene/document/FeatureQuery.java +++ b/lucene/core/src/java/org/apache/lucene/document/FeatureQuery.java @@ -30,7 +30,6 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; diff --git a/lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java b/lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java index 8b75fdf1c1ec..a24b7cdfae58 100644 --- a/lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java +++ b/lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java @@ -30,7 +30,6 @@ import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.ScorerSupplier; diff --git a/lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java b/lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java index ce1679746410..89b299734144 100644 --- a/lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java @@ -35,7 +35,6 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo public String toString() { return "weight(" + MatchAllDocsQuery.this + ")"; } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { return new ConstantScoreScorer(this, score(), DocIdSetIterator.all(context.reader().maxDoc())); diff --git a/lucene/core/src/java/org/apache/lucene/search/MatchNoDocsQuery.java b/lucene/core/src/java/org/apache/lucene/search/MatchNoDocsQuery.java index 2527539dd3a4..525a18395434 100644 --- a/lucene/core/src/java/org/apache/lucene/search/MatchNoDocsQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/MatchNoDocsQuery.java @@ -48,11 +48,6 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo public void extractTerms(Set terms) { } - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return null; - } - @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { return Explanation.noMatch(reason); diff --git a/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java b/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java index a534715d1e6b..689d64a50d74 100644 --- a/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java @@ -112,6 +112,7 @@ public final Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, fl // This is an inverted structure and should be used in the first pass: return new ConstantScoreWeight(this, boost) { + @Override public Scorer scorer(LeafReaderContext context) throws IOException { LeafReader reader = context.reader(); diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java b/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java index 244f9addd7db..25b58fdc39a0 100644 --- a/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java +++ b/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java @@ -161,5 +161,4 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio return Explanation.noMatch("no matching term"); } - } diff --git a/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java b/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java index 909e93290f04..1657f9b9ced1 100644 --- a/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java +++ b/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java @@ -247,11 +247,6 @@ public void extractTerms(Set terms) { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return null; - } - @Override public Explanation explain(LeafReaderContext context, int doc) { throw new UnsupportedOperationException(UNSUPPORTED_MSG); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java b/lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java index c76b1607c046..8a8379be3432 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java @@ -84,11 +84,6 @@ public void extractTerms(Set terms) { throw new UnsupportedOperationException(); } - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return null; - } - @Override public Explanation explain(LeafReaderContext context, int doc) { throw new UnsupportedOperationException(); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java b/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java index 5b052f6fed8d..9cbf29ce2cf3 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java @@ -349,11 +349,6 @@ private static class DummyQuery extends Query { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return null; - } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { return null; @@ -948,11 +943,6 @@ private static class BadQuery extends Query { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return null; - } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { return null; @@ -1302,11 +1292,6 @@ public void extractTerms(Set terms) { } - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return null; - } - @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { return null; @@ -1379,11 +1364,6 @@ private static class DummyQuery2 extends Query { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return null; - } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { return scorerSupplier(context).get(Long.MAX_VALUE); @@ -1484,11 +1464,6 @@ public int hashCode() { public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, 1) { - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return null; - } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { scorerCreatedCount.incrementAndGet(); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestQueryRescorer.java b/lucene/core/src/test/org/apache/lucene/search/TestQueryRescorer.java index 26f4d15a6906..d1f307d063ec 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestQueryRescorer.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestQueryRescorer.java @@ -426,11 +426,6 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo public void extractTerms(Set terms) { } - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return null; - } - @Override public Scorer scorer(final LeafReaderContext context) throws IOException { diff --git a/lucene/core/src/test/org/apache/lucene/search/TestScorerPerf.java b/lucene/core/src/test/org/apache/lucene/search/TestScorerPerf.java index 8469e4954bf4..59a246cb6647 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestScorerPerf.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestScorerPerf.java @@ -151,11 +151,6 @@ private static class BitSetQuery extends Query { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return null; - } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { return new ConstantScoreScorer(this, score(), new BitSetIterator(docs, docs.approximateCardinality())); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java b/lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java index d97662ab857b..05b016c31c35 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java @@ -231,11 +231,6 @@ public RandomQuery(long seed, float density, List docValues) { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return null; - } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { Random random = new Random(context.docBase ^ seed); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestUsageTrackingFilterCachingPolicy.java b/lucene/core/src/test/org/apache/lucene/search/TestUsageTrackingFilterCachingPolicy.java index 242051cffc44..187accfdc524 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestUsageTrackingFilterCachingPolicy.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestUsageTrackingFilterCachingPolicy.java @@ -120,11 +120,6 @@ public int hashCode() { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(DummyQuery.this, boost) { - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return null; - } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { return new ConstantScoreScorer(this, score(), DocIdSetIterator.all(1)); diff --git a/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysQuery.java b/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysQuery.java index 6383b7fd560b..130e1d43cea9 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysQuery.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysQuery.java @@ -32,7 +32,6 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -92,11 +91,6 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo @Override public void extractTerms(Set terms) {} - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return null; - } - @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { return baseWeight.explain(context, doc); diff --git a/lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRange.java b/lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRange.java index f92ccf207953..56910f215cb5 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRange.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRange.java @@ -27,7 +27,6 @@ import org.apache.lucene.search.DoubleValues; import org.apache.lucene.search.DoubleValuesSource; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -144,11 +143,6 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo : searcher.createWeight(fastMatchQuery, ScoreMode.COMPLETE_NO_SCORES, 1f); return new ConstantScoreWeight(this, boost) { - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return null; - } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { final int maxDoc = context.reader().maxDoc(); diff --git a/lucene/facet/src/java/org/apache/lucene/facet/range/LongRange.java b/lucene/facet/src/java/org/apache/lucene/facet/range/LongRange.java index 8d53ef16ed68..88b569d4f87f 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/range/LongRange.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/range/LongRange.java @@ -27,7 +27,6 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.LongValues; import org.apache.lucene.search.LongValuesSource; -import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -136,11 +135,6 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo : searcher.createWeight(fastMatchQuery, ScoreMode.COMPLETE_NO_SCORES, 1f); return new ConstantScoreWeight(this, boost) { - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return null; - } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { final int maxDoc = context.reader().maxDoc(); diff --git a/lucene/facet/src/test/org/apache/lucene/facet/TestDrillSideways.java b/lucene/facet/src/test/org/apache/lucene/facet/TestDrillSideways.java index 653999ea5020..c632f3ae51cb 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/TestDrillSideways.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/TestDrillSideways.java @@ -43,8 +43,25 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; -import org.apache.lucene.search.*; +import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanClause.Occur; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.SimpleCollector; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.TwoPhaseIterator; +import org.apache.lucene.search.Weight; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.IOUtils; @@ -706,11 +723,6 @@ public void testRandom() throws Exception { public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return null; - } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { DocIdSetIterator approximation = DocIdSetIterator.all(context.reader().maxDoc()); diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java b/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java index 895569726586..a0f86af925b1 100644 --- a/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java +++ b/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java @@ -524,11 +524,6 @@ public Weight createWeight(IndexSearcher searcher, org.apache.lucene.search.Scor public void extractTerms(Set terms) { } - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return null; - } - @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { return null; diff --git a/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java b/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java index a25ff0d1c3a9..db17a90464e7 100644 --- a/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java +++ b/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java @@ -525,7 +525,7 @@ public int docID() { public void extractTerms(Set terms) { inner.extractTerms(terms); } - + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { Scorer s = scorer(context); diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java index 82c119ab501f..a72d458ddc9d 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java @@ -31,7 +31,6 @@ import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java index 5248ce60bf18..1b2e3a6b6d17 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java @@ -97,6 +97,7 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo final GeoEncodingUtils.PolygonPredicate polygonPredicate = GeoEncodingUtils.createPolygonPredicate(polygons, tree); return new ConstantScoreWeight(this, boost) { + @Override public Scorer scorer(LeafReaderContext context) throws IOException { LeafReader reader = context.reader(); diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesNumbersQuery.java b/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesNumbersQuery.java index db0d2befa69a..e72e99241b60 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesNumbersQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesNumbersQuery.java @@ -98,6 +98,7 @@ public String toString(String defaultField) { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { + @Override public Scorer scorer(LeafReaderContext context) throws IOException { final SortedNumericDocValues values = DocValues.getSortedNumeric(context.reader(), field); diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesTermsQuery.java b/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesTermsQuery.java index 4d955c037c9c..0e615b4ecbeb 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesTermsQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesTermsQuery.java @@ -167,6 +167,7 @@ public String toString(String defaultField) { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { + @Override public Scorer scorer(LeafReaderContext context) throws IOException { final SortedSetDocValues values = DocValues.getSortedSet(context.reader(), field); diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/intervals/IntervalQuery.java b/lucene/sandbox/src/java/org/apache/lucene/search/intervals/IntervalQuery.java index 9d091dc962f1..934d553717b1 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/search/intervals/IntervalQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/search/intervals/IntervalQuery.java @@ -30,7 +30,6 @@ import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.LeafSimScorer; -import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -125,11 +124,6 @@ public void extractTerms(Set terms) { intervalsSource.extractTerms(field, terms); } - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.emptyMatches(context, doc, this, field); - } - @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { IntervalScorer scorer = (IntervalScorer) scorer(context); diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/CompositeVerifyQuery.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/CompositeVerifyQuery.java index 224c24e275e4..2bfa4d53e70a 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/CompositeVerifyQuery.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/CompositeVerifyQuery.java @@ -85,6 +85,7 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo final Weight indexQueryWeight = indexQuery.createWeight(searcher, ScoreMode.COMPLETE_NO_SCORES, boost);//scores aren't unsupported return new ConstantScoreWeight(this, boost) { + @Override public Scorer scorer(LeafReaderContext context) throws IOException { diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/IntersectsRPTVerifyQuery.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/IntersectsRPTVerifyQuery.java index 6671044be1ee..e6324dae28e6 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/IntersectsRPTVerifyQuery.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/IntersectsRPTVerifyQuery.java @@ -48,14 +48,12 @@ public class IntersectsRPTVerifyQuery extends Query { private final IntersectsDifferentiatingQuery intersectsDiffQuery; private final ShapeValuesPredicate predicateValueSource; - private final String field; public IntersectsRPTVerifyQuery(Shape queryShape, String fieldName, SpatialPrefixTree grid, int detailLevel, int prefixGridScanLevel, ShapeValuesPredicate predicateValueSource) { this.predicateValueSource = predicateValueSource; this.intersectsDiffQuery = new IntersectsDifferentiatingQuery(queryShape, fieldName, grid, detailLevel, prefixGridScanLevel); - this.field = fieldName; } @Override diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java index 74a50c5080d3..cd94bf429bf2 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java @@ -113,7 +113,7 @@ public DoubleValuesSource makeDistanceValueSource(Point queryPoint, double multi public Query makeQuery(SpatialArgs args) { ShapeValuesSource shapeValueSource = makeShapeValueSource(); ShapeValuesPredicate predicateValueSource = new ShapeValuesPredicate(shapeValueSource, args.getOperation(), args.getShape()); - return new PredicateValueSourceQuery(getFieldName(), predicateValueSource); + return new PredicateValueSourceQuery(predicateValueSource); } /** @@ -128,11 +128,9 @@ public ShapeValuesSource makeShapeValueSource() { */ static class PredicateValueSourceQuery extends Query { private final ShapeValuesPredicate predicateValueSource; - private final String field; - public PredicateValueSourceQuery(String field, ShapeValuesPredicate predicateValueSource) { + public PredicateValueSourceQuery(ShapeValuesPredicate predicateValueSource) { this.predicateValueSource = predicateValueSource; - this.field = field; } @Override diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java index 6ab550d6026b..c7904df72160 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java @@ -245,7 +245,7 @@ private Query rangeQuery(String fieldName, Double min, Double max) { throw new UnsupportedOperationException("An index is required for this operation."); } - private class DistanceRangeQuery extends Query { + private static class DistanceRangeQuery extends Query { final Query inner; final DoubleValuesSource distanceSource; diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionWeight.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionWeight.java index b3bfd2edaa22..1f97d559c4da 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionWeight.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionWeight.java @@ -25,7 +25,6 @@ import org.apache.lucene.index.Terms; import org.apache.lucene.search.BulkScorer; import org.apache.lucene.search.Explanation; -import org.apache.lucene.search.Matches; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; import org.apache.lucene.search.suggest.BitsProducer; @@ -148,11 +147,6 @@ public void extractTerms(Set terms) { // no-op } - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return null; - } - @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { //TODO diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/BlockScoreQueryWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/search/BlockScoreQueryWrapper.java index dfe721b681ac..3b9a740a448f 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/search/BlockScoreQueryWrapper.java +++ b/lucene/test-framework/src/java/org/apache/lucene/search/BlockScoreQueryWrapper.java @@ -204,11 +204,6 @@ public void extractTerms(Set terms) { inWeight.extractTerms(terms); } - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return inWeight.matches(context, doc); - } - @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { return inWeight.explain(context, doc); diff --git a/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java b/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java index fb1b6f3e0668..1b81c7f5ec45 100644 --- a/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java +++ b/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java @@ -25,8 +25,8 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermState; import org.apache.lucene.index.TermStates; +import org.apache.lucene.index.TermState; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.BooleanClause; diff --git a/solr/core/src/java/org/apache/solr/schema/LatLonType.java b/solr/core/src/java/org/apache/solr/schema/LatLonType.java index 15f8a6ec3b56..9f9dcd18dfb8 100644 --- a/solr/core/src/java/org/apache/solr/schema/LatLonType.java +++ b/solr/core/src/java/org/apache/solr/schema/LatLonType.java @@ -15,7 +15,6 @@ * limitations under the License. */ package org.apache.solr.schema; - import java.io.IOException; import java.util.ArrayList; import java.util.List; diff --git a/solr/core/src/java/org/apache/solr/search/GraphTermsQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/GraphTermsQParserPlugin.java index 700a044b387f..d1f7ff2e4707 100644 --- a/solr/core/src/java/org/apache/solr/search/GraphTermsQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/GraphTermsQParserPlugin.java @@ -37,8 +37,8 @@ import org.apache.lucene.index.PrefixCodedTerms; import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermState; import org.apache.lucene.index.TermStates; +import org.apache.lucene.index.TermState; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.BulkScorer; @@ -49,7 +49,6 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchNoDocsQuery; -import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; diff --git a/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java index 1f5827cd4b8e..ebf0ebcee782 100644 --- a/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java @@ -238,6 +238,7 @@ public void close() { DocSet resultSet; Filter filter; + @Override public Scorer scorer(LeafReaderContext context) throws IOException { if (filter == null) { diff --git a/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java b/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java index 695f616c3f95..4402a2667479 100644 --- a/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java +++ b/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java @@ -286,7 +286,7 @@ public boolean isCacheable(LeafReaderContext ctx) { public void extractTerms(Set terms) { // NoOp for now , not used.. / supported } - + } private static class GraphScorer extends Scorer { diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSortRandom.java b/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSortRandom.java index 42be74a8118f..58069cd060bb 100644 --- a/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSortRandom.java +++ b/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSortRandom.java @@ -43,7 +43,6 @@ import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -272,11 +271,6 @@ public RandomQuery(long seed, float density, List docValues) { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { return new ConstantScoreWeight(this, boost) { - @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return null; - } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { Random random = new Random(seed ^ context.docBase); From 88eac1e4883bb8e2d9a6c913311b4585c6ee3557 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Fri, 6 Apr 2018 08:51:51 +0100 Subject: [PATCH 28/45] Feedback --- .../java/org/apache/lucene/search/Matches.java | 18 ++++++++---------- .../apache/lucene/search/MatchesIterator.java | 10 ++++++++++ .../lucene/search/TestMatchesIterator.java | 6 +++++- .../apache/lucene/search/AssertingMatches.java | 6 +++--- 4 files changed, 26 insertions(+), 14 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/search/Matches.java b/lucene/core/src/java/org/apache/lucene/search/Matches.java index 83fae478f2cb..3c883db1b295 100644 --- a/lucene/core/src/java/org/apache/lucene/search/Matches.java +++ b/lucene/core/src/java/org/apache/lucene/search/Matches.java @@ -22,6 +22,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -31,11 +32,9 @@ * Reports the positions and optionally offsets of all matching terms in a query * for a single document * - * To find all fields that have matches, call {@link #getMatchFields()} - * * To obtain a {@link MatchesIterator} for a particular field, call {@link #getMatches(String)} */ -public class Matches { +public class Matches implements Iterable { public static final Matches MATCH_WITH_NO_TERMS = new Matches(Collections.emptyMap()); @@ -68,7 +67,9 @@ public static Matches fromSubMatches(List subMatches) throws IOExceptio Map matches = new HashMap<>(); Set allFields = new HashSet<>(); for (Matches m : subMatches) { - allFields.addAll(m.getMatchFields()); + for (String field : m) { + allFields.add(field); + } } for (String field : allFields) { List mis = new ArrayList<>(); @@ -105,11 +106,8 @@ public MatchesIterator getMatches(String field) { return matches.get(field); } - /** - * Returns the fields with matches for this document - */ - public Set getMatchFields() { - return matches.keySet(); + @Override + public Iterator iterator() { + return matches.keySet().iterator(); } - } diff --git a/lucene/core/src/java/org/apache/lucene/search/MatchesIterator.java b/lucene/core/src/java/org/apache/lucene/search/MatchesIterator.java index abca34eb3ccb..b874263915b2 100644 --- a/lucene/core/src/java/org/apache/lucene/search/MatchesIterator.java +++ b/lucene/core/src/java/org/apache/lucene/search/MatchesIterator.java @@ -43,26 +43,36 @@ public interface MatchesIterator { /** * The start position of the current match + * + * Should only be called after {@link #next()} has returned {@code true} */ int startPosition(); /** * The end position of the current match + * + * Should only be called after {@link #next()} has returned {@code true} */ int endPosition(); /** * The starting offset of the current match, or {@code -1} if offsets are not available + * + * Should only be called after {@link #next()} has returned {@code true} */ int startOffset() throws IOException; /** * The ending offset of the current match, or {@code -1} if offsets are not available + * + * Should only be called after {@link #next()} has returned {@code true} */ int endOffset() throws IOException; /** * The underlying term of the current match + * + * Should only be called after {@link #next()} has returned {@code true} */ BytesRef term(); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java b/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java index 9b74b590c3d2..318d09874a0c 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java @@ -18,6 +18,7 @@ package org.apache.lucene.search; import java.io.IOException; +import java.util.HashSet; import java.util.Set; import org.apache.lucene.analysis.MockAnalyzer; @@ -283,7 +284,10 @@ public void testMultipleFields() throws IOException { checkFieldMatches(m.getMatches(FIELD_WITH_OFFSETS), new int[]{ -1, 1, 1, 3, 5, 3, 3, 9, 11 }); assertNull(m.getMatches("bogus")); - Set fields = m.getMatchFields(); + Set fields = new HashSet<>(); + for (String field : m) { + fields.add(field); + } assertEquals(2, fields.size()); assertTrue(fields.contains(FIELD_WITH_OFFSETS)); assertTrue(fields.contains("id")); diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/AssertingMatches.java b/lucene/test-framework/src/java/org/apache/lucene/search/AssertingMatches.java index 50f49c758ff6..5d448ecaef65 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/search/AssertingMatches.java +++ b/lucene/test-framework/src/java/org/apache/lucene/search/AssertingMatches.java @@ -17,7 +17,7 @@ package org.apache.lucene.search; -import java.util.Set; +import java.util.Iterator; class AssertingMatches extends Matches { @@ -37,7 +37,7 @@ public MatchesIterator getMatches(String field) { } @Override - public Set getMatchFields() { - return in.getMatchFields(); + public Iterator iterator() { + return in.iterator(); } } From 9c051033a3c6d0f724406fe5efc0d5a08a4d7de8 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Fri, 6 Apr 2018 09:36:30 +0100 Subject: [PATCH 29/45] Clarify javadocs - DMI doesn't deduplicate --- .../org/apache/lucene/search/DisjunctionMatchesIterator.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java b/lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java index f00a8563fa8b..d8a13487174a 100644 --- a/lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java +++ b/lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java @@ -34,7 +34,8 @@ * A {@link MatchesIterator} that combines matches from a set of sub-iterators * * Matches are sorted by their start positions, and then by their end positions, so that - * prefixes sort first. Matches may overlap. + * prefixes sort first. Matches may overlap, or be duplicated if they appear in more + * than one of the sub-iterators. */ public final class DisjunctionMatchesIterator implements MatchesIterator { From 955f70a37b1dad7d919ddf867c43018d6762ad05 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Fri, 6 Apr 2018 10:00:48 +0100 Subject: [PATCH 30/45] A bit more testing; javadocs --- .../org/apache/lucene/search/Matches.java | 3 ++ .../lucene/search/TestMatchesIterator.java | 40 +++++++++++++------ 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/search/Matches.java b/lucene/core/src/java/org/apache/lucene/search/Matches.java index 3c883db1b295..f1b55cc26a2b 100644 --- a/lucene/core/src/java/org/apache/lucene/search/Matches.java +++ b/lucene/core/src/java/org/apache/lucene/search/Matches.java @@ -36,6 +36,9 @@ */ public class Matches implements Iterable { + /** + * Indicates a match with no term positions, for example on a Point or DocValues field + */ public static final Matches MATCH_WITH_NO_TERMS = new Matches(Collections.emptyMap()); private final Map matches; diff --git a/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java b/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java index 318d09874a0c..5434de66f055 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java @@ -80,7 +80,8 @@ public void setUp() throws Exception { "w1 w2 w3 w4 w5", "w1 w3 w2 w3 zz", "w1 xx w2 yy w4", - "w1 w2 w1 w4 w2 w3" + "w1 w2 w1 w4 w2 w3", + "nothing matches this document" }; void checkMatches(Query q, String field, int[][] expected) throws IOException { @@ -137,7 +138,8 @@ public void testTermQuery() throws IOException { { 0, 0, 0, 0, 2 }, { 1, 0, 0, 0, 2 }, { 2, 0, 0, 0, 2 }, - { 3, 0, 0, 0, 2, 2, 2, 6, 8 } + { 3, 0, 0, 0, 2, 2, 2, 6, 8 }, + { 4 } }); } @@ -147,13 +149,15 @@ public void testTermQueryNoStoredOffsets() throws IOException { { 0, 0, 0, -1, -1 }, { 1, 0, 0, -1, -1 }, { 2, 0, 0, -1, -1 }, - { 3, 0, 0, -1, -1, 2, 2, -1, -1 } + { 3, 0, 0, -1, -1, 2, 2, -1, -1 }, + { 4 } }); checkTerms(q, FIELD_NO_OFFSETS, new String[][]{ { "w1" }, { "w1" }, { "w1" }, { "w1", "w1" }, + {} }); } @@ -166,13 +170,15 @@ public void testDisjunction() throws IOException { { 0, 0, 0, 0, 2, 2, 2, 6, 8 }, { 1, 0, 0, 0, 2, 1, 1, 3, 5, 3, 3, 9, 11 }, { 2, 0, 0, 0, 2 }, - { 3, 0, 0, 0, 2, 2, 2, 6, 8, 5, 5, 15, 17 } + { 3, 0, 0, 0, 2, 2, 2, 6, 8, 5, 5, 15, 17 }, + { 4 } }); checkTerms(q, FIELD_WITH_OFFSETS, new String[][]{ { "w1", "w3" }, { "w1", "w3", "w3" }, { "w1" }, - { "w1", "w1", "w3" } + { "w1", "w1", "w3" }, + {} }); } @@ -185,7 +191,8 @@ public void testReqOpt() throws IOException { { 0, 0, 0, 0, 2, 2, 2, 6, 8 }, { 1, 0, 0, 0, 2, 1, 1, 3, 5, 3, 3, 9, 11 }, { 2 }, - { 3, 0, 0, 0, 2, 2, 2, 6, 8, 5, 5, 15, 17 } + { 3, 0, 0, 0, 2, 2, 2, 6, 8, 5, 5, 15, 17 }, + { 4 } }); } @@ -203,13 +210,15 @@ public void testMinShouldMatch() throws IOException { { 0, 0, 0, 0, 2, 2, 2, 6, 8, 3, 3, 9, 11 }, { 1, 1, 1, 3, 5, 3, 3, 9, 11 }, { 2, 0, 0, 0, 2, 1, 1, 3, 5, 4, 4, 12, 14 }, - { 3, 0, 0, 0, 2, 2, 2, 6, 8, 3, 3, 9, 11, 5, 5, 15, 17 } + { 3, 0, 0, 0, 2, 2, 2, 6, 8, 3, 3, 9, 11, 5, 5, 15, 17 }, + { 4 } }); checkTerms(q, FIELD_WITH_OFFSETS, new String[][]{ { "w1", "w3", "w4" }, { "w3", "w3" }, { "w1", "xx", "w4" }, - { "w1", "w1", "w4", "w3" } + { "w1", "w1", "w4", "w3" }, + {} }); } @@ -222,7 +231,8 @@ public void testExclusion() throws IOException { { 0, 2, 2, 6, 8 }, { 1 }, { 2 }, - { 3, 5, 5, 15, 17 } + { 3, 5, 5, 15, 17 }, + { 4 } }); } @@ -235,7 +245,8 @@ public void testConjunction() throws IOException { { 0, 2, 2, 6, 8, 3, 3, 9, 11 }, { 1 }, { 2 }, - { 3, 3, 3, 9, 11, 5, 5, 15, 17 } + { 3, 3, 3, 9, 11, 5, 5, 15, 17 }, + { 4 } }); } @@ -245,7 +256,8 @@ public void testWildcards() throws IOException { { 0 }, { 1 }, { 2, 1, 1, 3, 5 }, - { 0 } + { 3 }, + { 4 } }); checkTerms(q, FIELD_WITH_OFFSETS, new String[][]{ {}, {}, { "xx" }, {} @@ -256,7 +268,8 @@ public void testWildcards() throws IOException { { 0, 0, 0, 0, 2, 1, 1, 3, 5 }, { 1, 0, 0, 0, 2, 2, 2, 6, 8 }, { 2, 0, 0, 0, 2, 2, 2, 6, 8 }, - { 3, 0, 0, 0, 2, 1, 1, 3, 5, 2, 2, 6, 8, 4, 4, 12, 14 } + { 3, 0, 0, 0, 2, 1, 1, 3, 5, 2, 2, 6, 8, 4, 4, 12, 14 }, + { 4 } }); } @@ -266,7 +279,8 @@ public void testSynonymQuery() throws IOException { { 0, 0, 0, 0, 2, 1, 1, 3, 5 }, { 1, 0, 0, 0, 2, 2, 2, 6, 8 }, { 2, 0, 0, 0, 2, 2, 2, 6, 8 }, - { 3, 0, 0, 0, 2, 1, 1, 3, 5, 2, 2, 6, 8, 4, 4, 12, 14 } + { 3, 0, 0, 0, 2, 1, 1, 3, 5, 2, 2, 6, 8, 4, 4, 12, 14 }, + { 4 } }); } From 0a64dcfccbed1852ff144a389e3c07a944467d49 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Fri, 6 Apr 2018 15:40:40 +0100 Subject: [PATCH 31/45] Matches is now an interface, and getMatches() always returns a new iterator --- .../lucene/search/DocValuesRewriteMethod.java | 2 +- .../org/apache/lucene/search/Matches.java | 107 ++++++++++-------- .../MultiTermQueryConstantScoreWrapper.java | 5 +- .../apache/lucene/search/SynonymQuery.java | 2 +- .../apache/lucene/search/TermInSetQuery.java | 4 +- .../org/apache/lucene/search/TermQuery.java | 4 +- .../lucene/search/TestMatchesIterator.java | 8 ++ .../lucene/search/AssertingMatches.java | 6 +- 8 files changed, 79 insertions(+), 59 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/search/DocValuesRewriteMethod.java b/lucene/core/src/java/org/apache/lucene/search/DocValuesRewriteMethod.java index c745ced475c4..99720b7e5868 100644 --- a/lucene/core/src/java/org/apache/lucene/search/DocValuesRewriteMethod.java +++ b/lucene/core/src/java/org/apache/lucene/search/DocValuesRewriteMethod.java @@ -78,7 +78,7 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { final SortedSetDocValues fcsi = DocValues.getSortedSet(context.reader(), query.field); - return Matches.fromField(query.field, DisjunctionMatchesIterator.fromTermsEnum(context, doc, query.field, getTermsEnum(fcsi))); + return Matches.forField(query.field, () -> DisjunctionMatchesIterator.fromTermsEnum(context, doc, query.field, getTermsEnum(fcsi))); } private TermsEnum getTermsEnum(SortedSetDocValues fcsi) throws IOException { diff --git a/lucene/core/src/java/org/apache/lucene/search/Matches.java b/lucene/core/src/java/org/apache/lucene/search/Matches.java index f1b55cc26a2b..968722ad206a 100644 --- a/lucene/core/src/java/org/apache/lucene/search/Matches.java +++ b/lucene/core/src/java/org/apache/lucene/search/Matches.java @@ -20,11 +20,9 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -34,83 +32,96 @@ * * To obtain a {@link MatchesIterator} for a particular field, call {@link #getMatches(String)} */ -public class Matches implements Iterable { +public interface Matches extends Iterable { /** - * Indicates a match with no term positions, for example on a Point or DocValues field + * Returns a {@link MatchesIterator} over the matches for a single field, + * or {@code null} if there are no matches in that field. */ - public static final Matches MATCH_WITH_NO_TERMS = new Matches(Collections.emptyMap()); - - private final Map matches; + MatchesIterator getMatches(String field) throws IOException; /** - * Create a simple {@link Matches} for a single field + * Indicates a match with no term positions, for example on a Point or DocValues field */ - public static Matches fromField(String field, MatchesIterator it) { - if (it == null) { + Matches MATCH_WITH_NO_TERMS = new Matches() { + @Override + public Iterator iterator() { + return Collections.emptyIterator(); + } + + @Override + public MatchesIterator getMatches(String field) { return null; } - return new Matches(field, it); - } + }; /** * Amalgamate a collection of {@link Matches} into a single object */ - public static Matches fromSubMatches(List subMatches) throws IOException { + static Matches fromSubMatches(List subMatches) { if (subMatches == null || subMatches.size() == 0) { return null; } - subMatches = subMatches.stream().filter(m -> m != MATCH_WITH_NO_TERMS).collect(Collectors.toList()); - if (subMatches.size() == 0) { + List sm = subMatches.stream().filter(m -> m != MATCH_WITH_NO_TERMS).collect(Collectors.toList()); + if (sm.size() == 0) { return MATCH_WITH_NO_TERMS; } - if (subMatches.size() == 1) { - return subMatches.get(0); + if (sm.size() == 1) { + return sm.get(0); } - Map matches = new HashMap<>(); - Set allFields = new HashSet<>(); - for (Matches m : subMatches) { + Set fields = new HashSet<>(); + for (Matches m : sm) { for (String field : m) { - allFields.add(field); + fields.add(field); } } - for (String field : allFields) { - List mis = new ArrayList<>(); - for (Matches m : subMatches) { - MatchesIterator mi = m.getMatches(field); - if (mi != null) { - mis.add(mi); + return new Matches() { + @Override + public MatchesIterator getMatches(String field) throws IOException { + List subIterators = new ArrayList<>(); + for (Matches m : sm) { + MatchesIterator it = m.getMatches(field); + if (it != null) { + subIterators.add(it); + } } + return DisjunctionMatchesIterator.fromSubIterators(subIterators); } - matches.put(field, DisjunctionMatchesIterator.fromSubIterators(mis)); - } - return new Matches(matches); + + @Override + public Iterator iterator() { + return fields.iterator(); + } + }; } /** - * Create a {@link Matches} from a map of fields to iterators + * A functional interface that supplies a {@link MatchesIterator} */ - protected Matches(Map matches) { - this.matches = matches; - } - - private Matches(String field, MatchesIterator iterator) { - this.matches = Collections.singletonMap(field, iterator); + @FunctionalInterface + interface MatchesIteratorSupplier { + /** Return a new {@link MatchesIterator} */ + MatchesIterator get() throws IOException; } /** - * Returns a {@link MatchesIterator} over the matches for a single field, - * or {@code null} if there are no matches in that field. - * - * This method always returns the same iterator, so clients should only - * call it once per field + * Create a Matches for a single field */ - public MatchesIterator getMatches(String field) { - return matches.get(field); - } + static Matches forField(String field, MatchesIteratorSupplier mif) { + return new Matches() { + @Override + public MatchesIterator getMatches(String f) throws IOException { + if (field.equals(f) == false) { + return null; + } + return mif.get(); + } - @Override - public Iterator iterator() { - return matches.keySet().iterator(); + @Override + public Iterator iterator() { + return Collections.singleton(field).iterator(); + } + }; } + } diff --git a/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryConstantScoreWrapper.java b/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryConstantScoreWrapper.java index 51214e53f675..cb14d435a4b5 100644 --- a/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryConstantScoreWrapper.java +++ b/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryConstantScoreWrapper.java @@ -25,8 +25,8 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermStates; import org.apache.lucene.index.TermState; +import org.apache.lucene.index.TermStates; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.BooleanClause.Occur; @@ -208,7 +208,8 @@ public Matches matches(LeafReaderContext context, int doc) throws IOException { if (terms == null) { return null; } - return Matches.fromField(query.field, DisjunctionMatchesIterator.fromTermsEnum(context, doc, query.field, query.getTermsEnum(terms))); + return Matches.forField(query.field, + () -> DisjunctionMatchesIterator.fromTermsEnum(context, doc, query.field, query.getTermsEnum(terms))); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java b/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java index 00daec5d7bcf..8b47bfda5518 100644 --- a/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java @@ -162,7 +162,7 @@ public void extractTerms(Set terms) { @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { String field = terms[0].field(); - return Matches.fromField(field, DisjunctionMatchesIterator.fromTerms(context, doc, field, Arrays.asList(SynonymQuery.this.terms))); + return Matches.forField(field, () -> DisjunctionMatchesIterator.fromTerms(context, doc, field, Arrays.asList(SynonymQuery.this.terms))); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/search/TermInSetQuery.java b/lucene/core/src/java/org/apache/lucene/search/TermInSetQuery.java index be106f554641..7297d5ef7052 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TermInSetQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/TermInSetQuery.java @@ -221,8 +221,8 @@ public void extractTerms(Set terms) { } @Override - public Matches matches(LeafReaderContext context, int doc) throws IOException { - return Matches.fromField(field, DisjunctionMatchesIterator.fromTermsEnum(context, doc, field, termData.iterator())); + public Matches matches(LeafReaderContext context, int doc) { + return Matches.forField(field, () -> DisjunctionMatchesIterator.fromTermsEnum(context, doc, field, termData.iterator())); } /** diff --git a/lucene/core/src/java/org/apache/lucene/search/TermQuery.java b/lucene/core/src/java/org/apache/lucene/search/TermQuery.java index 0928aca9f7e9..ba078682d231 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TermQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/TermQuery.java @@ -28,8 +28,8 @@ import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermStates; import org.apache.lucene.index.TermState; +import org.apache.lucene.index.TermStates; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.similarities.Similarity; @@ -91,7 +91,7 @@ public Matches matches(LeafReaderContext context, int doc) throws IOException { if (pe.advance(doc) != doc) { return null; } - return Matches.fromField(term.field(), new TermMatchesIterator(term.bytes(), pe)); + return Matches.forField(term.field(), () -> new TermMatchesIterator(term.bytes(), pe)); } @Override diff --git a/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java b/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java index 5434de66f055..f5a8c18bff11 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java @@ -95,6 +95,10 @@ void checkMatches(Query q, String field, int[][] expected) throws IOException { continue; } MatchesIterator it = matches.getMatches(field); + if (expected[i].length == 1) { + assertNull(it); + return; + } checkFieldMatches(it, expected[i]); } } @@ -123,6 +127,10 @@ void checkTerms(Query q, String field, String[][] expected) throws IOException { continue; } MatchesIterator it = matches.getMatches(field); + if (it == null) { + assertEquals(expected[i].length, 0); + continue; + } int pos = 0; while (it.next()) { assertEquals(expected[i][pos], it.term().utf8ToString()); diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/AssertingMatches.java b/lucene/test-framework/src/java/org/apache/lucene/search/AssertingMatches.java index 5d448ecaef65..c5c6e981e1cc 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/search/AssertingMatches.java +++ b/lucene/test-framework/src/java/org/apache/lucene/search/AssertingMatches.java @@ -17,19 +17,19 @@ package org.apache.lucene.search; +import java.io.IOException; import java.util.Iterator; -class AssertingMatches extends Matches { +class AssertingMatches implements Matches { private final Matches in; AssertingMatches(Matches matches) { - super(null); this.in = matches; } @Override - public MatchesIterator getMatches(String field) { + public MatchesIterator getMatches(String field) throws IOException { MatchesIterator mi = in.getMatches(field); if (mi == null) return null; From 9f1ea552cc3513cc6df22c14872f55078c5e8a82 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Fri, 6 Apr 2018 15:52:26 +0100 Subject: [PATCH 32/45] Clean up DisjunctionMatchesIterator --- .../lucene/search/DisjunctionMatchesIterator.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java b/lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java index d8a13487174a..7c1b9971be82 100644 --- a/lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java +++ b/lucene/core/src/java/org/apache/lucene/search/DisjunctionMatchesIterator.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.PostingsEnum; @@ -37,14 +38,19 @@ * prefixes sort first. Matches may overlap, or be duplicated if they appear in more * than one of the sub-iterators. */ -public final class DisjunctionMatchesIterator implements MatchesIterator { +final class DisjunctionMatchesIterator implements MatchesIterator { /** * Create a {@link DisjunctionMatchesIterator} over a list of terms * * Only terms that have at least one match in the given document will be included */ - public static DisjunctionMatchesIterator fromTerms(LeafReaderContext context, int doc, String field, List terms) throws IOException { + static MatchesIterator fromTerms(LeafReaderContext context, int doc, String field, List terms) throws IOException { + for (Term term : terms) { + if (Objects.equals(field, term.field()) == false) { + throw new IllegalArgumentException("Tried to generate iterator from terms in multiple fields: expected [" + field + "] but got [" + term.field() + "]"); + } + } return fromTermsEnum(context, doc, field, asBytesRefIterator(terms)); } @@ -65,7 +71,7 @@ public BytesRef next() { * * Only terms that have at least one match in the given document will be included */ - public static DisjunctionMatchesIterator fromTermsEnum(LeafReaderContext context, int doc, String field, BytesRefIterator terms) throws IOException { + static MatchesIterator fromTermsEnum(LeafReaderContext context, int doc, String field, BytesRefIterator terms) throws IOException { List mis = new ArrayList<>(); Terms t = context.reader().terms(field); if (t == null) @@ -87,6 +93,8 @@ public static DisjunctionMatchesIterator fromTermsEnum(LeafReaderContext context } if (mis.size() == 0) return null; + if (mis.size() == 1) + return mis.get(0); return new DisjunctionMatchesIterator(mis); } From 66e8c94eda35836c1ee539bf27906d2397568fbd Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Fri, 6 Apr 2018 17:01:34 +0100 Subject: [PATCH 33/45] Fields with no positions should return Matches.MATCH_WITH_NO_TERMS --- .../org/apache/lucene/search/Matches.java | 3 +- .../MultiTermQueryConstantScoreWrapper.java | 3 + .../apache/lucene/search/SynonymQuery.java | 3 + .../apache/lucene/search/TermInSetQuery.java | 5 +- .../org/apache/lucene/search/TermQuery.java | 3 + .../java/org/apache/lucene/search/Weight.java | 3 + .../lucene/search/TestMatchesIterator.java | 112 +++++++++++++++++- 7 files changed, 127 insertions(+), 5 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/search/Matches.java b/lucene/core/src/java/org/apache/lucene/search/Matches.java index 968722ad206a..09ab45159287 100644 --- a/lucene/core/src/java/org/apache/lucene/search/Matches.java +++ b/lucene/core/src/java/org/apache/lucene/search/Matches.java @@ -41,7 +41,8 @@ public interface Matches extends Iterable { MatchesIterator getMatches(String field) throws IOException; /** - * Indicates a match with no term positions, for example on a Point or DocValues field + * Indicates a match with no term positions, for example on a Point or DocValues field, + * or a field indexed as docs and freqs only */ Matches MATCH_WITH_NO_TERMS = new Matches() { @Override diff --git a/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryConstantScoreWrapper.java b/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryConstantScoreWrapper.java index cb14d435a4b5..6244226f143d 100644 --- a/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryConstantScoreWrapper.java +++ b/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryConstantScoreWrapper.java @@ -208,6 +208,9 @@ public Matches matches(LeafReaderContext context, int doc) throws IOException { if (terms == null) { return null; } + if (terms.hasPositions() == false) { + return super.matches(context, doc); + } return Matches.forField(query.field, () -> DisjunctionMatchesIterator.fromTermsEnum(context, doc, query.field, query.getTermsEnum(terms))); } diff --git a/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java b/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java index 8b47bfda5518..d0c5854dab43 100644 --- a/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java @@ -162,6 +162,9 @@ public void extractTerms(Set terms) { @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { String field = terms[0].field(); + if (context.reader().terms(field).hasPositions() == false) { + return super.matches(context, doc); + } return Matches.forField(field, () -> DisjunctionMatchesIterator.fromTerms(context, doc, field, Arrays.asList(SynonymQuery.this.terms))); } diff --git a/lucene/core/src/java/org/apache/lucene/search/TermInSetQuery.java b/lucene/core/src/java/org/apache/lucene/search/TermInSetQuery.java index 7297d5ef7052..cf683a077f50 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TermInSetQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/TermInSetQuery.java @@ -221,7 +221,10 @@ public void extractTerms(Set terms) { } @Override - public Matches matches(LeafReaderContext context, int doc) { + public Matches matches(LeafReaderContext context, int doc) throws IOException { + if (context.reader().terms(field).hasPositions() == false) { + return super.matches(context, doc); + } return Matches.forField(field, () -> DisjunctionMatchesIterator.fromTermsEnum(context, doc, field, termData.iterator())); } diff --git a/lucene/core/src/java/org/apache/lucene/search/TermQuery.java b/lucene/core/src/java/org/apache/lucene/search/TermQuery.java index ba078682d231..e7439b921fd5 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TermQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/TermQuery.java @@ -83,6 +83,9 @@ public void extractTerms(Set terms) { @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { + if (context.reader().terms(term.field()).hasPositions() == false) { + return super.matches(context, doc); + } TermsEnum te = getTermsEnum(context); if (te == null) { return null; diff --git a/lucene/core/src/java/org/apache/lucene/search/Weight.java b/lucene/core/src/java/org/apache/lucene/search/Weight.java index 196de6a08f27..b46d724719df 100644 --- a/lucene/core/src/java/org/apache/lucene/search/Weight.java +++ b/lucene/core/src/java/org/apache/lucene/search/Weight.java @@ -73,6 +73,9 @@ protected Weight(Query query) { * Returns {@link Matches} for a specific document, or {@code null} if the document * does not match the parent query * + * A query match that contains no position information (for example, a Point or + * DocValues query) will return {@link Matches#MATCH_WITH_NO_TERMS} + * * @param context the reader's context to create the {@link Matches} for * @param doc the document's id relative to the given context's reader */ diff --git a/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java b/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java index f5a8c18bff11..7b6a6c10b737 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java @@ -41,14 +41,26 @@ public class TestMatchesIterator extends LuceneTestCase { protected Directory directory; protected IndexReader reader; - public static final String FIELD_WITH_OFFSETS = "field_offsets"; - public static final String FIELD_NO_OFFSETS = "field_no_offsets"; + private static final String FIELD_WITH_OFFSETS = "field_offsets"; + private static final String FIELD_NO_OFFSETS = "field_no_offsets"; + private static final String FIELD_DOCS_ONLY = "field_docs_only"; + private static final String FIELD_FREQS = "field_freqs"; - public static final FieldType OFFSETS = new FieldType(TextField.TYPE_STORED); + private static final FieldType OFFSETS = new FieldType(TextField.TYPE_STORED); static { OFFSETS.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS); } + private static final FieldType DOCS = new FieldType(TextField.TYPE_STORED); + static { + DOCS.setIndexOptions(IndexOptions.DOCS); + } + + private static final FieldType DOCS_AND_FREQS = new FieldType(TextField.TYPE_STORED); + static { + DOCS_AND_FREQS.setIndexOptions(IndexOptions.DOCS_AND_FREQS); + } + @Override public void tearDown() throws Exception { reader.close(); @@ -66,6 +78,8 @@ public void setUp() throws Exception { Document doc = new Document(); doc.add(newField(FIELD_WITH_OFFSETS, docFields[i], OFFSETS)); doc.add(newField(FIELD_NO_OFFSETS, docFields[i], TextField.TYPE_STORED)); + doc.add(newField(FIELD_DOCS_ONLY, docFields[i], DOCS)); + doc.add(newField(FIELD_FREQS, docFields[i], DOCS_AND_FREQS)); doc.add(new NumericDocValuesField("id", i)); doc.add(newField("id", Integer.toString(i), TextField.TYPE_STORED)); writer.addDocument(doc); @@ -116,6 +130,22 @@ void checkFieldMatches(MatchesIterator it, int[] expected) throws IOException { assertEquals(expected.length, pos); } + void checkNoPositionsMatches(Query q, String field, boolean[] expected) throws IOException { + Weight w = searcher.createNormalizedWeight(q, ScoreMode.COMPLETE_NO_SCORES); + for (int i = 0; i < expected.length; i++) { + LeafReaderContext ctx = searcher.leafContexts.get(ReaderUtil.subIndex(i, searcher.leafContexts)); + int doc = i - ctx.docBase; + Matches matches = w.matches(ctx, doc); + if (expected[i]) { + MatchesIterator mi = matches.getMatches(field); + assertNull(mi); + } + else { + assertNull(matches); + } + } + } + void checkTerms(Query q, String field, String[][] expected) throws IOException { Weight w = searcher.createNormalizedWeight(q, ScoreMode.COMPLETE_NO_SCORES); for (int i = 0; i < expected.length; i++) { @@ -169,6 +199,13 @@ public void testTermQueryNoStoredOffsets() throws IOException { }); } + public void testTermQueryNoPositions() throws IOException { + for (String field : new String[]{ FIELD_DOCS_ONLY, FIELD_FREQS }) { + Query q = new TermQuery(new Term(field, "w1")); + checkNoPositionsMatches(q, field, new boolean[]{ true, true, true, true, false }); + } + } + public void testDisjunction() throws IOException { Query q = new BooleanQuery.Builder() .add(new TermQuery(new Term(FIELD_WITH_OFFSETS, "w1")), BooleanClause.Occur.SHOULD) @@ -190,6 +227,16 @@ public void testDisjunction() throws IOException { }); } + public void testDisjunctionNoPositions() throws IOException { + for (String field : new String[]{ FIELD_DOCS_ONLY, FIELD_FREQS }) { + Query q = new BooleanQuery.Builder() + .add(new TermQuery(new Term(field, "w1")), BooleanClause.Occur.SHOULD) + .add(new TermQuery(new Term(field, "w3")), BooleanClause.Occur.SHOULD) + .build(); + checkNoPositionsMatches(q, field, new boolean[]{ true, true, true, true, false }); + } + } + public void testReqOpt() throws IOException { Query q = new BooleanQuery.Builder() .add(new TermQuery(new Term(FIELD_WITH_OFFSETS, "w1")), BooleanClause.Occur.SHOULD) @@ -204,6 +251,16 @@ public void testReqOpt() throws IOException { }); } + public void testReqOptNoPositions() throws IOException { + for (String field : new String[]{ FIELD_DOCS_ONLY, FIELD_FREQS }) { + Query q = new BooleanQuery.Builder() + .add(new TermQuery(new Term(field, "w1")), BooleanClause.Occur.SHOULD) + .add(new TermQuery(new Term(field, "w3")), BooleanClause.Occur.MUST) + .build(); + checkNoPositionsMatches(q, field, new boolean[]{ true, true, false, true, false }); + } + } + public void testMinShouldMatch() throws IOException { Query q = new BooleanQuery.Builder() .add(new TermQuery(new Term(FIELD_WITH_OFFSETS, "w3")), BooleanClause.Occur.SHOULD) @@ -230,6 +287,21 @@ public void testMinShouldMatch() throws IOException { }); } + public void testMinShouldMatchNoPositions() throws IOException { + for (String field : new String[]{ FIELD_FREQS, FIELD_DOCS_ONLY }) { + Query q = new BooleanQuery.Builder() + .add(new TermQuery(new Term(field, "w3")), BooleanClause.Occur.SHOULD) + .add(new BooleanQuery.Builder() + .add(new TermQuery(new Term(field, "w1")), BooleanClause.Occur.SHOULD) + .add(new TermQuery(new Term(field, "w4")), BooleanClause.Occur.SHOULD) + .add(new TermQuery(new Term(field, "xx")), BooleanClause.Occur.SHOULD) + .setMinimumNumberShouldMatch(2) + .build(), BooleanClause.Occur.SHOULD) + .build(); + checkNoPositionsMatches(q, field, new boolean[]{ true, true, true, true, false }); + } + } + public void testExclusion() throws IOException { Query q = new BooleanQuery.Builder() .add(new TermQuery(new Term(FIELD_WITH_OFFSETS, "w3")), BooleanClause.Occur.SHOULD) @@ -244,6 +316,16 @@ public void testExclusion() throws IOException { }); } + public void testExclusionNoPositions() throws IOException { + for (String field : new String[]{ FIELD_FREQS, FIELD_DOCS_ONLY }) { + Query q = new BooleanQuery.Builder() + .add(new TermQuery(new Term(field, "w3")), BooleanClause.Occur.SHOULD) + .add(new TermQuery(new Term(field, "zz")), BooleanClause.Occur.MUST_NOT) + .build(); + checkNoPositionsMatches(q, field, new boolean[]{ true, false, false, true, false }); + } + } + public void testConjunction() throws IOException { Query q = new BooleanQuery.Builder() .add(new TermQuery(new Term(FIELD_WITH_OFFSETS, "w3")), BooleanClause.Occur.MUST) @@ -258,6 +340,16 @@ public void testConjunction() throws IOException { }); } + public void testConjunctionNoPositions() throws IOException { + for (String field : new String[]{ FIELD_FREQS, FIELD_DOCS_ONLY }) { + Query q = new BooleanQuery.Builder() + .add(new TermQuery(new Term(field, "w3")), BooleanClause.Occur.MUST) + .add(new TermQuery(new Term(field, "w4")), BooleanClause.Occur.MUST) + .build(); + checkNoPositionsMatches(q, field, new boolean[]{ true, false, false, true, false }); + } + } + public void testWildcards() throws IOException { Query q = new PrefixQuery(new Term(FIELD_WITH_OFFSETS, "x")); checkMatches(q, FIELD_WITH_OFFSETS, new int[][]{ @@ -281,6 +373,13 @@ public void testWildcards() throws IOException { }); } + public void testWildcardsNoPositions() throws IOException { + for (String field : new String[]{ FIELD_FREQS, FIELD_DOCS_ONLY }) { + Query q = new PrefixQuery(new Term(field, "x")); + checkNoPositionsMatches(q, field, new boolean[]{ false, false, true, false, false }); + } + } + public void testSynonymQuery() throws IOException { Query q = new SynonymQuery(new Term(FIELD_WITH_OFFSETS, "w1"), new Term(FIELD_WITH_OFFSETS, "w2")); checkMatches(q, FIELD_WITH_OFFSETS, new int[][]{ @@ -292,6 +391,13 @@ public void testSynonymQuery() throws IOException { }); } + public void testSynonymQueryNoPositions() throws IOException { + for (String field : new String[]{ FIELD_FREQS, FIELD_DOCS_ONLY }) { + Query q = new SynonymQuery(new Term(field, "w1"), new Term(field, "w2")); + checkNoPositionsMatches(q, field, new boolean[]{ true, true, true, true, false }); + } + } + public void testMultipleFields() throws IOException { Query q = new BooleanQuery.Builder() .add(new TermQuery(new Term("id", "1")), BooleanClause.Occur.SHOULD) From f12b8b94e5d1815b4f659025b5e18684ffff19a8 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Sat, 7 Apr 2018 10:57:49 +0100 Subject: [PATCH 34/45] Address feedback: * Matches.getMatches() can be called multiple times (although not thread-safely) * Add assertion to CheckHits and QueryUtils that matching docs have non-null Matches --- .../org/apache/lucene/search/Matches.java | 20 +++++-- .../MultiTermQueryConstantScoreWrapper.java | 3 +- .../apache/lucene/search/SynonymQuery.java | 4 +- .../apache/lucene/search/TermInSetQuery.java | 3 +- .../org/apache/lucene/search/TermQuery.java | 16 +++--- .../java/org/apache/lucene/search/Weight.java | 2 +- .../lucene/search/TestMatchesIterator.java | 9 +++ .../org/apache/lucene/search/CheckHits.java | 56 ++++++++++++++++--- .../org/apache/lucene/search/QueryUtils.java | 2 + 9 files changed, 91 insertions(+), 24 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/search/Matches.java b/lucene/core/src/java/org/apache/lucene/search/Matches.java index 09ab45159287..cfa211cce4ef 100644 --- a/lucene/core/src/java/org/apache/lucene/search/Matches.java +++ b/lucene/core/src/java/org/apache/lucene/search/Matches.java @@ -23,6 +23,7 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -30,7 +31,9 @@ * Reports the positions and optionally offsets of all matching terms in a query * for a single document * - * To obtain a {@link MatchesIterator} for a particular field, call {@link #getMatches(String)} + * To obtain a {@link MatchesIterator} for a particular field, call {@link #getMatches(String)}. + * Note that you can call {@link #getMatches(String)} multiple times to retrieve new + * iterators, but it is not thread-safe. */ public interface Matches extends Iterable { @@ -108,14 +111,23 @@ interface MatchesIteratorSupplier { /** * Create a Matches for a single field */ - static Matches forField(String field, MatchesIteratorSupplier mif) { + static Matches forField(String field, MatchesIteratorSupplier mis) throws IOException { + MatchesIterator mi = mis.get(); + if (mi == null) { + return null; + } return new Matches() { + boolean cached = true; @Override public MatchesIterator getMatches(String f) throws IOException { - if (field.equals(f) == false) { + if (Objects.equals(field, f) == false) { return null; } - return mif.get(); + if (cached == false) { + return mis.get(); + } + cached = false; + return mi; } @Override diff --git a/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryConstantScoreWrapper.java b/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryConstantScoreWrapper.java index 6244226f143d..229977621196 100644 --- a/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryConstantScoreWrapper.java +++ b/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryConstantScoreWrapper.java @@ -211,8 +211,7 @@ public Matches matches(LeafReaderContext context, int doc) throws IOException { if (terms.hasPositions() == false) { return super.matches(context, doc); } - return Matches.forField(query.field, - () -> DisjunctionMatchesIterator.fromTermsEnum(context, doc, query.field, query.getTermsEnum(terms))); + return Matches.forField(query.field, () -> DisjunctionMatchesIterator.fromTermsEnum(context, doc, query.field, query.getTermsEnum(terms))); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java b/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java index d0c5854dab43..a364c9aacb06 100644 --- a/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java @@ -32,6 +32,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.index.TermState; import org.apache.lucene.index.TermStates; +import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.util.BytesRef; @@ -162,7 +163,8 @@ public void extractTerms(Set terms) { @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { String field = terms[0].field(); - if (context.reader().terms(field).hasPositions() == false) { + Terms terms = context.reader().terms(field); + if (terms == null || terms.hasPositions() == false) { return super.matches(context, doc); } return Matches.forField(field, () -> DisjunctionMatchesIterator.fromTerms(context, doc, field, Arrays.asList(SynonymQuery.this.terms))); diff --git a/lucene/core/src/java/org/apache/lucene/search/TermInSetQuery.java b/lucene/core/src/java/org/apache/lucene/search/TermInSetQuery.java index cf683a077f50..7145a83db30a 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TermInSetQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/TermInSetQuery.java @@ -222,7 +222,8 @@ public void extractTerms(Set terms) { @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - if (context.reader().terms(field).hasPositions() == false) { + Terms terms = context.reader().terms(field); + if (terms == null || terms.hasPositions() == false) { return super.matches(context, doc); } return Matches.forField(field, () -> DisjunctionMatchesIterator.fromTermsEnum(context, doc, field, termData.iterator())); diff --git a/lucene/core/src/java/org/apache/lucene/search/TermQuery.java b/lucene/core/src/java/org/apache/lucene/search/TermQuery.java index e7439b921fd5..b86f340f3346 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TermQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/TermQuery.java @@ -83,18 +83,20 @@ public void extractTerms(Set terms) { @Override public Matches matches(LeafReaderContext context, int doc) throws IOException { - if (context.reader().terms(term.field()).hasPositions() == false) { - return super.matches(context, doc); - } TermsEnum te = getTermsEnum(context); if (te == null) { return null; } - PostingsEnum pe = te.postings(null, PostingsEnum.OFFSETS); - if (pe.advance(doc) != doc) { - return null; + if (context.reader().terms(term.field()).hasPositions() == false) { + return super.matches(context, doc); } - return Matches.forField(term.field(), () -> new TermMatchesIterator(term.bytes(), pe)); + return Matches.forField(term.field(), () -> { + PostingsEnum pe = te.postings(null, PostingsEnum.OFFSETS); + if (pe.advance(doc) != doc) { + return null; + } + return new TermMatchesIterator(term.bytes(), pe); + }); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/search/Weight.java b/lucene/core/src/java/org/apache/lucene/search/Weight.java index b46d724719df..cb8ef46bc6d6 100644 --- a/lucene/core/src/java/org/apache/lucene/search/Weight.java +++ b/lucene/core/src/java/org/apache/lucene/search/Weight.java @@ -81,7 +81,7 @@ protected Weight(Query query) { */ public Matches matches(LeafReaderContext context, int doc) throws IOException { Scorer scorer = scorer(context); - if (scorer.iterator().advance(doc) != doc) { + if (scorer == null || scorer.iterator().advance(doc) != doc) { return null; } return Matches.MATCH_WITH_NO_TERMS; diff --git a/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java b/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java index 7b6a6c10b737..0d38cd588b72 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestMatchesIterator.java @@ -114,6 +114,7 @@ void checkMatches(Query q, String field, int[][] expected) throws IOException { return; } checkFieldMatches(it, expected[i]); + checkFieldMatches(matches.getMatches(field), expected[i]); // test multiple calls } } @@ -371,6 +372,14 @@ public void testWildcards() throws IOException { { 3, 0, 0, 0, 2, 1, 1, 3, 5, 2, 2, 6, 8, 4, 4, 12, 14 }, { 4 } }); + + } + + public void testNoMatchWildcards() throws IOException { + Query nomatch = new PrefixQuery(new Term(FIELD_WITH_OFFSETS, "wibble")); + Matches matches = searcher.createWeight(searcher.rewrite(nomatch), ScoreMode.COMPLETE_NO_SCORES, 1) + .matches(searcher.leafContexts.get(0), 0); + assertNull(matches); } public void testWildcardsNoPositions() throws IOException { diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/CheckHits.java b/lucene/test-framework/src/java/org/apache/lucene/search/CheckHits.java index 8d8b60b88278..ebd3a88d4fa3 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/search/CheckHits.java +++ b/lucene/test-framework/src/java/org/apache/lucene/search/CheckHits.java @@ -16,22 +16,22 @@ */ package org.apache.lucene.search; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - import java.io.IOException; import java.util.Locale; +import java.util.Random; import java.util.Set; import java.util.TreeSet; import java.util.regex.Pattern; -import java.util.Random; import junit.framework.Assert; - -import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.util.LuceneTestCase; +import static junit.framework.Assert.assertNotNull; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + /** * Utility class for asserting expected hits in tests. */ @@ -57,7 +57,7 @@ public static void checkNoMatchExplanations(Query q, String defaultFieldName, if (ignore.contains(Integer.valueOf(doc))) continue; Explanation exp = searcher.explain(q, doc); - Assert.assertNotNull("Explanation of [["+d+"]] for #"+doc+" is null", + assertNotNull("Explanation of [["+d+"]] for #"+doc+" is null", exp); Assert.assertFalse("Explanation of [["+d+"]] for #"+doc+ " doesn't indicate non-match: " + exp.toString(), @@ -301,6 +301,16 @@ public static void checkExplanations(Query query, (query, defaultFieldName, searcher, deep)); } + + /** + * Asserts that the result of calling {@link Weight#matches(LeafReaderContext, int)} + * for every document matching a query returns a non-null {@link Matches} + * @param query the query to test + * @param searcher the search to test against + */ + public static void checkMatches(Query query, IndexSearcher searcher) throws IOException { + searcher.search(query, new MatchesAsserter(query, searcher)); + } private static final Pattern COMPUTED_FROM_PATTERN = Pattern.compile(".*, computed as .* from:"); @@ -506,7 +516,7 @@ public void collect(int doc) throws IOException { ("exception in hitcollector of [["+d+"]] for #"+doc, e); } - Assert.assertNotNull("Explanation of [["+d+"]] for #"+doc+" is null", exp); + assertNotNull("Explanation of [["+d+"]] for #"+doc+" is null", exp); verifyExplanation(d,doc,scorer.score(),deep,exp); Assert.assertTrue("Explanation of [["+d+"]] for #"+ doc + " does not indicate match: " + exp.toString(), @@ -523,6 +533,36 @@ public ScoreMode scoreMode() { } } + /** + * Asserts that the {@link Matches} from a query is non-null whenever + * the document its created for is a hit + */ + public static class MatchesAsserter extends SimpleCollector { + + private final Weight weight; + private LeafReaderContext context; + + public MatchesAsserter(Query query, IndexSearcher searcher) throws IOException { + this.weight = searcher.createWeight(searcher.rewrite(query), ScoreMode.COMPLETE_NO_SCORES, 1); + } + + @Override + protected void doSetNextReader(LeafReaderContext context) throws IOException { + this.context = context; + } + + @Override + public void collect(int doc) throws IOException { + Matches matches = this.weight.matches(context, doc); + assertNotNull("Unexpected null Matches object in doc" + doc + " for query " + this.weight.getQuery(), matches); + } + + @Override + public ScoreMode scoreMode() { + return ScoreMode.COMPLETE_NO_SCORES; + } + } + public static void checkTopScores(Random random, Query query, IndexSearcher searcher) throws IOException { // Check it computed the top hits correctly doCheckTopScores(query, searcher, 1); diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java b/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java index fa113113f81a..865ab648c73d 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java +++ b/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java @@ -108,6 +108,7 @@ public static void checkExplanations (final Query q, final IndexSearcher s) thro * @see #checkSkipTo * @see #checkExplanations * @see #checkEqual + * @see CheckHits#checkMatches(Query, IndexSearcher) */ public static void check(Random random, Query q1, IndexSearcher s) { check(random, q1, s, true); @@ -125,6 +126,7 @@ public static void check(Random random, Query q1, IndexSearcher s, boolean wrap) check(random, q1, wrapUnderlyingReader(random, s, +1), false); } checkExplanations(q1,s); + CheckHits.checkMatches(q1, s); } } catch (IOException e) { throw new RuntimeException(e); From 050c2322ea322ea2aa5a1c1908768ecdb4bd1d08 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Sat, 7 Apr 2018 12:15:29 +0100 Subject: [PATCH 35/45] Fix ToParentBlockJoinQuery's matches --- .../lucene/search/join/ToParentBlockJoinQuery.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinQuery.java index 9f2bd9055061..36522381f81d 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinQuery.java +++ b/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinQuery.java @@ -28,6 +28,7 @@ import org.apache.lucene.search.Explanation; import org.apache.lucene.search.FilterWeight; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.ScorerSupplier; @@ -151,6 +152,17 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio } return Explanation.noMatch("Not a match"); } + + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + // The default implementation would delegate to the joinQuery's Weight, which + // matches on children. We need to match on the parent instead + Scorer scorer = scorer(context); + if (scorer == null || scorer.iterator().advance(doc) != doc) { + return null; + } + return Matches.MATCH_WITH_NO_TERMS; + } } private static class ParentApproximation extends DocIdSetIterator { From 73ef08e0609b20a2fedc227d2d213f51c1a370d1 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 9 Apr 2018 09:37:47 +0100 Subject: [PATCH 36/45] Add a (failing...) test --- .../matchhighlight/MatchHighlighter.java | 10 ++- .../matchhighlight/PassageCollector.java | 90 +++++++++++++++++++ .../matchhighlight/SnippetCollector.java | 5 +- .../matchhighlight/SourceAwareMatches.java | 29 ++++-- .../SourceAwareMatchesIterator.java | 12 +-- .../matchhighlight/TestMatchHighlighter.java | 88 ++++++++++++++++++ 6 files changed, 214 insertions(+), 20 deletions(-) create mode 100644 lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageCollector.java create mode 100644 lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchHighlighter.java diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchHighlighter.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchHighlighter.java index 16d485b00192..e2a6ade20a5d 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchHighlighter.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchHighlighter.java @@ -52,9 +52,11 @@ public TopHighlights highlight(Query query, TopDocs docs, Supplier fields; + + public PassageCollector(Set fields) { + this.fields = fields; + } + + @Override + public Document getHighlights() { + return document; + } + + @Override + public boolean needsField(String name) { + return fields.contains(name); + } + + @Override + public void collectSnippets(SourceAwareMatches matches, FieldInfo field, String text) throws IOException { + + BreakIterator breakIterator = BreakIterator.getSentenceInstance(Locale.ROOT); + breakIterator.setText(text); // nocommit is there a nicer way of doing this? + + StringBuilder snippet = new StringBuilder(); + int passageStart = breakIterator.first(); + int passageEnd = breakIterator.next(); + int snippetEnd = 0; + MatchesIterator mi = matches.getMatches(field, text); + + while (mi.next()) { + if (mi.startOffset() >= passageEnd) { + // finish the current snippet and advance the BreakIterator until we're surrounding the current match + if (snippet.length() > 0) { + snippet.append(text.substring(snippetEnd, passageEnd)); + document.add(new TextField(field.name, snippet.toString(), Field.Store.YES)); + snippet = new StringBuilder(); + } + while (passageEnd < mi.startOffset()) { + passageStart = passageEnd; + passageEnd = breakIterator.next(); + } + } + // append to the current snippet + snippet.append(text.substring(passageStart, mi.startOffset())); + snippet.append(""); // TODO make configurable + snippet.append(text.substring(mi.startOffset(), mi.endOffset())); + snippet.append(""); + passageStart = mi.endOffset() + 1; + } + + if (snippet.length() > 0) { + snippet.append(text.substring(snippetEnd, passageEnd)); + document.add(new TextField(field.name, snippet.toString(), Field.Store.YES)); + } + } + +} diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SnippetCollector.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SnippetCollector.java index b6f9c1e0febe..cc7e7390ef45 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SnippetCollector.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SnippetCollector.java @@ -17,7 +17,10 @@ package org.apache.lucene.search.matchhighlight; +import java.io.IOException; + import org.apache.lucene.document.Document; +import org.apache.lucene.index.FieldInfo; public interface SnippetCollector { @@ -25,6 +28,6 @@ public interface SnippetCollector { boolean needsField(String name); - void collectSnippets(SourceAwareMatches matches, String name, byte[] value); + void collectSnippets(SourceAwareMatches matches, FieldInfo field, String text) throws IOException; } diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SourceAwareMatches.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SourceAwareMatches.java index dff8f5775298..de1aa1c9ef95 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SourceAwareMatches.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SourceAwareMatches.java @@ -17,6 +17,8 @@ package org.apache.lucene.search.matchhighlight; +import java.io.Closeable; +import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -25,8 +27,9 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.search.Matches; import org.apache.lucene.search.MatchesIterator; +import org.apache.lucene.util.IOUtils; -class SourceAwareMatches { +class SourceAwareMatches implements Closeable { private final Matches in; private final Analyzer analyzer; @@ -38,13 +41,21 @@ class SourceAwareMatches { this.analyzer = analyzer; } - MatchesIterator getMatches(FieldInfo fi, byte[] value) { - SourceAwareMatchesIterator it = iterators.computeIfAbsent(fi, fieldinfo -> { - if (fieldinfo.getIndexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) - return SourceAwareMatchesIterator.wrapOffsets(in.getMatches(fieldinfo.name)); - return SourceAwareMatchesIterator.fromTokenStream(in.getMatches(fieldinfo.name), analyzer); - }); - it.addSource(value); - return it; + MatchesIterator getMatches(FieldInfo fi, String source) throws IOException { + if (iterators.containsKey(fi) == false) { + if (fi.getIndexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) { + iterators.put(fi, SourceAwareMatchesIterator.wrapOffsets(in.getMatches(fi.name))); + } + else { + iterators.put(fi, SourceAwareMatchesIterator.fromTokenStream(in.getMatches(fi.name), fi.name, analyzer)); + } + } + iterators.get(fi).addSource(source); + return iterators.get(fi); + } + + @Override + public void close() throws IOException { + IOUtils.close(iterators.values()); } } diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SourceAwareMatchesIterator.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SourceAwareMatchesIterator.java index 1466c63886f2..e5e0e54dced7 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SourceAwareMatchesIterator.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SourceAwareMatchesIterator.java @@ -17,10 +17,9 @@ package org.apache.lucene.search.matchhighlight; -import java.io.ByteArrayInputStream; import java.io.Closeable; import java.io.IOException; -import java.io.InputStreamReader; +import java.io.StringReader; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; @@ -31,7 +30,7 @@ public interface SourceAwareMatchesIterator extends MatchesIterator, Closeable { - void addSource(byte[] source); + void addSource(String source) throws IOException; static SourceAwareMatchesIterator wrapOffsets(MatchesIterator in) { return new SourceAwareMatchesIterator() { @@ -41,7 +40,7 @@ public void close() throws IOException { } @Override - public void addSource(byte[] source) { + public void addSource(String source) { // no-op - offsets already provided } @@ -89,14 +88,15 @@ static SourceAwareMatchesIterator fromTokenStream(MatchesIterator in, String fie int startPosition, endPosition, startOffset, endOffset; @Override - public void addSource(byte[] source) { + public void addSource(String source) throws IOException { sourceCount++; if (sourceCount > 0) { tsPosition += analyzer.getPositionIncrementGap(field); } - ts = analyzer.tokenStream(field, new InputStreamReader(new ByteArrayInputStream(source))); + ts = analyzer.tokenStream(field, new StringReader(source)); offsetAttribute = ts.getAttribute(OffsetAttribute.class); posIncAttribute = ts.getAttribute(PositionIncrementAttribute.class); + ts.reset(); } @Override diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchHighlighter.java b/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchHighlighter.java new file mode 100644 index 000000000000..8563d454dcca --- /dev/null +++ b/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchHighlighter.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.search.matchhighlight; + +import java.io.IOException; +import java.util.Collections; + +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.analysis.MockTokenizer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.LuceneTestCase; +import org.junit.After; +import org.junit.Before; + +public class TestMatchHighlighter extends LuceneTestCase { + + private MockAnalyzer indexAnalyzer; + private Directory dir; + + @Before + public void doBefore() { + indexAnalyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);//whitespace, punctuation, lowercase + indexAnalyzer.setEnableChecks(false); // highlighters will not necessarily exhaust tokenstreams + dir = newDirectory(); + } + + @After + public void doAfter() throws IOException { + dir.close(); + } + + public void testBasics() throws Exception { + RandomIndexWriter iw = new RandomIndexWriter(random(), dir, indexAnalyzer); + + Field body = new Field("body", "", TextField.TYPE_STORED); + Document doc = new Document(); + doc.add(body); + + body.setStringValue("This is a test. Just a test highlighting from postings. Feel free to ignore."); + iw.addDocument(doc); + body.setStringValue("Highlighting the first term. Hope it works."); + iw.addDocument(doc); + + IndexReader ir = iw.getReader(); + iw.close(); + + IndexSearcher searcher = newSearcher(ir); + Query query = new TermQuery(new Term("body", "highlighting")); + TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER); + assertEquals(2, topDocs.totalHits); + + MatchHighlighter highlighter = new MatchHighlighter(searcher, indexAnalyzer); + TopHighlights highlights = highlighter.highlight(query, topDocs, () -> new PassageCollector(Collections.singleton("body"))); + + assertEquals(2, highlights.docs.length); + assertEquals("Just a test highlighting from postings. ", highlights.docs[0].fields.get("body")); + assertEquals("Highlighting the first term. ", highlights.docs[0].fields.get("body")); + + ir.close(); + } + +} From a5f12ecde18e1cc3a44ffb58f10b4d63bb6982fe Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 9 Apr 2018 10:25:44 +0100 Subject: [PATCH 37/45] Check that non-matching docs have null Matches --- .../src/java/org/apache/lucene/search/CheckHits.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/CheckHits.java b/lucene/test-framework/src/java/org/apache/lucene/search/CheckHits.java index ebd3a88d4fa3..9ccf28f30d5e 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/search/CheckHits.java +++ b/lucene/test-framework/src/java/org/apache/lucene/search/CheckHits.java @@ -30,6 +30,7 @@ import static junit.framework.Assert.assertNotNull; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; /** @@ -535,12 +536,15 @@ public ScoreMode scoreMode() { /** * Asserts that the {@link Matches} from a query is non-null whenever - * the document its created for is a hit + * the document its created for is a hit. + * + * Also checks that the previous non-matching document has a {@code null} {@link Matches} */ public static class MatchesAsserter extends SimpleCollector { private final Weight weight; private LeafReaderContext context; + int lastCheckedDoc = -1; public MatchesAsserter(Query query, IndexSearcher searcher) throws IOException { this.weight = searcher.createWeight(searcher.rewrite(query), ScoreMode.COMPLETE_NO_SCORES, 1); @@ -549,12 +553,18 @@ public MatchesAsserter(Query query, IndexSearcher searcher) throws IOException { @Override protected void doSetNextReader(LeafReaderContext context) throws IOException { this.context = context; + this.lastCheckedDoc = -1; } @Override public void collect(int doc) throws IOException { Matches matches = this.weight.matches(context, doc); assertNotNull("Unexpected null Matches object in doc" + doc + " for query " + this.weight.getQuery(), matches); + if (lastCheckedDoc != doc - 1) { + assertNull("Unexpected non-null Matches object in non-matching doc" + doc + " for query " + this.weight.getQuery(), + this.weight.matches(context, doc - 1)); + } + lastCheckedDoc = doc; } @Override From 1eff4dd9e4d8306e0a9d9ffcc50b7de5801aaa99 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 16 Apr 2018 16:24:26 +0100 Subject: [PATCH 38/45] Basic tests --- .../matchhighlight/MatchHighlighter.java | 2 +- .../matchhighlight/PassageCollector.java | 8 ++--- .../SourceAwareMatchesIterator.java | 2 +- .../matchhighlight/TestMatchHighlighter.java | 30 ++++++++++++++++++- 4 files changed, 35 insertions(+), 7 deletions(-) diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchHighlighter.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchHighlighter.java index e2a6ade20a5d..b78be98b9ba5 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchHighlighter.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchHighlighter.java @@ -46,7 +46,7 @@ public MatchHighlighter(IndexSearcher searcher, Analyzer analyzer) { public TopHighlights highlight(Query query, TopDocs docs, Supplier collectorSupplier) throws IOException { HighlightDoc[] highlights = new HighlightDoc[docs.scoreDocs.length]; - Weight weight = searcher.createNormalizedWeight(query, ScoreMode.COMPLETE_NO_SCORES); + Weight weight = searcher.createWeight(searcher.rewrite(query), ScoreMode.COMPLETE_NO_SCORES, 1); int i = 0; for (ScoreDoc doc : docs.scoreDocs) { int contextOrd = ReaderUtil.subIndex(doc.doc, searcher.getIndexReader().leaves()); diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageCollector.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageCollector.java index 222b93bd99dd..d14badc42730 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageCollector.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageCollector.java @@ -75,14 +75,14 @@ public void collectSnippets(SourceAwareMatches matches, FieldInfo field, String } // append to the current snippet snippet.append(text.substring(passageStart, mi.startOffset())); - snippet.append(""); // TODO make configurable + snippet.append(""); // TODO make configurable snippet.append(text.substring(mi.startOffset(), mi.endOffset())); - snippet.append(""); - passageStart = mi.endOffset() + 1; + snippet.append(""); + passageStart = mi.endOffset(); } if (snippet.length() > 0) { - snippet.append(text.substring(snippetEnd, passageEnd)); + snippet.append(text.substring(passageStart, passageEnd)); document.add(new TextField(field.name, snippet.toString(), Field.Store.YES)); } } diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SourceAwareMatchesIterator.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SourceAwareMatchesIterator.java index e5e0e54dced7..916a7ca46766 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SourceAwareMatchesIterator.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SourceAwareMatchesIterator.java @@ -80,7 +80,7 @@ static SourceAwareMatchesIterator fromTokenStream(MatchesIterator in, String fie return new SourceAwareMatchesIterator() { int sourceCount = -1; - int tsPosition = 0; + int tsPosition = -1; TokenStream ts; OffsetAttribute offsetAttribute; PositionIncrementAttribute posIncAttribute; diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchHighlighter.java b/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchHighlighter.java index 8563d454dcca..73b95616413f 100644 --- a/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchHighlighter.java +++ b/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchHighlighter.java @@ -80,7 +80,35 @@ public void testBasics() throws Exception { assertEquals(2, highlights.docs.length); assertEquals("Just a test highlighting from postings. ", highlights.docs[0].fields.get("body")); - assertEquals("Highlighting the first term. ", highlights.docs[0].fields.get("body")); + assertEquals("Highlighting the first term. ", highlights.docs[1].fields.get("body")); + + ir.close(); + } + + // simple test highlighting last word. + public void testHighlightLastWord() throws Exception { + RandomIndexWriter iw = new RandomIndexWriter(random(), dir, indexAnalyzer); + + Field body = new Field("body", "", TextField.TYPE_STORED); + Document doc = new Document(); + doc.add(body); + + body.setStringValue("This is a test"); + iw.addDocument(doc); + + IndexReader ir = iw.getReader(); + iw.close(); + + IndexSearcher searcher = newSearcher(ir); + Query query = new TermQuery(new Term("body", "test")); + + TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER); + assertEquals(1, topDocs.totalHits); + + MatchHighlighter highlighter = new MatchHighlighter(searcher, indexAnalyzer); + TopHighlights highlights = highlighter.highlight(query, topDocs, () -> new PassageCollector(Collections.singleton("body"))); + assertEquals(1, highlights.docs.length); + assertEquals("This is a test", highlights.docs[0].fields.get("body")); ir.close(); } From 69ba22b7ee1016aa1c1deda8c77c893573269e95 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Tue, 17 Apr 2018 11:57:14 +0100 Subject: [PATCH 39/45] Some refactoring --- .../search/matchhighlight/PassageBuilder.java | 28 +++++++ .../matchhighlight/PassageCollector.java | 43 +++-------- .../SentencePassageBuilder.java | 74 +++++++++++++++++++ .../matchhighlight/TestMatchHighlighter.java | 6 +- 4 files changed, 117 insertions(+), 34 deletions(-) create mode 100644 lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageBuilder.java create mode 100644 lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SentencePassageBuilder.java diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageBuilder.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageBuilder.java new file mode 100644 index 000000000000..d135132956ed --- /dev/null +++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageBuilder.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.search.matchhighlight; + +import org.apache.lucene.util.BytesRef; + +public interface PassageBuilder { + + void addMatch(BytesRef term, int startOffset, int endOffset); + + Iterable getTopPassages(int topN); + +} diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageCollector.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageCollector.java index d14badc42730..12fbed2682c3 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageCollector.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageCollector.java @@ -18,9 +18,8 @@ package org.apache.lucene.search.matchhighlight; import java.io.IOException; -import java.text.BreakIterator; -import java.util.Locale; import java.util.Set; +import java.util.function.Function; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; @@ -33,9 +32,13 @@ public class PassageCollector implements SnippetCollector { private final Document document = new Document(); private final Set fields; + private final int maxPassagesPerField; + private final Function passageSource; - public PassageCollector(Set fields) { + public PassageCollector(Set fields, int maxPassagesPerField, Function passageSource) { this.fields = fields; + this.maxPassagesPerField = maxPassagesPerField; + this.passageSource = passageSource; } @Override @@ -51,40 +54,16 @@ public boolean needsField(String name) { @Override public void collectSnippets(SourceAwareMatches matches, FieldInfo field, String text) throws IOException { - BreakIterator breakIterator = BreakIterator.getSentenceInstance(Locale.ROOT); - breakIterator.setText(text); // nocommit is there a nicer way of doing this? - - StringBuilder snippet = new StringBuilder(); - int passageStart = breakIterator.first(); - int passageEnd = breakIterator.next(); - int snippetEnd = 0; MatchesIterator mi = matches.getMatches(field, text); - + PassageBuilder passageBuilder = passageSource.apply(text); while (mi.next()) { - if (mi.startOffset() >= passageEnd) { - // finish the current snippet and advance the BreakIterator until we're surrounding the current match - if (snippet.length() > 0) { - snippet.append(text.substring(snippetEnd, passageEnd)); - document.add(new TextField(field.name, snippet.toString(), Field.Store.YES)); - snippet = new StringBuilder(); - } - while (passageEnd < mi.startOffset()) { - passageStart = passageEnd; - passageEnd = breakIterator.next(); - } - } - // append to the current snippet - snippet.append(text.substring(passageStart, mi.startOffset())); - snippet.append(""); // TODO make configurable - snippet.append(text.substring(mi.startOffset(), mi.endOffset())); - snippet.append(""); - passageStart = mi.endOffset(); + passageBuilder.addMatch(mi.term(), mi.startOffset(), mi.endOffset()); } - if (snippet.length() > 0) { - snippet.append(text.substring(passageStart, passageEnd)); - document.add(new TextField(field.name, snippet.toString(), Field.Store.YES)); + for (String snippet : passageBuilder.getTopPassages(maxPassagesPerField)) { + document.add(new TextField(field.name, snippet, Field.Store.YES)); } + } } diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SentencePassageBuilder.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SentencePassageBuilder.java new file mode 100644 index 000000000000..929f7ddca414 --- /dev/null +++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SentencePassageBuilder.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.search.matchhighlight; + +import java.text.BreakIterator; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +import org.apache.lucene.util.BytesRef; + +public class SentencePassageBuilder implements PassageBuilder { + + private final BreakIterator breakIterator; + private final String source; + private final List passages = new ArrayList<>(); + + private StringBuilder currentPassage = new StringBuilder(); + private int matchStart = 0; + private int matchEnd; + + public SentencePassageBuilder(String source) { + this.breakIterator = BreakIterator.getSentenceInstance(Locale.ROOT); + this.breakIterator.setText(source); + this.source = source; + this.matchEnd = this.breakIterator.next(); + } + + @Override + public void addMatch(BytesRef term, int startOffset, int endOffset) { + if (startOffset > this.matchEnd) { + finishPassage(); + while (matchEnd <= startOffset) { + matchStart = matchEnd; + matchEnd = breakIterator.next(); + } + } + currentPassage.append(this.source.substring(matchStart, startOffset)); + currentPassage.append(""); + currentPassage.append(this.source.substring(startOffset, endOffset)); + currentPassage.append(""); + matchStart = endOffset; + } + + private void finishPassage() { + if (currentPassage.length() == 0) { + return; + } + currentPassage.append(this.source.substring(matchStart, matchEnd)); + passages.add(currentPassage.toString()); + currentPassage = new StringBuilder(); + } + + @Override + public Iterable getTopPassages(int topN) { + finishPassage(); + return passages; + } +} diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchHighlighter.java b/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchHighlighter.java index 73b95616413f..43a3b904caf1 100644 --- a/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchHighlighter.java +++ b/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchHighlighter.java @@ -76,7 +76,8 @@ public void testBasics() throws Exception { assertEquals(2, topDocs.totalHits); MatchHighlighter highlighter = new MatchHighlighter(searcher, indexAnalyzer); - TopHighlights highlights = highlighter.highlight(query, topDocs, () -> new PassageCollector(Collections.singleton("body"))); + TopHighlights highlights = highlighter.highlight(query, topDocs, + () -> new PassageCollector(Collections.singleton("body"), 1, SentencePassageBuilder::new)); assertEquals(2, highlights.docs.length); assertEquals("Just a test highlighting from postings. ", highlights.docs[0].fields.get("body")); @@ -106,7 +107,8 @@ public void testHighlightLastWord() throws Exception { assertEquals(1, topDocs.totalHits); MatchHighlighter highlighter = new MatchHighlighter(searcher, indexAnalyzer); - TopHighlights highlights = highlighter.highlight(query, topDocs, () -> new PassageCollector(Collections.singleton("body"))); + TopHighlights highlights = highlighter.highlight(query, topDocs, + () -> new PassageCollector(Collections.singleton("body"), 1, SentencePassageBuilder::new)); assertEquals(1, highlights.docs.length); assertEquals("This is a test", highlights.docs[0].fields.get("body")); From ff3b9387eab4d6f9114010013768f8e714f8a3da Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Tue, 17 Apr 2018 16:02:59 +0100 Subject: [PATCH 40/45] Multivalued fields --- ...Collector.java => HighlightCollector.java} | 4 +-- .../matchhighlight/MatchHighlighter.java | 8 ++--- .../search/matchhighlight/PassageBuilder.java | 2 +- .../matchhighlight/PassageCollector.java | 11 +++---- .../SentencePassageBuilder.java | 6 +++- .../matchhighlight/SourceAwareMatches.java | 3 +- .../SourceAwareMatchesIterator.java | 27 +++++++++++++---- .../matchhighlight/TestMatchHighlighter.java | 30 +++++++++++++++++++ 8 files changed, 71 insertions(+), 20 deletions(-) rename lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/{SnippetCollector.java => HighlightCollector.java} (88%) diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SnippetCollector.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/HighlightCollector.java similarity index 88% rename from lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SnippetCollector.java rename to lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/HighlightCollector.java index cc7e7390ef45..097f6fa9ee36 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SnippetCollector.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/HighlightCollector.java @@ -22,12 +22,12 @@ import org.apache.lucene.document.Document; import org.apache.lucene.index.FieldInfo; -public interface SnippetCollector { +public interface HighlightCollector { Document getHighlights(); boolean needsField(String name); - void collectSnippets(SourceAwareMatches matches, FieldInfo field, String text) throws IOException; + void collectHighlights(SourceAwareMatches matches, FieldInfo field, String text) throws IOException; } diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchHighlighter.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchHighlighter.java index b78be98b9ba5..59aa36fe30f1 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchHighlighter.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchHighlighter.java @@ -44,7 +44,7 @@ public MatchHighlighter(IndexSearcher searcher, Analyzer analyzer) { this.analyzer = analyzer; } - public TopHighlights highlight(Query query, TopDocs docs, Supplier collectorSupplier) throws IOException { + public TopHighlights highlight(Query query, TopDocs docs, Supplier collectorSupplier) throws IOException { HighlightDoc[] highlights = new HighlightDoc[docs.scoreDocs.length]; Weight weight = searcher.createWeight(searcher.rewrite(query), ScoreMode.COMPLETE_NO_SCORES, 1); int i = 0; @@ -64,9 +64,9 @@ public TopHighlights highlight(Query query, TopDocs docs, Supplier getTopPassages(int topN); diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageCollector.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageCollector.java index 12fbed2682c3..6ce6972eaa99 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageCollector.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageCollector.java @@ -25,9 +25,8 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.TextField; import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.search.MatchesIterator; -public class PassageCollector implements SnippetCollector { +public class PassageCollector implements HighlightCollector { private final Document document = new Document(); @@ -52,12 +51,14 @@ public boolean needsField(String name) { } @Override - public void collectSnippets(SourceAwareMatches matches, FieldInfo field, String text) throws IOException { + public void collectHighlights(SourceAwareMatches matches, FieldInfo field, String text) throws IOException { - MatchesIterator mi = matches.getMatches(field, text); + SourceAwareMatchesIterator mi = matches.getMatches(field, text); PassageBuilder passageBuilder = passageSource.apply(text); while (mi.next()) { - passageBuilder.addMatch(mi.term(), mi.startOffset(), mi.endOffset()); + if (passageBuilder.addMatch(mi.term(), mi.startOffset(), mi.endOffset()) == false) { + break; + } } for (String snippet : passageBuilder.getTopPassages(maxPassagesPerField)) { diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SentencePassageBuilder.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SentencePassageBuilder.java index 929f7ddca414..fab48993a726 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SentencePassageBuilder.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SentencePassageBuilder.java @@ -42,7 +42,10 @@ public SentencePassageBuilder(String source) { } @Override - public void addMatch(BytesRef term, int startOffset, int endOffset) { + public boolean addMatch(BytesRef term, int startOffset, int endOffset) { + if (startOffset > source.length()) { + return false; + } if (startOffset > this.matchEnd) { finishPassage(); while (matchEnd <= startOffset) { @@ -55,6 +58,7 @@ public void addMatch(BytesRef term, int startOffset, int endOffset) { currentPassage.append(this.source.substring(startOffset, endOffset)); currentPassage.append(""); matchStart = endOffset; + return true; } private void finishPassage() { diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SourceAwareMatches.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SourceAwareMatches.java index de1aa1c9ef95..1fa950fd0a15 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SourceAwareMatches.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SourceAwareMatches.java @@ -26,7 +26,6 @@ import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.search.Matches; -import org.apache.lucene.search.MatchesIterator; import org.apache.lucene.util.IOUtils; class SourceAwareMatches implements Closeable { @@ -41,7 +40,7 @@ class SourceAwareMatches implements Closeable { this.analyzer = analyzer; } - MatchesIterator getMatches(FieldInfo fi, String source) throws IOException { + SourceAwareMatchesIterator getMatches(FieldInfo fi, String source) throws IOException { if (iterators.containsKey(fi) == false) { if (fi.getIndexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) { iterators.put(fi, SourceAwareMatchesIterator.wrapOffsets(in.getMatches(fi.name))); diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SourceAwareMatchesIterator.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SourceAwareMatchesIterator.java index 916a7ca46766..c0b7d6702741 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SourceAwareMatchesIterator.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SourceAwareMatchesIterator.java @@ -34,6 +34,7 @@ public interface SourceAwareMatchesIterator extends MatchesIterator, Closeable { static SourceAwareMatchesIterator wrapOffsets(MatchesIterator in) { return new SourceAwareMatchesIterator() { + @Override public void close() throws IOException { // no-op @@ -81,17 +82,22 @@ static SourceAwareMatchesIterator fromTokenStream(MatchesIterator in, String fie int sourceCount = -1; int tsPosition = -1; + int offsetGap = 0; TokenStream ts; OffsetAttribute offsetAttribute; PositionIncrementAttribute posIncAttribute; int startPosition, endPosition, startOffset, endOffset; + boolean cached; + boolean restarted; @Override public void addSource(String source) throws IOException { sourceCount++; if (sourceCount > 0) { + ts.close(); tsPosition += analyzer.getPositionIncrementGap(field); + offsetGap += analyzer.getOffsetGap(field) - 1; } ts = analyzer.tokenStream(field, new StringReader(source)); offsetAttribute = ts.getAttribute(OffsetAttribute.class); @@ -101,17 +107,28 @@ public void addSource(String source) throws IOException { @Override public boolean next() throws IOException { - boolean next = in.next(); - if (next == false) { - return false; + if (cached) { + cached = false; + return true; + } + if (restarted == false) { + boolean next = in.next(); + if (next == false) { + return false; + } + } + else { + restarted = false; } startPosition = in.startPosition(); if (advanceTokenStream(startPosition) == false) { + restarted = true; return false; } startOffset = offsetAttribute.startOffset(); endPosition = in.endPosition(); if (advanceTokenStream(endPosition) == false) { + restarted = true; return false; } endOffset = offsetAttribute.endOffset(); @@ -140,12 +157,12 @@ public int endPosition() { @Override public int startOffset() throws IOException { - return startOffset; + return startOffset - offsetGap; } @Override public int endOffset() throws IOException { - return endOffset; + return endOffset - offsetGap; } @Override diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchHighlighter.java b/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchHighlighter.java index 43a3b904caf1..9c3dbaa8aa68 100644 --- a/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchHighlighter.java +++ b/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchHighlighter.java @@ -115,4 +115,34 @@ public void testHighlightLastWord() throws Exception { ir.close(); } + public void testMultiValuedField() throws Exception { + + RandomIndexWriter iw = new RandomIndexWriter(random(), dir, indexAnalyzer); + Document doc = new Document(); + doc.add(new TextField("body", "This is the first sentence, and a fine sentence it is too", Field.Store.YES)); + doc.add(new TextField("body", "And this is the second sentence", Field.Store.YES)); + doc.add(new TextField("body", "And a third sentence too!", Field.Store.YES)); + iw.addDocument(doc); + + IndexReader ir = iw.getReader(); + iw.close(); + + IndexSearcher searcher = newSearcher(ir); + Query query = new TermQuery(new Term("body", "sentence")); + TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER); + assertEquals(1, topDocs.totalHits); + + MatchHighlighter highlighter = new MatchHighlighter(searcher, indexAnalyzer); + TopHighlights highlights = highlighter.highlight(query, topDocs, + () -> new PassageCollector(Collections.singleton("body"), 1, SentencePassageBuilder::new)); + assertEquals(1, highlights.docs.length); + String[] values = highlights.docs[0].fields.getValues("body"); + assertEquals(3, values.length); + assertEquals("This is the first sentence, and a fine sentence it is too", values[0]); + assertEquals("And this is the second sentence", values[1]); + assertEquals("And a third sentence too!", values[2]); + + ir.close(); + } + } From bc560f8ccf2011d27c2f8f6d5fd8f696b35a480b Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Tue, 17 Apr 2018 16:21:59 +0100 Subject: [PATCH 41/45] Limit number of passages per field --- .../search/matchhighlight/PassageBuilder.java | 2 ++ .../matchhighlight/PassageCollector.java | 25 +++++++++++-------- .../SentencePassageBuilder.java | 13 +++++++--- .../matchhighlight/TestMatchHighlighter.java | 7 +++++- 4 files changed, 33 insertions(+), 14 deletions(-) diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageBuilder.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageBuilder.java index 79172bf8d1a6..034c2ecfe700 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageBuilder.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageBuilder.java @@ -21,6 +21,8 @@ public interface PassageBuilder { + void addSource(String source); + boolean addMatch(BytesRef term, int startOffset, int endOffset); Iterable getTopPassages(int topN); diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageCollector.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageCollector.java index 6ce6972eaa99..ba790675e538 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageCollector.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageCollector.java @@ -18,8 +18,10 @@ package org.apache.lucene.search.matchhighlight; import java.io.IOException; +import java.util.HashMap; +import java.util.Map; import java.util.Set; -import java.util.function.Function; +import java.util.function.Supplier; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; @@ -28,13 +30,12 @@ public class PassageCollector implements HighlightCollector { - private final Document document = new Document(); - private final Set fields; private final int maxPassagesPerField; - private final Function passageSource; + private final Supplier passageSource; + private final Map builders = new HashMap<>(); - public PassageCollector(Set fields, int maxPassagesPerField, Function passageSource) { + public PassageCollector(Set fields, int maxPassagesPerField, Supplier passageSource) { this.fields = fields; this.maxPassagesPerField = maxPassagesPerField; this.passageSource = passageSource; @@ -42,6 +43,12 @@ public PassageCollector(Set fields, int maxPassagesPerField, Function passages : builders.entrySet()) { + for (String snippet : passages.getValue().getTopPassages(maxPassagesPerField)) { + document.add(new TextField(passages.getKey(), snippet, Field.Store.YES)); + } + } return document; } @@ -54,17 +61,15 @@ public boolean needsField(String name) { public void collectHighlights(SourceAwareMatches matches, FieldInfo field, String text) throws IOException { SourceAwareMatchesIterator mi = matches.getMatches(field, text); - PassageBuilder passageBuilder = passageSource.apply(text); + PassageBuilder passageBuilder = builders.computeIfAbsent(field.name, t -> passageSource.get()); + passageBuilder.addSource(text); + while (mi.next()) { if (passageBuilder.addMatch(mi.term(), mi.startOffset(), mi.endOffset()) == false) { break; } } - for (String snippet : passageBuilder.getTopPassages(maxPassagesPerField)) { - document.add(new TextField(field.name, snippet, Field.Store.YES)); - } - } } diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SentencePassageBuilder.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SentencePassageBuilder.java index fab48993a726..b3193c06c462 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SentencePassageBuilder.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SentencePassageBuilder.java @@ -27,18 +27,25 @@ public class SentencePassageBuilder implements PassageBuilder { private final BreakIterator breakIterator; - private final String source; + private final List passages = new ArrayList<>(); private StringBuilder currentPassage = new StringBuilder(); private int matchStart = 0; private int matchEnd; + private String source; - public SentencePassageBuilder(String source) { + public SentencePassageBuilder() { this.breakIterator = BreakIterator.getSentenceInstance(Locale.ROOT); + } + + @Override + public void addSource(String source) { + finishPassage(); this.breakIterator.setText(source); this.source = source; this.matchEnd = this.breakIterator.next(); + this.matchStart = 0; } @Override @@ -73,6 +80,6 @@ private void finishPassage() { @Override public Iterable getTopPassages(int topN) { finishPassage(); - return passages; + return passages.subList(0, topN); } } diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchHighlighter.java b/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchHighlighter.java index 9c3dbaa8aa68..30a5b9eb02a8 100644 --- a/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchHighlighter.java +++ b/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchHighlighter.java @@ -134,7 +134,7 @@ public void testMultiValuedField() throws Exception { MatchHighlighter highlighter = new MatchHighlighter(searcher, indexAnalyzer); TopHighlights highlights = highlighter.highlight(query, topDocs, - () -> new PassageCollector(Collections.singleton("body"), 1, SentencePassageBuilder::new)); + () -> new PassageCollector(Collections.singleton("body"), 3, SentencePassageBuilder::new)); assertEquals(1, highlights.docs.length); String[] values = highlights.docs[0].fields.getValues("body"); assertEquals(3, values.length); @@ -142,6 +142,11 @@ public void testMultiValuedField() throws Exception { assertEquals("And this is the second sentence", values[1]); assertEquals("And a third sentence too!", values[2]); + // again, this time with only one passage per field + highlights = highlighter.highlight(query, topDocs, + () -> new PassageCollector(Collections.singleton("body"), 1, SentencePassageBuilder::new)); + assertEquals(1, highlights.docs[0].fields.getValues("body").length); + ir.close(); } From 72e7ac57aa3b80af5f3e31bce23916e86593eafa Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 6 Jun 2018 09:24:04 +0100 Subject: [PATCH 42/45] WIP --- .../lucene/index/LeafReaderContext.java | 2 +- .../matchhighlight/HighlightCollector.java | 12 +- .../matchhighlight/MatchHighlighter.java | 73 +++++-- .../search/matchhighlight/OffsetsReader.java | 108 +++++++++++ .../search/matchhighlight/PassageBuilder.java | 4 +- .../matchhighlight/PassageCollector.java | 37 +++- .../SentencePassageBuilder.java | 4 +- .../matchhighlight/SourceAwareMatches.java | 60 ------ .../SourceAwareMatchesIterator.java | 182 ------------------ 9 files changed, 209 insertions(+), 273 deletions(-) create mode 100644 lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsReader.java delete mode 100644 lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SourceAwareMatches.java delete mode 100644 lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SourceAwareMatchesIterator.java diff --git a/lucene/core/src/java/org/apache/lucene/index/LeafReaderContext.java b/lucene/core/src/java/org/apache/lucene/index/LeafReaderContext.java index 12dc8c18a5cf..23d38c2aee92 100644 --- a/lucene/core/src/java/org/apache/lucene/index/LeafReaderContext.java +++ b/lucene/core/src/java/org/apache/lucene/index/LeafReaderContext.java @@ -35,7 +35,7 @@ public final class LeafReaderContext extends IndexReaderContext { /** * Creates a new {@link LeafReaderContext} */ - LeafReaderContext(CompositeReaderContext parent, LeafReader reader, + public LeafReaderContext(CompositeReaderContext parent, LeafReader reader, int ord, int docBase, int leafOrd, int leafDocBase) { super(parent, ord, docBase); this.ord = leafOrd; diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/HighlightCollector.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/HighlightCollector.java index 097f6fa9ee36..ec6991f326bf 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/HighlightCollector.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/HighlightCollector.java @@ -18,16 +18,22 @@ package org.apache.lucene.search.matchhighlight; import java.io.IOException; +import java.util.Set; import org.apache.lucene.document.Document; -import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.search.Matches; public interface HighlightCollector { Document getHighlights(); - boolean needsField(String name); + Set requiredFields(); - void collectHighlights(SourceAwareMatches matches, FieldInfo field, String text) throws IOException; + default boolean needsField(String field) { + return requiredFields().contains(field); + } + void collectHighlights(String field, String text, int offset) throws IOException; + + void setMatches(Matches matches); } diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchHighlighter.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchHighlighter.java index 59aa36fe30f1..8bc9246f1b5c 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchHighlighter.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchHighlighter.java @@ -18,14 +18,22 @@ package org.apache.lucene.search.matchhighlight; import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; import java.util.function.Supplier; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Document; import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.index.StoredFieldVisitor; +import org.apache.lucene.index.Terms; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; @@ -49,25 +57,59 @@ public TopHighlights highlight(Query query, TopDocs docs, Supplier fromTermVectors = new HashMap<>(); + Set fromAnalysis = new HashSet<>(); + + for (String field : collector.requiredFields()) { + FieldInfo fi = fis.fieldInfo(field); + if (fi != null && fi.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) < 0) { + if (fi.hasVectors()) { + Terms terms = defaultContext.reader().getTermVector(doc, field); + if (terms != null && terms.hasOffsets()) { + fromTermVectors.put(field, terms); + continue; + } + } + fromAnalysis.add(field); + } + } + + if (fromTermVectors.size() == 0 && fromAnalysis.size() == 0) { + return defaultContext; + } + + LeafReader reader = new OffsetsReader(defaultContext.reader(), doc - defaultContext.docBase, fromTermVectors, fromAnalysis, analyzer); + return new LeafReaderContext(defaultContext.parent, reader, 0, 0, 0, 0); + } + private class HighlightingFieldVisitor extends StoredFieldVisitor { - final SourceAwareMatches matches; final HighlightCollector collector; + final Map offsets = new HashMap<>(); - private HighlightingFieldVisitor(SourceAwareMatches matches, HighlightCollector collector) { - this.matches = matches; + private HighlightingFieldVisitor(HighlightCollector collector) { this.collector = collector; } @@ -76,13 +118,20 @@ Document getHighlights() { } @Override - public Status needsField(FieldInfo fieldInfo) throws IOException { + public Status needsField(FieldInfo fieldInfo) { return collector.needsField(fieldInfo.name) ? Status.YES : Status.NO; } @Override public void stringField(FieldInfo fieldInfo, byte[] value) throws IOException { - collector.collectHighlights(matches, fieldInfo, new String(value)); + String text = new String(value); + collector.collectHighlights(fieldInfo.name, text, offsets.getOrDefault(fieldInfo.name, 0)); + offsets.compute(fieldInfo.name, (n, i) -> { + if (i == null) { + i = 0; + } + return i + text.length() + analyzer.getOffsetGap(n); + }); } } diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsReader.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsReader.java new file mode 100644 index 000000000000..7bdd22ac302e --- /dev/null +++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsReader.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.search.matchhighlight; + +import java.io.IOException; +import java.util.Map; +import java.util.Set; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FilterLeafReader; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.PostingsEnum; +import org.apache.lucene.index.StoredFieldVisitor; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.index.memory.MemoryIndex; + +class OffsetsReader extends FilterLeafReader { + + private final int doc; + private final Map fromTermVectors; + private final Set fromAnalysis; + private final LeafReader reanalyzedReader; + + OffsetsReader(LeafReader reader, int doc, Map fromTermVectors, Set fromAnalysis, Analyzer analyzer) throws IOException { + super(reader); + this.doc = doc; + this.fromTermVectors = fromTermVectors; + this.fromAnalysis = fromAnalysis; + MemoryIndex mi = new MemoryIndex(true); + reader.document(doc, new StoredFieldVisitor() { + @Override + public Status needsField(FieldInfo fieldInfo) throws IOException { + return fromAnalysis.contains(fieldInfo.name) ? Status.YES : Status.NO; + } + + @Override + public void stringField(FieldInfo fieldInfo, byte[] value) throws IOException { + mi.addField(fieldInfo.name, new String(value), analyzer); + } + }); + mi.freeze(); + this.reanalyzedReader = (LeafReader) mi.createSearcher().getIndexReader(); + } + + @Override + public Terms terms(String field) throws IOException { + if (fromTermVectors.containsKey(field)) { + return new SingleDocTerms(fromTermVectors.get(field)); + } + if (fromAnalysis.contains(field)) { + return new SingleDocTerms(reanalyzedReader.terms(field)); + } + return in.terms(field); + } + + @Override + public CacheHelper getCoreCacheHelper() { + return null; + } + + @Override + public CacheHelper getReaderCacheHelper() { + return null; + } + + private class SingleDocTerms extends FilterTerms { + + SingleDocTerms(Terms in) { + super(in); + } + + @Override + public TermsEnum iterator() throws IOException { + return new FilterTermsEnum(in.iterator()) { + @Override + public PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException { + return new FilterPostingsEnum(in.postings(reuse, flags)) { + @Override + public int advance(int target) throws IOException { + if (target == doc) { + in.advance(0); + return target; + } + return NO_MORE_DOCS; + } + }; + } + }; + } + } +} diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageBuilder.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageBuilder.java index 034c2ecfe700..51f0c02b46d4 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageBuilder.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageBuilder.java @@ -17,13 +17,11 @@ package org.apache.lucene.search.matchhighlight; -import org.apache.lucene.util.BytesRef; - public interface PassageBuilder { void addSource(String source); - boolean addMatch(BytesRef term, int startOffset, int endOffset); + boolean addMatch(int startOffset, int endOffset); Iterable getTopPassages(int topN); diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageCollector.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageCollector.java index ba790675e538..e600c296db35 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageCollector.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageCollector.java @@ -26,7 +26,8 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.search.Matches; +import org.apache.lucene.search.MatchesIterator; public class PassageCollector implements HighlightCollector { @@ -35,12 +36,20 @@ public class PassageCollector implements HighlightCollector { private final Supplier passageSource; private final Map builders = new HashMap<>(); + private Matches matches; + private final Map iterators = new HashMap<>(); + public PassageCollector(Set fields, int maxPassagesPerField, Supplier passageSource) { this.fields = fields; this.maxPassagesPerField = maxPassagesPerField; this.passageSource = passageSource; } + @Override + public void setMatches(Matches matches) { + this.matches = matches; + } + @Override public Document getHighlights() { Document document = new Document(); @@ -53,22 +62,32 @@ public Document getHighlights() { } @Override - public boolean needsField(String name) { - return fields.contains(name); + public Set requiredFields() { + return fields; } @Override - public void collectHighlights(SourceAwareMatches matches, FieldInfo field, String text) throws IOException { + public void collectHighlights(String field, String text, int offset) throws IOException { - SourceAwareMatchesIterator mi = matches.getMatches(field, text); - PassageBuilder passageBuilder = builders.computeIfAbsent(field.name, t -> passageSource.get()); + MatchesIterator mi = iterators.get(field); + if (mi == null) { + mi = matches.getMatches(field); + if (mi == null) { + return; + } + iterators.put(field, mi); + if (mi.next() == false) { + return; + } + } + PassageBuilder passageBuilder = builders.computeIfAbsent(field, t -> passageSource.get()); passageBuilder.addSource(text); - while (mi.next()) { - if (passageBuilder.addMatch(mi.term(), mi.startOffset(), mi.endOffset()) == false) { + do { + if (passageBuilder.addMatch(mi.startOffset() - offset, mi.endOffset() - offset) == false) { break; } - } + } while (mi.next()); } diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SentencePassageBuilder.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SentencePassageBuilder.java index b3193c06c462..d82f088b83d2 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SentencePassageBuilder.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SentencePassageBuilder.java @@ -22,8 +22,6 @@ import java.util.List; import java.util.Locale; -import org.apache.lucene.util.BytesRef; - public class SentencePassageBuilder implements PassageBuilder { private final BreakIterator breakIterator; @@ -49,7 +47,7 @@ public void addSource(String source) { } @Override - public boolean addMatch(BytesRef term, int startOffset, int endOffset) { + public boolean addMatch(int startOffset, int endOffset) { if (startOffset > source.length()) { return false; } diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SourceAwareMatches.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SourceAwareMatches.java deleted file mode 100644 index 1fa950fd0a15..000000000000 --- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SourceAwareMatches.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.lucene.search.matchhighlight; - -import java.io.Closeable; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.IndexOptions; -import org.apache.lucene.search.Matches; -import org.apache.lucene.util.IOUtils; - -class SourceAwareMatches implements Closeable { - - private final Matches in; - private final Analyzer analyzer; - - private final Map iterators = new HashMap<>(); - - SourceAwareMatches(Matches in, Analyzer analyzer) { - this.in = in; - this.analyzer = analyzer; - } - - SourceAwareMatchesIterator getMatches(FieldInfo fi, String source) throws IOException { - if (iterators.containsKey(fi) == false) { - if (fi.getIndexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) { - iterators.put(fi, SourceAwareMatchesIterator.wrapOffsets(in.getMatches(fi.name))); - } - else { - iterators.put(fi, SourceAwareMatchesIterator.fromTokenStream(in.getMatches(fi.name), fi.name, analyzer)); - } - } - iterators.get(fi).addSource(source); - return iterators.get(fi); - } - - @Override - public void close() throws IOException { - IOUtils.close(iterators.values()); - } -} diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SourceAwareMatchesIterator.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SourceAwareMatchesIterator.java deleted file mode 100644 index c0b7d6702741..000000000000 --- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SourceAwareMatchesIterator.java +++ /dev/null @@ -1,182 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.lucene.search.matchhighlight; - -import java.io.Closeable; -import java.io.IOException; -import java.io.StringReader; - -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; -import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; -import org.apache.lucene.search.MatchesIterator; -import org.apache.lucene.util.BytesRef; - -public interface SourceAwareMatchesIterator extends MatchesIterator, Closeable { - - void addSource(String source) throws IOException; - - static SourceAwareMatchesIterator wrapOffsets(MatchesIterator in) { - return new SourceAwareMatchesIterator() { - - @Override - public void close() throws IOException { - // no-op - } - - @Override - public void addSource(String source) { - // no-op - offsets already provided - } - - @Override - public boolean next() throws IOException { - return in.next(); - } - - @Override - public int startPosition() { - return in.startPosition(); - } - - @Override - public int endPosition() { - return in.endPosition(); - } - - @Override - public int startOffset() throws IOException { - return in.startOffset(); - } - - @Override - public int endOffset() throws IOException { - return in.endOffset(); - } - - @Override - public BytesRef term() { - return in.term(); - } - }; - } - - static SourceAwareMatchesIterator fromTokenStream(MatchesIterator in, String field, Analyzer analyzer) { - return new SourceAwareMatchesIterator() { - - int sourceCount = -1; - int tsPosition = -1; - int offsetGap = 0; - TokenStream ts; - OffsetAttribute offsetAttribute; - PositionIncrementAttribute posIncAttribute; - - int startPosition, endPosition, startOffset, endOffset; - boolean cached; - boolean restarted; - - @Override - public void addSource(String source) throws IOException { - sourceCount++; - if (sourceCount > 0) { - ts.close(); - tsPosition += analyzer.getPositionIncrementGap(field); - offsetGap += analyzer.getOffsetGap(field) - 1; - } - ts = analyzer.tokenStream(field, new StringReader(source)); - offsetAttribute = ts.getAttribute(OffsetAttribute.class); - posIncAttribute = ts.getAttribute(PositionIncrementAttribute.class); - ts.reset(); - } - - @Override - public boolean next() throws IOException { - if (cached) { - cached = false; - return true; - } - if (restarted == false) { - boolean next = in.next(); - if (next == false) { - return false; - } - } - else { - restarted = false; - } - startPosition = in.startPosition(); - if (advanceTokenStream(startPosition) == false) { - restarted = true; - return false; - } - startOffset = offsetAttribute.startOffset(); - endPosition = in.endPosition(); - if (advanceTokenStream(endPosition) == false) { - restarted = true; - return false; - } - endOffset = offsetAttribute.endOffset(); - return true; - } - - private boolean advanceTokenStream(int targetPos) throws IOException { - while (tsPosition < targetPos) { - if (ts.incrementToken() == false) { - return false; - } - tsPosition += posIncAttribute.getPositionIncrement(); - } - return true; - } - - @Override - public int startPosition() { - return startPosition; - } - - @Override - public int endPosition() { - return endPosition; - } - - @Override - public int startOffset() throws IOException { - return startOffset - offsetGap; - } - - @Override - public int endOffset() throws IOException { - return endOffset - offsetGap; - } - - @Override - public BytesRef term() { - return in.term(); - } - - @Override - public void close() throws IOException { - if (ts != null) { - ts.close(); - } - } - }; - } - -} From 22cdb4c113c4095d8d639835955b200194606c75 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 6 Jun 2018 15:19:12 +0100 Subject: [PATCH 43/45] Simplification --- .../matchhighlight/HighlightCollector.java | 39 ------- .../matchhighlight/MatchHighlighter.java | 12 +-- .../search/matchhighlight/PassageBuilder.java | 102 +++++++++++++++++- .../matchhighlight/PassageCollector.java | 46 ++++---- .../SentencePassageBuilder.java | 83 -------------- .../matchhighlight/TestMatchHighlighter.java | 11 +- 6 files changed, 137 insertions(+), 156 deletions(-) delete mode 100644 lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/HighlightCollector.java delete mode 100644 lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SentencePassageBuilder.java diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/HighlightCollector.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/HighlightCollector.java deleted file mode 100644 index ec6991f326bf..000000000000 --- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/HighlightCollector.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.lucene.search.matchhighlight; - -import java.io.IOException; -import java.util.Set; - -import org.apache.lucene.document.Document; -import org.apache.lucene.search.Matches; - -public interface HighlightCollector { - - Document getHighlights(); - - Set requiredFields(); - - default boolean needsField(String field) { - return requiredFields().contains(field); - } - - void collectHighlights(String field, String text, int offset) throws IOException; - - void setMatches(Matches matches); -} diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchHighlighter.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchHighlighter.java index 8bc9246f1b5c..ad5634c99f2c 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchHighlighter.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchHighlighter.java @@ -52,12 +52,12 @@ public MatchHighlighter(IndexSearcher searcher, Analyzer analyzer) { this.analyzer = analyzer; } - public TopHighlights highlight(Query query, TopDocs docs, Supplier collectorSupplier) throws IOException { + public TopHighlights highlight(Query query, TopDocs docs, Supplier collectorSupplier) throws IOException { HighlightDoc[] highlights = new HighlightDoc[docs.scoreDocs.length]; Weight weight = searcher.createWeight(searcher.rewrite(query), ScoreMode.COMPLETE_NO_SCORES, 1); int i = 0; for (ScoreDoc doc : docs.scoreDocs) { - HighlightCollector collector = collectorSupplier.get(); + PassageCollector collector = collectorSupplier.get(); LeafReaderContext ctx = getReaderContext(doc.doc, collector); Matches matches = weight.matches(ctx, doc.doc - ctx.docBase); collector.setMatches(matches); @@ -68,7 +68,7 @@ public TopHighlights highlight(Query query, TopDocs docs, Supplier offsets = new HashMap<>(); - private HighlightingFieldVisitor(HighlightCollector collector) { + private HighlightingFieldVisitor(PassageCollector collector) { this.collector = collector; } @@ -119,7 +119,7 @@ Document getHighlights() { @Override public Status needsField(FieldInfo fieldInfo) { - return collector.needsField(fieldInfo.name) ? Status.YES : Status.NO; + return collector.requiredFields().contains(fieldInfo.name) ? Status.YES : Status.NO; } @Override diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageBuilder.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageBuilder.java index 51f0c02b46d4..7bec9ff7df0e 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageBuilder.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageBuilder.java @@ -17,12 +17,106 @@ package org.apache.lucene.search.matchhighlight; -public interface PassageBuilder { +import java.io.IOException; +import java.text.BreakIterator; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; - void addSource(String source); +import org.apache.lucene.search.MatchesIterator; - boolean addMatch(int startOffset, int endOffset); +public class PassageBuilder { - Iterable getTopPassages(int topN); + private static final int PAD_WIDTH = 10; + + private final int snippetWidth; + private final String markupStart = ""; + private final String markupEnd = ""; + + private final BreakIterator wordBreakIterator = BreakIterator.getWordInstance(Locale.ROOT); + + private final List passages = new ArrayList<>(); + + private StringBuilder currentPassage; + private int passageStart; + private int passageEnd; + + public PassageBuilder(int snippetWidth) { + this.snippetWidth = snippetWidth; + } + + public Iterable getTopPassages(int topN) { + return passages.subList(0, topN); + } + + public boolean build(String source, MatchesIterator mi, int offset) throws IOException { + currentPassage = new StringBuilder(); + passageStart = passageEnd = 0; + wordBreakIterator.setText(source); + do { + int startOffset = mi.startOffset() - offset; + if (startOffset >= 0) { + if (startOffset > source.length()) { + finishPassage(source); + return true; + } + int endOffset = mi.endOffset() - offset; + if (currentPassage.length() == 0) { + passageStart = startOffset; + markup(source, startOffset, endOffset); + } + else { + // subsequent passage + // if total width of passage would exceed snippetWidth then finish this passage and start a new one + if ((endOffset - passageStart + PAD_WIDTH * 2) > snippetWidth) { + finishPassage(source); + currentPassage = new StringBuilder(); + passageStart = startOffset; + markup(source, startOffset, endOffset); + } + // otherwise we add to this passage + else { + currentPassage.append(source.substring(passageEnd, startOffset)); + markup(source, startOffset, endOffset); + } + } + } + } while (mi.next()); + finishPassage(source); + return false; + } + + private void markup(String source, int startOffset, int endOffset) { + currentPassage.append(markupStart).append(source.substring(startOffset, endOffset)).append(markupEnd); + passageEnd = endOffset; + } + + private void finishPassage(String source) { + int padWidth = Math.max(PAD_WIDTH, ((snippetWidth - (passageEnd - passageStart)) / 2)); + int start = Math.max(0, passageStart - padWidth); + if (start != 0) { + // we're not at the beginning of the source, so find an appropriate word boundary + // to start on + start = nextWordStartAfter(start, passageStart, source); + } + int endPadWidth = padWidth - (passageStart - start) + padWidth; + int end = Math.min(source.length(), passageEnd + endPadWidth); + if (end != source.length()) { + end = wordBreakIterator.preceding(end); + } + String passage = source.substring(start, passageStart) + currentPassage.toString() + source.substring(passageEnd, end); + passages.add(passage); + } + + private int nextWordStartAfter(int position, int maxLength, String source) { + int start = wordBreakIterator.following(position); + while (start <= maxLength) { + if (Character.isLetter(source.codePointAt(start))) { + break; + } + start++; + } + return start; + } } diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageCollector.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageCollector.java index e600c296db35..0e17c4d2b9e8 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageCollector.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageCollector.java @@ -29,7 +29,7 @@ import org.apache.lucene.search.Matches; import org.apache.lucene.search.MatchesIterator; -public class PassageCollector implements HighlightCollector { +public class PassageCollector { private final Set fields; private final int maxPassagesPerField; @@ -37,7 +37,16 @@ public class PassageCollector implements HighlightCollector { private final Map builders = new HashMap<>(); private Matches matches; - private final Map iterators = new HashMap<>(); + private final Map iterators = new HashMap<>(); + + private static class Iterator { + final MatchesIterator mi; + boolean exhausted = false; + + private Iterator(MatchesIterator mi) { + this.mi = mi; + } + } public PassageCollector(Set fields, int maxPassagesPerField, Supplier passageSource) { this.fields = fields; @@ -45,12 +54,10 @@ public PassageCollector(Set fields, int maxPassagesPerField, Supplier passages : builders.entrySet()) { @@ -61,33 +68,34 @@ public Document getHighlights() { return document; } - @Override public Set requiredFields() { return fields; } - @Override public void collectHighlights(String field, String text, int offset) throws IOException { - MatchesIterator mi = iterators.get(field); + Iterator mi = iterators.get(field); if (mi == null) { - mi = matches.getMatches(field); - if (mi == null) { - return; + mi = new Iterator(matches.getMatches(field)); + if (mi.mi == null) { + mi.exhausted = true; } iterators.put(field, mi); - if (mi.next() == false) { - return; + if (mi.exhausted == false) { + assert mi.mi != null; + if (mi.mi.next() == false) { + mi.exhausted = true; + } } } - PassageBuilder passageBuilder = builders.computeIfAbsent(field, t -> passageSource.get()); - passageBuilder.addSource(text); + if (mi.exhausted) { + return; + } - do { - if (passageBuilder.addMatch(mi.startOffset() - offset, mi.endOffset() - offset) == false) { - break; - } - } while (mi.next()); + PassageBuilder passageBuilder = builders.computeIfAbsent(field, t -> passageSource.get()); + if (passageBuilder.build(text, mi.mi, offset) == false) { + mi.exhausted = true; + } } diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SentencePassageBuilder.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SentencePassageBuilder.java deleted file mode 100644 index d82f088b83d2..000000000000 --- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/SentencePassageBuilder.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.lucene.search.matchhighlight; - -import java.text.BreakIterator; -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; - -public class SentencePassageBuilder implements PassageBuilder { - - private final BreakIterator breakIterator; - - private final List passages = new ArrayList<>(); - - private StringBuilder currentPassage = new StringBuilder(); - private int matchStart = 0; - private int matchEnd; - private String source; - - public SentencePassageBuilder() { - this.breakIterator = BreakIterator.getSentenceInstance(Locale.ROOT); - } - - @Override - public void addSource(String source) { - finishPassage(); - this.breakIterator.setText(source); - this.source = source; - this.matchEnd = this.breakIterator.next(); - this.matchStart = 0; - } - - @Override - public boolean addMatch(int startOffset, int endOffset) { - if (startOffset > source.length()) { - return false; - } - if (startOffset > this.matchEnd) { - finishPassage(); - while (matchEnd <= startOffset) { - matchStart = matchEnd; - matchEnd = breakIterator.next(); - } - } - currentPassage.append(this.source.substring(matchStart, startOffset)); - currentPassage.append(""); - currentPassage.append(this.source.substring(startOffset, endOffset)); - currentPassage.append(""); - matchStart = endOffset; - return true; - } - - private void finishPassage() { - if (currentPassage.length() == 0) { - return; - } - currentPassage.append(this.source.substring(matchStart, matchEnd)); - passages.add(currentPassage.toString()); - currentPassage = new StringBuilder(); - } - - @Override - public Iterable getTopPassages(int topN) { - finishPassage(); - return passages.subList(0, topN); - } -} diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchHighlighter.java b/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchHighlighter.java index 30a5b9eb02a8..a448e38159b2 100644 --- a/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchHighlighter.java +++ b/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchHighlighter.java @@ -77,11 +77,11 @@ public void testBasics() throws Exception { MatchHighlighter highlighter = new MatchHighlighter(searcher, indexAnalyzer); TopHighlights highlights = highlighter.highlight(query, topDocs, - () -> new PassageCollector(Collections.singleton("body"), 1, SentencePassageBuilder::new)); + () -> new PassageCollector(Collections.singleton("body"), 1, () -> new PassageBuilder(45))); assertEquals(2, highlights.docs.length); assertEquals("Just a test highlighting from postings. ", highlights.docs[0].fields.get("body")); - assertEquals("Highlighting the first term. ", highlights.docs[1].fields.get("body")); + assertEquals("Highlighting the first term. Hope it works.", highlights.docs[1].fields.get("body")); ir.close(); } @@ -108,7 +108,7 @@ public void testHighlightLastWord() throws Exception { MatchHighlighter highlighter = new MatchHighlighter(searcher, indexAnalyzer); TopHighlights highlights = highlighter.highlight(query, topDocs, - () -> new PassageCollector(Collections.singleton("body"), 1, SentencePassageBuilder::new)); + () -> new PassageCollector(Collections.singleton("body"), 1, () -> new PassageBuilder(60))); assertEquals(1, highlights.docs.length); assertEquals("This is a test", highlights.docs[0].fields.get("body")); @@ -122,6 +122,7 @@ public void testMultiValuedField() throws Exception { doc.add(new TextField("body", "This is the first sentence, and a fine sentence it is too", Field.Store.YES)); doc.add(new TextField("body", "And this is the second sentence", Field.Store.YES)); doc.add(new TextField("body", "And a third sentence too!", Field.Store.YES)); + doc.add(new TextField("body", "And then a final entry without a highlight", Field.Store.YES)); iw.addDocument(doc); IndexReader ir = iw.getReader(); @@ -134,7 +135,7 @@ public void testMultiValuedField() throws Exception { MatchHighlighter highlighter = new MatchHighlighter(searcher, indexAnalyzer); TopHighlights highlights = highlighter.highlight(query, topDocs, - () -> new PassageCollector(Collections.singleton("body"), 3, SentencePassageBuilder::new)); + () -> new PassageCollector(Collections.singleton("body"), 3, () -> new PassageBuilder(70))); assertEquals(1, highlights.docs.length); String[] values = highlights.docs[0].fields.getValues("body"); assertEquals(3, values.length); @@ -144,7 +145,7 @@ public void testMultiValuedField() throws Exception { // again, this time with only one passage per field highlights = highlighter.highlight(query, topDocs, - () -> new PassageCollector(Collections.singleton("body"), 1, SentencePassageBuilder::new)); + () -> new PassageCollector(Collections.singleton("body"), 1, () -> new PassageBuilder(60))); assertEquals(1, highlights.docs[0].fields.getValues("body").length); ir.close(); From 246d9f71dc37cef3105dbece27095f9389db8c0b Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 6 Jun 2018 15:29:30 +0100 Subject: [PATCH 44/45] Fix imports --- lucene/core/src/java/org/apache/lucene/search/Matches.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/search/Matches.java b/lucene/core/src/java/org/apache/lucene/search/Matches.java index b1982b362d28..de9a692ee6d8 100644 --- a/lucene/core/src/java/org/apache/lucene/search/Matches.java +++ b/lucene/core/src/java/org/apache/lucene/search/Matches.java @@ -20,12 +20,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; import java.util.Iterator; import java.util.List; import java.util.Objects; From 3b93008a8d9e832617d286e62aea2fa8349957b2 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 6 Jun 2018 15:36:28 +0100 Subject: [PATCH 45/45] Make PassageBuilder a bit more configurable --- .../search/matchhighlight/PassageBuilder.java | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageBuilder.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageBuilder.java index 7bec9ff7df0e..765bb38e8b69 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageBuilder.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageBuilder.java @@ -30,10 +30,9 @@ public class PassageBuilder { private static final int PAD_WIDTH = 10; private final int snippetWidth; - private final String markupStart = ""; - private final String markupEnd = ""; - - private final BreakIterator wordBreakIterator = BreakIterator.getWordInstance(Locale.ROOT); + private final String markupStart; + private final String markupEnd; + private final BreakIterator passageBreaker; private final List passages = new ArrayList<>(); @@ -41,8 +40,15 @@ public class PassageBuilder { private int passageStart; private int passageEnd; - public PassageBuilder(int snippetWidth) { + public PassageBuilder(int snippetWidth, BreakIterator passageBreaker, String markupStart, String markupEnd) { this.snippetWidth = snippetWidth; + this.passageBreaker = passageBreaker; + this.markupStart = markupStart; + this.markupEnd = markupEnd; + } + + public PassageBuilder(int snippetWidth) { + this(snippetWidth, BreakIterator.getWordInstance(Locale.ROOT), "", ""); } public Iterable getTopPassages(int topN) { @@ -52,7 +58,7 @@ public Iterable getTopPassages(int topN) { public boolean build(String source, MatchesIterator mi, int offset) throws IOException { currentPassage = new StringBuilder(); passageStart = passageEnd = 0; - wordBreakIterator.setText(source); + passageBreaker.setText(source); do { int startOffset = mi.startOffset() - offset; if (startOffset >= 0) { @@ -102,14 +108,14 @@ private void finishPassage(String source) { int endPadWidth = padWidth - (passageStart - start) + padWidth; int end = Math.min(source.length(), passageEnd + endPadWidth); if (end != source.length()) { - end = wordBreakIterator.preceding(end); + end = passageBreaker.preceding(end); } String passage = source.substring(start, passageStart) + currentPassage.toString() + source.substring(passageEnd, end); passages.add(passage); } private int nextWordStartAfter(int position, int maxLength, String source) { - int start = wordBreakIterator.following(position); + int start = passageBreaker.following(position); while (start <= maxLength) { if (Character.isLetter(source.codePointAt(start))) { break;