ALLOWED_INDEX_SORT_TYPES = EnumSet.of(SortField.Type.STRING,
SortField.Type.LONG,
@@ -471,10 +488,9 @@ public IndexWriterConfig setCommitOnClose(boolean commitOnClose) {
* Set the {@link Sort} order to use for all (flushed and merged) segments.
*/
public IndexWriterConfig setIndexSort(Sort sort) {
- for(SortField sortField : sort.getSort()) {
- final SortField.Type sortType = Sorter.getSortFieldType(sortField);
- if (ALLOWED_INDEX_SORT_TYPES.contains(sortType) == false) {
- throw new IllegalArgumentException("invalid SortField type: must be one of " + ALLOWED_INDEX_SORT_TYPES + " but got: " + sortField);
+ for (SortField sortField : sort.getSort()) {
+ if (sortField.getIndexSorter() == null) {
+ throw new IllegalArgumentException("Cannot sort index with sort field " + sortField);
}
}
this.indexSort = sort;
diff --git a/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java b/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java
index 1f48acc8d5f6..9b1d56c0a96d 100644
--- a/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java
+++ b/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java
@@ -109,6 +109,8 @@ public class LiveIndexWriterConfig {
/** soft deletes field */
protected String softDeletesField = null;
+ /** Amount of time to wait for merges returned by MergePolicy.findFullFlushMerges(...) */
+ protected volatile long maxCommitMergeWaitSeconds;
// used by IndexWriterConfig
LiveIndexWriterConfig(Analyzer analyzer) {
@@ -132,6 +134,7 @@ public class LiveIndexWriterConfig {
flushPolicy = new FlushByRamOrCountsPolicy();
readerPooling = IndexWriterConfig.DEFAULT_READER_POOLING;
perThreadHardLimitMB = IndexWriterConfig.DEFAULT_RAM_PER_THREAD_HARD_LIMIT_MB;
+ maxCommitMergeWaitSeconds = IndexWriterConfig.DEFAULT_MAX_COMMIT_MERGE_WAIT_SECONDS;
}
/** Returns the default analyzer to use for indexing documents. */
@@ -461,6 +464,15 @@ public String getSoftDeletesField() {
return softDeletesField;
}
+ /**
+ * Expert: return the amount of time to wait for merges returned by by MergePolicy.findFullFlushMerges(...).
+ * If this time is reached, we proceed with the commit based on segments merged up to that point.
+ * The merges are not cancelled, and may still run to completion independent of the commit.
+ */
+ public long getMaxCommitMergeWaitSeconds() {
+ return maxCommitMergeWaitSeconds;
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
@@ -484,6 +496,7 @@ public String toString() {
sb.append("indexSort=").append(getIndexSort()).append("\n");
sb.append("checkPendingFlushOnUpdate=").append(isCheckPendingFlushOnUpdate()).append("\n");
sb.append("softDeletesField=").append(getSoftDeletesField()).append("\n");
+ sb.append("maxCommitMergeWaitSeconds=").append(getMaxCommitMergeWaitSeconds()).append("\n");
return sb.toString();
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java b/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java
index 3ac391451911..5a090da82967 100644
--- a/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java
+++ b/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java
@@ -23,7 +23,12 @@
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Optional;
import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
@@ -37,6 +42,7 @@
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.IOSupplier;
import org.apache.lucene.util.InfoStream;
+import org.apache.lucene.util.ThreadInterruptedException;
/**
* Expert: a MergePolicy determines the sequence of
@@ -76,7 +82,7 @@ public abstract class MergePolicy {
* @lucene.experimental */
public static class OneMergeProgress {
/** Reason for pausing the merge thread. */
- public static enum PauseReason {
+ public enum PauseReason {
/** Stopped (because of throughput rate set to 0, typically). */
STOPPED,
/** Temporarily paused because of exceeded throughput rate. */
@@ -196,6 +202,7 @@ final void setMergeThread(Thread owner) {
*
* @lucene.experimental */
public static class OneMerge {
+ private final CompletableFuture mergeCompleted = new CompletableFuture<>();
SegmentCommitInfo info; // used by IndexWriter
boolean registerDone; // used by IndexWriter
long mergeGen; // used by IndexWriter
@@ -222,7 +229,7 @@ public static class OneMerge {
volatile long mergeStartNS = -1;
/** Total number of documents in segments to be merged, not accounting for deletions. */
- public final int totalMaxDoc;
+ final int totalMaxDoc;
Throwable error;
/** Sole constructor.
@@ -233,13 +240,8 @@ public OneMerge(List segments) {
throw new RuntimeException("segments must include at least one segment");
}
// clone the list, as the in list may be based off original SegmentInfos and may be modified
- this.segments = new ArrayList<>(segments);
- int count = 0;
- for(SegmentCommitInfo info : segments) {
- count += info.info.maxDoc();
- }
- totalMaxDoc = count;
-
+ this.segments = List.copyOf(segments);
+ totalMaxDoc = segments.stream().mapToInt(i -> i.info.maxDoc()).sum();
mergeProgress = new OneMergeProgress();
}
@@ -250,9 +252,15 @@ public OneMerge(List segments) {
public void mergeInit() throws IOException {
mergeProgress.setMergeThread(Thread.currentThread());
}
-
- /** Called by {@link IndexWriter} after the merge is done and all readers have been closed. */
- public void mergeFinished() throws IOException {
+
+ /** Called by {@link IndexWriter} after the merge is done and all readers have been closed.
+ * @param success true iff the merge finished successfully ie. was committed */
+ public void mergeFinished(boolean success) throws IOException {
+ mergeCompleted.complete(success);
+ // https://issues.apache.org/jira/browse/LUCENE-9408
+ // if (mergeCompleted.complete(success) == false) {
+ // throw new IllegalStateException("merge has already finished");
+ // }
}
/** Wrap the reader in order to add/remove information to the merged segment. */
@@ -362,6 +370,37 @@ public void checkAborted() throws MergeAbortedException {
public OneMergeProgress getMergeProgress() {
return mergeProgress;
}
+
+ /**
+ * Waits for this merge to be completed
+ * @return true if the merge finished within the specified timeout
+ */
+ boolean await(long timeout, TimeUnit timeUnit) {
+ try {
+ mergeCompleted.get(timeout, timeUnit);
+ return true;
+ } catch (InterruptedException e) {
+ throw new ThreadInterruptedException(e);
+ } catch (ExecutionException | TimeoutException e) {
+ return false;
+ }
+ }
+
+ /**
+ * Returns true if the merge has finished or false if it's still running or
+ * has not been started. This method will not block.
+ */
+ boolean isDone() {
+ return mergeCompleted.isDone();
+ }
+
+ /**
+ * Returns true iff the merge completed successfully or false if the merge succeeded with a failure.
+ * This method will not block and return an empty Optional if the merge has not finished yet
+ */
+ Optional hasCompletedSuccessfully() {
+ return Optional.ofNullable(mergeCompleted.getNow(null));
+ }
}
/**
@@ -399,6 +438,22 @@ public String segString(Directory dir) {
}
return b.toString();
}
+
+ /**
+ * Waits if necessary for at most the given time for all merges.
+ */
+ boolean await(long timeout, TimeUnit unit) {
+ try {
+ CompletableFuture future = CompletableFuture.allOf(merges.stream()
+ .map(m -> m.mergeCompleted).collect(Collectors.toList()).toArray(CompletableFuture[]::new));
+ future.get(timeout, unit);
+ return true;
+ } catch (InterruptedException e) {
+ throw new ThreadInterruptedException(e);
+ } catch (ExecutionException | TimeoutException e) {
+ return false;
+ }
+ }
}
/** Exception thrown if there are any problems while executing a merge. */
@@ -500,7 +555,7 @@ public abstract MergeSpecification findMerges(MergeTrigger mergeTrigger, Segment
* an original segment present in the
* to-be-merged index; else, it was a segment
* produced by a cascaded merge.
- * @param mergeContext the IndexWriter to find the merges on
+ * @param mergeContext the MergeContext to find the merges on
*/
public abstract MergeSpecification findForcedMerges(
SegmentInfos segmentInfos, int maxSegmentCount, Map segmentsToMerge, MergeContext mergeContext)
@@ -511,11 +566,33 @@ public abstract MergeSpecification findForcedMerges(
* deletes from the index.
* @param segmentInfos
* the total set of segments in the index
- * @param mergeContext the IndexWriter to find the merges on
+ * @param mergeContext the MergeContext to find the merges on
*/
public abstract MergeSpecification findForcedDeletesMerges(
SegmentInfos segmentInfos, MergeContext mergeContext) throws IOException;
+ /**
+ * Identifies merges that we want to execute (synchronously) on commit. By default, do not synchronously merge on commit.
+ *
+ * Any merges returned here will make {@link IndexWriter#commit()} or {@link IndexWriter#prepareCommit()} block until
+ * the merges complete or until {@link IndexWriterConfig#getMaxCommitMergeWaitSeconds()} have elapsed. This may be
+ * used to merge small segments that have just been flushed as part of the commit, reducing the number of segments in
+ * the commit. If a merge does not complete in the allotted time, it will continue to execute, but will not be reflected
+ * in the commit.
+ *
+ * If a {@link OneMerge} in the returned {@link MergeSpecification} includes a segment already included in a registered
+ * merge, then {@link IndexWriter#commit()} or {@link IndexWriter#prepareCommit()} will throw a {@link IllegalStateException}.
+ * Use {@link MergeContext#getMergingSegments()} to determine which segments are currently registered to merge.
+ *
+ * @param mergeTrigger the event that triggered the merge (COMMIT or FULL_FLUSH).
+ * @param segmentInfos the total set of segments in the index (while preparing the commit)
+ * @param mergeContext the MergeContext to find the merges on, which should be used to determine which segments are
+ * already in a registered merge (see {@link MergeContext#getMergingSegments()}).
+ */
+ public MergeSpecification findFullFlushMerges(MergeTrigger mergeTrigger, SegmentInfos segmentInfos, MergeContext mergeContext) throws IOException {
+ return null;
+ }
+
/**
* Returns true if a new segment (regardless of its origin) should use the
* compound file format. The default implementation returns true
diff --git a/lucene/core/src/java/org/apache/lucene/index/MergeTrigger.java b/lucene/core/src/java/org/apache/lucene/index/MergeTrigger.java
index d165a27008f4..01a6b15a0358 100644
--- a/lucene/core/src/java/org/apache/lucene/index/MergeTrigger.java
+++ b/lucene/core/src/java/org/apache/lucene/index/MergeTrigger.java
@@ -47,5 +47,10 @@ public enum MergeTrigger {
/**
* Merge was triggered by a closing IndexWriter.
*/
- CLOSING
+ CLOSING,
+
+ /**
+ * Merge was triggered on commit.
+ */
+ COMMIT,
}
diff --git a/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java b/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java
index b9ee2f58bb73..d515b6dfc9e1 100644
--- a/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java
+++ b/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java
@@ -434,7 +434,7 @@ public static SortedNumericDocValues getSortedNumericValues(final IndexReader r,
LeafReaderContext context = leaves.get(i);
SortedNumericDocValues v = context.reader().getSortedNumericDocValues(field);
if (v == null) {
- v = DocValues.emptySortedNumeric(context.reader().maxDoc());
+ v = DocValues.emptySortedNumeric();
} else {
anyReal = true;
}
diff --git a/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java b/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
index 23487014ea3b..35d1441155a7 100644
--- a/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
@@ -24,8 +24,6 @@
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.LongValues;
-import org.apache.lucene.util.NumericUtils;
import org.apache.lucene.util.PriorityQueue;
import org.apache.lucene.util.packed.PackedInts;
import org.apache.lucene.util.packed.PackedLongValues;
@@ -41,10 +39,14 @@ static MergeState.DocMap[] sort(Sort sort, List readers) throws IOE
// TODO: optimize if only 1 reader is incoming, though that's a rare case
SortField fields[] = sort.getSort();
- final ComparableProvider[][] comparables = new ComparableProvider[fields.length][];
+ final IndexSorter.ComparableProvider[][] comparables = new IndexSorter.ComparableProvider[fields.length][];
final int[] reverseMuls = new int[fields.length];
for(int i=0;i readers, SortField sortField) throws IOException {
-
- ComparableProvider[] providers = new ComparableProvider[readers.size()];
- final SortField.Type sortType = Sorter.getSortFieldType(sortField);
-
- switch(sortType) {
-
- case STRING:
- {
- // this uses the efficient segment-local ordinal map:
- final SortedDocValues[] values = new SortedDocValues[readers.size()];
- for(int i=0;i 0) {
- infos.minSegmentLuceneVersion = Version.fromBits(input.readVInt(), input.readVInt(), input.readVInt());
- } else {
- // else leave as null: no segments
- }
+ SegmentInfos infos = new SegmentInfos(indexCreatedVersion);
+ infos.id = id;
+ infos.generation = generation;
+ infos.lastGeneration = generation;
+ infos.luceneVersion = luceneVersion;
- long totalDocs = 0;
- for (int seg = 0; seg < numSegments; seg++) {
- String segName = input.readString();
- byte[] segmentID = new byte[StringHelper.ID_LENGTH];
- input.readBytes(segmentID, 0, segmentID.length);
- Codec codec = readCodec(input);
- SegmentInfo info = codec.segmentInfoFormat().read(directory, segName, segmentID, IOContext.READ);
- info.setCodec(codec);
- totalDocs += info.maxDoc();
- long delGen = input.readLong();
- int delCount = input.readInt();
- if (delCount < 0 || delCount > info.maxDoc()) {
- throw new CorruptIndexException("invalid deletion count: " + delCount + " vs maxDoc=" + info.maxDoc(), input);
- }
- long fieldInfosGen = input.readLong();
- long dvGen = input.readLong();
- int softDelCount = format > VERSION_72 ? input.readInt() : 0;
- if (softDelCount < 0 || softDelCount > info.maxDoc()) {
- throw new CorruptIndexException("invalid deletion count: " + softDelCount + " vs maxDoc=" + info.maxDoc(), input);
+ infos.version = input.readLong();
+ //System.out.println("READ sis version=" + infos.version);
+ if (format > VERSION_70) {
+ infos.counter = input.readVLong();
+ } else {
+ infos.counter = input.readInt();
}
- if (softDelCount + delCount > info.maxDoc()) {
- throw new CorruptIndexException("invalid deletion count: " + softDelCount + delCount + " vs maxDoc=" + info.maxDoc(), input);
+ int numSegments = input.readInt();
+ if (numSegments < 0) {
+ throw new CorruptIndexException("invalid segment count: " + numSegments, input);
}
- final byte[] sciId;
- if (format > VERSION_74) {
- byte marker = input.readByte();
- switch (marker) {
- case 1:
- sciId = new byte[StringHelper.ID_LENGTH];
- input.readBytes(sciId, 0, sciId.length);
- break;
- case 0:
- sciId = null;
- break;
- default:
- throw new CorruptIndexException("invalid SegmentCommitInfo ID marker: " + marker, input);
- }
+
+ if (numSegments > 0) {
+ infos.minSegmentLuceneVersion = Version.fromBits(input.readVInt(), input.readVInt(), input.readVInt());
} else {
- sciId = null;
+ // else leave as null: no segments
}
- SegmentCommitInfo siPerCommit = new SegmentCommitInfo(info, delCount, softDelCount, delGen, fieldInfosGen, dvGen, sciId);
- siPerCommit.setFieldInfosFiles(input.readSetOfStrings());
- final Map> dvUpdateFiles;
- final int numDVFields = input.readInt();
- if (numDVFields == 0) {
- dvUpdateFiles = Collections.emptyMap();
- } else {
- Map> map = new HashMap<>(numDVFields);
- for (int i = 0; i < numDVFields; i++) {
- map.put(input.readInt(), input.readSetOfStrings());
+
+ long totalDocs = 0;
+ for (int seg = 0; seg < numSegments; seg++) {
+ String segName = input.readString();
+ byte[] segmentID = new byte[StringHelper.ID_LENGTH];
+ input.readBytes(segmentID, 0, segmentID.length);
+ Codec codec = readCodec(input);
+ SegmentInfo info = codec.segmentInfoFormat().read(directory, segName, segmentID, IOContext.READ);
+ info.setCodec(codec);
+ totalDocs += info.maxDoc();
+ long delGen = input.readLong();
+ int delCount = input.readInt();
+ if (delCount < 0 || delCount > info.maxDoc()) {
+ throw new CorruptIndexException("invalid deletion count: " + delCount + " vs maxDoc=" + info.maxDoc(), input);
}
- dvUpdateFiles = Collections.unmodifiableMap(map);
- }
- siPerCommit.setDocValuesUpdatesFiles(dvUpdateFiles);
- infos.add(siPerCommit);
+ long fieldInfosGen = input.readLong();
+ long dvGen = input.readLong();
+ int softDelCount = format > VERSION_72 ? input.readInt() : 0;
+ if (softDelCount < 0 || softDelCount > info.maxDoc()) {
+ throw new CorruptIndexException("invalid deletion count: " + softDelCount + " vs maxDoc=" + info.maxDoc(), input);
+ }
+ if (softDelCount + delCount > info.maxDoc()) {
+ throw new CorruptIndexException("invalid deletion count: " + softDelCount + delCount + " vs maxDoc=" + info.maxDoc(), input);
+ }
+ final byte[] sciId;
+ if (format > VERSION_74) {
+ byte marker = input.readByte();
+ switch (marker) {
+ case 1:
+ sciId = new byte[StringHelper.ID_LENGTH];
+ input.readBytes(sciId, 0, sciId.length);
+ break;
+ case 0:
+ sciId = null;
+ break;
+ default:
+ throw new CorruptIndexException("invalid SegmentCommitInfo ID marker: " + marker, input);
+ }
+ } else {
+ sciId = null;
+ }
+ SegmentCommitInfo siPerCommit = new SegmentCommitInfo(info, delCount, softDelCount, delGen, fieldInfosGen, dvGen, sciId);
+ siPerCommit.setFieldInfosFiles(input.readSetOfStrings());
+ final Map> dvUpdateFiles;
+ final int numDVFields = input.readInt();
+ if (numDVFields == 0) {
+ dvUpdateFiles = Collections.emptyMap();
+ } else {
+ Map> map = new HashMap<>(numDVFields);
+ for (int i = 0; i < numDVFields; i++) {
+ map.put(input.readInt(), input.readSetOfStrings());
+ }
+ dvUpdateFiles = Collections.unmodifiableMap(map);
+ }
+ siPerCommit.setDocValuesUpdatesFiles(dvUpdateFiles);
+ infos.add(siPerCommit);
- Version segmentVersion = info.getVersion();
+ Version segmentVersion = info.getVersion();
- if (segmentVersion.onOrAfter(infos.minSegmentLuceneVersion) == false) {
- throw new CorruptIndexException("segments file recorded minSegmentLuceneVersion=" + infos.minSegmentLuceneVersion + " but segment=" + info + " has older version=" + segmentVersion, input);
- }
+ if (segmentVersion.onOrAfter(infos.minSegmentLuceneVersion) == false) {
+ throw new CorruptIndexException("segments file recorded minSegmentLuceneVersion=" + infos.minSegmentLuceneVersion + " but segment=" + info + " has older version=" + segmentVersion, input);
+ }
- if (infos.indexCreatedVersionMajor >= 7 && segmentVersion.major < infos.indexCreatedVersionMajor) {
- throw new CorruptIndexException("segments file recorded indexCreatedVersionMajor=" + infos.indexCreatedVersionMajor + " but segment=" + info + " has older version=" + segmentVersion, input);
- }
+ if (infos.indexCreatedVersionMajor >= 7 && segmentVersion.major < infos.indexCreatedVersionMajor) {
+ throw new CorruptIndexException("segments file recorded indexCreatedVersionMajor=" + infos.indexCreatedVersionMajor + " but segment=" + info + " has older version=" + segmentVersion, input);
+ }
- if (infos.indexCreatedVersionMajor >= 7 && info.getMinVersion() == null) {
- throw new CorruptIndexException("segments infos must record minVersion with indexCreatedVersionMajor=" + infos.indexCreatedVersionMajor, input);
+ if (infos.indexCreatedVersionMajor >= 7 && info.getMinVersion() == null) {
+ throw new CorruptIndexException("segments infos must record minVersion with indexCreatedVersionMajor=" + infos.indexCreatedVersionMajor, input);
+ }
}
- }
- infos.userData = input.readMapOfStrings();
+ infos.userData = input.readMapOfStrings();
- CodecUtil.checkFooter(input);
+ // LUCENE-6299: check we are in bounds
+ if (totalDocs > IndexWriter.getActualMaxDocs()) {
+ throw new CorruptIndexException("Too many documents: an index cannot exceed " + IndexWriter.getActualMaxDocs() + " but readers have total maxDoc=" + totalDocs, input);
+ }
- // LUCENE-6299: check we are in bounds
- if (totalDocs > IndexWriter.getActualMaxDocs()) {
- throw new CorruptIndexException("Too many documents: an index cannot exceed " + IndexWriter.getActualMaxDocs() + " but readers have total maxDoc=" + totalDocs, input);
+ return infos;
+ } catch (Throwable t) {
+ priorE = t;
+ } finally {
+ if (format >= VERSION_70) { // oldest supported version
+ CodecUtil.checkFooter(input, priorE);
+ } else {
+ throw IOUtils.rethrowAlways(priorE);
+ }
}
-
- return infos;
+ throw new Error("Unreachable code");
}
private static Codec readCodec(DataInput input) throws IOException {
diff --git a/lucene/core/src/java/org/apache/lucene/index/SortFieldProvider.java b/lucene/core/src/java/org/apache/lucene/index/SortFieldProvider.java
new file mode 100644
index 000000000000..290decd73c3b
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/index/SortFieldProvider.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.index;
+
+import java.io.IOException;
+import java.util.Set;
+
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.util.NamedSPILoader;
+
+/**
+ * Reads/Writes a named SortField from a segment info file, used to record index sorts
+ */
+public abstract class SortFieldProvider implements NamedSPILoader.NamedSPI {
+
+ private static class Holder {
+ private static final NamedSPILoader LOADER = new NamedSPILoader<>(SortFieldProvider.class);
+
+ static NamedSPILoader getLoader() {
+ if (LOADER == null) {
+ throw new IllegalStateException("You tried to lookup a SortFieldProvider by name before all SortFieldProviders could be initialized. "+
+ "This likely happens if you call SortFieldProvider#forName from a SortFieldProviders's ctor.");
+ }
+ return LOADER;
+ }
+ }
+
+ /**
+ * Looks up a SortFieldProvider by name
+ */
+ public static SortFieldProvider forName(String name) {
+ return Holder.getLoader().lookup(name);
+ }
+
+ /**
+ * Lists all available SortFieldProviders
+ */
+ public static Set availableSortFieldProviders() {
+ return Holder.getLoader().availableServices();
+ }
+
+ /**
+ * Reloads the SortFieldProvider list from the given {@link ClassLoader}.
+ * Changes to the list are visible after the method ends, all
+ * iterators ({@link #availableSortFieldProviders()} ()},...) stay consistent.
+ *
+ * NOTE: Only new SortFieldProviders are added, existing ones are
+ * never removed or replaced.
+ *
+ *
This method is expensive and should only be called for discovery
+ * of new SortFieldProviders on the given classpath/classloader!
+ */
+ public static void reloadSortFieldProviders(ClassLoader classLoader) {
+ Holder.getLoader().reload(classLoader);
+ }
+
+ /**
+ * Writes a SortField to a DataOutput
+ */
+ public static void write(SortField sf, DataOutput output) throws IOException {
+ IndexSorter sorter = sf.getIndexSorter();
+ if (sorter == null) {
+ throw new IllegalArgumentException("Cannot serialize sort field " + sf);
+ }
+ SortFieldProvider provider = SortFieldProvider.forName(sorter.getProviderName());
+ provider.writeSortField(sf, output);
+ }
+
+ /** The name this SortFieldProvider is registered under */
+ protected final String name;
+
+ /**
+ * Creates a new SortFieldProvider.
+ *
+ * The provided name will be written into the index segment: in order to
+ * for the segment to be read this class should be registered with Java's
+ * SPI mechanism (registered in META-INF/ of your jar file, etc).
+ * @param name must be all ascii alphanumeric, and less than 128 characters in length.
+ */
+ protected SortFieldProvider(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * Reads a SortField from serialized bytes
+ */
+ public abstract SortField readSortField(DataInput in) throws IOException;
+
+ /**
+ * Writes a SortField to a DataOutput
+ *
+ * This is used to record index sort information in segment headers
+ */
+ public abstract void writeSortField(SortField sf, DataOutput out) throws IOException;
+
+}
diff --git a/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesWriter.java b/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesWriter.java
index 86d0f0bab338..2252f003cb2d 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesWriter.java
@@ -21,7 +21,6 @@
import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.SortField;
import org.apache.lucene.util.ByteBlockPool;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefHash;
@@ -35,7 +34,7 @@
/** Buffers up pending byte[] per doc, deref and sorting via
* int ord, then flushes when segment flushes. */
-class SortedDocValuesWriter extends DocValuesWriter {
+class SortedDocValuesWriter extends DocValuesWriter {
final BytesRefHash hash;
private PackedLongValues.Builder pending;
private DocsWithFieldSet docsWithField;
@@ -79,11 +78,6 @@ public void addValue(int docID, BytesRef value) {
lastDocID = docID;
}
- @Override
- public void finish(int maxDoc) {
- updateBytesUsed();
- }
-
private void addOneValue(BytesRef value) {
int termID = hash.add(value);
if (termID < 0) {
@@ -107,20 +101,20 @@ private void updateBytesUsed() {
}
@Override
- Sorter.DocComparator getDocComparator(int maxDoc, SortField sortField) throws IOException {
- assert sortField.getType().equals(SortField.Type.STRING);
- assert finalSortedValues == null && finalOrdMap == null &&finalOrds == null;
+ SortedDocValues getDocValues() {
int valueCount = hash.size();
- finalSortedValues = hash.sort();
- finalOrds = pending.build();
- finalOrdMap = new int[valueCount];
+ if (finalSortedValues == null) {
+ updateBytesUsed();
+ assert finalOrdMap == null && finalOrds == null;
+ finalSortedValues = hash.sort();
+ finalOrds = pending.build();
+ finalOrdMap = new int[valueCount];
+ }
for (int ord = 0; ord < valueCount; ord++) {
finalOrdMap[finalSortedValues[ord]] = ord;
}
- final SortedDocValues docValues =
- new BufferedSortedDocValues(hash, valueCount, finalOrds, finalSortedValues, finalOrdMap,
+ return new BufferedSortedDocValues(hash, valueCount, finalOrds, finalSortedValues, finalOrdMap,
docsWithField.iterator());
- return Sorter.getDocComparator(maxDoc, sortField, () -> docValues, () -> null);
}
private int[] sortDocValues(int maxDoc, Sorter.DocMap sortMap, SortedDocValues oldValues) throws IOException {
@@ -137,26 +131,20 @@ private int[] sortDocValues(int maxDoc, Sorter.DocMap sortMap, SortedDocValues o
@Override
public void flush(SegmentWriteState state, Sorter.DocMap sortMap, DocValuesConsumer dvConsumer) throws IOException {
final int valueCount = hash.size();
- final PackedLongValues ords;
- final int[] sortedValues;
- final int[] ordMap;
if (finalOrds == null) {
- sortedValues = hash.sort();
- ords = pending.build();
- ordMap = new int[valueCount];
+ updateBytesUsed();
+ finalSortedValues = hash.sort();
+ finalOrds = pending.build();
+ finalOrdMap = new int[valueCount];
for (int ord = 0; ord < valueCount; ord++) {
- ordMap[sortedValues[ord]] = ord;
+ finalOrdMap[finalSortedValues[ord]] = ord;
}
- } else {
- sortedValues = finalSortedValues;
- ords = finalOrds;
- ordMap = finalOrdMap;
}
final int[] sorted;
if (sortMap != null) {
sorted = sortDocValues(state.segmentInfo.maxDoc(), sortMap,
- new BufferedSortedDocValues(hash, valueCount, ords, sortedValues, ordMap, docsWithField.iterator()));
+ new BufferedSortedDocValues(hash, valueCount, finalOrds, finalSortedValues, finalOrdMap, docsWithField.iterator()));
} else {
sorted = null;
}
@@ -168,7 +156,7 @@ public SortedDocValues getSorted(FieldInfo fieldInfoIn) {
throw new IllegalArgumentException("wrong fieldInfo");
}
final SortedDocValues buf =
- new BufferedSortedDocValues(hash, valueCount, ords, sortedValues, ordMap, docsWithField.iterator());
+ new BufferedSortedDocValues(hash, valueCount, finalOrds, finalSortedValues, finalOrdMap, docsWithField.iterator());
if (sorted == null) {
return buf;
}
@@ -245,8 +233,4 @@ public int getValueCount() {
}
}
- @Override
- DocIdSetIterator getDocIdSet() {
- return docsWithField.iterator();
- }
}
diff --git a/lucene/core/src/java/org/apache/lucene/index/SortedNumericDocValuesWriter.java b/lucene/core/src/java/org/apache/lucene/index/SortedNumericDocValuesWriter.java
index bdc65cc80574..83c394fc207b 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SortedNumericDocValuesWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SortedNumericDocValuesWriter.java
@@ -22,9 +22,6 @@
import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.SortField;
-import org.apache.lucene.search.SortedNumericSelector;
-import org.apache.lucene.search.SortedNumericSortField;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Counter;
import org.apache.lucene.util.RamUsageEstimator;
@@ -34,7 +31,7 @@
import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
/** Buffers up pending long[] per doc, sorts, then flushes when segment flushes. */
-class SortedNumericDocValuesWriter extends DocValuesWriter {
+class SortedNumericDocValuesWriter extends DocValuesWriter {
private PackedLongValues.Builder pending; // stream of all values
private PackedLongValues.Builder pendingCounts; // count of values per doc
private DocsWithFieldSet docsWithField;
@@ -85,11 +82,6 @@ private void finishCurrentDoc() {
docsWithField.add(currentDoc);
}
- @Override
- public void finish(int maxDoc) {
- finishCurrentDoc();
- }
-
private void addOneValue(long value) {
if (currentUpto == currentValues.length) {
currentValues = ArrayUtil.grow(currentValues, currentValues.length+1);
@@ -106,16 +98,14 @@ private void updateBytesUsed() {
}
@Override
- Sorter.DocComparator getDocComparator(int maxDoc, SortField sortField) throws IOException {
- assert sortField instanceof SortedNumericSortField;
- assert finalValues == null && finalValuesCount == null;
- finalValues = pending.build();
- finalValuesCount = pendingCounts.build();
- final SortedNumericDocValues docValues =
- new BufferedSortedNumericDocValues(finalValues, finalValuesCount, docsWithField.iterator());
- SortedNumericSortField sf = (SortedNumericSortField) sortField;
- return Sorter.getDocComparator(maxDoc, sf, () -> null,
- () -> SortedNumericSelector.wrap(docValues, sf.getSelector(), sf.getNumericType()));
+ SortedNumericDocValues getDocValues() {
+ if (finalValues == null) {
+ assert finalValuesCount == null;
+ finishCurrentDoc();
+ finalValues = pending.build();
+ finalValuesCount = pendingCounts.build();
+ }
+ return new BufferedSortedNumericDocValues(finalValues, finalValuesCount, docsWithField.iterator());
}
private long[][] sortDocValues(int maxDoc, Sorter.DocMap sortMap, SortedNumericDocValues oldValues) throws IOException {
@@ -137,6 +127,7 @@ public void flush(SegmentWriteState state, Sorter.DocMap sortMap, DocValuesConsu
final PackedLongValues values;
final PackedLongValues valueCounts;
if (finalValues == null) {
+ finishCurrentDoc();
values = pending.build();
valueCounts = pendingCounts.build();
} else {
@@ -232,8 +223,4 @@ public long cost() {
}
}
- @Override
- DocIdSetIterator getDocIdSet() {
- return docsWithField.iterator();
- }
}
diff --git a/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesWriter.java b/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesWriter.java
index 71a14a5cb7cb..022b17da0956 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesWriter.java
@@ -21,9 +21,6 @@
import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.SortField;
-import org.apache.lucene.search.SortedSetSelector;
-import org.apache.lucene.search.SortedSetSortField;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.ByteBlockPool;
import org.apache.lucene.util.BytesRef;
@@ -39,7 +36,7 @@
/** Buffers up pending byte[]s per doc, deref and sorting via
* int ord, then flushes when segment flushes. */
-class SortedSetDocValuesWriter extends DocValuesWriter {
+class SortedSetDocValuesWriter extends DocValuesWriter {
final BytesRefHash hash;
private PackedLongValues.Builder pending; // stream of all termIDs
private PackedLongValues.Builder pendingCounts; // termIDs per doc
@@ -115,11 +112,6 @@ private void finishCurrentDoc() {
docsWithField.add(currentDoc);
}
- @Override
- public void finish(int maxDoc) {
- finishCurrentDoc();
- }
-
private void addOneValue(BytesRef value) {
int termID = hash.add(value);
if (termID < 0) {
@@ -170,22 +162,20 @@ private long[][] sortDocValues(int maxDoc, Sorter.DocMap sortMap, SortedSetDocVa
}
@Override
- Sorter.DocComparator getDocComparator(int maxDoc, SortField sortField) throws IOException {
- assert sortField instanceof SortedSetSortField;
- assert finalOrds == null && finalOrdCounts == null && finalSortedValues == null && finalOrdMap == null;
- int valueCount = hash.size();
- finalOrds = pending.build();
- finalOrdCounts = pendingCounts.build();
- finalSortedValues = hash.sort();
- finalOrdMap = new int[valueCount];
- for (int ord = 0; ord < valueCount; ord++) {
+ SortedSetDocValues getDocValues() {
+ if (finalOrds == null) {
+ assert finalOrdCounts == null && finalSortedValues == null && finalOrdMap == null;
+ finishCurrentDoc();
+ int valueCount = hash.size();
+ finalOrds = pending.build();
+ finalOrdCounts = pendingCounts.build();
+ finalSortedValues = hash.sort();
+ finalOrdMap = new int[valueCount];
+ }
+ for (int ord = 0; ord < finalOrdMap.length; ord++) {
finalOrdMap[finalSortedValues[ord]] = ord;
}
-
- SortedSetSortField sf = (SortedSetSortField) sortField;
- final SortedSetDocValues dvs =
- new BufferedSortedSetDocValues(finalSortedValues, finalOrdMap, hash, finalOrds, finalOrdCounts, maxCount, docsWithField.iterator());
- return Sorter.getDocComparator(maxDoc, sf, () -> SortedSetSelector.wrap(dvs, sf.getSelector()), () -> null);
+ return new BufferedSortedSetDocValues(finalSortedValues, finalOrdMap, hash, finalOrds, finalOrdCounts, maxCount, docsWithField.iterator());
}
@Override
@@ -196,7 +186,9 @@ public void flush(SegmentWriteState state, Sorter.DocMap sortMap, DocValuesConsu
final int[] sortedValues;
final int[] ordMap;
- if (finalOrdCounts == null) {
+ if (finalOrds == null) {
+ assert finalOrdCounts == null && finalSortedValues == null && finalOrdMap == null;
+ finishCurrentDoc();
ords = pending.build();
ordCounts = pendingCounts.build();
sortedValues = hash.sort();
@@ -315,8 +307,5 @@ public BytesRef lookupOrd(long ord) {
return scratch;
}
}
- @Override
- DocIdSetIterator getDocIdSet() {
- return docsWithField.iterator();
- }
+
}
diff --git a/lucene/core/src/java/org/apache/lucene/index/Sorter.java b/lucene/core/src/java/org/apache/lucene/index/Sorter.java
index 5f43c5ad1566..c8605696fd35 100644
--- a/lucene/core/src/java/org/apache/lucene/index/Sorter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/Sorter.java
@@ -17,22 +17,13 @@
package org.apache.lucene.index;
import java.io.IOException;
-import java.util.Arrays;
-import java.util.Comparator;
-import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
-import org.apache.lucene.search.SortedNumericSelector;
-import org.apache.lucene.search.SortedNumericSortField;
-import org.apache.lucene.search.SortedSetSelector;
-import org.apache.lucene.search.SortedSetSortField;
import org.apache.lucene.util.TimSorter;
import org.apache.lucene.util.packed.PackedInts;
import org.apache.lucene.util.packed.PackedLongValues;
-import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
-
/**
* Sorts documents of a given index by returning a permutation on the document
* IDs.
@@ -84,21 +75,13 @@ static boolean isConsistent(DocMap docMap) {
return true;
}
- /** A comparator of doc IDs. */
- static abstract class DocComparator {
-
- /** Compare docID1 against docID2. The contract for the return value is the
- * same as {@link Comparator#compare(Object, Object)}. */
- public abstract int compare(int docID1, int docID2);
- }
-
private static final class DocValueSorter extends TimSorter {
private final int[] docs;
- private final Sorter.DocComparator comparator;
+ private final IndexSorter.DocComparator comparator;
private final int[] tmp;
- DocValueSorter(int[] docs, Sorter.DocComparator comparator) {
+ DocValueSorter(int[] docs, IndexSorter.DocComparator comparator) {
super(docs.length / 64);
this.docs = docs;
this.comparator = comparator;
@@ -139,7 +122,7 @@ protected int compareSaved(int i, int j) {
}
/** Computes the old-to-new permutation over the given comparator. */
- private static Sorter.DocMap sort(final int maxDoc, DocComparator comparator) {
+ private static Sorter.DocMap sort(final int maxDoc, IndexSorter.DocComparator comparator) {
// check if the index is sorted
boolean sorted = true;
for (int i = 1; i < maxDoc; ++i) {
@@ -202,196 +185,10 @@ public int size() {
};
}
- /** Returns the native sort type for {@link SortedSetSortField} and {@link SortedNumericSortField},
- * {@link SortField#getType()} otherwise */
- static SortField.Type getSortFieldType(SortField sortField) {
- if (sortField instanceof SortedSetSortField) {
- return SortField.Type.STRING;
- } else if (sortField instanceof SortedNumericSortField) {
- return ((SortedNumericSortField) sortField).getNumericType();
- } else {
- return sortField.getType();
- }
- }
-
- /** Wraps a {@link SortedNumericDocValues} as a single-valued view if the field is an instance of {@link SortedNumericSortField},
- * returns {@link NumericDocValues} for the field otherwise. */
- static NumericDocValues getOrWrapNumeric(LeafReader reader, SortField sortField) throws IOException {
- if (sortField instanceof SortedNumericSortField) {
- SortedNumericSortField sf = (SortedNumericSortField) sortField;
- return SortedNumericSelector.wrap(DocValues.getSortedNumeric(reader, sf.getField()), sf.getSelector(), sf.getNumericType());
- } else {
- return DocValues.getNumeric(reader, sortField.getField());
- }
- }
-
- /** Wraps a {@link SortedSetDocValues} as a single-valued view if the field is an instance of {@link SortedSetSortField},
- * returns {@link SortedDocValues} for the field otherwise. */
- static SortedDocValues getOrWrapSorted(LeafReader reader, SortField sortField) throws IOException {
- if (sortField instanceof SortedSetSortField) {
- SortedSetSortField sf = (SortedSetSortField) sortField;
- return SortedSetSelector.wrap(DocValues.getSortedSet(reader, sf.getField()), sf.getSelector());
- } else {
- return DocValues.getSorted(reader, sortField.getField());
- }
- }
-
- static DocComparator getDocComparator(LeafReader reader, SortField sortField) throws IOException {
- return getDocComparator(reader.maxDoc(), sortField,
- () -> getOrWrapSorted(reader, sortField),
- () -> getOrWrapNumeric(reader, sortField));
- }
-
- interface NumericDocValuesSupplier {
- NumericDocValues get() throws IOException;
- }
-
- interface SortedDocValuesSupplier {
- SortedDocValues get() throws IOException;
- }
-
- /** We cannot use the {@link FieldComparator} API because that API requires that you send it docIDs in order. Note that this API
- * allocates arrays[maxDoc] to hold the native values needed for comparison, but 1) they are transient (only alive while sorting this one
- * segment), and 2) in the typical index sorting case, they are only used to sort newly flushed segments, which will be smaller than
- * merged segments. */
- static DocComparator getDocComparator(int maxDoc,
- SortField sortField,
- SortedDocValuesSupplier sortedProvider,
- NumericDocValuesSupplier numericProvider) throws IOException {
-
- final int reverseMul = sortField.getReverse() ? -1 : 1;
- final SortField.Type sortType = getSortFieldType(sortField);
-
- switch(sortType) {
-
- case STRING:
- {
- final SortedDocValues sorted = sortedProvider.get();
- final int missingOrd;
- if (sortField.getMissingValue() == SortField.STRING_LAST) {
- missingOrd = Integer.MAX_VALUE;
- } else {
- missingOrd = Integer.MIN_VALUE;
- }
-
- final int[] ords = new int[maxDoc];
- Arrays.fill(ords, missingOrd);
- int docID;
- while ((docID = sorted.nextDoc()) != NO_MORE_DOCS) {
- ords[docID] = sorted.ordValue();
- }
-
- return new DocComparator() {
- @Override
- public int compare(int docID1, int docID2) {
- return reverseMul * Integer.compare(ords[docID1], ords[docID2]);
- }
- };
- }
-
- case LONG:
- {
- final NumericDocValues dvs = numericProvider.get();
- long[] values = new long[maxDoc];
- if (sortField.getMissingValue() != null) {
- Arrays.fill(values, (Long) sortField.getMissingValue());
- }
- while (true) {
- int docID = dvs.nextDoc();
- if (docID == NO_MORE_DOCS) {
- break;
- }
- values[docID] = dvs.longValue();
- }
-
- return new DocComparator() {
- @Override
- public int compare(int docID1, int docID2) {
- return reverseMul * Long.compare(values[docID1], values[docID2]);
- }
- };
- }
-
- case INT:
- {
- final NumericDocValues dvs = numericProvider.get();
- int[] values = new int[maxDoc];
- if (sortField.getMissingValue() != null) {
- Arrays.fill(values, (Integer) sortField.getMissingValue());
- }
-
- while (true) {
- int docID = dvs.nextDoc();
- if (docID == NO_MORE_DOCS) {
- break;
- }
- values[docID] = (int) dvs.longValue();
- }
-
- return new DocComparator() {
- @Override
- public int compare(int docID1, int docID2) {
- return reverseMul * Integer.compare(values[docID1], values[docID2]);
- }
- };
- }
-
- case DOUBLE:
- {
- final NumericDocValues dvs = numericProvider.get();
- double[] values = new double[maxDoc];
- if (sortField.getMissingValue() != null) {
- Arrays.fill(values, (Double) sortField.getMissingValue());
- }
- while (true) {
- int docID = dvs.nextDoc();
- if (docID == NO_MORE_DOCS) {
- break;
- }
- values[docID] = Double.longBitsToDouble(dvs.longValue());
- }
-
- return new DocComparator() {
- @Override
- public int compare(int docID1, int docID2) {
- return reverseMul * Double.compare(values[docID1], values[docID2]);
- }
- };
- }
-
- case FLOAT:
- {
- final NumericDocValues dvs = numericProvider.get();
- float[] values = new float[maxDoc];
- if (sortField.getMissingValue() != null) {
- Arrays.fill(values, (Float) sortField.getMissingValue());
- }
- while (true) {
- int docID = dvs.nextDoc();
- if (docID == NO_MORE_DOCS) {
- break;
- }
- values[docID] = Float.intBitsToFloat((int) dvs.longValue());
- }
-
- return new DocComparator() {
- @Override
- public int compare(int docID1, int docID2) {
- return reverseMul * Float.compare(values[docID1], values[docID2]);
- }
- };
- }
-
- default:
- throw new IllegalArgumentException("unhandled SortField.getType()=" + sortField.getType());
- }
- }
-
-
/**
* Returns a mapping from the old document ID to its new location in the
* sorted index. Implementations can use the auxiliary
- * {@link #sort(int, DocComparator)} to compute the old-to-new permutation
+ * {@link #sort(int, IndexSorter.DocComparator)} to compute the old-to-new permutation
* given a list of documents and their corresponding values.
*
* A return value of null is allowed and means that
@@ -401,28 +198,29 @@ public int compare(int docID1, int docID2) {
* well, they will however be marked as deleted in the sorted view.
*/
DocMap sort(LeafReader reader) throws IOException {
- SortField fields[] = sort.getSort();
- final DocComparator comparators[] = new DocComparator[fields.length];
+ SortField[] fields = sort.getSort();
+ final IndexSorter.DocComparator[] comparators = new IndexSorter.DocComparator[fields.length];
for (int i = 0; i < fields.length; i++) {
- comparators[i] = getDocComparator(reader, fields[i]);
+ IndexSorter sorter = fields[i].getIndexSorter();
+ if (sorter == null) {
+ throw new IllegalArgumentException("Cannot use sortfield + " + fields[i] + " to sort indexes");
+ }
+ comparators[i] = sorter.getDocComparator(reader, reader.maxDoc());
}
return sort(reader.maxDoc(), comparators);
}
- DocMap sort(int maxDoc, DocComparator[] comparators) throws IOException {
- final DocComparator comparator = new DocComparator() {
- @Override
- public int compare(int docID1, int docID2) {
- for (int i = 0; i < comparators.length; i++) {
- int comp = comparators[i].compare(docID1, docID2);
- if (comp != 0) {
- return comp;
- }
+ DocMap sort(int maxDoc, IndexSorter.DocComparator[] comparators) throws IOException {
+ final IndexSorter.DocComparator comparator = (docID1, docID2) -> {
+ for (int i = 0; i < comparators.length; i++) {
+ int comp = comparators[i].compare(docID1, docID2);
+ if (comp != 0) {
+ return comp;
}
- return Integer.compare(docID1, docID2); // docid order tiebreak
}
+ return Integer.compare(docID1, docID2); // docid order tiebreak
};
return sort(maxDoc, comparator);
diff --git a/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java b/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java
index f9b851760945..80213f92a9e9 100644
--- a/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java
+++ b/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java
@@ -44,11 +44,11 @@ class TermVectorsConsumer extends TermsHash {
final ByteSliceReader vectorSliceReaderOff = new ByteSliceReader();
boolean hasVectors;
- int numVectorFields;
+ private int numVectorFields;
int lastDocID;
private TermVectorsConsumerPerField[] perFields = new TermVectorsConsumerPerField[1];
- public TermVectorsConsumer(DocumentsWriterPerThread docWriter) {
+ TermVectorsConsumer(DocumentsWriterPerThread docWriter) {
super(docWriter, false, null);
this.docWriter = docWriter;
}
@@ -91,7 +91,7 @@ void initTermVectorsWriter() throws IOException {
}
@Override
- void finishDocument() throws IOException {
+ void finishDocument(int docID) throws IOException {
if (!hasVectors) {
return;
@@ -102,7 +102,7 @@ void finishDocument() throws IOException {
initTermVectorsWriter();
- fill(docState.docID);
+ fill(docID);
// Append term vectors to the real outputs:
writer.startDocument(numVectorFields);
@@ -111,7 +111,7 @@ void finishDocument() throws IOException {
}
writer.finishDocument();
- assert lastDocID == docState.docID: "lastDocID=" + lastDocID + " docState.docID=" + docState.docID;
+ assert lastDocID == docID: "lastDocID=" + lastDocID + " docID=" + docID;
lastDocID++;
diff --git a/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java b/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java
index 4e0aa3cdaadb..a1abd985f31c 100644
--- a/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java
+++ b/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java
@@ -20,27 +20,37 @@
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
+import org.apache.lucene.analysis.tokenattributes.TermFrequencyAttribute;
import org.apache.lucene.codecs.TermVectorsWriter;
+import org.apache.lucene.util.ByteBlockPool;
import org.apache.lucene.util.BytesRef;
final class TermVectorsConsumerPerField extends TermsHashPerField {
private TermVectorsPostingsArray termVectorsPostingsArray;
- final TermVectorsConsumer termsWriter;
+ private final TermVectorsConsumer termsWriter;
+ private final FieldInvertState fieldState;
+ private final FieldInfo fieldInfo;
- boolean doVectors;
- boolean doVectorPositions;
- boolean doVectorOffsets;
- boolean doVectorPayloads;
+ private boolean doVectors;
+ private boolean doVectorPositions;
+ private boolean doVectorOffsets;
+ private boolean doVectorPayloads;
- OffsetAttribute offsetAttribute;
- PayloadAttribute payloadAttribute;
- boolean hasPayloads; // if enabled, and we actually saw any for this field
+ private OffsetAttribute offsetAttribute;
+ private PayloadAttribute payloadAttribute;
+ private TermFrequencyAttribute termFreqAtt;
+ private final ByteBlockPool termBytePool;
- public TermVectorsConsumerPerField(FieldInvertState invertState, TermVectorsConsumer termsWriter, FieldInfo fieldInfo) {
- super(2, invertState, termsWriter, null, fieldInfo);
- this.termsWriter = termsWriter;
+ private boolean hasPayloads; // if enabled, and we actually saw any for this field
+
+ TermVectorsConsumerPerField(FieldInvertState invertState, TermVectorsConsumer termsHash, FieldInfo fieldInfo) {
+ super(2, termsHash.intPool, termsHash.bytePool, termsHash.termBytePool, termsHash.bytesUsed, null, fieldInfo.name, fieldInfo.getIndexOptions());
+ this.termsWriter = termsHash;
+ this.fieldInfo = fieldInfo;
+ this.fieldState = invertState;
+ termBytePool = termsHash.termBytePool;
}
/** Called once per field per document if term vectors
@@ -48,7 +58,7 @@ public TermVectorsConsumerPerField(FieldInvertState invertState, TermVectorsCons
* RAMOutputStream, which is then quickly flushed to
* the real term vectors files in the Directory. */ @Override
void finish() {
- if (!doVectors || bytesHash.size() == 0) {
+ if (!doVectors || getNumTerms() == 0) {
return;
}
termsWriter.addFieldToFlush(this);
@@ -61,7 +71,7 @@ void finishDocument() throws IOException {
doVectors = false;
- final int numPostings = bytesHash.size();
+ final int numPostings = getNumTerms();
final BytesRef flushTerm = termsWriter.flushTerm;
@@ -74,7 +84,8 @@ void finishDocument() throws IOException {
TermVectorsPostingsArray postings = termVectorsPostingsArray;
final TermVectorsWriter tv = termsWriter.writer;
- final int[] termIDs = sortPostings();
+ sortTerms();
+ final int[] termIDs = getSortedTermIDs();
tv.startField(fieldInfo, numPostings, doVectorPositions, doVectorOffsets, hasPayloads);
@@ -110,18 +121,19 @@ void finishDocument() throws IOException {
@Override
boolean start(IndexableField field, boolean first) {
super.start(field, first);
+ termFreqAtt = fieldState.termFreqAttribute;
assert field.fieldType().indexOptions() != IndexOptions.NONE;
if (first) {
- if (bytesHash.size() != 0) {
+ if (getNumTerms() != 0) {
// Only necessary if previous doc hit a
// non-aborting exception while writing vectors in
// this field:
reset();
}
- bytesHash.reinit();
+ reinitHash();
hasPayloads = false;
@@ -189,8 +201,8 @@ boolean start(IndexableField field, boolean first) {
return doVectors;
}
-
- void writeProx(TermVectorsPostingsArray postings, int termID) {
+
+ void writeProx(TermVectorsPostingsArray postings, int termID) {
if (doVectorOffsets) {
int startOffset = fieldState.offset + offsetAttribute.startOffset();
int endOffset = fieldState.offset + offsetAttribute.endOffset();
@@ -222,7 +234,7 @@ void writeProx(TermVectorsPostingsArray postings, int termID) {
}
@Override
- void newTerm(final int termID) {
+ void newTerm(final int termID, final int docID) {
TermVectorsPostingsArray postings = termVectorsPostingsArray;
postings.freqs[termID] = getTermFreq();
@@ -233,7 +245,7 @@ void newTerm(final int termID) {
}
@Override
- void addTerm(final int termID) {
+ void addTerm(final int termID, final int docID) {
TermVectorsPostingsArray postings = termVectorsPostingsArray;
postings.freqs[termID] += getTermFreq();
@@ -245,10 +257,10 @@ private int getTermFreq() {
int freq = termFreqAtt.getTermFrequency();
if (freq != 1) {
if (doVectorPositions) {
- throw new IllegalArgumentException("field \"" + fieldInfo.name + "\": cannot index term vector positions while using custom TermFrequencyAttribute");
+ throw new IllegalArgumentException("field \"" + getFieldName() + "\": cannot index term vector positions while using custom TermFrequencyAttribute");
}
if (doVectorOffsets) {
- throw new IllegalArgumentException("field \"" + fieldInfo.name + "\": cannot index term vector offsets while using custom TermFrequencyAttribute");
+ throw new IllegalArgumentException("field \"" + getFieldName() + "\": cannot index term vector offsets while using custom TermFrequencyAttribute");
}
}
@@ -266,7 +278,7 @@ ParallelPostingsArray createPostingsArray(int size) {
}
static final class TermVectorsPostingsArray extends ParallelPostingsArray {
- public TermVectorsPostingsArray(int size) {
+ TermVectorsPostingsArray(int size) {
super(size);
freqs = new int[size];
lastOffsets = new int[size];
diff --git a/lucene/core/src/java/org/apache/lucene/index/TermsHash.java b/lucene/core/src/java/org/apache/lucene/index/TermsHash.java
index f420aca65b23..0f702d925b8e 100644
--- a/lucene/core/src/java/org/apache/lucene/index/TermsHash.java
+++ b/lucene/core/src/java/org/apache/lucene/index/TermsHash.java
@@ -40,14 +40,10 @@ abstract class TermsHash {
final ByteBlockPool bytePool;
ByteBlockPool termBytePool;
final Counter bytesUsed;
-
- final DocumentsWriterPerThread.DocState docState;
-
final boolean trackAllocations;
TermsHash(final DocumentsWriterPerThread docWriter, boolean trackAllocations, TermsHash nextTermsHash) {
- this.docState = docWriter.docState;
- this.trackAllocations = trackAllocations;
+ this.trackAllocations = trackAllocations;
this.nextTermsHash = nextTermsHash;
this.bytesUsed = trackAllocations ? docWriter.bytesUsed : Counter.newCounter();
intPool = new IntBlockPool(docWriter.intBlockAllocator);
@@ -82,7 +78,7 @@ void flush(Map fieldsToFlush, final SegmentWriteState
if (nextTermsHash != null) {
Map nextChildFields = new HashMap<>();
for (final Map.Entry entry : fieldsToFlush.entrySet()) {
- nextChildFields.put(entry.getKey(), entry.getValue().nextPerField);
+ nextChildFields.put(entry.getKey(), entry.getValue().getNextPerField());
}
nextTermsHash.flush(nextChildFields, state, sortMap, norms);
}
@@ -90,9 +86,9 @@ void flush(Map fieldsToFlush, final SegmentWriteState
abstract TermsHashPerField addField(FieldInvertState fieldInvertState, FieldInfo fieldInfo);
- void finishDocument() throws IOException {
+ void finishDocument(int docID) throws IOException {
if (nextTermsHash != null) {
- nextTermsHash.finishDocument();
+ nextTermsHash.finishDocument(docID);
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/index/TermsHashPerField.java b/lucene/core/src/java/org/apache/lucene/index/TermsHashPerField.java
index 2586378267da..d3e048703d2d 100644
--- a/lucene/core/src/java/org/apache/lucene/index/TermsHashPerField.java
+++ b/lucene/core/src/java/org/apache/lucene/index/TermsHashPerField.java
@@ -19,182 +19,186 @@
import java.io.IOException;
-import org.apache.lucene.analysis.tokenattributes.TermFrequencyAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
import org.apache.lucene.util.ByteBlockPool;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefHash.BytesStartArray;
import org.apache.lucene.util.BytesRefHash;
import org.apache.lucene.util.Counter;
import org.apache.lucene.util.IntBlockPool;
+/**
+ * This class stores streams of information per term without knowing
+ * the size of the stream ahead of time. Each stream typically encodes one level
+ * of information like term frequency per document or term proximity. Internally
+ * this class allocates a linked list of slices that can be read by a {@link ByteSliceReader}
+ * for each term. Terms are first deduplicated in a {@link BytesRefHash} once this is done
+ * internal data-structures point to the current offset of each stream that can be written to.
+ */
abstract class TermsHashPerField implements Comparable {
private static final int HASH_INIT_SIZE = 4;
- final TermsHash termsHash;
-
- final TermsHashPerField nextPerField;
- protected final DocumentsWriterPerThread.DocState docState;
- protected final FieldInvertState fieldState;
- TermToBytesRefAttribute termAtt;
- protected TermFrequencyAttribute termFreqAtt;
-
- // Copied from our perThread
- final IntBlockPool intPool;
+ private final TermsHashPerField nextPerField;
+ private final IntBlockPool intPool;
final ByteBlockPool bytePool;
- final ByteBlockPool termBytePool;
-
- final int streamCount;
- final int numPostingInt;
-
- protected final FieldInfo fieldInfo;
-
- final BytesRefHash bytesHash;
+ // for each term we store an integer per stream that points into the bytePool above
+ // the address is updated once data is written to the stream to point to the next free offset
+ // in the terms stream. The start address for the stream is stored in postingsArray.byteStarts[termId]
+ // This is initialized in the #addTerm method, either to a brand new per term stream if the term is new or
+ // to the addresses where the term stream was written to when we saw it the last time.
+ private int[] termStreamAddressBuffer;
+ private int streamAddressOffset;
+ private final int streamCount;
+ private final String fieldName;
+ final IndexOptions indexOptions;
+ /* This stores the actual term bytes for postings and offsets into the parent hash in the case that this
+ * TermsHashPerField is hashing term vectors.*/
+ private final BytesRefHash bytesHash;
ParallelPostingsArray postingsArray;
- private final Counter bytesUsed;
+ private int lastDocID; // only with assert
/** streamCount: how many streams this field stores per term.
* E.g. doc(+freq) is 1 stream, prox+offset is a second. */
-
- public TermsHashPerField(int streamCount, FieldInvertState fieldState, TermsHash termsHash, TermsHashPerField nextPerField, FieldInfo fieldInfo) {
- intPool = termsHash.intPool;
- bytePool = termsHash.bytePool;
- termBytePool = termsHash.termBytePool;
- docState = termsHash.docState;
- this.termsHash = termsHash;
- bytesUsed = termsHash.bytesUsed;
- this.fieldState = fieldState;
+ TermsHashPerField(int streamCount, IntBlockPool intPool, ByteBlockPool bytePool, ByteBlockPool termBytePool,
+ Counter bytesUsed, TermsHashPerField nextPerField, String fieldName, IndexOptions indexOptions) {
+ this.intPool = intPool;
+ this.bytePool = bytePool;
this.streamCount = streamCount;
- numPostingInt = 2*streamCount;
- this.fieldInfo = fieldInfo;
+ this.fieldName = fieldName;
this.nextPerField = nextPerField;
+ assert indexOptions != IndexOptions.NONE;
+ this.indexOptions = indexOptions;
PostingsBytesStartArray byteStarts = new PostingsBytesStartArray(this, bytesUsed);
bytesHash = new BytesRefHash(termBytePool, HASH_INIT_SIZE, byteStarts);
}
void reset() {
bytesHash.clear(false);
+ sortedTermIDs = null;
if (nextPerField != null) {
nextPerField.reset();
}
}
- public void initReader(ByteSliceReader reader, int termID, int stream) {
+ final void initReader(ByteSliceReader reader, int termID, int stream) {
assert stream < streamCount;
- int intStart = postingsArray.intStarts[termID];
- final int[] ints = intPool.buffers[intStart >> IntBlockPool.INT_BLOCK_SHIFT];
- final int upto = intStart & IntBlockPool.INT_BLOCK_MASK;
+ int streamStartOffset = postingsArray.addressOffset[termID];
+ final int[] streamAddressBuffer = intPool.buffers[streamStartOffset >> IntBlockPool.INT_BLOCK_SHIFT];
+ final int offsetInAddressBuffer = streamStartOffset & IntBlockPool.INT_BLOCK_MASK;
reader.init(bytePool,
postingsArray.byteStarts[termID]+stream*ByteBlockPool.FIRST_LEVEL_SIZE,
- ints[upto+stream]);
+ streamAddressBuffer[offsetInAddressBuffer+stream]);
}
- int[] sortedTermIDs;
+ private int[] sortedTermIDs;
/** Collapse the hash table and sort in-place; also sets
- * this.sortedTermIDs to the results */
- public int[] sortPostings() {
+ * this.sortedTermIDs to the results
+ * This method must not be called twice unless {@link #reset()}
+ * or {@link #reinitHash()} was called. */
+ final void sortTerms() {
+ assert sortedTermIDs == null;
sortedTermIDs = bytesHash.sort();
+ }
+
+ /**
+ * Returns the sorted term IDs. {@link #sortTerms()} must be called before
+ */
+ final int[] getSortedTermIDs() {
+ assert sortedTermIDs != null;
return sortedTermIDs;
}
+ final void reinitHash() {
+ sortedTermIDs = null;
+ bytesHash.reinit();
+ }
+
private boolean doNextCall;
// Secondary entry point (for 2nd & subsequent TermsHash),
// because token text has already been "interned" into
// textStart, so we hash by textStart. term vectors use
// this API.
- public void add(int textStart) throws IOException {
+ private void add(int textStart, final int docID) throws IOException {
int termID = bytesHash.addByPoolOffset(textStart);
if (termID >= 0) { // New posting
// First time we are seeing this token since we last
// flushed the hash.
- // Init stream slices
- if (numPostingInt + intPool.intUpto > IntBlockPool.INT_BLOCK_SIZE) {
- intPool.nextBuffer();
- }
-
- if (ByteBlockPool.BYTE_BLOCK_SIZE - bytePool.byteUpto < numPostingInt*ByteBlockPool.FIRST_LEVEL_SIZE) {
- bytePool.nextBuffer();
- }
+ initStreamSlices(termID, docID);
+ } else {
+ positionStreamSlice(termID, docID);
+ }
+ }
- intUptos = intPool.buffer;
- intUptoStart = intPool.intUpto;
- intPool.intUpto += streamCount;
+ private void initStreamSlices(int termID, int docID) throws IOException {
+ // Init stream slices
+ // TODO: figure out why this is 2*streamCount here. streamCount should be enough?
+ if ((2*streamCount) + intPool.intUpto > IntBlockPool.INT_BLOCK_SIZE) {
+ // can we fit all the streams in the current buffer?
+ intPool.nextBuffer();
+ }
- postingsArray.intStarts[termID] = intUptoStart + intPool.intOffset;
+ if (ByteBlockPool.BYTE_BLOCK_SIZE - bytePool.byteUpto < (2*streamCount) * ByteBlockPool.FIRST_LEVEL_SIZE) {
+ // can we fit at least one byte per stream in the current buffer, if not allocate a new one
+ bytePool.nextBuffer();
+ }
- for(int i=0;i> IntBlockPool.INT_BLOCK_SHIFT];
- intUptoStart = intStart & IntBlockPool.INT_BLOCK_MASK;
- addTerm(termID);
+ for (int i = 0; i < streamCount; i++) {
+ // initialize each stream with a slice we start with ByteBlockPool.FIRST_LEVEL_SIZE)
+ // and grow as we need more space. see ByteBlockPool.LEVEL_SIZE_ARRAY
+ final int upto = bytePool.newSlice(ByteBlockPool.FIRST_LEVEL_SIZE);
+ termStreamAddressBuffer[streamAddressOffset + i] = upto + bytePool.byteOffset;
}
+ postingsArray.byteStarts[termID] = termStreamAddressBuffer[streamAddressOffset];
+ newTerm(termID, docID);
+ }
+
+ private boolean assertDocId(int docId) {
+ assert docId >= lastDocID : "docID must be >= " + lastDocID + " but was: " + docId;
+ lastDocID = docId;
+ return true;
}
/** Called once per inverted token. This is the primary
* entry point (for first TermsHash); postings use this
* API. */
- void add() throws IOException {
+ void add(BytesRef termBytes, final int docID) throws IOException {
+ assert assertDocId(docID);
// We are first in the chain so we must "intern" the
// term text into textStart address
// Get the text & hash of this term.
- int termID = bytesHash.add(termAtt.getBytesRef());
-
+ int termID = bytesHash.add(termBytes);
//System.out.println("add term=" + termBytesRef.utf8ToString() + " doc=" + docState.docID + " termID=" + termID);
-
- if (termID >= 0) {// New posting
- bytesHash.byteStart(termID);
+ if (termID >= 0) { // New posting
// Init stream slices
- if (numPostingInt + intPool.intUpto > IntBlockPool.INT_BLOCK_SIZE) {
- intPool.nextBuffer();
- }
-
- if (ByteBlockPool.BYTE_BLOCK_SIZE - bytePool.byteUpto < numPostingInt*ByteBlockPool.FIRST_LEVEL_SIZE) {
- bytePool.nextBuffer();
- }
-
- intUptos = intPool.buffer;
- intUptoStart = intPool.intUpto;
- intPool.intUpto += streamCount;
-
- postingsArray.intStarts[termID] = intUptoStart + intPool.intOffset;
-
- for(int i=0;i> IntBlockPool.INT_BLOCK_SHIFT];
- intUptoStart = intStart & IntBlockPool.INT_BLOCK_MASK;
- addTerm(termID);
+ termID = positionStreamSlice(termID, docID);
}
-
if (doNextCall) {
- nextPerField.add(postingsArray.textStarts[termID]);
+ nextPerField.add(postingsArray.textStarts[termID], docID);
}
}
- int[] intUptos;
- int intUptoStart;
+ private int positionStreamSlice(int termID, final int docID) throws IOException {
+ termID = (-termID) - 1;
+ int intStart = postingsArray.addressOffset[termID];
+ termStreamAddressBuffer = intPool.buffers[intStart >> IntBlockPool.INT_BLOCK_SHIFT];
+ streamAddressOffset = intStart & IntBlockPool.INT_BLOCK_MASK;
+ addTerm(termID, docID);
+ return termID;
+ }
- void writeByte(int stream, byte b) {
- int upto = intUptos[intUptoStart+stream];
+ final void writeByte(int stream, byte b) {
+ int streamAddress = streamAddressOffset + stream;
+ int upto = termStreamAddressBuffer[streamAddress];
byte[] bytes = bytePool.buffers[upto >> ByteBlockPool.BYTE_BLOCK_SHIFT];
assert bytes != null;
int offset = upto & ByteBlockPool.BYTE_BLOCK_MASK;
@@ -202,20 +206,20 @@ void writeByte(int stream, byte b) {
// End of slice; allocate a new one
offset = bytePool.allocSlice(bytes, offset);
bytes = bytePool.buffer;
- intUptos[intUptoStart+stream] = offset + bytePool.byteOffset;
+ termStreamAddressBuffer[streamAddress] = offset + bytePool.byteOffset;
}
bytes[offset] = b;
- (intUptos[intUptoStart+stream])++;
+ (termStreamAddressBuffer[streamAddress])++;
}
- public void writeBytes(int stream, byte[] b, int offset, int len) {
+ final void writeBytes(int stream, byte[] b, int offset, int len) {
// TODO: optimize
final int end = offset + len;
for(int i=offset;i LevenshteinAutomata.MAXIMUM_SUPPORTED_DISTANCE) {
+ throw new IllegalArgumentException("max edits must be 0.." + LevenshteinAutomata.MAXIMUM_SUPPORTED_DISTANCE + ", inclusive; got: " + maxEdits);
+ }
+ if (prefixLength < 0) {
+ throw new IllegalArgumentException("prefixLength cannot be less than 0");
+ }
+ this.term = term;
+ this.maxEdits = maxEdits;
+ int[] codePoints = stringToUTF32(term);
+ this.termLength = codePoints.length;
+ prefixLength = Math.min(prefixLength, codePoints.length);
+ int[] suffix = new int[codePoints.length - prefixLength];
+ System.arraycopy(codePoints, prefixLength, suffix, 0, suffix.length);
+ this.levBuilder = new LevenshteinAutomata(suffix, Character.MAX_CODE_POINT, transpositions);
+ this.prefix = UnicodeUtil.newString(codePoints, 0, prefixLength);
+ }
+
+ CompiledAutomaton[] buildAutomatonSet() {
+ CompiledAutomaton[] compiled = new CompiledAutomaton[maxEdits + 1];
+ for (int i = 0; i <= maxEdits; i++) {
+ try {
+ compiled[i] = new CompiledAutomaton(levBuilder.toAutomaton(i, prefix), true, false);
+ }
+ catch (TooComplexToDeterminizeException e) {
+ throw new FuzzyTermsEnum.FuzzyTermsException(term, e);
+ }
+ }
+ return compiled;
+ }
+
+ CompiledAutomaton buildMaxEditAutomaton() {
+ try {
+ return new CompiledAutomaton(levBuilder.toAutomaton(maxEdits, prefix), true, false);
+ } catch (TooComplexToDeterminizeException e) {
+ throw new FuzzyTermsEnum.FuzzyTermsException(term, e);
+ }
+ }
+
+ int getTermLength() {
+ return this.termLength;
+ }
+
+ private static int[] stringToUTF32(String text) {
+ int[] termText = new int[text.codePointCount(0, text.length())];
+ for (int cp, i = 0, j = 0; i < text.length(); i += Character.charCount(cp)) {
+ termText[j++] = cp = text.codePointAt(i);
+ }
+ return termText;
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/search/FuzzyQuery.java b/lucene/core/src/java/org/apache/lucene/search/FuzzyQuery.java
index c4b4d1b6adab..041f0ca180ae 100644
--- a/lucene/core/src/java/org/apache/lucene/search/FuzzyQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/FuzzyQuery.java
@@ -18,14 +18,13 @@
import java.io.IOException;
+import java.util.Objects;
import org.apache.lucene.index.SingleTermsEnum;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.AttributeSource;
-import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.automaton.CompiledAutomaton;
import org.apache.lucene.util.automaton.LevenshteinAutomata;
@@ -53,9 +52,7 @@
* not match an indexed term "ab", and FuzzyQuery on term "a" with maxEdits=2 will not
* match an indexed term "abc".
*/
-public class FuzzyQuery extends MultiTermQuery implements Accountable {
-
- private static final long BASE_RAM_BYTES = RamUsageEstimator.shallowSizeOfInstance(AutomatonQuery.class);
+public class FuzzyQuery extends MultiTermQuery {
public final static int defaultMaxEdits = LevenshteinAutomata.MAXIMUM_SUPPORTED_DISTANCE;
public final static int defaultPrefixLength = 0;
@@ -67,10 +64,6 @@ public class FuzzyQuery extends MultiTermQuery implements Accountable {
private final boolean transpositions;
private final int prefixLength;
private final Term term;
- private final int termLength;
- private final CompiledAutomaton[] automata;
-
- private final long ramBytesUsed;
/**
* Create a new FuzzyQuery that will match terms with an edit distance
@@ -106,22 +99,7 @@ public FuzzyQuery(Term term, int maxEdits, int prefixLength, int maxExpansions,
this.prefixLength = prefixLength;
this.transpositions = transpositions;
this.maxExpansions = maxExpansions;
- int[] codePoints = FuzzyTermsEnum.stringToUTF32(term.text());
- this.termLength = codePoints.length;
- this.automata = FuzzyTermsEnum.buildAutomata(term.text(), codePoints, prefixLength, transpositions, maxEdits);
setRewriteMethod(new MultiTermQuery.TopTermsBlendedFreqScoringRewrite(maxExpansions));
- this.ramBytesUsed = calculateRamBytesUsed(term, this.automata);
- }
-
- private static long calculateRamBytesUsed(Term term, CompiledAutomaton[] automata) {
- long bytes = BASE_RAM_BYTES + term.ramBytesUsed();
- for (CompiledAutomaton a : automata) {
- bytes += a.ramBytesUsed();
- }
- bytes += 4 * Integer.BYTES;
- bytes += Long.BYTES;
- bytes += 1;
- return bytes;
}
/**
@@ -173,8 +151,9 @@ public boolean getTranspositions() {
/**
* Returns the compiled automata used to match terms
*/
- public CompiledAutomaton[] getAutomata() {
- return automata;
+ public CompiledAutomaton getAutomata() {
+ FuzzyAutomatonBuilder builder = new FuzzyAutomatonBuilder(term.text(), maxEdits, prefixLength, transpositions);
+ return builder.buildMaxEditAutomaton();
}
@Override
@@ -183,17 +162,17 @@ public void visit(QueryVisitor visitor) {
if (maxEdits == 0 || prefixLength >= term.text().length()) {
visitor.consumeTerms(this, term);
} else {
- automata[automata.length - 1].visit(visitor, this, field);
+ visitor.consumeTermsMatching(this, term.field(), () -> getAutomata().runAutomaton);
}
}
}
@Override
protected TermsEnum getTermsEnum(Terms terms, AttributeSource atts) throws IOException {
- if (maxEdits == 0 || prefixLength >= term.text().length()) { // can only match if it's exact
+ if (maxEdits == 0) { // can only match if it's exact
return new SingleTermsEnum(terms.iterator(), term.bytes());
}
- return new FuzzyTermsEnum(terms, atts, getTerm(), termLength, maxEdits, automata);
+ return new FuzzyTermsEnum(terms, atts, getTerm(), maxEdits, prefixLength, transpositions);
}
/**
@@ -237,22 +216,9 @@ public boolean equals(Object obj) {
if (getClass() != obj.getClass())
return false;
FuzzyQuery other = (FuzzyQuery) obj;
- // Note that we don't need to compare termLength or automata because they
- // are entirely determined by the other fields
- if (maxEdits != other.maxEdits)
- return false;
- if (prefixLength != other.prefixLength)
- return false;
- if (maxExpansions != other.maxExpansions)
- return false;
- if (transpositions != other.transpositions)
- return false;
- if (term == null) {
- if (other.term != null)
- return false;
- } else if (!term.equals(other.term))
- return false;
- return true;
+ return Objects.equals(maxEdits, other.maxEdits) && Objects.equals(prefixLength, other.prefixLength)
+ && Objects.equals(maxExpansions, other.maxExpansions) && Objects.equals(transpositions, other.transpositions)
+ && Objects.equals(term, other.term);
}
/**
@@ -274,8 +240,4 @@ public static int floatToEdits(float minimumSimilarity, int termLen) {
}
}
- @Override
- public long ramBytesUsed() {
- return ramBytesUsed;
- }
}
diff --git a/lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java b/lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java
index 91a44d5245c0..4c49d8accd77 100644
--- a/lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java
+++ b/lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java
@@ -18,6 +18,7 @@
import java.io.IOException;
+import java.util.function.Supplier;
import org.apache.lucene.index.ImpactsEnum;
import org.apache.lucene.index.PostingsEnum;
@@ -25,14 +26,14 @@
import org.apache.lucene.index.TermState;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.util.Attribute;
+import org.apache.lucene.util.AttributeImpl;
+import org.apache.lucene.util.AttributeReflector;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.UnicodeUtil;
-import org.apache.lucene.util.automaton.Automaton;
import org.apache.lucene.util.automaton.CompiledAutomaton;
-import org.apache.lucene.util.automaton.LevenshteinAutomata;
-import org.apache.lucene.util.automaton.TooComplexToDeterminizeException;
/** Subclass of TermsEnum for enumerating all terms that are similar
* to the specified filter term.
@@ -57,21 +58,21 @@ public final class FuzzyTermsEnum extends TermsEnum {
private final MaxNonCompetitiveBoostAttribute maxBoostAtt;
private final CompiledAutomaton[] automata;
+ private final Terms terms;
+ private final int termLength;
+ private final Term term;
private float bottom;
private BytesRef bottomTerm;
private BytesRef queuedBottom;
- private final int termLength;
// Maximum number of edits we will accept. This is either 2 or 1 (or, degenerately, 0) passed by the user originally,
// but as we collect terms, we can lower this (e.g. from 2 to 1) if we detect that the term queue is full, and all
// collected terms are ed=1:
private int maxEdits;
- private final Terms terms;
- private final Term term;
/**
* Constructor for enumeration of all terms from specified reader which share a prefix of
@@ -88,43 +89,44 @@ public final class FuzzyTermsEnum extends TermsEnum {
* @throws IOException if there is a low-level IO error
*/
public FuzzyTermsEnum(Terms terms, Term term, int maxEdits, int prefixLength, boolean transpositions) throws IOException {
- this(terms, term, stringToUTF32(term.text()), maxEdits, prefixLength, transpositions);
- }
-
- private FuzzyTermsEnum(Terms terms, Term term, int[] codePoints, int maxEdits, int prefixLength, boolean transpositions) throws IOException {
- this(terms, new AttributeSource(), term, codePoints.length, maxEdits,
- buildAutomata(term.text(), codePoints, prefixLength, transpositions, maxEdits));
+ this(terms, new AttributeSource(), term, () -> new FuzzyAutomatonBuilder(term.text(), maxEdits, prefixLength, transpositions));
}
/**
* Constructor for enumeration of all terms from specified reader which share a prefix of
* length prefixLength with term and which have at most {@code maxEdits} edits.
*
- * After calling the constructor the enumeration is already pointing to the first
- * valid term if such a term exists.
- *
+ * After calling the constructor the enumeration is already pointing to the first
+ * valid term if such a term exists.
+ *
* @param terms Delivers terms.
- * @param atts {@link AttributeSource} created by the rewrite method of {@link MultiTermQuery}
- * that contains information about competitive boosts during rewrite
+ * @param atts An AttributeSource used to share automata between segments
* @param term Pattern term.
* @param maxEdits Maximum edit distance.
- * @param automata An array of levenshtein automata to match against terms,
- * see {@link #buildAutomata(String, int[], int, boolean, int)}
+ * @param prefixLength the length of the required common prefix
+ * @param transpositions whether transpositions should count as a single edit
* @throws IOException if there is a low-level IO error
*/
- public FuzzyTermsEnum(Terms terms, AttributeSource atts, Term term, int termLength,
- final int maxEdits, CompiledAutomaton[] automata) throws IOException {
+ FuzzyTermsEnum(Terms terms, AttributeSource atts, Term term, int maxEdits, int prefixLength, boolean transpositions) throws IOException {
+ this(terms, atts, term, () -> new FuzzyAutomatonBuilder(term.text(), maxEdits, prefixLength, transpositions));
+ }
+
+ private FuzzyTermsEnum(Terms terms, AttributeSource atts, Term term, Supplier automatonBuilder) throws IOException {
- this.maxEdits = maxEdits;
this.terms = terms;
- this.term = term;
this.atts = atts;
- this.termLength = termLength;
+ this.term = term;
this.maxBoostAtt = atts.addAttribute(MaxNonCompetitiveBoostAttribute.class);
this.boostAtt = atts.addAttribute(BoostAttribute.class);
- this.automata = automata;
+ atts.addAttributeImpl(new AutomatonAttributeImpl());
+ AutomatonAttribute aa = atts.addAttribute(AutomatonAttribute.class);
+ aa.init(automatonBuilder);
+
+ this.automata = aa.getAutomata();
+ this.termLength = aa.getTermLength();
+ this.maxEdits = this.automata.length - 1;
bottom = maxBoostAtt.getMaxNonCompetitiveBoost();
bottomTerm = maxBoostAtt.getCompetitiveTerm();
@@ -145,47 +147,6 @@ public void setMaxNonCompetitiveBoost(float boost) {
public float getBoost() {
return boostAtt.getBoost();
}
-
- static CompiledAutomaton[] buildAutomata(String text, int[] termText, int prefixLength, boolean transpositions, int maxEdits) {
- CompiledAutomaton[] compiled = new CompiledAutomaton[maxEdits + 1];
- Automaton[] automata = buildAutomata(termText, prefixLength, transpositions, maxEdits);
- for (int i = 0; i <= maxEdits; i++) {
- try {
- compiled[i] = new CompiledAutomaton(automata[i], true, false);
- }
- catch (TooComplexToDeterminizeException e) {
- throw new FuzzyTermsException(text, e);
- }
- }
- return compiled;
- }
-
- static int[] stringToUTF32(String text) {
- int[] termText = new int[text.codePointCount(0, text.length())];
- for (int cp, i = 0, j = 0; i < text.length(); i += Character.charCount(cp)) {
- termText[j++] = cp = text.codePointAt(i);
- }
- return termText;
- }
-
- private static Automaton[] buildAutomata(int[] termText, int prefixLength, boolean transpositions, int maxEdits) {
- if (maxEdits < 0 || maxEdits > LevenshteinAutomata.MAXIMUM_SUPPORTED_DISTANCE) {
- throw new IllegalArgumentException("max edits must be 0.." + LevenshteinAutomata.MAXIMUM_SUPPORTED_DISTANCE + ", inclusive; got: " + maxEdits);
- }
- if (prefixLength < 0) {
- throw new IllegalArgumentException("prefixLength cannot be less than 0");
- }
- Automaton[] automata = new Automaton[maxEdits + 1];
- int termLength = termText.length;
- prefixLength = Math.min(prefixLength, termLength);
- String suffix = UnicodeUtil.newString(termText, prefixLength, termText.length - prefixLength);
- LevenshteinAutomata builder = new LevenshteinAutomata(suffix, transpositions);
- String prefix = UnicodeUtil.newString(termText, 0, prefixLength);
- for (int i = 0; i <= maxEdits; i++) {
- automata[i] = builder.toAutomaton(i, prefix);
- }
- return automata;
- }
/**
* return an automata-based enum for matching up to editDistance from
@@ -274,7 +235,7 @@ public BytesRef next() throws IOException {
final float bottom = maxBoostAtt.getMaxNonCompetitiveBoost();
final BytesRef bottomTerm = maxBoostAtt.getCompetitiveTerm();
- if (term != null && (bottom != this.bottom || bottomTerm != this.bottomTerm)) {
+ if (bottom != this.bottom || bottomTerm != this.bottomTerm) {
this.bottom = bottom;
this.bottomTerm = bottomTerm;
// clone the term before potentially doing something with it
@@ -364,4 +325,60 @@ public static class FuzzyTermsException extends RuntimeException {
}
}
+ /**
+ * Used for sharing automata between segments
+ *
+ * Levenshtein automata are large and expensive to build; we don't want to build
+ * them directly on the query because this can blow up caches that use queries
+ * as keys; we also don't want to rebuild them for every segment. This attribute
+ * allows the FuzzyTermsEnum to build the automata once for its first segment
+ * and then share them for subsequent segment calls.
+ */
+ private interface AutomatonAttribute extends Attribute {
+ CompiledAutomaton[] getAutomata();
+ int getTermLength();
+ void init(Supplier builder);
+ }
+
+ private static class AutomatonAttributeImpl extends AttributeImpl implements AutomatonAttribute {
+
+ private CompiledAutomaton[] automata;
+ private int termLength;
+
+ @Override
+ public CompiledAutomaton[] getAutomata() {
+ return automata;
+ }
+
+ @Override
+ public int getTermLength() {
+ return termLength;
+ }
+
+ @Override
+ public void init(Supplier supplier) {
+ if (automata != null) {
+ return;
+ }
+ FuzzyAutomatonBuilder builder = supplier.get();
+ this.termLength = builder.getTermLength();
+ this.automata = builder.buildAutomatonSet();
+ }
+
+ @Override
+ public void clear() {
+ this.automata = null;
+ }
+
+ @Override
+ public void reflectWith(AttributeReflector reflector) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void copyTo(AttributeImpl target) {
+ throw new UnsupportedOperationException();
+ }
+ }
+
}
diff --git a/lucene/core/src/java/org/apache/lucene/search/MultiCollector.java b/lucene/core/src/java/org/apache/lucene/search/MultiCollector.java
index 82251e43cfe8..5cb6db8acd48 100644
--- a/lucene/core/src/java/org/apache/lucene/search/MultiCollector.java
+++ b/lucene/core/src/java/org/apache/lucene/search/MultiCollector.java
@@ -117,7 +117,7 @@ public ScoreMode scoreMode() {
@Override
public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {
- final List leafCollectors = new ArrayList<>();
+ final List leafCollectors = new ArrayList<>(collectors.length);
for (Collector collector : collectors) {
final LeafCollector leafCollector;
try {
@@ -134,7 +134,7 @@ public LeafCollector getLeafCollector(LeafReaderContext context) throws IOExcept
case 1:
return leafCollectors.get(0);
default:
- return new MultiLeafCollector(leafCollectors, cacheScores);
+ return new MultiLeafCollector(leafCollectors, cacheScores, scoreMode() == ScoreMode.TOP_SCORES);
}
}
@@ -142,12 +142,14 @@ private static class MultiLeafCollector implements LeafCollector {
private final boolean cacheScores;
private final LeafCollector[] collectors;
- private int numCollectors;
+ private final float[] minScores;
+ private final boolean skipNonCompetitiveScores;
- private MultiLeafCollector(List collectors, boolean cacheScores) {
+ private MultiLeafCollector(List collectors, boolean cacheScores, boolean skipNonCompetitive) {
this.collectors = collectors.toArray(new LeafCollector[collectors.size()]);
this.cacheScores = cacheScores;
- this.numCollectors = this.collectors.length;
+ this.skipNonCompetitiveScores = skipNonCompetitive;
+ this.minScores = this.skipNonCompetitiveScores ? new float[this.collectors.length] : null;
}
@Override
@@ -155,48 +157,89 @@ public void setScorer(Scorable scorer) throws IOException {
if (cacheScores) {
scorer = new ScoreCachingWrappingScorer(scorer);
}
- scorer = new FilterScorable(scorer) {
- @Override
- public void setMinCompetitiveScore(float minScore) {
- // Ignore calls to setMinCompetitiveScore so that if we wrap two
- // collectors and one of them wants to skip low-scoring hits, then
- // the other collector still sees all hits. We could try to reconcile
- // min scores and take the maximum min score across collectors, but
- // this is very unlikely to be helpful in practice.
+ if (skipNonCompetitiveScores) {
+ for (int i = 0; i < collectors.length; ++i) {
+ final LeafCollector c = collectors[i];
+ if (c != null) {
+ c.setScorer(new MinCompetitiveScoreAwareScorable(scorer, i, minScores));
+ }
}
+ } else {
+ scorer = new FilterScorable(scorer) {
+ @Override
+ public void setMinCompetitiveScore(float minScore) throws IOException {
+ // Ignore calls to setMinCompetitiveScore so that if we wrap two
+ // collectors and one of them wants to skip low-scoring hits, then
+ // the other collector still sees all hits.
+ }
- };
- for (int i = 0; i < numCollectors; ++i) {
- final LeafCollector c = collectors[i];
- c.setScorer(scorer);
+ };
+ for (int i = 0; i < collectors.length; ++i) {
+ final LeafCollector c = collectors[i];
+ if (c != null) {
+ c.setScorer(scorer);
+ }
+ }
}
}
- private void removeCollector(int i) {
- System.arraycopy(collectors, i + 1, collectors, i, numCollectors - i - 1);
- --numCollectors;
- collectors[numCollectors] = null;
- }
-
@Override
public void collect(int doc) throws IOException {
- final LeafCollector[] collectors = this.collectors;
- int numCollectors = this.numCollectors;
- for (int i = 0; i < numCollectors; ) {
+ for (int i = 0; i < collectors.length; i++) {
final LeafCollector collector = collectors[i];
- try {
- collector.collect(doc);
- ++i;
- } catch (CollectionTerminatedException e) {
- removeCollector(i);
- numCollectors = this.numCollectors;
- if (numCollectors == 0) {
- throw new CollectionTerminatedException();
+ if (collector != null) {
+ try {
+ collector.collect(doc);
+ } catch (CollectionTerminatedException e) {
+ collectors[i] = null;
+ if (allCollectorsTerminated()) {
+ throw new CollectionTerminatedException();
+ }
}
}
}
}
+ private boolean allCollectorsTerminated() {
+ for (int i = 0; i < collectors.length; i++) {
+ if (collectors[i] != null) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ }
+
+ final static class MinCompetitiveScoreAwareScorable extends FilterScorable {
+
+ private final int idx;
+ private final float[] minScores;
+
+ MinCompetitiveScoreAwareScorable(Scorable in, int idx, float[] minScores) {
+ super(in);
+ this.idx = idx;
+ this.minScores = minScores;
+ }
+
+ @Override
+ public void setMinCompetitiveScore(float minScore) throws IOException {
+ if (minScore > minScores[idx]) {
+ minScores[idx] = minScore;
+ in.setMinCompetitiveScore(minScore());
+ }
+ }
+
+ private float minScore() {
+ float min = Float.MAX_VALUE;
+ for (int i = 0; i < minScores.length; i++) {
+ if (minScores[i] < min) {
+ min = minScores[i];
+ }
+ }
+ return min;
+ }
+
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/search/MultiTermQuery.java b/lucene/core/src/java/org/apache/lucene/search/MultiTermQuery.java
index 327227afe151..e3c4ff7097f4 100644
--- a/lucene/core/src/java/org/apache/lucene/search/MultiTermQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/MultiTermQuery.java
@@ -286,9 +286,9 @@ public MultiTermQuery(final String field) {
* (should instead return {@link TermsEnum#EMPTY} if no
* terms match). The TermsEnum must already be
* positioned to the first matching term.
- * The given {@link AttributeSource} is passed by the {@link RewriteMethod} to
- * provide attributes, the rewrite method uses to inform about e.g. maximum competitive boosts.
- * This is currently only used by {@link TopTermsRewrite}
+ * The given {@link AttributeSource} is passed by the {@link RewriteMethod} to
+ * share information between segments, for example {@link TopTermsRewrite} uses
+ * it to share maximum competitive boosts
*/
protected abstract TermsEnum getTermsEnum(Terms terms, AttributeSource atts) throws IOException;
diff --git a/lucene/core/src/java/org/apache/lucene/search/SortField.java b/lucene/core/src/java/org/apache/lucene/search/SortField.java
index 2cfae46f01d8..7512ec934416 100644
--- a/lucene/core/src/java/org/apache/lucene/search/SortField.java
+++ b/lucene/core/src/java/org/apache/lucene/search/SortField.java
@@ -21,7 +21,13 @@
import java.util.Comparator;
import java.util.Objects;
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.IndexSorter;
+import org.apache.lucene.index.SortFieldProvider;
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.DataOutput;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.NumericUtils;
/**
* Stores information about how to sort documents by terms in an individual
@@ -120,6 +126,106 @@ public SortField(String field, Type type, boolean reverse) {
this.reverse = reverse;
}
+ /** A SortFieldProvider for field sorts */
+ public static final class Provider extends SortFieldProvider {
+
+ /** The name this Provider is registered under */
+ public static final String NAME = "SortField";
+
+ /** Creates a new Provider */
+ public Provider() {
+ super(NAME);
+ }
+
+ @Override
+ public SortField readSortField(DataInput in) throws IOException {
+ SortField sf = new SortField(in.readString(), readType(in), in.readInt() == 1);
+ if (in.readInt() == 1) {
+ // missing object
+ switch (sf.type) {
+ case STRING:
+ int missingString = in.readInt();
+ if (missingString == 1) {
+ sf.setMissingValue(STRING_FIRST);
+ }
+ else {
+ sf.setMissingValue(STRING_LAST);
+ }
+ break;
+ case INT:
+ sf.setMissingValue(in.readInt());
+ break;
+ case LONG:
+ sf.setMissingValue(in.readLong());
+ break;
+ case FLOAT:
+ sf.setMissingValue(NumericUtils.sortableIntToFloat(in.readInt()));
+ break;
+ case DOUBLE:
+ sf.setMissingValue(NumericUtils.sortableLongToDouble(in.readLong()));
+ break;
+ default:
+ throw new IllegalArgumentException("Cannot deserialize sort of type " + sf.type);
+ }
+ }
+ return sf;
+ }
+
+ @Override
+ public void writeSortField(SortField sf, DataOutput out) throws IOException {
+ sf.serialize(out);
+ }
+ }
+
+ protected static Type readType(DataInput in) throws IOException {
+ String type = in.readString();
+ try {
+ return Type.valueOf(type);
+ }
+ catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException("Can't deserialize SortField - unknown type " + type);
+ }
+ }
+
+ private void serialize(DataOutput out) throws IOException {
+ out.writeString(field);
+ out.writeString(type.toString());
+ out.writeInt(reverse ? 1 : 0);
+ if (missingValue == null) {
+ out.writeInt(0);
+ }
+ else {
+ out.writeInt(1);
+ switch (type) {
+ case STRING:
+ if (missingValue == STRING_LAST) {
+ out.writeInt(0);
+ }
+ else if (missingValue == STRING_FIRST) {
+ out.writeInt(1);
+ }
+ else {
+ throw new IllegalArgumentException("Cannot serialize missing value of " + missingValue + " for type STRING");
+ }
+ break;
+ case INT:
+ out.writeInt((int)missingValue);
+ break;
+ case LONG:
+ out.writeLong((long)missingValue);
+ break;
+ case FLOAT:
+ out.writeInt(NumericUtils.floatToSortableInt((float)missingValue));
+ break;
+ case DOUBLE:
+ out.writeLong(NumericUtils.doubleToSortableLong((double)missingValue));
+ break;
+ default:
+ throw new IllegalArgumentException("Cannot serialize SortField of type " + type);
+ }
+ }
+ }
+
/** Pass this to {@link #setMissingValue} to have missing
* string values sort first. */
public final static Object STRING_FIRST = new Object() {
@@ -392,4 +498,33 @@ public SortField rewrite(IndexSearcher searcher) throws IOException {
public boolean needsScores() {
return type == Type.SCORE;
}
+
+ /**
+ * Returns an {@link IndexSorter} used for sorting index segments by this SortField.
+ *
+ * If the SortField cannot be used for index sorting (for example, if it uses scores or
+ * other query-dependent values) then this method should return {@code null}
+ *
+ * SortFields that implement this method should also implement a companion
+ * {@link SortFieldProvider} to serialize and deserialize the sort in index segment
+ * headers
+ *
+ * @lucene.experimental
+ */
+ public IndexSorter getIndexSorter() {
+ switch (type) {
+ case STRING:
+ return new IndexSorter.StringSorter(Provider.NAME, missingValue, reverse, reader -> DocValues.getSorted(reader, field));
+ case INT:
+ return new IndexSorter.IntSorter(Provider.NAME, (Integer)missingValue, reverse, reader -> DocValues.getNumeric(reader, field));
+ case LONG:
+ return new IndexSorter.LongSorter(Provider.NAME, (Long)missingValue, reverse, reader -> DocValues.getNumeric(reader, field));
+ case DOUBLE:
+ return new IndexSorter.DoubleSorter(Provider.NAME, (Double)missingValue, reverse, reader -> DocValues.getNumeric(reader, field));
+ case FLOAT:
+ return new IndexSorter.FloatSorter(Provider.NAME, (Float)missingValue, reverse, reader -> DocValues.getNumeric(reader, field));
+ default: return null;
+ }
+ }
+
}
diff --git a/lucene/core/src/java/org/apache/lucene/search/SortedNumericSortField.java b/lucene/core/src/java/org/apache/lucene/search/SortedNumericSortField.java
index fff000b96f6a..6c5154a3ee35 100644
--- a/lucene/core/src/java/org/apache/lucene/search/SortedNumericSortField.java
+++ b/lucene/core/src/java/org/apache/lucene/search/SortedNumericSortField.java
@@ -20,9 +20,15 @@
import java.io.IOException;
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.IndexSorter;
+import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.index.SortFieldProvider;
import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.util.NumericUtils;
/**
* SortField for {@link SortedNumericDocValues}.
@@ -83,6 +89,86 @@ public SortedNumericSortField(String field, SortField.Type type, boolean reverse
this.type = type;
}
+ /** A SortFieldProvider for this sort field */
+ public static final class Provider extends SortFieldProvider {
+
+ /** The name this provider is registered under */
+ public static final String NAME = "SortedNumericSortField";
+
+ /** Creates a new Provider */
+ public Provider() {
+ super(NAME);
+ }
+
+ @Override
+ public SortField readSortField(DataInput in) throws IOException {
+ SortedNumericSortField sf = new SortedNumericSortField(in.readString(), readType(in), in.readInt() == 1, readSelectorType(in));
+ if (in.readInt() == 1) {
+ switch (sf.type) {
+ case INT:
+ sf.setMissingValue(in.readInt());
+ break;
+ case LONG:
+ sf.setMissingValue(in.readLong());
+ break;
+ case FLOAT:
+ sf.setMissingValue(NumericUtils.sortableIntToFloat(in.readInt()));
+ break;
+ case DOUBLE:
+ sf.setMissingValue(NumericUtils.sortableLongToDouble(in.readLong()));
+ break;
+ default:
+ throw new AssertionError();
+ }
+ }
+ return sf;
+ }
+
+ @Override
+ public void writeSortField(SortField sf, DataOutput out) throws IOException {
+ assert sf instanceof SortedNumericSortField;
+ ((SortedNumericSortField)sf).serialize(out);
+ }
+ }
+
+ private static SortedNumericSelector.Type readSelectorType(DataInput in) throws IOException {
+ int selectorType = in.readInt();
+ if (selectorType >= SortedNumericSelector.Type.values().length) {
+ throw new IllegalArgumentException("Can't deserialize SortedNumericSortField - unknown selector type " + selectorType);
+ }
+ return SortedNumericSelector.Type.values()[selectorType];
+ }
+
+ private void serialize(DataOutput out) throws IOException {
+ out.writeString(getField());
+ out.writeString(type.toString());
+ out.writeInt(reverse ? 1 : 0);
+ out.writeInt(selector.ordinal());
+ if (missingValue == null) {
+ out.writeInt(0);
+ }
+ else {
+ out.writeInt(1);
+ // oh for switch expressions...
+ switch (type) {
+ case INT:
+ out.writeInt((int)missingValue);
+ break;
+ case LONG:
+ out.writeLong((long)missingValue);
+ break;
+ case FLOAT:
+ out.writeInt(NumericUtils.floatToSortableInt((float)missingValue));
+ break;
+ case DOUBLE:
+ out.writeLong(NumericUtils.doubleToSortableLong((double)missingValue));
+ break;
+ default:
+ throw new AssertionError();
+ }
+ }
+ }
+
/** Returns the numeric type in use for this sort */
public SortField.Type getNumericType() {
return type;
@@ -170,4 +256,24 @@ protected NumericDocValues getNumericDocValues(LeafReaderContext context, String
throw new AssertionError();
}
}
+
+ private NumericDocValues getValue(LeafReader reader) throws IOException {
+ return SortedNumericSelector.wrap(DocValues.getSortedNumeric(reader, getField()), selector, type);
+ }
+
+ @Override
+ public IndexSorter getIndexSorter() {
+ switch(type) {
+ case INT:
+ return new IndexSorter.IntSorter(Provider.NAME, (Integer)missingValue, reverse, this::getValue);
+ case LONG:
+ return new IndexSorter.LongSorter(Provider.NAME, (Long)missingValue, reverse, this::getValue);
+ case DOUBLE:
+ return new IndexSorter.DoubleSorter(Provider.NAME, (Double)missingValue, reverse, this::getValue);
+ case FLOAT:
+ return new IndexSorter.FloatSorter(Provider.NAME, (Float)missingValue, reverse, this::getValue);
+ default:
+ throw new AssertionError();
+ }
+ }
}
diff --git a/lucene/core/src/java/org/apache/lucene/search/SortedSetSortField.java b/lucene/core/src/java/org/apache/lucene/search/SortedSetSortField.java
index b095c6e88fcd..2321a667bdb9 100644
--- a/lucene/core/src/java/org/apache/lucene/search/SortedSetSortField.java
+++ b/lucene/core/src/java/org/apache/lucene/search/SortedSetSortField.java
@@ -16,13 +16,17 @@
*/
package org.apache.lucene.search;
-
import java.io.IOException;
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.IndexSorter;
+import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.SortFieldProvider;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.DataOutput;
/**
* SortField for {@link SortedSetDocValues}.
@@ -68,6 +72,60 @@ public SortedSetSortField(String field, boolean reverse, SortedSetSelector.Type
}
this.selector = selector;
}
+
+ /** A SortFieldProvider for this sort */
+ public static final class Provider extends SortFieldProvider {
+
+ /** The name this provider is registered under */
+ public static final String NAME = "SortedSetSortField";
+
+ /** Creates a new Provider */
+ public Provider() {
+ super(NAME);
+ }
+
+ @Override
+ public SortField readSortField(DataInput in) throws IOException {
+ SortField sf = new SortedSetSortField(in.readString(), in.readInt() == 1, readSelectorType(in));
+ int missingValue = in.readInt();
+ if (missingValue == 1) {
+ sf.setMissingValue(SortField.STRING_FIRST);
+ }
+ else if (missingValue == 2) {
+ sf.setMissingValue(SortField.STRING_LAST);
+ }
+ return sf;
+ }
+
+ @Override
+ public void writeSortField(SortField sf, DataOutput out) throws IOException {
+ assert sf instanceof SortedSetSortField;
+ ((SortedSetSortField)sf).serialize(out);
+ }
+ }
+
+ private static SortedSetSelector.Type readSelectorType(DataInput in) throws IOException {
+ int type = in.readInt();
+ if (type >= SortedSetSelector.Type.values().length) {
+ throw new IllegalArgumentException("Cannot deserialize SortedSetSortField: unknown selector type " + type);
+ }
+ return SortedSetSelector.Type.values()[type];
+ }
+
+ private void serialize(DataOutput out) throws IOException {
+ out.writeString(getField());
+ out.writeInt(reverse ? 1 : 0);
+ out.writeInt(selector.ordinal());
+ if (missingValue == SortField.STRING_FIRST) {
+ out.writeInt(1);
+ }
+ else if (missingValue == SortField.STRING_LAST) {
+ out.writeInt(2);
+ }
+ else {
+ out.writeInt(0);
+ }
+ }
/** Returns the selector in use for this sort */
public SortedSetSelector.Type getSelector() {
@@ -126,4 +184,13 @@ protected SortedDocValues getSortedDocValues(LeafReaderContext context, String f
}
};
}
+
+ private SortedDocValues getValues(LeafReader reader) throws IOException {
+ return SortedSetSelector.wrap(DocValues.getSortedSet(reader, getField()), selector);
+ }
+
+ @Override
+ public IndexSorter getIndexSorter() {
+ return new IndexSorter.StringSorter(Provider.NAME, missingValue, reverse, this::getValues);
+ }
}
diff --git a/lucene/core/src/java/org/apache/lucene/store/BufferedChecksum.java b/lucene/core/src/java/org/apache/lucene/store/BufferedChecksum.java
index 4378c9a7705d..74b97d2d13cc 100644
--- a/lucene/core/src/java/org/apache/lucene/store/BufferedChecksum.java
+++ b/lucene/core/src/java/org/apache/lucene/store/BufferedChecksum.java
@@ -27,8 +27,8 @@ public class BufferedChecksum implements Checksum {
private final Checksum in;
private final byte buffer[];
private int upto;
- /** Default buffer size: 256 */
- public static final int DEFAULT_BUFFERSIZE = 256;
+ /** Default buffer size: 1024 */
+ public static final int DEFAULT_BUFFERSIZE = 1024;
/** Create a new BufferedChecksum with {@link #DEFAULT_BUFFERSIZE} */
public BufferedChecksum(Checksum in) {
diff --git a/lucene/core/src/java/org/apache/lucene/store/ByteBuffersIndexOutput.java b/lucene/core/src/java/org/apache/lucene/store/ByteBuffersIndexOutput.java
index 19dc4004853b..f6785bef79d8 100644
--- a/lucene/core/src/java/org/apache/lucene/store/ByteBuffersIndexOutput.java
+++ b/lucene/core/src/java/org/apache/lucene/store/ByteBuffersIndexOutput.java
@@ -81,24 +81,10 @@ public long getChecksum() throws IOException {
if (lastChecksumPosition != delegate.size()) {
lastChecksumPosition = delegate.size();
checksum.reset();
- byte [] buffer = null;
for (ByteBuffer bb : delegate.toBufferList()) {
- if (bb.hasArray()) {
- checksum.update(bb.array(), bb.arrayOffset() + bb.position(), bb.remaining());
- } else {
- if (buffer == null) buffer = new byte [1024 * 4];
-
- bb = bb.asReadOnlyBuffer();
- int remaining = bb.remaining();
- while (remaining > 0) {
- int len = Math.min(remaining, buffer.length);
- bb.get(buffer, 0, len);
- checksum.update(buffer, 0, len);
- remaining -= len;
- }
- }
+ checksum.update(bb);
}
- lastChecksum = checksum.getValue();
+ lastChecksum = checksum.getValue();
}
return lastChecksum;
}
diff --git a/lucene/core/src/java/org/apache/lucene/util/IntBlockPool.java b/lucene/core/src/java/org/apache/lucene/util/IntBlockPool.java
index 46500fcdf073..d09cb83e2cc3 100644
--- a/lucene/core/src/java/org/apache/lucene/util/IntBlockPool.java
+++ b/lucene/core/src/java/org/apache/lucene/util/IntBlockPool.java
@@ -175,7 +175,7 @@ private int newSlice(final int size) {
return upto;
}
- private static final boolean assertSliceBuffer(int[] buffer) {
+ private static boolean assertSliceBuffer(int[] buffer) {
int count = 0;
for (int i = 0; i < buffer.length; i++) {
count += buffer[i]; // for slices the buffer must only have 0 values
diff --git a/lucene/core/src/java/org/apache/lucene/util/Version.java b/lucene/core/src/java/org/apache/lucene/util/Version.java
index 5ed1a959da30..f5dbcc1e8e72 100644
--- a/lucene/core/src/java/org/apache/lucene/util/Version.java
+++ b/lucene/core/src/java/org/apache/lucene/util/Version.java
@@ -102,6 +102,13 @@ public final class Version {
@Deprecated
public static final Version LUCENE_8_5_1 = new Version(8, 5, 1);
+ /**
+ * Match settings and bugs in Lucene's 8.5.2 release.
+ * @deprecated Use latest
+ */
+ @Deprecated
+ public static final Version LUCENE_8_5_2 = new Version(8, 5, 2);
+
/**
* Match settings and bugs in Lucene's 8.6.0 release.
* @deprecated Use latest
diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/RegExp.java b/lucene/core/src/java/org/apache/lucene/util/automaton/RegExp.java
index e8b37e59a3ff..0874cde71015 100644
--- a/lucene/core/src/java/org/apache/lucene/util/automaton/RegExp.java
+++ b/lucene/core/src/java/org/apache/lucene/util/automaton/RegExp.java
@@ -290,6 +290,55 @@
* | (a single non-reserved character) |
* |
*
+ *
+ *
+ * |
+ * | |
+ * \d |
+ * (a digit [0-9]) |
+ * |
+ *
+ *
+ *
+ * |
+ * | |
+ * \D |
+ * (a non-digit [^0-9]) |
+ * |
+ *
+ *
+ *
+ * |
+ * | |
+ * \s |
+ * (whitespace [ \t\n\r]) |
+ * |
+ *
+ *
+ *
+ * |
+ * | |
+ * \S |
+ * (non whitespace [^\s]) |
+ * |
+ *
+ *
+ *
+ * |
+ * | |
+ * \w |
+ * (a word character [a-zA-Z_0-9]) |
+ * |
+ *
+ *
+ *
+ * |
+ * | |
+ * \W |
+ * (a non word character [^\w]) |
+ * |
+ *
+ *
*
* |
* | |
@@ -316,8 +365,44 @@
*/
public class RegExp {
- enum Kind {
- REGEXP_UNION, REGEXP_CONCATENATION, REGEXP_INTERSECTION, REGEXP_OPTIONAL, REGEXP_REPEAT, REGEXP_REPEAT_MIN, REGEXP_REPEAT_MINMAX, REGEXP_COMPLEMENT, REGEXP_CHAR, REGEXP_CHAR_RANGE, REGEXP_ANYCHAR, REGEXP_EMPTY, REGEXP_STRING, REGEXP_ANYSTRING, REGEXP_AUTOMATON, REGEXP_INTERVAL
+ /**
+ * The type of expression represented by a RegExp node.
+ */
+ public enum Kind {
+ /** The union of two expressions */
+ REGEXP_UNION,
+ /** A sequence of two expressions */
+ REGEXP_CONCATENATION,
+ /** The intersection of two expressions */
+ REGEXP_INTERSECTION,
+ /** An optional expression */
+ REGEXP_OPTIONAL,
+ /** An expression that repeats */
+ REGEXP_REPEAT,
+ /** An expression that repeats a minimum number of times*/
+ REGEXP_REPEAT_MIN,
+ /** An expression that repeats a minimum and maximum number of times*/
+ REGEXP_REPEAT_MINMAX,
+ /** The complement of an expression */
+ REGEXP_COMPLEMENT,
+ /** A Character */
+ REGEXP_CHAR,
+ /** A Character range*/
+ REGEXP_CHAR_RANGE,
+ /** Any Character allowed*/
+ REGEXP_ANYCHAR,
+ /** An empty expression*/
+ REGEXP_EMPTY,
+ /** A string expression*/
+ REGEXP_STRING,
+ /** Any string allowed */
+ REGEXP_ANYSTRING,
+ /** An Automaton expression*/
+ REGEXP_AUTOMATON,
+ /** An Interval expression */
+ REGEXP_INTERVAL,
+ /** An expression for a pre-defined class e.g. \w */
+ REGEXP_PRE_CLASS
}
/**
@@ -361,21 +446,37 @@ enum Kind {
*/
public static final int NONE = 0x0000;
+ //Immutable parsed state
+ /**
+ * The type of expression
+ */
+ public final Kind kind;
+ /**
+ * Child expressions held by a container type expression
+ */
+ public final RegExp exp1, exp2;
+ /**
+ * String expression
+ */
+ public final String s;
+ /**
+ * Character expression
+ */
+ public final int c;
+ /**
+ * Limits for repeatable type expressions
+ */
+ public final int min, max, digits;
+ /**
+ * Extents for range type expressions
+ */
+ public final int from, to;
+
+ // Parser variables
private final String originalString;
- Kind kind;
- RegExp exp1, exp2;
- String s;
- int c;
- int min, max, digits;
- int from, to;
-
int flags;
int pos;
-
- RegExp() {
- this.originalString = null;
- }
-
+
/**
* Constructs new RegExp from a string. Same as
* RegExp(s, ALL).
@@ -418,6 +519,37 @@ public RegExp(String s, int syntax_flags) throws IllegalArgumentException {
from = e.from;
to = e.to;
}
+
+ RegExp(Kind kind, RegExp exp1, RegExp exp2, String s, int c, int min, int max, int digits, int from, int to){
+ this.originalString = null;
+ this.kind = kind;
+ this.flags = 0;
+ this.exp1 = exp1;
+ this.exp2 = exp2;
+ this.s = s;
+ this.c = c;
+ this.min = min;
+ this.max = max;
+ this.digits = digits;
+ this.from = from;
+ this.to = to;
+ }
+
+ // Simplified construction of container nodes
+ static RegExp newContainerNode(Kind kind, RegExp exp1, RegExp exp2) {
+ return new RegExp(kind, exp1, exp2, null, 0, 0, 0, 0, 0, 0);
+ }
+
+ // Simplified construction of repeating nodes
+ static RegExp newRepeatingNode(Kind kind, RegExp exp, int min, int max) {
+ return new RegExp(kind, exp, null, null, 0, min, max, 0, 0, 0);
+ }
+
+
+ // Simplified construction of leaf nodes
+ static RegExp newLeafNode(Kind kind, String s, int c, int min, int max, int digits, int from, int to) {
+ return new RegExp(kind, null, null, s, c, min, max, digits, from, to);
+ }
/**
* Constructs new Automaton from this RegExp. Same
@@ -506,6 +638,10 @@ private Automaton toAutomatonInternal(Map automata,
List list;
Automaton a = null;
switch (kind) {
+ case REGEXP_PRE_CLASS:
+ RegExp expanded = expandPredefined();
+ a = expanded.toAutomatonInternal(automata, automaton_provider, maxDeterminizedStates);
+ break;
case REGEXP_UNION:
list = new ArrayList<>();
findLeaves(exp1, Kind.REGEXP_UNION, list, automata, automaton_provider,
@@ -716,6 +852,9 @@ void toStringBuilder(StringBuilder b) {
b.append('0');
b.append(s2).append(">");
break;
+ case REGEXP_PRE_CLASS:
+ b.append("\\").appendCodePoint(from);
+ break;
}
}
@@ -774,6 +913,13 @@ void toStringTree(StringBuilder b, String indent) {
b.appendCodePoint(c);
b.append('\n');
break;
+ case REGEXP_PRE_CLASS:
+ b.append(indent);
+ b.append(kind);
+ b.append(" class=\\");
+ b.appendCodePoint(from);
+ b.append('\n');
+ break;
case REGEXP_CHAR_RANGE:
b.append(indent);
b.append(kind);
@@ -855,34 +1001,29 @@ void getIdentifiers(Set set) {
}
static RegExp makeUnion(RegExp exp1, RegExp exp2) {
- RegExp r = new RegExp();
- r.kind = Kind.REGEXP_UNION;
- r.exp1 = exp1;
- r.exp2 = exp2;
- return r;
+ return newContainerNode(Kind.REGEXP_UNION, exp1, exp2);
}
static RegExp makeConcatenation(RegExp exp1, RegExp exp2) {
if ((exp1.kind == Kind.REGEXP_CHAR || exp1.kind == Kind.REGEXP_STRING)
&& (exp2.kind == Kind.REGEXP_CHAR || exp2.kind == Kind.REGEXP_STRING)) return makeString(
exp1, exp2);
- RegExp r = new RegExp();
- r.kind = Kind.REGEXP_CONCATENATION;
+ RegExp rexp1, rexp2;
if (exp1.kind == Kind.REGEXP_CONCATENATION
&& (exp1.exp2.kind == Kind.REGEXP_CHAR || exp1.exp2.kind == Kind.REGEXP_STRING)
&& (exp2.kind == Kind.REGEXP_CHAR || exp2.kind == Kind.REGEXP_STRING)) {
- r.exp1 = exp1.exp1;
- r.exp2 = makeString(exp1.exp2, exp2);
+ rexp1 = exp1.exp1;
+ rexp2 = makeString(exp1.exp2, exp2);
} else if ((exp1.kind == Kind.REGEXP_CHAR || exp1.kind == Kind.REGEXP_STRING)
&& exp2.kind == Kind.REGEXP_CONCATENATION
&& (exp2.exp1.kind == Kind.REGEXP_CHAR || exp2.exp1.kind == Kind.REGEXP_STRING)) {
- r.exp1 = makeString(exp1, exp2.exp1);
- r.exp2 = exp2.exp2;
+ rexp1 = makeString(exp1, exp2.exp1);
+ rexp2 = exp2.exp2;
} else {
- r.exp1 = exp1;
- r.exp2 = exp2;
+ rexp1 = exp1;
+ rexp2 = exp2;
}
- return r;
+ return newContainerNode(Kind.REGEXP_CONCATENATION, rexp1, rexp2);
}
static private RegExp makeString(RegExp exp1, RegExp exp2) {
@@ -895,107 +1036,61 @@ static private RegExp makeString(RegExp exp1, RegExp exp2) {
}
static RegExp makeIntersection(RegExp exp1, RegExp exp2) {
- RegExp r = new RegExp();
- r.kind = Kind.REGEXP_INTERSECTION;
- r.exp1 = exp1;
- r.exp2 = exp2;
- return r;
+ return newContainerNode(Kind.REGEXP_INTERSECTION, exp1, exp2);
}
static RegExp makeOptional(RegExp exp) {
- RegExp r = new RegExp();
- r.kind = Kind.REGEXP_OPTIONAL;
- r.exp1 = exp;
- return r;
+ return newContainerNode(Kind.REGEXP_OPTIONAL, exp, null);
}
static RegExp makeRepeat(RegExp exp) {
- RegExp r = new RegExp();
- r.kind = Kind.REGEXP_REPEAT;
- r.exp1 = exp;
- return r;
+ return newContainerNode(Kind.REGEXP_REPEAT, exp, null);
}
static RegExp makeRepeat(RegExp exp, int min) {
- RegExp r = new RegExp();
- r.kind = Kind.REGEXP_REPEAT_MIN;
- r.exp1 = exp;
- r.min = min;
- return r;
+ return newRepeatingNode(Kind.REGEXP_REPEAT_MIN, exp, min, 0);
}
static RegExp makeRepeat(RegExp exp, int min, int max) {
- RegExp r = new RegExp();
- r.kind = Kind.REGEXP_REPEAT_MINMAX;
- r.exp1 = exp;
- r.min = min;
- r.max = max;
- return r;
+ return newRepeatingNode(Kind.REGEXP_REPEAT_MINMAX, exp, min, max);
}
static RegExp makeComplement(RegExp exp) {
- RegExp r = new RegExp();
- r.kind = Kind.REGEXP_COMPLEMENT;
- r.exp1 = exp;
- return r;
+ return newContainerNode(Kind.REGEXP_COMPLEMENT, exp, null);
}
static RegExp makeChar(int c) {
- RegExp r = new RegExp();
- r.kind = Kind.REGEXP_CHAR;
- r.c = c;
- return r;
+ return newLeafNode(Kind.REGEXP_CHAR, null, c, 0, 0, 0, 0, 0);
}
static RegExp makeCharRange(int from, int to) {
if (from > to)
throw new IllegalArgumentException("invalid range: from (" + from + ") cannot be > to (" + to + ")");
- RegExp r = new RegExp();
- r.kind = Kind.REGEXP_CHAR_RANGE;
- r.from = from;
- r.to = to;
- return r;
+ return newLeafNode(Kind.REGEXP_CHAR_RANGE, null, 0, 0, 0, 0, from, to);
}
static RegExp makeAnyChar() {
- RegExp r = new RegExp();
- r.kind = Kind.REGEXP_ANYCHAR;
- return r;
+ return newContainerNode(Kind.REGEXP_ANYCHAR, null, null);
}
static RegExp makeEmpty() {
- RegExp r = new RegExp();
- r.kind = Kind.REGEXP_EMPTY;
- return r;
+ return newContainerNode(Kind.REGEXP_EMPTY, null, null);
}
static RegExp makeString(String s) {
- RegExp r = new RegExp();
- r.kind = Kind.REGEXP_STRING;
- r.s = s;
- return r;
+ return newLeafNode(Kind.REGEXP_STRING, s, 0, 0, 0, 0, 0, 0);
}
static RegExp makeAnyString() {
- RegExp r = new RegExp();
- r.kind = Kind.REGEXP_ANYSTRING;
- return r;
+ return newContainerNode(Kind.REGEXP_ANYSTRING, null, null);
}
static RegExp makeAutomaton(String s) {
- RegExp r = new RegExp();
- r.kind = Kind.REGEXP_AUTOMATON;
- r.s = s;
- return r;
+ return newLeafNode(Kind.REGEXP_AUTOMATON, s, 0, 0, 0, 0, 0, 0);
}
static RegExp makeInterval(int min, int max, int digits) {
- RegExp r = new RegExp();
- r.kind = Kind.REGEXP_INTERVAL;
- r.min = min;
- r.max = max;
- r.digits = digits;
- return r;
+ return newLeafNode(Kind.REGEXP_INTERVAL, null, 0, min, max, digits, 0, 0);
}
private boolean peek(String s) {
@@ -1101,10 +1196,60 @@ final RegExp parseCharClasses() throws IllegalArgumentException {
}
final RegExp parseCharClass() throws IllegalArgumentException {
+ RegExp predefinedExp = matchPredefinedCharacterClass();
+ if (predefinedExp != null) {
+ return predefinedExp;
+ }
+
int c = parseCharExp();
if (match('-')) return makeCharRange(c, parseCharExp());
else return makeChar(c);
}
+
+ RegExp expandPredefined() {
+ //See https://docs.oracle.com/javase/tutorial/essential/regex/pre_char_classes.html
+ switch (from) {
+ case 'd':
+ return new RegExp("[0-9]"); // digit
+ case 'D':
+ return new RegExp("[^0-9]"); // non-digit
+ case 's':
+ return new RegExp("[ \t\n\r]"); // whitespace
+ case 'S':
+ return new RegExp("[^\\s]"); // non-whitespace
+ case 'w':
+ return new RegExp("[a-zA-Z_0-9]"); // word
+ case 'W':
+ return new RegExp("[^\\w]"); // non-word
+ default:
+ throw new IllegalArgumentException(
+ "invalid character class " + from);
+ }
+ }
+
+
+ final RegExp matchPredefinedCharacterClass() {
+ //See https://docs.oracle.com/javase/tutorial/essential/regex/pre_char_classes.html
+ if (match('\\')) {
+ if (peek("dDwWsS")) {
+ return newLeafNode(Kind.REGEXP_PRE_CLASS, null, 0, 0, 0, 0, next(), 0);
+ }
+
+ if (peek("\\")) {
+ return makeChar(next());
+ }
+
+ // From https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html#bs
+ // "It is an error to use a backslash prior to any alphabetic character that does not denote an escaped
+ // construct;"
+ if (peek("abcefghijklmnopqrtuvxyz") || peek("ABCEFGHIJKLMNOPQRTUVXYZ")) {
+ throw new IllegalArgumentException("invalid character class \\" + next());
+ }
+ }
+
+ return null;
+ }
+
final RegExp parseSimpleExp() throws IllegalArgumentException {
if (match('.')) return makeAnyChar();
@@ -1158,7 +1303,13 @@ else if (match('"')) {
"interval syntax error at position " + (pos - 1));
}
}
- } else return makeChar(parseCharExp());
+ } else {
+ RegExp predefined = matchPredefinedCharacterClass();
+ if (predefined != null) {
+ return predefined;
+ }
+ return makeChar(parseCharExp());
+ }
}
final int parseCharExp() throws IllegalArgumentException {
diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java
index 490432693efc..1e3702db84c9 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java
@@ -24,11 +24,7 @@
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.PointValues;
import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.store.ByteArrayDataInput;
-import org.apache.lucene.store.ByteBufferIndexInput;
-import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.MathUtil;
@@ -36,124 +32,7 @@
*
* @lucene.experimental */
-public final class BKDReader extends PointValues implements Accountable {
-
- private static abstract class BKDInput extends DataInput implements Cloneable {
- abstract long getMinLeafBlockFP();
- abstract long ramBytesUsed();
-
- abstract int getPosition();
- abstract void setPosition(int pos) throws IOException;
-
- @Override
- public BKDInput clone() {
- return (BKDInput)super.clone();
- }
- }
-
- private static class BKDOffHeapInput extends BKDInput implements Cloneable {
-
- private final IndexInput packedIndex;
- private final long minLeafBlockFP;
-
- BKDOffHeapInput(IndexInput packedIndex) throws IOException {
- this.packedIndex = packedIndex;
- this.minLeafBlockFP = packedIndex.clone().readVLong();
- }
-
- private BKDOffHeapInput(IndexInput packedIndex, long minLeadBlockFP) {
- this.packedIndex = packedIndex;
- this.minLeafBlockFP = minLeadBlockFP;
- }
-
- @Override
- public BKDOffHeapInput clone() {
- return new BKDOffHeapInput(packedIndex.clone(), minLeafBlockFP);
- }
-
- @Override
- long getMinLeafBlockFP() {
- return minLeafBlockFP;
- }
-
- @Override
- long ramBytesUsed() {
- return 0;
- }
-
- @Override
- int getPosition() {
- return (int)packedIndex.getFilePointer();
- }
-
- @Override
- void setPosition(int pos) throws IOException {
- packedIndex.seek(pos);
- }
-
- @Override
- public byte readByte() throws IOException {
- return packedIndex.readByte();
- }
-
- @Override
- public void readBytes(byte[] b, int offset, int len) throws IOException {
- packedIndex.readBytes(b, offset, len);
- }
- }
-
- private static class BKDOnHeapInput extends BKDInput implements Cloneable {
-
- private final ByteArrayDataInput packedIndex;
- private final long minLeafBlockFP;
-
- BKDOnHeapInput(IndexInput packedIndex, int numBytes) throws IOException {
- byte[] packedBytes = new byte[numBytes];
- packedIndex.readBytes(packedBytes, 0, numBytes);
- this.packedIndex = new ByteArrayDataInput(packedBytes);
- this.minLeafBlockFP = this.packedIndex.clone().readVLong();
- }
-
- private BKDOnHeapInput(ByteArrayDataInput packedIndex, long minLeadBlockFP) {
- this.packedIndex = packedIndex;
- this.minLeafBlockFP = minLeadBlockFP;
- }
-
- @Override
- public BKDOnHeapInput clone() {
- return new BKDOnHeapInput((ByteArrayDataInput)packedIndex.clone(), minLeafBlockFP);
- }
-
- @Override
- long getMinLeafBlockFP() {
- return minLeafBlockFP;
- }
-
- @Override
- long ramBytesUsed() {
- return packedIndex.length();
- }
-
- @Override
- int getPosition() {
- return packedIndex.getPosition();
- }
-
- @Override
- void setPosition(int pos) {
- packedIndex.setPosition(pos);
- }
-
- @Override
- public byte readByte() throws IOException {
- return packedIndex.readByte();
- }
-
- @Override
- public void readBytes(byte[] b, int offset, int len) throws IOException {
- packedIndex.readBytes(b, offset, len);
- }
- }
+public final class BKDReader extends PointValues {
// Packed array of byte[] holding all split values in the full binary tree:
final int leafNodeOffset;
@@ -170,67 +49,64 @@ public void readBytes(byte[] b, int offset, int len) throws IOException {
final int version;
protected final int packedBytesLength;
protected final int packedIndexBytesLength;
+ final long minLeafBlockFP;
- final BKDInput packedIndex;
-
- /** Caller must pre-seek the provided {@link IndexInput} to the index location that {@link BKDWriter#finish} returned */
- public BKDReader(IndexInput in) throws IOException {
- this(in, in instanceof ByteBufferIndexInput);
- }
+ final IndexInput packedIndex;
- /**
- * Caller must pre-seek the provided {@link IndexInput} to the index location that {@link BKDWriter#finish} returned
- * and specify {@code true} to store BKD off-heap ({@code false} otherwise)
- */
- public BKDReader(IndexInput in, boolean offHeap) throws IOException {
- version = CodecUtil.checkHeader(in, BKDWriter.CODEC_NAME, BKDWriter.VERSION_START, BKDWriter.VERSION_CURRENT);
- numDataDims = in.readVInt();
+ /** Caller must pre-seek the provided {@link IndexInput} to the index location that {@link BKDWriter#finish} returned.
+ * BKD tree is always stored off-heap. */
+ public BKDReader(IndexInput metaIn, IndexInput indexIn, IndexInput dataIn) throws IOException {
+ version = CodecUtil.checkHeader(metaIn, BKDWriter.CODEC_NAME, BKDWriter.VERSION_START, BKDWriter.VERSION_CURRENT);
+ numDataDims = metaIn.readVInt();
if (version >= BKDWriter.VERSION_SELECTIVE_INDEXING) {
- numIndexDims = in.readVInt();
+ numIndexDims = metaIn.readVInt();
} else {
numIndexDims = numDataDims;
}
- maxPointsInLeafNode = in.readVInt();
- bytesPerDim = in.readVInt();
+ maxPointsInLeafNode = metaIn.readVInt();
+ bytesPerDim = metaIn.readVInt();
packedBytesLength = numDataDims * bytesPerDim;
packedIndexBytesLength = numIndexDims * bytesPerDim;
// Read index:
- numLeaves = in.readVInt();
+ numLeaves = metaIn.readVInt();
assert numLeaves > 0;
leafNodeOffset = numLeaves;
minPackedValue = new byte[packedIndexBytesLength];
maxPackedValue = new byte[packedIndexBytesLength];
- in.readBytes(minPackedValue, 0, packedIndexBytesLength);
- in.readBytes(maxPackedValue, 0, packedIndexBytesLength);
+ metaIn.readBytes(minPackedValue, 0, packedIndexBytesLength);
+ metaIn.readBytes(maxPackedValue, 0, packedIndexBytesLength);
for(int dim=0;dim 0) {
- throw new CorruptIndexException("minPackedValue " + new BytesRef(minPackedValue) + " is > maxPackedValue " + new BytesRef(maxPackedValue) + " for dim=" + dim, in);
+ throw new CorruptIndexException("minPackedValue " + new BytesRef(minPackedValue) + " is > maxPackedValue " + new BytesRef(maxPackedValue) + " for dim=" + dim, metaIn);
}
}
- pointCount = in.readVLong();
- docCount = in.readVInt();
-
- int numBytes = in.readVInt();
- IndexInput slice = in.slice("packedIndex", in.getFilePointer(), numBytes);
- if (offHeap) {
- packedIndex = new BKDOffHeapInput(slice);
+ pointCount = metaIn.readVLong();
+ docCount = metaIn.readVInt();
+
+ int numIndexBytes = metaIn.readVInt();
+ long indexStartPointer;
+ if (version >= BKDWriter.VERSION_META_FILE) {
+ minLeafBlockFP = metaIn.readLong();
+ indexStartPointer = metaIn.readLong();
} else {
- packedIndex = new BKDOnHeapInput(slice, numBytes);
+ indexStartPointer = indexIn.getFilePointer();
+ minLeafBlockFP = indexIn.readVLong();
+ indexIn.seek(indexStartPointer);
}
-
- this.in = in;
+ this.packedIndex = indexIn.slice("packedIndex", indexStartPointer, numIndexBytes);
+ this.in = dataIn;
}
long getMinLeafBlockFP() {
- return packedIndex.getMinLeafBlockFP();
+ return minLeafBlockFP;
}
- /** Used to walk the in-heap index. The format takes advantage of the limited
+ /** Used to walk the off-heap index. The format takes advantage of the limited
* access pattern to the BKD tree at search time, i.e. starting at the root
* node and recursing downwards one child at a time.
* @lucene.internal */
@@ -240,13 +116,11 @@ public class IndexTree implements Cloneable {
private int level;
private int splitDim;
private final byte[][] splitPackedValueStack;
- // used to read the packed byte[]
- private final BKDInput in;
+ // used to read the packed tree off-heap
+ private final IndexInput in;
// holds the minimum (left most) leaf block file pointer for each level we've recursed to:
private final long[] leafBlockFPStack;
- // holds the address, in the packed byte[] index, of the left-node of each level:
- private final int[] leftNodePositions;
- // holds the address, in the packed byte[] index, of the right-node of each level:
+ // holds the address, in the off-heap index, of the right-node of each level:
private final int[] rightNodePositions;
// holds the splitDim for each level:
private final int[] splitDims;
@@ -260,52 +134,41 @@ public class IndexTree implements Cloneable {
private final BytesRef scratch;
IndexTree() {
+ this(packedIndex.clone(), 1, 1);
+ // read root node
+ readNodeData(false);
+ }
+
+ private IndexTree(IndexInput in, int nodeID, int level) {
int treeDepth = getTreeDepth();
splitPackedValueStack = new byte[treeDepth+1][];
- nodeID = 1;
- level = 1;
+ this.nodeID = nodeID;
+ this.level = level;
splitPackedValueStack[level] = new byte[packedIndexBytesLength];
leafBlockFPStack = new long[treeDepth+1];
- leftNodePositions = new int[treeDepth+1];
rightNodePositions = new int[treeDepth+1];
splitValuesStack = new byte[treeDepth+1][];
splitDims = new int[treeDepth+1];
negativeDeltas = new boolean[numIndexDims*(treeDepth+1)];
-
- in = packedIndex.clone();
+ this.in = in;
splitValuesStack[0] = new byte[packedIndexBytesLength];
- readNodeData(false);
scratch = new BytesRef();
scratch.length = bytesPerDim;
}
public void pushLeft() {
- int nodePosition = leftNodePositions[level];
nodeID *= 2;
level++;
- if (splitPackedValueStack[level] == null) {
- splitPackedValueStack[level] = new byte[packedIndexBytesLength];
- }
- System.arraycopy(negativeDeltas, (level-1)*numIndexDims, negativeDeltas, level*numIndexDims, numIndexDims);
- assert splitDim != -1;
- negativeDeltas[level*numIndexDims+splitDim] = true;
- try {
- in.setPosition(nodePosition);
- } catch (IOException e) {
- throw new UncheckedIOException(e);
- }
readNodeData(true);
}
/** Clone, but you are not allowed to pop up past the point where the clone happened. */
@Override
public IndexTree clone() {
- IndexTree index = new IndexTree();
- index.nodeID = nodeID;
- index.level = level;
+ IndexTree index = new IndexTree(in.clone(), nodeID, level);
+ // copy node data
index.splitDim = splitDim;
index.leafBlockFPStack[level] = leafBlockFPStack[level];
- index.leftNodePositions[level] = leftNodePositions[level];
index.rightNodePositions[level] = rightNodePositions[level];
index.splitValuesStack[index.level] = splitValuesStack[index.level].clone();
System.arraycopy(negativeDeltas, level*numIndexDims, index.negativeDeltas, level*numIndexDims, numIndexDims);
@@ -314,17 +177,12 @@ public IndexTree clone() {
}
public void pushRight() {
- int nodePosition = rightNodePositions[level];
+ final int nodePosition = rightNodePositions[level];
+ assert nodePosition >= in.getFilePointer() : "nodePosition = " + nodePosition + " < currentPosition=" + in.getFilePointer();
nodeID = nodeID * 2 + 1;
level++;
- if (splitPackedValueStack[level] == null) {
- splitPackedValueStack[level] = new byte[packedIndexBytesLength];
- }
- System.arraycopy(negativeDeltas, (level-1)*numIndexDims, negativeDeltas, level*numIndexDims, numIndexDims);
- assert splitDim != -1;
- negativeDeltas[level*numIndexDims+splitDim] = false;
try {
- in.setPosition(nodePosition);
+ in.seek(nodePosition);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
@@ -412,6 +270,13 @@ private int getNumLeavesSlow(int node) {
}
private void readNodeData(boolean isLeft) {
+ if (splitPackedValueStack[level] == null) {
+ splitPackedValueStack[level] = new byte[packedIndexBytesLength];
+ }
+ System.arraycopy(negativeDeltas, (level-1)*numIndexDims, negativeDeltas, level*numIndexDims, numIndexDims);
+ assert splitDim != -1;
+ negativeDeltas[level*numIndexDims+splitDim] = isLeft;
+
try {
leafBlockFPStack[level] = leafBlockFPStack[level - 1];
@@ -454,9 +319,7 @@ private void readNodeData(boolean isLeft) {
} else {
leftNumBytes = 0;
}
-
- leftNodePositions[level] = in.getPosition();
- rightNodePositions[level] = leftNodePositions[level] + leftNumBytes;
+ rightNodePositions[level] = Math.toIntExact(in.getFilePointer()) + leftNumBytes;
}
} catch (IOException e) {
throw new UncheckedIOException(e);
@@ -880,11 +743,6 @@ private long estimatePointCount(IntersectState state, byte[] cellMinPacked, byte
}
}
- @Override
- public long ramBytesUsed() {
- return packedIndex.ramBytesUsed();
- }
-
@Override
public byte[] getMinPackedValue() {
return minPackedValue.clone();
diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
index 727b824f7214..de71941ec52e 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
@@ -18,6 +18,7 @@
import java.io.Closeable;
import java.io.IOException;
+import java.io.UncheckedIOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -57,9 +58,10 @@
* Recursively builds a block KD-tree to assign all incoming points in N-dim space to smaller
* and smaller N-dim rectangles (cells) until the number of points in a given
* rectangle is <= maxPointsInLeafNode. The tree is
- * fully balanced, which means the leaf nodes will have between 50% and 100% of
- * the requested maxPointsInLeafNode. Values that fall exactly
- * on a cell boundary may be in either cell.
+ * partially balanced, which means the leaf nodes will have
+ * the requested maxPointsInLeafNode except one that might have less.
+ * Leaf nodes may straddle the two bottom levels of the binary tree.
+ * Values that fall exactly on a cell boundary may be in either cell.
*
* The number of dimensions can be 1 to 8, but every byte[] value is fixed length.
*
@@ -68,7 +70,7 @@
* {@code maxMBSortInHeap} heap space for writing.
*
*
- * NOTE: This can write at most Integer.MAX_VALUE * maxPointsInLeafNode / (1+bytesPerDim)
+ * NOTE: This can write at most Integer.MAX_VALUE * maxPointsInLeafNode / bytesPerDim
* total points.
*
* @lucene.experimental */
@@ -81,13 +83,14 @@ public class BKDWriter implements Closeable {
public static final int VERSION_LEAF_STORES_BOUNDS = 5;
public static final int VERSION_SELECTIVE_INDEXING = 6;
public static final int VERSION_LOW_CARDINALITY_LEAVES = 7;
- public static final int VERSION_CURRENT = VERSION_LOW_CARDINALITY_LEAVES;
+ public static final int VERSION_META_FILE = 9;
+ public static final int VERSION_CURRENT = VERSION_META_FILE;
/** How many bytes each docs takes in the fixed-width offline format */
private final int bytesPerDoc;
/** Default maximum number of point in each leaf block */
- public static final int DEFAULT_MAX_POINTS_IN_LEAF_NODE = 1024;
+ public static final int DEFAULT_MAX_POINTS_IN_LEAF_NODE = 512;
/** Default maximum heap to use, before spilling to (slower) disk */
public static final float DEFAULT_MAX_MB_SORT_IN_HEAP = 16.0f;
@@ -252,11 +255,6 @@ public void add(byte[] packedValue, int docID) throws IOException {
docsSeen.set(docID);
}
- /** How many points have been added so far */
- public long getPointCount() {
- return pointCount;
- }
-
private static class MergeReader {
final BKDReader bkd;
final BKDReader.IntersectState state;
@@ -371,16 +369,32 @@ public boolean lessThan(MergeReader a, MergeReader b) {
}
}
+ /** flat representation of a kd-tree */
+ private interface BKDTreeLeafNodes {
+ /** number of leaf nodes */
+ int numLeaves();
+ /** pointer to the leaf node previously written. Leaves are order from
+ * left to right, so leaf at {@code index} 0 is the leftmost leaf and
+ * the the leaf at {@code numleaves()} -1 is the rightmost leaf */
+ long getLeafLP(int index);
+ /** split value between two leaves. The split value at position n corresponds to the
+ * leaves at (n -1) and n. */
+ BytesRef getSplitValue(int index);
+ /** split dimension between two leaves. The split dimension at position n corresponds to the
+ * leaves at (n -1) and n.*/
+ int getSplitDimension(int index);
+ }
+
/** Write a field from a {@link MutablePointValues}. This way of writing
* points is faster than regular writes with {@link BKDWriter#add} since
* there is opportunity for reordering points before writing them to
* disk. This method does not use transient disk in order to reorder points.
*/
- public long writeField(IndexOutput out, String fieldName, MutablePointValues reader) throws IOException {
+ public Runnable writeField(IndexOutput metaOut, IndexOutput indexOut, IndexOutput dataOut, String fieldName, MutablePointValues reader) throws IOException {
if (numDataDims == 1) {
- return writeField1Dim(out, fieldName, reader);
+ return writeField1Dim(metaOut, indexOut, dataOut, fieldName, reader);
} else {
- return writeFieldNDims(out, fieldName, reader);
+ return writeFieldNDims(metaOut, indexOut, dataOut, fieldName, reader);
}
}
@@ -407,7 +421,7 @@ private void computePackedValueBounds(MutablePointValues values, int from, int t
/* In the 2+D case, we recursively pick the split dimension, compute the
* median value and partition other values around it. */
- private long writeFieldNDims(IndexOutput out, String fieldName, MutablePointValues values) throws IOException {
+ private Runnable writeFieldNDims(IndexOutput metaOut, IndexOutput indexOut, IndexOutput dataOut, String fieldName, MutablePointValues values) throws IOException {
if (pointCount != 0) {
throw new IllegalStateException("cannot mix add and writeField");
}
@@ -420,19 +434,15 @@ private long writeFieldNDims(IndexOutput out, String fieldName, MutablePointValu
// Mark that we already finished:
finished = true;
- long countPerLeaf = pointCount = values.size();
- long innerNodeCount = 1;
-
- while (countPerLeaf > maxPointsInLeafNode) {
- countPerLeaf = (countPerLeaf+1)/2;
- innerNodeCount *= 2;
- }
+ pointCount = values.size();
- int numLeaves = Math.toIntExact(innerNodeCount);
+ final int numLeaves = Math.toIntExact((pointCount + maxPointsInLeafNode - 1) / maxPointsInLeafNode);
+ final int numSplits = numLeaves - 1;
checkMaxLeafNodeCount(numLeaves);
- final byte[] splitPackedValues = new byte[numLeaves * (bytesPerDim + 1)];
+ final byte[] splitPackedValues = new byte[numSplits * bytesPerDim];
+ final byte[] splitDimensionValues = new byte[numSplits];
final long[] leafBlockFPs = new long[numLeaves];
// compute the min/max for this slice
@@ -441,24 +451,55 @@ private long writeFieldNDims(IndexOutput out, String fieldName, MutablePointValu
docsSeen.set(values.getDocID(i));
}
+ final long dataStartFP = dataOut.getFilePointer();
final int[] parentSplits = new int[numIndexDims];
- build(1, numLeaves, values, 0, Math.toIntExact(pointCount), out,
+ build(0, numLeaves, values, 0, Math.toIntExact(pointCount), dataOut,
minPackedValue.clone(), maxPackedValue.clone(), parentSplits,
- splitPackedValues, leafBlockFPs,
+ splitPackedValues, splitDimensionValues, leafBlockFPs,
new int[maxPointsInLeafNode]);
assert Arrays.equals(parentSplits, new int[numIndexDims]);
- long indexFP = out.getFilePointer();
- writeIndex(out, Math.toIntExact(countPerLeaf), leafBlockFPs, splitPackedValues);
- return indexFP;
+ scratchBytesRef1.length = bytesPerDim;
+ scratchBytesRef1.bytes = splitPackedValues;
+
+ BKDTreeLeafNodes leafNodes = new BKDTreeLeafNodes() {
+ @Override
+ public long getLeafLP(int index) {
+ return leafBlockFPs[index];
+ }
+
+ @Override
+ public BytesRef getSplitValue(int index) {
+ scratchBytesRef1.offset = index * bytesPerDim;
+ return scratchBytesRef1;
+ }
+
+ @Override
+ public int getSplitDimension(int index) {
+ return splitDimensionValues[index] & 0xff;
+ }
+
+ @Override
+ public int numLeaves() {
+ return leafBlockFPs.length;
+ }
+ };
+
+ return () -> {
+ try {
+ writeIndex(metaOut, indexOut, maxPointsInLeafNode, leafNodes, dataStartFP);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ };
}
/* In the 1D case, we can simply sort points in ascending order and use the
* same writing logic as we use at merge time. */
- private long writeField1Dim(IndexOutput out, String fieldName, MutablePointValues reader) throws IOException {
+ private Runnable writeField1Dim(IndexOutput metaOut, IndexOutput indexOut, IndexOutput dataOut, String fieldName, MutablePointValues reader) throws IOException {
MutablePointsReaderUtils.sort(maxDoc, packedIndexBytesLength, reader, 0, Math.toIntExact(reader.size()));
- final OneDimensionBKDWriter oneDimWriter = new OneDimensionBKDWriter(out);
+ final OneDimensionBKDWriter oneDimWriter = new OneDimensionBKDWriter(metaOut, indexOut, dataOut);
reader.intersect(new IntersectVisitor() {
@@ -484,7 +525,7 @@ public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
/** More efficient bulk-add for incoming {@link BKDReader}s. This does a merge sort of the already
* sorted values and currently only works when numDims==1. This returns -1 if all documents containing
* dimensional values were deleted. */
- public long merge(IndexOutput out, List docMaps, List readers) throws IOException {
+ public Runnable merge(IndexOutput metaOut, IndexOutput indexOut, IndexOutput dataOut, List docMaps, List readers) throws IOException {
assert docMaps == null || readers.size() == docMaps.size();
BKDMergeQueue queue = new BKDMergeQueue(bytesPerDim, readers.size());
@@ -503,7 +544,7 @@ public long merge(IndexOutput out, List docMaps, List docMaps, List leafBlockFPs = new ArrayList<>();
final List leafBlockStartValues = new ArrayList<>();
final byte[] leafValues = new byte[maxPointsInLeafNode * packedBytesLength];
@@ -536,7 +578,7 @@ private class OneDimensionBKDWriter {
private int leafCount;
private int leafCardinality;
- OneDimensionBKDWriter(IndexOutput out) {
+ OneDimensionBKDWriter(IndexOutput metaOut, IndexOutput indexOut, IndexOutput dataOut) {
if (numIndexDims != 1) {
throw new UnsupportedOperationException("numIndexDims must be 1 but got " + numIndexDims);
}
@@ -552,7 +594,10 @@ private class OneDimensionBKDWriter {
// Mark that we already finished:
finished = true;
- this.out = out;
+ this.metaOut = metaOut;
+ this.indexOut = indexOut;
+ this.dataOut = dataOut;
+ this.dataStartFP = dataOut.getFilePointer();
lastPackedValue = new byte[packedBytesLength];
}
@@ -588,7 +633,7 @@ assert valueInOrder(valueCount + leafCount,
assert (lastDocID = docID) >= 0; // only assign when asserts are enabled
}
- public long finish() throws IOException {
+ public Runnable finish() throws IOException {
if (leafCount > 0) {
writeLeafBlock(leafCardinality);
leafCardinality = 0;
@@ -596,25 +641,43 @@ public long finish() throws IOException {
}
if (valueCount == 0) {
- return -1;
+ return null;
}
pointCount = valueCount;
- long indexFP = out.getFilePointer();
+ scratchBytesRef1.length = bytesPerDim;
+ scratchBytesRef1.offset = 0;
+ assert leafBlockStartValues.size() + 1 == leafBlockFPs.size();
+ BKDTreeLeafNodes leafNodes = new BKDTreeLeafNodes() {
+ @Override
+ public long getLeafLP(int index) {
+ return leafBlockFPs.get(index);
+ }
- int numInnerNodes = leafBlockStartValues.size();
+ @Override
+ public BytesRef getSplitValue(int index) {
+ scratchBytesRef1.bytes = leafBlockStartValues.get(index);
+ return scratchBytesRef1;
+ }
- //System.out.println("BKDW: now rotate numInnerNodes=" + numInnerNodes + " leafBlockStarts=" + leafBlockStartValues.size());
+ @Override
+ public int getSplitDimension(int index) {
+ return 0;
+ }
- byte[] index = new byte[(1+numInnerNodes) * (1+bytesPerDim)];
- rotateToTree(1, 0, numInnerNodes, index, leafBlockStartValues);
- long[] arr = new long[leafBlockFPs.size()];
- for(int i=0;i {
+ try {
+ writeIndex(metaOut, indexOut, maxPointsInLeafNode, leafNodes, dataStartFP);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ };
}
private void writeLeafBlock(int leafCardinality) throws IOException {
@@ -630,7 +693,7 @@ private void writeLeafBlock(int leafCardinality) throws IOException {
// Save the first (minimum) value in each leaf block except the first, to build the split value index in the end:
leafBlockStartValues.add(ArrayUtil.copyOfSubArray(leafValues, 0, packedBytesLength));
}
- leafBlockFPs.add(out.getFilePointer());
+ leafBlockFPs.add(dataOut.getFilePointer());
checkMaxLeafNodeCount(leafBlockFPs.size());
// Find per-dim common prefix:
@@ -660,58 +723,26 @@ assert valuesInOrderAndBounds(leafCount, 0, ArrayUtil.copyOfSubArray(leafValues,
ArrayUtil.copyOfSubArray(leafValues, (leafCount - 1) * packedBytesLength, leafCount * packedBytesLength),
packedValues, leafDocs, 0);
writeLeafBlockPackedValues(scratchOut, commonPrefixLengths, leafCount, 0, packedValues, leafCardinality);
- scratchOut.copyTo(out);
+ scratchOut.copyTo(dataOut);
scratchOut.reset();
}
}
- // TODO: there must be a simpler way?
- private void rotateToTree(int nodeID, int offset, int count, byte[] index, List leafBlockStartValues) {
- //System.out.println("ROTATE: nodeID=" + nodeID + " offset=" + offset + " count=" + count + " bpd=" + bytesPerDim + " index.length=" + index.length);
- if (count == 1) {
- // Leaf index node
- //System.out.println(" leaf index node");
- //System.out.println(" index[" + nodeID + "] = blockStartValues[" + offset + "]");
- System.arraycopy(leafBlockStartValues.get(offset), 0, index, nodeID*(1+bytesPerDim)+1, bytesPerDim);
- } else if (count > 1) {
- // Internal index node: binary partition of count
- int countAtLevel = 1;
- int totalCount = 0;
- while (true) {
- int countLeft = count - totalCount;
- //System.out.println(" cycle countLeft=" + countLeft + " coutAtLevel=" + countAtLevel);
- if (countLeft <= countAtLevel) {
- // This is the last level, possibly partially filled:
- int lastLeftCount = Math.min(countAtLevel/2, countLeft);
- assert lastLeftCount >= 0;
- int leftHalf = (totalCount-1)/2 + lastLeftCount;
-
- int rootOffset = offset + leftHalf;
- /*
- System.out.println(" last left count " + lastLeftCount);
- System.out.println(" leftHalf " + leftHalf + " rightHalf=" + (count-leftHalf-1));
- System.out.println(" rootOffset=" + rootOffset);
- */
-
- System.arraycopy(leafBlockStartValues.get(rootOffset), 0, index, nodeID*(1+bytesPerDim)+1, bytesPerDim);
- //System.out.println(" index[" + nodeID + "] = blockStartValues[" + rootOffset + "]");
-
- // TODO: we could optimize/specialize, when we know it's simply fully balanced binary tree
- // under here, to save this while loop on each recursion
-
- // Recurse left
- rotateToTree(2*nodeID, offset, leftHalf, index, leafBlockStartValues);
-
- // Recurse right
- rotateToTree(2*nodeID+1, rootOffset+1, count-leftHalf-1, index, leafBlockStartValues);
- return;
- }
- totalCount += countAtLevel;
- countAtLevel *= 2;
- }
- } else {
- assert count == 0;
- }
+ private int getNumLeftLeafNodes(int numLeaves) {
+ assert numLeaves > 1: "getNumLeftLeaveNodes() called with " + numLeaves;
+ // return the level that can be filled with this number of leaves
+ int lastFullLevel = 31 - Integer.numberOfLeadingZeros(numLeaves);
+ // how many leaf nodes are in the full level
+ int leavesFullLevel = 1 << lastFullLevel;
+ // half of the leaf nodes from the full level goes to the left
+ int numLeftLeafNodes = leavesFullLevel / 2;
+ // leaf nodes that do not fit in the full level
+ int unbalancedLeafNodes = numLeaves - leavesFullLevel;
+ // distribute unbalanced leaf nodes
+ numLeftLeafNodes += Math.min(unbalancedLeafNodes, numLeftLeafNodes);
+ // we should always place unbalanced leaf nodes on the left
+ assert numLeftLeafNodes >= numLeaves - numLeftLeafNodes && numLeftLeafNodes <= 2L * (numLeaves - numLeftLeafNodes);
+ return numLeftLeafNodes;
}
// TODO: if we fixed each partition step to just record the file offset at the "split point", we could probably handle variable length
@@ -736,13 +767,14 @@ private void printPathSlice(String desc, PathSlice slice, int dim) throws IOExce
*/
private void checkMaxLeafNodeCount(int numLeaves) {
- if ((1+bytesPerDim) * (long) numLeaves > ArrayUtil.MAX_ARRAY_LENGTH) {
+ if (bytesPerDim * (long) numLeaves > ArrayUtil.MAX_ARRAY_LENGTH) {
throw new IllegalStateException("too many nodes; increase maxPointsInLeafNode (currently " + maxPointsInLeafNode + ") and reindex");
}
}
- /** Writes the BKD tree to the provided {@link IndexOutput} and returns the file offset where index was written. */
- public long finish(IndexOutput out) throws IOException {
+ /** Writes the BKD tree to the provided {@link IndexOutput}s and returns a {@link Runnable} that
+ * writes the index of the tree if at least one point has been added, or {@code null} otherwise. */
+ public Runnable finish(IndexOutput metaOut, IndexOutput indexOut, IndexOutput dataOut) throws IOException {
// System.out.println("\nBKDTreeWriter.finish pointCount=" + pointCount + " out=" + out + " heapWriter=" + heapPointWriter);
// TODO: specialize the 1D case? it's much faster at indexing time (no partitioning on recurse...)
@@ -753,7 +785,7 @@ public long finish(IndexOutput out) throws IOException {
}
if (pointCount == 0) {
- throw new IllegalStateException("must index at least one point");
+ return null;
}
//mark as finished
@@ -765,16 +797,8 @@ public long finish(IndexOutput out) throws IOException {
tempInput = null;
pointWriter = null;
-
- long countPerLeaf = pointCount;
- long innerNodeCount = 1;
-
- while (countPerLeaf > maxPointsInLeafNode) {
- countPerLeaf = (countPerLeaf+1)/2;
- innerNodeCount *= 2;
- }
-
- int numLeaves = (int) innerNodeCount;
+ final int numLeaves = Math.toIntExact((pointCount + maxPointsInLeafNode - 1) / maxPointsInLeafNode);
+ final int numSplits = numLeaves - 1;
checkMaxLeafNodeCount(numLeaves);
@@ -782,7 +806,8 @@ public long finish(IndexOutput out) throws IOException {
// step of the recursion to recompute the split dim:
// Indexed by nodeID, but first (root) nodeID is 1. We do 1+ because the lead byte at each recursion says which dim we split on.
- byte[] splitPackedValues = new byte[Math.toIntExact(numLeaves*(1+bytesPerDim))];
+ byte[] splitPackedValues = new byte[Math.toIntExact(numSplits*bytesPerDim)];
+ byte[] splitDimensionValues = new byte[numSplits];
// +1 because leaf count is power of 2 (e.g. 8), and innerNodeCount is power of 2 minus 1 (e.g. 7)
long[] leafBlockFPs = new long[numLeaves];
@@ -793,15 +818,17 @@ public long finish(IndexOutput out) throws IOException {
//We re-use the selector so we do not need to create an object every time.
BKDRadixSelector radixSelector = new BKDRadixSelector(numDataDims, numIndexDims, bytesPerDim, maxPointsSortInHeap, tempDir, tempFileNamePrefix);
+ final long dataStartFP = dataOut.getFilePointer();
boolean success = false;
try {
final int[] parentSplits = new int[numIndexDims];
- build(1, numLeaves, points,
- out, radixSelector,
+ build(0, numLeaves, points,
+ dataOut, radixSelector,
minPackedValue.clone(), maxPackedValue.clone(),
parentSplits,
splitPackedValues,
+ splitDimensionValues,
leafBlockFPs,
new int[maxPointsInLeafNode]);
assert Arrays.equals(parentSplits, new int[numIndexDims]);
@@ -818,43 +845,43 @@ public long finish(IndexOutput out) throws IOException {
}
}
- //System.out.println("Total nodes: " + innerNodeCount);
-
- // Write index:
- long indexFP = out.getFilePointer();
- writeIndex(out, Math.toIntExact(countPerLeaf), leafBlockFPs, splitPackedValues);
- return indexFP;
- }
+ scratchBytesRef1.bytes = splitPackedValues;
+ scratchBytesRef1.length = bytesPerDim;
+ BKDTreeLeafNodes leafNodes = new BKDTreeLeafNodes() {
+ @Override
+ public long getLeafLP(int index) {
+ return leafBlockFPs[index];
+ }
- /** Packs the two arrays, representing a balanced binary tree, into a compact byte[] structure. */
- private byte[] packIndex(long[] leafBlockFPs, byte[] splitPackedValues) throws IOException {
+ @Override
+ public BytesRef getSplitValue(int index) {
+ scratchBytesRef1.offset = index * bytesPerDim;
+ return scratchBytesRef1;
+ }
- int numLeaves = leafBlockFPs.length;
+ @Override
+ public int getSplitDimension(int index) {
+ return splitDimensionValues[index] & 0xff;
+ }
- // Possibly rotate the leaf block FPs, if the index not fully balanced binary tree (only happens
- // if it was created by OneDimensionBKDWriter). In this case the leaf nodes may straddle the two bottom
- // levels of the binary tree:
- if (numIndexDims == 1 && numLeaves > 1) {
- int levelCount = 2;
- while (true) {
- if (numLeaves >= levelCount && numLeaves <= 2*levelCount) {
- int lastLevel = 2*(numLeaves - levelCount);
- assert lastLevel >= 0;
- if (lastLevel != 0) {
- // Last level is partially filled, so we must rotate the leaf FPs to match. We do this here, after loading
- // at read-time, so that we can still delta code them on disk at write:
- long[] newLeafBlockFPs = new long[numLeaves];
- System.arraycopy(leafBlockFPs, lastLevel, newLeafBlockFPs, 0, leafBlockFPs.length - lastLevel);
- System.arraycopy(leafBlockFPs, 0, newLeafBlockFPs, leafBlockFPs.length - lastLevel, lastLevel);
- leafBlockFPs = newLeafBlockFPs;
- }
- break;
- }
+ @Override
+ public int numLeaves() {
+ return leafBlockFPs.length;
+ }
+ };
- levelCount *= 2;
+ return () -> {
+ // Write index:
+ try {
+ writeIndex(metaOut, indexOut, maxPointsInLeafNode, leafNodes, dataStartFP);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
}
- }
+ };
+ }
+ /** Packs the two arrays, representing a semi-balanced binary tree, into a compact byte[] structure. */
+ private byte[] packIndex(BKDTreeLeafNodes leafNodes) throws IOException {
/** Reused while packing the index */
ByteBuffersDataOutput writeBuffer = ByteBuffersDataOutput.newResettableInstance();
@@ -862,7 +889,8 @@ private byte[] packIndex(long[] leafBlockFPs, byte[] splitPackedValues) throws I
List blocks = new ArrayList<>();
byte[] lastSplitValues = new byte[bytesPerDim * numIndexDims];
//System.out.println("\npack index");
- int totalSize = recursePackIndex(writeBuffer, leafBlockFPs, splitPackedValues, 0l, blocks, 1, lastSplitValues, new boolean[numIndexDims], false);
+ int totalSize = recursePackIndex(writeBuffer, leafNodes, 0l, blocks, lastSplitValues, new boolean[numIndexDims], false,
+ 0, leafNodes.numLeaves());
// Compact the byte[] blocks into single byte index:
byte[] index = new byte[totalSize];
@@ -887,45 +915,43 @@ private int appendBlock(ByteBuffersDataOutput writeBuffer, List blocks)
/**
* lastSplitValues is per-dimension split value previously seen; we use this to prefix-code the split byte[] on each inner node
*/
- private int recursePackIndex(ByteBuffersDataOutput writeBuffer, long[] leafBlockFPs, byte[] splitPackedValues, long minBlockFP, List blocks,
- int nodeID, byte[] lastSplitValues, boolean[] negativeDeltas, boolean isLeft) throws IOException {
- if (nodeID >= leafBlockFPs.length) {
- int leafID = nodeID - leafBlockFPs.length;
- //System.out.println("recursePack leaf nodeID=" + nodeID);
-
- // In the unbalanced case it's possible the left most node only has one child:
- if (leafID < leafBlockFPs.length) {
- long delta = leafBlockFPs[leafID] - minBlockFP;
- if (isLeft) {
- assert delta == 0;
- return 0;
- } else {
- assert nodeID == 1 || delta > 0: "nodeID=" + nodeID;
- writeBuffer.writeVLong(delta);
- return appendBlock(writeBuffer, blocks);
- }
- } else {
+ private int recursePackIndex(ByteBuffersDataOutput writeBuffer, BKDTreeLeafNodes leafNodes, long minBlockFP, List blocks,
+ byte[] lastSplitValues, boolean[] negativeDeltas, boolean isLeft, int leavesOffset, int numLeaves) throws IOException {
+ if (numLeaves == 1) {
+ if (isLeft) {
+ assert leafNodes.getLeafLP(leavesOffset) - minBlockFP == 0;
return 0;
+ } else {
+ long delta = leafNodes.getLeafLP(leavesOffset) - minBlockFP;
+ assert leafNodes.numLeaves() == numLeaves || delta > 0 : "expected delta > 0; got numLeaves =" + numLeaves + " and delta=" + delta;
+ writeBuffer.writeVLong(delta);
+ return appendBlock(writeBuffer, blocks);
}
} else {
long leftBlockFP;
- if (isLeft == false) {
- leftBlockFP = getLeftMostLeafBlockFP(leafBlockFPs, nodeID);
- long delta = leftBlockFP - minBlockFP;
- assert nodeID == 1 || delta > 0 : "expected nodeID=1 or delta > 0; got nodeID=" + nodeID + " and delta=" + delta;
- writeBuffer.writeVLong(delta);
- } else {
+ if (isLeft) {
// The left tree's left most leaf block FP is always the minimal FP:
+ assert leafNodes.getLeafLP(leavesOffset) == minBlockFP;
leftBlockFP = minBlockFP;
+ } else {
+ leftBlockFP = leafNodes.getLeafLP(leavesOffset);
+ long delta = leftBlockFP - minBlockFP;
+ assert leafNodes.numLeaves() == numLeaves || delta > 0 : "expected delta > 0; got numLeaves =" + numLeaves + " and delta=" + delta;
+ writeBuffer.writeVLong(delta);
}
- int address = nodeID * (1+bytesPerDim);
- int splitDim = splitPackedValues[address++] & 0xff;
+ int numLeftLeafNodes = getNumLeftLeafNodes(numLeaves);
+ final int rightOffset = leavesOffset + numLeftLeafNodes;
+ final int splitOffset = rightOffset - 1;
+
+ int splitDim = leafNodes.getSplitDimension(splitOffset);
+ BytesRef splitValue = leafNodes.getSplitValue(splitOffset);
+ int address = splitValue.offset;
//System.out.println("recursePack inner nodeID=" + nodeID + " splitDim=" + splitDim + " splitValue=" + new BytesRef(splitPackedValues, address, bytesPerDim));
// find common prefix with last split value in this dim:
- int prefix = Arrays.mismatch(splitPackedValues, address, address + bytesPerDim, lastSplitValues,
+ int prefix = Arrays.mismatch(splitValue.bytes, address, address + bytesPerDim, lastSplitValues,
splitDim * bytesPerDim, splitDim * bytesPerDim + bytesPerDim);
if (prefix == -1) {
prefix = bytesPerDim;
@@ -936,7 +962,7 @@ private int recursePackIndex(ByteBuffersDataOutput writeBuffer, long[] leafBlock
int firstDiffByteDelta;
if (prefix < bytesPerDim) {
//System.out.println(" delta byte cur=" + Integer.toHexString(splitPackedValues[address+prefix]&0xFF) + " prev=" + Integer.toHexString(lastSplitValues[splitDim * bytesPerDim + prefix]&0xFF) + " negated?=" + negativeDeltas[splitDim]);
- firstDiffByteDelta = (splitPackedValues[address+prefix]&0xFF) - (lastSplitValues[splitDim * bytesPerDim + prefix]&0xFF);
+ firstDiffByteDelta = (splitValue.bytes[address+prefix]&0xFF) - (lastSplitValues[splitDim * bytesPerDim + prefix]&0xFF);
if (negativeDeltas[splitDim]) {
firstDiffByteDelta = -firstDiffByteDelta;
}
@@ -958,7 +984,7 @@ private int recursePackIndex(ByteBuffersDataOutput writeBuffer, long[] leafBlock
int suffix = bytesPerDim - prefix;
byte[] savSplitValue = new byte[suffix];
if (suffix > 1) {
- writeBuffer.writeBytes(splitPackedValues, address+prefix+1, suffix-1);
+ writeBuffer.writeBytes(splitValue.bytes, address+prefix+1, suffix-1);
}
byte[] cmp = lastSplitValues.clone();
@@ -966,7 +992,7 @@ private int recursePackIndex(ByteBuffersDataOutput writeBuffer, long[] leafBlock
System.arraycopy(lastSplitValues, splitDim * bytesPerDim + prefix, savSplitValue, 0, suffix);
// copy our split value into lastSplitValues for our children to prefix-code against
- System.arraycopy(splitPackedValues, address+prefix, lastSplitValues, splitDim * bytesPerDim + prefix, suffix);
+ System.arraycopy(splitValue.bytes, address+prefix, lastSplitValues, splitDim * bytesPerDim + prefix, suffix);
int numBytes = appendBlock(writeBuffer, blocks);
@@ -978,9 +1004,11 @@ private int recursePackIndex(ByteBuffersDataOutput writeBuffer, long[] leafBlock
boolean savNegativeDelta = negativeDeltas[splitDim];
negativeDeltas[splitDim] = true;
- int leftNumBytes = recursePackIndex(writeBuffer, leafBlockFPs, splitPackedValues, leftBlockFP, blocks, 2*nodeID, lastSplitValues, negativeDeltas, true);
- if (nodeID * 2 < leafBlockFPs.length) {
+ int leftNumBytes = recursePackIndex(writeBuffer, leafNodes, leftBlockFP, blocks, lastSplitValues, negativeDeltas, true,
+ leavesOffset, numLeftLeafNodes);
+
+ if (numLeftLeafNodes != 1) {
writeBuffer.writeVInt(leftNumBytes);
} else {
assert leftNumBytes == 0: "leftNumBytes=" + leftNumBytes;
@@ -992,7 +1020,8 @@ private int recursePackIndex(ByteBuffersDataOutput writeBuffer, long[] leafBlock
blocks.set(idxSav, bytes2);
negativeDeltas[splitDim] = false;
- int rightNumBytes = recursePackIndex(writeBuffer, leafBlockFPs, splitPackedValues, leftBlockFP, blocks, 2*nodeID+1, lastSplitValues, negativeDeltas, false);
+ int rightNumBytes = recursePackIndex(writeBuffer, leafNodes, leftBlockFP, blocks, lastSplitValues, negativeDeltas, false,
+ rightOffset, numLeaves - numLeftLeafNodes);
negativeDeltas[splitDim] = savNegativeDelta;
@@ -1005,46 +1034,32 @@ private int recursePackIndex(ByteBuffersDataOutput writeBuffer, long[] leafBlock
}
}
- private long getLeftMostLeafBlockFP(long[] leafBlockFPs, int nodeID) {
- // TODO: can we do this cheaper, e.g. a closed form solution instead of while loop? Or
- // change the recursion while packing the index to return this left-most leaf block FP
- // from each recursion instead?
- //
- // Still, the overall cost here is minor: this method's cost is O(log(N)), and while writing
- // we call it O(N) times (N = number of leaf blocks)
- while (nodeID < leafBlockFPs.length) {
- nodeID *= 2;
- }
- int leafID = nodeID - leafBlockFPs.length;
- long result = leafBlockFPs[leafID];
- if (result < 0) {
- throw new AssertionError(result + " for leaf " + leafID);
- }
- return result;
- }
-
- private void writeIndex(IndexOutput out, int countPerLeaf, long[] leafBlockFPs, byte[] splitPackedValues) throws IOException {
- byte[] packedIndex = packIndex(leafBlockFPs, splitPackedValues);
- writeIndex(out, countPerLeaf, leafBlockFPs.length, packedIndex);
+ private void writeIndex(IndexOutput metaOut, IndexOutput indexOut, int countPerLeaf, BKDTreeLeafNodes leafNodes, long dataStartFP) throws IOException {
+ byte[] packedIndex = packIndex(leafNodes);
+ writeIndex(metaOut, indexOut, countPerLeaf, leafNodes.numLeaves(), packedIndex, dataStartFP);
}
- private void writeIndex(IndexOutput out, int countPerLeaf, int numLeaves, byte[] packedIndex) throws IOException {
-
- CodecUtil.writeHeader(out, CODEC_NAME, VERSION_CURRENT);
- out.writeVInt(numDataDims);
- out.writeVInt(numIndexDims);
- out.writeVInt(countPerLeaf);
- out.writeVInt(bytesPerDim);
+ private void writeIndex(IndexOutput metaOut, IndexOutput indexOut, int countPerLeaf, int numLeaves, byte[] packedIndex, long dataStartFP) throws IOException {
+ CodecUtil.writeHeader(metaOut, CODEC_NAME, VERSION_CURRENT);
+ metaOut.writeVInt(numDataDims);
+ metaOut.writeVInt(numIndexDims);
+ metaOut.writeVInt(countPerLeaf);
+ metaOut.writeVInt(bytesPerDim);
assert numLeaves > 0;
- out.writeVInt(numLeaves);
- out.writeBytes(minPackedValue, 0, packedIndexBytesLength);
- out.writeBytes(maxPackedValue, 0, packedIndexBytesLength);
-
- out.writeVLong(pointCount);
- out.writeVInt(docsSeen.cardinality());
- out.writeVInt(packedIndex.length);
- out.writeBytes(packedIndex, 0, packedIndex.length);
+ metaOut.writeVInt(numLeaves);
+ metaOut.writeBytes(minPackedValue, 0, packedIndexBytesLength);
+ metaOut.writeBytes(maxPackedValue, 0, packedIndexBytesLength);
+
+ metaOut.writeVLong(pointCount);
+ metaOut.writeVInt(docsSeen.cardinality());
+ metaOut.writeVInt(packedIndex.length);
+ metaOut.writeLong(dataStartFP);
+ // If metaOut and indexOut are the same file, we account for the fact that
+ // writing a long makes the index start 8 bytes later.
+ metaOut.writeLong(indexOut.getFilePointer() + (metaOut == indexOut ? Long.BYTES : 0));
+
+ indexOut.writeBytes(packedIndex, 0, packedIndex.length);
}
private void writeLeafBlockDocs(DataOutput out, int[] docIDs, int start, int count) throws IOException {
@@ -1319,16 +1334,17 @@ private HeapPointWriter switchToHeap(PointWriter source) throws IOException {
/* Recursively reorders the provided reader and writes the bkd-tree on the fly; this method is used
* when we are writing a new segment directly from IndexWriter's indexing buffer (MutablePointsReader). */
- private void build(int nodeID, int leafNodeOffset,
+ private void build(int leavesOffset, int numLeaves,
MutablePointValues reader, int from, int to,
IndexOutput out,
byte[] minPackedValue, byte[] maxPackedValue,
int[] parentSplits,
byte[] splitPackedValues,
+ byte[] splitDimensionValues,
long[] leafBlockFPs,
int[] spareDocIds) throws IOException {
- if (nodeID >= leafNodeOffset) {
+ if (numLeaves == 1) {
// leaf node
final int count = to - from;
assert count <= maxPointsInLeafNode;
@@ -1402,7 +1418,7 @@ private void build(int nodeID, int leafNodeOffset,
}
}
// Save the block file pointer:
- leafBlockFPs[nodeID - leafNodeOffset] = out.getFilePointer();
+ leafBlockFPs[leavesOffset] = out.getFilePointer();
assert scratchOut.size() == 0;
@@ -1443,13 +1459,16 @@ assert valuesInOrderAndBounds(count, sortedDim, minPackedValue, maxPackedValue,
// for dimensions > 2 we recompute the bounds for the current inner node to help the algorithm choose best
// split dimensions. Because it is an expensive operation, the frequency we recompute the bounds is given
// by SPLITS_BEFORE_EXACT_BOUNDS.
- if (nodeID > 1 && numIndexDims > 2 && Arrays.stream(parentSplits).sum() % SPLITS_BEFORE_EXACT_BOUNDS == 0) {
+ if (numLeaves != leafBlockFPs.length && numIndexDims > 2 && Arrays.stream(parentSplits).sum() % SPLITS_BEFORE_EXACT_BOUNDS == 0) {
computePackedValueBounds(reader, from, to, minPackedValue, maxPackedValue, scratchBytesRef1);
}
splitDim = split(minPackedValue, maxPackedValue, parentSplits);
}
- final int mid = (from + to + 1) >>> 1;
+ // How many leaves will be in the left tree:
+ int numLeftLeafNodes = getNumLeftLeafNodes(numLeaves);
+ // How many points will be in the left tree:
+ final int mid = from + numLeftLeafNodes * maxPointsInLeafNode;
int commonPrefixLen = Arrays.mismatch(minPackedValue, splitDim * bytesPerDim,
splitDim * bytesPerDim + bytesPerDim, maxPackedValue, splitDim * bytesPerDim,
@@ -1461,11 +1480,13 @@ assert valuesInOrderAndBounds(count, sortedDim, minPackedValue, maxPackedValue,
MutablePointsReaderUtils.partition(numDataDims, numIndexDims, maxDoc, splitDim, bytesPerDim, commonPrefixLen,
reader, from, to, mid, scratchBytesRef1, scratchBytesRef2);
+ final int rightOffset = leavesOffset + numLeftLeafNodes;
+ final int splitOffset = rightOffset - 1;
// set the split value
- final int address = nodeID * (1+bytesPerDim);
- splitPackedValues[address] = (byte) splitDim;
+ final int address = splitOffset * bytesPerDim;
+ splitDimensionValues[splitOffset] = (byte) splitDim;
reader.getValue(mid, scratchBytesRef1);
- System.arraycopy(scratchBytesRef1.bytes, scratchBytesRef1.offset + splitDim * bytesPerDim, splitPackedValues, address + 1, bytesPerDim);
+ System.arraycopy(scratchBytesRef1.bytes, scratchBytesRef1.offset + splitDim * bytesPerDim, splitPackedValues, address, bytesPerDim);
byte[] minSplitPackedValue = ArrayUtil.copyOfSubArray(minPackedValue, 0, packedIndexBytesLength);
byte[] maxSplitPackedValue = ArrayUtil.copyOfSubArray(maxPackedValue, 0, packedIndexBytesLength);
@@ -1476,12 +1497,12 @@ assert valuesInOrderAndBounds(count, sortedDim, minPackedValue, maxPackedValue,
// recurse
parentSplits[splitDim]++;
- build(nodeID * 2, leafNodeOffset, reader, from, mid, out,
+ build(leavesOffset, numLeftLeafNodes, reader, from, mid, out,
minPackedValue, maxSplitPackedValue, parentSplits,
- splitPackedValues, leafBlockFPs, spareDocIds);
- build(nodeID * 2 + 1, leafNodeOffset, reader, mid, to, out,
+ splitPackedValues, splitDimensionValues, leafBlockFPs, spareDocIds);
+ build(rightOffset, numLeaves - numLeftLeafNodes, reader, mid, to, out,
minSplitPackedValue, maxPackedValue, parentSplits,
- splitPackedValues, leafBlockFPs, spareDocIds);
+ splitPackedValues, splitDimensionValues, leafBlockFPs, spareDocIds);
parentSplits[splitDim]--;
}
}
@@ -1512,17 +1533,18 @@ private void computePackedValueBounds(BKDRadixSelector.PathSlice slice, byte[] m
/** The point writer contains the data that is going to be splitted using radix selection.
/* This method is used when we are merging previously written segments, in the numDims > 1 case. */
- private void build(int nodeID, int leafNodeOffset,
+ private void build(int leavesOffset, int numLeaves,
BKDRadixSelector.PathSlice points,
IndexOutput out,
BKDRadixSelector radixSelector,
byte[] minPackedValue, byte[] maxPackedValue,
int[] parentSplits,
byte[] splitPackedValues,
+ byte[] splitDimensionValues,
long[] leafBlockFPs,
int[] spareDocIds) throws IOException {
- if (nodeID >= leafNodeOffset) {
+ if (numLeaves == 1) {
// Leaf node: write block
// We can write the block in any order so by default we write it sorted by the dimension that has the
@@ -1573,13 +1595,13 @@ private void build(int nodeID, int leafNodeOffset,
int leafCardinality = heapSource.computeCardinality(from ,to, numDataDims, bytesPerDim, commonPrefixLengths);
// Save the block file pointer:
- leafBlockFPs[nodeID - leafNodeOffset] = out.getFilePointer();
+ leafBlockFPs[leavesOffset] = out.getFilePointer();
//System.out.println(" write leaf block @ fp=" + out.getFilePointer());
// Write docIDs first, as their own chunk, so that at intersect time we can add all docIDs w/o
// loading the values:
int count = to - from;
- assert count > 0: "nodeID=" + nodeID + " leafNodeOffset=" + leafNodeOffset;
+ assert count > 0: "numLeaves=" + numLeaves + " leavesOffset=" + leavesOffset;
assert count <= spareDocIds.length : "count=" + count + " > length=" + spareDocIds.length;
// Write doc IDs
int[] docIDs = spareDocIds;
@@ -1622,17 +1644,18 @@ assert valuesInOrderAndBounds(count, sortedDim, minPackedValue, maxPackedValue,
// for dimensions > 2 we recompute the bounds for the current inner node to help the algorithm choose best
// split dimensions. Because it is an expensive operation, the frequency we recompute the bounds is given
// by SPLITS_BEFORE_EXACT_BOUNDS.
- if (nodeID > 1 && numIndexDims > 2 && Arrays.stream(parentSplits).sum() % SPLITS_BEFORE_EXACT_BOUNDS == 0) {
+ if (numLeaves != leafBlockFPs.length && numIndexDims > 2 && Arrays.stream(parentSplits).sum() % SPLITS_BEFORE_EXACT_BOUNDS == 0) {
computePackedValueBounds(points, minPackedValue, maxPackedValue);
}
splitDim = split(minPackedValue, maxPackedValue, parentSplits);
}
- assert nodeID < splitPackedValues.length : "nodeID=" + nodeID + " splitValues.length=" + splitPackedValues.length;
+ assert numLeaves <= leafBlockFPs.length : "numLeaves=" + numLeaves + " leafBlockFPs.length=" + leafBlockFPs.length;
+ // How many leaves will be in the left tree:
+ final int numLeftLeafNodes = getNumLeftLeafNodes(numLeaves);
// How many points will be in the left tree:
- long rightCount = points.count / 2;
- long leftCount = points.count - rightCount;
+ final long leftCount = numLeftLeafNodes * maxPointsInLeafNode;
BKDRadixSelector.PathSlice[] slices = new BKDRadixSelector.PathSlice[2];
@@ -1645,9 +1668,12 @@ assert valuesInOrderAndBounds(count, sortedDim, minPackedValue, maxPackedValue,
byte[] splitValue = radixSelector.select(points, slices, points.start, points.start + points.count, points.start + leftCount, splitDim, commonPrefixLen);
- int address = nodeID * (1 + bytesPerDim);
- splitPackedValues[address] = (byte) splitDim;
- System.arraycopy(splitValue, 0, splitPackedValues, address + 1, bytesPerDim);
+ final int rightOffset = leavesOffset + numLeftLeafNodes;
+ final int splitValueOffset = rightOffset - 1;
+
+ splitDimensionValues[splitValueOffset] = (byte) splitDim;
+ int address = splitValueOffset * bytesPerDim;
+ System.arraycopy(splitValue, 0, splitPackedValues, address, bytesPerDim);
byte[] minSplitPackedValue = new byte[packedIndexBytesLength];
System.arraycopy(minPackedValue, 0, minSplitPackedValue, 0, packedIndexBytesLength);
@@ -1660,14 +1686,14 @@ assert valuesInOrderAndBounds(count, sortedDim, minPackedValue, maxPackedValue,
parentSplits[splitDim]++;
// Recurse on left tree:
- build(2 * nodeID, leafNodeOffset, slices[0],
+ build(leavesOffset, numLeftLeafNodes, slices[0],
out, radixSelector, minPackedValue, maxSplitPackedValue,
- parentSplits, splitPackedValues, leafBlockFPs, spareDocIds);
+ parentSplits, splitPackedValues, splitDimensionValues, leafBlockFPs, spareDocIds);
// Recurse on right tree:
- build(2 * nodeID + 1, leafNodeOffset, slices[1],
- out, radixSelector, minSplitPackedValue, maxPackedValue
- , parentSplits, splitPackedValues, leafBlockFPs, spareDocIds);
+ build(rightOffset, numLeaves - numLeftLeafNodes, slices[1],
+ out, radixSelector, minSplitPackedValue, maxPackedValue,
+ parentSplits, splitPackedValues, splitDimensionValues, leafBlockFPs, spareDocIds);
parentSplits[splitDim]--;
}
diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/FST.java b/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
index 260e9dab1aeb..c3cf67bfbdd5 100644
--- a/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
+++ b/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
@@ -26,6 +26,7 @@
import java.nio.file.Path;
import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.store.ByteBuffersDataOutput;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.DataOutput;
@@ -407,26 +408,26 @@ private static boolean flag(int flags, int bit) {
private static final int DEFAULT_MAX_BLOCK_BITS = Constants.JRE_IS_64BIT ? 30 : 28;
/** Load a previously saved FST. */
- public FST(DataInput in, Outputs outputs) throws IOException {
- this(in, outputs, new OnHeapFSTStore(DEFAULT_MAX_BLOCK_BITS));
+ public FST(DataInput metaIn, DataInput in, Outputs outputs) throws IOException {
+ this(metaIn, in, outputs, new OnHeapFSTStore(DEFAULT_MAX_BLOCK_BITS));
}
/** Load a previously saved FST; maxBlockBits allows you to
* control the size of the byte[] pages used to hold the FST bytes. */
- public FST(DataInput in, Outputs outputs, FSTStore fstStore) throws IOException {
+ public FST(DataInput metaIn, DataInput in, Outputs outputs, FSTStore fstStore) throws IOException {
bytes = null;
this.fstStore = fstStore;
this.outputs = outputs;
// NOTE: only reads formats VERSION_START up to VERSION_CURRENT; we don't have
// back-compat promise for FSTs (they are experimental), but we are sometimes able to offer it
- CodecUtil.checkHeader(in, FILE_FORMAT_NAME, VERSION_START, VERSION_CURRENT);
- if (in.readByte() == 1) {
+ CodecUtil.checkHeader(metaIn, FILE_FORMAT_NAME, VERSION_START, VERSION_CURRENT);
+ if (metaIn.readByte() == 1) {
// accepts empty string
// 1 KB blocks:
BytesStore emptyBytes = new BytesStore(10);
- int numBytes = in.readVInt();
- emptyBytes.copyBytes(in, numBytes);
+ int numBytes = metaIn.readVInt();
+ emptyBytes.copyBytes(metaIn, numBytes);
// De-serialize empty-string output:
BytesReader reader = emptyBytes.getReverseReader();
@@ -440,7 +441,7 @@ public FST(DataInput in, Outputs outputs, FSTStore fstStore) throws IOExcepti
} else {
emptyOutput = null;
}
- final byte t = in.readByte();
+ final byte t = metaIn.readByte();
switch(t) {
case 0:
inputType = INPUT_TYPE.BYTE1;
@@ -452,11 +453,11 @@ public FST(DataInput in, Outputs outputs, FSTStore fstStore) throws IOExcepti
inputType = INPUT_TYPE.BYTE4;
break;
default:
- throw new IllegalStateException("invalid input type " + t);
+ throw new CorruptIndexException("invalid input type " + t, in);
}
- startNode = in.readVLong();
+ startNode = metaIn.readVLong();
- long numBytes = in.readVLong();
+ long numBytes = metaIn.readVLong();
this.fstStore.init(in, numBytes);
}
@@ -501,16 +502,16 @@ void setEmptyOutput(T v) {
}
}
- public void save(DataOutput out) throws IOException {
+ public void save(DataOutput metaOut, DataOutput out) throws IOException {
if (startNode == -1) {
throw new IllegalStateException("call finish first");
}
- CodecUtil.writeHeader(out, FILE_FORMAT_NAME, VERSION_CURRENT);
+ CodecUtil.writeHeader(metaOut, FILE_FORMAT_NAME, VERSION_CURRENT);
// TODO: really we should encode this as an arc, arriving
// to the root node, instead of special casing here:
if (emptyOutput != null) {
// Accepts empty string
- out.writeByte((byte) 1);
+ metaOut.writeByte((byte) 1);
// Serialize empty-string output:
ByteBuffersDataOutput ros = new ByteBuffersDataOutput();
@@ -527,10 +528,10 @@ public void save(DataOutput out) throws IOException {
emptyOutputBytes[emptyLen - upto - 1] = b;
upto++;
}
- out.writeVInt(emptyLen);
- out.writeBytes(emptyOutputBytes, 0, emptyLen);
+ metaOut.writeVInt(emptyLen);
+ metaOut.writeBytes(emptyOutputBytes, 0, emptyLen);
} else {
- out.writeByte((byte) 0);
+ metaOut.writeByte((byte) 0);
}
final byte t;
if (inputType == INPUT_TYPE.BYTE1) {
@@ -540,11 +541,11 @@ public void save(DataOutput out) throws IOException {
} else {
t = 2;
}
- out.writeByte(t);
- out.writeVLong(startNode);
+ metaOut.writeByte(t);
+ metaOut.writeVLong(startNode);
if (bytes != null) {
long numBytes = bytes.getPosition();
- out.writeVLong(numBytes);
+ metaOut.writeVLong(numBytes);
bytes.writeTo(out);
} else {
assert fstStore != null;
@@ -557,7 +558,8 @@ public void save(DataOutput out) throws IOException {
*/
public void save(final Path path) throws IOException {
try (OutputStream os = new BufferedOutputStream(Files.newOutputStream(path))) {
- save(new OutputStreamDataOutput(os));
+ DataOutput out = new OutputStreamDataOutput(os);
+ save(out, out);
}
}
@@ -566,7 +568,8 @@ public void save(final Path path) throws IOException {
*/
public static FST read(Path path, Outputs outputs) throws IOException {
try (InputStream is = Files.newInputStream(path)) {
- return new FST<>(new InputStreamDataInput(new BufferedInputStream(is)), outputs);
+ DataInput in = new InputStreamDataInput(new BufferedInputStream(is));
+ return new FST<>(in, in, outputs);
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/PositiveIntOutputs.java b/lucene/core/src/java/org/apache/lucene/util/fst/PositiveIntOutputs.java
index bc9806dcdddc..c6ea6f263f44 100644
--- a/lucene/core/src/java/org/apache/lucene/util/fst/PositiveIntOutputs.java
+++ b/lucene/core/src/java/org/apache/lucene/util/fst/PositiveIntOutputs.java
@@ -33,6 +33,7 @@
@SuppressForbidden(reason = "Uses a Long instance as a marker")
public final class PositiveIntOutputs extends Outputs {
+ @SuppressWarnings("deprecation")
private final static Long NO_OUTPUT = new Long(0);
private final static PositiveIntOutputs singleton = new PositiveIntOutputs();
diff --git a/lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java b/lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java
index 85e7ea8e0780..dab9a4b9a234 100644
--- a/lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java
+++ b/lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java
@@ -90,6 +90,7 @@ public int get(int index, long[] arr, int off, int len) {
// bulk get
assert index % valuesPerBlock == 0;
+ @SuppressWarnings("deprecation")
final PackedInts.Decoder decoder = BulkOperation.of(PackedInts.Format.PACKED_SINGLE_BLOCK, bitsPerValue);
assert decoder.longBlockCount() == 1;
assert decoder.longValueCount() == valuesPerBlock;
@@ -134,6 +135,7 @@ public int set(int index, long[] arr, int off, int len) {
// bulk set
assert index % valuesPerBlock == 0;
+ @SuppressWarnings("deprecation")
final BulkOperation op = BulkOperation.of(PackedInts.Format.PACKED_SINGLE_BLOCK, bitsPerValue);
assert op.longBlockCount() == 1;
assert op.longValueCount() == valuesPerBlock;
@@ -195,6 +197,7 @@ public void fill(int fromIndex, int toIndex, long val) {
}
@Override
+ @SuppressWarnings("deprecation")
protected PackedInts.Format getFormat() {
return PackedInts.Format.PACKED_SINGLE_BLOCK;
}
diff --git a/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec b/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
index 611e237384c3..2897a8ac7c1a 100644
--- a/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
+++ b/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
@@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-org.apache.lucene.codecs.lucene84.Lucene84Codec
+org.apache.lucene.codecs.lucene86.Lucene86Codec
diff --git a/lucene/core/src/resources/META-INF/services/org.apache.lucene.index.SortFieldProvider b/lucene/core/src/resources/META-INF/services/org.apache.lucene.index.SortFieldProvider
new file mode 100644
index 000000000000..a96a47b7b810
--- /dev/null
+++ b/lucene/core/src/resources/META-INF/services/org.apache.lucene.index.SortFieldProvider
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.lucene.search.SortField$Provider
+org.apache.lucene.search.SortedNumericSortField$Provider
+org.apache.lucene.search.SortedSetSortField$Provider
\ No newline at end of file
diff --git a/lucene/core/src/test/org/apache/lucene/analysis/TestCharArraySet.java b/lucene/core/src/test/org/apache/lucene/analysis/TestCharArraySet.java
index 8430eb82def1..1a32a800bbba 100644
--- a/lucene/core/src/test/org/apache/lucene/analysis/TestCharArraySet.java
+++ b/lucene/core/src/test/org/apache/lucene/analysis/TestCharArraySet.java
@@ -61,15 +61,19 @@ public void testNonZeroOffset() {
public void testObjectContains() {
CharArraySet set = new CharArraySet(10, true);
Integer val = Integer.valueOf(1);
+ @SuppressWarnings("deprecation")
+ Integer val1 = new Integer(1);
+ // Verify explicitly the case of different Integer instances
+ assertNotSame(val, val1);
set.add(val);
assertTrue(set.contains(val));
- assertTrue(set.contains(new Integer(1))); // another integer
+ assertTrue(set.contains(val1)); // another integer
assertTrue(set.contains("1"));
assertTrue(set.contains(new char[]{'1'}));
// test unmodifiable
set = CharArraySet.unmodifiableSet(set);
assertTrue(set.contains(val));
- assertTrue(set.contains(new Integer(1))); // another integer
+ assertTrue(set.contains(val1)); // another integer
assertTrue(set.contains("1"));
assertTrue(set.contains(new char[]{'1'}));
}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/TestCodecUtil.java b/lucene/core/src/test/org/apache/lucene/codecs/TestCodecUtil.java
index 0a11a9b5b495..ea0972d5a2af 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/TestCodecUtil.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/TestCodecUtil.java
@@ -26,6 +26,8 @@
import org.apache.lucene.store.ByteBuffersIndexInput;
import org.apache.lucene.store.ByteBuffersIndexOutput;
import org.apache.lucene.store.ChecksumIndexInput;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.LuceneTestCase;
@@ -148,13 +150,13 @@ public void testCheckFooterValidPastFooter() throws Exception {
// bogusly read a byte too far (can happen)
input.readByte();
Exception mine = new RuntimeException("fake exception");
- RuntimeException expected = expectThrows(RuntimeException.class, () -> {
+ CorruptIndexException expected = expectThrows(CorruptIndexException.class, () -> {
CodecUtil.checkFooter(input, mine);
});
- assertEquals("fake exception", expected.getMessage());
+ assertTrue(expected.getMessage().contains("checksum status indeterminate"));
Throwable suppressed[] = expected.getSuppressed();
assertEquals(1, suppressed.length);
- assertTrue(suppressed[0].getMessage().contains("checksum status indeterminate"));
+ assertEquals("fake exception", suppressed[0].getMessage());
input.close();
}
@@ -172,13 +174,13 @@ public void testCheckFooterInvalid() throws Exception {
CodecUtil.checkHeader(input, "FooBar", 5, 5);
assertEquals("this is the data", input.readString());
Exception mine = new RuntimeException("fake exception");
- RuntimeException expected = expectThrows(RuntimeException.class, () -> {
+ CorruptIndexException expected = expectThrows(CorruptIndexException.class, () -> {
CodecUtil.checkFooter(input, mine);
});
- assertEquals("fake exception", expected.getMessage());
+ assertTrue(expected.getMessage().contains("checksum failed"));
Throwable suppressed[] = expected.getSuppressed();
assertEquals(1, suppressed.length);
- assertTrue(suppressed[0].getMessage().contains("checksum failed"));
+ assertEquals("fake exception", suppressed[0].getMessage());
input.close();
}
@@ -319,4 +321,47 @@ public void testTruncatedFileThrowsCorruptIndexException() throws IOException {
() -> CodecUtil.retrieveChecksum(input));
assertTrue(e.getMessage(), e.getMessage().contains("misplaced codec footer (file truncated?): length=0 but footerLength==16 (resource"));
}
+
+ public void testRetrieveChecksum() throws IOException {
+ Directory dir = newDirectory();
+ try (IndexOutput out = dir.createOutput("foo", IOContext.DEFAULT)) {
+ out.writeByte((byte) 42);
+ CodecUtil.writeFooter(out);
+ }
+ try (IndexInput in = dir.openInput("foo", IOContext.DEFAULT)) {
+ CodecUtil.retrieveChecksum(in, in.length()); // no exception
+
+ CorruptIndexException exception = expectThrows(CorruptIndexException.class,
+ () -> CodecUtil.retrieveChecksum(in, in.length() - 1));
+ assertTrue(exception.getMessage().contains("too long"));
+ assertArrayEquals(new Throwable[0], exception.getSuppressed());
+
+ exception = expectThrows(CorruptIndexException.class,
+ () -> CodecUtil.retrieveChecksum(in, in.length() + 1));
+ assertTrue(exception.getMessage().contains("truncated"));
+ assertArrayEquals(new Throwable[0], exception.getSuppressed());
+ }
+
+ try (IndexOutput out = dir.createOutput("bar", IOContext.DEFAULT)) {
+ for (int i = 0; i <= CodecUtil.footerLength(); ++i) {
+ out.writeByte((byte) i);
+ }
+ }
+ try (IndexInput in = dir.openInput("bar", IOContext.DEFAULT)) {
+ CorruptIndexException exception = expectThrows(CorruptIndexException.class,
+ () -> CodecUtil.retrieveChecksum(in, in.length()));
+ assertTrue(exception.getMessage().contains("codec footer mismatch"));
+ assertArrayEquals(new Throwable[0], exception.getSuppressed());
+
+ exception = expectThrows(CorruptIndexException.class,
+ () -> CodecUtil.retrieveChecksum(in, in.length() - 1));
+ assertTrue(exception.getMessage().contains("too long"));
+
+ exception = expectThrows(CorruptIndexException.class,
+ () -> CodecUtil.retrieveChecksum(in, in.length() + 1));
+ assertTrue(exception.getMessage().contains("truncated"));
+ }
+
+ dir.close();
+ }
}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormatHighCompression.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormatHighCompression.java
index 6a3ce93a0f03..cccee736d46b 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormatHighCompression.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormatHighCompression.java
@@ -17,9 +17,10 @@
package org.apache.lucene.codecs.lucene50;
+import com.carrotsearch.randomizedtesting.generators.RandomPicks;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.Mode;
-import org.apache.lucene.codecs.lucene84.Lucene84Codec;
+import org.apache.lucene.codecs.lucene86.Lucene86Codec;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.BaseStoredFieldsFormatTestCase;
@@ -28,12 +29,10 @@
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.store.Directory;
-import com.carrotsearch.randomizedtesting.generators.RandomPicks;
-
public class TestLucene50StoredFieldsFormatHighCompression extends BaseStoredFieldsFormatTestCase {
@Override
protected Codec getCodec() {
- return new Lucene84Codec(Mode.BEST_COMPRESSION);
+ return new Lucene86Codec(Mode.BEST_COMPRESSION);
}
/**
@@ -44,7 +43,7 @@ public void testMixedCompressions() throws Exception {
Directory dir = newDirectory();
for (int i = 0; i < 10; i++) {
IndexWriterConfig iwc = newIndexWriterConfig();
- iwc.setCodec(new Lucene84Codec(RandomPicks.randomFrom(random(), Mode.values())));
+ iwc.setCodec(new Lucene86Codec(RandomPicks.randomFrom(random(), Mode.values())));
IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig());
Document doc = new Document();
doc.add(new StoredField("field1", "value1"));
@@ -71,7 +70,7 @@ public void testMixedCompressions() throws Exception {
public void testInvalidOptions() {
expectThrows(NullPointerException.class, () -> {
- new Lucene84Codec(null);
+ new Lucene86Codec(null);
});
expectThrows(NullPointerException.class, () -> {
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene80/TestLucene80NormsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene80/TestLucene80NormsFormat.java
index 4eadf05ef5a2..b6e7268d67f1 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene80/TestLucene80NormsFormat.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene80/TestLucene80NormsFormat.java
@@ -18,14 +18,14 @@
import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.lucene84.Lucene84Codec;
+import org.apache.lucene.codecs.lucene86.Lucene86Codec;
import org.apache.lucene.index.BaseNormsFormatTestCase;
/**
* Tests Lucene80NormsFormat
*/
public class TestLucene80NormsFormat extends BaseNormsFormatTestCase {
- private final Codec codec = new Lucene84Codec();
+ private final Codec codec = new Lucene86Codec();
@Override
protected Codec getCodec() {
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene86/TestLucene86PointsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene86/TestLucene86PointsFormat.java
new file mode 100644
index 000000000000..8d5ce08f346d
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene86/TestLucene86PointsFormat.java
@@ -0,0 +1,393 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene86;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.FilterCodec;
+import org.apache.lucene.codecs.PointsFormat;
+import org.apache.lucene.codecs.PointsReader;
+import org.apache.lucene.codecs.PointsWriter;
+import org.apache.lucene.document.BinaryPoint;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.BasePointsFormatTestCase;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.MockRandomMergePolicy;
+import org.apache.lucene.index.PointValues;
+import org.apache.lucene.index.PointValues.IntersectVisitor;
+import org.apache.lucene.index.PointValues.Relation;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.TestUtil;
+import org.apache.lucene.util.bkd.BKDWriter;
+
+public class TestLucene86PointsFormat extends BasePointsFormatTestCase {
+
+ private final Codec codec;
+ private final int maxPointsInLeafNode;
+
+ public TestLucene86PointsFormat() {
+ // standard issue
+ Codec defaultCodec = new Lucene86Codec();
+ if (random().nextBoolean()) {
+ // randomize parameters
+ maxPointsInLeafNode = TestUtil.nextInt(random(), 50, 500);
+ double maxMBSortInHeap = 3.0 + (3*random().nextDouble());
+ if (VERBOSE) {
+ System.out.println("TEST: using Lucene60PointsFormat with maxPointsInLeafNode=" + maxPointsInLeafNode + " and maxMBSortInHeap=" + maxMBSortInHeap);
+ }
+
+ // sneaky impersonation!
+ codec = new FilterCodec(defaultCodec.getName(), defaultCodec) {
+ @Override
+ public PointsFormat pointsFormat() {
+ return new PointsFormat() {
+ @Override
+ public PointsWriter fieldsWriter(SegmentWriteState writeState) throws IOException {
+ return new Lucene86PointsWriter(writeState, maxPointsInLeafNode, maxMBSortInHeap);
+ }
+
+ @Override
+ public PointsReader fieldsReader(SegmentReadState readState) throws IOException {
+ return new Lucene86PointsReader(readState);
+ }
+ };
+ }
+ };
+ } else {
+ // standard issue
+ codec = defaultCodec;
+ maxPointsInLeafNode = BKDWriter.DEFAULT_MAX_POINTS_IN_LEAF_NODE;
+ }
+ }
+
+ @Override
+ protected Codec getCodec() {
+ return codec;
+ }
+
+ @Override
+ public void testMergeStability() throws Exception {
+ assumeFalse("TODO: mess with the parameters and test gets angry!", codec instanceof FilterCodec);
+ super.testMergeStability();
+ }
+
+ public void testEstimatePointCount() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = newIndexWriterConfig();
+ // Avoid mockRandomMP since it may cause non-optimal merges that make the
+ // number of points per leaf hard to predict
+ while (iwc.getMergePolicy() instanceof MockRandomMergePolicy) {
+ iwc.setMergePolicy(newMergePolicy());
+ }
+ IndexWriter w = new IndexWriter(dir, iwc);
+ byte[] pointValue = new byte[3];
+ byte[] uniquePointValue = new byte[3];
+ random().nextBytes(uniquePointValue);
+ final int numDocs = TEST_NIGHTLY ? atLeast(10000) : atLeast(500); // at night, make sure we have several leaves
+ final boolean multiValues = random().nextBoolean();
+ for (int i = 0; i < numDocs; ++i) {
+ Document doc = new Document();
+ if (i == numDocs / 2) {
+ doc.add(new BinaryPoint("f", uniquePointValue));
+ } else {
+ final int numValues = (multiValues) ? TestUtil.nextInt(random(), 2, 100) : 1;
+ for (int j = 0; j < numValues; j ++) {
+ do {
+ random().nextBytes(pointValue);
+ } while (Arrays.equals(pointValue, uniquePointValue));
+ doc.add(new BinaryPoint("f", pointValue));
+ }
+ }
+ w.addDocument(doc);
+ }
+ w.forceMerge(1);
+ final IndexReader r = DirectoryReader.open(w);
+ w.close();
+ final LeafReader lr = getOnlyLeafReader(r);
+ PointValues points = lr.getPointValues("f");
+
+ // If all points match, then the point count is numLeaves * maxPointsInLeafNode
+ final int numLeaves = (int) Math.ceil((double) points.size() / maxPointsInLeafNode);
+
+ IntersectVisitor allPointsVisitor = new IntersectVisitor() {
+ @Override
+ public void visit(int docID, byte[] packedValue) throws IOException {}
+
+ @Override
+ public void visit(int docID) throws IOException {}
+
+ @Override
+ public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
+ return Relation.CELL_INSIDE_QUERY;
+ }
+ };
+
+ assertEquals(numLeaves * maxPointsInLeafNode, points.estimatePointCount(allPointsVisitor));
+ assertEquals(numDocs, points.estimateDocCount(allPointsVisitor));
+
+ IntersectVisitor noPointsVisitor = new IntersectVisitor() {
+ @Override
+ public void visit(int docID, byte[] packedValue) throws IOException {}
+
+ @Override
+ public void visit(int docID) throws IOException {}
+
+ @Override
+ public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
+ return Relation.CELL_OUTSIDE_QUERY;
+ }
+ };
+
+ // Return 0 if no points match
+ assertEquals(0, points.estimatePointCount(noPointsVisitor));
+ assertEquals(0, points.estimateDocCount(noPointsVisitor));
+
+ IntersectVisitor onePointMatchVisitor = new IntersectVisitor() {
+ @Override
+ public void visit(int docID, byte[] packedValue) throws IOException {}
+
+ @Override
+ public void visit(int docID) throws IOException {}
+
+ @Override
+ public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
+ if (Arrays.compareUnsigned(uniquePointValue, 0, 3, maxPackedValue, 0, 3) > 0 ||
+ Arrays.compareUnsigned(uniquePointValue, 0, 3, minPackedValue, 0, 3) < 0) {
+ return Relation.CELL_OUTSIDE_QUERY;
+ }
+ return Relation.CELL_CROSSES_QUERY;
+ }
+ };
+
+ // If only one point matches, then the point count is (maxPointsInLeafNode + 1) / 2
+ // in general, or maybe 2x that if the point is a split value
+ final long pointCount = points.estimatePointCount(onePointMatchVisitor);
+ assertTrue(""+pointCount,
+ pointCount == (maxPointsInLeafNode + 1) / 2 || // common case
+ pointCount == 2*((maxPointsInLeafNode + 1) / 2)); // if the point is a split value
+
+ final long docCount = points.estimateDocCount(onePointMatchVisitor);
+
+ if (multiValues) {
+ assertEquals(docCount, (long) (docCount * (1d - Math.pow( (numDocs - pointCount) / points.size() , points.size() / docCount))));
+ } else {
+ assertEquals(Math.min(pointCount, numDocs), docCount);
+ }
+ r.close();
+ dir.close();
+ }
+
+ // The tree is always balanced in the N dims case, and leaves are
+ // not all full so things are a bit different
+ public void testEstimatePointCount2Dims() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig());
+ byte[][] pointValue = new byte[2][];
+ pointValue[0] = new byte[3];
+ pointValue[1] = new byte[3];
+ byte[][] uniquePointValue = new byte[2][];
+ uniquePointValue[0] = new byte[3];
+ uniquePointValue[1] = new byte[3];
+ random().nextBytes(uniquePointValue[0]);
+ random().nextBytes(uniquePointValue[1]);
+ final int numDocs = TEST_NIGHTLY? atLeast(10000) : atLeast(1000); // in nightly, make sure we have several leaves
+ final boolean multiValues = random().nextBoolean();
+ for (int i = 0; i < numDocs; ++i) {
+ Document doc = new Document();
+ if (i == numDocs / 2) {
+ doc.add(new BinaryPoint("f", uniquePointValue));
+ } else {
+ final int numValues = (multiValues) ? TestUtil.nextInt(random(), 2, 100) : 1;
+ for (int j = 0; j < numValues; j ++) {
+ do {
+ random().nextBytes(pointValue[0]);
+ random().nextBytes(pointValue[1]);
+ } while (Arrays.equals(pointValue[0], uniquePointValue[0]) || Arrays.equals(pointValue[1], uniquePointValue[1]));
+ doc.add(new BinaryPoint("f", pointValue));
+ }
+ }
+ w.addDocument(doc);
+ }
+ w.forceMerge(1);
+ final IndexReader r = DirectoryReader.open(w);
+ w.close();
+ final LeafReader lr = getOnlyLeafReader(r);
+ PointValues points = lr.getPointValues("f");
+
+ IntersectVisitor allPointsVisitor = new IntersectVisitor() {
+ @Override
+ public void visit(int docID, byte[] packedValue) throws IOException {}
+
+ @Override
+ public void visit(int docID) throws IOException {}
+
+ @Override
+ public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
+ return Relation.CELL_INSIDE_QUERY;
+ }
+ };
+
+ // If all points match, then the point count is numLeaves * maxPointsInLeafNode
+ final int numLeaves = (int) Math.ceil((double) points.size() / maxPointsInLeafNode);
+
+ assertEquals(numLeaves * maxPointsInLeafNode, points.estimatePointCount(allPointsVisitor));
+ assertEquals(numDocs, points.estimateDocCount(allPointsVisitor));
+
+ IntersectVisitor noPointsVisitor = new IntersectVisitor() {
+ @Override
+ public void visit(int docID, byte[] packedValue) throws IOException {}
+
+ @Override
+ public void visit(int docID) throws IOException {}
+
+ @Override
+ public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
+ return Relation.CELL_OUTSIDE_QUERY;
+ }
+ };
+
+ // Return 0 if no points match
+ assertEquals(0, points.estimatePointCount(noPointsVisitor));
+ assertEquals(0, points.estimateDocCount(noPointsVisitor));
+
+ IntersectVisitor onePointMatchVisitor = new IntersectVisitor() {
+ @Override
+ public void visit(int docID, byte[] packedValue) throws IOException {}
+
+ @Override
+ public void visit(int docID) throws IOException {}
+
+ @Override
+ public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
+ for (int dim = 0; dim < 2; ++dim) {
+ if (Arrays.compareUnsigned(uniquePointValue[dim], 0, 3, maxPackedValue, dim * 3, dim * 3 + 3) > 0 ||
+ Arrays.compareUnsigned(uniquePointValue[dim], 0, 3, minPackedValue, dim * 3, dim * 3 + 3) < 0) {
+ return Relation.CELL_OUTSIDE_QUERY;
+ }
+ }
+ return Relation.CELL_CROSSES_QUERY;
+ }
+ };
+
+ final long pointCount = points.estimatePointCount(onePointMatchVisitor);
+ // The number of matches needs to be multiple of count per leaf
+ final long countPerLeaf = (maxPointsInLeafNode + 1) / 2;
+ assertTrue(""+pointCount, pointCount % countPerLeaf == 0);
+ // in extreme cases, a point can be be shared by 4 leaves
+ assertTrue(""+pointCount, pointCount / countPerLeaf <= 4 && pointCount / countPerLeaf >= 1);
+
+ final long docCount = points.estimateDocCount(onePointMatchVisitor);
+ if (multiValues) {
+ assertEquals(docCount, (long) (docCount * (1d - Math.pow( (numDocs - pointCount) / points.size() , points.size() / docCount))));
+ } else {
+ assertEquals(Math.min(pointCount, numDocs), docCount);
+ }
+ r.close();
+ dir.close();
+ }
+
+ public void testDocCountEdgeCases() {
+ PointValues values = getPointValues(Long.MAX_VALUE, 1, Long.MAX_VALUE);
+ long docs = values.estimateDocCount(null);
+ assertEquals(1, docs);
+ values = getPointValues(Long.MAX_VALUE, 1, 1);
+ docs = values.estimateDocCount(null);
+ assertEquals(1, docs);
+ values = getPointValues(Long.MAX_VALUE, Integer.MAX_VALUE, Long.MAX_VALUE);
+ docs = values.estimateDocCount(null);
+ assertEquals(Integer.MAX_VALUE, docs);
+ values = getPointValues(Long.MAX_VALUE, Integer.MAX_VALUE, Long.MAX_VALUE / 2);
+ docs = values.estimateDocCount(null);
+ assertEquals(Integer.MAX_VALUE, docs);
+ values = getPointValues(Long.MAX_VALUE, Integer.MAX_VALUE, 1);
+ docs = values.estimateDocCount(null);
+ assertEquals(1, docs);
+ }
+
+ public void testRandomDocCount() {
+ for (int i = 0; i < 100; i++) {
+ long size = TestUtil.nextLong(random(), 1, Long.MAX_VALUE);
+ int maxDoc = (size > Integer.MAX_VALUE) ? Integer.MAX_VALUE : Math.toIntExact(size);
+ int docCount = TestUtil.nextInt(random(), 1, maxDoc);
+ long estimatedPointCount = TestUtil.nextLong(random(), 0, size);
+ PointValues values = getPointValues(size, docCount, estimatedPointCount);
+ long docs = values.estimateDocCount(null);
+ assertTrue(docs <= estimatedPointCount);
+ assertTrue(docs <= maxDoc);
+ assertTrue(docs >= estimatedPointCount / (size/docCount));
+ }
+ }
+
+
+ private PointValues getPointValues(long size, int docCount, long estimatedPointCount) {
+ return new PointValues() {
+ @Override
+ public void intersect(IntersectVisitor visitor) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long estimatePointCount(IntersectVisitor visitor) {
+ return estimatedPointCount;
+ }
+
+ @Override
+ public byte[] getMinPackedValue() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public byte[] getMaxPackedValue() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int getNumDimensions() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int getNumIndexDimensions() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int getBytesPerDimension() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long size() {
+ return size;
+ }
+
+ @Override
+ public int getDocCount() {
+ return docCount;
+ }
+ };
+ }
+
+}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene86/TestLucene86SegmentInfoFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene86/TestLucene86SegmentInfoFormat.java
new file mode 100644
index 000000000000..e462d3f4ac45
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene86/TestLucene86SegmentInfoFormat.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.codecs.lucene86;
+
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.index.BaseSegmentInfoFormatTestCase;
+import org.apache.lucene.util.TestUtil;
+import org.apache.lucene.util.Version;
+
+public class TestLucene86SegmentInfoFormat extends BaseSegmentInfoFormatTestCase {
+
+ @Override
+ protected Version[] getVersions() {
+ return new Version[] { Version.LATEST };
+ }
+
+ @Override
+ protected Codec getCodec() {
+ return TestUtil.getDefaultCodec();
+ }
+
+}
diff --git a/lucene/core/src/test/org/apache/lucene/document/TestDocument.java b/lucene/core/src/test/org/apache/lucene/document/TestDocument.java
index 50c1ed0a919a..ac5c43452b54 100644
--- a/lucene/core/src/test/org/apache/lucene/document/TestDocument.java
+++ b/lucene/core/src/test/org/apache/lucene/document/TestDocument.java
@@ -214,14 +214,10 @@ public void testGetValuesForIndexedDocument() throws Exception {
public void testGetValues() {
Document doc = makeDocumentWithFields();
- assertEquals(new String[] {"test1", "test2"},
- doc.getValues("keyword"));
- assertEquals(new String[] {"test1", "test2"},
- doc.getValues("text"));
- assertEquals(new String[] {"test1", "test2"},
- doc.getValues("unindexed"));
- assertEquals(new String[0],
- doc.getValues("nope"));
+ assertArrayEquals(new String[] {"test1", "test2"}, doc.getValues("keyword"));
+ assertArrayEquals(new String[] {"test1", "test2"}, doc.getValues("text"));
+ assertArrayEquals(new String[] {"test1", "test2"}, doc.getValues("unindexed"));
+ assertArrayEquals(new String[0], doc.getValues("nope"));
}
public void testPositionIncrementMultiFields() throws Exception {
diff --git a/lucene/core/src/test/org/apache/lucene/geo/TestTessellator.java b/lucene/core/src/test/org/apache/lucene/geo/TestTessellator.java
index 0d1f270776e5..2410ba13a5db 100644
--- a/lucene/core/src/test/org/apache/lucene/geo/TestTessellator.java
+++ b/lucene/core/src/test/org/apache/lucene/geo/TestTessellator.java
@@ -573,6 +573,14 @@ public void testComplexPolygon40() throws Exception {
}
}
+ public void testComplexPolygon41() throws Exception {
+ String wkt = "POLYGON((-1.569137181294115 54.4855283059375, -1.5692505240440333 54.48535373128068, -1.5684753656387294 54.48534438253056, -1.568606793880459 54.485674703738624, -1.5694141387939453 54.48611720532629, -1.569137181294115 54.4855283059375)," +
+ "(-1.569137181294115 54.4855283059375, -1.5690783030431206 54.48545352137167, -1.5689449291711688 54.48547663706703, -1.569137181294115 54.4855283059375)," +
+ "(-1.5689449291711688 54.48547663706703, -1.5689437289004642 54.48535482680399, -1.5687730514221028 54.48538045082698, -1.5689449291711688 54.48547663706703)," +
+ "(-1.5689449291711688 54.48547663706703, -1.5689879483854345 54.485580118416785, -1.5687756358893499 54.485612860811244, -1.568765285875931 54.485496217554285, -1.5689449291711688 54.48547663706703))";
+ checkPolygon(wkt);
+ }
+
private void checkPolygon(String wkt) throws Exception {
Polygon polygon = (Polygon) SimpleWKTShapeParser.parse(wkt);
List tessellation = Tessellator.tessellate(polygon);
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectBitFlips.java b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectBitFlips.java
new file mode 100644
index 000000000000..f63387b74d03
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectBitFlips.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.index;
+
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.store.BaseDirectoryWrapper;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.LineFileDocs;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
+import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems;
+import org.apache.lucene.util.TestUtil;
+
+/**
+ * Test that the default codec detects bit flips at open or checkIntegrity time.
+ */
+@SuppressFileSystems("ExtrasFS")
+@AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/LUCENE-9356")
+public class TestAllFilesDetectBitFlips extends LuceneTestCase {
+
+ public void test() throws Exception {
+ doTest(false);
+ }
+
+ public void testCFS() throws Exception {
+ doTest(true);
+ }
+
+ public void doTest(boolean cfs) throws Exception {
+ Directory dir = newDirectory();
+
+ IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
+ conf.setCodec(TestUtil.getDefaultCodec());
+
+ if (cfs == false) {
+ conf.setUseCompoundFile(false);
+ conf.getMergePolicy().setNoCFSRatio(0.0);
+ }
+
+ RandomIndexWriter riw = new RandomIndexWriter(random(), dir, conf);
+ // Use LineFileDocs so we (hopefully) get most Lucene features
+ // tested, e.g. IntPoint was recently added to it:
+ LineFileDocs docs = new LineFileDocs(random());
+ for (int i = 0; i < 100; i++) {
+ riw.addDocument(docs.nextDoc());
+ if (random().nextInt(7) == 0) {
+ riw.commit();
+ }
+ if (random().nextInt(20) == 0) {
+ riw.deleteDocuments(new Term("docid", Integer.toString(i)));
+ }
+ if (random().nextInt(15) == 0) {
+ riw.updateNumericDocValue(new Term("docid", Integer.toString(i)), "docid_intDV", Long.valueOf(i));
+ }
+ }
+ if (TEST_NIGHTLY == false) {
+ riw.forceMerge(1);
+ }
+ riw.close();
+ checkBitFlips(dir);
+ dir.close();
+ }
+
+ private void checkBitFlips(Directory dir) throws IOException {
+ for(String name : dir.listAll()) {
+ if (name.equals(IndexWriter.WRITE_LOCK_NAME) == false) {
+ corruptFile(dir, name);
+ }
+ }
+ }
+
+ private void corruptFile(Directory dir, String victim) throws IOException {
+ try (BaseDirectoryWrapper dirCopy = newDirectory()) {
+ dirCopy.setCheckIndexOnClose(false);
+
+ long victimLength = dir.fileLength(victim);
+ long flipOffset = TestUtil.nextLong(random(), 0, victimLength - 1);
+
+ if (VERBOSE) {
+ System.out.println("TEST: now corrupt file " + victim + " by changing byte at offset " + flipOffset + " (length= " + victimLength + ")");
+ }
+
+ for(String name : dir.listAll()) {
+ if (name.equals(victim) == false) {
+ dirCopy.copyFrom(dir, name, name, IOContext.DEFAULT);
+ } else {
+ try (IndexOutput out = dirCopy.createOutput(name, IOContext.DEFAULT);
+ IndexInput in = dir.openInput(name, IOContext.DEFAULT)) {
+ out.copyBytes(in, flipOffset);
+ out.writeByte((byte) (in.readByte() + TestUtil.nextInt(random(), 0x01, 0xFF)));
+ out.copyBytes(in, victimLength - flipOffset - 1);
+ }
+ try (IndexInput in = dirCopy.openInput(name, IOContext.DEFAULT)) {
+ try {
+ CodecUtil.checksumEntireFile(in);
+ System.out.println("TEST: changing a byte in " + victim + " did not update the checksum)");
+ return;
+ } catch (CorruptIndexException e) {
+ // ok
+ }
+ }
+ }
+ dirCopy.sync(Collections.singleton(name));
+ }
+
+ // corruption must be detected
+ expectThrowsAnyOf(Arrays.asList(CorruptIndexException.class, IndexFormatTooOldException.class, IndexFormatTooNewException.class),
+ () -> {
+ try (IndexReader reader = DirectoryReader.open(dirCopy)) {
+ for (LeafReaderContext context : reader.leaves()) {
+ context.reader().checkIntegrity();
+ }
+ }
+ }
+ );
+ }
+ }
+}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java b/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java
index 050aed7864a9..a31194caf7ac 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java
@@ -113,9 +113,13 @@ protected boolean isOK(Throwable th) {
ioe.printStackTrace(System.out);
}
failure.clearDoFail();
- assertTrue(writer.isClosed());
+ // make sure we are closed or closing - if we are unlucky a merge does
+ // the actual closing for us. this is rare but might happen since the
+ // tragicEvent is checked by IFD and that might throw during a merge
+ expectThrows(AlreadyClosedException.class, writer::ensureOpen);
// Abort should have closed the deleter:
assertTrue(writer.isDeleterClosed());
+ writer.close(); // now wait for the close to actually happen if a merge thread did the close.
break outer;
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java b/lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java
index 4b82800fd188..7fdad3ba2de9 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java
@@ -538,7 +538,7 @@ public CodecReader wrapForMerge(CodecReader reader) throws IOException {
}
@Override
- public void mergeFinished() throws IOException {
+ public void mergeFinished(boolean success) throws IOException {
Throwable th = null;
for (ParallelLeafReader r : parallelReaders) {
try {
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
index ae944de1bc32..042e2a80ff58 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
@@ -2146,7 +2146,7 @@ public void testBadSort() throws Exception {
IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
iwc.setIndexSort(Sort.RELEVANCE);
});
- assertEquals("invalid SortField type: must be one of [STRING, INT, FLOAT, LONG, DOUBLE] but got: ", expected.getMessage());
+ assertEquals("Cannot sort index with sort field ", expected.getMessage());
}
// you can't change the index sort on an existing index:
@@ -2498,6 +2498,7 @@ public void testRandom3() throws Exception {
System.out.println(" float=" + docValues.floatValue);
System.out.println(" double=" + docValues.doubleValue);
System.out.println(" bytes=" + new BytesRef(docValues.bytesValue));
+ System.out.println(" mvf=" + Arrays.toString(docValues.floatValues));
}
Document doc = new Document();
@@ -2741,7 +2742,7 @@ public void testWrongSortFieldType() throws Exception {
Document doc = new Document();
doc.add(dvs.get(j));
IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> w.addDocument(doc));
- assertThat(exc.getMessage(), containsString("invalid doc value type"));
+ assertThat(exc.getMessage(), containsString("expected field [field] to be "));
doc.clear();
doc.add(dvs.get(i));
w.addDocument(doc);
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
index 8fb1ce5d75f7..5c6164a254ca 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
@@ -344,7 +344,7 @@ private static int getSegmentCount(Directory dir) throws IOException {
// Make sure it's OK to change RAM buffer size and
// maxBufferedDocs in a write session
public void testChangingRAMBuffer() throws IOException {
- Directory dir = newDirectory();
+ Directory dir = newDirectory();
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
writer.getConfig().setMaxBufferedDocs(10);
writer.getConfig().setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH);
@@ -607,7 +607,7 @@ public void testVariableSchema() throws Exception {
doc.add(newField("content4", contents, customType));
type = customType;
} else
- type = TextField.TYPE_NOT_STORED;
+ type = TextField.TYPE_NOT_STORED;
doc.add(newTextField("content1", contents, Field.Store.NO));
doc.add(newField("content3", "", customType));
doc.add(newField("content5", "", type));
@@ -663,13 +663,13 @@ public void testEmptyFieldName() throws IOException {
writer.close();
dir.close();
}
-
+
public void testEmptyFieldNameTerms() throws IOException {
Directory dir = newDirectory();
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
Document doc = new Document();
doc.add(newTextField("", "a b c", Field.Store.NO));
- writer.addDocument(doc);
+ writer.addDocument(doc);
writer.close();
DirectoryReader reader = DirectoryReader.open(dir);
LeafReader subreader = getOnlyLeafReader(reader);
@@ -681,7 +681,7 @@ public void testEmptyFieldNameTerms() throws IOException {
reader.close();
dir.close();
}
-
+
public void testEmptyFieldNameWithEmptyTerm() throws IOException {
Directory dir = newDirectory();
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
@@ -690,7 +690,7 @@ public void testEmptyFieldNameWithEmptyTerm() throws IOException {
doc.add(newStringField("", "a", Field.Store.NO));
doc.add(newStringField("", "b", Field.Store.NO));
doc.add(newStringField("", "c", Field.Store.NO));
- writer.addDocument(doc);
+ writer.addDocument(doc);
writer.close();
DirectoryReader reader = DirectoryReader.open(dir);
LeafReader subreader = getOnlyLeafReader(reader);
@@ -834,7 +834,7 @@ public void testDeadlock() throws Exception {
customType.setStoreTermVectors(true);
customType.setStoreTermVectorPositions(true);
customType.setStoreTermVectorOffsets(true);
-
+
doc.add(newField("content", "aaa bbb ccc ddd eee fff ggg hhh iii", customType));
writer.addDocument(doc);
writer.addDocument(doc);
@@ -922,7 +922,7 @@ public void run() {
// open/close slowly sometimes
dir.setUseSlowOpenClosers(true);
-
+
// throttle a little
dir.setThrottling(MockDirectoryWrapper.Throttling.SOMETIMES);
@@ -1148,7 +1148,7 @@ public void testIndexStoreCombos() throws Exception {
FieldType customType = new FieldType(StoredField.TYPE);
customType.setTokenized(true);
-
+
Field f = new Field("binary", b, 10, 17, customType);
// TODO: this is evil, changing the type after creating the field:
customType.setIndexOptions(IndexOptions.DOCS);
@@ -1157,7 +1157,7 @@ public void testIndexStoreCombos() throws Exception {
f.setTokenStream(doc1field1);
FieldType customType2 = new FieldType(TextField.TYPE_STORED);
-
+
Field f2 = newField("string", "value", customType2);
final MockTokenizer doc1field2 = new MockTokenizer(MockTokenizer.WHITESPACE, false);
doc1field2.setReader(new StringReader("doc1field2"));
@@ -1233,7 +1233,7 @@ public void testNoDocsIndex() throws Throwable {
public void testDeleteUnusedFiles() throws Exception {
assumeFalse("test relies on exact filenames", Codec.getDefault() instanceof SimpleTextCodec);
assumeWorkingMMapOnWindows();
-
+
for(int iter=0;iter<2;iter++) {
// relies on windows semantics
Path path = createTempDir();
@@ -1250,7 +1250,7 @@ public void testDeleteUnusedFiles() throws Exception {
}
MergePolicy mergePolicy = newLogMergePolicy(true);
-
+
// This test expects all of its segments to be in CFS
mergePolicy.setNoCFSRatio(1.0);
mergePolicy.setMaxCFSSegmentSizeMB(Double.POSITIVE_INFINITY);
@@ -1338,7 +1338,7 @@ public void testDeleteUnusedFiles2() throws Exception {
customType.setStoreTermVectors(true);
customType.setStoreTermVectorPositions(true);
customType.setStoreTermVectorOffsets(true);
-
+
doc.add(newField("c", "val", customType));
writer.addDocument(doc);
writer.commit();
@@ -1379,7 +1379,7 @@ public void testEmptyDirRollback() throws Exception {
// indexed, flushed (but not committed) and then IW rolls back, then no
// files are left in the Directory.
Directory dir = newDirectory();
-
+
String[] origFiles = dir.listAll();
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
.setMaxBufferedDocs(2)
@@ -1409,8 +1409,8 @@ public void testEmptyDirRollback() throws Exception {
// Adding just one document does not call flush yet.
int computedExtraFileCount = 0;
for (String file : dir.listAll()) {
- if (IndexWriter.WRITE_LOCK_NAME.equals(file) ||
- file.startsWith(IndexFileNames.SEGMENTS) ||
+ if (IndexWriter.WRITE_LOCK_NAME.equals(file) ||
+ file.startsWith(IndexFileNames.SEGMENTS) ||
IndexFileNames.CODEC_FILE_PATTERN.matcher(file).matches()) {
if (file.lastIndexOf('.') < 0
// don't count stored fields and term vectors in, or any temporary files they might
@@ -1458,7 +1458,7 @@ public void testNoUnwantedTVFiles() throws Exception {
FieldType customType3 = new FieldType(TextField.TYPE_STORED);
customType3.setTokenized(false);
customType3.setOmitNorms(true);
-
+
for (int i=0; i<2; i++) {
Document doc = new Document();
doc.add(new Field("id", Integer.toString(i)+BIG, customType3));
@@ -1478,7 +1478,7 @@ public void testNoUnwantedTVFiles() throws Exception {
SegmentReader sr = (SegmentReader) ctx.reader();
assertFalse(sr.getFieldInfos().hasVectors());
}
-
+
r0.close();
dir.close();
}
@@ -1501,7 +1501,7 @@ public StringSplitTokenizer() {
@Override
public final boolean incrementToken() {
- clearAttributes();
+ clearAttributes();
if (upto < tokens.length) {
termAtt.setEmpty();
termAtt.append(tokens[upto]);
@@ -1724,7 +1724,7 @@ public void testPrepareCommitThenRollback2() throws Exception {
r.close();
dir.close();
}
-
+
public void testDontInvokeAnalyzerForUnAnalyzedFields() throws Exception {
Analyzer analyzer = new Analyzer() {
@Override
@@ -1759,13 +1759,13 @@ public int getOffsetGap(String fieldName) {
w.close();
dir.close();
}
-
+
//LUCENE-1468 -- make sure opening an IndexWriter with
// create=true does not remove non-index files
-
+
public void testOtherFiles() throws Throwable {
Directory dir = newDirectory();
- IndexWriter iw = new IndexWriter(dir,
+ IndexWriter iw = new IndexWriter(dir,
newIndexWriterConfig(new MockAnalyzer(random())));
iw.addDocument(new Document());
iw.close();
@@ -1774,15 +1774,15 @@ public void testOtherFiles() throws Throwable {
IndexOutput out = dir.createOutput("myrandomfile", newIOContext(random()));
out.writeByte((byte) 42);
out.close();
-
+
new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))).close();
-
+
assertTrue(slowFileExists(dir, "myrandomfile"));
} finally {
dir.close();
}
}
-
+
// LUCENE-3849
public void testStopwordsPosIncHole() throws Exception {
Directory dir = newDirectory();
@@ -1811,7 +1811,7 @@ protected TokenStreamComponents createComponents(String fieldName) {
ir.close();
dir.close();
}
-
+
// LUCENE-3849
public void testStopwordsPosIncHole2() throws Exception {
// use two stopfilters for testing here
@@ -1843,23 +1843,23 @@ protected TokenStreamComponents createComponents(String fieldName) {
ir.close();
dir.close();
}
-
+
// LUCENE-4575
public void testCommitWithUserDataOnly() throws Exception {
Directory dir = newDirectory();
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(null));
writer.commit(); // first commit to complete IW create transaction.
-
+
// this should store the commit data, even though no other changes were made
writer.setLiveCommitData(new HashMap() {{
put("key", "value");
}}.entrySet());
writer.commit();
-
+
DirectoryReader r = DirectoryReader.open(dir);
assertEquals("value", r.getIndexCommit().getUserData().get("key"));
r.close();
-
+
// now check setCommitData and prepareCommit/commit sequence
writer.setLiveCommitData(new HashMap() {{
put("key", "value1");
@@ -1873,7 +1873,7 @@ public void testCommitWithUserDataOnly() throws Exception {
r = DirectoryReader.open(dir);
assertEquals("value1", r.getIndexCommit().getUserData().get("key"));
r.close();
-
+
// now should commit the second commitData - there was a bug where
// IndexWriter.finishCommit overrode the second commitData
writer.commit();
@@ -1881,7 +1881,7 @@ public void testCommitWithUserDataOnly() throws Exception {
assertEquals("IndexWriter.finishCommit may have overridden the second commitData",
"value2", r.getIndexCommit().getUserData().get("key"));
r.close();
-
+
writer.close();
dir.close();
}
@@ -1896,7 +1896,7 @@ private Map getLiveCommitData(IndexWriter writer) {
}
return data;
}
-
+
@Test
public void testGetCommitData() throws Exception {
Directory dir = newDirectory();
@@ -1906,16 +1906,16 @@ public void testGetCommitData() throws Exception {
}}.entrySet());
assertEquals("value", getLiveCommitData(writer).get("key"));
writer.close();
-
+
// validate that it's also visible when opening a new IndexWriter
writer = new IndexWriter(dir, newIndexWriterConfig(null)
.setOpenMode(OpenMode.APPEND));
assertEquals("value", getLiveCommitData(writer).get("key"));
writer.close();
-
+
dir.close();
}
-
+
public void testNullAnalyzer() throws IOException {
Directory dir = newDirectory();
IndexWriterConfig iwConf = newIndexWriterConfig(null);
@@ -1942,7 +1942,7 @@ public void testNullAnalyzer() throws IOException {
iw.close();
dir.close();
}
-
+
public void testNullDocument() throws IOException {
Directory dir = newDirectory();
RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
@@ -1967,7 +1967,7 @@ public void testNullDocument() throws IOException {
iw.close();
dir.close();
}
-
+
public void testNullDocuments() throws IOException {
Directory dir = newDirectory();
RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
@@ -1992,7 +1992,7 @@ public void testNullDocuments() throws IOException {
iw.close();
dir.close();
}
-
+
public void testIterableFieldThrowsException() throws IOException {
Directory dir = newDirectory();
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
@@ -2000,7 +2000,7 @@ public void testIterableFieldThrowsException() throws IOException {
int docCount = 0;
int docId = 0;
Set liveIds = new HashSet<>();
- for (int i = 0; i < iters; i++) {
+ for (int i = 0; i < iters; i++) {
int numDocs = atLeast(4);
for (int j = 0; j < numDocs; j++) {
String id = Integer.toString(docId++);
@@ -2008,7 +2008,7 @@ public void testIterableFieldThrowsException() throws IOException {
fields.add(new StringField("id", id, Field.Store.YES));
fields.add(new StringField("foo", TestUtil.randomSimpleString(random()), Field.Store.NO));
docId++;
-
+
boolean success = false;
try {
w.addDocument(new RandomFailingIterable(fields, random()));
@@ -2040,7 +2040,7 @@ public void testIterableFieldThrowsException() throws IOException {
w.close();
IOUtils.close(reader, dir);
}
-
+
public void testIterableThrowsException() throws IOException {
Directory dir = newDirectory();
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
@@ -2088,7 +2088,7 @@ public void testIterableThrowsException() throws IOException {
w.close();
IOUtils.close(reader, dir);
}
-
+
public void testIterableThrowsException2() throws IOException {
Directory dir = newDirectory();
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
@@ -2128,7 +2128,7 @@ public RandomFailingIterable(Iterable list, Random random) {
this.list = list;
this.failOn = random.nextInt(5);
}
-
+
@Override
public Iterator iterator() {
final Iterator docIter = list.iterator();
@@ -2254,7 +2254,7 @@ public void testHasUncommittedChanges() throws IOException {
writer.close();
dir.close();
}
-
+
public void testMergeAllDeleted() throws IOException {
Directory dir = newDirectory();
IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
@@ -2477,12 +2477,12 @@ public void testIds() throws Exception {
IndexWriter w = new IndexWriter(d, newIndexWriterConfig(new MockAnalyzer(random())));
w.addDocument(new Document());
w.close();
-
+
SegmentInfos sis = SegmentInfos.readLatestCommit(d);
byte[] id1 = sis.getId();
assertNotNull(id1);
assertEquals(StringHelper.ID_LENGTH, id1.length);
-
+
byte[] id2 = sis.info(0).info.getId();
byte[] sciId2 = sis.info(0).getId();
assertNotNull(id2);
@@ -2514,7 +2514,7 @@ public void testIds() throws Exception {
ids.add(id);
}
}
-
+
public void testEmptyNorm() throws Exception {
Directory d = newDirectory();
IndexWriter w = new IndexWriter(d, newIndexWriterConfig(new MockAnalyzer(random())));
@@ -2579,7 +2579,7 @@ public void testNRTSegmentsFile() throws Exception {
assertEquals(1, r2.getIndexCommit().getGeneration());
assertEquals("segments_1", r2.getIndexCommit().getSegmentsFileName());
r2.close();
-
+
// make a change and another commit
w.addDocument(new Document());
w.commit();
@@ -2866,7 +2866,7 @@ public void testLeftoverTempFiles() throws Exception {
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
IndexWriter w = new IndexWriter(dir, iwc);
w.close();
-
+
IndexOutput out = dir.createTempOutput("_0", "bkd", IOContext.DEFAULT);
String tempName = out.getName();
out.close();
@@ -3151,7 +3151,7 @@ public void testSoftUpdateDocuments() throws IOException {
expectThrows(IllegalArgumentException.class, () -> {
writer.softUpdateDocument(null, new Document(), new NumericDocValuesField("soft_delete", 1));
});
-
+
expectThrows(IllegalArgumentException.class, () -> {
writer.softUpdateDocument(new Term("id", "1"), new Document());
});
@@ -4167,4 +4167,76 @@ public void testSegmentCommitInfoId() throws IOException {
}
}
}
+
+ public void testMergeZeroDocsMergeIsClosedOnce() throws IOException {
+ LogDocMergePolicy keepAllSegments = new LogDocMergePolicy() {
+ @Override
+ public boolean keepFullyDeletedSegment(IOSupplier readerIOSupplier) {
+ return true;
+ }
+ };
+ try (Directory dir = newDirectory()) {
+ try (IndexWriter writer = new IndexWriter(dir,
+ new IndexWriterConfig().setMergePolicy(new OneMergeWrappingMergePolicy(keepAllSegments, merge -> {
+ SetOnce onlyFinishOnce = new SetOnce<>();
+ return new MergePolicy.OneMerge(merge.segments) {
+ @Override
+ public void mergeFinished(boolean success) {
+ onlyFinishOnce.set(true);
+ }
+ };
+ })))) {
+ Document doc = new Document();
+ doc.add(new StringField("id", "1", Field.Store.NO));
+ writer.addDocument(doc);
+ writer.flush();
+ writer.addDocument(doc);
+ writer.flush();
+ writer.deleteDocuments(new Term("id", "1"));
+ writer.flush();
+ assertEquals(2, writer.getSegmentCount());
+ assertEquals(0, writer.getDocStats().numDocs);
+ assertEquals(2, writer.getDocStats().maxDoc);
+ writer.forceMerge(1);
+ }
+ }
+ }
+
+ public void testMergeOnCommitKeepFullyDeletedSegments() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = newIndexWriterConfig();
+ iwc.setMaxCommitMergeWaitSeconds(30);
+ iwc.mergePolicy = new FilterMergePolicy(newMergePolicy()) {
+ @Override
+ public boolean keepFullyDeletedSegment(IOSupplier readerIOSupplier) {
+ return true;
+ }
+
+ @Override
+ public MergeSpecification findFullFlushMerges(MergeTrigger mergeTrigger,
+ SegmentInfos segmentInfos,
+ MergeContext mergeContext) {
+ List fullyDeletedSegments = segmentInfos.asList().stream()
+ .filter(s -> s.info.maxDoc() - s.getDelCount() == 0)
+ .collect(Collectors.toList());
+ if (fullyDeletedSegments.isEmpty()) {
+ return null;
+ }
+ MergeSpecification spec = new MergeSpecification();
+ spec.add(new OneMerge(fullyDeletedSegments));
+ return spec;
+ }
+ };
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document d = new Document();
+ d.add(new StringField("id", "1", Field.Store.YES));
+ w.addDocument(d);
+ w.commit();
+ w.updateDocument(new Term("id", "1"), d);
+ w.commit();
+ try (DirectoryReader reader = w.getReader()) {
+ assertEquals(1, reader.numDocs());
+ }
+ IOUtils.close(w, dir);
+ }
}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
index bb16884269cc..e7dbfd294da4 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
@@ -702,8 +702,8 @@ private void doTestOperationsOnDiskFull(boolean updates) throws IOException {
}
dir.close();
- // Try again with 10 more bytes of free space:
- diskFree += 10;
+ // Try again with more bytes of free space:
+ diskFree += Math.max(10, diskFree >>> 3);
}
startDir.close();
}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
index ce591a280c6e..2577f6b28ecb 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
@@ -18,17 +18,42 @@
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
public class TestIndexWriterMergePolicy extends LuceneTestCase {
-
+
+ private static final MergePolicy MERGE_ON_COMMIT_POLICY = new LogDocMergePolicy() {
+ @Override
+ public MergeSpecification findFullFlushMerges(MergeTrigger mergeTrigger, SegmentInfos segmentInfos, MergeContext mergeContext) {
+ // Optimize down to a single segment on commit
+ if (mergeTrigger == MergeTrigger.COMMIT && segmentInfos.size() > 1) {
+ List nonMergingSegments = new ArrayList<>();
+ for (SegmentCommitInfo sci : segmentInfos) {
+ if (mergeContext.getMergingSegments().contains(sci) == false) {
+ nonMergingSegments.add(sci);
+ }
+ }
+ if (nonMergingSegments.size() > 1) {
+ MergeSpecification mergeSpecification = new MergeSpecification();
+ mergeSpecification.add(new OneMerge(nonMergingSegments));
+ return mergeSpecification;
+ }
+ }
+ return null;
+ }
+ };
+
// Test the normal case
public void testNormalCase() throws IOException {
Directory dir = newDirectory();
@@ -278,6 +303,50 @@ public void testSetters() {
assertSetters(new LogDocMergePolicy());
}
+ // Test basic semantics of merge on commit
+ public void testMergeOnCommit() throws IOException {
+ Directory dir = newDirectory();
+
+ IndexWriter firstWriter = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
+ .setMergePolicy(NoMergePolicy.INSTANCE));
+ for (int i = 0; i < 5; i++) {
+ TestIndexWriter.addDoc(firstWriter);
+ firstWriter.flush();
+ }
+ DirectoryReader firstReader = DirectoryReader.open(firstWriter);
+ assertEquals(5, firstReader.leaves().size());
+ firstReader.close();
+ firstWriter.close(); // When this writer closes, it does not merge on commit.
+
+ IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()))
+ .setMergePolicy(MERGE_ON_COMMIT_POLICY).setMaxCommitMergeWaitSeconds(30);
+
+
+ IndexWriter writerWithMergePolicy = new IndexWriter(dir, iwc);
+ writerWithMergePolicy.commit(); // No changes. Commit doesn't trigger a merge.
+
+ DirectoryReader unmergedReader = DirectoryReader.open(writerWithMergePolicy);
+ assertEquals(5, unmergedReader.leaves().size());
+ unmergedReader.close();
+
+ TestIndexWriter.addDoc(writerWithMergePolicy);
+ writerWithMergePolicy.commit(); // Doc added, do merge on commit.
+ assertEquals(1, writerWithMergePolicy.getSegmentCount()); //
+
+ DirectoryReader mergedReader = DirectoryReader.open(writerWithMergePolicy);
+ assertEquals(1, mergedReader.leaves().size());
+ mergedReader.close();
+
+ try (IndexReader reader = writerWithMergePolicy.getReader()) {
+ IndexSearcher searcher = new IndexSearcher(reader);
+ assertEquals(6, reader.numDocs());
+ assertEquals(6, searcher.count(new MatchAllDocsQuery()));
+ }
+
+ writerWithMergePolicy.close();
+ dir.close();
+ }
+
private void assertSetters(MergePolicy lmp) {
lmp.setMaxCFSSegmentSizeMB(2.0);
assertEquals(2.0, lmp.getMaxCFSSegmentSizeMB(), EPSILON);
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java
index 5f8650fe0aaa..228c34366db5 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java
@@ -809,11 +809,16 @@ private Directory getAssertNoDeletesDirectory(Directory directory) {
// Stress test reopen during add/delete
public void testDuringAddDelete() throws Exception {
Directory dir1 = newDirectory();
+ IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()))
+ .setMergePolicy(newLogMergePolicy(2));
+ if (TEST_NIGHTLY) {
+ // if we have a ton of iterations we need to make sure we don't do unnecessary
+ // extra flushing otherwise we will timeout on nightly
+ iwc.setRAMBufferSizeMB(IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB);
+ iwc.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ }
final IndexWriter writer = new IndexWriter(
- dir1,
- newIndexWriterConfig(new MockAnalyzer(random()))
- .setMergePolicy(newLogMergePolicy(2))
- );
+ dir1,iwc);
// create the index
createIndexNoClose(false, "test", writer);
@@ -822,7 +827,7 @@ public void testDuringAddDelete() throws Exception {
DirectoryReader r = writer.getReader();
final int iters = TEST_NIGHTLY ? 1000 : 10;
- final List excs = Collections.synchronizedList(new ArrayList());
+ final List excs = Collections.synchronizedList(new ArrayList<>());
final Thread[] threads = new Thread[numThreads];
final AtomicInteger remainingThreads = new AtomicInteger(numThreads);
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestMergePolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestMergePolicy.java
new file mode 100644
index 000000000000..e5f5635e1f19
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/index/TestMergePolicy.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.index;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.StringHelper;
+import org.apache.lucene.util.TestUtil;
+import org.apache.lucene.util.Version;
+
+public class TestMergePolicy extends LuceneTestCase {
+
+ public void testWaitForOneMerge() throws IOException, InterruptedException {
+ try (Directory dir = newDirectory()) {
+ MergePolicy.MergeSpecification ms = createRandomMergeSpecification(dir, 1 + random().nextInt(10));
+ for (MergePolicy.OneMerge m : ms.merges) {
+ assertFalse(m.hasCompletedSuccessfully().isPresent());
+ }
+ Thread t = new Thread(() -> {
+ try {
+ for (MergePolicy.OneMerge m : ms.merges) {
+ m.mergeFinished(true);
+ }
+ } catch (IOException e) {
+ throw new AssertionError(e);
+ }
+ });
+ t.start();
+ assertTrue(ms.await(100, TimeUnit.HOURS));
+ for (MergePolicy.OneMerge m : ms.merges) {
+ assertTrue(m.hasCompletedSuccessfully().get());
+ }
+ t.join();
+ }
+ }
+
+ public void testTimeout() throws IOException, InterruptedException {
+ try (Directory dir = newDirectory()) {
+ MergePolicy.MergeSpecification ms = createRandomMergeSpecification(dir, 3);
+ for (MergePolicy.OneMerge m : ms.merges) {
+ assertFalse(m.hasCompletedSuccessfully().isPresent());
+ }
+ Thread t = new Thread(() -> {
+ try {
+ ms.merges.get(0).mergeFinished(true);
+ } catch (IOException e) {
+ throw new AssertionError(e);
+ }
+ });
+ t.start();
+ assertFalse(ms.await(10, TimeUnit.MILLISECONDS));
+ assertFalse(ms.merges.get(1).hasCompletedSuccessfully().isPresent());
+ t.join();
+ }
+ }
+
+ public void testTimeoutLargeNumberOfMerges() throws IOException, InterruptedException {
+ try (Directory dir = newDirectory()) {
+ MergePolicy.MergeSpecification ms = createRandomMergeSpecification(dir, 10000);
+ for (MergePolicy.OneMerge m : ms.merges) {
+ assertFalse(m.hasCompletedSuccessfully().isPresent());
+ }
+ AtomicInteger i = new AtomicInteger(0);
+ AtomicBoolean stop = new AtomicBoolean(false);
+ Thread t = new Thread(() -> {
+ while (stop.get() == false) {
+ try {
+ ms.merges.get(i.getAndIncrement()).mergeFinished(true);
+ Thread.sleep(1);
+ } catch (IOException | InterruptedException e) {
+ throw new AssertionError(e);
+ }
+ }
+ });
+ t.start();
+ assertFalse(ms.await(10, TimeUnit.MILLISECONDS));
+ stop.set(true);
+ t.join();
+ for (int j = 0; j < ms.merges.size(); j++) {
+ if (j < i.get()) {
+ assertTrue(ms.merges.get(j).hasCompletedSuccessfully().get());
+ } else {
+ assertFalse(ms.merges.get(j).hasCompletedSuccessfully().isPresent());
+ }
+ }
+ }
+ }
+
+ @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/LUCENE-9408")
+ public void testFinishTwice() throws IOException {
+ try (Directory dir = newDirectory()) {
+ MergePolicy.MergeSpecification spec = createRandomMergeSpecification(dir, 1);
+ MergePolicy.OneMerge oneMerge = spec.merges.get(0);
+ oneMerge.mergeFinished(true);
+ expectThrows(IllegalStateException.class, () -> oneMerge.mergeFinished(false));
+ }
+ }
+
+ public void testTotalMaxDoc() throws IOException {
+ try (Directory dir = newDirectory()) {
+ MergePolicy.MergeSpecification spec = createRandomMergeSpecification(dir, 1);
+ int docs = 0;
+ MergePolicy.OneMerge oneMerge = spec.merges.get(0);
+ for (SegmentCommitInfo info : oneMerge.segments) {
+ docs += info.info.maxDoc();
+ }
+ assertEquals(docs, oneMerge.totalMaxDoc);
+ }
+ }
+
+ private static MergePolicy.MergeSpecification createRandomMergeSpecification(Directory dir, int numMerges) {
+ MergePolicy.MergeSpecification ms = new MergePolicy.MergeSpecification();
+ for (int ii = 0; ii < numMerges; ++ii) {
+ final SegmentInfo si = new SegmentInfo(
+ dir, // dir
+ Version.LATEST, // version
+ Version.LATEST, // min version
+ TestUtil.randomSimpleString(random()), // name
+ random().nextInt(1000), // maxDoc
+ random().nextBoolean(), // isCompoundFile
+ null, // codec
+ Collections.emptyMap(), // diagnostics
+ TestUtil.randomSimpleString(// id
+ random(),
+ StringHelper.ID_LENGTH,
+ StringHelper.ID_LENGTH).getBytes(StandardCharsets.US_ASCII),
+ Collections.emptyMap(), // attributes
+ null /* indexSort */);
+ final List segments = new LinkedList();
+ segments.add(new SegmentCommitInfo(si, 0, 0, 0, 0, 0, StringHelper.randomId()));
+ ms.add(new MergePolicy.OneMerge(segments));
+ }
+ return ms;
+ }
+}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java b/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java
index ee778ed90027..d982953a2f62 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java
@@ -396,7 +396,7 @@ public void testIllegalTooManyDimensions() throws Exception {
public void testDifferentCodecs1() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- iwc.setCodec(Codec.forName("Lucene84"));
+ iwc.setCodec(Codec.forName("Lucene86"));
IndexWriter w = new IndexWriter(dir, iwc);
Document doc = new Document();
doc.add(new IntPoint("int", 1));
@@ -427,7 +427,7 @@ public void testDifferentCodecs2() throws Exception {
w.close();
iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- iwc.setCodec(Codec.forName("Lucene84"));
+ iwc.setCodec(Codec.forName("Lucene86"));
w = new IndexWriter(dir, iwc);
doc = new Document();
doc.add(new IntPoint("int", 1));
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSegmentInfos.java b/lucene/core/src/test/org/apache/lucene/index/TestSegmentInfos.java
index 19d821481e00..4570f0906001 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestSegmentInfos.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestSegmentInfos.java
@@ -18,15 +18,21 @@
import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.mockfile.ExtrasFS;
import org.apache.lucene.search.Sort;
import org.apache.lucene.store.BaseDirectoryWrapper;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.StringHelper;
+import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.Version;
import java.io.IOException;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
@@ -178,5 +184,64 @@ public void testIDChangesOnAdvance() throws IOException {
assertEquals("clone changed but shouldn't", StringHelper.idToString(id), StringHelper.idToString(clone.getId()));
}
}
+
+ public void testBitFlippedTriggersCorruptIndexException() throws IOException {
+ BaseDirectoryWrapper dir = newDirectory();
+ dir.setCheckIndexOnClose(false);
+ byte id[] = StringHelper.randomId();
+ Codec codec = Codec.getDefault();
+
+ SegmentInfos sis = new SegmentInfos(Version.LATEST.major);
+ SegmentInfo info = new SegmentInfo(dir, Version.LATEST, Version.LATEST, "_0", 1, false, Codec.getDefault(),
+ Collections.emptyMap(), id, Collections.emptyMap(), null);
+ info.setFiles(Collections.emptySet());
+ codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT);
+ SegmentCommitInfo commitInfo = new SegmentCommitInfo(info, 0, 0, -1, -1, -1, StringHelper.randomId());
+ sis.add(commitInfo);
+
+ info = new SegmentInfo(dir, Version.LATEST, Version.LATEST, "_1", 1, false, Codec.getDefault(),
+ Collections.emptyMap(), id, Collections.emptyMap(), null);
+ info.setFiles(Collections.emptySet());
+ codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT);
+ commitInfo = new SegmentCommitInfo(info, 0, 0,-1, -1, -1, StringHelper.randomId());
+ sis.add(commitInfo);
+
+ sis.commit(dir);
+
+ BaseDirectoryWrapper corruptDir = newDirectory();
+ corruptDir.setCheckIndexOnClose(false);
+ boolean corrupt = false;
+ for (String file : dir.listAll()) {
+ if (file.startsWith(IndexFileNames.SEGMENTS)) {
+ try (IndexInput in = dir.openInput(file, IOContext.DEFAULT);
+ IndexOutput out = corruptDir.createOutput(file, IOContext.DEFAULT)) {
+ final long corruptIndex = TestUtil.nextLong(random(), 0, in.length() - 1);
+ out.copyBytes(in, corruptIndex);
+ final int b = Byte.toUnsignedInt(in.readByte()) + TestUtil.nextInt(random(), 0x01, 0xff);
+ out.writeByte((byte) b);
+ out.copyBytes(in, in.length() - in.getFilePointer());
+ }
+ try (IndexInput in = corruptDir.openInput(file, IOContext.DEFAULT)) {
+ CodecUtil.checksumEntireFile(in);
+ if (VERBOSE) {
+ System.out.println("TEST: Altering the file did not update the checksum, aborting...");
+ }
+ return;
+ } catch (CorruptIndexException e) {
+ // ok
+ }
+ corrupt = true;
+ } else if (ExtrasFS.isExtra(file) == false) {
+ corruptDir.copyFrom(dir, file, file, IOContext.DEFAULT);
+ }
+ }
+ assertTrue("No segments file found", corrupt);
+
+ expectThrowsAnyOf(
+ Arrays.asList(CorruptIndexException.class, IndexFormatTooOldException.class, IndexFormatTooNewException.class),
+ () -> SegmentInfos.readLatestCommit(corruptDir));
+ dir.close();
+ corruptDir.close();
+ }
}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestTermsHashPerField.java b/lucene/core/src/test/org/apache/lucene/index/TestTermsHashPerField.java
new file mode 100644
index 000000000000..a8a879bf5c10
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/index/TestTermsHashPerField.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.index;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+
+import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+import com.carrotsearch.randomizedtesting.generators.RandomStrings;
+import org.apache.lucene.util.ByteBlockPool;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.Counter;
+import org.apache.lucene.util.IntBlockPool;
+import org.apache.lucene.util.LuceneTestCase;
+
+public class TestTermsHashPerField extends LuceneTestCase {
+
+ private static TermsHashPerField createNewHash(AtomicInteger newCalled, AtomicInteger addCalled) {
+ IntBlockPool intBlockPool = new IntBlockPool();
+ ByteBlockPool byteBlockPool = new ByteBlockPool(new ByteBlockPool.DirectAllocator());
+ ByteBlockPool termBlockPool = new ByteBlockPool(new ByteBlockPool.DirectAllocator());
+
+ TermsHashPerField hash = new TermsHashPerField(1, intBlockPool, byteBlockPool, termBlockPool, Counter.newCounter(),
+ null, "testfield", IndexOptions.DOCS_AND_FREQS) {
+
+ private FreqProxTermsWriterPerField.FreqProxPostingsArray freqProxPostingsArray;
+
+ @Override
+ void newTerm(int termID, int docID) {
+ newCalled.incrementAndGet();
+ FreqProxTermsWriterPerField.FreqProxPostingsArray postings = freqProxPostingsArray;
+ postings.lastDocIDs[termID] = docID;
+ postings.lastDocCodes[termID] = docID << 1;
+ postings.termFreqs[termID] = 1;
+ }
+
+ @Override
+ void addTerm(int termID, int docID) {
+ addCalled.incrementAndGet();
+ FreqProxTermsWriterPerField.FreqProxPostingsArray postings = freqProxPostingsArray;
+ if (docID != postings.lastDocIDs[termID]) {
+ if (1 == postings.termFreqs[termID]) {
+ writeVInt(0, postings.lastDocCodes[termID]|1);
+ } else {
+ writeVInt(0, postings.lastDocCodes[termID]);
+ writeVInt(0, postings.termFreqs[termID]);
+ }
+ postings.termFreqs[termID] = 1;
+ postings.lastDocCodes[termID] = (docID - postings.lastDocIDs[termID]) << 1;
+ postings.lastDocIDs[termID] = docID;
+ } else {
+ postings.termFreqs[termID] = Math.addExact(postings.termFreqs[termID], 1);
+ }
+ }
+
+ @Override
+ void newPostingsArray() {
+ freqProxPostingsArray = (FreqProxTermsWriterPerField.FreqProxPostingsArray) postingsArray;
+ }
+
+ @Override
+ ParallelPostingsArray createPostingsArray(int size) {
+ return new FreqProxTermsWriterPerField.FreqProxPostingsArray(size, true, false, false);
+ }
+ };
+ return hash;
+ }
+
+ boolean assertDocAndFreq(ByteSliceReader reader, FreqProxTermsWriterPerField.FreqProxPostingsArray postingsArray, int prevDoc, int termId, int doc, int frequency) throws IOException {
+ int docId = prevDoc;
+ int freq;
+ boolean eof = reader.eof();
+ if (eof) {
+ docId = postingsArray.lastDocIDs[termId];
+ freq = postingsArray.termFreqs[termId];
+ } else {
+ int code = reader.readVInt();
+ docId += code >>> 1;
+ if ((code & 1) != 0) {
+ freq = 1;
+ } else {
+ freq = reader.readVInt();
+ }
+ }
+ assertEquals("docID mismatch eof: " + eof, doc, docId);
+ assertEquals("freq mismatch eof: " + eof, frequency, freq);
+ return eof;
+ }
+
+ public void testAddAndUpdateTerm() throws IOException {
+ AtomicInteger newCalled = new AtomicInteger(0);
+ AtomicInteger addCalled = new AtomicInteger(0);
+ TermsHashPerField hash = createNewHash(newCalled, addCalled);
+ hash.start(null, true);
+
+ hash.add(new BytesRef("start"), 0); // tid = 0;
+ hash.add(new BytesRef("foo"), 0); // tid = 1;
+ hash.add(new BytesRef("bar"), 0); // tid = 2;
+ hash.finish();
+ hash.add(new BytesRef("bar"), 1);
+ hash.add(new BytesRef("foobar"), 1); // tid = 3;
+ hash.add(new BytesRef("bar"), 1);
+ hash.add(new BytesRef("bar"), 1);
+ hash.add(new BytesRef("foobar"), 1);
+ hash.add(new BytesRef("verylongfoobarbaz"), 1); // tid = 4;
+ hash.finish();
+ hash.add(new BytesRef("verylongfoobarbaz"), 2);
+ hash.add(new BytesRef("boom"), 2); // tid = 5;
+ hash.finish();
+ hash.add(new BytesRef("verylongfoobarbaz"), 3);
+ hash.add(new BytesRef("end"), 3); // tid = 6;
+ hash.finish();
+
+ assertEquals(7, newCalled.get());
+ assertEquals(6, addCalled.get());
+ final ByteSliceReader reader = new ByteSliceReader();
+ hash.initReader(reader, 0, 0);
+ assertTrue(assertDocAndFreq(reader, (FreqProxTermsWriterPerField.FreqProxPostingsArray) hash.postingsArray, 0, 0, 0, 1));
+ hash.initReader(reader, 1, 0);
+ assertTrue(assertDocAndFreq(reader, (FreqProxTermsWriterPerField.FreqProxPostingsArray) hash.postingsArray, 0, 1, 0, 1));
+ hash.initReader(reader, 2, 0);
+ assertFalse(assertDocAndFreq(reader, (FreqProxTermsWriterPerField.FreqProxPostingsArray) hash.postingsArray, 0, 2, 0, 1));
+ assertTrue(assertDocAndFreq(reader, (FreqProxTermsWriterPerField.FreqProxPostingsArray) hash.postingsArray, 2, 2, 1, 3));
+ hash.initReader(reader, 3, 0);
+ assertTrue(assertDocAndFreq(reader, (FreqProxTermsWriterPerField.FreqProxPostingsArray) hash.postingsArray, 0, 3, 1, 2));
+ hash.initReader(reader, 4, 0);
+ assertFalse(assertDocAndFreq(reader, (FreqProxTermsWriterPerField.FreqProxPostingsArray) hash.postingsArray, 0, 4, 1, 1));
+ assertFalse(assertDocAndFreq(reader, (FreqProxTermsWriterPerField.FreqProxPostingsArray) hash.postingsArray, 1, 4, 2, 1));
+ assertTrue(assertDocAndFreq(reader, (FreqProxTermsWriterPerField.FreqProxPostingsArray) hash.postingsArray, 2, 4, 3, 1));
+ hash.initReader(reader, 5, 0);
+ assertTrue(assertDocAndFreq(reader, (FreqProxTermsWriterPerField.FreqProxPostingsArray) hash.postingsArray, 0, 5, 2, 1));
+ hash.initReader(reader, 6, 0);
+ assertTrue(assertDocAndFreq(reader, (FreqProxTermsWriterPerField.FreqProxPostingsArray) hash.postingsArray, 0, 6, 3, 1));
+ }
+
+ public void testAddAndUpdateRandom() throws IOException {
+ AtomicInteger newCalled = new AtomicInteger(0);
+ AtomicInteger addCalled = new AtomicInteger(0);
+ TermsHashPerField hash = createNewHash(newCalled, addCalled);
+ hash.start(null, true);
+ class Posting {
+ int termId = -1;
+ final TreeMap docAndFreq = new TreeMap<>();
+ }
+ Map postingMap = new HashMap<>();
+ int numStrings = 1 + random().nextInt(200);
+ for (int i = 0; i < numStrings; i++) {
+ String randomString = RandomStrings.randomRealisticUnicodeOfCodepointLengthBetween(random(), 1, 10);
+ postingMap.putIfAbsent(new BytesRef(randomString), new Posting());
+ }
+ List bytesRefs = Arrays.asList(postingMap.keySet().toArray(new BytesRef[0]));
+ Collections.sort(bytesRefs);
+ int numDocs = 1 + random().nextInt(200);
+ int termOrd = 0;
+ for (int i = 0; i < numDocs; i++) {
+ int numTerms = 1 + random().nextInt(200);
+ int doc = i;
+ for (int j = 0; i < numTerms; i++) {
+ BytesRef ref = RandomPicks.randomFrom(random(), bytesRefs);
+ Posting posting = postingMap.get(ref);
+ if (posting.termId == -1) {
+ posting.termId = termOrd++;
+ }
+ posting.docAndFreq.putIfAbsent(doc, 0);
+ posting.docAndFreq.compute(doc, (key, oldVal) -> oldVal+1);
+ hash.add(ref, doc);
+ }
+ hash.finish();
+ }
+ List values = postingMap.values().stream().filter( x -> x.termId != -1)
+ .collect(Collectors.toList());
+ Collections.shuffle(values, random()); // term order doesn't matter
+ final ByteSliceReader reader = new ByteSliceReader();
+ for (Posting p : values) {
+ hash.initReader(reader, p.termId, 0);
+ boolean eof = false;
+ int prefDoc = 0;
+ for (Map.Entry entry : p.docAndFreq.entrySet()) {
+ assertFalse("the reader must not be EOF here", eof);
+ eof = assertDocAndFreq(reader, (FreqProxTermsWriterPerField.FreqProxPostingsArray) hash.postingsArray,
+ prefDoc, p.termId, entry.getKey(), entry.getValue());
+ prefDoc = entry.getKey();
+ }
+ assertTrue("the last posting must be EOF on the reader", eof);
+ }
+ }
+}
diff --git a/lucene/core/src/test/org/apache/lucene/search/MultiCollectorTest.java b/lucene/core/src/test/org/apache/lucene/search/MultiCollectorTest.java
index c3b4f42650ac..80a5a9a99583 100644
--- a/lucene/core/src/test/org/apache/lucene/search/MultiCollectorTest.java
+++ b/lucene/core/src/test/org/apache/lucene/search/MultiCollectorTest.java
@@ -163,4 +163,176 @@ public void testCacheScoresIfNecessary() throws IOException {
reader.close();
dir.close();
}
+
+ public void testScorerWrappingForTopScores() throws IOException {
+ Directory dir = newDirectory();
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
+ iw.addDocument(new Document());
+ DirectoryReader reader = iw.getReader();
+ iw.close();
+ final LeafReaderContext ctx = reader.leaves().get(0);
+ Collector c1 = collector(ScoreMode.TOP_SCORES, MultiCollector.MinCompetitiveScoreAwareScorable.class);
+ Collector c2 = collector(ScoreMode.TOP_SCORES, MultiCollector.MinCompetitiveScoreAwareScorable.class);
+ MultiCollector.wrap(c1, c2).getLeafCollector(ctx).setScorer(new ScoreAndDoc());
+
+ c1 = collector(ScoreMode.TOP_SCORES, ScoreCachingWrappingScorer.class);
+ c2 = collector(ScoreMode.COMPLETE, ScoreCachingWrappingScorer.class);
+ MultiCollector.wrap(c1, c2).getLeafCollector(ctx).setScorer(new ScoreAndDoc());
+
+ reader.close();
+ dir.close();
+ }
+
+ public void testMinCompetitiveScore() throws IOException {
+ float[] currentMinScores = new float[3];
+ float[] minCompetitiveScore = new float[1];
+ Scorable scorer = new Scorable() {
+
+ @Override
+ public float score() throws IOException {
+ return 0;
+ }
+
+ @Override
+ public int docID() {
+ return 0;
+ }
+
+ @Override
+ public void setMinCompetitiveScore(float minScore) throws IOException {
+ minCompetitiveScore[0] = minScore;
+ }
+ };
+ Scorable s0 = new MultiCollector.MinCompetitiveScoreAwareScorable(scorer, 0, currentMinScores);
+ Scorable s1 = new MultiCollector.MinCompetitiveScoreAwareScorable(scorer, 1, currentMinScores);
+ Scorable s2 = new MultiCollector.MinCompetitiveScoreAwareScorable(scorer, 2, currentMinScores);
+ assertEquals(0f, minCompetitiveScore[0], 0);
+ s0.setMinCompetitiveScore(0.5f);
+ assertEquals(0f, minCompetitiveScore[0], 0);
+ s1.setMinCompetitiveScore(0.8f);
+ assertEquals(0f, minCompetitiveScore[0], 0);
+ s2.setMinCompetitiveScore(0.3f);
+ assertEquals(0.3f, minCompetitiveScore[0], 0);
+ s2.setMinCompetitiveScore(0.1f);
+ assertEquals(0.3f, minCompetitiveScore[0], 0);
+ s1.setMinCompetitiveScore(Float.MAX_VALUE);
+ assertEquals(0.3f, minCompetitiveScore[0], 0);
+ s2.setMinCompetitiveScore(Float.MAX_VALUE);
+ assertEquals(0.5f, minCompetitiveScore[0], 0);
+ s0.setMinCompetitiveScore(Float.MAX_VALUE);
+ assertEquals(Float.MAX_VALUE, minCompetitiveScore[0], 0);
+ }
+
+ public void testCollectionTermination() throws IOException {
+ Directory dir = newDirectory();
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
+ iw.addDocument(new Document());
+ DirectoryReader reader = iw.getReader();
+ iw.close();
+ final LeafReaderContext ctx = reader.leaves().get(0);
+ DummyCollector c1 = new TerminatingDummyCollector(1, ScoreMode.COMPLETE);
+ DummyCollector c2 = new TerminatingDummyCollector(2, ScoreMode.COMPLETE);
+
+ Collector mc = MultiCollector.wrap(c1, c2);
+ LeafCollector lc = mc.getLeafCollector(ctx);
+ lc.setScorer(new ScoreAndDoc());
+ lc.collect(0); // OK
+ assertTrue("c1's collect should be called", c1.collectCalled);
+ assertTrue("c2's collect should be called", c2.collectCalled);
+ c1.collectCalled = false;
+ c2.collectCalled = false;
+ lc.collect(1); // OK, but c1 should terminate
+ assertFalse("c1 should be removed already", c1.collectCalled);
+ assertTrue("c2's collect should be called", c2.collectCalled);
+ c2.collectCalled = false;
+
+ expectThrows(CollectionTerminatedException.class, () -> {
+ lc.collect(2);
+ });
+ assertFalse("c1 should be removed already", c1.collectCalled);
+ assertFalse("c2 should be removed already", c2.collectCalled);
+
+ reader.close();
+ dir.close();
+ }
+
+ public void testSetScorerOnCollectionTerminationSkipNonCompetitive() throws IOException {
+ doTestSetScorerOnCollectionTermination(true);
+ }
+
+ public void testSetScorerOnCollectionTerminationSkipNoSkips() throws IOException {
+ doTestSetScorerOnCollectionTermination(false);
+ }
+
+ private void doTestSetScorerOnCollectionTermination(boolean allowSkipNonCompetitive) throws IOException {
+ Directory dir = newDirectory();
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
+ iw.addDocument(new Document());
+ DirectoryReader reader = iw.getReader();
+ iw.close();
+ final LeafReaderContext ctx = reader.leaves().get(0);
+
+ DummyCollector c1 = new TerminatingDummyCollector(1, allowSkipNonCompetitive? ScoreMode.TOP_SCORES : ScoreMode.COMPLETE);
+ DummyCollector c2 = new TerminatingDummyCollector(2, allowSkipNonCompetitive? ScoreMode.TOP_SCORES : ScoreMode.COMPLETE);
+
+ Collector mc = MultiCollector.wrap(c1, c2);
+ LeafCollector lc = mc.getLeafCollector(ctx);
+ assertFalse(c1.setScorerCalled);
+ assertFalse(c2.setScorerCalled);
+ lc.setScorer(new ScoreAndDoc());
+ assertTrue(c1.setScorerCalled);
+ assertTrue(c2.setScorerCalled);
+ c1.setScorerCalled = false;
+ c2.setScorerCalled = false;
+ lc.collect(0); // OK
+
+ lc.setScorer(new ScoreAndDoc());
+ assertTrue(c1.setScorerCalled);
+ assertTrue(c2.setScorerCalled);
+ c1.setScorerCalled = false;
+ c2.setScorerCalled = false;
+
+ lc.collect(1); // OK, but c1 should terminate
+ lc.setScorer(new ScoreAndDoc());
+ assertFalse(c1.setScorerCalled);
+ assertTrue(c2.setScorerCalled);
+ c2.setScorerCalled = false;
+
+ expectThrows(CollectionTerminatedException.class, () -> {
+ lc.collect(2);
+ });
+ lc.setScorer(new ScoreAndDoc());
+ assertFalse(c1.setScorerCalled);
+ assertFalse(c2.setScorerCalled);
+
+ reader.close();
+ dir.close();
+ }
+
+ private static class TerminatingDummyCollector extends DummyCollector {
+
+ private final int terminateOnDoc;
+ private final ScoreMode scoreMode;
+
+ public TerminatingDummyCollector(int terminateOnDoc, ScoreMode scoreMode) {
+ super();
+ this.terminateOnDoc = terminateOnDoc;
+ this.scoreMode = scoreMode;
+ }
+
+ @Override
+ public void collect(int doc) throws IOException {
+ if (doc == terminateOnDoc) {
+ throw new CollectionTerminatedException();
+ }
+ super.collect(doc);
+ }
+
+ @Override
+ public ScoreMode scoreMode() {
+ return scoreMode;
+ }
+
+ }
+
}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestBoolean2.java b/lucene/core/src/test/org/apache/lucene/search/TestBoolean2.java
index 2ccfd9aa8cb7..3400f0e6dd67 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestBoolean2.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestBoolean2.java
@@ -96,7 +96,7 @@ public static void beforeClass() throws Exception {
IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
// randomized codecs are sometimes too costly for this test:
- iwc.setCodec(Codec.forName("Lucene84"));
+ iwc.setCodec(Codec.forName("Lucene86"));
iwc.setMergePolicy(newLogMergePolicy());
RandomIndexWriter writer= new RandomIndexWriter(random(), directory, iwc);
// we'll make a ton of docs, disable store/norms/vectors
@@ -141,7 +141,7 @@ public static void beforeClass() throws Exception {
iwc = newIndexWriterConfig(new MockAnalyzer(random()));
// we need docID order to be preserved:
// randomized codecs are sometimes too costly for this test:
- iwc.setCodec(Codec.forName("Lucene84"));
+ iwc.setCodec(Codec.forName("Lucene86"));
iwc.setMergePolicy(newLogMergePolicy());
try (IndexWriter w = new IndexWriter(singleSegmentDirectory, iwc)) {
w.forceMerge(1, true);
@@ -167,7 +167,7 @@ public static void beforeClass() throws Exception {
iwc = newIndexWriterConfig(new MockAnalyzer(random()));
// randomized codecs are sometimes too costly for this test:
- iwc.setCodec(Codec.forName("Lucene84"));
+ iwc.setCodec(Codec.forName("Lucene86"));
RandomIndexWriter w = new RandomIndexWriter(random(), dir2, iwc);
w.addIndexes(copy);
copy.close();
@@ -179,7 +179,7 @@ public static void beforeClass() throws Exception {
iwc = newIndexWriterConfig(new MockAnalyzer(random()));
iwc.setMaxBufferedDocs(TestUtil.nextInt(random(), 50, 1000));
// randomized codecs are sometimes too costly for this test:
- iwc.setCodec(Codec.forName("Lucene84"));
+ iwc.setCodec(Codec.forName("Lucene86"));
RandomIndexWriter w = new RandomIndexWriter(random(), dir2, iwc);
doc = new Document();
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestFuzzyQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestFuzzyQuery.java
index a95ad39cf59a..4e9bf9dcfdb1 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestFuzzyQuery.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestFuzzyQuery.java
@@ -24,6 +24,8 @@
import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Supplier;
import com.carrotsearch.randomizedtesting.RandomizedTest;
import org.apache.lucene.analysis.MockAnalyzer;
@@ -36,15 +38,15 @@
import org.apache.lucene.index.MultiReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.Terms;
-import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.similarities.ClassicSimilarity;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
+import org.apache.lucene.util.automaton.ByteRunAutomaton;
import org.apache.lucene.util.automaton.LevenshteinAutomata;
import org.apache.lucene.util.automaton.Operations;
@@ -222,7 +224,46 @@ public void testFuzziness() throws Exception {
reader.close();
directory.close();
}
-
+
+ public void testPrefixLengthEqualStringLength() throws Exception {
+ Directory directory = newDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(random(), directory);
+ addDoc("b*a", writer);
+ addDoc("b*ab", writer);
+ addDoc("b*abc", writer);
+ addDoc("b*abcd", writer);
+ String multibyte = "아프리카코끼리속";
+ addDoc(multibyte, writer);
+ IndexReader reader = writer.getReader();
+ IndexSearcher searcher = newSearcher(reader);
+ writer.close();
+
+ int maxEdits = 0;
+ int prefixLength = 3;
+ FuzzyQuery query = new FuzzyQuery(new Term("field", "b*a"), maxEdits, prefixLength);
+ ScoreDoc[] hits = searcher.search(query, 1000).scoreDocs;
+ assertEquals(1, hits.length);
+
+ maxEdits = 1;
+ query = new FuzzyQuery(new Term("field", "b*a"), maxEdits, prefixLength);
+ hits = searcher.search(query, 1000).scoreDocs;
+ assertEquals(2, hits.length);
+
+ maxEdits = 2;
+ query = new FuzzyQuery(new Term("field", "b*a"), maxEdits, prefixLength);
+ hits = searcher.search(query, 1000).scoreDocs;
+ assertEquals(3, hits.length);
+
+ maxEdits = 1;
+ prefixLength = multibyte.length() - 1;
+ query = new FuzzyQuery(new Term("field", multibyte.substring(0, prefixLength)), maxEdits, prefixLength);
+ hits = searcher.search(query, 1000).scoreDocs;
+ assertEquals(1, hits.length);
+
+ reader.close();
+ directory.close();
+ }
+
public void test2() throws Exception {
Directory directory = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), directory, new MockAnalyzer(random(), MockTokenizer.KEYWORD, false));
@@ -409,7 +450,6 @@ public void testBoostOnlyRewrite() throws Exception {
public void testGiga() throws Exception {
- MockAnalyzer analyzer = new MockAnalyzer(random());
Directory index = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), index);
@@ -441,6 +481,7 @@ public void testGiga() throws Exception {
assertEquals(1, hits.length);
assertEquals("Giga byte", searcher.doc(hits[0].doc).get("field"));
r.close();
+ w.close();
index.close();
}
@@ -515,54 +556,13 @@ public void testErrorMessage() {
final String value = randomRealisticMultiByteUnicode(length);
FuzzyTermsEnum.FuzzyTermsException expected = expectThrows(FuzzyTermsEnum.FuzzyTermsException.class, () -> {
- new FuzzyQuery(new Term("field", value)).getTermsEnum(new Terms() {
- @Override
- public TermsEnum iterator() {
- return TermsEnum.EMPTY;
- }
-
- @Override
- public long size() {
- return 0;
- }
-
- @Override
- public long getSumTotalTermFreq() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public long getSumDocFreq() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public int getDocCount() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public boolean hasFreqs() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public boolean hasOffsets() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public boolean hasPositions() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public boolean hasPayloads() {
- throw new UnsupportedOperationException();
- }
- });
+ new FuzzyAutomatonBuilder(value, 2, 0, true).buildMaxEditAutomaton();
});
assertThat(expected.getMessage(), containsString(value));
+
+ expected = expectThrows(FuzzyTermsEnum.FuzzyTermsException.class,
+ () -> new FuzzyAutomatonBuilder(value, 2, 0, true).buildAutomatonSet());
+ assertThat(expected.getMessage(), containsString(value));
}
private void addDoc(String text, RandomIndexWriter writer) throws IOException {
@@ -600,6 +600,7 @@ public void testRandom() throws Exception {
w.addDocument(doc);
}
DirectoryReader r = w.getReader();
+ w.close();
//System.out.println("TEST: reader=" + r);
IndexSearcher s = newSearcher(r);
int iters = atLeast(200);
@@ -677,7 +678,7 @@ public void testRandom() throws Exception {
}
}
- IOUtils.close(r, w, dir);
+ IOUtils.close(r, dir);
}
private static class TermAndScore implements Comparable {
@@ -777,4 +778,31 @@ private static IntsRef toIntsRef(String s) {
}
return ref;
}
+
+ public void testVisitor() {
+ FuzzyQuery q = new FuzzyQuery(new Term("field", "blob"), 2);
+ AtomicBoolean visited = new AtomicBoolean(false);
+ q.visit(new QueryVisitor() {
+ @Override
+ public void consumeTermsMatching(Query query, String field, Supplier automaton) {
+ visited.set(true);
+ ByteRunAutomaton a = automaton.get();
+ assertMatches(a, "blob");
+ assertMatches(a, "bolb");
+ assertMatches(a, "blobby");
+ assertNoMatches(a, "bolbby");
+ }
+ });
+ assertTrue(visited.get());
+ }
+
+ private static void assertMatches(ByteRunAutomaton automaton, String text) {
+ BytesRef b = new BytesRef(text);
+ assertTrue(automaton.run(b.bytes, b.offset, b.length));
+ }
+
+ private static void assertNoMatches(ByteRunAutomaton automaton, String text) {
+ BytesRef b = new BytesRef(text);
+ assertFalse(automaton.run(b.bytes, b.offset, b.length));
+ }
}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java b/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java
index 7993beb7427e..ef023752a66b 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java
@@ -181,14 +181,17 @@ public ScoreMode scoreMode() {
thread.join();
}
- if (error.get() != null) {
- throw error.get();
+ try {
+ if (error.get() != null) {
+ throw error.get();
+ }
+ queryCache.assertConsistent();
+ } finally {
+ mgr.close();
+ w.close();
+ dir.close();
+ queryCache.assertConsistent();
}
- queryCache.assertConsistent();
- mgr.close();
- w.close();
- dir.close();
- queryCache.assertConsistent();
}
public void testLRUEviction() throws Exception {
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestMultiTermConstantScore.java b/lucene/core/src/test/org/apache/lucene/search/TestMultiTermConstantScore.java
index 69e1e1039b22..f4a2543925f9 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestMultiTermConstantScore.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestMultiTermConstantScore.java
@@ -28,13 +28,12 @@
import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
import org.junit.AfterClass;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import java.io.IOException;
-import junit.framework.Assert;
-
public class TestMultiTermConstantScore extends BaseTestRangeFilter {
/** threshold for comparing floats */
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java b/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java
index 3c6e29403e52..7b0c41896e4e 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java
@@ -38,8 +38,8 @@
import org.apache.lucene.codecs.PointsFormat;
import org.apache.lucene.codecs.PointsReader;
import org.apache.lucene.codecs.PointsWriter;
-import org.apache.lucene.codecs.lucene60.Lucene60PointsReader;
-import org.apache.lucene.codecs.lucene60.Lucene60PointsWriter;
+import org.apache.lucene.codecs.lucene86.Lucene86PointsReader;
+import org.apache.lucene.codecs.lucene86.Lucene86PointsWriter;
import org.apache.lucene.document.BinaryPoint;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.DoublePoint;
@@ -1173,12 +1173,12 @@ public PointsFormat pointsFormat() {
return new PointsFormat() {
@Override
public PointsWriter fieldsWriter(SegmentWriteState writeState) throws IOException {
- return new Lucene60PointsWriter(writeState, maxPointsInLeafNode, maxMBSortInHeap);
+ return new Lucene86PointsWriter(writeState, maxPointsInLeafNode, maxMBSortInHeap);
}
@Override
public PointsReader fieldsReader(SegmentReadState readState) throws IOException {
- return new Lucene60PointsReader(readState);
+ return new Lucene86PointsReader(readState);
}
};
}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestRegexpQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestRegexpQuery.java
index 6a8e183e0d45..767ee20a7d45 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestRegexpQuery.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestRegexpQuery.java
@@ -50,7 +50,7 @@ public void setUp() throws Exception {
directory = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), directory);
Document doc = new Document();
- doc.add(newTextField(FN, "the quick brown fox jumps over the lazy ??? dog 493432 49344", Field.Store.NO));
+ doc.add(newTextField(FN, "the quick brown fox jumps over the lazy ??? dog 493432 49344 [foo] 12.3 \\", Field.Store.NO));
writer.addDocument(doc);
reader = writer.getReader();
writer.close();
@@ -90,6 +90,41 @@ public void testNumericRange() throws IOException {
assertEquals(0, regexQueryNrHits("<493433-600000>"));
}
+ public void testCharacterClasses() throws IOException {
+ assertEquals(0, regexQueryNrHits("\\d"));
+ assertEquals(1, regexQueryNrHits("\\d*"));
+ assertEquals(1, regexQueryNrHits("\\d{6}"));
+ assertEquals(1, regexQueryNrHits("[a\\d]{6}"));
+ assertEquals(1, regexQueryNrHits("\\d{2,7}"));
+ assertEquals(0, regexQueryNrHits("\\d{4}"));
+ assertEquals(0, regexQueryNrHits("\\dog"));
+ assertEquals(1, regexQueryNrHits("493\\d32"));
+
+ assertEquals(1, regexQueryNrHits("\\wox"));
+ assertEquals(1, regexQueryNrHits("493\\w32"));
+ assertEquals(1, regexQueryNrHits("\\?\\?\\?"));
+ assertEquals(1, regexQueryNrHits("\\?\\W\\?"));
+ assertEquals(1, regexQueryNrHits("\\?\\S\\?"));
+
+ assertEquals(1, regexQueryNrHits("\\[foo\\]"));
+ assertEquals(1, regexQueryNrHits("\\[\\w{3}\\]"));
+
+ assertEquals(0, regexQueryNrHits("\\s.*")); // no matches because all whitespace stripped
+ assertEquals(1, regexQueryNrHits("\\S*ck")); //matches quick
+ assertEquals(1, regexQueryNrHits("[\\d\\.]{3,10}")); // matches 12.3
+ assertEquals(1, regexQueryNrHits("\\d{1,3}(\\.(\\d{1,2}))+")); // matches 12.3
+
+ assertEquals(1, regexQueryNrHits("\\\\"));
+ assertEquals(1, regexQueryNrHits("\\\\.*"));
+
+ IllegalArgumentException expected = expectThrows(
+ IllegalArgumentException.class, () -> {
+ regexQueryNrHits("\\p");
+ }
+ );
+ assertTrue(expected.getMessage().contains("invalid character class"));
+ }
+
public void testRegexComplement() throws IOException {
assertEquals(1, regexQueryNrHits("4934~[3]"));
// not the empty lang, i.e. match all docs
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSameScoresWithThreads.java b/lucene/core/src/test/org/apache/lucene/search/TestSameScoresWithThreads.java
index a615a6a7fbb6..4b284dfd31b0 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestSameScoresWithThreads.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestSameScoresWithThreads.java
@@ -119,6 +119,7 @@ public void run() {
thread.join();
}
}
+ docs.close();
r.close();
dir.close();
}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSearchAfter.java b/lucene/core/src/test/org/apache/lucene/search/TestSearchAfter.java
index 9dc9843d5106..59a8de9027fb 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestSearchAfter.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestSearchAfter.java
@@ -295,7 +295,7 @@ void assertPage(int pageStart, TopDocs all, TopDocs paged) throws IOException {
assertEquals(sd1.score, sd2.score, 0f);
if (sd1 instanceof FieldDoc) {
assertTrue(sd2 instanceof FieldDoc);
- assertEquals(((FieldDoc) sd1).fields, ((FieldDoc) sd2).fields);
+ assertArrayEquals(((FieldDoc) sd1).fields, ((FieldDoc) sd2).fields);
}
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java b/lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java
index b92386669163..1d8edccf092d 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java
@@ -310,6 +310,7 @@ public void testReferenceDecrementIllegally() throws Exception {
Directory dir = newDirectory();
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
new MockAnalyzer(random())).setMergeScheduler(new ConcurrentMergeScheduler()));
+ @SuppressWarnings("resource")
SearcherManager sm = new SearcherManager(writer, false, false, new SearcherFactory());
writer.addDocument(new Document());
writer.commit();
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTermQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestTermQuery.java
index e460e26ddba3..65986d968485 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestTermQuery.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestTermQuery.java
@@ -21,6 +21,7 @@
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.CompositeReaderContext;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.FilterDirectoryReader;
import org.apache.lucene.index.FilterLeafReader;
@@ -47,9 +48,13 @@ public void testEquals() throws IOException {
QueryUtils.checkUnequal(
new TermQuery(new Term("foo", "bar")),
new TermQuery(new Term("foo", "baz")));
+ final CompositeReaderContext context;
+ try (MultiReader multiReader = new MultiReader()) {
+ context = multiReader.getContext();
+ }
QueryUtils.checkEqual(
new TermQuery(new Term("foo", "bar")),
- new TermQuery(new Term("foo", "bar"), TermStates.build(new MultiReader().getContext(), new Term("foo", "bar"), true)));
+ new TermQuery(new Term("foo", "bar"), TermStates.build(context, new Term("foo", "bar"), true)));
}
public void testCreateWeightDoesNotSeekIfScoresAreNotNeeded() throws IOException {
diff --git a/lucene/core/src/test/org/apache/lucene/util/TestCharsRef.java b/lucene/core/src/test/org/apache/lucene/util/TestCharsRef.java
index 079b3b774b53..fd9d639ea888 100644
--- a/lucene/core/src/test/org/apache/lucene/util/TestCharsRef.java
+++ b/lucene/core/src/test/org/apache/lucene/util/TestCharsRef.java
@@ -20,6 +20,8 @@
public class TestCharsRef extends LuceneTestCase {
+
+ @SuppressWarnings("deprecation")
public void testUTF16InUTF8Order() {
final int numStrings = atLeast(1000);
BytesRef utf8[] = new BytesRef[numStrings];
diff --git a/lucene/core/src/test/org/apache/lucene/util/TestOfflineSorter.java b/lucene/core/src/test/org/apache/lucene/util/TestOfflineSorter.java
index 927af148afc7..f902cd58cc05 100644
--- a/lucene/core/src/test/org/apache/lucene/util/TestOfflineSorter.java
+++ b/lucene/core/src/test/org/apache/lucene/util/TestOfflineSorter.java
@@ -17,7 +17,6 @@
package org.apache.lucene.util;
-import java.io.EOFException;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
@@ -351,14 +350,12 @@ protected void corruptFile() throws IOException {
IndexOutput unsorted = dir.createTempOutput("unsorted", "tmp", IOContext.DEFAULT);
writeAll(unsorted, generateFixed(5*1024));
- // This corruption made OfflineSorter fail with its own exception, but we verify it also went and added (as suppressed) that the
- // checksum was wrong:
- EOFException e = expectThrows(EOFException.class, () -> {
+ // This corruption made OfflineSorter fail with its own exception, but we verify and throw a CorruptIndexException
+ // instead when checksums don't match.
+ CorruptIndexException e = expectThrows(CorruptIndexException.class, () -> {
new OfflineSorter(dir, "foo").sort(unsorted.getName());
});
- assertEquals(1, e.getSuppressed().length);
- assertTrue(e.getSuppressed()[0] instanceof CorruptIndexException);
- assertTrue(e.getSuppressed()[0].getMessage().contains("checksum failed (hardware problem?)"));
+ assertTrue(e.getMessage().contains("checksum failed (hardware problem?)"));
}
}
@@ -436,12 +433,10 @@ protected void corruptFile() throws IOException {
IndexOutput unsorted = dir.createTempOutput("unsorted", "tmp", IOContext.DEFAULT);
writeAll(unsorted, generateFixed((int) (OfflineSorter.MB * 3)));
- EOFException e = expectThrows(EOFException.class, () -> {
+ CorruptIndexException e = expectThrows(CorruptIndexException.class, () -> {
new OfflineSorter(dir, "foo", OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(1), 10, -1, null, 0).sort(unsorted.getName());
});
- assertEquals(1, e.getSuppressed().length);
- assertTrue(e.getSuppressed()[0] instanceof CorruptIndexException);
- assertTrue(e.getSuppressed()[0].getMessage().contains("checksum failed (hardware problem?)"));
+ assertTrue(e.getMessage().contains("checksum failed (hardware problem?)"));
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/util/TestVersion.java b/lucene/core/src/test/org/apache/lucene/util/TestVersion.java
index e19c61508128..02d566ea10ae 100644
--- a/lucene/core/src/test/org/apache/lucene/util/TestVersion.java
+++ b/lucene/core/src/test/org/apache/lucene/util/TestVersion.java
@@ -23,6 +23,7 @@
import java.util.Locale;
import java.util.Random;
+@SuppressWarnings("deprecation")
public class TestVersion extends LuceneTestCase {
public void testOnOrAfter() throws Exception {
diff --git a/lucene/core/src/test/org/apache/lucene/util/automaton/TestRegExp.java b/lucene/core/src/test/org/apache/lucene/util/automaton/TestRegExp.java
index 7d24939c3478..ce36eacfb10d 100644
--- a/lucene/core/src/test/org/apache/lucene/util/automaton/TestRegExp.java
+++ b/lucene/core/src/test/org/apache/lucene/util/automaton/TestRegExp.java
@@ -17,8 +17,12 @@
package org.apache.lucene.util.automaton;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
public class TestRegExp extends LuceneTestCase {
/**
@@ -83,4 +87,148 @@ public void testRepeatWithEmptyLanguage() throws Exception {
a = new RegExp("#?").toAutomaton(1000);
assertTrue(a.toString().length() > 0);
}
+
+ public void testCoreJavaParity() {
+ // Generate random doc values and random regular expressions
+ // and check for same matching behaviour as Java's Pattern class.
+ for (int i = 0; i < 1000; i++) {
+ checkRandomExpression(randomDocValue(1 + random().nextInt(30)));
+ }
+ }
+
+ public void testIllegalBackslashChars() {
+ String illegalChars = "abcefghijklmnopqrtuvxyzABCEFGHIJKLMNOPQRTUVXYZ";
+ for (int i = 0; i < illegalChars.length(); i++) {
+ String illegalExpression = "\\" + illegalChars.charAt(i);
+ IllegalArgumentException expected = expectThrows(
+ IllegalArgumentException.class, () -> {
+ new RegExp(illegalExpression);
+ }
+ );
+ assertTrue(expected.getMessage().contains("invalid character class"));
+ }
+ }
+
+ public void testLegalBackslashChars() {
+ String legalChars = "dDsSWw0123456789[]*&^$@!{}\\/";
+ for (int i = 0; i < legalChars.length(); i++) {
+ String legalExpression = "\\" + legalChars.charAt(i);
+ new RegExp(legalExpression);
+ }
+ }
+
+ static String randomDocValue(int minLength) {
+ String charPalette = "AAAaaaBbbCccc123456 \t";
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < minLength; i++) {
+ sb.append(charPalette.charAt(randomInt(charPalette.length() - 1)));
+ }
+ return sb.toString();
+ }
+
+ private static int randomInt(int bound) {
+ return bound == 0 ? 0 : random().nextInt(bound);
+ }
+
+ protected String checkRandomExpression(String docValue) {
+ // Generate and test a random regular expression which should match the given docValue
+ StringBuilder result = new StringBuilder();
+ // Pick a part of the string to change
+ int substitutionPoint = randomInt(docValue.length() - 1);
+ int substitutionLength = 1 + randomInt(Math.min(10, docValue.length() - substitutionPoint));
+
+ // Add any head to the result, unchanged
+ if (substitutionPoint > 0) {
+ result.append(docValue.substring(0, substitutionPoint));
+ }
+
+ // Modify the middle...
+ String replacementPart = docValue.substring(substitutionPoint, substitutionPoint + substitutionLength);
+ int mutation = random().nextInt(13);
+ switch (mutation) {
+ case 0:
+ // OR with random alpha of same length
+ result.append("(" + replacementPart + "|d" + randomDocValue(replacementPart.length()) + ")");
+ break;
+ case 1:
+ // OR with non-existant value
+ result.append("(" + replacementPart + "|doesnotexist)");
+ break;
+ case 2:
+ // OR with another randomised regex (used to create nested levels of expression).
+ result.append("(" + checkRandomExpression(replacementPart) + "|doesnotexist)");
+ break;
+ case 3:
+ // Star-replace all ab sequences.
+ result.append(replacementPart.replaceAll("ab", ".*"));
+ break;
+ case 4:
+ // .-replace all b chars
+ result.append(replacementPart.replaceAll("b", "."));
+ break;
+ case 5:
+ // length-limited stars {1,2}
+ result.append(".{1," + replacementPart.length() + "}");
+ break;
+ case 6:
+ // replace all chars with .
+ result.append(replacementPart.replaceAll(".", "."));
+ break;
+ case 7:
+ // OR with uppercase chars eg [aA] (many of these sorts of expression in the wild..
+ char[] chars = replacementPart.toCharArray();
+ for (char c : chars) {
+ result.append("[" + c + Character.toUpperCase(c) + "]");
+ }
+ break;
+ case 8:
+ // NOT a character - replace all b's with "not a"
+ result.append(replacementPart.replaceAll("b", "[^a]"));
+ break;
+ case 9:
+ // Make whole part repeatable 1 or more times
+ result.append("(" + replacementPart + ")+");
+ break;
+ case 10:
+ // Make whole part repeatable 0 or more times
+ result.append("(" + replacementPart + ")?");
+ break;
+ case 11:
+ // Make any digits replaced by character class
+ result.append(replacementPart.replaceAll("\\d", "\\\\d"));
+ break;
+ case 12:
+ // Make any whitespace chars replaced by not word class
+ result.append(replacementPart.replaceAll("\\s", "\\\\W"));
+ break;
+ case 13:
+ // Make any whitespace chars replace by whitespace class
+ result.append(replacementPart.replaceAll("\\s", "\\\\s"));
+ break;
+ default:
+ break;
+ }
+ // add any remaining tail, unchanged
+ if (substitutionPoint + substitutionLength <= docValue.length() - 1) {
+ result.append(docValue.substring(substitutionPoint + substitutionLength));
+ }
+
+ String regexPattern = result.toString();
+ // Assert our randomly generated regex actually matches the provided raw input using java's expression matcher
+ Pattern pattern = Pattern.compile(regexPattern);
+ Matcher matcher = pattern.matcher(docValue);
+ assertTrue("Java regex " + regexPattern + " did not match doc value " + docValue, matcher.matches());
+
+ RegExp regex = new RegExp(regexPattern);
+ Automaton automaton = regex.toAutomaton();
+ ByteRunAutomaton bytesMatcher = new ByteRunAutomaton(automaton);
+ BytesRef br = new BytesRef(docValue);
+ assertTrue(
+ "[" + regexPattern + "]should match [" + docValue + "]" + substitutionPoint + "-" + substitutionLength + "/"
+ + docValue.length(),
+ bytesMatcher.run(br.bytes, br.offset, br.length)
+ );
+ return regexPattern;
+ }
+
}
diff --git a/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java b/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java
index deccdf0498a6..27b5511a9822 100644
--- a/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java
+++ b/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java
@@ -58,12 +58,14 @@ public void test1D() throws Exception {
}
}
IndexOutput out = dir.createOutput("1d.bkd", IOContext.DEFAULT);
- long indexFP = w.finish(out);
+ Runnable finalizer = w.finish(out, out, out);
+ long indexFP = out.getFilePointer();
+ finalizer.run();
out.close();
IndexInput in = dir.openInput("1d.bkd", IOContext.DEFAULT);
in.seek(indexFP);
- BKDReader r = new BKDReader(in);
+ BKDReader r = new BKDReader(in, in, in);
CheckIndex.VerifyPointsVisitor visitor = new CheckIndex.VerifyPointsVisitor("1d", numDocs, r);
r.intersect(visitor);
assertEquals(r.size(), visitor.getPointCountSeen());
@@ -98,12 +100,14 @@ public void test2D() throws Exception {
}
}
IndexOutput out = dir.createOutput("2d.bkd", IOContext.DEFAULT);
- long indexFP = w.finish(out);
+ Runnable finalizer = w.finish(out, out, out);
+ long indexFP = out.getFilePointer();
+ finalizer.run();
out.close();
IndexInput in = dir.openInput("2d.bkd", IOContext.DEFAULT);
in.seek(indexFP);
- BKDReader r = new BKDReader(in);
+ BKDReader r = new BKDReader(in, in, in);
CheckIndex.VerifyPointsVisitor visitor = new CheckIndex.VerifyPointsVisitor("2d", numDocs, r);
r.intersect(visitor);
assertEquals(r.size(), visitor.getPointCountSeen());
diff --git a/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java b/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java
index 38d91f20f9b0..f9bb9ea56b59 100644
--- a/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java
+++ b/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java
@@ -46,8 +46,6 @@
import org.apache.lucene.util.NumericUtils;
import org.apache.lucene.util.TestUtil;
-import static com.carrotsearch.randomizedtesting.RandomizedTest.randomBoolean;
-
public class TestBKD extends LuceneTestCase {
public void testBasicInts1D() throws Exception {
@@ -61,12 +59,14 @@ public void testBasicInts1D() throws Exception {
long indexFP;
try (IndexOutput out = dir.createOutput("bkd", IOContext.DEFAULT)) {
- indexFP = w.finish(out);
+ Runnable finalizer = w.finish(out, out, out);
+ indexFP = out.getFilePointer();
+ finalizer.run();
}
try (IndexInput in = dir.openInput("bkd", IOContext.DEFAULT)) {
in.seek(indexFP);
- BKDReader r = new BKDReader(in, randomBoolean());
+ BKDReader r = new BKDReader(in, in, in);
// Simple 1D range query:
final int queryMin = 42;
@@ -163,12 +163,14 @@ public void testRandomIntsNDims() throws Exception {
long indexFP;
try (IndexOutput out = dir.createOutput("bkd", IOContext.DEFAULT)) {
- indexFP = w.finish(out);
+ Runnable finalizer = w.finish(out, out, out);
+ indexFP = out.getFilePointer();
+ finalizer.run();
}
try (IndexInput in = dir.openInput("bkd", IOContext.DEFAULT)) {
in.seek(indexFP);
- BKDReader r = new BKDReader(in, randomBoolean());
+ BKDReader r = new BKDReader(in, in, in);
byte[] minPackedValue = r.getMinPackedValue();
byte[] maxPackedValue = r.getMaxPackedValue();
@@ -290,13 +292,15 @@ public void testBigIntNDims() throws Exception {
}
long indexFP;
- try (IndexOutput out = dir.createOutput("bkd", IOContext.DEFAULT)) {
- indexFP = w.finish(out);
+ try (IndexOutput out = dir.createOutput("bkd", IOContext.DEFAULT)) {
+ Runnable finalizer = w.finish(out, out, out);
+ indexFP = out.getFilePointer();
+ finalizer.run();
}
try (IndexInput in = dir.openInput("bkd", IOContext.DEFAULT)) {
in.seek(indexFP);
- BKDReader r = new BKDReader(in, randomBoolean());
+ BKDReader r = new BKDReader(in, in, in);
int iters = atLeast(100);
for(int iter=0;iter 0) {
- toMerge.add(w.finish(out));
+ Runnable finalizer = w.finish(out, out, out);
+ toMerge.add(out.getFilePointer());
+ finalizer.run();
final int curDocIDBase = lastDocIDBase;
docMaps.add(new MergeState.DocMap() {
@Override
@@ -788,21 +796,25 @@ public int get(int docID) {
List readers = new ArrayList<>();
for(long fp : toMerge) {
in.seek(fp);
- readers.add(new BKDReader(in, randomBoolean()));
+ readers.add(new BKDReader(in, in, in));
}
out = dir.createOutput("bkd2", IOContext.DEFAULT);
- indexFP = w.merge(out, docMaps, readers);
+ Runnable finalizer = w.merge(out, out, out, docMaps, readers);
+ indexFP = out.getFilePointer();
+ finalizer.run();
out.close();
in.close();
in = dir.openInput("bkd2", IOContext.DEFAULT);
} else {
- indexFP = w.finish(out);
+ Runnable finalizer = w.finish(out, out, out);
+ indexFP = out.getFilePointer();
+ finalizer.run();
out.close();
in = dir.openInput("bkd", IOContext.DEFAULT);
}
in.seek(indexFP);
- BKDReader r = new BKDReader(in, randomBoolean());
+ BKDReader r = new BKDReader(in, in, in);
int iters = atLeast(100);
for(int iter=0;iter maxPointsInLeafNode) {
- actualMaxPointsInLeafNode = (actualMaxPointsInLeafNode + 1) / 2;
- }
+ BKDReader points = new BKDReader(pointsIn, pointsIn, pointsIn);
// If all points match, then the point count is numLeaves * maxPointsInLeafNode
- final int numLeaves = Integer.highestOneBit((numValues - 1) / actualMaxPointsInLeafNode) << 1;
- assertEquals(numLeaves * actualMaxPointsInLeafNode,
+ int numLeaves = numValues / maxPointsInLeafNode;
+ if (numValues % maxPointsInLeafNode != 0) {
+ numLeaves++;
+ }
+ assertEquals(numLeaves * maxPointsInLeafNode,
points.estimatePointCount(new IntersectVisitor() {
@Override
public void visit(int docID, byte[] packedValue) throws IOException {}
@@ -1363,8 +1383,8 @@ public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
}
});
assertTrue(""+pointCount,
- pointCount == (actualMaxPointsInLeafNode + 1) / 2 || // common case
- pointCount == 2*((actualMaxPointsInLeafNode + 1) / 2)); // if the point is a split value
+ pointCount == (maxPointsInLeafNode + 1) / 2 || // common case
+ pointCount == 2*((maxPointsInLeafNode + 1) / 2)); // if the point is a split value
pointsIn.close();
dir.close();
@@ -1452,7 +1472,7 @@ public byte getByteAt(int i, int k) {
BKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP, numValues);
expectThrows(IllegalStateException.class, () -> {
try (IndexOutput out = dir.createOutput("bkd", IOContext.DEFAULT)) {
- w.writeField(out, "test_field_name", reader);
+ w.writeField(out, out, out, "test_field_name", reader);
} finally {
w.close();
dir.close();
@@ -1563,7 +1583,7 @@ public int getDocCount() {
}
};
try (IndexOutput out = dir.createOutput("bkd", IOContext.DEFAULT)) {
- IllegalStateException ex = expectThrows(IllegalStateException.class, () -> { w.writeField(out, "", val);});
+ IllegalStateException ex = expectThrows(IllegalStateException.class, () -> { w.writeField(out, out, out, "", val);});
assertEquals("totalPointCount=10 was passed when we were created, but we just hit 11 values", ex.getMessage());
w.close();
}
diff --git a/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java b/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java
index 9ee6947b2e3f..8725b032af1f 100644
--- a/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java
+++ b/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java
@@ -119,10 +119,10 @@ public void test() throws Exception {
if (verify == 0) {
System.out.println("\nTEST: save/load FST and re-verify");
IndexOutput out = dir.createOutput("fst", IOContext.DEFAULT);
- fst.save(out);
+ fst.save(out, out);
out.close();
IndexInput in = dir.openInput("fst", IOContext.DEFAULT);
- fst = new FST<>(in, outputs);
+ fst = new FST<>(in, in, outputs);
in.close();
} else {
dir.deleteFile("fst");
@@ -198,10 +198,10 @@ public void test() throws Exception {
if (verify == 0) {
System.out.println("\nTEST: save/load FST and re-verify");
IndexOutput out = dir.createOutput("fst", IOContext.DEFAULT);
- fst.save(out);
+ fst.save(out, out);
out.close();
IndexInput in = dir.openInput("fst", IOContext.DEFAULT);
- fst = new FST<>(in, outputs);
+ fst = new FST<>(in, in, outputs);
in.close();
} else {
dir.deleteFile("fst");
@@ -256,7 +256,9 @@ public void test() throws Exception {
// forward lookup:
assertEquals(output, Util.get(fst, input).longValue());
// reverse lookup:
- assertEquals(input, Util.getByOutput(fst, output));
+ @SuppressWarnings("deprecation")
+ IntsRef inputResult = Util.getByOutput(fst, output);
+ assertEquals(input, inputResult);
output += 1 + r.nextInt(10);
nextInput(r, ints);
}
@@ -284,10 +286,10 @@ public void test() throws Exception {
if (verify == 0) {
System.out.println("\nTEST: save/load FST and re-verify");
IndexOutput out = dir.createOutput("fst", IOContext.DEFAULT);
- fst.save(out);
+ fst.save(out, out);
out.close();
IndexInput in = dir.openInput("fst", IOContext.DEFAULT);
- fst = new FST<>(in, outputs);
+ fst = new FST<>(in, in, outputs);
in.close();
} else {
dir.deleteFile("fst");
diff --git a/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTDirectAddressing.java b/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTDirectAddressing.java
index 25ea6f6fe0e4..6cf2d76f3f14 100644
--- a/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTDirectAddressing.java
+++ b/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTDirectAddressing.java
@@ -174,7 +174,7 @@ public static void main(String... args) throws Exception {
private static void countFSTArcs(String fstFilePath) throws IOException {
byte[] buf = Files.readAllBytes(Paths.get(fstFilePath));
DataInput in = new ByteArrayDataInput(buf);
- FST fst = new FST<>(in, ByteSequenceOutputs.getSingleton());
+ FST fst = new FST<>(in, in, ByteSequenceOutputs.getSingleton());
BytesRefFSTEnum fstEnum = new BytesRefFSTEnum<>(fst);
int binarySearchArcCount = 0, directAddressingArcCount = 0, listArcCount = 0;
while(fstEnum.next() != null) {
@@ -228,7 +228,7 @@ private static void recompileAndWalk(String fstFilePath) throws IOException {
System.out.println("Reading FST");
long startTimeMs = System.currentTimeMillis();
- FST originalFst = new FST<>(in, CharSequenceOutputs.getSingleton());
+ FST originalFst = new FST<>(in, in, CharSequenceOutputs.getSingleton());
long endTimeMs = System.currentTimeMillis();
System.out.println("time = " + (endTimeMs - startTimeMs) + " ms");
diff --git a/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java b/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java
index e081f6c2f5e6..10319f9eb54e 100644
--- a/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java
+++ b/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java
@@ -529,7 +529,7 @@ public void run(int limit, boolean verify, boolean verifyByOutput) throws IOExce
Directory dir = FSDirectory.open(dirOut);
IndexOutput out = dir.createOutput("fst.bin", IOContext.DEFAULT);
- fst.save(out);
+ fst.save(out, out);
out.close();
System.out.println("Saved FST to fst.bin.");
@@ -570,7 +570,7 @@ public void run(int limit, boolean verify, boolean verifyByOutput) throws IOExce
} else {
// Get by output
final Long output = (Long) getOutput(intsRef.get(), ord);
- @SuppressWarnings("unchecked") final IntsRef actual = Util.getByOutput((FST) fst, output.longValue());
+ @SuppressWarnings({"unchecked", "deprecation"}) final IntsRef actual = Util.getByOutput((FST) fst, output.longValue());
if (actual == null) {
throw new RuntimeException("unexpected null input from output=" + output);
}
@@ -833,13 +833,17 @@ public void testSimple() throws Exception {
assertEquals(b, seekResult.input);
assertEquals(42, (long) seekResult.output);
- assertEquals(Util.toIntsRef(new BytesRef("c"), new IntsRefBuilder()),
- Util.getByOutput(fst, 13824324872317238L));
- assertNull(Util.getByOutput(fst, 47));
- assertEquals(Util.toIntsRef(new BytesRef("b"), new IntsRefBuilder()),
- Util.getByOutput(fst, 42));
- assertEquals(Util.toIntsRef(new BytesRef("a"), new IntsRefBuilder()),
- Util.getByOutput(fst, 17));
+ @SuppressWarnings("deprecation") IntsRef byOutput = Util.getByOutput(fst, 13824324872317238L);
+ assertEquals(Util.toIntsRef(new BytesRef("c"), new IntsRefBuilder()), byOutput);
+
+ @SuppressWarnings("deprecation") IntsRef byOutput47 = Util.getByOutput(fst, 47);
+ assertNull(byOutput47);
+
+ @SuppressWarnings("deprecation") IntsRef byOutput42 = Util.getByOutput(fst, 42);
+ assertEquals(Util.toIntsRef(new BytesRef("b"), new IntsRefBuilder()), byOutput42);
+
+ @SuppressWarnings("deprecation") IntsRef byOutput17 = Util.getByOutput(fst, 17);
+ assertEquals(Util.toIntsRef(new BytesRef("a"), new IntsRefBuilder()), byOutput17);
}
public void testPrimaryKeys() throws Exception {
@@ -1191,11 +1195,11 @@ public void testNonFinalStopNode() throws Exception {
// Make sure it still works after save/load:
Directory dir = newDirectory();
IndexOutput out = dir.createOutput("fst", IOContext.DEFAULT);
- fst.save(out);
+ fst.save(out, out);
out.close();
IndexInput in = dir.openInput("fst", IOContext.DEFAULT);
- final FST fst2 = new FST<>(in, outputs);
+ final FST fst2 = new FST<>(in, in, outputs);
checkStopNodes(fst2, outputs);
in.close();
dir.close();
diff --git a/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java b/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
index 27ac460e9c4e..497052baed99 100644
--- a/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
+++ b/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
@@ -836,7 +836,9 @@ public void testEncodeDecode() {
final long[] blocks = new long[blocksOffset + blocksLen];
for (int i = 0; i < blocks.length; ++i) {
blocks[i] = random().nextLong();
- if (format == PackedInts.Format.PACKED_SINGLE_BLOCK && 64 % bpv != 0) {
+ @SuppressWarnings("deprecation")
+ PackedInts.Format PACKED_SINGLE_BLOCK = PackedInts.Format.PACKED_SINGLE_BLOCK;
+ if (format == PACKED_SINGLE_BLOCK && 64 % bpv != 0) {
// clear highest bits for packed
final int toClear = 64 % bpv;
blocks[i] = (blocks[i] << toClear) >>> toClear;
diff --git a/lucene/demo/build.gradle b/lucene/demo/build.gradle
index 2871774bbf88..8407fa6b5d63 100644
--- a/lucene/demo/build.gradle
+++ b/lucene/demo/build.gradle
@@ -17,6 +17,8 @@
apply plugin: 'java-library'
+description = 'Simple example code for Apache Lucene'
+
dependencies {
implementation project(':lucene:core')
implementation project(':lucene:facet')
diff --git a/lucene/expressions/build.gradle b/lucene/expressions/build.gradle
index 62d197830e76..7ba76a7f3a97 100644
--- a/lucene/expressions/build.gradle
+++ b/lucene/expressions/build.gradle
@@ -17,6 +17,8 @@
apply plugin: 'java-library'
+description = 'Dynamically computed values to sort/facet/search on based on a pluggable grammar'
+
dependencies {
api project(':lucene:core')
diff --git a/lucene/expressions/src/java/org/apache/lucene/expressions/ExpressionValueSource.java b/lucene/expressions/src/java/org/apache/lucene/expressions/ExpressionValueSource.java
index f4fa894af8a6..1f8713dd38f3 100644
--- a/lucene/expressions/src/java/org/apache/lucene/expressions/ExpressionValueSource.java
+++ b/lucene/expressions/src/java/org/apache/lucene/expressions/ExpressionValueSource.java
@@ -174,7 +174,7 @@ public DoubleValuesSource rewrite(IndexSearcher searcher) throws IOException {
DoubleValuesSource[] rewritten = new DoubleValuesSource[variables.length];
for (int i = 0; i < variables.length; i++) {
rewritten[i] = variables[i].rewrite(searcher);
- changed |= (rewritten[i] == variables[i]);
+ changed |= (rewritten[i] != variables[i]);
}
if (changed) {
return new ExpressionValueSource(rewritten, expression, needsScores);
diff --git a/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionValueSource.java b/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionValueSource.java
index d5dbabee5478..3b5589b2d7e3 100644
--- a/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionValueSource.java
+++ b/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionValueSource.java
@@ -28,9 +28,12 @@
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.search.DoubleValues;
import org.apache.lucene.search.DoubleValuesSource;
+import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
+import java.io.IOException;
+
public class TestExpressionValueSource extends LuceneTestCase {
DirectoryReader reader;
Directory dir;
@@ -125,4 +128,59 @@ public void testDoubleValuesSourceEquals() throws Exception {
assertFalse(vs1.equals(vs4));
}
+ public void testRewrite() throws Exception {
+ Expression expr = JavascriptCompiler.compile("a");
+
+ ExpressionValueSource rewritingExpressionSource = new ExpressionValueSource(
+ new DoubleValuesSource[]{createDoubleValuesSourceMock(true)},
+ expr,
+ false);
+ ExpressionValueSource notRewritingExpressionSource = new ExpressionValueSource(
+ new DoubleValuesSource[]{createDoubleValuesSourceMock(false)},
+ expr,
+ false);
+
+ assertNotSame(rewritingExpressionSource, rewritingExpressionSource.rewrite(null));
+ assertSame(notRewritingExpressionSource, notRewritingExpressionSource.rewrite(null));
+ }
+
+ private static DoubleValuesSource createDoubleValuesSourceMock(boolean rewriting) {
+ return new DoubleValuesSource() {
+ @Override
+ public DoubleValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
+ return null;
+ }
+
+ @Override
+ public boolean needsScores() {
+ return false;
+ }
+
+ @Override
+ public DoubleValuesSource rewrite(IndexSearcher reader) throws IOException {
+ return rewriting ? createDoubleValuesSourceMock(true) : this;
+ }
+
+ @Override
+ public int hashCode() {
+ return 0;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return null;
+ }
+
+ @Override
+ public boolean isCacheable(LeafReaderContext ctx) {
+ return false;
+ }
+ };
+ }
+
}
diff --git a/lucene/facet/build.gradle b/lucene/facet/build.gradle
index c0ffc91e9298..6b6a6ef137e8 100644
--- a/lucene/facet/build.gradle
+++ b/lucene/facet/build.gradle
@@ -18,6 +18,7 @@
apply plugin: 'java-library'
+description = 'Faceted indexing and search capabilities'
dependencies {
api project(':lucene:core')
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/DrillSideways.java b/lucene/facet/src/java/org/apache/lucene/facet/DrillSideways.java
index 15df7e3d957b..6dbacf19578f 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/DrillSideways.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/DrillSideways.java
@@ -299,7 +299,7 @@ public DrillSidewaysResult search(ScoreDoc after, DrillDownQuery query, int topN
if (executor != null) { // We have an executor, let use the multi-threaded version
final CollectorManager collectorManager =
- new CollectorManager() {
+ new CollectorManager<>() {
@Override
public TopScoreDocCollector newCollector() throws IOException {
@@ -404,6 +404,7 @@ private DrillDownQuery getDrillDownQuery(final DrillDownQuery query, Query[] que
}
/** Runs a search, using a {@link CollectorManager} to gather and merge search results */
+ @SuppressWarnings("unchecked")
public ConcurrentDrillSidewaysResult search(final DrillDownQuery query,
final CollectorManager hitCollectorManager) throws IOException {
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/FacetsConfig.java b/lucene/facet/src/java/org/apache/lucene/facet/FacetsConfig.java
index f958af187805..84f69d7bc0a4 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/FacetsConfig.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/FacetsConfig.java
@@ -496,8 +496,15 @@ private void checkTaxoWriter(TaxonomyWriter taxoWriter) {
}
}
- // Joins the path components together:
- private static final char DELIM_CHAR = '\u001F';
+ /**
+ * Character used to join the category path components together into a single
+ * drill down term for indexing. Applications and unit-tests can reference this for
+ * creating their own drill-down terms, or use existing APIs (for example,
+ * {@link #pathToString}).
+ *
+ * @lucene.internal
+ */
+ public static final char DELIM_CHAR = '\u001F';
// Escapes any occurrence of the path component inside the label:
private static final char ESCAPE_CHAR = '\u001E';
diff --git a/lucene/grouping/build.gradle b/lucene/grouping/build.gradle
index b18d3d9cf4dd..b0838e3dfd23 100644
--- a/lucene/grouping/build.gradle
+++ b/lucene/grouping/build.gradle
@@ -18,6 +18,8 @@
apply plugin: 'java-library'
+description = 'Collectors for grouping search results'
+
dependencies {
api project(':lucene:core')
diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/BlockGroupingCollector.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/BlockGroupingCollector.java
index 23601ca994b0..bbeb2ee204b4 100644
--- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/BlockGroupingCollector.java
+++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/BlockGroupingCollector.java
@@ -67,6 +67,10 @@
* @lucene.experimental
*/
+// TODO: TopGroups.merge() won't work with TopGroups returned by this collector, because
+// each block will be on a different shard. Add a specialized merge() static method
+// to this collector?
+
public class BlockGroupingCollector extends SimpleCollector {
private int[] pendingSubDocs;
diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/DoubleRange.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/DoubleRange.java
new file mode 100644
index 000000000000..df34f6b1e0df
--- /dev/null
+++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/DoubleRange.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.search.grouping;
+
+import java.util.Objects;
+
+/**
+ * Represents a contiguous range of double values, with an inclusive minimum and
+ * exclusive maximum
+ */
+public class DoubleRange {
+
+ /** The inclusive minimum value of this range */
+ public double min;
+ /** The exclusive maximum value of this range */
+ public double max;
+
+ /**
+ * Creates a new double range, running from {@code min} inclusive to {@code max} exclusive
+ */
+ public DoubleRange(double min, double max) {
+ this.min = min;
+ this.max = max;
+ }
+
+ @Override
+ public String toString() {
+ return "DoubleRange(" + min + ", " + max + ")";
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ DoubleRange that = (DoubleRange) o;
+ return Double.compare(that.min, min) == 0 &&
+ Double.compare(that.max, max) == 0;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(min, max);
+ }
+}
diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/DoubleRangeFactory.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/DoubleRangeFactory.java
new file mode 100644
index 000000000000..3ea4606a6431
--- /dev/null
+++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/DoubleRangeFactory.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.search.grouping;
+
+/**
+ * Groups double values into ranges
+ */
+public class DoubleRangeFactory {
+
+ private final double min;
+ private final double width;
+ private final double max;
+
+ /**
+ * Creates a new DoubleRangeFactory
+ * @param min a minimum value; all doubles below this value are grouped into a single range
+ * @param width a standard width; all ranges between {@code min} and {@code max} are this wide,
+ * with the exception of the final range which may be up to this width. Ranges
+ * are inclusive at the lower end, and exclusive at the upper end.
+ * @param max a maximum value; all doubles above this value are grouped into a single range
+ */
+ public DoubleRangeFactory(double min, double width, double max) {
+ this.min = min;
+ this.width = width;
+ this.max = max;
+ }
+
+ /**
+ * Finds the DoubleRange that a value should be grouped into
+ * @param value the value to group
+ * @param reuse an existing DoubleRange object to reuse
+ */
+ public DoubleRange getRange(double value, DoubleRange reuse) {
+ if (reuse == null)
+ reuse = new DoubleRange(Double.MIN_VALUE, Double.MAX_VALUE);
+ if (value < min) {
+ reuse.max = min;
+ reuse.min = Double.MIN_VALUE;
+ return reuse;
+ }
+ if (value >= max) {
+ reuse.min = max;
+ reuse.max = Double.MAX_VALUE;
+ return reuse;
+ }
+ double bucket = Math.floor((value - min) / width);
+ reuse.min = min + (bucket * width);
+ reuse.max = reuse.min + width;
+ return reuse;
+ }
+
+}
diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/DoubleRangeGroupSelector.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/DoubleRangeGroupSelector.java
new file mode 100644
index 000000000000..4a6a65a72366
--- /dev/null
+++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/DoubleRangeGroupSelector.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.search.grouping;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.DoubleValues;
+import org.apache.lucene.search.DoubleValuesSource;
+import org.apache.lucene.search.Scorable;
+
+/**
+ * A GroupSelector implementation that groups documents by double values
+ */
+public class DoubleRangeGroupSelector extends GroupSelector {
+
+ private final DoubleValuesSource source;
+ private final DoubleRangeFactory rangeFactory;
+
+ private Set inSecondPass;
+ private boolean includeEmpty = true;
+ private boolean positioned;
+ private DoubleRange current;
+
+ private LeafReaderContext context;
+ private DoubleValues values;
+
+ /**
+ * Creates a new DoubleRangeGroupSelector
+ * @param source a DoubleValuesSource to retrieve double values per document
+ * @param rangeFactory a DoubleRangeFactory that defines how to group the double values into range buckets
+ */
+ public DoubleRangeGroupSelector(DoubleValuesSource source, DoubleRangeFactory rangeFactory) {
+ this.source = source;
+ this.rangeFactory = rangeFactory;
+ }
+
+ @Override
+ public void setNextReader(LeafReaderContext readerContext) throws IOException {
+ this.context = readerContext;
+ }
+
+ @Override
+ public void setScorer(Scorable scorer) throws IOException {
+ this.values = source.getValues(context, DoubleValuesSource.fromScorer(scorer));
+ }
+
+ @Override
+ public State advanceTo(int doc) throws IOException {
+ positioned = values.advanceExact(doc);
+ if (positioned == false) {
+ return includeEmpty ? State.ACCEPT : State.SKIP;
+ }
+ this.current = rangeFactory.getRange(values.doubleValue(), this.current);
+ if (inSecondPass == null) {
+ return State.ACCEPT;
+ }
+ return inSecondPass.contains(this.current) ? State.ACCEPT : State.SKIP;
+ }
+
+ @Override
+ public DoubleRange currentValue() throws IOException {
+ return positioned ? this.current : null;
+ }
+
+ @Override
+ public DoubleRange copyValue() throws IOException {
+ return positioned ? new DoubleRange(this.current.min, this.current.max) : null;
+ }
+
+ @Override
+ public void setGroups(Collection> searchGroups) {
+ inSecondPass = new HashSet<>();
+ includeEmpty = false;
+ for (SearchGroup group : searchGroups) {
+ if (group.groupValue == null)
+ includeEmpty = true;
+ else
+ inSecondPass.add(group.groupValue);
+ }
+ }
+}
diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/FirstPassGroupingCollector.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/FirstPassGroupingCollector.java
index 6a745b8e7352..f5b05974e416 100644
--- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/FirstPassGroupingCollector.java
+++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/FirstPassGroupingCollector.java
@@ -151,6 +151,7 @@ public Collection> getTopGroups(int groupOffset) throws IOExcepti
@Override
public void setScorer(Scorable scorer) throws IOException {
+ groupSelector.setScorer(scorer);
for (LeafFieldComparator comparator : leafComparators) {
comparator.setScorer(scorer);
}
diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/GroupSelector.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/GroupSelector.java
index dbb09329f8ae..92962a4d4570 100644
--- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/GroupSelector.java
+++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/GroupSelector.java
@@ -21,6 +21,7 @@
import java.util.Collection;
import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.Scorable;
/**
* Defines a group, for use by grouping collectors
@@ -43,6 +44,11 @@ public enum State { SKIP, ACCEPT }
*/
public abstract void setNextReader(LeafReaderContext readerContext) throws IOException;
+ /**
+ * Set the current Scorer
+ */
+ public abstract void setScorer(Scorable scorer) throws IOException;
+
/**
* Advance the GroupSelector's iterator to the given document
*/
@@ -53,12 +59,12 @@ public enum State { SKIP, ACCEPT }
*
* N.B. this object may be reused, for a persistent version use {@link #copyValue()}
*/
- public abstract T currentValue();
+ public abstract T currentValue() throws IOException;
/**
* @return a copy of the group value of the current document
*/
- public abstract T copyValue();
+ public abstract T copyValue() throws IOException;
/**
* Set a restriction on the group values returned by this selector
diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/GroupingSearch.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/GroupingSearch.java
index b88fb743f087..25ed3770addc 100644
--- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/GroupingSearch.java
+++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/GroupingSearch.java
@@ -71,6 +71,14 @@ public GroupingSearch(String groupField) {
this(new TermGroupSelector(groupField), null);
}
+ /**
+ * Constructs a GroupingSearch instance that groups documents using a {@link GroupSelector}
+ * @param groupSelector a {@link GroupSelector} that defines groups for this GroupingSearch
+ */
+ public GroupingSearch(GroupSelector groupSelector) {
+ this(groupSelector, null);
+ }
+
/**
* Constructs a GroupingSearch instance that groups documents by function using a {@link ValueSource}
* instance.
@@ -78,7 +86,7 @@ public GroupingSearch(String groupField) {
* @param groupFunction The function to group by specified as {@link ValueSource}
* @param valueSourceContext The context of the specified groupFunction
*/
- public GroupingSearch(ValueSource groupFunction, Map valueSourceContext) {
+ public GroupingSearch(ValueSource groupFunction, Map valueSourceContext) {
this(new ValueSourceGroupSelector(groupFunction, valueSourceContext), null);
}
diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/LongRange.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/LongRange.java
new file mode 100644
index 000000000000..7b6c845aa4a9
--- /dev/null
+++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/LongRange.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.search.grouping;
+
+import java.util.Objects;
+
+/**
+ * Represents a contiguous range of long values, with an inclusive minimum and
+ * exclusive maximum
+ */
+public class LongRange {
+
+ /** The inclusive minimum value of this range */
+ public long min;
+ /** The exclusive maximum value of this range */
+ public long max;
+
+ /**
+ * Creates a new double range, running from {@code min} inclusive to {@code max} exclusive
+ */
+ public LongRange(long min, long max) {
+ this.min = min;
+ this.max = max;
+ }
+
+ @Override
+ public String toString() {
+ return "LongRange(" + min + ", " + max + ")";
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ LongRange that = (LongRange) o;
+ return that.min == min && that.max == max;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(min, max);
+ }
+}
diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/LongRangeFactory.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/LongRangeFactory.java
new file mode 100644
index 000000000000..be66647a5e99
--- /dev/null
+++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/LongRangeFactory.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.search.grouping;
+
+/**
+ * Groups double values into ranges
+ */
+public class LongRangeFactory {
+
+ private final long min;
+ private final long width;
+ private final long max;
+
+ /**
+ * Creates a new LongRangeFactory
+ * @param min a minimum value; all longs below this value are grouped into a single range
+ * @param width a standard width; all ranges between {@code min} and {@code max} are this wide,
+ * with the exception of the final range which may be up to this width. Ranges
+ * are inclusive at the lower end, and exclusive at the upper end.
+ * @param max a maximum value; all longs above this value are grouped into a single range
+ */
+ public LongRangeFactory(long min, long width, long max) {
+ this.min = min;
+ this.width = width;
+ this.max = max;
+ }
+
+ /**
+ * Finds the LongRange that a value should be grouped into
+ * @param value the value to group
+ * @param reuse an existing LongRange object to reuse
+ */
+ public LongRange getRange(long value, LongRange reuse) {
+ if (reuse == null)
+ reuse = new LongRange(Long.MIN_VALUE, Long.MAX_VALUE);
+ if (value < min) {
+ reuse.max = min;
+ reuse.min = Long.MIN_VALUE;
+ return reuse;
+ }
+ if (value >= max) {
+ reuse.min = max;
+ reuse.max = Long.MAX_VALUE;
+ return reuse;
+ }
+ long bucket = (value - min) / width;
+ reuse.min = min + (bucket * width);
+ reuse.max = reuse.min + width;
+ return reuse;
+ }
+
+}
diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/LongRangeGroupSelector.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/LongRangeGroupSelector.java
new file mode 100644
index 000000000000..7dd0c238d925
--- /dev/null
+++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/LongRangeGroupSelector.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.search.grouping;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.DoubleValuesSource;
+import org.apache.lucene.search.LongValues;
+import org.apache.lucene.search.LongValuesSource;
+import org.apache.lucene.search.Scorable;
+
+/**
+ * A GroupSelector implementation that groups documents by long values
+ */
+public class LongRangeGroupSelector extends GroupSelector {
+
+ private final LongValuesSource source;
+ private final LongRangeFactory rangeFactory;
+
+ private Set inSecondPass;
+ private boolean includeEmpty = true;
+ private boolean positioned;
+ private LongRange current;
+
+ private LeafReaderContext context;
+ private LongValues values;
+
+ /**
+ * Creates a new LongRangeGroupSelector
+ * @param source a LongValuesSource to retrieve long values per document
+ * @param rangeFactory a LongRangeFactory that defines how to group the long values into range buckets
+ */
+ public LongRangeGroupSelector(LongValuesSource source, LongRangeFactory rangeFactory) {
+ this.source = source;
+ this.rangeFactory = rangeFactory;
+ }
+
+ @Override
+ public void setNextReader(LeafReaderContext readerContext) throws IOException {
+ this.context = readerContext;
+ }
+
+ @Override
+ public void setScorer(Scorable scorer) throws IOException {
+ this.values = source.getValues(context, DoubleValuesSource.fromScorer(scorer));
+ }
+
+ @Override
+ public State advanceTo(int doc) throws IOException {
+ positioned = values.advanceExact(doc);
+ if (positioned == false) {
+ return includeEmpty ? State.ACCEPT : State.SKIP;
+ }
+ this.current = rangeFactory.getRange(values.longValue(), this.current);
+ if (inSecondPass == null) {
+ return State.ACCEPT;
+ }
+ return inSecondPass.contains(this.current) ? State.ACCEPT : State.SKIP;
+ }
+
+ @Override
+ public LongRange currentValue() throws IOException {
+ return positioned ? this.current : null;
+ }
+
+ @Override
+ public LongRange copyValue() throws IOException {
+ return positioned ? new LongRange(this.current.min, this.current.max) : null;
+ }
+
+ @Override
+ public void setGroups(Collection> searchGroups) {
+ inSecondPass = new HashSet<>();
+ includeEmpty = false;
+ for (SearchGroup group : searchGroups) {
+ if (group.groupValue == null)
+ includeEmpty = true;
+ else
+ inSecondPass.add(group.groupValue);
+ }
+ }
+}
diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/SecondPassGroupingCollector.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/SecondPassGroupingCollector.java
index 0d5fc9daa26c..dc7d0aabde76 100644
--- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/SecondPassGroupingCollector.java
+++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/SecondPassGroupingCollector.java
@@ -78,6 +78,7 @@ public ScoreMode scoreMode() {
@Override
public void setScorer(Scorable scorer) throws IOException {
+ groupSelector.setScorer(scorer);
groupReducer.setScorer(scorer);
}
diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/TermGroupSelector.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/TermGroupSelector.java
index 5b8f77c2ce83..65213b2dc9a4 100644
--- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/TermGroupSelector.java
+++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/TermGroupSelector.java
@@ -25,6 +25,7 @@
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.search.Scorable;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefHash;
@@ -64,6 +65,9 @@ public void setNextReader(LeafReaderContext readerContext) throws IOException {
}
}
+ @Override
+ public void setScorer(Scorable scorer) throws IOException { }
+
@Override
public State advanceTo(int doc) throws IOException {
if (this.docValues.advanceExact(doc) == false) {
diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/ValueSourceGroupSelector.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/ValueSourceGroupSelector.java
index 249016018855..54373de8be25 100644
--- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/ValueSourceGroupSelector.java
+++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/ValueSourceGroupSelector.java
@@ -26,6 +26,7 @@
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
+import org.apache.lucene.search.Scorable;
import org.apache.lucene.util.mutable.MutableValue;
/**
@@ -34,7 +35,7 @@
public class ValueSourceGroupSelector extends GroupSelector {
private final ValueSource valueSource;
- private final Map context;
+ private final Map context;
private Set secondPassGroups;
@@ -43,7 +44,7 @@ public class ValueSourceGroupSelector extends GroupSelector {
* @param valueSource the ValueSource to group by
* @param context a context map for the ValueSource
*/
- public ValueSourceGroupSelector(ValueSource valueSource, Map context) {
+ public ValueSourceGroupSelector(ValueSource valueSource, Map context) {
this.valueSource = valueSource;
this.context = context;
}
@@ -56,6 +57,9 @@ public void setNextReader(LeafReaderContext readerContext) throws IOException {
this.filler = values.getValueFiller();
}
+ @Override
+ public void setScorer(Scorable scorer) throws IOException { }
+
@Override
public State advanceTo(int doc) throws IOException {
this.filler.fillValue(doc);
@@ -67,7 +71,7 @@ public State advanceTo(int doc) throws IOException {
}
@Override
- public MutableValue currentValue() {
+ public MutableValue currentValue() throws IOException {
return filler.getValue();
}
diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/package-info.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/package-info.java
index 7e3745e61f28..36d94a5a92e3 100644
--- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/package-info.java
+++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/package-info.java
@@ -28,11 +28,9 @@
* Grouping requires a number of inputs:
*
*
- * groupField: this is the field used for grouping.
- * For example, if you use the author field then each
- * group has all books by the same author. Documents that don't
- * have this field are grouped under a single group with
- * a null group value.
+ * groupSelector: this defines how groups are created
+ * from values per-document. The grouping module ships with
+ * selectors for grouping by term, and by long and double ranges.
*
* groupSort: how the groups are sorted. For sorting
* purposes, each group is "represented" by the highest-sorted
@@ -80,6 +78,10 @@
* the value of a {@link org.apache.lucene.index.SortedDocValues} field- {@link org.apache.lucene.search.grouping.ValueSourceGroupSelector} groups based on
* the value of a {@link org.apache.lucene.queries.function.ValueSource}
+ * - {@link org.apache.lucene.search.grouping.DoubleRangeGroupSelector} groups based on
+ * the value of a {@link org.apache.lucene.search.DoubleValuesSource}
+ * - {@link org.apache.lucene.search.grouping.LongRangeGroupSelector} groups based on
+ * the value of a {@link org.apache.lucene.search.LongValuesSource}
*
*
* Known limitations:
@@ -137,17 +139,10 @@
* writer.addDocuments(oneGroup);
*
*
- * Then, at search time, do this up front:
+ * Then, at search time:
*
*
- * // Set this once in your app & save away for reusing across all queries:
- * Filter groupEndDocs = new CachingWrapperFilter(new QueryWrapperFilter(new TermQuery(new Term("groupEnd", "x"))));
- *
- *
- * Finally, do this per search:
- *
- *
- * // Per search:
+ * Query groupEndDocs = new TermQuery(new Term("groupEnd", "x"));
* BlockGroupingCollector c = new BlockGroupingCollector(groupSort, groupOffset+topNGroups, needsScores, groupEndDocs);
* s.search(new TermQuery(new Term("content", searchTerm)), c);
* TopGroups groupsResult = c.getTopGroups(withinGroupSort, groupOffset, docOffset, docOffset+docsPerGroup, fillFields);
diff --git a/lucene/grouping/src/test/org/apache/lucene/search/grouping/AbstractGroupingTestCase.java b/lucene/grouping/src/test/org/apache/lucene/search/grouping/AbstractGroupingTestCase.java
index 3c3b9f900d60..e25666734db3 100644
--- a/lucene/grouping/src/test/org/apache/lucene/search/grouping/AbstractGroupingTestCase.java
+++ b/lucene/grouping/src/test/org/apache/lucene/search/grouping/AbstractGroupingTestCase.java
@@ -16,6 +16,15 @@
*/
package org.apache.lucene.search.grouping;
+import java.io.Closeable;
+import java.io.IOException;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@@ -36,4 +45,40 @@ protected String generateRandomNonEmptyString() {
} while ("".equals(randomValue));
return randomValue;
}
+
+ protected static void assertScoreDocsEquals(ScoreDoc[] expected, ScoreDoc[] actual) {
+ assertEquals(expected.length, actual.length);
+ for (int i = 0; i < expected.length; i++) {
+ assertEquals(expected[i].doc, actual[i].doc);
+ assertEquals(expected[i].score, actual[i].score, 0);
+ }
+ }
+
+ protected static class Shard implements Closeable {
+
+ final Directory directory;
+ final RandomIndexWriter writer;
+ IndexSearcher searcher;
+
+ Shard() throws IOException {
+ this.directory = newDirectory();
+ this.writer = new RandomIndexWriter(random(), directory,
+ newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy()));
+ }
+
+ IndexSearcher getIndexSearcher() throws IOException {
+ if (searcher == null) {
+ searcher = new IndexSearcher(this.writer.getReader());
+ }
+ return searcher;
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (searcher != null) {
+ searcher.getIndexReader().close();
+ }
+ IOUtils.close(writer, directory);
+ }
+ }
}
diff --git a/lucene/grouping/src/test/org/apache/lucene/search/grouping/BaseGroupSelectorTestCase.java b/lucene/grouping/src/test/org/apache/lucene/search/grouping/BaseGroupSelectorTestCase.java
new file mode 100644
index 000000000000..bb2a946f2a47
--- /dev/null
+++ b/lucene/grouping/src/test/org/apache/lucene/search/grouping/BaseGroupSelectorTestCase.java
@@ -0,0 +1,365 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.search.grouping;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.FieldDoc;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+
+public abstract class BaseGroupSelectorTestCase extends AbstractGroupingTestCase {
+
+ protected abstract void addGroupField(Document document, int id);
+
+ protected abstract GroupSelector getGroupSelector();
+
+ protected abstract Query filterQuery(T groupValue);
+
+ public void testSortByRelevance() throws IOException {
+
+ Shard shard = new Shard();
+ indexRandomDocs(shard.writer);
+
+ String[] query = new String[]{ "foo", "bar", "baz" };
+ Query topLevel = new TermQuery(new Term("text", query[random().nextInt(query.length)]));
+
+ IndexSearcher searcher = shard.getIndexSearcher();
+ GroupingSearch grouper = new GroupingSearch(getGroupSelector());
+ grouper.setGroupDocsLimit(10);
+ TopGroups topGroups = grouper.search(searcher, topLevel, 0, 5);
+ TopDocs topDoc = searcher.search(topLevel, 1);
+ for (int i = 0; i < topGroups.groups.length; i++) {
+ // Each group should have a result set equal to that returned by the top-level query,
+ // filtered by the group value.
+ Query filtered = new BooleanQuery.Builder()
+ .add(topLevel, BooleanClause.Occur.MUST)
+ .add(filterQuery(topGroups.groups[i].groupValue), BooleanClause.Occur.FILTER)
+ .build();
+ TopDocs td = searcher.search(filtered, 10);
+ assertScoreDocsEquals(topGroups.groups[i].scoreDocs, td.scoreDocs);
+ if (i == 0) {
+ assertEquals(td.scoreDocs[0].doc, topDoc.scoreDocs[0].doc);
+ assertEquals(td.scoreDocs[0].score, topDoc.scoreDocs[0].score, 0);
+ }
+ }
+
+ shard.close();
+ }
+
+ public void testSortGroups() throws IOException {
+
+ Shard shard = new Shard();
+ indexRandomDocs(shard.writer);
+ IndexSearcher searcher = shard.getIndexSearcher();
+
+ String[] query = new String[]{ "foo", "bar", "baz" };
+ Query topLevel = new TermQuery(new Term("text", query[random().nextInt(query.length)]));
+
+ GroupingSearch grouper = new GroupingSearch(getGroupSelector());
+ grouper.setGroupDocsLimit(10);
+ Sort sort = new Sort(new SortField("sort1", SortField.Type.STRING), new SortField("sort2", SortField.Type.LONG));
+ grouper.setGroupSort(sort);
+ TopGroups topGroups = grouper.search(searcher, topLevel, 0, 5);
+ TopDocs topDoc = searcher.search(topLevel, 1, sort);
+ for (int i = 0; i < topGroups.groups.length; i++) {
+ // We're sorting the groups by a defined Sort, but each group itself should be ordered
+ // by doc relevance, and should be equal to the results of a top-level query filtered
+ // by the group value
+ Query filtered = new BooleanQuery.Builder()
+ .add(topLevel, BooleanClause.Occur.MUST)
+ .add(filterQuery(topGroups.groups[i].groupValue), BooleanClause.Occur.FILTER)
+ .build();
+ TopDocs td = searcher.search(filtered, 10);
+ assertScoreDocsEquals(topGroups.groups[i].scoreDocs, td.scoreDocs);
+ // The top group should have sort values equal to the sort values of the top doc of
+ // a top-level search sorted by the same Sort; subsequent groups should have sort values
+ // that compare lower than their predecessor.
+ if (i > 0) {
+ assertSortsBefore(topGroups.groups[i - 1], topGroups.groups[i]);
+ } else {
+ assertArrayEquals(((FieldDoc)topDoc.scoreDocs[0]).fields, topGroups.groups[0].groupSortValues);
+ }
+ }
+
+ shard.close();
+ }
+
+ public void testSortWithinGroups() throws IOException {
+
+ Shard shard = new Shard();
+ indexRandomDocs(shard.writer);
+ IndexSearcher searcher = shard.getIndexSearcher();
+
+ String[] query = new String[]{ "foo", "bar", "baz" };
+ Query topLevel = new TermQuery(new Term("text", query[random().nextInt(query.length)]));
+
+ GroupingSearch grouper = new GroupingSearch(getGroupSelector());
+ grouper.setGroupDocsLimit(10);
+ Sort sort = new Sort(new SortField("sort1", SortField.Type.STRING), new SortField("sort2", SortField.Type.LONG));
+ grouper.setSortWithinGroup(sort);
+
+ TopGroups topGroups = grouper.search(searcher, topLevel, 0, 5);
+ TopDocs topDoc = searcher.search(topLevel, 1);
+
+ for (int i = 0; i < topGroups.groups.length; i++) {
+ // Check top-level ordering by score: first group's maxScore should be equal to the
+ // top score returned by a simple search with no grouping; subsequent groups should
+ // all have equal or lower maxScores
+ if (i == 0) {
+ assertEquals(topDoc.scoreDocs[0].score, topGroups.groups[0].maxScore, 0);
+ } else {
+ assertTrue(topGroups.groups[i].maxScore <= topGroups.groups[i - 1].maxScore);
+ }
+ // Groups themselves are ordered by a defined Sort, and each should give the same result as
+ // the top-level query, filtered by the group value, with the same Sort
+ Query filtered = new BooleanQuery.Builder()
+ .add(topLevel, BooleanClause.Occur.MUST)
+ .add(filterQuery(topGroups.groups[i].groupValue), BooleanClause.Occur.FILTER)
+ .build();
+ TopDocs td = searcher.search(filtered, 10, sort);
+ assertScoreDocsEquals(td.scoreDocs, topGroups.groups[i].scoreDocs);
+ }
+
+ shard.close();
+
+ }
+
+ public void testGroupHeads() throws IOException {
+
+ Shard shard = new Shard();
+ indexRandomDocs(shard.writer);
+ IndexSearcher searcher = shard.getIndexSearcher();
+
+ String[] query = new String[]{ "foo", "bar", "baz" };
+ Query topLevel = new TermQuery(new Term("text", query[random().nextInt(query.length)]));
+
+ GroupSelector groupSelector = getGroupSelector();
+ GroupingSearch grouping = new GroupingSearch(groupSelector);
+ grouping.setAllGroups(true);
+ grouping.setAllGroupHeads(true);
+
+ grouping.search(searcher, topLevel, 0, 1);
+ Collection matchingGroups = grouping.getAllMatchingGroups();
+
+ // The number of hits from the top-level query should equal the sum of
+ // the number of hits from the query filtered by each group value in turn
+ int totalHits = searcher.count(topLevel);
+ int groupHits = 0;
+ for (T groupValue : matchingGroups) {
+ Query filtered = new BooleanQuery.Builder()
+ .add(topLevel, BooleanClause.Occur.MUST)
+ .add(filterQuery(groupValue), BooleanClause.Occur.FILTER)
+ .build();
+ groupHits += searcher.count(filtered);
+ }
+ assertEquals(totalHits, groupHits);
+
+ Bits groupHeads = grouping.getAllGroupHeads();
+ int cardinality = 0;
+ for (int i = 0; i < groupHeads.length(); i++) {
+ if (groupHeads.get(i)) {
+ cardinality++;
+ }
+ }
+ assertEquals(matchingGroups.size(), cardinality); // We should have one set bit per matching group
+
+ // Each group head should correspond to the topdoc of a search filtered by
+ // that group
+ for (T groupValue : matchingGroups) {
+ Query filtered = new BooleanQuery.Builder()
+ .add(topLevel, BooleanClause.Occur.MUST)
+ .add(filterQuery(groupValue), BooleanClause.Occur.FILTER)
+ .build();
+ TopDocs td = searcher.search(filtered, 1);
+ assertTrue(groupHeads.get(td.scoreDocs[0].doc));
+ }
+
+ shard.close();
+ }
+
+ public void testGroupHeadsWithSort() throws IOException {
+
+ Shard shard = new Shard();
+ indexRandomDocs(shard.writer);
+ IndexSearcher searcher = shard.getIndexSearcher();
+
+ String[] query = new String[]{ "foo", "bar", "baz" };
+ Query topLevel = new TermQuery(new Term("text", query[random().nextInt(query.length)]));
+
+ Sort sort = new Sort(new SortField("sort1", SortField.Type.STRING), new SortField("sort2", SortField.Type.LONG));
+ GroupSelector groupSelector = getGroupSelector();
+ GroupingSearch grouping = new GroupingSearch(groupSelector);
+ grouping.setAllGroups(true);
+ grouping.setAllGroupHeads(true);
+ grouping.setSortWithinGroup(sort);
+
+ grouping.search(searcher, topLevel, 0, 1);
+ Collection matchingGroups = grouping.getAllMatchingGroups();
+
+ Bits groupHeads = grouping.getAllGroupHeads();
+ int cardinality = 0;
+ for (int i = 0; i < groupHeads.length(); i++) {
+ if (groupHeads.get(i)) {
+ cardinality++;
+ }
+ }
+ assertEquals(matchingGroups.size(), cardinality); // We should have one set bit per matching group
+
+ // Each group head should correspond to the topdoc of a search filtered by
+ // that group using the same within-group sort
+ for (T groupValue : matchingGroups) {
+ Query filtered = new BooleanQuery.Builder()
+ .add(topLevel, BooleanClause.Occur.MUST)
+ .add(filterQuery(groupValue), BooleanClause.Occur.FILTER)
+ .build();
+ TopDocs td = searcher.search(filtered, 1, sort);
+ assertTrue(groupHeads.get(td.scoreDocs[0].doc));
+ }
+
+ shard.close();
+ }
+
+ public void testShardedGrouping() throws IOException {
+
+ Shard control = new Shard();
+
+ int shardCount = random().nextInt(3) + 2; // between 2 and 4 shards
+ Shard[] shards = new Shard[shardCount];
+ for (int i = 0; i < shardCount; i++) {
+ shards[i] = new Shard();
+ }
+
+ String[] texts = new String[]{ "foo", "bar", "bar baz", "foo foo bar" };
+
+ // Create a bunch of random documents, and index them - once into the control index,
+ // and once into a randomly picked shard.
+
+ int numDocs = atLeast(200);
+ for (int i = 0; i < numDocs; i++) {
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", i));
+ doc.add(new TextField("name", Integer.toString(i), Field.Store.YES));
+ doc.add(new TextField("text", texts[random().nextInt(texts.length)], Field.Store.NO));
+ doc.add(new SortedDocValuesField("sort1", new BytesRef("sort" + random().nextInt(4))));
+ doc.add(new NumericDocValuesField("sort2", random().nextLong()));
+ addGroupField(doc, i);
+ control.writer.addDocument(doc);
+ int shard = random().nextInt(shardCount);
+ shards[shard].writer.addDocument(doc);
+ }
+
+ String[] query = new String[]{ "foo", "bar", "baz" };
+ Query topLevel = new TermQuery(new Term("text", query[random().nextInt(query.length)]));
+
+ Sort sort = new Sort(new SortField("sort1", SortField.Type.STRING), new SortField("sort2", SortField.Type.LONG));
+
+ // A grouped query run in two phases against the control should give us the same
+ // result as the query run against shards and merged back together after each phase.
+
+ FirstPassGroupingCollector singletonFirstPass = new FirstPassGroupingCollector<>(getGroupSelector(), sort, 5);
+ control.getIndexSearcher().search(topLevel, singletonFirstPass);
+ Collection> singletonGroups = singletonFirstPass.getTopGroups(0);
+
+ List>> shardGroups = new ArrayList<>();
+ for (Shard shard : shards) {
+ FirstPassGroupingCollector fc = new FirstPassGroupingCollector<>(getGroupSelector(), sort, 5);
+ shard.getIndexSearcher().search(topLevel, fc);
+ shardGroups.add(fc.getTopGroups(0));
+ }
+ Collection> mergedGroups = SearchGroup.merge(shardGroups, 0, 5, sort);
+ assertEquals(singletonGroups, mergedGroups);
+
+ TopGroupsCollector singletonSecondPass = new TopGroupsCollector<>(getGroupSelector(), singletonGroups, sort,
+ Sort.RELEVANCE, 5, true);
+ control.getIndexSearcher().search(topLevel, singletonSecondPass);
+ TopGroups singletonTopGroups = singletonSecondPass.getTopGroups(0);
+
+ // TODO why does SearchGroup.merge() take a list but TopGroups.merge() take an array?
+ @SuppressWarnings("unchecked")
+ TopGroups[] shardTopGroups = (TopGroups[]) new TopGroups[shards.length];
+ int j = 0;
+ for (Shard shard : shards) {
+ TopGroupsCollector sc = new TopGroupsCollector<>(getGroupSelector(), mergedGroups, sort, Sort.RELEVANCE, 5, true);
+ shard.getIndexSearcher().search(topLevel, sc);
+ shardTopGroups[j] = sc.getTopGroups(0);
+ j++;
+ }
+ TopGroups mergedTopGroups = TopGroups.merge(shardTopGroups, sort, Sort.RELEVANCE, 0, 5, TopGroups.ScoreMergeMode.None);
+ assertNotNull(mergedTopGroups);
+
+ assertEquals(singletonTopGroups.totalGroupedHitCount, mergedTopGroups.totalGroupedHitCount);
+ assertEquals(singletonTopGroups.totalHitCount, mergedTopGroups.totalHitCount);
+ assertEquals(singletonTopGroups.totalGroupCount, mergedTopGroups.totalGroupCount);
+ assertEquals(singletonTopGroups.groups.length, mergedTopGroups.groups.length);
+ for (int i = 0; i < singletonTopGroups.groups.length; i++) {
+ assertEquals(singletonTopGroups.groups[i].groupValue, mergedTopGroups.groups[i].groupValue);
+ assertEquals(singletonTopGroups.groups[i].scoreDocs.length, mergedTopGroups.groups[i].scoreDocs.length);
+ }
+
+ control.close();
+ for (Shard shard : shards) {
+ shard.close();
+ }
+
+ }
+
+ private void indexRandomDocs(RandomIndexWriter w) throws IOException {
+ String[] texts = new String[]{ "foo", "bar", "bar baz", "foo foo bar" };
+
+ int numDocs = atLeast(200);
+ for (int i = 0; i < numDocs; i++) {
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", i));
+ doc.add(new TextField("name", Integer.toString(i), Field.Store.YES));
+ doc.add(new TextField("text", texts[random().nextInt(texts.length)], Field.Store.NO));
+ doc.add(new SortedDocValuesField("sort1", new BytesRef("sort" + random().nextInt(4))));
+ doc.add(new NumericDocValuesField("sort2", random().nextLong()));
+ addGroupField(doc, i);
+ w.addDocument(doc);
+ }
+ }
+
+ private void assertSortsBefore(GroupDocs first, GroupDocs second) {
+ Object[] groupSortValues = second.groupSortValues;
+ Object[] prevSortValues = first.groupSortValues;
+ assertTrue(((BytesRef)prevSortValues[0]).compareTo((BytesRef)groupSortValues[0]) <= 0);
+ if (prevSortValues[0].equals(groupSortValues[0])) {
+ assertTrue((long)prevSortValues[1] <= (long)groupSortValues[1]);
+ }
+ }
+
+}
diff --git a/lucene/grouping/src/test/org/apache/lucene/search/grouping/BlockGroupingTest.java b/lucene/grouping/src/test/org/apache/lucene/search/grouping/BlockGroupingTest.java
new file mode 100644
index 000000000000..fc03dfdd84a5
--- /dev/null
+++ b/lucene/grouping/src/test/org/apache/lucene/search/grouping/BlockGroupingTest.java
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.search.grouping;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.FieldDoc;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.util.BytesRef;
+
+public class BlockGroupingTest extends AbstractGroupingTestCase {
+
+ public void testSimple() throws IOException {
+
+ Shard shard = new Shard();
+ indexRandomDocs(shard.writer);
+ IndexSearcher searcher = shard.getIndexSearcher();
+
+ Query blockEndQuery = new TermQuery(new Term("blockEnd", "true"));
+ GroupingSearch grouper = new GroupingSearch(blockEndQuery);
+ grouper.setGroupDocsLimit(10);
+
+ Query topLevel = new TermQuery(new Term("text", "grandmother"));
+ TopGroups tg = grouper.search(searcher, topLevel, 0, 5);
+
+ // We're sorting by score, so the score of the top group should be the same as the
+ // score of the top document from the same query with no grouping
+ TopDocs topDoc = searcher.search(topLevel, 1);
+ assertEquals(topDoc.scoreDocs[0].score, tg.groups[0].scoreDocs[0].score, 0);
+ assertEquals(topDoc.scoreDocs[0].doc, tg.groups[0].scoreDocs[0].doc);
+
+ for (int i = 0; i < tg.groups.length; i++) {
+ String bookName = searcher.doc(tg.groups[i].scoreDocs[0].doc).get("book");
+ // The contents of each group should be equal to the results of a search for
+ // that group alone
+ Query filtered = new BooleanQuery.Builder()
+ .add(topLevel, BooleanClause.Occur.MUST)
+ .add(new TermQuery(new Term("book", bookName)), BooleanClause.Occur.FILTER)
+ .build();
+ TopDocs td = searcher.search(filtered, 10);
+ assertScoreDocsEquals(td.scoreDocs, tg.groups[i].scoreDocs);
+ }
+
+ shard.close();
+
+ }
+
+ public void testTopLevelSort() throws IOException {
+
+ Shard shard = new Shard();
+ indexRandomDocs(shard.writer);
+ IndexSearcher searcher = shard.getIndexSearcher();
+
+ Sort sort = new Sort(new SortField("length", SortField.Type.LONG));
+
+ Query blockEndQuery = new TermQuery(new Term("blockEnd", "true"));
+ GroupingSearch grouper = new GroupingSearch(blockEndQuery);
+ grouper.setGroupDocsLimit(10);
+ grouper.setGroupSort(sort); // groups returned sorted by length, chapters within group sorted by relevancy
+
+ Query topLevel = new TermQuery(new Term("text", "grandmother"));
+ TopGroups tg = grouper.search(searcher, topLevel, 0, 5);
+
+ // The sort value of the top doc in the top group should be the same as the sort value
+ // of the top result from the same search done with no grouping
+ TopDocs topDoc = searcher.search(topLevel, 1, sort);
+ assertEquals(((FieldDoc)topDoc.scoreDocs[0]).fields[0], tg.groups[0].groupSortValues[0]);
+
+ for (int i = 0; i < tg.groups.length; i++) {
+ String bookName = searcher.doc(tg.groups[i].scoreDocs[0].doc).get("book");
+ // The contents of each group should be equal to the results of a search for
+ // that group alone, sorted by score
+ Query filtered = new BooleanQuery.Builder()
+ .add(topLevel, BooleanClause.Occur.MUST)
+ .add(new TermQuery(new Term("book", bookName)), BooleanClause.Occur.FILTER)
+ .build();
+ TopDocs td = searcher.search(filtered, 10);
+ assertScoreDocsEquals(td.scoreDocs, tg.groups[i].scoreDocs);
+ if (i > 1) {
+ assertSortsBefore(tg.groups[i - 1], tg.groups[i]);
+ }
+ }
+
+ shard.close();
+
+ }
+
+ public void testWithinGroupSort() throws IOException {
+
+ Shard shard = new Shard();
+ indexRandomDocs(shard.writer);
+ IndexSearcher searcher = shard.getIndexSearcher();
+
+ Sort sort = new Sort(new SortField("length", SortField.Type.LONG));
+
+ Query blockEndQuery = new TermQuery(new Term("blockEnd", "true"));
+ GroupingSearch grouper = new GroupingSearch(blockEndQuery);
+ grouper.setGroupDocsLimit(10);
+ grouper.setSortWithinGroup(sort); // groups returned sorted by relevancy, chapters within group sorted by length
+
+ Query topLevel = new TermQuery(new Term("text", "grandmother"));
+ TopGroups tg = grouper.search(searcher, topLevel, 0, 5);
+
+ // We're sorting by score, so the score of the top group should be the same as the
+ // score of the top document from the same query with no grouping
+ TopDocs topDoc = searcher.search(topLevel, 1);
+ assertEquals(topDoc.scoreDocs[0].score, (float)tg.groups[0].groupSortValues[0], 0);
+
+ for (int i = 0; i < tg.groups.length; i++) {
+ String bookName = searcher.doc(tg.groups[i].scoreDocs[0].doc).get("book");
+ // The contents of each group should be equal to the results of a search for
+ // that group alone, sorted by length
+ Query filtered = new BooleanQuery.Builder()
+ .add(topLevel, BooleanClause.Occur.MUST)
+ .add(new TermQuery(new Term("book", bookName)), BooleanClause.Occur.FILTER)
+ .build();
+ TopDocs td = searcher.search(filtered, 10, sort);
+ assertFieldDocsEquals(td.scoreDocs, tg.groups[i].scoreDocs);
+ // We're sorting by score, so the group sort value for each group should be a float,
+ // and the value for the previous group should be higher or equal to the value for this one
+ if (i > 0) {
+ float prevScore = (float) tg.groups[i - 1].groupSortValues[0];
+ float thisScore = (float) tg.groups[i].groupSortValues[0];
+ assertTrue(prevScore >= thisScore);
+ }
+ }
+
+ shard.close();
+ }
+
+ private static void indexRandomDocs(RandomIndexWriter writer) throws IOException {
+ int bookCount = atLeast(20);
+ for (int i = 0; i < bookCount; i++) {
+ writer.addDocuments(createRandomBlock(i));
+ }
+ }
+
+ private static List createRandomBlock(int book) {
+ List block = new ArrayList<>();
+ String bookName = "book" + book;
+ int chapterCount = atLeast(10);
+ for (int j = 0; j < chapterCount; j++) {
+ Document doc = new Document();
+ String chapterName = "chapter" + j;
+ String chapterText = randomText();
+ doc.add(new TextField("book", bookName, Field.Store.YES));
+ doc.add(new TextField("chapter", chapterName, Field.Store.YES));
+ doc.add(new TextField("text", chapterText, Field.Store.NO));
+ doc.add(new NumericDocValuesField("length", chapterText.length()));
+ doc.add(new SortedDocValuesField("book", new BytesRef(bookName)));
+ if (j == chapterCount - 1) {
+ doc.add(new TextField("blockEnd", "true", Field.Store.NO));
+ }
+ block.add(doc);
+ }
+ return block;
+ }
+
+ private static final String[] TEXT = new String[]{
+ "It was the day my grandmother exploded",
+ "It was the best of times, it was the worst of times",
+ "It was a bright cold morning in April",
+ "It is a truth universally acknowledged",
+ "I have just returned from a visit to my landlord",
+ "I've been here and I've been there"
+ };
+
+ private static String randomText() {
+ StringBuilder sb = new StringBuilder(TEXT[random().nextInt(TEXT.length)]);
+ int sentences = random().nextInt(20);
+ for (int i = 0; i < sentences; i++) {
+ sb.append(" ").append(TEXT[random().nextInt(TEXT.length)]);
+ }
+ return sb.toString();
+ }
+
+ private void assertSortsBefore(GroupDocs first, GroupDocs second) {
+ Object[] groupSortValues = second.groupSortValues;
+ Object[] prevSortValues = first.groupSortValues;
+ assertTrue(((Long)prevSortValues[0]).compareTo((Long)groupSortValues[0]) <= 0);
+ }
+
+ protected static void assertFieldDocsEquals(ScoreDoc[] expected, ScoreDoc[] actual) {
+ assertEquals(expected.length, actual.length);
+ for (int i = 0; i < expected.length; i++) {
+ assertEquals(expected[i].doc, actual[i].doc);
+ FieldDoc e = (FieldDoc) expected[i];
+ FieldDoc a = (FieldDoc) actual[i];
+ assertArrayEquals(e.fields, a.fields);
+ }
+ }
+
+}
diff --git a/lucene/grouping/src/test/org/apache/lucene/search/grouping/DoubleRangeGroupSelectorTest.java b/lucene/grouping/src/test/org/apache/lucene/search/grouping/DoubleRangeGroupSelectorTest.java
new file mode 100644
index 000000000000..c9385898ade5
--- /dev/null
+++ b/lucene/grouping/src/test/org/apache/lucene/search/grouping/DoubleRangeGroupSelectorTest.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.search.grouping;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.DoublePoint;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.DocValuesFieldExistsQuery;
+import org.apache.lucene.search.DoubleValuesSource;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Query;
+
+public class DoubleRangeGroupSelectorTest extends BaseGroupSelectorTestCase {
+
+ @Override
+ protected void addGroupField(Document document, int id) {
+ if (rarely()) {
+ return; // missing value
+ }
+ // numbers between 0 and 1000, groups are 100 wide from 100 to 900
+ double value = random().nextDouble() * 1000;
+ document.add(new DoublePoint("double", value));
+ document.add(new NumericDocValuesField("double", Double.doubleToLongBits(value)));
+ }
+
+ @Override
+ protected GroupSelector getGroupSelector() {
+ return new DoubleRangeGroupSelector(DoubleValuesSource.fromDoubleField("double"),
+ new DoubleRangeFactory(100, 100, 900));
+ }
+
+ @Override
+ protected Query filterQuery(DoubleRange groupValue) {
+ if (groupValue == null) {
+ return new BooleanQuery.Builder()
+ .add(new MatchAllDocsQuery(), BooleanClause.Occur.FILTER)
+ .add(new DocValuesFieldExistsQuery("double"), BooleanClause.Occur.MUST_NOT)
+ .build();
+ }
+ return DoublePoint.newRangeQuery("double", groupValue.min, Math.nextDown(groupValue.max));
+ }
+}
diff --git a/lucene/grouping/src/test/org/apache/lucene/search/grouping/LongRangeGroupSelectorTest.java b/lucene/grouping/src/test/org/apache/lucene/search/grouping/LongRangeGroupSelectorTest.java
new file mode 100644
index 000000000000..6075f700a28f
--- /dev/null
+++ b/lucene/grouping/src/test/org/apache/lucene/search/grouping/LongRangeGroupSelectorTest.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.search.grouping;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.LongPoint;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.DocValuesFieldExistsQuery;
+import org.apache.lucene.search.LongValuesSource;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Query;
+
+public class LongRangeGroupSelectorTest extends BaseGroupSelectorTestCase {
+
+ @Override
+ protected void addGroupField(Document document, int id) {
+ if (rarely()) {
+ return; // missing value
+ }
+ // numbers between 0 and 1000, groups are 100 wide from 100 to 900
+ long value = random().nextInt(1000);
+ document.add(new LongPoint("long", value));
+ document.add(new NumericDocValuesField("long", value));
+ }
+
+ @Override
+ protected GroupSelector getGroupSelector() {
+ return new LongRangeGroupSelector(LongValuesSource.fromLongField("long"),
+ new LongRangeFactory(100, 100, 900));
+ }
+
+ @Override
+ protected Query filterQuery(LongRange groupValue) {
+ if (groupValue == null) {
+ return new BooleanQuery.Builder()
+ .add(new MatchAllDocsQuery(), BooleanClause.Occur.FILTER)
+ .add(new DocValuesFieldExistsQuery("long"), BooleanClause.Occur.MUST_NOT)
+ .build();
+ }
+ return LongPoint.newRangeQuery("long", groupValue.min, groupValue.max - 1);
+ }
+}
diff --git a/lucene/grouping/src/test/org/apache/lucene/search/grouping/TermGroupSelectorTest.java b/lucene/grouping/src/test/org/apache/lucene/search/grouping/TermGroupSelectorTest.java
new file mode 100644
index 000000000000..831a0147d1d0
--- /dev/null
+++ b/lucene/grouping/src/test/org/apache/lucene/search/grouping/TermGroupSelectorTest.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.search.grouping;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.DocValuesFieldExistsQuery;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.util.BytesRef;
+
+public class TermGroupSelectorTest extends BaseGroupSelectorTestCase {
+
+ @Override
+ protected void addGroupField(Document document, int id) {
+ if (rarely()) {
+ return; // missing value
+ }
+ String groupValue = "group" + random().nextInt(10);
+ document.add(new SortedDocValuesField("groupField", new BytesRef(groupValue)));
+ document.add(new TextField("groupField", groupValue, Field.Store.NO));
+ }
+
+ @Override
+ protected GroupSelector getGroupSelector() {
+ return new TermGroupSelector("groupField");
+ }
+
+ @Override
+ protected Query filterQuery(BytesRef groupValue) {
+ if (groupValue == null) {
+ return new BooleanQuery.Builder()
+ .add(new MatchAllDocsQuery(), BooleanClause.Occur.FILTER)
+ .add(new DocValuesFieldExistsQuery("groupField"), BooleanClause.Occur.MUST_NOT)
+ .build();
+ }
+ return new TermQuery(new Term("groupField", groupValue));
+ }
+}
diff --git a/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestDoubleRangeFactory.java b/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestDoubleRangeFactory.java
new file mode 100644
index 000000000000..d6e05c336426
--- /dev/null
+++ b/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestDoubleRangeFactory.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.search.grouping;
+
+import org.apache.lucene.util.LuceneTestCase;
+
+public class TestDoubleRangeFactory extends LuceneTestCase {
+
+ public void test() {
+
+ DoubleRangeFactory factory = new DoubleRangeFactory(10, 10, 50);
+ DoubleRange scratch = new DoubleRange(0, 0);
+
+ assertEquals(new DoubleRange(Double.MIN_VALUE, 10), factory.getRange(4, scratch));
+ assertEquals(new DoubleRange(10, 20), factory.getRange(10, scratch));
+ assertEquals(new DoubleRange(20, 30), factory.getRange(20, scratch));
+ assertEquals(new DoubleRange(10, 20), factory.getRange(15, scratch));
+ assertEquals(new DoubleRange(30, 40), factory.getRange(35, scratch));
+ assertEquals(new DoubleRange(50, Double.MAX_VALUE), factory.getRange(50, scratch));
+ assertEquals(new DoubleRange(50, Double.MAX_VALUE), factory.getRange(500, scratch));
+
+ }
+
+}
diff --git a/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java b/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java
index f1ce508e134a..4ef6c02d70e7 100644
--- a/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java
+++ b/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java
@@ -287,9 +287,11 @@ private void compareGroupValue(String expected, GroupDocs group) {
private Collection> getSearchGroups(FirstPassGroupingCollector c, int groupOffset) throws IOException {
if (TermGroupSelector.class.isAssignableFrom(c.getGroupSelector().getClass())) {
+ @SuppressWarnings("unchecked")
FirstPassGroupingCollector collector = (FirstPassGroupingCollector) c;
return collector.getTopGroups(groupOffset);
} else if (ValueSourceGroupSelector.class.isAssignableFrom(c.getGroupSelector().getClass())) {
+ @SuppressWarnings("unchecked")
FirstPassGroupingCollector collector = (FirstPassGroupingCollector) c;
Collection> mutableValueGroups = collector.getTopGroups(groupOffset);
if (mutableValueGroups == null) {
diff --git a/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestLongRangeFactory.java b/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestLongRangeFactory.java
new file mode 100644
index 000000000000..0677631476eb
--- /dev/null
+++ b/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestLongRangeFactory.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.search.grouping;
+
+import org.apache.lucene.util.LuceneTestCase;
+
+public class TestLongRangeFactory extends LuceneTestCase {
+
+ public void test() {
+
+ LongRangeFactory factory = new LongRangeFactory(10, 10, 50);
+ LongRange scratch = new LongRange(0, 0);
+
+ assertEquals(new LongRange(Long.MIN_VALUE, 10), factory.getRange(4, scratch));
+ assertEquals(new LongRange(10, 20), factory.getRange(10, scratch));
+ assertEquals(new LongRange(20, 30), factory.getRange(20, scratch));
+ assertEquals(new LongRange(10, 20), factory.getRange(15, scratch));
+ assertEquals(new LongRange(30, 40), factory.getRange(35, scratch));
+ assertEquals(new LongRange(50, Long.MAX_VALUE), factory.getRange(50, scratch));
+ assertEquals(new LongRange(50, Long.MAX_VALUE), factory.getRange(500, scratch));
+
+ }
+
+}
diff --git a/lucene/grouping/src/test/org/apache/lucene/search/grouping/ValueSourceGroupSelectorTest.java b/lucene/grouping/src/test/org/apache/lucene/search/grouping/ValueSourceGroupSelectorTest.java
new file mode 100644
index 000000000000..941ad8a26c19
--- /dev/null
+++ b/lucene/grouping/src/test/org/apache/lucene/search/grouping/ValueSourceGroupSelectorTest.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.search.grouping;
+
+import java.util.HashMap;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.queries.function.valuesource.SortedSetFieldSource;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.mutable.MutableValue;
+
+public class ValueSourceGroupSelectorTest extends BaseGroupSelectorTestCase {
+
+ @Override
+ protected void addGroupField(Document document, int id) {
+ String groupValue = "group" + random().nextInt(10);
+ document.add(new SortedDocValuesField("groupField", new BytesRef(groupValue)));
+ document.add(new TextField("groupField", groupValue, Field.Store.NO));
+ }
+
+ @Override
+ protected GroupSelector getGroupSelector() {
+ return new ValueSourceGroupSelector(new SortedSetFieldSource("groupField"), new HashMap<>());
+ }
+
+ @Override
+ protected Query filterQuery(MutableValue groupValue) {
+ return new TermQuery(new Term("groupField", groupValue.toObject().toString()));
+ }
+}
diff --git a/lucene/highlighter/build.gradle b/lucene/highlighter/build.gradle
index f369aef62c02..6e105d59b6e1 100644
--- a/lucene/highlighter/build.gradle
+++ b/lucene/highlighter/build.gradle
@@ -18,6 +18,8 @@
apply plugin: 'java-library'
+description = 'Highlights search keywords in results'
+
dependencies {
api project(':lucene:core')
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/UnifiedHighlighter.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/UnifiedHighlighter.java
index 74de2483b8a2..5d0dc94f271e 100644
--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/UnifiedHighlighter.java
+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/UnifiedHighlighter.java
@@ -61,6 +61,7 @@
import org.apache.lucene.search.Weight;
import org.apache.lucene.search.spans.SpanQuery;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.InPlaceMergeSorter;
/**
@@ -643,6 +644,7 @@ protected Map highlightFieldsAsObjects(String[] fieldsIn, Quer
batchDocIdx += fieldValsByDoc.size();
}
+ IOUtils.close(indexReaderWithTermVecCache);
assert docIdIter.docID() == DocIdSetIterator.NO_MORE_DOCS
|| docIdIter.nextDoc() == DocIdSetIterator.NO_MORE_DOCS;
diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java b/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
index 0c3a0f65d95b..2e703175bd4f 100644
--- a/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
+++ b/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
@@ -1362,24 +1362,25 @@ public void testOverlapAnalyzer() throws Exception {
public void run() throws Exception {
HashMap synonyms = new HashMap<>();
synonyms.put("football", "soccer,footie");
- Analyzer analyzer = new SynonymAnalyzer(synonyms);
+ try (Analyzer analyzer = new SynonymAnalyzer(synonyms)) {
- String s = "football-soccer in the euro 2004 footie competition";
+ String s = "football-soccer in the euro 2004 footie competition";
- BooleanQuery.Builder query = new BooleanQuery.Builder();
- query.add(new TermQuery(new Term("bookid", "football")), Occur.SHOULD);
- query.add(new TermQuery(new Term("bookid", "soccer")), Occur.SHOULD);
- query.add(new TermQuery(new Term("bookid", "footie")), Occur.SHOULD);
+ BooleanQuery.Builder query = new BooleanQuery.Builder();
+ query.add(new TermQuery(new Term("bookid", "football")), Occur.SHOULD);
+ query.add(new TermQuery(new Term("bookid", "soccer")), Occur.SHOULD);
+ query.add(new TermQuery(new Term("bookid", "footie")), Occur.SHOULD);
- Highlighter highlighter = getHighlighter(query.build(), null, HighlighterTest.this);
+ Highlighter highlighter = getHighlighter(query.build(), null, HighlighterTest.this);
- // Get 3 best fragments and separate with a "..."
- TokenStream tokenStream = analyzer.tokenStream(null, s);
+ // Get 3 best fragments and separate with a "..."
+ TokenStream tokenStream = analyzer.tokenStream(null, s);
- String result = highlighter.getBestFragments(tokenStream, s, 3, "...");
- String expectedResult = "football-soccer in the euro 2004 footie competition";
- assertTrue("overlapping analyzer should handle highlights OK, expected:" + expectedResult
- + " actual:" + result, expectedResult.equals(result));
+ String result = highlighter.getBestFragments(tokenStream, s, 3, "...");
+ String expectedResult = "football-soccer in the euro 2004 footie competition";
+ assertTrue("overlapping analyzer should handle highlights OK, expected:" + expectedResult
+ + " actual:" + result, expectedResult.equals(result));
+ }
}
};
diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java b/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java
index 825133c3f71f..30cf7117de8f 100644
--- a/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java
+++ b/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java
@@ -391,6 +391,7 @@ public void testRandomizedRoundTrip() throws Exception {
if (startOffsets[i] == startOffsets[i-1]) {
if (VERBOSE)
System.out.println("Skipping test because can't easily validate random token-stream is correct.");
+ rTokenStream.close();
return;
}
}
@@ -438,6 +439,7 @@ public void testRandomizedRoundTrip() throws Exception {
reader.close();
dir.close();
+ rTokenStream.close();
}
public void testMaxStartOffsetConsistency() throws IOException {
diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/highlight/custom/HighlightCustomQueryTest.java b/lucene/highlighter/src/test/org/apache/lucene/search/highlight/custom/HighlightCustomQueryTest.java
index b8ce3dd48408..115a51a96737 100644
--- a/lucene/highlighter/src/test/org/apache/lucene/search/highlight/custom/HighlightCustomQueryTest.java
+++ b/lucene/highlighter/src/test/org/apache/lucene/search/highlight/custom/HighlightCustomQueryTest.java
@@ -105,17 +105,18 @@ protected void extractUnknownQuery(Query query, Map ter
*/
private String highlightField(Query query, String fieldName,
String text) throws IOException, InvalidTokenOffsetsException {
- TokenStream tokenStream = new MockAnalyzer(random(), MockTokenizer.SIMPLE,
- true, MockTokenFilter.ENGLISH_STOPSET).tokenStream(fieldName, text);
- // Assuming "", "" used to highlight
- SimpleHTMLFormatter formatter = new SimpleHTMLFormatter();
- MyQueryScorer scorer = new MyQueryScorer(query, fieldName, FIELD_NAME);
- Highlighter highlighter = new Highlighter(formatter, scorer);
- highlighter.setTextFragmenter(new SimpleFragmenter(Integer.MAX_VALUE));
-
- String rv = highlighter.getBestFragments(tokenStream, text, 1,
- "(FIELD TEXT TRUNCATED)");
- return rv.length() == 0 ? text : rv;
+ try (MockAnalyzer mockAnalyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE,true,
+ MockTokenFilter.ENGLISH_STOPSET); TokenStream tokenStream = mockAnalyzer.tokenStream(fieldName, text)) {
+ // Assuming "", "" used to highlight
+ SimpleHTMLFormatter formatter = new SimpleHTMLFormatter();
+ MyQueryScorer scorer = new MyQueryScorer(query, fieldName, FIELD_NAME);
+ Highlighter highlighter = new Highlighter(formatter, scorer);
+ highlighter.setTextFragmenter(new SimpleFragmenter(Integer.MAX_VALUE));
+
+ String rv = highlighter.getBestFragments(tokenStream, text, 1,
+ "(FIELD TEXT TRUNCATED)");
+ return rv.length() == 0 ? text : rv;
+ }
}
public static class MyWeightedSpanTermExtractor extends
diff --git a/lucene/ivy-versions.properties b/lucene/ivy-versions.properties
index 8ab9fe8604b4..761cb72a1e53 100644
--- a/lucene/ivy-versions.properties
+++ b/lucene/ivy-versions.properties
@@ -10,7 +10,7 @@ com.carrotsearch.randomizedtesting.version = 2.7.6
/com.carrotsearch.thirdparty/simple-xml-safe = 2.7.1
-/com.carrotsearch/hppc = 0.8.1
+/com.carrotsearch/hppc = 0.8.2
/com.cybozu.labs/langdetect = 1.1-20120112
/com.drewnoakes/metadata-extractor = 2.11.0
@@ -23,7 +23,7 @@ com.fasterxml.jackson.core.version = 2.10.1
/com.fasterxml.jackson.core/jackson-databind = ${com.fasterxml.jackson.core.version}
/com.fasterxml.jackson.dataformat/jackson-dataformat-smile = ${com.fasterxml.jackson.core.version}
-/com.github.ben-manes.caffeine/caffeine = 2.8.0
+/com.github.ben-manes.caffeine/caffeine = 2.8.4
/com.github.virtuald/curvesapi = 1.06
/com.github.zafarkhaja/java-semver = 0.9.0
@@ -52,10 +52,9 @@ com.sun.jersey.version = 1.19
/com.tdunning/t-digest = 3.1
/com.vaadin.external.google/android-json = 0.0.20131108.vaadin1
-/commons-cli/commons-cli = 1.2
+/commons-cli/commons-cli = 1.4
/commons-codec/commons-codec = 1.13
/commons-collections/commons-collections = 3.2.2
-/commons-fileupload/commons-fileupload = 1.3.3
/commons-io/commons-io = 2.6
/commons-logging/commons-logging = 1.1.3
/de.l3s.boilerpipe/boilerpipe = 1.1.0
@@ -179,7 +178,7 @@ org.apache.kerby.version = 1.0.1
/org.apache.kerby/kerby-pkix = ${org.apache.kerby.version}
/org.apache.kerby/kerby-util = ${org.apache.kerby.version}
-org.apache.logging.log4j.version = 2.11.2
+org.apache.logging.log4j.version = 2.13.2
/org.apache.logging.log4j/log4j-1.2-api = ${org.apache.logging.log4j.version}
/org.apache.logging.log4j/log4j-api = ${org.apache.logging.log4j.version}
/org.apache.logging.log4j/log4j-core = ${org.apache.logging.log4j.version}
diff --git a/lucene/join/build.gradle b/lucene/join/build.gradle
index eab862245879..143daa13711a 100644
--- a/lucene/join/build.gradle
+++ b/lucene/join/build.gradle
@@ -17,6 +17,8 @@
apply plugin: 'java-library'
+description = 'Index-time and Query-time joins for normalized content'
+
dependencies {
api project(':lucene:core')
testImplementation project(':lucene:test-framework')
diff --git a/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinSortField.java b/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinSortField.java
index fef167e62f93..605e859543b0 100644
--- a/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinSortField.java
+++ b/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinSortField.java
@@ -30,6 +30,8 @@
import org.apache.lucene.util.BitSet;
import org.apache.lucene.util.NumericUtils;
+import static org.apache.lucene.search.join.BlockJoinSelector.toIter;
+
/**
* A special sort field that allows sorting parent docs based on nested / child level fields.
* Based on the sort order it either takes the document with the lowest or highest field value into account.
@@ -118,7 +120,7 @@ protected SortedDocValues getSortedDocValues(LeafReaderContext context, String f
if (children == null) {
return DocValues.emptySorted();
}
- return BlockJoinSelector.wrap(sortedSet, type, parents, children);
+ return BlockJoinSelector.wrap(sortedSet, type, parents, toIter(children));
}
};
@@ -137,7 +139,7 @@ protected NumericDocValues getNumericDocValues(LeafReaderContext context, String
if (children == null) {
return DocValues.emptyNumeric();
}
- return BlockJoinSelector.wrap(sortedNumeric, type, parents, children);
+ return BlockJoinSelector.wrap(sortedNumeric, type, parents, toIter(children));
}
};
}
@@ -155,7 +157,7 @@ protected NumericDocValues getNumericDocValues(LeafReaderContext context, String
if (children == null) {
return DocValues.emptyNumeric();
}
- return BlockJoinSelector.wrap(sortedNumeric, type, parents, children);
+ return BlockJoinSelector.wrap(sortedNumeric, type, parents, toIter(children));
}
};
}
@@ -173,7 +175,7 @@ protected NumericDocValues getNumericDocValues(LeafReaderContext context, String
if (children == null) {
return DocValues.emptyNumeric();
}
- return new FilterNumericDocValues(BlockJoinSelector.wrap(sortedNumeric, type, parents, children)) {
+ return new FilterNumericDocValues(BlockJoinSelector.wrap(sortedNumeric, type, parents, toIter(children))) {
@Override
public long longValue() throws IOException {
// undo the numericutils sortability
@@ -197,7 +199,7 @@ protected NumericDocValues getNumericDocValues(LeafReaderContext context, String
if (children == null) {
return DocValues.emptyNumeric();
}
- return new FilterNumericDocValues(BlockJoinSelector.wrap(sortedNumeric, type, parents, children)) {
+ return new FilterNumericDocValues(BlockJoinSelector.wrap(sortedNumeric, type, parents, toIter(children))) {
@Override
public long longValue() throws IOException {
// undo the numericutils sortability
diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoinSelector.java b/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoinSelector.java
index 838826fcbb4a..29e088bccbcc 100644
--- a/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoinSelector.java
+++ b/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoinSelector.java
@@ -17,6 +17,7 @@
package org.apache.lucene.search.join;
import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
+import static org.apache.lucene.search.join.BlockJoinSelector.toIter;
import java.io.IOException;
import java.util.Arrays;
@@ -132,14 +133,14 @@ public void testSortedSelector() throws IOException {
ords[12] = 10;
ords[18] = 10;
- final SortedDocValues mins = BlockJoinSelector.wrap(DocValues.singleton(new CannedSortedDocValues(ords)), BlockJoinSelector.Type.MIN, parents, children);
+ final SortedDocValues mins = BlockJoinSelector.wrap(DocValues.singleton(new CannedSortedDocValues(ords)), BlockJoinSelector.Type.MIN, parents, toIter(children));
assertEquals(5, nextDoc(mins,5));
assertEquals(3, mins.ordValue());
assertEquals(15, nextDoc(mins,15));
assertEquals(10, mins.ordValue());
assertNoMoreDoc(mins, 20);
- final SortedDocValues maxs = BlockJoinSelector.wrap(DocValues.singleton(new CannedSortedDocValues(ords)), BlockJoinSelector.Type.MAX, parents, children);
+ final SortedDocValues maxs = BlockJoinSelector.wrap(DocValues.singleton(new CannedSortedDocValues(ords)), BlockJoinSelector.Type.MAX, parents, toIter(children));
assertEquals(5, nextDoc(maxs,5));
assertEquals(7, maxs.ordValue());
assertEquals(15, nextDoc(maxs,15));
@@ -246,14 +247,14 @@ public void testNumericSelector() throws Exception {
docsWithValue.set(18);
longs[18] = 10;
- final NumericDocValues mins = BlockJoinSelector.wrap(DocValues.singleton(new CannedNumericDocValues(longs, docsWithValue)), BlockJoinSelector.Type.MIN, parents, children);
+ final NumericDocValues mins = BlockJoinSelector.wrap(DocValues.singleton(new CannedNumericDocValues(longs, docsWithValue)), BlockJoinSelector.Type.MIN, parents, toIter(children));
assertEquals(5, nextDoc(mins,5));
assertEquals(3, mins.longValue());
assertEquals(15, nextDoc(mins,15));
assertEquals(10, mins.longValue());
assertNoMoreDoc(mins, 20);
- final NumericDocValues maxs = BlockJoinSelector.wrap(DocValues.singleton(new CannedNumericDocValues(longs, docsWithValue)), BlockJoinSelector.Type.MAX, parents, children);
+ final NumericDocValues maxs = BlockJoinSelector.wrap(DocValues.singleton(new CannedNumericDocValues(longs, docsWithValue)), BlockJoinSelector.Type.MAX, parents, toIter(children));
assertEquals(5, nextDoc(maxs, 5));
assertEquals(7, maxs.longValue());
assertEquals(15, nextDoc(maxs, 15));
diff --git a/lucene/licenses/hppc-0.8.1.jar.sha1 b/lucene/licenses/hppc-0.8.1.jar.sha1
deleted file mode 100644
index 7006e68f4482..000000000000
--- a/lucene/licenses/hppc-0.8.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-ffc7ba8f289428b9508ab484b8001dea944ae603
diff --git a/lucene/licenses/hppc-0.8.2.jar.sha1 b/lucene/licenses/hppc-0.8.2.jar.sha1
new file mode 100644
index 000000000000..a73358b9c66f
--- /dev/null
+++ b/lucene/licenses/hppc-0.8.2.jar.sha1
@@ -0,0 +1 @@
+ccb3ef933ead6b5d766fa571582ddb9b447e48c4
diff --git a/lucene/licenses/log4j-api-2.11.2.jar.sha1 b/lucene/licenses/log4j-api-2.11.2.jar.sha1
deleted file mode 100644
index 0cdea100b72b..000000000000
--- a/lucene/licenses/log4j-api-2.11.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f5e9a2ffca496057d6891a3de65128efc636e26e
diff --git a/lucene/licenses/log4j-api-2.13.2.jar.sha1 b/lucene/licenses/log4j-api-2.13.2.jar.sha1
new file mode 100644
index 000000000000..a98264912453
--- /dev/null
+++ b/lucene/licenses/log4j-api-2.13.2.jar.sha1
@@ -0,0 +1 @@
+567ea514dedd8679c429c5b5b39b0d67b6464c3c
diff --git a/lucene/licenses/log4j-core-2.11.2.jar.sha1 b/lucene/licenses/log4j-core-2.11.2.jar.sha1
deleted file mode 100644
index ec2acae4df7f..000000000000
--- a/lucene/licenses/log4j-core-2.11.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-6c2fb3f5b7cd27504726aef1b674b542a0c9cf53
diff --git a/lucene/licenses/log4j-core-2.13.2.jar.sha1 b/lucene/licenses/log4j-core-2.13.2.jar.sha1
new file mode 100644
index 000000000000..ce26d9c45a85
--- /dev/null
+++ b/lucene/licenses/log4j-core-2.13.2.jar.sha1
@@ -0,0 +1 @@
+8eb1fc1914eb2550bf3ddea26917c9a7cbb00593
diff --git a/lucene/luke/build.gradle b/lucene/luke/build.gradle
index aaa0d7336d90..6e32b1b35a5e 100644
--- a/lucene/luke/build.gradle
+++ b/lucene/luke/build.gradle
@@ -17,6 +17,8 @@
apply plugin: 'java-library'
+description = 'Luke - Lucene Toolbox'
+
dependencies {
api project(':lucene:core')
diff --git a/lucene/luke/src/java/org/apache/lucene/luke/app/AbstractHandler.java b/lucene/luke/src/java/org/apache/lucene/luke/app/AbstractHandler.java
index ab967a8d149d..bca88f18632d 100644
--- a/lucene/luke/src/java/org/apache/lucene/luke/app/AbstractHandler.java
+++ b/lucene/luke/src/java/org/apache/lucene/luke/app/AbstractHandler.java
@@ -33,7 +33,9 @@ public abstract class AbstractHandler {
public void addObserver(T observer) {
observers.add(observer);
- log.debug("{} registered.", observer.getClass().getName());
+ if (log.isDebugEnabled()) {
+ log.debug("{} registered.", observer.getClass().getName());
+ }
}
void notifyObservers() {
diff --git a/lucene/luke/src/java/org/apache/lucene/luke/app/desktop/components/dialog/documents/AddDocumentDialogFactory.java b/lucene/luke/src/java/org/apache/lucene/luke/app/desktop/components/dialog/documents/AddDocumentDialogFactory.java
index 0bbeb3eb6f5c..1b9508f79eb1 100644
--- a/lucene/luke/src/java/org/apache/lucene/luke/app/desktop/components/dialog/documents/AddDocumentDialogFactory.java
+++ b/lucene/luke/src/java/org/apache/lucene/luke/app/desktop/components/dialog/documents/AddDocumentDialogFactory.java
@@ -365,7 +365,7 @@ void addDocument(ActionEvent e) {
}
addDocument(doc);
- log.info("Added document: {}", doc.toString());
+ log.info("Added document: {}", doc);
}
@SuppressWarnings("unchecked")
diff --git a/lucene/luke/src/java/org/apache/lucene/luke/app/desktop/components/dialog/menubar/ExportTermsDialogFactory.java b/lucene/luke/src/java/org/apache/lucene/luke/app/desktop/components/dialog/menubar/ExportTermsDialogFactory.java
index 471094223c1e..c091bea5c47d 100644
--- a/lucene/luke/src/java/org/apache/lucene/luke/app/desktop/components/dialog/menubar/ExportTermsDialogFactory.java
+++ b/lucene/luke/src/java/org/apache/lucene/luke/app/desktop/components/dialog/menubar/ExportTermsDialogFactory.java
@@ -248,10 +248,10 @@ protected Void doInBackground() {
try {
filename = toolsModel.exportTerms(directory, field, selectedDelimiter);
} catch (LukeException e) {
- log.error("Error while exporting terms from field " + field, e);
+ log.error("Error while exporting terms from field {}", field, e);
statusLbl.setText(MessageUtils.getLocalizedMessage("export.terms.label.error", e.getMessage()));
} catch (Exception e) {
- log.error("Error while exporting terms from field " + field, e);
+ log.error("Error while exporting terms from field {}", field, e);
statusLbl.setText(MessageUtils.getLocalizedMessage("message.error.unknown"));
throw e;
} finally {
diff --git a/lucene/luke/src/java/org/apache/lucene/luke/models/documents/DocumentsImpl.java b/lucene/luke/src/java/org/apache/lucene/luke/models/documents/DocumentsImpl.java
index e4b25296fb4b..d2f6c9b5f860 100644
--- a/lucene/luke/src/java/org/apache/lucene/luke/models/documents/DocumentsImpl.java
+++ b/lucene/luke/src/java/org/apache/lucene/luke/models/documents/DocumentsImpl.java
@@ -245,7 +245,9 @@ public Optional nextTermDoc() {
if (penum.nextDoc() == PostingsEnum.NO_MORE_DOCS) {
// end of the iterator
resetPostingsIterator();
- log.info("Reached the end of the postings iterator for term: {} in field: {}", BytesRefUtils.decode(tenum.term()), curField);
+ if (log.isInfoEnabled()) {
+ log.info("Reached the end of the postings iterator for term: {} in field: {}", BytesRefUtils.decode(tenum.term()), curField);
+ }
return Optional.empty();
} else {
return Optional.of(penum.docID());
diff --git a/lucene/luke/src/java/org/apache/lucene/luke/models/util/IndexUtils.java b/lucene/luke/src/java/org/apache/lucene/luke/models/util/IndexUtils.java
index 71e8070af465..9b98e92b9ede 100644
--- a/lucene/luke/src/java/org/apache/lucene/luke/models/util/IndexUtils.java
+++ b/lucene/luke/src/java/org/apache/lucene/luke/models/util/IndexUtils.java
@@ -93,7 +93,9 @@ public FileVisitResult preVisitDirectory(Path path, BasicFileAttributes attrs) t
throw new RuntimeException("No valid directory at the location: " + indexPath);
}
- log.info(String.format(Locale.ENGLISH, "IndexReaders (%d leaf readers) successfully opened. Index path=%s", readers.size(), indexPath));
+ if (log.isInfoEnabled()) {
+ log.info(String.format(Locale.ENGLISH, "IndexReaders (%d leaf readers) successfully opened. Index path=%s", readers.size(), indexPath));
+ }
if (readers.size() == 1) {
return readers.get(0);
@@ -115,7 +117,9 @@ public FileVisitResult preVisitDirectory(Path path, BasicFileAttributes attrs) t
public static Directory openDirectory(String dirPath, String dirImpl) throws IOException {
final Path path = FileSystems.getDefault().getPath(Objects.requireNonNull(dirPath));
Directory dir = openDirectory(path, dirImpl);
- log.info(String.format(Locale.ENGLISH, "DirectoryReader successfully opened. Directory path=%s", dirPath));
+ if (log.isInfoEnabled()) {
+ log.info(String.format(Locale.ENGLISH, "DirectoryReader successfully opened. Directory path=%s", dirPath));
+ }
return dir;
}
diff --git a/lucene/luke/src/java/org/apache/lucene/luke/models/util/twentynewsgroups/MessageFilesParser.java b/lucene/luke/src/java/org/apache/lucene/luke/models/util/twentynewsgroups/MessageFilesParser.java
index 5a2fe7398496..a11997450d13 100644
--- a/lucene/luke/src/java/org/apache/lucene/luke/models/util/twentynewsgroups/MessageFilesParser.java
+++ b/lucene/luke/src/java/org/apache/lucene/luke/models/util/twentynewsgroups/MessageFilesParser.java
@@ -54,7 +54,7 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attr) {
}
}
} catch (IOException e) {
- log.warn("Invalid file? " + file.toString());
+ log.warn("Invalid file? {}", file);
}
return FileVisitResult.CONTINUE;
}
diff --git a/lucene/luke/src/java/org/apache/lucene/luke/util/reflection/SubtypeCollector.java b/lucene/luke/src/java/org/apache/lucene/luke/util/reflection/SubtypeCollector.java
index 4f22a871434d..de40a187eeff 100644
--- a/lucene/luke/src/java/org/apache/lucene/luke/util/reflection/SubtypeCollector.java
+++ b/lucene/luke/src/java/org/apache/lucene/luke/util/reflection/SubtypeCollector.java
@@ -84,7 +84,7 @@ public void run() {
}
}
} catch (IOException e) {
- log.error("Cannot load jar " + url.toString(), e);
+ log.error("Cannot load jar {}", url, e);
}
}
}
diff --git a/lucene/memory/build.gradle b/lucene/memory/build.gradle
index a52f88cdbcd0..0d3bc1f7b03c 100644
--- a/lucene/memory/build.gradle
+++ b/lucene/memory/build.gradle
@@ -18,6 +18,8 @@
apply plugin: 'java-library'
+description = 'Single-document in-memory index implementation'
+
dependencies {
api project(':lucene:core')
diff --git a/lucene/misc/build.gradle b/lucene/misc/build.gradle
index eab862245879..bdf575df5628 100644
--- a/lucene/misc/build.gradle
+++ b/lucene/misc/build.gradle
@@ -17,6 +17,8 @@
apply plugin: 'java-library'
+description = 'Index tools and other miscellaneous code'
+
dependencies {
api project(':lucene:core')
testImplementation project(':lucene:test-framework')
diff --git a/lucene/misc/src/java/org/apache/lucene/util/fst/UpToTwoPositiveIntOutputs.java b/lucene/misc/src/java/org/apache/lucene/util/fst/UpToTwoPositiveIntOutputs.java
index af1766e23b5b..a6e0a66fe088 100644
--- a/lucene/misc/src/java/org/apache/lucene/util/fst/UpToTwoPositiveIntOutputs.java
+++ b/lucene/misc/src/java/org/apache/lucene/util/fst/UpToTwoPositiveIntOutputs.java
@@ -82,7 +82,8 @@ public int hashCode() {
return (int) ((first^(first>>>32)) ^ (second^(second>>32)));
}
}
-
+
+ @SuppressWarnings("deprecation")
private final static Long NO_OUTPUT = new Long(0);
private final boolean doShare;
diff --git a/lucene/misc/src/test/org/apache/lucene/search/similarity/TestLegacyBM25Similarity.java b/lucene/misc/src/test/org/apache/lucene/search/similarity/TestLegacyBM25Similarity.java
index b3a0cd24ca13..9ffdc63b2edb 100644
--- a/lucene/misc/src/test/org/apache/lucene/search/similarity/TestLegacyBM25Similarity.java
+++ b/lucene/misc/src/test/org/apache/lucene/search/similarity/TestLegacyBM25Similarity.java
@@ -23,6 +23,7 @@
import org.apache.lucene.search.similarities.BaseSimilarityTestCase;
import org.apache.lucene.search.similarities.Similarity;
+@Deprecated
public class TestLegacyBM25Similarity extends BaseSimilarityTestCase {
public void testIllegalK1() {
diff --git a/lucene/monitor/build.gradle b/lucene/monitor/build.gradle
index cc30dbed8afb..3dd65a4da7a2 100644
--- a/lucene/monitor/build.gradle
+++ b/lucene/monitor/build.gradle
@@ -17,6 +17,8 @@
apply plugin: 'java-library'
+description = 'Reverse-search implementation for monitoring and classification'
+
dependencies {
api project(':lucene:core')
diff --git a/lucene/queries/build.gradle b/lucene/queries/build.gradle
index 199b39b9f20d..64c2bf41dad0 100644
--- a/lucene/queries/build.gradle
+++ b/lucene/queries/build.gradle
@@ -17,6 +17,8 @@
apply plugin: 'java-library'
+description = 'Filters and Queries that add to core Lucene'
+
dependencies {
api project(':lucene:core')
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java
index c804523bcb97..9bd9ccc3e34e 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java
@@ -56,7 +56,7 @@ public ValueSource getValueSource() {
protected class FunctionWeight extends Weight {
protected final IndexSearcher searcher;
protected final float boost;
- protected final Map context;
+ protected final Map context;
public FunctionWeight(IndexSearcher searcher, float boost) throws IOException {
super(FunctionQuery.this);
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionRangeQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionRangeQuery.java
index f72e12b9e1c8..7848dc615733 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionRangeQuery.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionRangeQuery.java
@@ -124,8 +124,7 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo
}
private class FunctionRangeWeight extends Weight {
- @SuppressWarnings("rawtypes")
- private final Map vsContext;
+ private final Map vsContext;
public FunctionRangeWeight(IndexSearcher searcher) throws IOException {
super(FunctionRangeQuery.this);
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionScoreQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionScoreQuery.java
index 2ba77e2afb77..afd1db309969 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionScoreQuery.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionScoreQuery.java
@@ -105,7 +105,13 @@ public static FunctionScoreQuery boostByQuery(Query in, Query boostMatch, float
@Override
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
- Weight inner = in.createWeight(searcher, scoreMode.needsScores() && source.needsScores() ? scoreMode : ScoreMode.COMPLETE_NO_SCORES, 1f);
+ ScoreMode sm;
+ if (scoreMode.needsScores() && source.needsScores()) {
+ sm = ScoreMode.COMPLETE;
+ } else {
+ sm = ScoreMode.COMPLETE_NO_SCORES;
+ }
+ Weight inner = in.createWeight(searcher, sm, 1f);
if (scoreMode.needsScores() == false)
return inner;
return new FunctionScoreWeight(this, inner, source.rewrite(searcher), boost);
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSource.java
index 1ba35804002b..0b3421da4b44 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSource.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSource.java
@@ -50,7 +50,7 @@ public abstract class ValueSource {
* docID manner, and you must call this method again to iterate through
* the values again.
*/
- public abstract FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException;
+ public abstract FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException;
@Override
public abstract boolean equals(Object o);
@@ -74,14 +74,14 @@ public String toString() {
* weight info in the context. The context object will be passed to getValues()
* where this info can be retrieved.
*/
- public void createWeight(Map context, IndexSearcher searcher) throws IOException {
+ public void createWeight(Map context, IndexSearcher searcher) throws IOException {
}
/**
* Returns a new non-threadsafe context map.
*/
- public static Map newContext(IndexSearcher searcher) {
- Map context = new IdentityHashMap();
+ public static Map newContext(IndexSearcher searcher) {
+ Map context = new IdentityHashMap<>();
context.put("searcher", searcher);
return context;
}
@@ -119,7 +119,7 @@ private WrappedLongValuesSource(ValueSource in) {
@Override
public LongValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
- Map context = new IdentityHashMap<>();
+ Map context = new IdentityHashMap<>();
ScoreAndDoc scorer = new ScoreAndDoc();
context.put("scorer", scorer);
final FunctionValues fv = in.getValues(context, ctx);
@@ -196,7 +196,7 @@ private WrappedDoubleValuesSource(ValueSource in, IndexSearcher searcher) {
@Override
public DoubleValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
- Map context = new HashMap<>();
+ Map context = new HashMap<>();
ScoreAndDoc scorer = new ScoreAndDoc();
context.put("scorer", scorer);
context.put("searcher", searcher);
@@ -236,7 +236,7 @@ public boolean isCacheable(LeafReaderContext ctx) {
@Override
public Explanation explain(LeafReaderContext ctx, int docId, Explanation scoreExplanation) throws IOException {
- Map context = new HashMap<>();
+ Map context = new HashMap<>();
ScoreAndDoc scorer = new ScoreAndDoc();
scorer.score = scoreExplanation.getValue().floatValue();
context.put("scorer", scorer);
@@ -283,7 +283,7 @@ private FromDoubleValuesSource(DoubleValuesSource in) {
}
@Override
- public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
+ public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
Scorable scorer = (Scorable) context.get("scorer");
DoubleValues scores = scorer == null ? null : DoubleValuesSource.fromScorer(scorer);
@@ -365,16 +365,16 @@ public ValueSourceSortField(boolean reverse) {
@Override
public SortField rewrite(IndexSearcher searcher) throws IOException {
- Map context = newContext(searcher);
+ Map context = newContext(searcher);
createWeight(context, searcher);
return new SortField(getField(), new ValueSourceComparatorSource(context), getReverse());
}
}
class ValueSourceComparatorSource extends FieldComparatorSource {
- private final Map context;
+ private final Map context;
- public ValueSourceComparatorSource(Map context) {
+ public ValueSourceComparatorSource(Map context) {
this.context = context;
}
@@ -394,10 +394,10 @@ class ValueSourceComparator extends SimpleFieldComparator {
private final double[] values;
private FunctionValues docVals;
private double bottom;
- private final Map fcontext;
+ private final Map fcontext;
private double topValue;
- ValueSourceComparator(Map fcontext, int numHits) {
+ ValueSourceComparator(Map fcontext, int numHits) {
this.fcontext = fcontext;
values = new double[numHits];
}
@@ -429,7 +429,7 @@ public void setBottom(final int bottom) {
@Override
public void setTopValue(final Double value) {
- this.topValue = value.doubleValue();
+ this.topValue = value;
}
@Override
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java
index bb33122bdcb6..af7ae3de0ec7 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java
@@ -41,7 +41,7 @@ public BytesRefFieldSource(String field) {
}
@Override
- public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
+ public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
final FieldInfo fieldInfo = readerContext.reader().getFieldInfos().fieldInfo(field);
// To be sorted or not to be sorted, that is the question
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ComparisonBoolFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ComparisonBoolFunction.java
index 1a936d98ea45..57adf6b19415 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ComparisonBoolFunction.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ComparisonBoolFunction.java
@@ -50,7 +50,7 @@ public String name() {
}
@Override
- public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
+ public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
final FunctionValues lhsVal = this.lhs.getValues(context, readerContext);
final FunctionValues rhsVal = this.rhs.getValues(context, readerContext);
final String compLabel = this.name();
@@ -97,7 +97,7 @@ public String description() {
}
@Override
- public void createWeight(Map context, IndexSearcher searcher) throws IOException {
+ public void createWeight(Map context, IndexSearcher searcher) throws IOException {
lhs.createWeight(context, searcher);
rhs.createWeight(context, searcher);
}
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ConstValueSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ConstValueSource.java
index 125422fb2dee..b9fda157eaae 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ConstValueSource.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ConstValueSource.java
@@ -41,7 +41,7 @@ public String description() {
}
@Override
- public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
+ public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
return new FloatDocValues(this) {
@Override
public float floatVal(int doc) {
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DefFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DefFunction.java
index f3fd004675b6..2c6f2a9edddf 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DefFunction.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DefFunction.java
@@ -42,7 +42,7 @@ protected String name() {
@Override
- public FunctionValues getValues(Map fcontext, LeafReaderContext readerContext) throws IOException {
+ public FunctionValues getValues(Map fcontext, LeafReaderContext readerContext) throws IOException {
return new Values(valsArr(sources, fcontext, readerContext)) {
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DocFreqValueSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DocFreqValueSource.java
index e03e3160495e..f7abe36c0e65 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DocFreqValueSource.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DocFreqValueSource.java
@@ -29,94 +29,6 @@
import java.util.Map;
-class ConstIntDocValues extends IntDocValues {
- final int ival;
- final float fval;
- final double dval;
- final long lval;
- final String sval;
- final ValueSource parent;
-
- ConstIntDocValues(int val, ValueSource parent) {
- super(parent);
- ival = val;
- fval = val;
- dval = val;
- lval = val;
- sval = Integer.toString(val);
- this.parent = parent;
- }
-
- @Override
- public float floatVal(int doc) {
- return fval;
- }
- @Override
- public int intVal(int doc) {
- return ival;
- }
- @Override
- public long longVal(int doc) {
- return lval;
- }
- @Override
- public double doubleVal(int doc) {
- return dval;
- }
- @Override
- public String strVal(int doc) {
- return sval;
- }
- @Override
- public String toString(int doc) {
- return parent.description() + '=' + sval;
- }
-}
-
-class ConstDoubleDocValues extends DoubleDocValues {
- final int ival;
- final float fval;
- final double dval;
- final long lval;
- final String sval;
- final ValueSource parent;
-
- ConstDoubleDocValues(double val, ValueSource parent) {
- super(parent);
- ival = (int)val;
- fval = (float)val;
- dval = val;
- lval = (long)val;
- sval = Double.toString(val);
- this.parent = parent;
- }
-
- @Override
- public float floatVal(int doc) {
- return fval;
- }
- @Override
- public int intVal(int doc) {
- return ival;
- }
- @Override
- public long longVal(int doc) {
- return lval;
- }
- @Override
- public double doubleVal(int doc) {
- return dval;
- }
- @Override
- public String strVal(int doc) {
- return sval;
- }
- @Override
- public String toString(int doc) {
- return parent.description() + '=' + sval;
- }
-}
-
/**
* DocFreqValueSource returns the number of documents containing the term.
@@ -145,15 +57,15 @@ public String description() {
}
@Override
- public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
+ public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
IndexSearcher searcher = (IndexSearcher)context.get("searcher");
int docfreq = searcher.getIndexReader().docFreq(new Term(indexedField, indexedBytes));
return new ConstIntDocValues(docfreq, this);
}
@Override
- public void createWeight(Map context, IndexSearcher searcher) throws IOException {
- context.put("searcher",searcher);
+ public void createWeight(Map