Revert "LUCENE-8962: Fix intermittent test failures"

This reverts commit a5475de.

Revert "Add CHANGES entry for LUCENE-8962"

This reverts commit fdac6d8.

Revert "LUCENE-8962: Add ability to selectively merge on commit (#1155)"

This reverts commit a1791e7.

Author: Michael Sokolov

lucene/CHANGES.txt

@@ -85,6 +85,11 @@ Optimizations
 
 * LUCENE-9068: FuzzyQuery builds its Automaton up-front (Alan Woodward, Mike Drob)
 
+* LUCENE-9253: KoreanTokenizer now supports custom dictionaries(system, unknown). (Namgyu Kim)
+
+Optimizations
+---------------------
+
 * LUCENE-9113: Faster merging of SORTED/SORTED_SET doc values. (Adrien Grand)
 
 * LUCENE-9125: Optimize Automaton.step() with binary search and introduce Automaton.next(). (Bruno Roustant)

lucene/core/src/java/org/apache/lucene/index/FilterMergePolicy.java

@@ -57,11 +57,6 @@ public MergeSpecification findForcedDeletesMerges(SegmentInfos segmentInfos, Mer
     return in.findForcedDeletesMerges(segmentInfos, mergeContext);
   }
 
-  @Override
-  public MergeSpecification findFullFlushMerges(MergeTrigger mergeTrigger, SegmentInfos segmentInfos, MergeContext mergeContext) throws IOException {
-    return in.findFullFlushMerges(mergeTrigger, segmentInfos, mergeContext);
-  }
-
   @Override
   public boolean useCompoundFile(SegmentInfos infos, SegmentCommitInfo mergedInfo, MergeContext mergeContext)
       throws IOException {

lucene/core/src/java/org/apache/lucene/index/IndexWriter.java

@@ -32,8 +32,6 @@
 import java.util.Queue;
 import java.util.Set;
 import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
@@ -3166,42 +3164,6 @@ public final boolean flushNextBuffer() throws IOException {
     }
   }
 
-  private MergePolicy.OneMerge updateSegmentInfosOnMergeFinish(MergePolicy.OneMerge merge, final SegmentInfos toCommit,
-                                                                AtomicReference<CountDownLatch> mergeLatchRef) {
-    return new MergePolicy.OneMerge(merge.segments) {
-      public void mergeFinished() throws IOException {
-        super.mergeFinished();
-        CountDownLatch mergeAwaitLatch = mergeLatchRef.get();
-        if (mergeAwaitLatch == null) {
-          // Commit thread timed out waiting for this merge and moved on. No need to manipulate toCommit.
-          return;
-        }
-        if (isAborted() == false) {
-          deleter.incRef(this.info.files());
-          // Resolve "live" SegmentInfos segments to their toCommit cloned equivalents, based on segment name.
-          Set<String> mergedSegmentNames = new HashSet<>();
-          for (SegmentCommitInfo sci : this.segments) {
-            deleter.decRef(sci.files());
-            mergedSegmentNames.add(sci.info.name);
-          }
-          List<SegmentCommitInfo> toCommitMergedAwaySegments = new ArrayList<>();
-          for (SegmentCommitInfo sci : toCommit) {
-            if (mergedSegmentNames.contains(sci.info.name)) {
-              toCommitMergedAwaySegments.add(sci);
-            }
-          }
-          // Construct a OneMerge that applies to toCommit
-          MergePolicy.OneMerge applicableMerge = new MergePolicy.OneMerge(toCommitMergedAwaySegments);
-          applicableMerge.info = this.info.clone();
-          long segmentCounter = Long.parseLong(this.info.info.name.substring(1), Character.MAX_RADIX);
-          toCommit.counter = Math.max(toCommit.counter, segmentCounter + 1);
-          toCommit.applyMergeChanges(applicableMerge, false);
-        }
-        mergeAwaitLatch.countDown();
-      }
-    };
-  }
-
   private long prepareCommitInternal() throws IOException {
     startCommitTime = System.nanoTime();
     synchronized(commitLock) {
@@ -3224,8 +3186,6 @@ private long prepareCommitInternal() throws IOException {
       SegmentInfos toCommit = null;
       boolean anyChanges = false;
       long seqNo;
-      List<MergePolicy.OneMerge> commitMerges = null;
-      AtomicReference<CountDownLatch> mergeAwaitLatchRef = null;
 
       // This is copied from doFlush, except it's modified to
       // clone & incRef the flushed SegmentInfos inside the
@@ -3280,38 +3240,15 @@ private long prepareCommitInternal() throws IOException {
               // sneak into the commit point:
               toCommit = segmentInfos.clone();
 
-              if (anyChanges) {
-                // Find any merges that can execute on commit (per MergePolicy).
-                MergePolicy.MergeSpecification mergeSpec =
-                    config.getMergePolicy().findFullFlushMerges(MergeTrigger.COMMIT, segmentInfos, this);
-                if (mergeSpec != null && mergeSpec.merges.size() > 0) {
-                  int mergeCount = mergeSpec.merges.size();
-                  commitMerges = new ArrayList<>(mergeCount);
-                  mergeAwaitLatchRef = new AtomicReference<>(new CountDownLatch(mergeCount));
-                  for (MergePolicy.OneMerge oneMerge : mergeSpec.merges) {
-                    MergePolicy.OneMerge trackedMerge =
-                        updateSegmentInfosOnMergeFinish(oneMerge, toCommit, mergeAwaitLatchRef);
-                    if (registerMerge(trackedMerge) == false) {
-                      throw new IllegalStateException("MergePolicy " + config.getMergePolicy().getClass() +
-                          " returned merging segments from findFullFlushMerges");
-                    }
-                    commitMerges.add(trackedMerge);
-                  }
-                  if (infoStream.isEnabled("IW")) {
-                    infoStream.message("IW", "Registered " + mergeCount + " commit merges");
-                    infoStream.message("IW", "Before executing commit merges, had " + toCommit.size() + " segments");
-                  }
-                }
-              }
-
               pendingCommitChangeCount = changeCount.get();
 
               // This protects the segmentInfos we are now going
               // to commit.  This is important in case, eg, while
               // we are trying to sync all referenced files, a
               // merge completes which would otherwise have
               // removed the files we are now syncing.    
-              deleter.incRef(toCommit.files(false));
+              filesToCommit = toCommit.files(false); 
+              deleter.incRef(filesToCommit);
             }
             success = true;
           } finally {
@@ -3332,52 +3269,6 @@ private long prepareCommitInternal() throws IOException {
       } finally {
         maybeCloseOnTragicEvent();
       }
-
-      if (mergeAwaitLatchRef != null) {
-        CountDownLatch mergeAwaitLatch = mergeAwaitLatchRef.get();
-        // If we found and registered any merges above, within the flushLock, then we want to ensure that they
-        // complete execution. Note that since we released the lock, other merges may have been scheduled. We will
-        // block until  the merges that we registered complete. As they complete, they will update toCommit to
-        // replace merged segments with the result of each merge.
-        config.getIndexWriterEvents().beginMergeOnCommit();
-        mergeScheduler.merge(this, MergeTrigger.COMMIT, true);
-        long mergeWaitStart = System.nanoTime();
-        int abandonedCount = 0;
-        long waitTimeMillis = (long) (config.getMaxCommitMergeWaitSeconds() * 1000.0);
-        try {
-          if (mergeAwaitLatch.await(waitTimeMillis, TimeUnit.MILLISECONDS) == false) {
-            synchronized (this) {
-              // Need to do this in a synchronized block, to make sure none of our commit merges are currently
-              // executing mergeFinished (since mergeFinished itself is called from within the IndexWriter lock).
-              // After we clear the value from mergeAwaitLatchRef, the merges we schedule will still execute as
-              // usual, but when they finish, they won't attempt to update toCommit or modify segment reference
-              // counts.
-              mergeAwaitLatchRef.set(null);
-              for (MergePolicy.OneMerge commitMerge : commitMerges) {
-                if (runningMerges.contains(commitMerge) || pendingMerges.contains(commitMerge)) {
-                  abandonedCount++;
-                }
-              }
-            }
-          }
-        } catch (InterruptedException ie) {
-          throw new ThreadInterruptedException(ie);
-        } finally {
-          if (infoStream.isEnabled("IW")) {
-            infoStream.message("IW", String.format(Locale.ROOT, "Waited %.1f ms for commit merges",
-                (System.nanoTime() - mergeWaitStart)/1_000_000.0));
-            infoStream.message("IW", "After executing commit merges, had " + toCommit.size() + " segments");
-            if (abandonedCount > 0) {
-              infoStream.message("IW", "Abandoned " + abandonedCount + " commit merges after " + waitTimeMillis + " ms");
-            }
-          }
-          if (abandonedCount > 0) {
-            config.getIndexWriterEvents().abandonedMergesOnCommit(abandonedCount);
-          }
-          config.getIndexWriterEvents().finishMergeOnCommit();
-        }
-      }
-      filesToCommit = toCommit.files(false);
 
       try {
         if (anyChanges) {

lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java

@@ -114,9 +114,6 @@ public static enum OpenMode {
 
   /** Default value for whether calls to {@link IndexWriter#close()} include a commit. */
   public final static boolean DEFAULT_COMMIT_ON_CLOSE = true;
-
-  /** Default value for time to wait for merges on commit (when using a {@link MergePolicy} that implements findFullFlushMerges). */
-  public static final double DEFAULT_MAX_COMMIT_MERGE_WAIT_SECONDS = 30.0;
 
   // indicates whether this config instance is already attached to a writer.
   // not final so that it can be cloned properly.
@@ -489,24 +486,6 @@ public IndexWriterConfig setCommitOnClose(boolean commitOnClose) {
     return this;
   }
 
-  /**
-   * Expert: sets the amount of time to wait for merges returned by MergePolicy.findFullFlushMerges(...).
-   * If this time is reached, we proceed with the commit based on segments merged up to that point.
-   * The merges are not cancelled, and may still run to completion independent of the commit.
-   */
-  public IndexWriterConfig setMaxCommitMergeWaitSeconds(double maxCommitMergeWaitSeconds) {
-    this.maxCommitMergeWaitSeconds = maxCommitMergeWaitSeconds;
-    return this;
-  }
-
-  /**
-   * Set the callback that gets invoked when IndexWriter performs various actions.
-   */
-  public IndexWriterConfig setIndexWriterEvents(IndexWriterEvents indexWriterEvents) {
-    this.indexWriterEvents = indexWriterEvents;
-    return this;
-  }
-
   /** We only allow sorting on these types */
   private static final EnumSet<SortField.Type> ALLOWED_INDEX_SORT_TYPES = EnumSet.of(SortField.Type.STRING,
                                                                                      SortField.Type.LONG,

lucene/core/src/java/org/apache/lucene/index/IndexWriterEvents.java

@@ -117,12 +117,6 @@ public class LiveIndexWriterConfig {
   /** the attributes for the NRT readers */
   protected Map<String, String> readerAttributes = Collections.emptyMap();
 
-  /** Amount of time to wait for merges returned by MergePolicy.findFullFlushMerges(...) */
-  protected volatile double maxCommitMergeWaitSeconds;
-
-  /** Callback interface called on index writer actions. */
-  protected IndexWriterEvents indexWriterEvents;
-
 
   // used by IndexWriterConfig
   LiveIndexWriterConfig(Analyzer analyzer) {
@@ -147,8 +141,6 @@ public class LiveIndexWriterConfig {
     readerPooling = IndexWriterConfig.DEFAULT_READER_POOLING;
     indexerThreadPool = new DocumentsWriterPerThreadPool();
     perThreadHardLimitMB = IndexWriterConfig.DEFAULT_RAM_PER_THREAD_HARD_LIMIT_MB;
-    maxCommitMergeWaitSeconds = IndexWriterConfig.DEFAULT_MAX_COMMIT_MERGE_WAIT_SECONDS;
-    indexWriterEvents = IndexWriterEvents.NULL_EVENTS;
   }
 
   /** Returns the default analyzer to use for indexing documents. */
@@ -488,22 +480,6 @@ public String getSoftDeletesField() {
     return softDeletesField;
   }
 
-  /**
-   * Expert: return the amount of time to wait for merges returned by by MergePolicy.findFullFlushMerges(...).
-   * If this time is reached, we proceed with the commit based on segments merged up to that point.
-   * The merges are not cancelled, and may still run to completion independent of the commit.
-   */
-  public double getMaxCommitMergeWaitSeconds() {
-    return maxCommitMergeWaitSeconds;
-  }
-
-  /**
-   * Returns a callback used to signal actions taken by the {@link IndexWriter}.
-   */
-  public IndexWriterEvents getIndexWriterEvents() {
-    return indexWriterEvents;
-  }
-
   @Override
   public String toString() {
     StringBuilder sb = new StringBuilder();
@@ -529,8 +505,6 @@ public String toString() {
     sb.append("checkPendingFlushOnUpdate=").append(isCheckPendingFlushOnUpdate()).append("\n");
     sb.append("softDeletesField=").append(getSoftDeletesField()).append("\n");
     sb.append("readerAttributes=").append(getReaderAttributes()).append("\n");
-    sb.append("maxCommitMergeWaitSeconds=").append(getMaxCommitMergeWaitSeconds()).append("\n");
-    sb.append("indexWriterEvents=").append(getIndexWriterEvents().getClass().getName()).append("\n");
     return sb.toString();
   }
 

lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java

@@ -510,7 +510,7 @@ public abstract MergeSpecification findMerges(MergeTrigger mergeTrigger, Segment
  *          an original segment present in the
  *          to-be-merged index; else, it was a segment
  *          produced by a cascaded merge.
-   * @param mergeContext the MergeContext to find the merges on
+   * @param mergeContext the IndexWriter to find the merges on
    */
   public abstract MergeSpecification findForcedMerges(
       SegmentInfos segmentInfos, int maxSegmentCount, Map<SegmentCommitInfo,Boolean> segmentsToMerge, MergeContext mergeContext)
@@ -521,33 +521,11 @@ public abstract MergeSpecification findForcedMerges(
    * deletes from the index.
    *  @param segmentInfos
    *          the total set of segments in the index
-   * @param mergeContext the MergeContext to find the merges on
+   * @param mergeContext the IndexWriter to find the merges on
    */
   public abstract MergeSpecification findForcedDeletesMerges(
       SegmentInfos segmentInfos, MergeContext mergeContext) throws IOException;
 
-  /**
-   * Identifies merges that we want to execute (synchronously) on commit. By default, do not synchronously merge on commit.
-   *
-   * Any merges returned here will make {@link IndexWriter#commit()} or {@link IndexWriter#prepareCommit()} block until
-   * the merges complete or until {@link IndexWriterConfig#getMaxCommitMergeWaitSeconds()} have elapsed. This may be
-   * used to merge small segments that have just been flushed as part of the commit, reducing the number of segments in
-   * the commit. If a merge does not complete in the allotted time, it will continue to execute, but will not be reflected
-   * in the commit.
-   *
-   * If a {@link OneMerge} in the returned {@link MergeSpecification} includes a segment already included in a registered
-   * merge, then {@link IndexWriter#commit()} or {@link IndexWriter#prepareCommit()} will throw a {@link IllegalStateException}.
-   * Use {@link MergeContext#getMergingSegments()} to determine which segments are currently registered to merge.
-   *
-   * @param mergeTrigger the event that triggered the merge (COMMIT or FULL_FLUSH).
-   * @param segmentInfos the total set of segments in the index (while preparing the commit)
-   * @param mergeContext the MergeContext to find the merges on, which should be used to determine which segments are
- *                     already in a registered merge (see {@link MergeContext#getMergingSegments()}).
-   */
-  public MergeSpecification findFullFlushMerges(MergeTrigger mergeTrigger, SegmentInfos segmentInfos, MergeContext mergeContext) throws IOException {
-    return null;
-  }
-
   /**
    * Returns true if a new segment (regardless of its origin) should use the
    * compound file format. The default implementation returns <code>true</code>

lucene/core/src/java/org/apache/lucene/index/MergePolicy.java

@@ -47,10 +47,5 @@ public enum MergeTrigger {
   /**
    * Merge was triggered by a closing IndexWriter.
    */
-  CLOSING,
-
-  /**
-   * Merge was triggered on commit.
-   */
-  COMMIT,
+  CLOSING
 }

lucene/core/src/java/org/apache/lucene/index/MergeTrigger.java

@@ -45,9 +45,6 @@ public MergeSpecification findForcedMerges(SegmentInfos segmentInfos,
   @Override
   public MergeSpecification findForcedDeletesMerges(SegmentInfos segmentInfos, MergeContext mergeContext) { return null; }
 
-  @Override
-  public MergeSpecification findFullFlushMerges(MergeTrigger mergeTrigger, SegmentInfos segmentInfos, MergeContext mergeContext) { return null; }
-
   @Override
   public boolean useCompoundFile(SegmentInfos segments, SegmentCommitInfo newSegment, MergeContext mergeContext) {
     return newSegment.info.getUseCompoundFile();