Replace exact numDocs by soft-del count in SegmentCommitInfo

buildSrc/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 7.0.0-alpha1
-lucene            = 7.4.0-snapshot-1cbadda4d3
+lucene            = 7.4.0-snapshot-0a7c3f462f
 
 # optional dependencies
 spatial4j         = 0.7

modules/lang-expression/licenses/lucene-expressions-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+bf2cfa0551ebdf08a2cf3079f3c74643bd9dbb76

plugins/analysis-icu/licenses/lucene-analyzers-icu-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+82d83fcac1d9c8948aa0247fc9c87f177ddbd59b

plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+73fd4364f2931e7c8303b5927e140a7d21116c36

plugins/analysis-nori/licenses/lucene-analyzers-nori-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+0a2c4417fa9a8be078864f590a5a66b98d551cf5

plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+6fa179924f139a30fc0e5399256e1a44562ed32b

plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+5ed135d34d7868b71a725257a46dc8d8735a15d4

plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+875911b36b99c2103719f94559878a0ecb862fb6

plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+e7191628df8cb72382a20da79224aef677117849

server/licenses/lucene-analyzers-common-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+8cd761f40c4a89ed977167f0518d12e409eaf3d8

server/licenses/lucene-backward-codecs-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+8c93ed67599d345b9359586248ab92342d7d3033

server/licenses/lucene-core-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+003ed080e5184661e606091cd321c229798b22f8

server/licenses/lucene-grouping-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+0b4be9f96edfd3dbcff5aa9b3f0914e86eb9cc51

server/licenses/lucene-highlighter-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+a5dcceb5bc017cee6ab5d3ee1943aca1ac6fe074

server/licenses/lucene-join-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+b59e7441f121da969bef8eef2c0c61743b4230a8

server/licenses/lucene-memory-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+46736dbb07b432f0a7c1b3080f62932c483e5cb9

server/licenses/lucene-misc-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+ee203718d525da0c6258a51a5a32d877089fe5af

server/licenses/lucene-queries-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+cf17a332d8e42a45e8f013d5df408f4391d2620a

server/licenses/lucene-queryparser-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+04832303d70502d2ece44501cb1716f42e24fe35

server/licenses/lucene-sandbox-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+639313e3a9573779b6a28b45a7f57fc1f73ffa46

server/licenses/lucene-spatial-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+6144b493ba3588a638858d0058054758acc619b9

server/licenses/lucene-spatial-extras-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+9d00c6b8bbbbb496aecd555406267fee9e0af914

server/licenses/lucene-spatial3d-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+159cdb6d36845690cb1972d02cc0b472bb14b7f3

server/licenses/lucene-suggest-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+af1dd0218d58990cca5c1592d9722e67d233c996

server/src/main/java/org/elasticsearch/common/lucene/Lucene.java

@@ -44,7 +44,6 @@
 import org.apache.lucene.index.SegmentCommitInfo;
 import org.apache.lucene.index.SegmentInfos;
 import org.apache.lucene.index.SegmentReader;
-import org.apache.lucene.index.SoftDeletesDirectoryReaderWrapper;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.FieldDoc;
@@ -145,21 +144,11 @@ public static Iterable<String> files(SegmentInfos infos) throws IOException {
     public static int getNumDocs(SegmentInfos info) {
         int numDocs = 0;
         for (SegmentCommitInfo si : info) {
-            numDocs += si.info.maxDoc() - si.getDelCount();
+            numDocs += si.info.maxDoc() - si.getDelCount() - si.getSoftDelCount();
         }
         return numDocs;
     }
 
-    /**
-     * Unlike {@link #getNumDocs(SegmentInfos)} this method returns a numDocs that always excludes soft-deleted docs.
-     * This method is expensive thus prefer using {@link #getNumDocs(SegmentInfos)} unless an exact numDocs is required.
-     */
-    public static int getExactNumDocs(IndexCommit commit) throws IOException {
-        try (DirectoryReader reader = DirectoryReader.open(commit)) {
-            return new SoftDeletesDirectoryReaderWrapper(reader, Lucene.SOFT_DELETE_FIELD).numDocs();
-        }
-    }
-
     /**
      * Reads the segments infos from the given commit, failing if it fails to load
      */
@@ -212,6 +201,7 @@ public static SegmentInfos pruneUnreferencedFiles(String segmentsFileName, Direc
         }
         final CommitPoint cp = new CommitPoint(si, directory);
         try (IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)
+                .setSoftDeletesField(Lucene.SOFT_DELETE_FIELD)
                 .setIndexCommit(cp)
                 .setCommitOnClose(false)
                 .setMergePolicy(NoMergePolicy.INSTANCE)
@@ -235,6 +225,7 @@ public static void cleanLuceneIndex(Directory directory) throws IOException {
             }
         }
         try (IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)
+                .setSoftDeletesField(Lucene.SOFT_DELETE_FIELD)
                 .setMergePolicy(NoMergePolicy.INSTANCE) // no merges
                 .setCommitOnClose(false) // no commits
                 .setOpenMode(IndexWriterConfig.OpenMode.CREATE))) // force creation - don't append...

server/src/main/java/org/elasticsearch/index/engine/CommitStats.java

@@ -39,13 +39,13 @@ public final class CommitStats implements Streamable, ToXContentFragment {
     private String id; // lucene commit id in base 64;
     private int numDocs;
 
-    public CommitStats(SegmentInfos segmentInfos, int numDocs) {
+    public CommitStats(SegmentInfos segmentInfos) {
         // clone the map to protect against concurrent changes
         userData = MapBuilder.<String, String>newMapBuilder().putAll(segmentInfos.getUserData()).immutableMap();
         // lucene calls the current generation, last generation.
         generation = segmentInfos.getLastGeneration();
         id = Base64.getEncoder().encodeToString(segmentInfos.getId());
-        this.numDocs = numDocs;
+        numDocs = Lucene.getNumDocs(segmentInfos);
     }
 
     private CommitStats() {

server/src/main/java/org/elasticsearch/index/engine/Engine.java

@@ -632,9 +632,7 @@ protected final void ensureOpen() {
 
     /** get commits stats for the last commit */
     public CommitStats commitStats() {
-        try (Engine.Searcher searcher = acquireSearcher("commit_stats", Engine.SearcherScope.INTERNAL)) {
-            return new CommitStats(getLastCommittedSegmentInfos(), searcher.reader().numDocs());
-        }
+        return new CommitStats(getLastCommittedSegmentInfos());
     }
 
     /**

server/src/main/java/org/elasticsearch/index/engine/TranslogLeafReader.java

@@ -56,13 +56,13 @@ final class TranslogLeafReader extends LeafReader {
     private final Translog.Index operation;
     private static final FieldInfo FAKE_SOURCE_FIELD
         = new FieldInfo(SourceFieldMapper.NAME, 1, false, false, false, IndexOptions.NONE, DocValuesType.NONE, -1, Collections.emptyMap(),
-        0,0);
+        0, 0, false);
     private static final FieldInfo FAKE_ROUTING_FIELD
         = new FieldInfo(RoutingFieldMapper.NAME, 2, false, false, false, IndexOptions.NONE, DocValuesType.NONE, -1, Collections.emptyMap(),
-        0,0);
+        0, 0, false);
     private static final FieldInfo FAKE_ID_FIELD
         = new FieldInfo(IdFieldMapper.NAME, 3, false, false, false, IndexOptions.NONE, DocValuesType.NONE, -1, Collections.emptyMap(),
-        0,0);
+        0, 0, false);
     private final Version indexVersionCreated;
 
     TranslogLeafReader(Translog.Index operation, Version indexVersionCreated) {

server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java

@@ -156,6 +156,7 @@ void addIndices(final RecoveryState.Index indexRecoveryStats, final Directory ta
         final Directory hardLinkOrCopyTarget = new org.apache.lucene.store.HardlinkCopyDirectoryWrapper(target);
 
         IndexWriterConfig iwc = new IndexWriterConfig(null)
+            .setSoftDeletesField(Lucene.SOFT_DELETE_FIELD)
             .setCommitOnClose(false)
             // we don't want merges to happen here - we call maybe merge on the engine
             // later once we stared it up otherwise we would need to wait for it here

server/src/main/java/org/elasticsearch/index/store/Store.java

@@ -864,7 +864,7 @@ static LoadedMetadata loadMetadata(IndexCommit commit, Directory directory, Logg
             Map<String, String> commitUserDataBuilder = new HashMap<>();
             try {
                 final SegmentInfos segmentCommitInfos = Store.readSegmentsInfo(commit, directory);
-                numDocs = Lucene.getExactNumDocs(commit != null ? commit : findIndexCommit(directory, segmentCommitInfos));
+                numDocs = Lucene.getNumDocs(segmentCommitInfos);
                 commitUserDataBuilder.putAll(segmentCommitInfos.getUserData());
                 Version maxVersion = segmentCommitInfos.getMinSegmentLuceneVersion(); // we don't know which version was used to write so we take the max version.
                 for (SegmentCommitInfo info : segmentCommitInfos) {
@@ -947,16 +947,6 @@ public static void hashFile(BytesRefBuilder fileHash, InputStream in, long size)
             assert fileHash.length() == len : Integer.toString(fileHash.length()) + " != " + Integer.toString(len);
         }
 
-        private static IndexCommit findIndexCommit(Directory directory, SegmentInfos sis) throws IOException {
-            List<IndexCommit> commits = DirectoryReader.listCommits(directory);
-            for (IndexCommit commit : commits) {
-                if (commit.getSegmentsFileName().equals(sis.getSegmentsFileName())) {
-                    return commit;
-                }
-            }
-            throw new IOException("Index commit [" + sis.getSegmentsFileName() + "] is not found");
-        }
-
         @Override
         public Iterator<StoreFileMetaData> iterator() {
             return metadata.values().iterator();
@@ -1604,6 +1594,7 @@ private static IndexWriter newIndexWriter(final IndexWriterConfig.OpenMode openM
         throws IOException {
         assert openMode == IndexWriterConfig.OpenMode.APPEND || commit == null : "can't specify create flag with a commit";
         IndexWriterConfig iwc = new IndexWriterConfig(null)
+            .setSoftDeletesField(Lucene.SOFT_DELETE_FIELD)
             .setCommitOnClose(false)
             .setIndexCommit(commit)
             // we don't want merges to happen here - we call maybe merge on the engine

server/src/main/java/org/elasticsearch/index/translog/TruncateTranslogCommand.java

@@ -33,6 +33,7 @@
 import org.apache.lucene.store.LockObtainFailedException;
 import org.apache.lucene.store.NativeFSLockFactory;
 import org.apache.lucene.store.OutputStreamDataOutput;
+import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.cli.EnvironmentAwareCommand;
@@ -179,6 +180,7 @@ protected void execute(Terminal terminal, OptionSet options, Environment env) th
             terminal.println("Marking index with the new history uuid");
             // commit the new histroy id
             IndexWriterConfig iwc = new IndexWriterConfig(null)
+                .setSoftDeletesField(Lucene.SOFT_DELETE_FIELD)
                 .setCommitOnClose(false)
                 // we don't want merges to happen here - we call maybe merge on the engine
                 // later once we stared it up otherwise we would need to wait for it here

server/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java

@@ -19,7 +19,6 @@
 package org.elasticsearch.indices.flush;
 
 import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.apache.lucene.index.SegmentInfos;
 import org.elasticsearch.Assertions;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
@@ -42,13 +41,13 @@
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.common.util.concurrent.CountDown;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.engine.CommitStats;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.shard.IndexEventListener;
 import org.elasticsearch.index.shard.IndexShard;
@@ -468,19 +467,15 @@ public String executor() {
         }
     }
 
-    private PreSyncedFlushResponse performPreSyncedFlush(PreShardSyncedFlushRequest request) throws IOException {
+    private PreSyncedFlushResponse performPreSyncedFlush(PreShardSyncedFlushRequest request) {
         IndexShard indexShard = indicesService.indexServiceSafe(request.shardId().getIndex()).getShard(request.shardId().id());
         FlushRequest flushRequest = new FlushRequest().force(false).waitIfOngoing(true);
         logger.trace("{} performing pre sync flush", request.shardId());
         indexShard.flush(flushRequest);
-        try (Engine.IndexCommitRef commitRef = indexShard.acquireLastIndexCommit(false)) {
-            final SegmentInfos segmentInfos = Lucene.readSegmentInfos(commitRef.getIndexCommit());
-            final int numDocs = Lucene.getExactNumDocs(commitRef.getIndexCommit());
-            final Engine.CommitId commitId = new Engine.CommitId(segmentInfos.getId());
-            final String syncId = segmentInfos.userData.get(Engine.SYNC_COMMIT_ID);
-            logger.trace("{} pre sync flush done. commit id {}, num docs {}", request.shardId(), commitId, numDocs);
-            return new PreSyncedFlushResponse(commitId, numDocs, syncId);
-        }
+        final CommitStats commitStats = indexShard.commitStats();
+        final Engine.CommitId commitId = commitStats.getRawCommitId();
+        logger.trace("{} pre sync flush done. commit id {}, num docs {}", request.shardId(), commitId, commitStats.getNumDocs());
+        return new PreSyncedFlushResponse(commitId, commitStats.getNumDocs(), commitStats.syncId());
     }
 
     private ShardSyncedFlushResponse performSyncedFlush(ShardSyncedFlushRequest request) {

server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java

@@ -289,7 +289,7 @@ public RecoveryResponse newInstance() {
      * @param recoveryTarget the target of the recovery
      * @return a snapshot of the store metadata
      */
-    static Store.MetadataSnapshot getStoreMetadataSnapshot(final Logger logger, final RecoveryTarget recoveryTarget) {
+    private Store.MetadataSnapshot getStoreMetadataSnapshot(final RecoveryTarget recoveryTarget) {
         try {
             return recoveryTarget.indexShard().snapshotStoreMetadata();
         } catch (final org.apache.lucene.index.IndexNotFoundException e) {
@@ -312,7 +312,7 @@ private StartRecoveryRequest getStartRecoveryRequest(final RecoveryTarget recove
         final StartRecoveryRequest request;
         logger.trace("{} collecting local files for [{}]", recoveryTarget.shardId(), recoveryTarget.sourceNode());
 
-        final Store.MetadataSnapshot metadataSnapshot = getStoreMetadataSnapshot(logger, recoveryTarget);
+        final Store.MetadataSnapshot metadataSnapshot = getStoreMetadataSnapshot(recoveryTarget);
         logger.trace("{} local file count [{}]", recoveryTarget.shardId(), metadataSnapshot.size());
 
         final long startingSeqNo;

server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java

@@ -1425,6 +1425,7 @@ public void restore() throws IOException {
                     // empty shard would cause exceptions to be thrown.  Since there is no data to restore from an empty
                     // shard anyway, we just create the empty shard here and then exit.
                     IndexWriter writer = new IndexWriter(store.directory(), new IndexWriterConfig(null)
+                        .setSoftDeletesField(Lucene.SOFT_DELETE_FIELD)
                         .setOpenMode(IndexWriterConfig.OpenMode.CREATE)
                         .setCommitOnClose(true));
                     writer.close();

server/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java

@@ -23,7 +23,6 @@
 import org.apache.lucene.index.CheckIndex;
 import org.apache.lucene.index.IndexFileNames;
 import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
@@ -108,7 +107,6 @@
 import static org.hamcrest.Matchers.notNullValue;
 
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE)
-@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/pull/30228") // What if DV is corrupted?
 public class CorruptedFileIT extends ESIntegTestCase {
 
     @Override