Merge pull request #11539 from mikemccand/fake_disk_full_corruption

Improve exception message when shard has a partial commit (segments_N file) due to prior disk full

src/main/java/org/elasticsearch/common/lucene/Lucene.java

@@ -789,10 +789,10 @@ public void delete() {
      * Returns <code>true</code> iff the store contains an index that contains segments that were
      * not upgraded to the lucene 4.x format.
      */
-    static boolean indexNeeds3xUpgrading(Directory directory) throws IOException {
-        final String si =  SegmentInfos.getLastCommitSegmentsFileName(directory);
-        if (si != null) {
-            try (IndexInput input = directory.openInput(si, IOContext.READONCE)) {
+    public static boolean indexNeeds3xUpgrading(Directory directory) throws IOException {
+        final String segmentsFile = SegmentInfos.getLastCommitSegmentsFileName(directory);
+        if (segmentsFile != null) {
+            try (IndexInput input = directory.openInput(segmentsFile, IOContext.READONCE)) {
                 return input.readInt() != CodecUtil.CODEC_MAGIC; // check if it's a 4.x commit point
             }
         }
@@ -801,20 +801,18 @@ static boolean indexNeeds3xUpgrading(Directory directory) throws IOException {
 
     /**
      * Upgrades the segments metadata of the index to match a lucene 4.x index format. In particular it ensures that each
-     * segment has a .si file even if it was written with lucene 3.x
+     * segment has a .si file even if it was written with lucene 3.x.  Only call this if {@link #indexNeeds3xUpgrading}
+     * returned true.
      */
-    public static boolean upgradeLucene3xSegmentsMetadata(Directory directory) throws IOException {
-        if (indexNeeds3xUpgrading(directory)) {
-            try (final IndexWriter iw = new IndexWriter(directory, new IndexWriterConfig(Version.LATEST, Lucene.STANDARD_ANALYZER)
-                    .setMergePolicy(NoMergePolicy.INSTANCE)
-                    .setOpenMode(IndexWriterConfig.OpenMode.APPEND))) {
-                Map<String, String> commitData = iw.getCommitData(); // this is a trick to make IW to actually do a commit - we have to preserve the last committed data as well
-                // for ES to get the translog ID back
-                iw.setCommitData(commitData);
-                iw.commit();
-            }
-            return true;
+    public static void upgradeLucene3xSegmentsMetadata(Directory directory) throws IOException {
+        try (final IndexWriter iw = new IndexWriter(directory, new IndexWriterConfig(Version.LATEST, Lucene.STANDARD_ANALYZER)
+                                                    .setMergePolicy(NoMergePolicy.INSTANCE)
+                                                    .setOpenMode(IndexWriterConfig.OpenMode.APPEND))) {
+            Map<String, String> commitData = iw.getCommitData();
+            // this is a trick to make IW to actually do a commit - we have to preserve the last committed data as well
+            // for ES to get the translog ID back
+            iw.setCommitData(commitData);
+            iw.commit();
         }
-        return false;
     }
 }

src/main/java/org/elasticsearch/index/engine/InternalEngine.java

@@ -119,19 +119,12 @@ public InternalEngine(EngineConfig engineConfig) throws EngineException {
         SearcherManager manager = null;
         boolean success = false;
         try {
-            try {
-                boolean autoUpgrade = true;
-                // If the index was created on 0.20.7 (Lucene 3.x) or earlier,
-                // it needs to be upgraded
-                autoUpgrade = Version.indexCreated(engineConfig.getIndexSettings()).onOrBefore(Version.V_0_20_7);
-                if (autoUpgrade) {
-                    logger.debug("[{}] checking for 3x segments to upgrade", shardId);
-                    upgrade3xSegments(store);
-                } else {
-                    logger.debug("[{}] skipping check for 3x segments", shardId);
-                }
-            } catch (IOException ex) {
-                throw new EngineCreationFailureException(shardId, "failed to upgrade 3x segments", ex);
+            // If the index was created on 0.20.7 (Lucene 3.x) or earlier, its commit point (segments_N file) needs to be upgraded:
+            if (Version.indexCreated(engineConfig.getIndexSettings()).onOrBefore(Version.V_0_20_7)) {
+                logger.debug("[{}] checking for 3x segments to upgrade", shardId);
+                maybeUpgrade3xSegments(store);
+            } else {
+                logger.debug("[{}] skipping check for 3x segments", shardId);
             }
             this.onGoingRecoveries = new FlushingRecoveryCounter(this, store, logger);
             this.lastDeleteVersionPruneTimeMSec = engineConfig.getThreadPool().estimatedTimeInMillis();
@@ -1139,10 +1132,25 @@ public void warm(AtomicReader reader) throws IOException {
         }
     }
 
-    protected void upgrade3xSegments(Store store) throws IOException {
+    protected void maybeUpgrade3xSegments(Store store) throws EngineException {
         store.incRef();
         try {
-            if (Lucene.upgradeLucene3xSegmentsMetadata(store.directory())) {
+            boolean doUpgrade;
+            try {
+                doUpgrade = Lucene.indexNeeds3xUpgrading(store.directory());
+            } catch (IOException ex) {
+                // This can happen if commit was truncated (e.g. due to prior disk full), and this case requires user intervention (remove the broken
+                // commit file so Lucene falls back to a previous good one, and also clear ES's corrupted_XXX marker file), and the shard
+                // should be OK:
+                throw new EngineCreationFailureException(shardId, "failed to read commit", ex);
+            }
+
+            if (doUpgrade) {
+                try {
+                    Lucene.upgradeLucene3xSegmentsMetadata(store.directory());
+                } catch (IOException ex) {
+                    throw new EngineCreationFailureException(shardId, "failed to upgrade 3.x segments_N commit point", ex);
+                }
                 logger.debug("upgraded current 3.x segments file on startup");
             } else {
                 logger.debug("segments file is already after 3.x; not upgrading");

src/test/java/org/elasticsearch/common/lucene/LuceneTest.java

@@ -330,14 +330,7 @@ public void testNeedsUpgrading() throws URISyntaxException, IOException {
                         assertTrue(Lucene.indexNeeds3xUpgrading(dir));
                     }
 
-                    for (int i = 0; i < 2; i++) {
-                        boolean upgraded = Lucene.upgradeLucene3xSegmentsMetadata(dir);
-                        if (i == 0) {
-                            assertTrue(upgraded);
-                        } else {
-                            assertFalse(upgraded);
-                        }
-                    }
+                    Lucene.upgradeLucene3xSegmentsMetadata(dir);
 
                     for (int i = 0; i < 2; i++) {
                         assertFalse(Lucene.indexNeeds3xUpgrading(dir));