Fix issue where entire shards are lost due to too many open files exc…

…eptions and a but in Lucene's

IndexWriter / DirectoryReader where an existsing index was not detected and then deleted due to a wrong
creation mode. See LUCENE-4870

Closes elastic#2812

src/main/java/org/elasticsearch/common/lucene/Lucene.java

@@ -22,7 +22,9 @@
 import org.apache.lucene.analysis.KeywordAnalyzer;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.index.*;
+import org.apache.lucene.index.SegmentInfos.FindSegmentsFile;
 import org.apache.lucene.search.*;
+import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.Version;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -32,6 +34,7 @@
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.field.data.FieldDataType;
 
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.lang.reflect.Field;
 
@@ -355,4 +358,28 @@ public boolean acceptsDocsOutOfOrder() {
     private Lucene() {
 
     }
+
+    // LUCENE UPGRADE this is a workaround for LUCENE-4870
+    public static final boolean indexExists(final Directory directory) {
+        try {
+            new FindSegmentsFile(directory) {
+                @Override
+                protected Object doBody(String segmentFileName) throws IOException {
+                    try {
+                        new SegmentInfos().read(directory, segmentFileName);
+                    } catch (FileNotFoundException ex) {
+                        if (!directory.fileExists(segmentFileName)) {
+                            throw ex;
+                        }
+                        // this is ok - we might have run into a access
+                        // exception here or even worse a too many open files exception.
+                    }
+                    return null;
+                }
+            }.run();
+            return true;
+        } catch (IOException ioe) {
+            return false;
+        }
+    }
 }

src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java

@@ -254,7 +254,7 @@ public void start() throws EngineException {
             try {
                 // commit on a just opened writer will commit even if there are no changes done to it
                 // we rely on that for the commit data translog id key
-                if (IndexReader.indexExists(store.directory())) {
+                if (Lucene.indexExists(store.directory())) {
                     Map<String, String> commitUserData = IndexReader.getCommitUserData(store.directory());
                     if (commitUserData.containsKey(Translog.TRANSLOG_ID_KEY)) {
                         translogIdGenerator.set(Long.parseLong(commitUserData.get(Translog.TRANSLOG_ID_KEY)));
@@ -1349,7 +1349,7 @@ private IndexWriter createWriter() throws IOException {
                 logger.warn("shard is locked, releasing lock");
                 IndexWriter.unlock(store.directory());
             }
-            boolean create = !IndexReader.indexExists(store.directory());
+            boolean create = !Lucene.indexExists(store.directory());
             IndexWriterConfig config = new IndexWriterConfig(Lucene.VERSION, analysisService.defaultIndexAnalyzer());
             config.setOpenMode(create ? IndexWriterConfig.OpenMode.CREATE : IndexWriterConfig.OpenMode.APPEND);
             config.setIndexDeletionPolicy(deletionPolicy);
@@ -1368,7 +1368,7 @@ private IndexWriter createWriter() throws IOException {
         }
         return indexWriter;
     }
-
+    
     public static final String INDEX_TERM_INDEX_INTERVAL = "index.term_index_interval";
     public static final String INDEX_TERM_INDEX_DIVISOR = "index.term_index_divisor";
     public static final String INDEX_INDEX_CONCURRENCY = "index.index_concurrency";

src/main/java/org/elasticsearch/index/gateway/blobstore/BlobStoreIndexShardGateway.java

@@ -31,6 +31,7 @@
 import org.elasticsearch.common.io.FastByteArrayInputStream;
 import org.elasticsearch.common.io.FastByteArrayOutputStream;
 import org.elasticsearch.common.io.stream.BytesStreamInput;
+import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.store.InputStreamIndexInput;
 import org.elasticsearch.common.lucene.store.ThreadSafeInputStreamIndexInput;
 import org.elasticsearch.common.settings.Settings;
@@ -608,7 +609,7 @@ private void recoverIndex(CommitPoint commitPoint, ImmutableMap<String, BlobMeta
         // read the gateway data persisted
         long version = -1;
         try {
-            if (IndexReader.indexExists(store.directory())) {
+            if (Lucene.indexExists(store.directory())) {
                 version = IndexReader.getCurrentVersion(store.directory());
             }
         } catch (IOException e) {

src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java

@@ -36,6 +36,7 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.FastByteArrayOutputStream;
+import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.metrics.MeanMetric;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
@@ -809,7 +810,7 @@ private void checkIndex(boolean throwException) throws IndexShardException {
         try {
             checkIndexTook = 0;
             long time = System.currentTimeMillis();
-            if (!IndexReader.indexExists(store.directory())) {
+            if (!Lucene.indexExists(store.directory())) {
                 return;
             }
             CheckIndex checkIndex = new CheckIndex(store.directory());