Failed search on a shard tries a local replica on a network thread

When a search on a shard to a remove node fails, and then replica exists on the local node, then the execution of the search is done on the network thread. This is problematic since we need to execute it on the actual search thread pool, but can also explain elastic#4519, where the get happens on the network thread and it waits to send the get request till the network thread we use is freed (deadlock...) fixes elastic#4526 note, re-enable the geo shape fetch test, this fix should solve it as well
brusic · Jan 19, 2014 · 1d7ce08 · 1d7ce08
1 parent a10cc78
commit 1d7ce08
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 5 deletions.
diff --git a/src/main/java/org/elasticsearch/action/search/type/TransportSearchTypeAction.java b/src/main/java/org/elasticsearch/action/search/type/TransportSearchTypeAction.java
@@ -278,14 +278,23 @@ void onFirstPhaseResult(final int shardIndex, @Nullable ShardRouting shard, @Nul
                     }
                 }
             } else {
-                ShardRouting nextShard = shardIt.nextOrNull();
+                final ShardRouting nextShard = shardIt.nextOrNull();
                 final boolean lastShard = nextShard == null;
                 // trace log this exception
                 if (logger.isTraceEnabled() && t != null) {
                     logger.trace(executionFailureMsg(shard, shardIt, request, lastShard), t);
                 }
                 if (!lastShard) {
-                    performFirstPhase(shardIndex, shardIt, nextShard);
+                    try {
+                        threadPool.executor(ThreadPool.Names.SEARCH).execute(new Runnable() {
+                            @Override
+                            public void run() {
+                                performFirstPhase(shardIndex, shardIt, nextShard);
+                            }
+                        });
+                    } catch (Throwable t1) {
+                        onFirstPhaseResult(shardIndex, shard, shard.currentNodeId(), shardIt, t1);
+                    }
                 } else {
                     // no more shards active, add a failure
                     if (logger.isDebugEnabled() && !logger.isTraceEnabled()) { // do not double log this exception

diff --git a/src/test/java/org/elasticsearch/search/geo/GeoShapeIntegrationTests.java b/src/test/java/org/elasticsearch/search/geo/GeoShapeIntegrationTests.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.search.geo;
 
-import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.geo.builders.ShapeBuilder;
@@ -147,9 +146,7 @@ public void testEdgeCases() throws Exception {
         assertThat(searchResponse.getHits().getAt(0).id(), equalTo("blakely"));
     }
 
-    // TODO this test causes hangs, blocking on the action get when fetching the shape for some reason
     @Test
-    @AwaitsFix(bugUrl = "this test causes hangs, blocking on the action get when fetching the shape for some reason")
     public void testIndexedShapeReference() throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
                 .startObject("properties").startObject("location")