Failed search on a shard tries a local replica on a network thread

When a search on a shard to a remove node fails, and then replica exists on the local node, then the execution of the search is done on the network thread. This is problematic since we need to execute it on the actual search thread pool, but can also explain elastic#4519, where the get happens on the network thread and it waits to send the get request till the network thread we use is freed (deadlock...) fixes elastic#4526 note, re-enable the geo shape fetch test, this fix should solve it as well
mute · Dec 19, 2013 · 14f1c04 · 14f1c04
1 parent 179152f
commit 14f1c04
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 4 deletions.
diff --git a/src/main/java/org/elasticsearch/action/search/type/TransportSearchTypeAction.java b/src/main/java/org/elasticsearch/action/search/type/TransportSearchTypeAction.java
@@ -278,14 +278,23 @@ void onFirstPhaseResult(final int shardIndex, @Nullable ShardRouting shard, @Nul
                     }
                 }
             } else {
-                ShardRouting nextShard = shardIt.nextOrNull();
+                final ShardRouting nextShard = shardIt.nextOrNull();
                 final boolean lastShard = nextShard == null;
                 // trace log this exception
                 if (logger.isTraceEnabled() && t != null) {
                     logger.trace(executionFailureMsg(shard, shardIt, request, lastShard), t);
                 }
                 if (!lastShard) {
-                    performFirstPhase(shardIndex, shardIt, nextShard);
+                    try {
+                        threadPool.executor(ThreadPool.Names.SEARCH).execute(new Runnable() {
+                            @Override
+                            public void run() {
+                                performFirstPhase(shardIndex, shardIt, nextShard);
+                            }
+                        });
+                    } catch (Throwable t1) {
+                        onFirstPhaseResult(shardIndex, shard, shard.currentNodeId(), shardIt, t1);
+                    }
                 } else {
                     // no more shards active, add a failure
                     if (logger.isDebugEnabled() && !logger.isTraceEnabled()) { // do not double log this exception

diff --git a/src/test/java/org/elasticsearch/search/geo/GeoShapeIntegrationTests.java b/src/test/java/org/elasticsearch/search/geo/GeoShapeIntegrationTests.java
@@ -132,9 +132,7 @@ public void testEdgeCases() throws Exception {
         assertThat(searchResponse.getHits().getAt(0).id(), equalTo("blakely"));
     }
 
-    // TODO for some reason it get stuck on the get when fetching the source document
     @Test()
-    @AwaitsFix(bugUrl = "for some reason it get stuck on the get when fetching the source document")
     public void testIndexedShapeReference() throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
                 .startObject("properties").startObject("location")