Skip to content

Commit fe7fe79

Browse files
hossmanhossman
committed
SOLR-17744: Solr now enables Jetty's Graceful Shutdown features to prevent client connections from being abruptly terminated on orderly shutdown
(cherry picked from commit d0d71a7)
1 parent 31e86bc commit fe7fe79

File tree

8 files changed

+208
-0
lines changed

8 files changed

+208
-0
lines changed

‎solr/CHANGES.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,8 @@ Improvements
4949

5050
* SOLR-17732: Score-based return fields other than "score" can now be returned in distributed queries. (Houston Putman)
5151

52+
* SOLR-17744: Solr now enables Jetty's Graceful Shutdown features to prevent client connections from being abruptly terminated on orderly shutdown (hossman)
53+
5254
Optimizations
5355
---------------------
5456
* SOLR-17578: Remove ZkController internal core supplier, for slightly faster reconnection after Zookeeper session loss. (Pierre Salagnac)

‎solr/bin/solr‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -304,6 +304,11 @@ else
304304
fi
305305
export SOLR_URL_SCHEME
306306

307+
# Gracefully wait for existing requests on shutdown
308+
if [ "${SOLR_JETTY_GRACEFUL:-true}" == "true" ]; then
309+
SOLR_JETTY_CONFIG+=("--module=graceful")
310+
fi
311+
307312
# Requestlog options
308313
if [ "${SOLR_REQUESTLOG_ENABLED:-true}" == "true" ]; then
309314
SOLR_JETTY_CONFIG+=("--module=requestlog")

‎solr/core/src/test/org/apache/solr/cloud/SplitShardTest.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -390,6 +390,7 @@ private boolean testColl(
390390
(s, replica) -> {
391391
if (replica.getNodeName().equals(jetty.getNodeName())
392392
&& !replica.isLeader()
393+
&& replica.getState().equals(Replica.State.ACTIVE)
393394
&& set.contains(replica.shard)) {
394395
set.remove(replica.shard);
395396
}

‎solr/core/src/test/org/apache/solr/cloud/TestGracefulJettyShutdown.java‎

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.solr.cloud;
19+
20+
import java.lang.invoke.MethodHandles;
21+
import java.util.ArrayList;
22+
import java.util.Arrays;
23+
import java.util.List;
24+
import java.util.concurrent.ExecutorService;
25+
import java.util.concurrent.Future;
26+
import java.util.concurrent.Semaphore;
27+
import java.util.concurrent.TimeUnit;
28+
import org.apache.solr.SolrTestCaseJ4;
29+
import org.apache.solr.client.solrj.SolrClient;
30+
import org.apache.solr.client.solrj.impl.CloudSolrClient;
31+
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
32+
import org.apache.solr.client.solrj.request.QueryRequest;
33+
import org.apache.solr.client.solrj.response.QueryResponse;
34+
import org.apache.solr.common.SolrException;
35+
import org.apache.solr.common.util.ExecutorUtil;
36+
import org.apache.solr.common.util.NamedList;
37+
import org.apache.solr.core.SolrCore;
38+
import org.apache.solr.embedded.JettyConfig;
39+
import org.apache.solr.embedded.JettySolrRunner;
40+
import org.apache.solr.handler.component.SearchHandler;
41+
import org.apache.solr.request.SolrQueryRequest;
42+
import org.apache.solr.response.SolrQueryResponse;
43+
import org.slf4j.Logger;
44+
import org.slf4j.LoggerFactory;
45+
46+
public class TestGracefulJettyShutdown extends SolrTestCaseJ4 {
47+
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
48+
49+
public void testSingleShardInFlightRequestsDuringShutDown() throws Exception {
50+
final String collection = getSaferTestName();
51+
final String handler = "/foo";
52+
53+
final Semaphore handlerGate = new Semaphore(0);
54+
final Semaphore handlerSignal = new Semaphore(0);
55+
56+
final ExecutorService exec = ExecutorUtil.newMDCAwareCachedThreadPool("client-requests");
57+
final MiniSolrCloudCluster cluster =
58+
new MiniSolrCloudCluster(1, createTempDir(), JettyConfig.builder().build());
59+
try {
60+
assertTrue(
61+
CollectionAdminRequest.createCollection(collection, "_default", 1, 1)
62+
.process(cluster.getSolrClient())
63+
.isSuccess());
64+
65+
final JettySolrRunner nodeToStop = cluster.getJettySolrRunner(0);
66+
67+
// register our custom handler with "all" (one) of our SolrCores
68+
for (String coreName : nodeToStop.getCoreContainer().getLoadedCoreNames()) {
69+
try (SolrCore core = nodeToStop.getCoreContainer().getCore(coreName)) {
70+
final BlockingSearchHandler h = new BlockingSearchHandler(handlerGate, handlerSignal);
71+
h.inform(core);
72+
core.registerRequestHandler(handler, h);
73+
}
74+
}
75+
76+
final CloudSolrClient cloudClient = cluster.getSolrClient();
77+
78+
// add a few docs...
79+
cloudClient.add(collection, sdoc("id", "xxx", "foo_s", "aaa"));
80+
cloudClient.add(collection, sdoc("id", "yyy", "foo_s", "bbb"));
81+
cloudClient.add(collection, sdoc("id", "zzz", "foo_s", "aaa"));
82+
cloudClient.commit(collection);
83+
84+
final List<Future<QueryResponse>> results = new ArrayList<>(13);
85+
86+
try (SolrClient jettyClient = nodeToStop.newClient()) {
87+
final QueryRequest req = new QueryRequest(params("q", "foo_s:aaa"));
88+
req.setPath(handler);
89+
90+
// check inflight requests using both clients...
91+
for (SolrClient client : Arrays.asList(cloudClient, jettyClient)) {
92+
results.add(
93+
exec.submit(
94+
() -> {
95+
return req.process(client, collection);
96+
}));
97+
}
98+
99+
// wait for our handlers to indicate they have recieved the requests and started processing
100+
log.info("Waiting for signals from both requests");
101+
assertTrue(handlerSignal.tryAcquire(2, 300, TimeUnit.SECONDS)); // safety valve
102+
103+
// stop our node (via executor so it doesn't block) and open the gate for our handlers
104+
log.info("Stopping jetty node");
105+
final Future<Boolean> stopped =
106+
exec.submit(
107+
() -> {
108+
nodeToStop.stop();
109+
return true;
110+
});
111+
log.info("Releasing gate for requests");
112+
handlerGate.release(2);
113+
log.info("Released gate for requests");
114+
115+
// confirm success of requests
116+
assertEquals(2, results.size());
117+
for (Future<QueryResponse> f : results) {
118+
final QueryResponse rsp = f.get(300, TimeUnit.SECONDS); // safety valve
119+
assertEquals(2, rsp.getResults().getNumFound());
120+
}
121+
assertTrue(stopped.get(300, TimeUnit.SECONDS)); // safety valve
122+
}
123+
124+
} finally {
125+
handlerGate.release(9999999); // safety valve
126+
handlerSignal.release(9999999); // safety valve
127+
cluster.shutdown();
128+
ExecutorUtil.shutdownAndAwaitTermination(exec);
129+
}
130+
}
131+
132+
/**
133+
* On every request, prior to handling, releases a ticket to it's signal Semaphre, and then blocks
134+
* until it can acquire a ticket from it's gate semaphore. Has a safety valve that gives up on the
135+
* gate after 300 seconds
136+
*/
137+
private static final class BlockingSearchHandler extends SearchHandler {
138+
private final Semaphore gate;
139+
private final Semaphore signal;
140+
141+
public BlockingSearchHandler(final Semaphore gate, final Semaphore signal) {
142+
super();
143+
this.gate = gate;
144+
this.signal = signal;
145+
super.init(new NamedList<>());
146+
}
147+
148+
@Override
149+
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
150+
log.info("Starting request");
151+
signal.release();
152+
if (gate.tryAcquire(300, TimeUnit.SECONDS)) {
153+
super.handleRequestBody(req, rsp);
154+
} else {
155+
log.error("Gate safety valve timeout");
156+
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Gate timeout");
157+
}
158+
log.info("Finishing request");
159+
}
160+
}
161+
}

‎solr/server/etc/jetty-graceful.xml‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
<?xml version="1.0"?>
2+
<!DOCTYPE Configure PUBLIC "-//Jetty//Configure//EN" "https://www.eclipse.org/jetty/configure_10_0.dtd">
3+
4+
<!-- =============================================================== -->
5+
<!-- Configure Jetty to finish existing requests on STOP. -->
6+
<!-- -->
7+
<!-- This is inspired by the "official" Jetty 12 graceful module, -->
8+
<!-- adapted to use StatisticsHandler for Jetty 10 and 11. -->
9+
<!-- -->
10+
<!-- IF THIS IS MODIFIED, JettySolrRunner MUST BE MODIFIED! -->
11+
<!-- =============================================================== -->
12+
13+
<Configure id="Server" class="org.eclipse.jetty.server.Server">
14+
<Call name="insertHandler">
15+
<Arg>
16+
<New id="GracefulHandler" class="org.eclipse.jetty.server.handler.StatisticsHandler">
17+
<Set name="gracefulShutdownWaitsForRequests">true</Set>
18+
</New>
19+
</Arg>
20+
</Call>
21+
<Set name="stopTimeout"><Property name="solr.jetty.stop.timeout" default="15000"/></Set>
22+
</Configure>

‎solr/server/modules/graceful.mod‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
[description]
2+
Enables Graceful processing of requests on STOP. Inspired by graceful.mod for Jetty-12, this works with Jetty 10 and 11.
3+
4+
[depend]
5+
server
6+
7+
[xml]
8+
etc/jetty-graceful.xml

‎solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -853,9 +853,11 @@ private Exception checkForExceptions(String message, Collection<Future<JettySolr
853853
try {
854854
future.get();
855855
} catch (ExecutionException e) {
856+
log.error(message, e);
856857
parsed.addSuppressed(e.getCause());
857858
ok = false;
858859
} catch (InterruptedException e) {
860+
log.error(message, e);
859861
Thread.interrupted();
860862
throw e;
861863
}

‎solr/test-framework/src/java/org/apache/solr/embedded/JettySolrRunner.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@
8787
import org.eclipse.jetty.server.ServerConnector;
8888
import org.eclipse.jetty.server.SslConnectionFactory;
8989
import org.eclipse.jetty.server.handler.HandlerWrapper;
90+
import org.eclipse.jetty.server.handler.StatisticsHandler;
9091
import org.eclipse.jetty.server.handler.gzip.GzipHandler;
9192
import org.eclipse.jetty.server.session.DefaultSessionIdManager;
9293
import org.eclipse.jetty.servlet.FilterHolder;
@@ -438,6 +439,12 @@ public void contextInitialized(ServletContextEvent event) {
438439
gzipHandler.setIncludedMethods("GET");
439440

440441
server.setHandler(gzipHandler);
442+
443+
// Mimic "graceful.mod"
444+
final StatisticsHandler graceful = new StatisticsHandler();
445+
graceful.setGracefulShutdownWaitsForRequests(true);
446+
server.insertHandler(graceful);
447+
server.setStopTimeout(15 * 1000);
441448
}
442449

443450
/**

0 commit comments

Comments
 (0)