Upgrade Jetty to 9.4.0.v20161208 (#1875)

* Upgrade Jetty to 9.4.0.v20161208

Hopefully, it's just a version bump.

Resolves #1874.

* Increase waiting interval for batch HTTP/2 test

* Delegate calculation amount of acceptors and selectors threads to Jetty

The issue is reported in #523. The OS on CI Linux machines returns an
astonishingly big amount of CPUs (~ 128). Jetty changed their algorithm
for calculating the maximum amount of selector and acceptor threads,
see `eclipse/jetty.project@2d52280`. But, in Dropwizard we still set
the amount of acceptors as #CPUs/2 and selectors #CPUs. Looks like
that's too much for Jetty, and it can't handle such a big amount of threads.
Because of that we have random errors on our CI environment and,
what worse, possibly hurt users who actually run their applications on
machines with a big amount of CPU. A solution is to delegate calculating
the amount to Jetty (which has more sane defaults) and document the
defaults.

docs/source/manual/configuration.rst

@@ -341,8 +341,10 @@ blockingTimeout          (none)              The timeout applied to blocking ope
 minBufferPoolSize        64 bytes            The minimum size of the buffer pool.
 bufferPoolIncrement      1KiB                The increment by which the buffer pool should be increased.
 maxBufferPoolSize        64KiB               The maximum size of the buffer pool.
-acceptorThreads          # of CPUs/2         The number of worker threads dedicated to accepting connections.
-selectorThreads          # of CPUs           The number of worker threads dedicated to sending and receiving data.
+acceptorThreads          (Jetty's default)   The number of worker threads dedicated to accepting connections.
+                                             By default is *max*(1, *min*(4, #CPUs/8)).
+selectorThreads          (Jetty's default)   The number of worker threads dedicated to sending and receiving data.
+                                             By default is *max*(1, *min*(4, #CPUs/2)).
 acceptQueueSize          (OS default)        The size of the TCP/IP accept queue for the listening socket.
 reuseAddress             true                Whether or not ``SO_REUSEADDR`` is enabled on the listening socket.
 soLingerTime             (disabled)          Enable/disable ``SO_LINGER`` with the specified linger time.

dropwizard-bom/pom.xml

@@ -25,7 +25,7 @@
         <guava.version>21.0</guava.version>
         <jersey.version>2.25</jersey.version>
         <jackson.version>2.8.6</jackson.version>
-        <jetty.version>9.3.15.v20161220</jetty.version>
+        <jetty.version>9.4.0.v20161208</jetty.version>
         <servlet.version>3.0.0.v201112011016</servlet.version>
         <metrics3.version>3.1.2</metrics3.version>
         <slf4j.version>1.7.22</slf4j.version>

dropwizard-http2/src/test/java/io/dropwizard/http2/AbstractHttp2Test.java

@@ -44,6 +44,6 @@ public void onComplete(Result result) {
                     });
         }
 
-        assertThat(latch.await(5, TimeUnit.SECONDS)).isTrue();
+        assertThat(latch.await(30, TimeUnit.SECONDS)).isTrue();
     }
 }

dropwizard-jetty/src/main/java/io/dropwizard/jetty/HttpConnectorFactory.java

@@ -25,6 +25,7 @@
 
 import javax.validation.constraints.Min;
 import javax.validation.constraints.NotNull;
+import java.util.Optional;
 import java.util.concurrent.TimeUnit;
 
 import static com.codahale.metrics.MetricRegistry.name;
@@ -141,13 +142,15 @@
  *     </tr>
  *     <tr>
  *         <td>{@code acceptorThreads}</td>
- *         <td>half the # of CPUs</td>
- *         <td>The number of worker threads dedicated to accepting connections.</td>
+ *         <td>(Jetty's default)</td>
+ *         <td>The number of worker threads dedicated to accepting connections.
+ *         By default is <i>max</i>(1, <i>min</i>(4, #CPUs/8)).</td>
  *     </tr>
  *     <tr>
  *         <td>{@code selectorThreads}</td>
- *         <td>the # of CPUs</td>
- *         <td>The number of worker threads dedicated to sending and receiving data.</td>
+ *         <td>(Jetty's default)</td>
+ *         <td>The number of worker threads dedicated to sending and receiving data.
+ *         By default is <i>max</i>(1, <i>min</i>(4, #CPUs/2)).</td>
  *     </tr>
  *     <tr>
  *         <td>{@code acceptQueueSize}</td>
@@ -259,10 +262,10 @@ public static ConnectorFactory admin() {
     private Size maxBufferPoolSize = Size.kilobytes(64);
 
     @Min(1)
-    private int acceptorThreads = Math.max(1, Runtime.getRuntime().availableProcessors() / 2);
+    private Optional<Integer> acceptorThreads = Optional.empty();
 
     @Min(1)
-    private int selectorThreads = Runtime.getRuntime().availableProcessors();
+    private Optional<Integer> selectorThreads = Optional.empty();
 
     @Min(0)
     private Integer acceptQueueSize;
@@ -405,22 +408,22 @@ public void setMaxBufferPoolSize(Size maxBufferPoolSize) {
     }
 
     @JsonProperty
-    public int getAcceptorThreads() {
+    public Optional<Integer> getAcceptorThreads() {
         return acceptorThreads;
     }
 
     @JsonProperty
-    public void setAcceptorThreads(int acceptorThreads) {
+    public void setAcceptorThreads(Optional<Integer> acceptorThreads) {
         this.acceptorThreads = acceptorThreads;
     }
 
     @JsonProperty
-    public int getSelectorThreads() {
+    public Optional<Integer> getSelectorThreads() {
         return selectorThreads;
     }
 
     @JsonProperty
-    public void setSelectorThreads(int selectorThreads) {
+    public void setSelectorThreads(Optional<Integer> selectorThreads) {
         this.selectorThreads = selectorThreads;
     }
 
@@ -530,8 +533,8 @@ protected ServerConnector buildConnector(Server server,
                                                               threadPool,
                                                               scheduler,
                                                               bufferPool,
-                                                              acceptorThreads,
-                                                              selectorThreads,
+                                                              acceptorThreads.orElse(-1),
+                                                              selectorThreads.orElse(-1),
                                                               factories);
         connector.setPort(port);
         connector.setHost(bindHost);

dropwizard-jetty/src/test/java/io/dropwizard/jetty/HttpConnectorFactoryTest.java

@@ -28,6 +28,7 @@
 
 import javax.validation.Validator;
 import java.io.File;
+import java.util.Optional;
 
 import static org.apache.commons.lang3.reflect.FieldUtils.getField;
 import static org.assertj.core.api.Assertions.assertThat;
@@ -67,8 +68,8 @@ public void testParseMinimalConfiguration() throws Exception {
         assertThat(http.getMinBufferPoolSize()).isEqualTo(Size.bytes(64));
         assertThat(http.getBufferPoolIncrement()).isEqualTo(Size.bytes(1024));
         assertThat(http.getMaxBufferPoolSize()).isEqualTo(Size.kilobytes(64));
-        assertThat(http.getAcceptorThreads()).isEqualTo(Math.max(1, Runtime.getRuntime().availableProcessors() / 2));
-        assertThat(http.getSelectorThreads()).isEqualTo(Runtime.getRuntime().availableProcessors());
+        assertThat(http.getAcceptorThreads()).isEmpty();
+        assertThat(http.getSelectorThreads()).isEmpty();
         assertThat(http.getAcceptQueueSize()).isNull();
         assertThat(http.isReuseAddress()).isTrue();
         assertThat(http.getSoLingerTime()).isNull();
@@ -97,8 +98,8 @@ public void testParseFullConfiguration() throws Exception {
         assertThat(http.getMinBufferPoolSize()).isEqualTo(Size.bytes(128));
         assertThat(http.getBufferPoolIncrement()).isEqualTo(Size.bytes(500));
         assertThat(http.getMaxBufferPoolSize()).isEqualTo(Size.kilobytes(32));
-        assertThat(http.getAcceptorThreads()).isEqualTo(1);
-        assertThat(http.getSelectorThreads()).isEqualTo(4);
+        assertThat(http.getAcceptorThreads()).contains(1);
+        assertThat(http.getSelectorThreads()).contains(4);
         assertThat(http.getAcceptQueueSize()).isEqualTo(1024);
         assertThat(http.isReuseAddress()).isFalse();
         assertThat(http.getSoLingerTime()).isEqualTo(Duration.seconds(30));
@@ -113,8 +114,8 @@ public void testParseFullConfiguration() throws Exception {
     public void testBuildConnector() throws Exception {
         HttpConnectorFactory http = new HttpConnectorFactory();
         http.setBindHost("127.0.0.1");
-        http.setAcceptorThreads(1);
-        http.setSelectorThreads(2);
+        http.setAcceptorThreads(Optional.of(1));
+        http.setSelectorThreads(Optional.of(2));
         http.setAcceptQueueSize(1024);
         http.setSoLingerTime(Duration.seconds(30));
         http.setBlockingTimeout(Duration.minutes(1));
@@ -175,8 +176,8 @@ public void testBuildConnector() throws Exception {
     public void testDefaultAcceptQueueSize() throws Exception {
         HttpConnectorFactory http = new HttpConnectorFactory();
         http.setBindHost("127.0.0.1");
-        http.setAcceptorThreads(1);
-        http.setSelectorThreads(2);
+        http.setAcceptorThreads(Optional.of(1));
+        http.setSelectorThreads(Optional.of(2));
         http.setSoLingerTime(Duration.seconds(30));
 
         Server server = new Server();