Make it possible to turn off HTTP/2 support

We're running into problems with HTTP/2 and rebalancing

misk-testing/src/main/kotlin/misk/web/WebTestingModule.kt

@@ -12,22 +12,27 @@ import misk.security.ssl.SslLoader
  * both plaintext and TLS.
  */
 class WebTestingModule(
-  private val webConfig: WebConfig = WebConfig(
-      port = 0,
-      idle_timeout = 500000,
-      host = "127.0.0.1",
-      ssl = WebSslConfig(
-          port = 0,
-          cert_store = CertStoreConfig(
-              resource = "classpath:/ssl/server_cert_key_combo.pem",
-              passphrase = "serverpassword",
-              format = SslLoader.FORMAT_PEM
-          ),
-          mutual_auth = WebSslConfig.MutualAuth.NONE))
+  private val webConfig: WebConfig = TESTING_WEB_CONFIG
 ) : KAbstractModule() {
   override fun configure() {
     install(EnvironmentModule(Environment.TESTING))
     install(MiskTestingServiceModule())
     install(MiskWebModule(webConfig))
   }
+
+  companion object {
+    val TESTING_WEB_CONFIG = WebConfig(
+        port = 0,
+        idle_timeout = 500000,
+        host = "127.0.0.1",
+        ssl = WebSslConfig(
+            port = 0,
+            cert_store = CertStoreConfig(
+                resource = "classpath:/ssl/server_cert_key_combo.pem",
+                passphrase = "serverpassword",
+                format = SslLoader.FORMAT_PEM
+            ),
+            mutual_auth = WebSslConfig.MutualAuth.NONE)
+    )
+  }
 }

misk/src/main/kotlin/misk/web/WebConfig.kt

@@ -6,21 +6,44 @@ import misk.security.ssl.TrustStoreConfig
 import misk.web.exceptions.ActionExceptionLogLevelConfig
 
 data class WebConfig(
+  /** HTTP port to listen on, or 0 for any available port. */
   val port: Int,
+
+  /** If a connection is unused for this many milliseconds, it is closed. */
   val idle_timeout: Long,
+
+  /** The network interface to bind to. Null or 0.0.0.0 to bind to all interfaces. */
   val host: String? = null,
+
   val ssl: WebSslConfig? = null,
+
+  /** HTTP/2 support is currently opt-in because we can't load balance it dynamically. */
+  val http2: Boolean = true,
+
+  /** Number of NIO selector threads. */
   val selectors: Int? = null,
+
+  /** Number of acceptor threads. */
   val acceptors: Int? = null,
+
+  /** The accept backlog. */
   val queue_size: Int? = null,
+
+  /** Maximum number of threads in Jetty's thread pool. */
   val jetty_max_thread_pool_size: Int? = null,
+
   val action_exception_log_level: ActionExceptionLogLevelConfig = ActionExceptionLogLevelConfig(),
+
+  /** The maximum number of streams per HTTP/2 connection. */
   val jetty_max_concurrent_streams: Int? = null,
+
+  /** A value in [0.0..100.0]. Include 'Connection: close' in this percentage of responses. */
   // TODO(jayestrella): Add a sane default value to this.
   val close_connection_percent: Double = 0.0
 ) : Config
 
 data class WebSslConfig(
+  /** HTTPS port to listen on, or 0 for any available port. */
   val port: Int,
   val cert_store: CertStoreConfig,
   val trust_store: TrustStoreConfig? = null,

misk/src/main/kotlin/misk/web/jetty/JettyService.kt

@@ -11,6 +11,7 @@ import misk.web.WebSslConfig
 import okhttp3.HttpUrl
 import org.eclipse.jetty.alpn.server.ALPNServerConnectionFactory
 import org.eclipse.jetty.http2.server.HTTP2ServerConnectionFactory
+import org.eclipse.jetty.server.ConnectionFactory
 import org.eclipse.jetty.server.HttpConfiguration
 import org.eclipse.jetty.server.HttpConnectionFactory
 import org.eclipse.jetty.server.NetworkConnector
@@ -48,16 +49,26 @@ class JettyService @Inject internal constructor(
     val stopwatch = Stopwatch.createStarted()
     logger.info("Starting Jetty")
 
+    val httpConnectionFactories = mutableListOf<ConnectionFactory>()
+
     val httpConfig = HttpConfiguration()
     httpConfig.customizeForGrpc()
     httpConfig.sendServerVersion = false
     if (webConfig.ssl != null) {
       httpConfig.securePort = webConfig.ssl.port
     }
+    httpConnectionFactories += HttpConnectionFactory(httpConfig)
 
     // TODO(mmihic): Allow require running only on HTTPS?
-    val httpConnector = ServerConnector(server, null, null, null,
-        webConfig.acceptors ?: -1, webConfig.selectors ?: -1, HttpConnectionFactory(httpConfig))
+    val httpConnector = ServerConnector(
+        server,
+        null /* executor */,
+        null /* scheduler */,
+        null /* buffer pool */,
+        webConfig.acceptors ?: -1,
+        webConfig.selectors ?: -1,
+        httpConnectionFactories.toTypedArray()
+    )
     httpConnector.port = webConfig.port
     httpConnector.idleTimeout = webConfig.idle_timeout
     httpConnector.reuseAddress = true
@@ -87,6 +98,8 @@ class JettyService @Inject internal constructor(
         }
       }
 
+      val httpsConnectionFactories = mutableListOf<ConnectionFactory>()
+
       // By default, Jetty excludes a number of common cipher suites. This default set is too
       // restrictive. Clear the set of excluded suites and define the suites to include below.
       sslContextFactory.setExcludeCipherSuites()
@@ -96,16 +109,34 @@ class JettyService @Inject internal constructor(
       val httpsConfig = HttpConfiguration(httpConfig)
       httpsConfig.addCustomizer(SecureRequestCustomizer())
 
-      val alpn = ALPNServerConnectionFactory("h2", "http/1.1")
+      val ssl = SslConnectionFactory(sslContextFactory, "alpn")
+      httpsConnectionFactories += ssl
+
+      val alpnProtocols = if (webConfig.http2) listOf("h2", "http/1.1") else listOf("http/1.1")
+      val alpn = ALPNServerConnectionFactory(*alpnProtocols.toTypedArray())
       alpn.defaultProtocol = "http/1.1"
-      val ssl = SslConnectionFactory(sslContextFactory, alpn.protocol)
-      val http2 = HTTP2ServerConnectionFactory(httpsConfig)
-      if (webConfig.jetty_max_concurrent_streams != null) {
-        http2.maxConcurrentStreams = webConfig.jetty_max_concurrent_streams
+      httpsConnectionFactories += alpn
+
+      if (webConfig.http2) {
+        val http2 = HTTP2ServerConnectionFactory(httpsConfig)
+        if (webConfig.jetty_max_concurrent_streams != null) {
+          http2.maxConcurrentStreams = webConfig.jetty_max_concurrent_streams
+        }
+        httpsConnectionFactories += http2
       }
+
       val http1 = HttpConnectionFactory(httpsConfig)
-      val httpsConnector = ServerConnector(server, null, null, null,
-          webConfig.acceptors ?: -1, webConfig.selectors ?: -1, ssl, alpn, http2, http1)
+      httpsConnectionFactories += http1
+
+      val httpsConnector = ServerConnector(
+          server,
+          null /* executor */,
+          null /* scheduler */,
+          null /* buffer pool */,
+          webConfig.acceptors ?: -1,
+          webConfig.selectors ?: -1,
+          httpsConnectionFactories.toTypedArray()
+      )
       httpsConnector.port = webConfig.ssl.port
       httpsConnector.idleTimeout = webConfig.idle_timeout
       httpsConnector.reuseAddress = true
@@ -138,10 +169,11 @@ class JettyService @Inject internal constructor(
     server.start()
 
     logger.info {
-      if (webConfig.ssl != null)
+      if (webConfig.ssl != null) {
         "Started Jetty in $stopwatch on port ${webConfig.port}/${webConfig.ssl.port}"
-      else
+      } else {
         "Started Jetty in $stopwatch on port ${webConfig.port}"
+      }
     }
   }
 

misk/src/test/kotlin/misk/web/AbstractRebalancingTest.kt

@@ -1,9 +1,6 @@
 package misk.web
 
 import misk.inject.KAbstractModule
-import misk.security.ssl.CertStoreConfig
-import misk.security.ssl.SslLoader
-import misk.security.ssl.TrustStoreConfig
 import misk.testing.MiskTest
 import misk.testing.MiskTestModule
 import misk.web.jetty.JettyService
@@ -38,22 +35,8 @@ abstract class AbstractRebalancingTest(
 
   inner class TestModule : KAbstractModule() {
     override fun configure() {
-      install(WebTestingModule(webConfig = WebConfig(
-          port = 0,
-          idle_timeout = 500000,
-          host = "127.0.0.1",
-          close_connection_percent = percent,
-          ssl = WebSslConfig(0,
-              cert_store = CertStoreConfig(
-                  resource = "classpath:/ssl/server_cert_key_combo.pem",
-                  passphrase = "serverpassword",
-                  format = SslLoader.FORMAT_PEM
-              ),
-              trust_store = TrustStoreConfig(
-                  resource = "classpath:/ssl/client_cert.pem",
-                  format = SslLoader.FORMAT_PEM
-              ),
-              mutual_auth = WebSslConfig.MutualAuth.REQUIRED)
+      install(WebTestingModule(webConfig = WebTestingModule.TESTING_WEB_CONFIG.copy(
+          close_connection_percent = percent
       )))
     }
   }
@@ -71,4 +54,4 @@ class RebalancingDisabledTest : AbstractRebalancingTest(0.0) {
   override fun checkResponse(response: Response) {
     assertThat(response.header("Connection")).isNull()
   }
-}
+}

misk/src/test/kotlin/misk/web/ssl/Http2ConnectivityTest.kt

@@ -93,7 +93,9 @@ class Http2ConnectivityTest {
 
   class TestModule : KAbstractModule() {
     override fun configure() {
-      install(WebTestingModule())
+      install(WebTestingModule(webConfig = WebTestingModule.TESTING_WEB_CONFIG.copy(
+          http2 = true
+      )))
       install(WebActionModule.create<HelloAction>())
     }
   }

misk/src/test/kotlin/misk/web/ssl/JceksSslClientServerTest.kt

@@ -25,7 +25,6 @@ import misk.web.RequestBody
 import misk.web.RequestContentType
 import misk.web.ResponseContentType
 import misk.web.WebActionModule
-import misk.web.WebConfig
 import misk.web.WebSslConfig
 import misk.web.WebTestingModule
 import misk.web.actions.WebAction
@@ -95,10 +94,7 @@ internal class JceksSslClientServerTest {
   class TestModule : KAbstractModule() {
     override fun configure() {
       install(WebActionModule.create<HelloAction>())
-      install(WebTestingModule(WebConfig(
-          port = 0,
-          idle_timeout = 500000,
-          host = "127.0.0.1",
+      install(WebTestingModule(WebTestingModule.TESTING_WEB_CONFIG.copy(
           ssl = WebSslConfig(0,
               cert_store = CertStoreConfig(
                   resource = "classpath:/ssl/server_keystore.jceks",

misk/src/test/kotlin/misk/web/ssl/PemSslClientServerTest.kt

@@ -25,7 +25,6 @@ import misk.web.RequestBody
 import misk.web.RequestContentType
 import misk.web.ResponseContentType
 import misk.web.WebActionModule
-import misk.web.WebConfig
 import misk.web.WebSslConfig
 import misk.web.WebTestingModule
 import misk.web.actions.WebAction
@@ -100,10 +99,7 @@ internal class PemSslClientServerTest {
 
   class TestModule : KAbstractModule() {
     override fun configure() {
-      install(WebTestingModule(WebConfig(
-          port = 0,
-          idle_timeout = 500000,
-          host = "127.0.0.1",
+      install(WebTestingModule(WebTestingModule.TESTING_WEB_CONFIG.copy(
           ssl = WebSslConfig(0,
               cert_store = CertStoreConfig(
                   resource = "classpath:/ssl/server_cert_key_combo.pem",