From 44855ba60e616ebd35bef9f4b7a5ca9013bd5c7e Mon Sep 17 00:00:00 2001 From: Ethan Li Date: Tue, 24 Oct 2017 17:12:56 -0500 Subject: [PATCH] [STORM-1927] upgrade to jetty 9 --- flux/flux-examples/pom.xml | 4 + pom.xml | 10 +- storm-client/pom.xml | 6 +- storm-core/pom.xml | 6 +- .../src/clj/org/apache/storm/ui/core.clj | 6 +- .../src/clj/org/apache/storm/ui/helpers.clj | 8 -- .../jvm/org/apache/storm/ui/UIHelpers.java | 104 +++++++++++------- .../apache/storm/daemon/nimbus/Nimbus.java | 2 +- .../apache/storm/daemon/drpc/DRPCServer.java | 10 +- 9 files changed, 90 insertions(+), 66 deletions(-) diff --git a/flux/flux-examples/pom.xml b/flux/flux-examples/pom.xml index 72042705712..3a15f2e0108 100644 --- a/flux/flux-examples/pom.xml +++ b/flux/flux-examples/pom.xml @@ -86,6 +86,10 @@ javax.servlet servlet-api + + javax.servlet + javax.servlet-api + diff --git a/pom.xml b/pom.xml index 8b84873fc54..0b740c75425 100644 --- a/pom.xml +++ b/pom.xml @@ -250,9 +250,9 @@ 4.0.0 2.12.0 1.1 - 1.3.1 - 0.3.1 - 7.6.13.v20130916 + 1.6.2 + 0.4.0 + 9.4.7.v20170914 0.2.3 0.0.1 1.5.0 @@ -281,7 +281,7 @@ ${hadoop.version} 1.1.0 3.0.3 - 2.5 + 3.1.0 2.3 0.9.3 4.11 @@ -726,7 +726,7 @@ javax.servlet - servlet-api + javax.servlet-api ${servlet.version} diff --git a/storm-client/pom.xml b/storm-client/pom.xml index b37f458797c..851206b6d0f 100644 --- a/storm-client/pom.xml +++ b/storm-client/pom.xml @@ -87,6 +87,10 @@ javax.servlet servlet-api + + javax.servlet + javax.servlet-api + @@ -158,7 +162,7 @@ javax.servlet - servlet-api + javax.servlet-api diff --git a/storm-core/pom.xml b/storm-core/pom.xml index 7f5b6f2c3dd..70abfbced08 100644 --- a/storm-core/pom.xml +++ b/storm-core/pom.xml @@ -186,6 +186,10 @@ javax.servlet servlet-api + + javax.servlet + javax.servlet-api + @@ -276,7 +280,7 @@ javax.servlet - servlet-api + javax.servlet-api org.slf4j diff --git a/storm-core/src/clj/org/apache/storm/ui/core.clj b/storm-core/src/clj/org/apache/storm/ui/core.clj index 574c9341086..247ff4a48cc 100644 --- a/storm-core/src/clj/org/apache/storm/ui/core.clj +++ b/storm-core/src/clj/org/apache/storm/ui/core.clj @@ -1616,6 +1616,7 @@ (UIHelpers/stormRunJetty (int (conf UI-PORT)) (conf UI-HOST) https-port + header-buffer-size (reify IConfigurator (execute [this server] (UIHelpers/configSsl server @@ -1628,9 +1629,8 @@ https-ts-password https-ts-type https-need-client-auth - https-want-client-auth) - (doseq [connector (.getConnectors server)] - (.setRequestHeaderSize connector header-buffer-size)) + https-want-client-auth + header-buffer-size) (UIHelpers/configFilter server (ring.util.servlet/servlet app) filters-confs))))) (catch Exception ex (log-error ex)))) diff --git a/storm-core/src/clj/org/apache/storm/ui/helpers.clj b/storm-core/src/clj/org/apache/storm/ui/helpers.clj index ac1ecd1b41f..5764529c3c4 100644 --- a/storm-core/src/clj/org/apache/storm/ui/helpers.clj +++ b/storm-core/src/clj/org/apache/storm/ui/helpers.clj @@ -28,14 +28,6 @@ (:import [org.apache.storm.logging.filters AccessLoggingFilter]) (:import [java.util EnumSet] [java.net URLEncoder]) - (:import [org.eclipse.jetty.server Server] - [org.eclipse.jetty.server.nio SelectChannelConnector] - [org.eclipse.jetty.server.ssl SslSocketConnector] - [org.eclipse.jetty.servlet ServletHolder FilterMapping] - [org.eclipse.jetty.util.ssl SslContextFactory] - [org.eclipse.jetty.server DispatcherType] - [org.eclipse.jetty.servlets CrossOriginFilter] - (org.json.simple JSONValue)) (:require [ring.util servlet] [ring.util.response :as response]) (:require [compojure.route :as route] diff --git a/storm-core/src/jvm/org/apache/storm/ui/UIHelpers.java b/storm-core/src/jvm/org/apache/storm/ui/UIHelpers.java index d43873fe008..75559984dbf 100644 --- a/storm-core/src/jvm/org/apache/storm/ui/UIHelpers.java +++ b/storm-core/src/jvm/org/apache/storm/ui/UIHelpers.java @@ -19,29 +19,33 @@ import com.google.common.base.Joiner; import com.google.common.collect.ImmutableMap; - +import java.io.PrintWriter; +import java.io.StringWriter; +import java.net.URLEncoder; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import javax.servlet.DispatcherType; +import javax.servlet.Servlet; import org.apache.storm.generated.ExecutorInfo; import org.apache.storm.logging.filters.AccessLoggingFilter; import org.apache.storm.utils.ObjectReader; -import org.eclipse.jetty.server.Connector; -import org.eclipse.jetty.server.DispatcherType; +import org.eclipse.jetty.http.HttpVersion; +import org.eclipse.jetty.server.HttpConfiguration; +import org.eclipse.jetty.server.HttpConnectionFactory; +import org.eclipse.jetty.server.SecureRequestCustomizer; import org.eclipse.jetty.server.Server; -import org.eclipse.jetty.server.nio.SelectChannelConnector; -import org.eclipse.jetty.server.ssl.SslSocketConnector; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.SslConnectionFactory; import org.eclipse.jetty.servlet.FilterHolder; -import org.eclipse.jetty.servlet.FilterMapping; import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; import org.eclipse.jetty.servlets.CrossOriginFilter; import org.eclipse.jetty.util.ssl.SslContextFactory; import org.json.simple.JSONValue; -import javax.servlet.Servlet; -import java.io.PrintWriter; -import java.io.StringWriter; -import java.net.URLEncoder; -import java.util.*; - public class UIHelpers { private static final Object[][] PRETTY_SEC_DIVIDERS = { @@ -110,20 +114,20 @@ public static Map unauthorizedUserJson(String user) { "errorMessage", String.format("User %s is not authorized.", user)); } - private static SslSocketConnector mkSslConnector(Integer port, String ksPath, String ksPassword, String ksType, + private static ServerConnector mkSslConnector(Server server, Integer port, String ksPath, String ksPassword, String ksType, String keyPassword, String tsPath, String tsPassword, String tsType, - Boolean needClientAuth, Boolean wantClientAuth) { + Boolean needClientAuth, Boolean wantClientAuth, Integer headerBufferSize) { SslContextFactory factory = new SslContextFactory(); factory.setExcludeCipherSuites("SSL_RSA_WITH_RC4_128_MD5", "SSL_RSA_WITH_RC4_128_SHA"); factory.setExcludeProtocols("SSLv3"); - factory.setAllowRenegotiate(false); + factory.setRenegotiationAllowed(false); factory.setKeyStorePath(ksPath); factory.setKeyStoreType(ksType); factory.setKeyStorePassword(ksPassword); factory.setKeyManagerPassword(keyPassword); if (tsPath != null && tsPassword != null && tsType != null) { - factory.setTrustStore(tsPath); + factory.setTrustStorePath(tsPath); factory.setTrustStoreType(tsType); factory.setTrustStorePassword(tsPassword); } @@ -134,16 +138,31 @@ private static SslSocketConnector mkSslConnector(Integer port, String ksPath, St factory.setWantClientAuth(true); } - SslSocketConnector sslConnector = new SslSocketConnector(factory); + HttpConfiguration httpsConfig = new HttpConfiguration(); + httpsConfig.addCustomizer(new SecureRequestCustomizer()); + if (null != headerBufferSize) { + httpsConfig.setRequestHeaderSize(headerBufferSize); + } + ServerConnector sslConnector = new ServerConnector(server, + new SslConnectionFactory(factory, HttpVersion.HTTP_1_1.asString()), + new HttpConnectionFactory(httpsConfig)); sslConnector.setPort(port); return sslConnector; } public static void configSsl(Server server, Integer port, String ksPath, String ksPassword, String ksType, - String keyPassword, String tsPath, String tsPassword, String tsType, Boolean needClientAuth, Boolean wantClientAuth) { + String keyPassword, String tsPath, String tsPassword, String tsType, + Boolean needClientAuth, Boolean wantClientAuth) { + configSsl(server, port, ksPath, ksPassword, ksType, keyPassword, + tsPath, tsPassword, tsType, needClientAuth, wantClientAuth, null); + } + + public static void configSsl(Server server, Integer port, String ksPath, String ksPassword, String ksType, + String keyPassword, String tsPath, String tsPassword, String tsType, + Boolean needClientAuth, Boolean wantClientAuth, Integer headerBufferSize) { if (port > 0) { - server.addConnector(mkSslConnector(port, ksPath, ksPassword, ksType, keyPassword, - tsPath, tsPassword, tsType, needClientAuth, wantClientAuth)); + server.addConnector(mkSslConnector(server, port, ksPath, ksPassword, ksType, keyPassword, + tsPath, tsPassword, tsType, needClientAuth, wantClientAuth, headerBufferSize)); } } @@ -197,37 +216,38 @@ public static void configFilters(ServletContextHandler context, List()); } - context.addFilter(filterHolder, "/*", FilterMapping.ALL); + context.addFilter(filterHolder, "/*", EnumSet.allOf(DispatcherType.class)); } } context.addFilter(mkAccessLoggingFilterHandle(), "/*", EnumSet.allOf(DispatcherType.class)); } - - private static Server removeNonSslConnector(Server server) { - for (Connector c : server.getConnectors()) { - if (c != null && !(c instanceof SslSocketConnector)) { - server.removeConnector(c); - } - } - return server; - } /** * Construct a Jetty Server instance. */ public static Server jettyCreateServer(Integer port, String host, Integer httpsPort) { - SelectChannelConnector connector = new SelectChannelConnector(); - connector.setPort(ObjectReader.getInt(port, 80)); - connector.setHost(host); - connector.setMaxIdleTime(200000); + return jettyCreateServer(port, host, httpsPort, null); + } + /** + * Construct a Jetty Server instance. + */ + public static Server jettyCreateServer(Integer port, String host, Integer httpsPort, Integer headerBufferSize) { Server server = new Server(); - server.addConnector(connector); - server.setSendDateHeader(true); - if (httpsPort != null && httpsPort > 0) { - removeNonSslConnector(server); + if (httpsPort == null || httpsPort <= 0) { + HttpConfiguration httpConfig = new HttpConfiguration(); + httpConfig.setSendDateHeader(true); + if (null != headerBufferSize) { + httpConfig.setRequestHeaderSize(headerBufferSize); + } + ServerConnector httpConnector = new ServerConnector(server, new HttpConnectionFactory(httpConfig)); + httpConnector.setPort(ObjectReader.getInt(port, 80)); + httpConnector.setIdleTimeout(200000); + httpConnector.setHost(host); + server.addConnector(httpConnector); } + return server; } @@ -235,16 +255,16 @@ public static Server jettyCreateServer(Integer port, String host, Integer httpsP * Modified version of run-jetty * Assumes configurator sets handler. */ - public static void stormRunJetty(Integer port, String host, Integer httpsPort, IConfigurator configurator) throws Exception { - Server s = jettyCreateServer(port, host, httpsPort); + public static void stormRunJetty(Integer port, String host, Integer httpsPort, Integer headerBufferSize, IConfigurator configurator) throws Exception { + Server s = jettyCreateServer(port, host, httpsPort, headerBufferSize); if (configurator != null) { configurator.execute(s); } s.start(); } - public static void stormRunJetty(Integer port, IConfigurator configurator) throws Exception { - stormRunJetty(port, null, null, configurator); + public static void stormRunJetty(Integer port, Integer headerBufferSize, IConfigurator configurator) throws Exception { + stormRunJetty(port, null, null, headerBufferSize, configurator); } public static String wrapJsonInCallback(String callback, String response) { diff --git a/storm-server/src/main/java/org/apache/storm/daemon/nimbus/Nimbus.java b/storm-server/src/main/java/org/apache/storm/daemon/nimbus/Nimbus.java index 3b914baa810..73ceb42ef01 100644 --- a/storm-server/src/main/java/org/apache/storm/daemon/nimbus/Nimbus.java +++ b/storm-server/src/main/java/org/apache/storm/daemon/nimbus/Nimbus.java @@ -185,7 +185,7 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon { private static final Logger LOG = LoggerFactory.getLogger(Nimbus.class); - + // Metrics private static final Meter submitTopologyWithOptsCalls = StormMetricsRegistry.registerMeter("nimbus:num-submitTopologyWithOpts-calls"); private static final Meter submitTopologyCalls = StormMetricsRegistry.registerMeter("nimbus:num-submitTopology-calls"); diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/DRPCServer.java b/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/DRPCServer.java index e3ef31cbee7..2dc5434b683 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/DRPCServer.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/DRPCServer.java @@ -20,11 +20,11 @@ import com.codahale.metrics.Meter; import com.google.common.annotations.VisibleForTesting; - import java.util.Arrays; +import java.util.EnumSet; import java.util.List; import java.util.Map; - +import javax.servlet.DispatcherType; import org.apache.storm.Config; import org.apache.storm.DaemonConfig; import org.apache.storm.daemon.drpc.webapp.DRPCApplication; @@ -41,8 +41,8 @@ import org.apache.storm.utils.ObjectReader; import org.apache.storm.utils.Utils; import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.servlet.FilterHolder; -import org.eclipse.jetty.servlet.FilterMapping; import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; import org.glassfish.jersey.servlet.ServletContainer; @@ -64,7 +64,7 @@ public class DRPCServer implements AutoCloseable { public static void addRequestContextFilter(ServletContextHandler context, String configName, Map conf) { IHttpCredentialsPlugin auth = AuthUtils.GetHttpCredentialsPlugin(conf, (String)conf.get(configName)); ReqContextFilter filter = new ReqContextFilter(auth); - context.addFilter(new FilterHolder(filter), "/*", FilterMapping.ALL); + context.addFilter(new FilterHolder(filter), "/*", EnumSet.allOf(DispatcherType.class)); } private static ThriftServer mkHandlerServer(final DistributedRPC.Iface service, Integer port, Map conf) { @@ -210,7 +210,7 @@ public int getDrpcInvokePort() { public int getHttpServerPort() { assert httpServer.getConnectors().length == 1; - return httpServer.getConnectors()[0].getLocalPort(); + return ((ServerConnector) (httpServer.getConnectors()[0])).getLocalPort(); } /**