diff --git a/engine/pom.xml b/engine/pom.xml index 5e7a5f26d6..b2d8a10d5c 100644 --- a/engine/pom.xml +++ b/engine/pom.xml @@ -289,6 +289,16 @@ commons-codec 1.10 + + commons-configuration + commons-configuration + 1.10 + + + org.mortbay.jetty + jetty + 6.1.2 + org.eclipse.jetty jetty-servlet diff --git a/engine/src/main/java/com/datatorrent/stram/client/StramAppLauncher.java b/engine/src/main/java/com/datatorrent/stram/client/StramAppLauncher.java index d3079d0fd6..41ba522ce1 100644 --- a/engine/src/main/java/com/datatorrent/stram/client/StramAppLauncher.java +++ b/engine/src/main/java/com/datatorrent/stram/client/StramAppLauncher.java @@ -14,8 +14,7 @@ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations - * under the License. - */ + * under the License.*/ package com.datatorrent.stram.client; import java.io.File; @@ -606,16 +605,16 @@ public ApplicationId launchApp(AppFactory appConfig) throws Exception conf.setEnum(StreamingApplication.ENVIRONMENT, StreamingApplication.Environment.CLUSTER); LogicalPlan dag = appConfig.createApp(propertiesBuilder); if (UserGroupInformation.isSecurityEnabled()) { - long hdfsTokenMaxLifeTime = conf.getLong(StramClientUtils.DT_HDFS_TOKEN_MAX_LIFE_TIME, conf.getLong(StramClientUtils.HDFS_TOKEN_MAX_LIFE_TIME, StramClientUtils.DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT)); + long hdfsTokenMaxLifeTime = conf.getLong(StramClientUtils.DT_HDFS_TOKEN_MAX_LIFE_TIME, conf.getLong(StramClientUtils.HDFS_TOKEN_MAX_LIFE_TIME, StramClientUtils.RM_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT)); dag.setAttribute(LogicalPlan.HDFS_TOKEN_LIFE_TIME, hdfsTokenMaxLifeTime); LOG.debug("HDFS token life time {}", hdfsTokenMaxLifeTime); - long hdfsTokenRenewInterval = conf.getLong(StramClientUtils.DT_HDFS_TOKEN_RENEW_INTERVAL, conf.getLong(StramClientUtils.HDFS_TOKEN_RENEW_INTERVAL, StramClientUtils.DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT)); + long hdfsTokenRenewInterval = conf.getLong(StramClientUtils.DT_HDFS_TOKEN_RENEW_INTERVAL, conf.getLong(StramClientUtils.HDFS_TOKEN_RENEW_INTERVAL, StramClientUtils.RM_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT)); dag.setAttribute(LogicalPlan.HDFS_TOKEN_RENEWAL_INTERVAL, hdfsTokenRenewInterval); LOG.debug("HDFS token renew interval {}", hdfsTokenRenewInterval); - long rmTokenMaxLifeTime = conf.getLong(StramClientUtils.DT_RM_TOKEN_MAX_LIFE_TIME, conf.getLong(YarnConfiguration.DELEGATION_TOKEN_MAX_LIFETIME_KEY, YarnConfiguration.DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT)); + long rmTokenMaxLifeTime = conf.getLong(StramClientUtils.DT_RM_TOKEN_MAX_LIFE_TIME, conf.getLong(YarnConfiguration.RM_DELEGATION_TOKEN_MAX_LIFETIME_KEY, YarnConfiguration.RM_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT)); dag.setAttribute(LogicalPlan.RM_TOKEN_LIFE_TIME, rmTokenMaxLifeTime); LOG.debug("RM token life time {}", rmTokenMaxLifeTime); - long rmTokenRenewInterval = conf.getLong(StramClientUtils.DT_RM_TOKEN_RENEW_INTERVAL, conf.getLong(YarnConfiguration.DELEGATION_TOKEN_RENEW_INTERVAL_KEY, YarnConfiguration.DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT)); + long rmTokenRenewInterval = conf.getLong(StramClientUtils.DT_RM_TOKEN_RENEW_INTERVAL, conf.getLong(YarnConfiguration.RM_DELEGATION_TOKEN_RENEW_INTERVAL_KEY, YarnConfiguration.RM_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT)); dag.setAttribute(LogicalPlan.RM_TOKEN_RENEWAL_INTERVAL, rmTokenRenewInterval); LOG.debug("RM token renew interval {}", rmTokenRenewInterval); setTokenRefreshCredentials(dag, conf); diff --git a/engine/src/main/java/com/datatorrent/stram/client/StramClientUtils.java b/engine/src/main/java/com/datatorrent/stram/client/StramClientUtils.java index d4f190fb64..54eabe793f 100644 --- a/engine/src/main/java/com/datatorrent/stram/client/StramClientUtils.java +++ b/engine/src/main/java/com/datatorrent/stram/client/StramClientUtils.java @@ -119,8 +119,8 @@ public class StramClientUtils @Deprecated public static final String KEY_TAB_FILE = StramUserLogin.DT_AUTH_PREFIX + "store.keytab"; public static final String TOKEN_ANTICIPATORY_REFRESH_FACTOR = StramUserLogin.DT_AUTH_PREFIX + "token.refresh.factor"; - public static final long DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT = 7 * 24 * 60 * 60 * 1000; - public static final long DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT = 24 * 60 * 60 * 1000; + public static final long RM_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT = 7 * 24 * 60 * 60 * 1000; + public static final long RM_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT = 24 * 60 * 60 * 1000; public static final String TOKEN_REFRESH_PRINCIPAL = StramUserLogin.DT_AUTH_PREFIX + "token.refresh.principal"; public static final String TOKEN_REFRESH_KEYTAB = StramUserLogin.DT_AUTH_PREFIX + "token.refresh.keytab"; /** @@ -828,7 +828,12 @@ private static YarnConfiguration getYarnConfiguration(Configuration conf) public static InetSocketAddress getRMWebAddress(Configuration conf, String rmId) { - boolean sslEnabled = conf.getBoolean(CommonConfigurationKeysPublic.HADOOP_SSL_ENABLED_KEY, CommonConfigurationKeysPublic.HADOOP_SSL_ENABLED_DEFAULT); + /* + * This is far from confirmed as the "correct" fix. The previous constants HADOOP_SSL_ENABLED_DEFAULT and HADOOP_SSL_ENABLED_KEY have + * been deprecated as far back as I can find Hadoop versions (2.4.1 at least), HADOOP_CALLER_CONTEXT_ENABLED_DEFAULT and + * HADOOP_CALLER_CONTEXT_ENABLED_KEY are the only two constants in 3.1.0 that seem even remotely related... + * */ + boolean sslEnabled = conf.getBoolean(CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_ENABLED_KEY, CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_ENABLED_DEFAULT); return getRMWebAddress(conf, sslEnabled, rmId); } diff --git a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java index 74510a66a7..cdf9df7045 100644 --- a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java +++ b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java @@ -163,8 +163,8 @@ public class LogicalPlan implements Serializable, DAG public static Attribute FAST_PUBLISHER_SUBSCRIBER = new Attribute<>(false); public static Attribute HDFS_TOKEN_RENEWAL_INTERVAL = new Attribute<>(86400000L); public static Attribute HDFS_TOKEN_LIFE_TIME = new Attribute<>(604800000L); - public static Attribute RM_TOKEN_RENEWAL_INTERVAL = new Attribute<>(YarnConfiguration.DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT); - public static Attribute RM_TOKEN_LIFE_TIME = new Attribute<>(YarnConfiguration.DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT); + public static Attribute RM_TOKEN_RENEWAL_INTERVAL = new Attribute<>(YarnConfiguration.RM_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT); + public static Attribute RM_TOKEN_LIFE_TIME = new Attribute<>(YarnConfiguration.RM_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT); public static Attribute PRINCIPAL = new Attribute<>(null, StringCodec.String2String.getInstance()); public static Attribute KEY_TAB_FILE = new Attribute<>((String)null, StringCodec.String2String.getInstance()); public static Attribute TOKEN_REFRESH_ANTICIPATORY_FACTOR = new Attribute<>(0.7); diff --git a/engine/src/main/java/com/datatorrent/stram/util/FSPartFileCollection.java b/engine/src/main/java/com/datatorrent/stram/util/FSPartFileCollection.java index 357716a720..cdffdec040 100644 --- a/engine/src/main/java/com/datatorrent/stram/util/FSPartFileCollection.java +++ b/engine/src/main/java/com/datatorrent/stram/util/FSPartFileCollection.java @@ -172,7 +172,11 @@ public boolean isReadyTurnoverPartFile() try { return (syncRequested || (partOutStr.getPos() > bytesPerPartFile) || (currentPartFileTimeStamp + millisPerPartFile < System.currentTimeMillis())) && partOutStr.getPos() > 0; - } catch (IOException ex) { + /* Not really sure this is a legitimate fix...getting the following error when compiling: + * exception java.io.IOException is never thrown in body of corresponding try statement + * The remedy for this appears to be to use a more specific exception as the current try/catch block could + * not spit out IOException. For now, just using Exception to try and get this to build, will revisit later. */ + } catch (Exception ex) { return true; } } diff --git a/engine/src/test/java/com/datatorrent/stram/client/StramClientUtilsTest.java b/engine/src/test/java/com/datatorrent/stram/client/StramClientUtilsTest.java index ed61efb03a..d0b7387ef8 100644 --- a/engine/src/test/java/com/datatorrent/stram/client/StramClientUtilsTest.java +++ b/engine/src/test/java/com/datatorrent/stram/client/StramClientUtilsTest.java @@ -124,7 +124,13 @@ public InetSocketAddress getSocketAddr(String name, String defaultAddress, int d }; // basic test - conf.setBoolean(CommonConfigurationKeysPublic.HADOOP_SSL_ENABLED_KEY, false); + + /* + * This is far from confirmed as the "correct" fix. The previous constants HADOOP_SSL_ENABLED_DEFAULT and HADOOP_SSL_ENABLED_KEY have + * been deprecated as far back as I can find Hadoop versions (2.4.1 at least), HADOOP_CALLER_CONTEXT_ENABLED_DEFAULT and + * HADOOP_CALLER_CONTEXT_ENABLED_KEY are the only two constants in 3.1.0 that seem even remotely related... + * */ + conf.setBoolean(CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_ENABLED_KEY, false); conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS, "192.168.1.1:8032"); conf.set(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, "192.168.1.2:8032"); Assert.assertEquals(getHostString("192.168.1.1") + ":8032", StramClientUtils.getSocketConnectString(StramClientUtils.getRMWebAddress(conf, null))); @@ -132,7 +138,7 @@ public InetSocketAddress getSocketAddr(String name, String defaultAddress, int d Assert.assertEquals(1, addresses.size()); Assert.assertEquals(getHostString("192.168.1.1") + ":8032", StramClientUtils.getSocketConnectString(addresses.get(0))); - conf.setBoolean(CommonConfigurationKeysPublic.HADOOP_SSL_ENABLED_KEY, true); + conf.setBoolean(CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_ENABLED_KEY, true); Assert.assertEquals(getHostString("192.168.1.2") + ":8032", StramClientUtils.getSocketConnectString(StramClientUtils.getRMWebAddress(conf, null))); addresses = StramClientUtils.getRMAddresses(conf); Assert.assertEquals(1, addresses.size()); diff --git a/pom.xml b/pom.xml index 3482d473f0..a30d0f408e 100644 --- a/pom.xml +++ b/pom.xml @@ -106,7 +106,7 @@ UTF-8 false yyyyMMdd - 2.6.0 + 3.1.0 github 1.9.13 1.9