diff --git a/engine/pom.xml b/engine/pom.xml
index 5e7a5f26d6..b2d8a10d5c 100644
--- a/engine/pom.xml
+++ b/engine/pom.xml
@@ -289,6 +289,16 @@
commons-codec
1.10
+
+ commons-configuration
+ commons-configuration
+ 1.10
+
+
+ org.mortbay.jetty
+ jetty
+ 6.1.2
+
org.eclipse.jetty
jetty-servlet
diff --git a/engine/src/main/java/com/datatorrent/stram/client/StramAppLauncher.java b/engine/src/main/java/com/datatorrent/stram/client/StramAppLauncher.java
index d3079d0fd6..41ba522ce1 100644
--- a/engine/src/main/java/com/datatorrent/stram/client/StramAppLauncher.java
+++ b/engine/src/main/java/com/datatorrent/stram/client/StramAppLauncher.java
@@ -14,8 +14,7 @@
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
- * under the License.
- */
+ * under the License.*/
package com.datatorrent.stram.client;
import java.io.File;
@@ -606,16 +605,16 @@ public ApplicationId launchApp(AppFactory appConfig) throws Exception
conf.setEnum(StreamingApplication.ENVIRONMENT, StreamingApplication.Environment.CLUSTER);
LogicalPlan dag = appConfig.createApp(propertiesBuilder);
if (UserGroupInformation.isSecurityEnabled()) {
- long hdfsTokenMaxLifeTime = conf.getLong(StramClientUtils.DT_HDFS_TOKEN_MAX_LIFE_TIME, conf.getLong(StramClientUtils.HDFS_TOKEN_MAX_LIFE_TIME, StramClientUtils.DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT));
+ long hdfsTokenMaxLifeTime = conf.getLong(StramClientUtils.DT_HDFS_TOKEN_MAX_LIFE_TIME, conf.getLong(StramClientUtils.HDFS_TOKEN_MAX_LIFE_TIME, StramClientUtils.RM_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT));
dag.setAttribute(LogicalPlan.HDFS_TOKEN_LIFE_TIME, hdfsTokenMaxLifeTime);
LOG.debug("HDFS token life time {}", hdfsTokenMaxLifeTime);
- long hdfsTokenRenewInterval = conf.getLong(StramClientUtils.DT_HDFS_TOKEN_RENEW_INTERVAL, conf.getLong(StramClientUtils.HDFS_TOKEN_RENEW_INTERVAL, StramClientUtils.DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT));
+ long hdfsTokenRenewInterval = conf.getLong(StramClientUtils.DT_HDFS_TOKEN_RENEW_INTERVAL, conf.getLong(StramClientUtils.HDFS_TOKEN_RENEW_INTERVAL, StramClientUtils.RM_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT));
dag.setAttribute(LogicalPlan.HDFS_TOKEN_RENEWAL_INTERVAL, hdfsTokenRenewInterval);
LOG.debug("HDFS token renew interval {}", hdfsTokenRenewInterval);
- long rmTokenMaxLifeTime = conf.getLong(StramClientUtils.DT_RM_TOKEN_MAX_LIFE_TIME, conf.getLong(YarnConfiguration.DELEGATION_TOKEN_MAX_LIFETIME_KEY, YarnConfiguration.DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT));
+ long rmTokenMaxLifeTime = conf.getLong(StramClientUtils.DT_RM_TOKEN_MAX_LIFE_TIME, conf.getLong(YarnConfiguration.RM_DELEGATION_TOKEN_MAX_LIFETIME_KEY, YarnConfiguration.RM_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT));
dag.setAttribute(LogicalPlan.RM_TOKEN_LIFE_TIME, rmTokenMaxLifeTime);
LOG.debug("RM token life time {}", rmTokenMaxLifeTime);
- long rmTokenRenewInterval = conf.getLong(StramClientUtils.DT_RM_TOKEN_RENEW_INTERVAL, conf.getLong(YarnConfiguration.DELEGATION_TOKEN_RENEW_INTERVAL_KEY, YarnConfiguration.DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT));
+ long rmTokenRenewInterval = conf.getLong(StramClientUtils.DT_RM_TOKEN_RENEW_INTERVAL, conf.getLong(YarnConfiguration.RM_DELEGATION_TOKEN_RENEW_INTERVAL_KEY, YarnConfiguration.RM_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT));
dag.setAttribute(LogicalPlan.RM_TOKEN_RENEWAL_INTERVAL, rmTokenRenewInterval);
LOG.debug("RM token renew interval {}", rmTokenRenewInterval);
setTokenRefreshCredentials(dag, conf);
diff --git a/engine/src/main/java/com/datatorrent/stram/client/StramClientUtils.java b/engine/src/main/java/com/datatorrent/stram/client/StramClientUtils.java
index d4f190fb64..54eabe793f 100644
--- a/engine/src/main/java/com/datatorrent/stram/client/StramClientUtils.java
+++ b/engine/src/main/java/com/datatorrent/stram/client/StramClientUtils.java
@@ -119,8 +119,8 @@ public class StramClientUtils
@Deprecated
public static final String KEY_TAB_FILE = StramUserLogin.DT_AUTH_PREFIX + "store.keytab";
public static final String TOKEN_ANTICIPATORY_REFRESH_FACTOR = StramUserLogin.DT_AUTH_PREFIX + "token.refresh.factor";
- public static final long DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT = 7 * 24 * 60 * 60 * 1000;
- public static final long DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT = 24 * 60 * 60 * 1000;
+ public static final long RM_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT = 7 * 24 * 60 * 60 * 1000;
+ public static final long RM_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT = 24 * 60 * 60 * 1000;
public static final String TOKEN_REFRESH_PRINCIPAL = StramUserLogin.DT_AUTH_PREFIX + "token.refresh.principal";
public static final String TOKEN_REFRESH_KEYTAB = StramUserLogin.DT_AUTH_PREFIX + "token.refresh.keytab";
/**
@@ -828,7 +828,12 @@ private static YarnConfiguration getYarnConfiguration(Configuration conf)
public static InetSocketAddress getRMWebAddress(Configuration conf, String rmId)
{
- boolean sslEnabled = conf.getBoolean(CommonConfigurationKeysPublic.HADOOP_SSL_ENABLED_KEY, CommonConfigurationKeysPublic.HADOOP_SSL_ENABLED_DEFAULT);
+ /*
+ * This is far from confirmed as the "correct" fix. The previous constants HADOOP_SSL_ENABLED_DEFAULT and HADOOP_SSL_ENABLED_KEY have
+ * been deprecated as far back as I can find Hadoop versions (2.4.1 at least), HADOOP_CALLER_CONTEXT_ENABLED_DEFAULT and
+ * HADOOP_CALLER_CONTEXT_ENABLED_KEY are the only two constants in 3.1.0 that seem even remotely related...
+ * */
+ boolean sslEnabled = conf.getBoolean(CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_ENABLED_KEY, CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_ENABLED_DEFAULT);
return getRMWebAddress(conf, sslEnabled, rmId);
}
diff --git a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
index 74510a66a7..cdf9df7045 100644
--- a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
+++ b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
@@ -163,8 +163,8 @@ public class LogicalPlan implements Serializable, DAG
public static Attribute FAST_PUBLISHER_SUBSCRIBER = new Attribute<>(false);
public static Attribute HDFS_TOKEN_RENEWAL_INTERVAL = new Attribute<>(86400000L);
public static Attribute HDFS_TOKEN_LIFE_TIME = new Attribute<>(604800000L);
- public static Attribute RM_TOKEN_RENEWAL_INTERVAL = new Attribute<>(YarnConfiguration.DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT);
- public static Attribute RM_TOKEN_LIFE_TIME = new Attribute<>(YarnConfiguration.DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT);
+ public static Attribute RM_TOKEN_RENEWAL_INTERVAL = new Attribute<>(YarnConfiguration.RM_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT);
+ public static Attribute RM_TOKEN_LIFE_TIME = new Attribute<>(YarnConfiguration.RM_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT);
public static Attribute PRINCIPAL = new Attribute<>(null, StringCodec.String2String.getInstance());
public static Attribute KEY_TAB_FILE = new Attribute<>((String)null, StringCodec.String2String.getInstance());
public static Attribute TOKEN_REFRESH_ANTICIPATORY_FACTOR = new Attribute<>(0.7);
diff --git a/engine/src/main/java/com/datatorrent/stram/util/FSPartFileCollection.java b/engine/src/main/java/com/datatorrent/stram/util/FSPartFileCollection.java
index 357716a720..cdffdec040 100644
--- a/engine/src/main/java/com/datatorrent/stram/util/FSPartFileCollection.java
+++ b/engine/src/main/java/com/datatorrent/stram/util/FSPartFileCollection.java
@@ -172,7 +172,11 @@ public boolean isReadyTurnoverPartFile()
try {
return (syncRequested || (partOutStr.getPos() > bytesPerPartFile) ||
(currentPartFileTimeStamp + millisPerPartFile < System.currentTimeMillis())) && partOutStr.getPos() > 0;
- } catch (IOException ex) {
+ /* Not really sure this is a legitimate fix...getting the following error when compiling:
+ * exception java.io.IOException is never thrown in body of corresponding try statement
+ * The remedy for this appears to be to use a more specific exception as the current try/catch block could
+ * not spit out IOException. For now, just using Exception to try and get this to build, will revisit later. */
+ } catch (Exception ex) {
return true;
}
}
diff --git a/engine/src/test/java/com/datatorrent/stram/client/StramClientUtilsTest.java b/engine/src/test/java/com/datatorrent/stram/client/StramClientUtilsTest.java
index ed61efb03a..d0b7387ef8 100644
--- a/engine/src/test/java/com/datatorrent/stram/client/StramClientUtilsTest.java
+++ b/engine/src/test/java/com/datatorrent/stram/client/StramClientUtilsTest.java
@@ -124,7 +124,13 @@ public InetSocketAddress getSocketAddr(String name, String defaultAddress, int d
};
// basic test
- conf.setBoolean(CommonConfigurationKeysPublic.HADOOP_SSL_ENABLED_KEY, false);
+
+ /*
+ * This is far from confirmed as the "correct" fix. The previous constants HADOOP_SSL_ENABLED_DEFAULT and HADOOP_SSL_ENABLED_KEY have
+ * been deprecated as far back as I can find Hadoop versions (2.4.1 at least), HADOOP_CALLER_CONTEXT_ENABLED_DEFAULT and
+ * HADOOP_CALLER_CONTEXT_ENABLED_KEY are the only two constants in 3.1.0 that seem even remotely related...
+ * */
+ conf.setBoolean(CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_ENABLED_KEY, false);
conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS, "192.168.1.1:8032");
conf.set(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, "192.168.1.2:8032");
Assert.assertEquals(getHostString("192.168.1.1") + ":8032", StramClientUtils.getSocketConnectString(StramClientUtils.getRMWebAddress(conf, null)));
@@ -132,7 +138,7 @@ public InetSocketAddress getSocketAddr(String name, String defaultAddress, int d
Assert.assertEquals(1, addresses.size());
Assert.assertEquals(getHostString("192.168.1.1") + ":8032", StramClientUtils.getSocketConnectString(addresses.get(0)));
- conf.setBoolean(CommonConfigurationKeysPublic.HADOOP_SSL_ENABLED_KEY, true);
+ conf.setBoolean(CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_ENABLED_KEY, true);
Assert.assertEquals(getHostString("192.168.1.2") + ":8032", StramClientUtils.getSocketConnectString(StramClientUtils.getRMWebAddress(conf, null)));
addresses = StramClientUtils.getRMAddresses(conf);
Assert.assertEquals(1, addresses.size());
diff --git a/pom.xml b/pom.xml
index 3482d473f0..a30d0f408e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -106,7 +106,7 @@
UTF-8
false
yyyyMMdd
- 2.6.0
+ 3.1.0
github
1.9.13
1.9