Skip to content
Browse files

Improve resource specification

- as memory for container parameter: Xmx parameter = 0.8 * the arg
- this can be overriden by the user in container vm parameters (for app master and S4 nodes)
  • Loading branch information...
1 parent ba7b94b commit 830f5dfa896a92f0362a8405fcbe898084fb2d0b @matthieumorel matthieumorel committed Nov 26, 2012
View
22 subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/CommonS4YarnArgs.java
@@ -5,19 +5,19 @@
import org.apache.s4.core.Main.InlineConfigParameterConverter;
-import com.beust.jcommander.IParameterValidator;
import com.beust.jcommander.Parameter;
-import com.beust.jcommander.ParameterException;
public class CommonS4YarnArgs {
+ public static final String JAVA_MEMORY_ALLOCATION_NOTICE = "For a Java process, like an S4 node, the total amount of memory is : java heap size + rest of process memory, including thread stack size, direct memory etc... By default, the S4 node, as a Java process, will be started with a java heap with a size "
+ + Utils.CONTAINER_MEMORY_REDUCTION_FACTOR + " times the container size. Can be overriden in ";
public static final String NAMED_STRING_PARAMETERS = "-namedStringParameters";
public static final String EXTRA_MODULES_CLASSES = "-extraModulesClasses";
public static final String PRIORITY = "-priority";
- public static final String S4_NODE_MEMORY = "-s4NodeMemory";
+ public static final String S4_NODE_CONTAINER_MEMORY = "-s4NodeContainerMemory";
public static final String S4_NODE_JVM_PARAMETERS = "-s4NodeJVMParameters";
@@ -30,7 +30,8 @@
@Parameter(names = "-zk", description = "S4 Zookeeper cluster manager connection string", required = true)
String zkString;
- @Parameter(names = { S4_NODE_MEMORY, "-container_memory" }, description = "YARN parameter: Amount of memory in MB to be requested to host the S4 node", required = false, validateWith = S4CLIYarnArgs.MemoryValidator.class)
+ @Parameter(names = { S4_NODE_CONTAINER_MEMORY, "-container_memory" }, description = "YARN parameter: Amount of memory in MB to be requested to host the S4 node. "
+ + JAVA_MEMORY_ALLOCATION_NOTICE + S4_NODE_JVM_PARAMETERS, required = false, validateWith = S4CLIYarnArgs.MemoryValidator.class)
int containerMemory = 256;
@Parameter(names = PRIORITY, description = "YARN parameter: Application priority", required = false)
@@ -55,18 +56,7 @@
List<String> extraNamedParameters = new ArrayList<String>();
// TODO parse JVM parameters that include commas
- @Parameter(names = S4_NODE_JVM_PARAMETERS, description = "Extra JVM parameter for running the nodes, specified as a comma separated list. The memory parameter -Xmx must be configured through "
- + S4_NODE_MEMORY, required = false, validateWith = NodeJVMParametersValidator.class)
+ @Parameter(names = S4_NODE_JVM_PARAMETERS, description = "Extra JVM parameter for running the nodes, specified as a comma separated list.", required = false)
List<String> extraS4NodeJVMParams = new ArrayList<String>();
- public static class NodeJVMParametersValidator implements IParameterValidator {
-
- @Override
- public void validate(String name, String value) throws ParameterException {
- if (value.matches(".*-Xmx\\d+.*")) {
- throw new ParameterException("-Xmx JVM parameter cannot be specified here. You must use the "
- + S4_NODE_MEMORY + " parameter instead.");
- }
- }
- }
}
View
2 subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/S4ApplicationMaster.java
@@ -178,6 +178,8 @@ public void uncaughtException(Thread t, Throwable e) {
*/
public void init() throws IOException {
+ containerMemory = Utils.extractMemoryParam(yarnArgs.containerMemory, yarnArgs.extraS4NodeJVMParams);
+
Map<String, String> envs = System.getenv();
appAttemptID = Records.newRecord(ApplicationAttemptId.class);
View
13 subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/S4CLIYarnArgs.java
@@ -1,5 +1,8 @@
package org.apache.s4.tools.yarn;
+import java.util.ArrayList;
+import java.util.List;
+
import com.beust.jcommander.IParameterValidator;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.ParameterException;
@@ -8,7 +11,9 @@
@Parameters(separators = "=")
class S4CLIYarnArgs extends CommonS4YarnArgs {
- public static final String S4_YARN_MASTER_MEMORY = "-s4YarnMasterMemory";
+ private static final String S4_YARN_APP_MASTER_JVM_PARAMETERS = "-s4YarnAppMasterJVMParameters";
+
+ public static final String S4_YARN_MASTER_MEMORY = "-s4YarnMasterContainerMemory";
public static final String QUEUE = "-queue";
@@ -35,9 +40,13 @@
@Parameter(names = "-timeout", description = "YARN parameter: Application timeout in milliseconds (default is: -1 = no timeout)", required = false)
int timeout = -1;
- @Parameter(names = { "-master_memory", S4_YARN_MASTER_MEMORY }, description = "YARN parameter: Amount of memory in MB to be requested to run the application master", required = false, validateWith = S4CLIYarnArgs.MemoryValidator.class)
+ @Parameter(names = { "-master_memory", S4_YARN_MASTER_MEMORY }, description = "YARN parameter: Amount of memory in MB to be requested to run the application master."
+ + JAVA_MEMORY_ALLOCATION_NOTICE + S4_YARN_APP_MASTER_JVM_PARAMETERS, required = false, validateWith = S4CLIYarnArgs.MemoryValidator.class)
int masterMemory = 256;
+ @Parameter(names = { S4_YARN_APP_MASTER_JVM_PARAMETERS }, description = "JVM parameters for the S4 YARN application master process")
+ List<String> appMasterJVMParameters = new ArrayList<String>();
+
@Parameter(names = "-log_properties", description = "YARN parameter: log4j.properties file", required = false)
String logProperties = "";
View
5 subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/S4YarnClient.java
@@ -138,6 +138,7 @@ public static void main(String[] args) {
logger.info("S4YarnClient args = " + Arrays.toString(args));
Tools.parseArgs(yarnArgs, args);
+
boolean result = false;
try {
@@ -181,6 +182,8 @@ public boolean accept(File pathname) {
public S4YarnClient(S4CLIYarnArgs yarnArgs, Configuration conf) throws Exception {
this.yarnArgs = yarnArgs;
this.conf = conf;
+ amMemory = Utils.extractMemoryParam(yarnArgs.containerMemory, yarnArgs.appMasterJVMParameters);
+
init(this.conf);
}
@@ -321,7 +324,7 @@ public boolean run(boolean testMode) throws IOException {
// Set Application Master class name
vargs.add(S4ApplicationMaster.class.getName());
// Set params for Application Master
- vargs.add(CommonS4YarnArgs.S4_NODE_MEMORY + " " + String.valueOf(yarnArgs.containerMemory));
+ vargs.add(CommonS4YarnArgs.S4_NODE_CONTAINER_MEMORY + " " + String.valueOf(yarnArgs.containerMemory));
vargs.add(CommonS4YarnArgs.NB_S4_NODES + " " + String.valueOf(yarnArgs.numContainers));
vargs.add(CommonS4YarnArgs.PRIORITY + " " + String.valueOf(yarnArgs.priority));
View
49 subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/Utils.java
@@ -0,0 +1,49 @@
+package org.apache.s4.tools.yarn;
+
+import java.util.List;
+import java.util.regex.Pattern;
+
+public class Utils {
+
+ /**
+ * The container memory reduction factor as used in: {@link CommonS4YarnArgs#JAVA_MEMORY_ALLOCATION_NOTICE}
+ *
+ */
+ public static final double CONTAINER_MEMORY_REDUCTION_FACTOR = 0.8;
+
+ /**
+ * Extracts Xmx JVM arg by evaluating container memory arg and JVM overriding args
+ *
+ * @param containerMemoryArg
+ * The container memory parameter, in MB
+ * @param jvmParamsArg
+ * The list of JVM parameters, which may include Xmx parameter
+ * @return Memory to reserve as Xmx java parameter, in MB
+ */
+ static int extractMemoryParam(int containerMemoryArg, List<String> jvmParamsArg) {
+ int memory = (int) (containerMemoryArg * CONTAINER_MEMORY_REDUCTION_FACTOR);
+ // override maximum heap size
+ for (String appVmArg : jvmParamsArg) {
+ if (appVmArg.trim().matches("-Xmx\\d+[gGmMkK]")) {
+ java.util.regex.Matcher m = Pattern.compile("-Xmx(\\d+)([gGmMkK])").matcher(appVmArg);
+ m.matches();
+ int memoryInKBytes = Integer.valueOf(m.group(1)) * HeapSizeMultiplier.valueOf(m.group(2)).multiplier
+ / 1000000;
+ memory = memoryInKBytes;
+ break;
+ }
+ }
+ return memory;
+ }
+
+}
+
+enum HeapSizeMultiplier {
+ g((int) Math.pow(10, 9)), G((int) Math.pow(10, 9)), m((int) Math.pow(10, 6)), M((int) Math.pow(10, 6)), k(
+ (int) Math.pow(10, 3)), K((int) Math.pow(10, 3));
+ int multiplier;
+
+ HeapSizeMultiplier(int multiplier) {
+ this.multiplier = multiplier;
+ }
+}
View
21 subprojects/s4-yarn/src/test/java/org/apache/s4/tools/yarn/TestUtils.java
@@ -0,0 +1,21 @@
+package org.apache.s4.tools.yarn;
+
+import junit.framework.Assert;
+
+import org.junit.Test;
+
+import com.google.common.collect.ImmutableList;
+
+public class TestUtils {
+
+ @Test
+ public void testUtils() {
+ Assert.assertEquals(200,
+ Utils.extractMemoryParam(10, ImmutableList.of(CommonS4YarnArgs.S4_NODE_JVM_PARAMETERS, "-Xmx200M")));
+ Assert.assertEquals(200,
+ Utils.extractMemoryParam(500, ImmutableList.of(CommonS4YarnArgs.S4_NODE_JVM_PARAMETERS, "-Xmx200M")));
+ Assert.assertEquals(((int) (500 * Utils.CONTAINER_MEMORY_REDUCTION_FACTOR)),
+ Utils.extractMemoryParam(500, ImmutableList.of(CommonS4YarnArgs.S4_NODE_JVM_PARAMETERS, "-Xms200M")));
+ }
+
+}
View
1 subprojects/s4-yarn/src/test/java/org/apache/s4/tools/yarn/TestYarnDeployment.java
@@ -180,4 +180,5 @@ public void run() {
client.killApplication(client.getApplicationList().get(0).getApplicationId());
}
+
}

0 comments on commit 830f5df

Please sign in to comment.
Something went wrong with that request. Please try again.