Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Improve resource specification

- as memory for container parameter: Xmx parameter = 0.8 * the arg
- this can be overriden by the user in container vm parameters (for app master and S4 nodes)
  • Loading branch information...
commit 830f5dfa896a92f0362a8405fcbe898084fb2d0b 1 parent ba7b94b
Matthieu Morel matthieumorel authored
22 subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/CommonS4YarnArgs.java
@@ -5,19 +5,19 @@
5 5
6 6 import org.apache.s4.core.Main.InlineConfigParameterConverter;
7 7
8   -import com.beust.jcommander.IParameterValidator;
9 8 import com.beust.jcommander.Parameter;
10   -import com.beust.jcommander.ParameterException;
11 9
12 10 public class CommonS4YarnArgs {
13 11
  12 + public static final String JAVA_MEMORY_ALLOCATION_NOTICE = "For a Java process, like an S4 node, the total amount of memory is : java heap size + rest of process memory, including thread stack size, direct memory etc... By default, the S4 node, as a Java process, will be started with a java heap with a size "
  13 + + Utils.CONTAINER_MEMORY_REDUCTION_FACTOR + " times the container size. Can be overriden in ";
14 14 public static final String NAMED_STRING_PARAMETERS = "-namedStringParameters";
15 15
16 16 public static final String EXTRA_MODULES_CLASSES = "-extraModulesClasses";
17 17
18 18 public static final String PRIORITY = "-priority";
19 19
20   - public static final String S4_NODE_MEMORY = "-s4NodeMemory";
  20 + public static final String S4_NODE_CONTAINER_MEMORY = "-s4NodeContainerMemory";
21 21
22 22 public static final String S4_NODE_JVM_PARAMETERS = "-s4NodeJVMParameters";
23 23
@@ -30,7 +30,8 @@
30 30 @Parameter(names = "-zk", description = "S4 Zookeeper cluster manager connection string", required = true)
31 31 String zkString;
32 32
33   - @Parameter(names = { S4_NODE_MEMORY, "-container_memory" }, description = "YARN parameter: Amount of memory in MB to be requested to host the S4 node", required = false, validateWith = S4CLIYarnArgs.MemoryValidator.class)
  33 + @Parameter(names = { S4_NODE_CONTAINER_MEMORY, "-container_memory" }, description = "YARN parameter: Amount of memory in MB to be requested to host the S4 node. "
  34 + + JAVA_MEMORY_ALLOCATION_NOTICE + S4_NODE_JVM_PARAMETERS, required = false, validateWith = S4CLIYarnArgs.MemoryValidator.class)
34 35 int containerMemory = 256;
35 36
36 37 @Parameter(names = PRIORITY, description = "YARN parameter: Application priority", required = false)
@@ -55,18 +56,7 @@
55 56 List<String> extraNamedParameters = new ArrayList<String>();
56 57
57 58 // TODO parse JVM parameters that include commas
58   - @Parameter(names = S4_NODE_JVM_PARAMETERS, description = "Extra JVM parameter for running the nodes, specified as a comma separated list. The memory parameter -Xmx must be configured through "
59   - + S4_NODE_MEMORY, required = false, validateWith = NodeJVMParametersValidator.class)
  59 + @Parameter(names = S4_NODE_JVM_PARAMETERS, description = "Extra JVM parameter for running the nodes, specified as a comma separated list.", required = false)
60 60 List<String> extraS4NodeJVMParams = new ArrayList<String>();
61 61
62   - public static class NodeJVMParametersValidator implements IParameterValidator {
63   -
64   - @Override
65   - public void validate(String name, String value) throws ParameterException {
66   - if (value.matches(".*-Xmx\\d+.*")) {
67   - throw new ParameterException("-Xmx JVM parameter cannot be specified here. You must use the "
68   - + S4_NODE_MEMORY + " parameter instead.");
69   - }
70   - }
71   - }
72 62 }
2  subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/S4ApplicationMaster.java
@@ -178,6 +178,8 @@ public void uncaughtException(Thread t, Throwable e) {
178 178 */
179 179 public void init() throws IOException {
180 180
  181 + containerMemory = Utils.extractMemoryParam(yarnArgs.containerMemory, yarnArgs.extraS4NodeJVMParams);
  182 +
181 183 Map<String, String> envs = System.getenv();
182 184
183 185 appAttemptID = Records.newRecord(ApplicationAttemptId.class);
13 subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/S4CLIYarnArgs.java
... ... @@ -1,5 +1,8 @@
1 1 package org.apache.s4.tools.yarn;
2 2
  3 +import java.util.ArrayList;
  4 +import java.util.List;
  5 +
3 6 import com.beust.jcommander.IParameterValidator;
4 7 import com.beust.jcommander.Parameter;
5 8 import com.beust.jcommander.ParameterException;
@@ -8,7 +11,9 @@
8 11 @Parameters(separators = "=")
9 12 class S4CLIYarnArgs extends CommonS4YarnArgs {
10 13
11   - public static final String S4_YARN_MASTER_MEMORY = "-s4YarnMasterMemory";
  14 + private static final String S4_YARN_APP_MASTER_JVM_PARAMETERS = "-s4YarnAppMasterJVMParameters";
  15 +
  16 + public static final String S4_YARN_MASTER_MEMORY = "-s4YarnMasterContainerMemory";
12 17
13 18 public static final String QUEUE = "-queue";
14 19
@@ -35,9 +40,13 @@
35 40 @Parameter(names = "-timeout", description = "YARN parameter: Application timeout in milliseconds (default is: -1 = no timeout)", required = false)
36 41 int timeout = -1;
37 42
38   - @Parameter(names = { "-master_memory", S4_YARN_MASTER_MEMORY }, description = "YARN parameter: Amount of memory in MB to be requested to run the application master", required = false, validateWith = S4CLIYarnArgs.MemoryValidator.class)
  43 + @Parameter(names = { "-master_memory", S4_YARN_MASTER_MEMORY }, description = "YARN parameter: Amount of memory in MB to be requested to run the application master."
  44 + + JAVA_MEMORY_ALLOCATION_NOTICE + S4_YARN_APP_MASTER_JVM_PARAMETERS, required = false, validateWith = S4CLIYarnArgs.MemoryValidator.class)
39 45 int masterMemory = 256;
40 46
  47 + @Parameter(names = { S4_YARN_APP_MASTER_JVM_PARAMETERS }, description = "JVM parameters for the S4 YARN application master process")
  48 + List<String> appMasterJVMParameters = new ArrayList<String>();
  49 +
41 50 @Parameter(names = "-log_properties", description = "YARN parameter: log4j.properties file", required = false)
42 51 String logProperties = "";
43 52
5 subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/S4YarnClient.java
@@ -138,6 +138,7 @@ public static void main(String[] args) {
138 138 logger.info("S4YarnClient args = " + Arrays.toString(args));
139 139
140 140 Tools.parseArgs(yarnArgs, args);
  141 +
141 142 boolean result = false;
142 143 try {
143 144
@@ -181,6 +182,8 @@ public boolean accept(File pathname) {
181 182 public S4YarnClient(S4CLIYarnArgs yarnArgs, Configuration conf) throws Exception {
182 183 this.yarnArgs = yarnArgs;
183 184 this.conf = conf;
  185 + amMemory = Utils.extractMemoryParam(yarnArgs.containerMemory, yarnArgs.appMasterJVMParameters);
  186 +
184 187 init(this.conf);
185 188 }
186 189
@@ -321,7 +324,7 @@ public boolean run(boolean testMode) throws IOException {
321 324 // Set Application Master class name
322 325 vargs.add(S4ApplicationMaster.class.getName());
323 326 // Set params for Application Master
324   - vargs.add(CommonS4YarnArgs.S4_NODE_MEMORY + " " + String.valueOf(yarnArgs.containerMemory));
  327 + vargs.add(CommonS4YarnArgs.S4_NODE_CONTAINER_MEMORY + " " + String.valueOf(yarnArgs.containerMemory));
325 328 vargs.add(CommonS4YarnArgs.NB_S4_NODES + " " + String.valueOf(yarnArgs.numContainers));
326 329 vargs.add(CommonS4YarnArgs.PRIORITY + " " + String.valueOf(yarnArgs.priority));
327 330
49 subprojects/s4-yarn/src/main/java/org/apache/s4/tools/yarn/Utils.java
... ... @@ -0,0 +1,49 @@
  1 +package org.apache.s4.tools.yarn;
  2 +
  3 +import java.util.List;
  4 +import java.util.regex.Pattern;
  5 +
  6 +public class Utils {
  7 +
  8 + /**
  9 + * The container memory reduction factor as used in: {@link CommonS4YarnArgs#JAVA_MEMORY_ALLOCATION_NOTICE}
  10 + *
  11 + */
  12 + public static final double CONTAINER_MEMORY_REDUCTION_FACTOR = 0.8;
  13 +
  14 + /**
  15 + * Extracts Xmx JVM arg by evaluating container memory arg and JVM overriding args
  16 + *
  17 + * @param containerMemoryArg
  18 + * The container memory parameter, in MB
  19 + * @param jvmParamsArg
  20 + * The list of JVM parameters, which may include Xmx parameter
  21 + * @return Memory to reserve as Xmx java parameter, in MB
  22 + */
  23 + static int extractMemoryParam(int containerMemoryArg, List<String> jvmParamsArg) {
  24 + int memory = (int) (containerMemoryArg * CONTAINER_MEMORY_REDUCTION_FACTOR);
  25 + // override maximum heap size
  26 + for (String appVmArg : jvmParamsArg) {
  27 + if (appVmArg.trim().matches("-Xmx\\d+[gGmMkK]")) {
  28 + java.util.regex.Matcher m = Pattern.compile("-Xmx(\\d+)([gGmMkK])").matcher(appVmArg);
  29 + m.matches();
  30 + int memoryInKBytes = Integer.valueOf(m.group(1)) * HeapSizeMultiplier.valueOf(m.group(2)).multiplier
  31 + / 1000000;
  32 + memory = memoryInKBytes;
  33 + break;
  34 + }
  35 + }
  36 + return memory;
  37 + }
  38 +
  39 +}
  40 +
  41 +enum HeapSizeMultiplier {
  42 + g((int) Math.pow(10, 9)), G((int) Math.pow(10, 9)), m((int) Math.pow(10, 6)), M((int) Math.pow(10, 6)), k(
  43 + (int) Math.pow(10, 3)), K((int) Math.pow(10, 3));
  44 + int multiplier;
  45 +
  46 + HeapSizeMultiplier(int multiplier) {
  47 + this.multiplier = multiplier;
  48 + }
  49 +}
21 subprojects/s4-yarn/src/test/java/org/apache/s4/tools/yarn/TestUtils.java
... ... @@ -0,0 +1,21 @@
  1 +package org.apache.s4.tools.yarn;
  2 +
  3 +import junit.framework.Assert;
  4 +
  5 +import org.junit.Test;
  6 +
  7 +import com.google.common.collect.ImmutableList;
  8 +
  9 +public class TestUtils {
  10 +
  11 + @Test
  12 + public void testUtils() {
  13 + Assert.assertEquals(200,
  14 + Utils.extractMemoryParam(10, ImmutableList.of(CommonS4YarnArgs.S4_NODE_JVM_PARAMETERS, "-Xmx200M")));
  15 + Assert.assertEquals(200,
  16 + Utils.extractMemoryParam(500, ImmutableList.of(CommonS4YarnArgs.S4_NODE_JVM_PARAMETERS, "-Xmx200M")));
  17 + Assert.assertEquals(((int) (500 * Utils.CONTAINER_MEMORY_REDUCTION_FACTOR)),
  18 + Utils.extractMemoryParam(500, ImmutableList.of(CommonS4YarnArgs.S4_NODE_JVM_PARAMETERS, "-Xms200M")));
  19 + }
  20 +
  21 +}
1  subprojects/s4-yarn/src/test/java/org/apache/s4/tools/yarn/TestYarnDeployment.java
@@ -180,4 +180,5 @@ public void run() {
180 180 client.killApplication(client.getApplicationList().get(0).getApplicationId());
181 181
182 182 }
  183 +
183 184 }

0 comments on commit 830f5df

Please sign in to comment.
Something went wrong with that request. Please try again.