From 62ebc7a22fde3f0974f381a89c48c8e5d43a1ce4 Mon Sep 17 00:00:00 2001 From: Devaraj K Date: Tue, 31 May 2016 15:08:50 +0530 Subject: [PATCH 1/2] [SPARK-15665] [CORE] spark-submit --kill and --status are not working --- .../org/apache/spark/launcher/SparkSubmitCommandBuilder.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java index 824500d980eee..701c382b4de68 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java +++ b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java @@ -420,6 +420,9 @@ protected boolean handle(String opt, String value) { allowsMixedArguments = true; appResource = specialClasses.get(value); } + } else if (opt.equals(KILL_SUBMISSION) || opt.equals(STATUS)) { + sparkArgs.add(opt); + appResource = value; } else if (opt.equals(HELP) || opt.equals(USAGE_ERROR)) { infoRequested = true; sparkArgs.add(opt); From 30dfb0f5a53492e72dd45d0a2e512546e935b2cb Mon Sep 17 00:00:00 2001 From: Devaraj K Date: Thu, 2 Jun 2016 16:33:27 +0530 Subject: [PATCH 2/2] Fixed the review comments and added a test --- .../launcher/SparkSubmitCommandBuilder.java | 25 ++++++++++--------- .../SparkSubmitCommandBuilderSuite.java | 14 +++++++++++ 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java index 701c382b4de68..b3ccc4805f2c5 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java +++ b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java @@ -89,7 +89,7 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder { } final List sparkArgs; - private final boolean printInfo; + private final boolean isAppResourceReq; private final boolean isExample; /** @@ -101,7 +101,7 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder { SparkSubmitCommandBuilder() { this.sparkArgs = new ArrayList<>(); - this.printInfo = false; + this.isAppResourceReq = true; this.isExample = false; } @@ -133,19 +133,19 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder { this.isExample = isExample; OptionParser parser = new OptionParser(); parser.parse(submitArgs); - this.printInfo = parser.infoRequested; + this.isAppResourceReq = parser.isAppResourceReq; } else { this.isExample = isExample; - this.printInfo = true; + this.isAppResourceReq = false; } } @Override public List buildCommand(Map env) throws IOException, IllegalArgumentException { - if (PYSPARK_SHELL.equals(appResource) && !printInfo) { + if (PYSPARK_SHELL.equals(appResource) && isAppResourceReq) { return buildPySparkShellCommand(env); - } else if (SPARKR_SHELL.equals(appResource) && !printInfo) { + } else if (SPARKR_SHELL.equals(appResource) && isAppResourceReq) { return buildSparkRCommand(env); } else { return buildSparkSubmitCommand(env); @@ -156,7 +156,7 @@ List buildSparkSubmitArgs() { List args = new ArrayList<>(); SparkSubmitOptionParser parser = new SparkSubmitOptionParser(); - if (!allowsMixedArguments && !printInfo) { + if (!allowsMixedArguments && isAppResourceReq) { checkArgument(appResource != null, "Missing application resource."); } @@ -208,7 +208,7 @@ List buildSparkSubmitArgs() { args.add(join(",", pyFiles)); } - if (!printInfo) { + if (isAppResourceReq) { checkArgument(!isExample || mainClass != null, "Missing example class name."); } if (mainClass != null) { @@ -388,7 +388,7 @@ private List findExamplesJars() { private class OptionParser extends SparkSubmitOptionParser { - boolean infoRequested = false; + boolean isAppResourceReq = true; @Override protected boolean handle(String opt, String value) { @@ -421,13 +421,14 @@ protected boolean handle(String opt, String value) { appResource = specialClasses.get(value); } } else if (opt.equals(KILL_SUBMISSION) || opt.equals(STATUS)) { + isAppResourceReq = false; sparkArgs.add(opt); - appResource = value; + sparkArgs.add(value); } else if (opt.equals(HELP) || opt.equals(USAGE_ERROR)) { - infoRequested = true; + isAppResourceReq = false; sparkArgs.add(opt); } else if (opt.equals(VERSION)) { - infoRequested = true; + isAppResourceReq = false; sparkArgs.add(opt); } else { sparkArgs.add(opt); diff --git a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java index d451651555768..16e5a22401ca8 100644 --- a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java +++ b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java @@ -72,6 +72,12 @@ public void testCliHelpAndNoArg() throws Exception { cmd.contains("org.apache.spark.deploy.SparkSubmit")); } + @Test + public void testCliKillAndStatus() throws Exception { + testCLIOpts(parser.STATUS); + testCLIOpts(parser.KILL_SUBMISSION); + } + @Test public void testCliParser() throws Exception { List sparkSubmitArgs = Arrays.asList( @@ -326,4 +332,12 @@ private List buildCommand(List args, Map env) th return newCommandBuilder(args).buildCommand(env); } + private void testCLIOpts(String opt) throws Exception { + List helpArgs = Arrays.asList(opt, "driver-20160531171222-0000"); + Map env = new HashMap<>(); + List cmd = buildCommand(helpArgs, env); + assertTrue(opt + " should be contained in the final cmd.", + cmd.contains(opt)); + } + }