From c59c618f1035e54dbb6ff74e392b51f8e620c255 Mon Sep 17 00:00:00 2001 From: Andrea Sella Date: Tue, 21 Jun 2016 14:13:58 +0200 Subject: [PATCH 01/26] [FLINK-4084] --configDir shell script --- flink-dist/src/main/flink-bin/bin/flink | 12 ++++++++++++ 1 file changed, 12 insertions(+) mode change 100644 => 100755 flink-dist/src/main/flink-bin/bin/flink diff --git a/flink-dist/src/main/flink-bin/bin/flink b/flink-dist/src/main/flink-bin/bin/flink old mode 100644 new mode 100755 index bc0eb6e645d7d..2375f716c084a --- a/flink-dist/src/main/flink-bin/bin/flink +++ b/flink-dist/src/main/flink-bin/bin/flink @@ -18,6 +18,18 @@ ################################################################################ target="$0" + + + +#Search --configDir into the parameters and set it as FLINK_CONF_DIR +args=("$@") +for i in "${!args[@]}"; do + if [ ${args[$i]} = "--configDir" ]; then + FLINK_CONF_DIR=${args[$(($i+1))]} + break + fi +done + # For the case, the executable has been directly symlinked, figure out # the correct bin path by following its symlink up to an upper bound. # Note: we can't use the readlink utility here if we want to be POSIX From 0a451119c496efb1765c7eadc2ae9f145f0016c6 Mon Sep 17 00:00:00 2001 From: Andrea Sella Date: Tue, 21 Jun 2016 17:07:50 +0200 Subject: [PATCH 02/26] [FLINK-4084] Add configDir in CliFrontendParser --- .../org/apache/flink/client/cli/CliFrontendParser.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java index d479d350b507e..b412a493d7719 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java +++ b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java @@ -75,6 +75,9 @@ public class CliFrontendParser { static final Option SAVEPOINT_DISPOSE_OPTION = new Option("d", "dispose", true, "Disposes an existing savepoint."); + static final Option CONFIGURATION_OPTION = new Option("D", "configDir", true, + "The configuration directory with which to run the program."); + // list specific options static final Option RUNNING_OPTION = new Option("r", "running", false, "Show only running programs and their JobIDs"); @@ -115,6 +118,9 @@ public class CliFrontendParser { SAVEPOINT_DISPOSE_OPTION.setRequired(false); SAVEPOINT_DISPOSE_OPTION.setArgName("savepointPath"); + + CONFIGURATION_OPTION.setRequired(false); + CONFIGURATION_OPTION.setArgName("/path/to/configuration/directory"); } private static final Options RUN_OPTIONS = getRunOptions(buildGeneralOptions(new Options())); @@ -144,6 +150,7 @@ public static Options getProgramSpecificOptions(Options options) { options.addOption(LOGGING_OPTION); options.addOption(DETACHED_OPTION); options.addOption(SAVEPOINT_PATH_OPTION); + options.addOption(CONFIGURATION_OPTION); return options; } @@ -154,6 +161,7 @@ private static Options getProgramSpecificOptionsWithoutDeprecatedOptions(Options options.addOption(LOGGING_OPTION); options.addOption(DETACHED_OPTION); options.addOption(SAVEPOINT_PATH_OPTION); + options.addOption(CONFIGURATION_OPTION); return options; } From 3f4b75ffb9f3168e10e149a5127f7c83b336f6fb Mon Sep 17 00:00:00 2001 From: Andrea Sella Date: Wed, 22 Jun 2016 13:07:03 +0200 Subject: [PATCH 03/26] [FLINK-4084] check if configDir value is a directory --- flink-dist/src/main/flink-bin/bin/flink | 27 ++++++++++++++----------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/flink-dist/src/main/flink-bin/bin/flink b/flink-dist/src/main/flink-bin/bin/flink index 2375f716c084a..cedc03f6bd772 100755 --- a/flink-dist/src/main/flink-bin/bin/flink +++ b/flink-dist/src/main/flink-bin/bin/flink @@ -18,18 +18,6 @@ ################################################################################ target="$0" - - - -#Search --configDir into the parameters and set it as FLINK_CONF_DIR -args=("$@") -for i in "${!args[@]}"; do - if [ ${args[$i]} = "--configDir" ]; then - FLINK_CONF_DIR=${args[$(($i+1))]} - break - fi -done - # For the case, the executable has been directly symlinked, figure out # the correct bin path by following its symlink up to an upper bound. # Note: we can't use the readlink utility here if we want to be POSIX @@ -45,6 +33,21 @@ while [ -L "$target" ]; do iteration=$((iteration + 1)) done +#Search --configDir into the parameters and set it as FLINK_CONF_DIR +args=("$@") +for i in "${!args[@]}"; do + if [ ${args[$i]} = "--configDir" ]; then + dir=${args[$(($i+1))]} + if [ -d "$dir" ]; then + FLINK_CONF_DIR=${dir} + else + echo "ERROR: --configDir is not a directory" + exit 1 + fi + break + fi +done + # Convert relative path to absolute path bin=`dirname "$target"` From fd9cf92ddff634cea6789e7b46034c9d5f4063f7 Mon Sep 17 00:00:00 2001 From: Andrea Sella Date: Wed, 22 Jun 2016 14:50:06 +0200 Subject: [PATCH 04/26] [FLINK-4084] add abspath to bin/flink, renamed CONFIGURATION to CONFIGDIR --- .../flink/client/cli/CliFrontendParser.java | 10 ++++----- .../flink/client/cli/ProgramOptions.java | 14 ++++++++++++- .../flink/client/CliFrontendRunTest.java | 21 ++++++++++++------- flink-dist/src/main/flink-bin/bin/flink | 20 ++++++++++++++++-- 4 files changed, 50 insertions(+), 15 deletions(-) diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java index b412a493d7719..ad214cfda605d 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java +++ b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java @@ -75,7 +75,7 @@ public class CliFrontendParser { static final Option SAVEPOINT_DISPOSE_OPTION = new Option("d", "dispose", true, "Disposes an existing savepoint."); - static final Option CONFIGURATION_OPTION = new Option("D", "configDir", true, + static final Option CONFIGDIR_OPTION = new Option("D", "configDir", true, "The configuration directory with which to run the program."); // list specific options @@ -119,8 +119,8 @@ public class CliFrontendParser { SAVEPOINT_DISPOSE_OPTION.setRequired(false); SAVEPOINT_DISPOSE_OPTION.setArgName("savepointPath"); - CONFIGURATION_OPTION.setRequired(false); - CONFIGURATION_OPTION.setArgName("/path/to/configuration/directory"); + CONFIGDIR_OPTION.setRequired(false); + CONFIGDIR_OPTION.setArgName("/path/to/configuration/directory"); } private static final Options RUN_OPTIONS = getRunOptions(buildGeneralOptions(new Options())); @@ -150,7 +150,7 @@ public static Options getProgramSpecificOptions(Options options) { options.addOption(LOGGING_OPTION); options.addOption(DETACHED_OPTION); options.addOption(SAVEPOINT_PATH_OPTION); - options.addOption(CONFIGURATION_OPTION); + options.addOption(CONFIGDIR_OPTION); return options; } @@ -161,7 +161,7 @@ private static Options getProgramSpecificOptionsWithoutDeprecatedOptions(Options options.addOption(LOGGING_OPTION); options.addOption(DETACHED_OPTION); options.addOption(SAVEPOINT_PATH_OPTION); - options.addOption(CONFIGURATION_OPTION); + options.addOption(CONFIGDIR_OPTION); return options; } diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/ProgramOptions.java b/flink-clients/src/main/java/org/apache/flink/client/cli/ProgramOptions.java index 368ec1956f155..a03870094d111 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/cli/ProgramOptions.java +++ b/flink-clients/src/main/java/org/apache/flink/client/cli/ProgramOptions.java @@ -34,7 +34,7 @@ import static org.apache.flink.client.cli.CliFrontendParser.LOGGING_OPTION; import static org.apache.flink.client.cli.CliFrontendParser.PARALLELISM_OPTION; import static org.apache.flink.client.cli.CliFrontendParser.SAVEPOINT_PATH_OPTION; - +import static org.apache.flink.client.cli.CliFrontendParser.CONFIGDIR_OPTION; /** * Base class for command line options that refer to a JAR file program. */ @@ -56,6 +56,8 @@ public abstract class ProgramOptions extends CommandLineOptions { private final String savepointPath; + private final String configDir; + protected ProgramOptions(CommandLine line) throws CliArgsException { super(line); @@ -115,6 +117,12 @@ else if (args.length > 0) { } else { savepointPath = null; } + + if (line.hasOption(CONFIGDIR_OPTION.getOpt())) { + configDir = line.getOptionValue(CONFIGDIR_OPTION.getOpt()); + } else { + configDir = null; + } } public String getJarFilePath() { @@ -148,4 +156,8 @@ public boolean getDetachedMode() { public String getSavepointPath() { return savepointPath; } + + public String getConfigDir() { + return configDir; + } } diff --git a/flink-clients/src/test/java/org/apache/flink/client/CliFrontendRunTest.java b/flink-clients/src/test/java/org/apache/flink/client/CliFrontendRunTest.java index fa554c6088a2b..d0105c6124e52 100644 --- a/flink-clients/src/test/java/org/apache/flink/client/CliFrontendRunTest.java +++ b/flink-clients/src/test/java/org/apache/flink/client/CliFrontendRunTest.java @@ -100,15 +100,22 @@ public void testRun() { } // test jar arguments + { + String[] parameters = + {"-m", "localhost:6123", getTestJarPath(), "-arg1", "value1", "justavalue", "--arg2", "value2"}; + RunOptions options = CliFrontendParser.parseRunCommand(parameters); + assertEquals("-arg1", options.getProgramArgs()[0]); + assertEquals("value1", options.getProgramArgs()[1]); + assertEquals("justavalue", options.getProgramArgs()[2]); + assertEquals("--arg2", options.getProgramArgs()[3]); + assertEquals("value2", options.getProgramArgs()[4]); + } + + // test configure configDir { - String[] parameters = - {"-m", "localhost:6123", getTestJarPath(), "-arg1", "value1", "justavalue", "--arg2", "value2"}; + String[] parameters = {"-D", "excpectedConfigDirectory", getTestJarPath()}; RunOptions options = CliFrontendParser.parseRunCommand(parameters); - assertEquals("-arg1", options.getProgramArgs()[0]); - assertEquals("value1", options.getProgramArgs()[1]); - assertEquals("justavalue", options.getProgramArgs()[2]); - assertEquals("--arg2", options.getProgramArgs()[3]); - assertEquals("value2", options.getProgramArgs()[4]); + assertEquals("excpectedConfigDirectory", options.getConfigDir()); } } catch (Exception e) { diff --git a/flink-dist/src/main/flink-bin/bin/flink b/flink-dist/src/main/flink-bin/bin/flink index cedc03f6bd772..4d6d9f1d6cf73 100755 --- a/flink-dist/src/main/flink-bin/bin/flink +++ b/flink-dist/src/main/flink-bin/bin/flink @@ -17,6 +17,20 @@ # limitations under the License. ################################################################################ +function abspath() { + if [ -d "$1" ]; then + # dir + (cd "$1"; pwd) + elif [ -f "$1" ]; then + # file + if [[ $1 == */* ]]; then + echo "$(cd "${1%/*}"; pwd)/${1##*/}" + else + echo "$(pwd)/$1" + fi + fi +} + target="$0" # For the case, the executable has been directly symlinked, figure out # the correct bin path by following its symlink up to an upper bound. @@ -33,13 +47,14 @@ while [ -L "$target" ]; do iteration=$((iteration + 1)) done + #Search --configDir into the parameters and set it as FLINK_CONF_DIR args=("$@") for i in "${!args[@]}"; do - if [ ${args[$i]} = "--configDir" ]; then + if [[ ${args[$i]} = "--configDir" || ${args[$i]} = "-D" ]]; then dir=${args[$(($i+1))]} if [ -d "$dir" ]; then - FLINK_CONF_DIR=${dir} + FLINK_CONF_DIR= abspath ${dir} else echo "ERROR: --configDir is not a directory" exit 1 @@ -68,3 +83,4 @@ export FLINK_CONF_DIR # Add HADOOP_CLASSPATH to allow the usage of Hadoop file systems $JAVA_RUN $JVM_ARGS "${log_setting[@]}" -classpath "`manglePathList "$CC_CLASSPATH:$INTERNAL_HADOOP_CLASSPATHS"`" org.apache.flink.client.CliFrontend "$@" + From e89097c02a13cacd3d131e08de11dced72cdf3e0 Mon Sep 17 00:00:00 2001 From: Andrea Sella Date: Wed, 22 Jun 2016 15:26:45 +0200 Subject: [PATCH 05/26] [FLINK-4084] typo in CliFrontendRunTest --- .../test/java/org/apache/flink/client/CliFrontendRunTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flink-clients/src/test/java/org/apache/flink/client/CliFrontendRunTest.java b/flink-clients/src/test/java/org/apache/flink/client/CliFrontendRunTest.java index d0105c6124e52..1292919fadc89 100644 --- a/flink-clients/src/test/java/org/apache/flink/client/CliFrontendRunTest.java +++ b/flink-clients/src/test/java/org/apache/flink/client/CliFrontendRunTest.java @@ -113,9 +113,9 @@ public void testRun() { // test configure configDir { - String[] parameters = {"-D", "excpectedConfigDirectory", getTestJarPath()}; + String[] parameters = {"-D", "expectedConfigDirectory", getTestJarPath()}; RunOptions options = CliFrontendParser.parseRunCommand(parameters); - assertEquals("excpectedConfigDirectory", options.getConfigDir()); + assertEquals("expectedConfigDirectory", options.getConfigDir()); } } catch (Exception e) { From 059d7dec6fb34b7599ac098468c37452556dcff3 Mon Sep 17 00:00:00 2001 From: Andrea Sella Date: Wed, 22 Jun 2016 16:59:12 +0200 Subject: [PATCH 06/26] [FLINK-4084] Read configDir before CliFrontend's initialization --- .../src/main/java/org/apache/flink/client/CliFrontend.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flink-clients/src/main/java/org/apache/flink/client/CliFrontend.java b/flink-clients/src/main/java/org/apache/flink/client/CliFrontend.java index 5c4791be7f9f3..58e6afebf9097 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/CliFrontend.java +++ b/flink-clients/src/main/java/org/apache/flink/client/CliFrontend.java @@ -1004,7 +1004,8 @@ public static void main(String[] args) { EnvironmentInformation.logEnvironmentInfo(LOG, "Command Line Client", args); try { - CliFrontend cli = new CliFrontend(); + String configDirPath = CliFrontendParser.parseRunCommand(args).getConfigDir(); + CliFrontend cli = configDirPath == null ? new CliFrontend() : new CliFrontend(configDirPath); int retCode = cli.parseParameters(args); System.exit(retCode); } From 15b3e294eb624a65cc073915a607c7c98decf2ed Mon Sep 17 00:00:00 2001 From: Andrea Sella Date: Thu, 23 Jun 2016 10:12:56 +0200 Subject: [PATCH 07/26] [FLINK-4084] --configDir enabled just for run action --- .../org/apache/flink/client/CliFrontend.java | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/flink-clients/src/main/java/org/apache/flink/client/CliFrontend.java b/flink-clients/src/main/java/org/apache/flink/client/CliFrontend.java index 58e6afebf9097..d98be86048d94 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/CliFrontend.java +++ b/flink-clients/src/main/java/org/apache/flink/client/CliFrontend.java @@ -132,9 +132,9 @@ public class CliFrontend { - private final Configuration config; + private Configuration config; - private final FiniteDuration clientTimeout; + private FiniteDuration clientTimeout; /** * @@ -145,6 +145,10 @@ public CliFrontend() throws Exception { } public CliFrontend(String configDir) throws Exception { + configureConfigDir(configDir); + } + + private void configureConfigDir(String configDir) throws Exception { // configure the config directory File configDirectory = new File(configDir); @@ -209,6 +213,15 @@ protected int run(String[] args) { return handleError(t); } + //load configDir + if(options.getConfigDir() != null) { + try { + configureConfigDir(options.getConfigDir()); + } catch(Exception e) { + return handleArgException(new CliArgsException(e.toString())); + } + } + // evaluate help flag if (options.isPrintHelp()) { CliFrontendParser.printHelpForRun(); @@ -1004,8 +1017,7 @@ public static void main(String[] args) { EnvironmentInformation.logEnvironmentInfo(LOG, "Command Line Client", args); try { - String configDirPath = CliFrontendParser.parseRunCommand(args).getConfigDir(); - CliFrontend cli = configDirPath == null ? new CliFrontend() : new CliFrontend(configDirPath); + CliFrontend cli = new CliFrontend(); int retCode = cli.parseParameters(args); System.exit(retCode); } From 665b6492cc61f1096c09af23e922036946126982 Mon Sep 17 00:00:00 2001 From: Andrea Sella Date: Thu, 23 Jun 2016 10:59:39 +0200 Subject: [PATCH 08/26] [FLINK-4084] format issue: add space after if --- .../src/main/java/org/apache/flink/client/CliFrontend.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flink-clients/src/main/java/org/apache/flink/client/CliFrontend.java b/flink-clients/src/main/java/org/apache/flink/client/CliFrontend.java index d98be86048d94..c42c000a5a0a3 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/CliFrontend.java +++ b/flink-clients/src/main/java/org/apache/flink/client/CliFrontend.java @@ -214,7 +214,7 @@ protected int run(String[] args) { } //load configDir - if(options.getConfigDir() != null) { + if (options.getConfigDir() != null) { try { configureConfigDir(options.getConfigDir()); } catch(Exception e) { From fddd3c07c0300074b2676552e1965d2230d42151 Mon Sep 17 00:00:00 2001 From: Andrea Sella Date: Thu, 23 Jun 2016 11:51:15 +0200 Subject: [PATCH 09/26] [FLINK-4084] Add configDir in docs --- docs/apis/cli.md | 2 ++ .../java/org/apache/flink/client/cli/CliFrontendParser.java | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/apis/cli.md b/docs/apis/cli.md index 511862c00e356..a45b662dd3650 100644 --- a/docs/apis/cli.md +++ b/docs/apis/cli.md @@ -187,6 +187,8 @@ Action "run" compiles and runs a program. java.net.URLClassLoader}. -d,--detached If present, runs the job in detached mode + -D,--configDir The configuration directory with which + to run the program. -m,--jobmanager Address of the JobManager (master) to which to connect. Specify 'yarn-cluster' as the JobManager to diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java index ad214cfda605d..456d9363e7072 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java +++ b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java @@ -120,7 +120,7 @@ public class CliFrontendParser { SAVEPOINT_DISPOSE_OPTION.setArgName("savepointPath"); CONFIGDIR_OPTION.setRequired(false); - CONFIGDIR_OPTION.setArgName("/path/to/configuration/directory"); + CONFIGDIR_OPTION.setArgName("/path/to/confdir"); } private static final Options RUN_OPTIONS = getRunOptions(buildGeneralOptions(new Options())); From 66bf7b550679e1a2d36eba2a057953980b055ab5 Mon Sep 17 00:00:00 2001 From: Andrea Sella Date: Mon, 27 Jun 2016 18:52:30 +0200 Subject: [PATCH 10/26] Address mxm's comments --- .../flink/client/cli/CliFrontendParser.java | 2 +- .../flink/client/cli/ProgramOptions.java | 4 +- .../flink/client/CliFrontendRunTest.java | 2 +- flink-dist/src/main/flink-bin/bin/flink | 45 ++++++++----------- 4 files changed, 22 insertions(+), 31 deletions(-) diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java index 456d9363e7072..dfb5c296ceb37 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java +++ b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java @@ -75,7 +75,7 @@ public class CliFrontendParser { static final Option SAVEPOINT_DISPOSE_OPTION = new Option("d", "dispose", true, "Disposes an existing savepoint."); - static final Option CONFIGDIR_OPTION = new Option("D", "configDir", true, + static final Option CONFIGDIR_OPTION = new Option(null, "configDir", true, "The configuration directory with which to run the program."); // list specific options diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/ProgramOptions.java b/flink-clients/src/main/java/org/apache/flink/client/cli/ProgramOptions.java index a03870094d111..761e1f6bc5e2c 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/cli/ProgramOptions.java +++ b/flink-clients/src/main/java/org/apache/flink/client/cli/ProgramOptions.java @@ -118,8 +118,8 @@ else if (args.length > 0) { savepointPath = null; } - if (line.hasOption(CONFIGDIR_OPTION.getOpt())) { - configDir = line.getOptionValue(CONFIGDIR_OPTION.getOpt()); + if (line.hasOption(CONFIGDIR_OPTION.getLongOpt())) { + configDir = line.getOptionValue(CONFIGDIR_OPTION.getLongOpt()); } else { configDir = null; } diff --git a/flink-clients/src/test/java/org/apache/flink/client/CliFrontendRunTest.java b/flink-clients/src/test/java/org/apache/flink/client/CliFrontendRunTest.java index 1292919fadc89..6516487348db7 100644 --- a/flink-clients/src/test/java/org/apache/flink/client/CliFrontendRunTest.java +++ b/flink-clients/src/test/java/org/apache/flink/client/CliFrontendRunTest.java @@ -113,7 +113,7 @@ public void testRun() { // test configure configDir { - String[] parameters = {"-D", "expectedConfigDirectory", getTestJarPath()}; + String[] parameters = {"--configDir", "expectedConfigDirectory", getTestJarPath()}; RunOptions options = CliFrontendParser.parseRunCommand(parameters); assertEquals("expectedConfigDirectory", options.getConfigDir()); } diff --git a/flink-dist/src/main/flink-bin/bin/flink b/flink-dist/src/main/flink-bin/bin/flink index 4d6d9f1d6cf73..b4b72e9c0c29a 100755 --- a/flink-dist/src/main/flink-bin/bin/flink +++ b/flink-dist/src/main/flink-bin/bin/flink @@ -17,44 +17,35 @@ # limitations under the License. ################################################################################ -function abspath() { - if [ -d "$1" ]; then - # dir - (cd "$1"; pwd) - elif [ -f "$1" ]; then - # file - if [[ $1 == */* ]]; then - echo "$(cd "${1%/*}"; pwd)/${1##*/}" - else - echo "$(pwd)/$1" - fi - fi -} - -target="$0" # For the case, the executable has been directly symlinked, figure out # the correct bin path by following its symlink up to an upper bound. # Note: we can't use the readlink utility here if we want to be POSIX # compatible. -iteration=0 -while [ -L "$target" ]; do - if [ "$iteration" -gt 100 ]; then - echo "Cannot resolve path: You have a cyclic symlink in $target." - break - fi - ls=`ls -ld -- "$target"` - target=`expr "$ls" : '.* -> \(.*\)$'` - iteration=$((iteration + 1)) -done +followSymLink() { + local iteration=0 + local bar=$1 + while [ -L "$bar" ]; do + if [ "$iteration" -gt 100 ]; then + echo "Cannot resolve path: You have a cyclic symlink in $bar." + break + fi + ls=`ls -ld -- "$bar"` + bar=`expr "$ls" : '.* -> \(.*\)$'` + iteration=$((iteration + 1)) + done + + echo "$bar" +} +target=$(followSymLink "$0") #Search --configDir into the parameters and set it as FLINK_CONF_DIR args=("$@") for i in "${!args[@]}"; do if [[ ${args[$i]} = "--configDir" || ${args[$i]} = "-D" ]]; then - dir=${args[$(($i+1))]} + dir=$(followSymLink "${args[$(($i+1))]}" ) if [ -d "$dir" ]; then - FLINK_CONF_DIR= abspath ${dir} + FLINK_CONF_DIR=`cd "${dir}"; pwd -P` else echo "ERROR: --configDir is not a directory" exit 1 From c4a01fd3db1f387392d18d73137797bbb0d0702c Mon Sep 17 00:00:00 2001 From: Andrea Sella Date: Mon, 27 Jun 2016 19:10:29 +0200 Subject: [PATCH 11/26] Update docs --- docs/apis/cli.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/apis/cli.md b/docs/apis/cli.md index a45b662dd3650..5b8c3d2de4b67 100644 --- a/docs/apis/cli.md +++ b/docs/apis/cli.md @@ -187,7 +187,7 @@ Action "run" compiles and runs a program. java.net.URLClassLoader}. -d,--detached If present, runs the job in detached mode - -D,--configDir The configuration directory with which + --configDir The configuration directory with which to run the program. -m,--jobmanager Address of the JobManager (master) to which to connect. Specify From 76283d3f0fbf5454b208ba1887d847acea09640d Mon Sep 17 00:00:00 2001 From: Andrea Sella Date: Tue, 28 Jun 2016 15:46:20 +0200 Subject: [PATCH 12/26] remove short option from bin/flink --- flink-dist/src/main/flink-bin/bin/flink | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flink-dist/src/main/flink-bin/bin/flink b/flink-dist/src/main/flink-bin/bin/flink index b4b72e9c0c29a..426c274a15555 100755 --- a/flink-dist/src/main/flink-bin/bin/flink +++ b/flink-dist/src/main/flink-bin/bin/flink @@ -42,7 +42,7 @@ target=$(followSymLink "$0") #Search --configDir into the parameters and set it as FLINK_CONF_DIR args=("$@") for i in "${!args[@]}"; do - if [[ ${args[$i]} = "--configDir" || ${args[$i]} = "-D" ]]; then + if [ ${args[$i]} = "--configDir" ]; then dir=$(followSymLink "${args[$(($i+1))]}" ) if [ -d "$dir" ]; then FLINK_CONF_DIR=`cd "${dir}"; pwd -P` From ad93b699a64cccb3152d3a67feb9b1293510e06c Mon Sep 17 00:00:00 2001 From: Andrea Sella Date: Tue, 28 Jun 2016 16:01:38 +0200 Subject: [PATCH 13/26] rename local bar in target, add space --- flink-dist/src/main/flink-bin/bin/flink | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/flink-dist/src/main/flink-bin/bin/flink b/flink-dist/src/main/flink-bin/bin/flink index 426c274a15555..3f633d33114ea 100755 --- a/flink-dist/src/main/flink-bin/bin/flink +++ b/flink-dist/src/main/flink-bin/bin/flink @@ -23,18 +23,18 @@ # compatible. followSymLink() { local iteration=0 - local bar=$1 - while [ -L "$bar" ]; do + local target=$1 + while [ -L "$target" ]; do if [ "$iteration" -gt 100 ]; then - echo "Cannot resolve path: You have a cyclic symlink in $bar." + echo "Cannot resolve path: You have a cyclic symlink in $target." break fi - ls=`ls -ld -- "$bar"` - bar=`expr "$ls" : '.* -> \(.*\)$'` + ls=`ls -ld -- "$target"` + target=`expr "$ls" : '.* -> \(.*\)$'` iteration=$((iteration + 1)) done - echo "$bar" + echo "$target" } target=$(followSymLink "$0") @@ -43,7 +43,7 @@ target=$(followSymLink "$0") args=("$@") for i in "${!args[@]}"; do if [ ${args[$i]} = "--configDir" ]; then - dir=$(followSymLink "${args[$(($i+1))]}" ) + dir=$( followSymLink "${args[$(($i+1))]}" ) if [ -d "$dir" ]; then FLINK_CONF_DIR=`cd "${dir}"; pwd -P` else From d8618ffba04b01f1a25cccb64cf97d21b66920f7 Mon Sep 17 00:00:00 2001 From: Andrea Sella Date: Tue, 28 Jun 2016 16:22:18 +0200 Subject: [PATCH 14/26] indent CliFrontendRunTest --- .../test/java/org/apache/flink/client/CliFrontendRunTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flink-clients/src/test/java/org/apache/flink/client/CliFrontendRunTest.java b/flink-clients/src/test/java/org/apache/flink/client/CliFrontendRunTest.java index 6516487348db7..7584d06502ac1 100644 --- a/flink-clients/src/test/java/org/apache/flink/client/CliFrontendRunTest.java +++ b/flink-clients/src/test/java/org/apache/flink/client/CliFrontendRunTest.java @@ -102,7 +102,7 @@ public void testRun() { // test jar arguments { String[] parameters = - {"-m", "localhost:6123", getTestJarPath(), "-arg1", "value1", "justavalue", "--arg2", "value2"}; + {"-m", "localhost:6123", getTestJarPath(), "-arg1", "value1", "justavalue", "--arg2", "value2"}; RunOptions options = CliFrontendParser.parseRunCommand(parameters); assertEquals("-arg1", options.getProgramArgs()[0]); assertEquals("value1", options.getProgramArgs()[1]); From cae20d490232f3367fcd58c6c72865d4737e557a Mon Sep 17 00:00:00 2001 From: Andrea Sella Date: Tue, 28 Jun 2016 16:32:00 +0200 Subject: [PATCH 15/26] format CliFrontendRunTest --- .../flink/client/CliFrontendRunTest.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/flink-clients/src/test/java/org/apache/flink/client/CliFrontendRunTest.java b/flink-clients/src/test/java/org/apache/flink/client/CliFrontendRunTest.java index 7584d06502ac1..4523211253c69 100644 --- a/flink-clients/src/test/java/org/apache/flink/client/CliFrontendRunTest.java +++ b/flink-clients/src/test/java/org/apache/flink/client/CliFrontendRunTest.java @@ -100,16 +100,16 @@ public void testRun() { } // test jar arguments - { - String[] parameters = - {"-m", "localhost:6123", getTestJarPath(), "-arg1", "value1", "justavalue", "--arg2", "value2"}; - RunOptions options = CliFrontendParser.parseRunCommand(parameters); - assertEquals("-arg1", options.getProgramArgs()[0]); - assertEquals("value1", options.getProgramArgs()[1]); - assertEquals("justavalue", options.getProgramArgs()[2]); - assertEquals("--arg2", options.getProgramArgs()[3]); - assertEquals("value2", options.getProgramArgs()[4]); - } + { + String[] parameters = + {"-m", "localhost:6123", getTestJarPath(), "-arg1", "value1", "justavalue", "--arg2", "value2"}; + RunOptions options = CliFrontendParser.parseRunCommand(parameters); + assertEquals("-arg1", options.getProgramArgs()[0]); + assertEquals("value1", options.getProgramArgs()[1]); + assertEquals("justavalue", options.getProgramArgs()[2]); + assertEquals("--arg2", options.getProgramArgs()[3]); + assertEquals("value2", options.getProgramArgs()[4]); + } // test configure configDir { From 1ef5011bbbef7aa976f83714fd2b88e2fa556c58 Mon Sep 17 00:00:00 2001 From: Andrea Sella Date: Tue, 28 Jun 2016 16:40:10 +0200 Subject: [PATCH 16/26] replace semicolon with chained version --- flink-dist/src/main/flink-bin/bin/flink | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flink-dist/src/main/flink-bin/bin/flink b/flink-dist/src/main/flink-bin/bin/flink index 3f633d33114ea..bce417990bbec 100755 --- a/flink-dist/src/main/flink-bin/bin/flink +++ b/flink-dist/src/main/flink-bin/bin/flink @@ -45,7 +45,7 @@ for i in "${!args[@]}"; do if [ ${args[$i]} = "--configDir" ]; then dir=$( followSymLink "${args[$(($i+1))]}" ) if [ -d "$dir" ]; then - FLINK_CONF_DIR=`cd "${dir}"; pwd -P` + FLINK_CONF_DIR=`cd "${dir}" && pwd -P` else echo "ERROR: --configDir is not a directory" exit 1 From 29e1d6b160b639b3c29e1bbc3bfa28d8b6e701da Mon Sep 17 00:00:00 2001 From: Andrea Sella Date: Wed, 29 Jun 2016 16:36:19 +0200 Subject: [PATCH 17/26] address mxm comments --- .../org/apache/flink/client/CliFrontend.java | 30 +++++++-------- .../flink/client/cli/CliFrontendParser.java | 37 ++++++++++++++++--- .../apache/flink/client/cli/MainOptions.java | 10 +++++ .../flink/client/CliFrontendMainTest.java | 35 ++++++++++++++++++ .../flink/client/CliFrontendRunTest.java | 7 +--- 5 files changed, 92 insertions(+), 27 deletions(-) create mode 100644 flink-clients/src/main/java/org/apache/flink/client/cli/MainOptions.java create mode 100644 flink-clients/src/test/java/org/apache/flink/client/CliFrontendMainTest.java diff --git a/flink-clients/src/main/java/org/apache/flink/client/CliFrontend.java b/flink-clients/src/main/java/org/apache/flink/client/CliFrontend.java index c42c000a5a0a3..9429740eecce3 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/CliFrontend.java +++ b/flink-clients/src/main/java/org/apache/flink/client/CliFrontend.java @@ -32,6 +32,7 @@ import org.apache.flink.client.cli.DefaultCLI; import org.apache.flink.client.cli.InfoOptions; import org.apache.flink.client.cli.ListOptions; +import org.apache.flink.client.cli.MainOptions; import org.apache.flink.client.cli.ProgramOptions; import org.apache.flink.client.cli.RunOptions; import org.apache.flink.client.cli.SavepointOptions; @@ -132,9 +133,9 @@ public class CliFrontend { - private Configuration config; + private final Configuration config; - private FiniteDuration clientTimeout; + private final FiniteDuration clientTimeout; /** * @@ -145,10 +146,6 @@ public CliFrontend() throws Exception { } public CliFrontend(String configDir) throws Exception { - configureConfigDir(configDir); - } - - private void configureConfigDir(String configDir) throws Exception { // configure the config directory File configDirectory = new File(configDir); @@ -213,15 +210,6 @@ protected int run(String[] args) { return handleError(t); } - //load configDir - if (options.getConfigDir() != null) { - try { - configureConfigDir(options.getConfigDir()); - } catch(Exception e) { - return handleArgException(new CliArgsException(e.toString())); - } - } - // evaluate help flag if (options.isPrintHelp()) { CliFrontendParser.printHelpForRun(); @@ -1017,7 +1005,17 @@ public static void main(String[] args) { EnvironmentInformation.logEnvironmentInfo(LOG, "Command Line Client", args); try { - CliFrontend cli = new CliFrontend(); + final MainOptions mainOptions = CliFrontendParser.parseMainCommand(args); + + final CliFrontend cli; + if (mainOptions.getConfigDir() == null) { + cli = new CliFrontend(); + } else { + + // remove configDir key and value from arguments + args = Arrays.copyOfRange(args, 2, args.length); + cli = new CliFrontend(mainOptions.getConfigDir()); + } int retCode = cli.parseParameters(args); System.exit(retCode); } diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java index dfb5c296ceb37..ea5656e2fa1db 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java +++ b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java @@ -76,7 +76,7 @@ public class CliFrontendParser { "Disposes an existing savepoint."); static final Option CONFIGDIR_OPTION = new Option(null, "configDir", true, - "The configuration directory with which to run the program."); + "Path to a custom configuration directory with which to run the client."); // list specific options static final Option RUNNING_OPTION = new Option("r", "running", false, @@ -129,6 +129,7 @@ public class CliFrontendParser { private static final Options CANCEL_OPTIONS = getCancelOptions(buildGeneralOptions(new Options())); private static final Options STOP_OPTIONS = getStopOptions(buildGeneralOptions(new Options())); private static final Options SAVEPOINT_OPTIONS = getSavepointOptions(buildGeneralOptions(new Options())); + private static final Options MAIN_OPTIONS = getMainOptions(buildGeneralOptions(new Options())); private static Options buildGeneralOptions(Options options) { options.addOption(HELP_OPTION); @@ -150,7 +151,6 @@ public static Options getProgramSpecificOptions(Options options) { options.addOption(LOGGING_OPTION); options.addOption(DETACHED_OPTION); options.addOption(SAVEPOINT_PATH_OPTION); - options.addOption(CONFIGDIR_OPTION); return options; } @@ -161,7 +161,6 @@ private static Options getProgramSpecificOptionsWithoutDeprecatedOptions(Options options.addOption(LOGGING_OPTION); options.addOption(DETACHED_OPTION); options.addOption(SAVEPOINT_PATH_OPTION); - options.addOption(CONFIGDIR_OPTION); return options; } @@ -206,7 +205,11 @@ private static Options getSavepointOptions(Options options) { return addCustomCliOptions(options, false); } - // -------------------------------------------------------------------------------------------- + private static Options getMainOptions(Options options) { + options.addOption(CONFIGDIR_OPTION); + return options; + } +// -------------------------------------------------------------------------------------------- // Help // -------------------------------------------------------------------------------------------- @@ -249,8 +252,11 @@ private static Options getSavepointOptionsWithoutDeprecatedOptions(Options optio * Prints the help for the client. */ public static void printHelp() { - System.out.println("./flink [OPTIONS] [ARGUMENTS]"); + System.out.println("./flink [CONFIGDIR] [ACTION-OPTIONS] [ARGUMENTS]"); System.out.println(); + + printHelpForMain(); + System.out.println("The following actions are available:"); printHelpForRun(); @@ -263,6 +269,17 @@ public static void printHelp() { System.out.println(); } + public static void printHelpForMain() { + HelpFormatter formatter = new HelpFormatter(); + formatter.setLeftPadding(5); + formatter.setWidth(80); + + System.out.println("General option to point a custom configuration directory"); + formatter.printHelp(" ", getMainOptions(new Options())); + + System.out.println(); + } + public static void printHelpForRun() { HelpFormatter formatter = new HelpFormatter(); formatter.setLeftPadding(5); @@ -459,4 +476,14 @@ public static InfoOptions parseInfoCommand(String[] args) throws CliArgsExceptio } } + public static MainOptions parseMainCommand(String[] args) throws CliArgsException { + try { + DefaultParser parser = new DefaultParser(); + CommandLine line = parser.parse(MAIN_OPTIONS, args, false); + return new MainOptions(line); + } catch (ParseException e){ + throw new CliArgsException(e.getMessage()); + } + } + } diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/MainOptions.java b/flink-clients/src/main/java/org/apache/flink/client/cli/MainOptions.java new file mode 100644 index 0000000000000..c0fe71a274c23 --- /dev/null +++ b/flink-clients/src/main/java/org/apache/flink/client/cli/MainOptions.java @@ -0,0 +1,10 @@ +package org.apache.flink.client.cli; + +import org.apache.commons.cli.CommandLine; + +public class MainOptions extends ProgramOptions { + + public MainOptions(CommandLine line) throws CliArgsException { + super(line); + } +} diff --git a/flink-clients/src/test/java/org/apache/flink/client/CliFrontendMainTest.java b/flink-clients/src/test/java/org/apache/flink/client/CliFrontendMainTest.java new file mode 100644 index 0000000000000..37d138480c5d1 --- /dev/null +++ b/flink-clients/src/test/java/org/apache/flink/client/CliFrontendMainTest.java @@ -0,0 +1,35 @@ +package org.apache.flink.client; + + +import org.apache.flink.client.cli.CliArgsException; +import org.apache.flink.client.cli.CliFrontendParser; +import org.apache.flink.client.cli.MainOptions; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.FileNotFoundException; +import java.net.MalformedURLException; + +import static org.apache.flink.client.CliFrontendTestUtils.getTestJarPath; +import static org.junit.Assert.assertEquals; + +public class CliFrontendMainTest { + + + @BeforeClass + public static void init() { + CliFrontendTestUtils.pipeSystemOutToNull(); + CliFrontendTestUtils.clearGlobalConfiguration(); + } + + @Test + public void testMain() throws CliArgsException, FileNotFoundException, MalformedURLException { + // test configure configDir + { + String[] parameters = {"--configDir", "expectedConfigDirectory", getTestJarPath()}; + MainOptions options = CliFrontendParser.parseMainCommand(parameters); + assertEquals("expectedConfigDirectory", options.getConfigDir()); + } + } + +} diff --git a/flink-clients/src/test/java/org/apache/flink/client/CliFrontendRunTest.java b/flink-clients/src/test/java/org/apache/flink/client/CliFrontendRunTest.java index 4523211253c69..8b5ed48202083 100644 --- a/flink-clients/src/test/java/org/apache/flink/client/CliFrontendRunTest.java +++ b/flink-clients/src/test/java/org/apache/flink/client/CliFrontendRunTest.java @@ -111,12 +111,7 @@ public void testRun() { assertEquals("value2", options.getProgramArgs()[4]); } - // test configure configDir - { - String[] parameters = {"--configDir", "expectedConfigDirectory", getTestJarPath()}; - RunOptions options = CliFrontendParser.parseRunCommand(parameters); - assertEquals("expectedConfigDirectory", options.getConfigDir()); - } + } catch (Exception e) { e.printStackTrace(); From b7f588686967a5016544c2b55284dd9338cd72e4 Mon Sep 17 00:00:00 2001 From: Andrea Sella Date: Thu, 30 Jun 2016 14:27:45 +0200 Subject: [PATCH 18/26] update script: configDir is positional, it must be after target --- flink-dist/src/main/flink-bin/bin/flink | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/flink-dist/src/main/flink-bin/bin/flink b/flink-dist/src/main/flink-bin/bin/flink index bce417990bbec..87cfde4897d18 100755 --- a/flink-dist/src/main/flink-bin/bin/flink +++ b/flink-dist/src/main/flink-bin/bin/flink @@ -39,20 +39,17 @@ followSymLink() { target=$(followSymLink "$0") -#Search --configDir into the parameters and set it as FLINK_CONF_DIR -args=("$@") -for i in "${!args[@]}"; do - if [ ${args[$i]} = "--configDir" ]; then - dir=$( followSymLink "${args[$(($i+1))]}" ) - if [ -d "$dir" ]; then - FLINK_CONF_DIR=`cd "${dir}" && pwd -P` - else - echo "ERROR: --configDir is not a directory" - exit 1 - fi - break +#Check if --configDir is present and set is value as FLINK_CONF_DIR +if [ "$1" = "--configDir" ]; then + dir=$( followSymLink "$2" ) + if [ -d "$dir" ]; then + FLINK_CONF_DIR=`cd "${dir}" && pwd -P` + else + echo "ERROR: --configDir is not a directory" + exit 1 fi -done + break +fi # Convert relative path to absolute path bin=`dirname "$target"` From 972c9124a9486986e01769367b4d4510ec888b9c Mon Sep 17 00:00:00 2001 From: Andrea Sella Date: Fri, 1 Jul 2016 18:15:41 +0200 Subject: [PATCH 19/26] drop args after action for MainOptions --- .../org/apache/flink/client/CliFrontend.java | 9 +++++++ .../flink/client/cli/CliFrontendParser.java | 14 ++++++++++ .../apache/flink/client/cli/MainOptions.java | 27 +++++++++++++++++-- .../flink/client/cli/ProgramOptions.java | 11 -------- .../flink/client/CliFrontendMainTest.java | 18 ++++++++----- 5 files changed, 59 insertions(+), 20 deletions(-) diff --git a/flink-clients/src/main/java/org/apache/flink/client/CliFrontend.java b/flink-clients/src/main/java/org/apache/flink/client/CliFrontend.java index 9429740eecce3..a7e6560fdef29 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/CliFrontend.java +++ b/flink-clients/src/main/java/org/apache/flink/client/CliFrontend.java @@ -113,6 +113,15 @@ public class CliFrontend { private static final String ACTION_STOP = "stop"; private static final String ACTION_SAVEPOINT = "savepoint"; + public static final List ACTIONS = Arrays.asList( + ACTION_RUN, + ACTION_INFO, + ACTION_LIST, + ACTION_CANCEL, + ACTION_STOP, + ACTION_SAVEPOINT + ); + // config dir parameters private static final String ENV_CONFIG_DIRECTORY = "FLINK_CONF_DIR"; private static final String CONFIG_DIRECTORY_FALLBACK_1 = "../conf"; diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java index ea5656e2fa1db..3831c4203b3da 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java +++ b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java @@ -27,6 +27,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Arrays; +import java.util.List; + /** * A simple command line parser (based on Apache Commons CLI) that extracts command @@ -477,6 +480,17 @@ public static InfoOptions parseInfoCommand(String[] args) throws CliArgsExceptio } public static MainOptions parseMainCommand(String[] args) throws CliArgsException { + + // drop all arguments after an action + final List params= Arrays.asList(args); + for (String action: CliFrontend.ACTIONS) { + int index = params.indexOf(action); + if(index != -1) { + args = Arrays.copyOfRange(args, 0, index); + break; + } + } + try { DefaultParser parser = new DefaultParser(); CommandLine line = parser.parse(MAIN_OPTIONS, args, false); diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/MainOptions.java b/flink-clients/src/main/java/org/apache/flink/client/cli/MainOptions.java index c0fe71a274c23..bc0d71bbf10b8 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/cli/MainOptions.java +++ b/flink-clients/src/main/java/org/apache/flink/client/cli/MainOptions.java @@ -1,10 +1,33 @@ package org.apache.flink.client.cli; import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; -public class MainOptions extends ProgramOptions { +import static org.apache.flink.client.cli.CliFrontendParser.CONFIGDIR_OPTION; + +public class MainOptions { + + private final String configDir; + + private final Option[] options; public MainOptions(CommandLine line) throws CliArgsException { - super(line); + + if (line.hasOption(CONFIGDIR_OPTION.getLongOpt())) { + configDir = line.getOptionValue(CONFIGDIR_OPTION.getLongOpt()); + } else { + configDir = null; + } + + this.options = line.getOptions(); + } + + public Option[] getOptions() { + return options == null ? new Option[0] : options; + } + + + public String getConfigDir(){ + return configDir; } } diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/ProgramOptions.java b/flink-clients/src/main/java/org/apache/flink/client/cli/ProgramOptions.java index 761e1f6bc5e2c..b04246f80eb8c 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/cli/ProgramOptions.java +++ b/flink-clients/src/main/java/org/apache/flink/client/cli/ProgramOptions.java @@ -34,7 +34,6 @@ import static org.apache.flink.client.cli.CliFrontendParser.LOGGING_OPTION; import static org.apache.flink.client.cli.CliFrontendParser.PARALLELISM_OPTION; import static org.apache.flink.client.cli.CliFrontendParser.SAVEPOINT_PATH_OPTION; -import static org.apache.flink.client.cli.CliFrontendParser.CONFIGDIR_OPTION; /** * Base class for command line options that refer to a JAR file program. */ @@ -56,8 +55,6 @@ public abstract class ProgramOptions extends CommandLineOptions { private final String savepointPath; - private final String configDir; - protected ProgramOptions(CommandLine line) throws CliArgsException { super(line); @@ -118,11 +115,6 @@ else if (args.length > 0) { savepointPath = null; } - if (line.hasOption(CONFIGDIR_OPTION.getLongOpt())) { - configDir = line.getOptionValue(CONFIGDIR_OPTION.getLongOpt()); - } else { - configDir = null; - } } public String getJarFilePath() { @@ -157,7 +149,4 @@ public String getSavepointPath() { return savepointPath; } - public String getConfigDir() { - return configDir; - } } diff --git a/flink-clients/src/test/java/org/apache/flink/client/CliFrontendMainTest.java b/flink-clients/src/test/java/org/apache/flink/client/CliFrontendMainTest.java index 37d138480c5d1..d11721280b795 100644 --- a/flink-clients/src/test/java/org/apache/flink/client/CliFrontendMainTest.java +++ b/flink-clients/src/test/java/org/apache/flink/client/CliFrontendMainTest.java @@ -23,13 +23,17 @@ public static void init() { } @Test - public void testMain() throws CliArgsException, FileNotFoundException, MalformedURLException { - // test configure configDir - { - String[] parameters = {"--configDir", "expectedConfigDirectory", getTestJarPath()}; - MainOptions options = CliFrontendParser.parseMainCommand(parameters); - assertEquals("expectedConfigDirectory", options.getConfigDir()); - } + public void simpleParse() throws FileNotFoundException, MalformedURLException, CliArgsException { + String[] parameters = {"--configDir", "expectedConfigDirectory", "run", getTestJarPath()}; + MainOptions options = CliFrontendParser.parseMainCommand(parameters); + assertEquals("expectedConfigDirectory", options.getConfigDir()); } + @Test + public void dropAfterAction() throws FileNotFoundException, MalformedURLException, CliArgsException { + String[] parameters = {"--configDir", "expectedConfigDirectory", "run", getTestJarPath(), "--configDir", "notExpected"}; + MainOptions options = CliFrontendParser.parseMainCommand(parameters); + assertEquals("expectedConfigDirectory", options.getConfigDir()); + assertEquals(options.getOptions().length, 1); + } } From 856704c857853120c9e3c4c56512be2dce47d0c9 Mon Sep 17 00:00:00 2001 From: chobeat Date: Wed, 7 Sep 2016 10:16:43 +0200 Subject: [PATCH 20/26] Removed args dropping in parseMainCommand --- .../flink/client/cli/CliFrontendParser.java | 22 ++----------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java index 3831c4203b3da..8c1a5235805b0 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java +++ b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java @@ -17,19 +17,11 @@ */ package org.apache.flink.client.cli; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.DefaultParser; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.Options; -import org.apache.commons.cli.ParseException; +import org.apache.commons.cli.*; import org.apache.flink.client.CliFrontend; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Arrays; -import java.util.List; - /** * A simple command line parser (based on Apache Commons CLI) that extracts command @@ -481,19 +473,9 @@ public static InfoOptions parseInfoCommand(String[] args) throws CliArgsExceptio public static MainOptions parseMainCommand(String[] args) throws CliArgsException { - // drop all arguments after an action - final List params= Arrays.asList(args); - for (String action: CliFrontend.ACTIONS) { - int index = params.indexOf(action); - if(index != -1) { - args = Arrays.copyOfRange(args, 0, index); - break; - } - } - try { DefaultParser parser = new DefaultParser(); - CommandLine line = parser.parse(MAIN_OPTIONS, args, false); + CommandLine line = parser.parse(MAIN_OPTIONS, args, true); return new MainOptions(line); } catch (ParseException e){ throw new CliArgsException(e.getMessage()); From 8150c2dff8bfdb8a8b243ff8e06cb965946d5bc1 Mon Sep 17 00:00:00 2001 From: chobeat Date: Wed, 7 Sep 2016 10:34:38 +0200 Subject: [PATCH 21/26] Removed unnecessary changes --- .../main/java/org/apache/flink/client/cli/ProgramOptions.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/ProgramOptions.java b/flink-clients/src/main/java/org/apache/flink/client/cli/ProgramOptions.java index b04246f80eb8c..368ec1956f155 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/cli/ProgramOptions.java +++ b/flink-clients/src/main/java/org/apache/flink/client/cli/ProgramOptions.java @@ -34,6 +34,7 @@ import static org.apache.flink.client.cli.CliFrontendParser.LOGGING_OPTION; import static org.apache.flink.client.cli.CliFrontendParser.PARALLELISM_OPTION; import static org.apache.flink.client.cli.CliFrontendParser.SAVEPOINT_PATH_OPTION; + /** * Base class for command line options that refer to a JAR file program. */ @@ -114,7 +115,6 @@ else if (args.length > 0) { } else { savepointPath = null; } - } public String getJarFilePath() { @@ -148,5 +148,4 @@ public boolean getDetachedMode() { public String getSavepointPath() { return savepointPath; } - } From db1f473388ed9738e115046e7b44989e30e529f3 Mon Sep 17 00:00:00 2001 From: chobeat Date: Wed, 7 Sep 2016 10:48:00 +0200 Subject: [PATCH 22/26] Removed unnecessary changes from CLIFrontendRunTest --- .../java/org/apache/flink/client/CliFrontendRunTest.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/flink-clients/src/test/java/org/apache/flink/client/CliFrontendRunTest.java b/flink-clients/src/test/java/org/apache/flink/client/CliFrontendRunTest.java index 8b5ed48202083..687dc033faf3c 100644 --- a/flink-clients/src/test/java/org/apache/flink/client/CliFrontendRunTest.java +++ b/flink-clients/src/test/java/org/apache/flink/client/CliFrontendRunTest.java @@ -19,9 +19,6 @@ package org.apache.flink.client; -import static org.apache.flink.client.CliFrontendTestUtils.*; -import static org.junit.Assert.*; - import org.apache.flink.client.cli.CliFrontendParser; import org.apache.flink.client.cli.CommandLineOptions; import org.apache.flink.client.cli.RunOptions; @@ -30,6 +27,9 @@ import org.junit.BeforeClass; import org.junit.Test; +import static org.apache.flink.client.CliFrontendTestUtils.getTestJarPath; +import static org.junit.Assert.*; + public class CliFrontendRunTest { @@ -110,8 +110,6 @@ public void testRun() { assertEquals("--arg2", options.getProgramArgs()[3]); assertEquals("value2", options.getProgramArgs()[4]); } - - } catch (Exception e) { e.printStackTrace(); From 7d43c9b518d728a7d62108289e18103c579971bd Mon Sep 17 00:00:00 2001 From: chobeat Date: Wed, 7 Sep 2016 10:48:48 +0200 Subject: [PATCH 23/26] Renamed CLIFrontendMainTest in CliFrontendArgsMainTest --- .../{CliFrontendMainTest.java => CliFrontendArgsMainTest.java} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename flink-clients/src/test/java/org/apache/flink/client/{CliFrontendMainTest.java => CliFrontendArgsMainTest.java} (97%) diff --git a/flink-clients/src/test/java/org/apache/flink/client/CliFrontendMainTest.java b/flink-clients/src/test/java/org/apache/flink/client/CliFrontendArgsMainTest.java similarity index 97% rename from flink-clients/src/test/java/org/apache/flink/client/CliFrontendMainTest.java rename to flink-clients/src/test/java/org/apache/flink/client/CliFrontendArgsMainTest.java index d11721280b795..904666db220ed 100644 --- a/flink-clients/src/test/java/org/apache/flink/client/CliFrontendMainTest.java +++ b/flink-clients/src/test/java/org/apache/flink/client/CliFrontendArgsMainTest.java @@ -13,7 +13,7 @@ import static org.apache.flink.client.CliFrontendTestUtils.getTestJarPath; import static org.junit.Assert.assertEquals; -public class CliFrontendMainTest { +public class CliFrontendArgsMainTest { @BeforeClass From b92019b12e2906eac70c2241da6dbf8d6a187bb2 Mon Sep 17 00:00:00 2001 From: chobeat Date: Wed, 7 Sep 2016 11:04:04 +0200 Subject: [PATCH 24/26] Added licenses --- .../apache/flink/client/cli/MainOptions.java | 17 +++++++++++++++++ .../flink/client/CliFrontendArgsMainTest.java | 17 +++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/MainOptions.java b/flink-clients/src/main/java/org/apache/flink/client/cli/MainOptions.java index bc0d71bbf10b8..441b1296d06e2 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/cli/MainOptions.java +++ b/flink-clients/src/main/java/org/apache/flink/client/cli/MainOptions.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.flink.client.cli; import org.apache.commons.cli.CommandLine; diff --git a/flink-clients/src/test/java/org/apache/flink/client/CliFrontendArgsMainTest.java b/flink-clients/src/test/java/org/apache/flink/client/CliFrontendArgsMainTest.java index 904666db220ed..ef9134e66e6cf 100644 --- a/flink-clients/src/test/java/org/apache/flink/client/CliFrontendArgsMainTest.java +++ b/flink-clients/src/test/java/org/apache/flink/client/CliFrontendArgsMainTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.flink.client; From f2d106ab700e1cbe3ac9c2defa90d718c40978f3 Mon Sep 17 00:00:00 2001 From: chobeat Date: Wed, 7 Sep 2016 11:12:54 +0200 Subject: [PATCH 25/26] Checkstyle import --- .../org/apache/flink/client/cli/CliFrontendParser.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java index 8c1a5235805b0..ef90fcbf26518 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java +++ b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java @@ -17,7 +17,12 @@ */ package org.apache.flink.client.cli; -import org.apache.commons.cli.*; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.ParseException; import org.apache.flink.client.CliFrontend; import org.slf4j.Logger; import org.slf4j.LoggerFactory; From 5a6b116fbb87036ccfe8ff8dfb5a4d0a4c05b4b3 Mon Sep 17 00:00:00 2001 From: chobeat Date: Tue, 20 Sep 2016 15:55:10 +0200 Subject: [PATCH 26/26] Fixed CLI configdir option explanation --- .../java/org/apache/flink/client/cli/CliFrontendParser.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java index a127282a66c99..28add6f771233 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java +++ b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java @@ -262,7 +262,7 @@ private static Options getSavepointOptionsWithoutDeprecatedOptions(Options optio * Prints the help for the client. */ public static void printHelp() { - System.out.println("./flink [CONFIGDIR] [ACTION-OPTIONS] [ARGUMENTS]"); + System.out.println("./flink [--configdir CONFIGDIR] [ACTION-OPTIONS] [ARGUMENTS]"); System.out.println(); printHelpForMain();