SHDP-211 Container Grouping

- Initial import from dev branches
- Container grouping and clustering implementation.
- Internal statemachine which is intentionally left
  un-documented.
- Integration with boot autoconfiguration and
  configuration properties.
- Enhanced libs for building custom client
  side apps. This brings 'app moded' up to
  date in terms of how end to end lifecycle is handled.

build.gradle

@@ -321,6 +321,7 @@ configure(javaProjects()) {
 	eclipse.classpath.file {
 		whenMerged { classpath ->
 			classpath.entries.removeAll { entry -> entry.toString().contains(".pom") }
+			classpath.entries.removeAll { entry -> entry.toString().contains("servlet-api") }
 		}
 	}
 
@@ -964,6 +965,7 @@ if (gradle.ext.mr2) {
 		description = 'Spring Yarn Core'
 		dependencies {
 			compile project(":spring-data-hadoop")
+			compile "org.springframework:spring-messaging:$springVersion"
 			compile("org.apache.hadoop:hadoop-yarn-client:$hadoopVersion") { dep ->
 				exclude group: "org.slf4j", module: "slf4j-log4j12"
 			}
@@ -1028,12 +1030,35 @@ if (gradle.ext.mr2) {
 
 	project('spring-yarn:spring-yarn-boot') {
 		description = 'Spring Yarn Boot'
+		configurations.all {
+			exclude group: 'javax.servlet', module: 'servlet-api', version: '2.5'
+		}
 		dependencies {
 			compile project(":spring-yarn:spring-yarn-core")
 			provided project(":spring-yarn:spring-yarn-batch")
 			provided "org.springframework:spring-web:$springVersion"
+			provided "org.springframework:spring-webmvc:$springVersion"
 			compile "org.springframework.boot:spring-boot-autoconfigure:$springBootVersion"
+			compile "org.springframework.boot:spring-boot-actuator:$springBootVersion"
+			compile "org.apache.httpcomponents:httpclient:$httpclientVersion"
 			runtime "org.yaml:snakeyaml:$snakeYamlVersion"
+			testRuntime("org.springframework.boot:spring-boot-starter-web:$springBootVersion") {
+				exclude group:'ch.qos.logback'
+			}
+			testCompile("org.mockito:mockito-core:$mockitoVersion") { dep ->
+				exclude group: "org.hamcrest"
+			}
+			testCompile "com.jayway.jsonpath:json-path:$jsonpathVersion"
+			testCompile "com.jayway.jsonpath:json-path-assert:$jsonpathVersion"
+		}
+	}
+
+	project('spring-yarn:spring-yarn-boot-cli') {
+		description = 'Spring Yarn Boot Cli'
+		dependencies {
+			compile project(":spring-yarn:spring-yarn-boot")
+			compile "org.springframework.boot:spring-boot-cli:$springBootVersion"
+			runtime "org.springframework:spring-web:$springVersion"
 		}
 	}
 

gradle.properties

@@ -67,6 +67,8 @@ commonsioVersion = 2.4
 cglibVersion = 3.1
 snakeYamlVersion = 1.13
 kiteVersion = 0.14.0
+httpclientVersion = 4.3.3
+jsonpathVersion = 0.8.1
 
 ## Common testing libraries
 junitVersion = 4.11

settings.gradle

@@ -60,5 +60,5 @@ include 'spring-hadoop-build-tests'
 rootProject.children.find{ it.name == 'spring-hadoop-build-tests' }.name = 'spring-data-hadoop-build-tests'
 if (mr2) {
   print "Based on selected distro (${hadoopDistro}) we are including spring-yarn\n"
-  include 'spring-yarn:spring-yarn-core','spring-yarn:spring-yarn-integration','spring-yarn:spring-yarn-batch','spring-yarn:spring-yarn-test','spring-yarn:spring-yarn-build-tests','spring-yarn:spring-yarn-boot','spring-yarn:spring-yarn-boot-test','spring-yarn:spring-yarn-boot-build-tests'
+  include 'spring-yarn:spring-yarn-core','spring-yarn:spring-yarn-integration','spring-yarn:spring-yarn-batch','spring-yarn:spring-yarn-test','spring-yarn:spring-yarn-build-tests','spring-yarn:spring-yarn-boot','spring-yarn:spring-yarn-boot-cli','spring-yarn:spring-yarn-boot-test','spring-yarn:spring-yarn-boot-build-tests'
 }

spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/AbstractApplicationCommand.java

@@ -0,0 +1,59 @@
+/*
+ * Copyright 2014 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.yarn.boot.cli;
+
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+
+import org.springframework.boot.cli.command.OptionParsingCommand;
+import org.springframework.boot.cli.command.options.OptionHandler;
+import org.springframework.boot.cli.command.status.ExitStatus;
+
+/**
+ * Base class for all commands implemented by
+ * this cli package.
+ *
+ * @author Janne Valkealahti
+ *
+ */
+public class AbstractApplicationCommand extends OptionParsingCommand {
+
+	protected AbstractApplicationCommand(String name, String description, OptionHandler handler) {
+		super(name, description, handler);
+	}
+
+	protected abstract static class ApplicationOptionHandler extends OptionHandler {
+
+		@Override
+		protected final ExitStatus run(OptionSet options) throws Exception {
+			runApplication(options);
+			return ExitStatus.OK;
+		}
+
+		protected abstract void runApplication(OptionSet options) throws Exception;
+
+		protected boolean isFlagOn(OptionSet options, OptionSpec<Boolean> option) {
+			return options.has(option) ? options.valueOf(option) : false;
+		}
+
+	}
+
+	@Override
+	public String getUsageHelp() {
+		return "[options]";
+	}
+
+}

spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/AbstractCli.java

@@ -0,0 +1,89 @@
+/*
+ * Copyright 2014 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.yarn.boot.cli;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.springframework.boot.cli.command.Command;
+import org.springframework.boot.cli.command.CommandRunner;
+import org.springframework.boot.cli.command.core.HelpCommand;
+import org.springframework.boot.cli.command.core.VersionCommand;
+import org.springframework.boot.loader.tools.LogbackInitializer;
+
+/**
+ * Base Spring YARN Cli implementation.
+ *
+ * @author Janne Valkealahti
+ *
+ */
+public abstract class AbstractCli {
+
+	private final List<Command> commands = new ArrayList<Command>();
+
+	/**
+	 * Register command.
+	 *
+	 * @param command the command
+	 */
+	protected void registerCommand(Command command) {
+		commands.add(command);
+	}
+
+	/**
+	 * Register commands.
+	 *
+	 * @param commands the commands
+	 */
+	protected void registerCommands(List<Command> commands) {
+		this.commands.addAll(commands);
+	}
+
+	/**
+	 * Main method which should be called from implementing class.
+	 *
+	 * @param args the program args
+	 */
+	protected void doMain(String[] args) {
+		System.setProperty("java.awt.headless", Boolean.toString(true));
+		LogbackInitializer.initialize();
+
+		CommandRunner runner = new CommandRunner(getMainCommandName());
+		runner.addCommand(new HelpCommand(runner));
+
+		for (Command command : commands) {
+			runner.addCommand(command);
+		}
+		runner.setOptionCommands(HelpCommand.class, VersionCommand.class);
+
+		int exitCode = runner.runAndHandleErrors(args);
+		if (exitCode != 0) {
+			// If successful, leave it to run in case it's a server app
+			System.exit(exitCode);
+		}
+	}
+
+	/**
+	 * Get a main command name which can be overwritten
+	 * if default is not suitable for user cli implementation.
+	 *
+	 * @return the main command name
+	 */
+	protected String getMainCommandName() {
+		return "java -jar <jar>";
+	}
+
+}

spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/CliSystemConstants.java

@@ -0,0 +1,49 @@
+package org.springframework.yarn.boot.cli;
+
+import static java.util.Arrays.asList;
+
+import java.util.List;
+
+public abstract class CliSystemConstants {
+
+	public final static List<String> OPTIONS_APPLICATION_ID  = asList("application-id", "a");
+
+	public final static List<String> OPTIONS_APPLICATION_TYPE  = asList("application-type", "t");
+
+	public final static List<String> OPTIONS_APPLICATION_VERSION  = asList("application-version", "v");
+
+	public final static List<String> OPTIONS_CLUSTER_ID  = asList("cluster-id", "c");
+
+	public final static List<String> OPTIONS_CLUSTER_DEF  = asList("cluster-def", "i");
+
+	public final static List<String> OPTIONS_VERBOSE  = asList("verbose", "v");
+
+	public final static List<String> OPTIONS_PROJECTION_TYPE  = asList("projection-type", "p");
+
+	public final static List<String> OPTIONS_PROJECTION_ANY  = asList("projection-any", "w");
+
+	public final static List<String> OPTIONS_PROJECTION_HOSTS  = asList("projection-hosts", "h");
+
+	public final static List<String> OPTIONS_PROJECTION_RACKS  = asList("projection-racks", "r");
+
+	public final static String DESC_APPLICATION_ID = "Specify YARN application id";
+
+	public final static String DESC_CLUSTER_ID = "Specify cluster id";
+
+	public final static String DESC_CLUSTER_DEF = "Specify cluster def id";
+
+	public final static String DESC_APPLICATION_TYPE = "Application type";
+
+	public final static String DESC_APPLICATION_VERSION = "Application version";
+
+	public final static String DESC_VERBOSE = "Verbose output";
+
+	public final static String DESC_PROJECTION_TYPE = "Projection type";
+
+	public final static String DESC_PROJECTION_ANY = "Projection any count";
+
+	public final static String DESC_PROJECTION_HOSTS = "Projection hosts counts";
+
+	public final static String DESC_PROJECTION_RACKS = "Projection racks counts";
+
+}