[SW-889] Port AWS preparation scripts into SW codebase (#769)

doc/build.gradle

@@ -25,11 +25,17 @@ task substitute(){
     doLast {
         def siteDir = "${buildDir}/site"
 
+        def (first, second, third) = version.replace("-SNAPSHOT", "").tokenize(".")
+        def majorVersion = "${first}.${second}"
+        def minorVersion = third
+
         new File(siteDir).eachFileRecurse(FILES) {
             if (it.name.endsWith('.html')) {
                 def contents = file(it).getText('UTF-8')
                 contents = contents
                         .replaceAll("SUBST_SW_VERSION", version.replace("-SNAPSHOT", ""))
+                        .replaceAll("SUBST_SW_MAJOR_VERSION", majorVersion)
+                        .replaceAll("SUBST_SW_MINOR_VERSION", minorVersion)
                         .replaceAll("SUBST_SPARK_VERSION", sparkVersion)
                         .replaceAll("SUBST_SPARK_MAJOR_VERSION", sparkVersion.count(".") == 1 ? sparkVersion : sparkVersion.substring(0, sparkVersion.lastIndexOf('.')))
                         .replaceAll("SUBST_H2O_VERSION", h2oVersion)

make-dist.sh

@@ -65,6 +65,10 @@ SCALADOC_DST_DIR="$DIST_BUILD_DIR/scaladoc/"
 USERDOC_SRC_DIR="$TOPDIR/doc/build/site/"
 USERDOC_DST_DIR="$DIST_BUILD_DIR/doc/"
 
+# Source and destination folders for deployement templates
+TEMPLATES_SRC_DIR="$TOPDIR/templates/build/"
+TEMPLATES_DST_DIR="$DIST_BUILD_DIR/templates/"
+
 # Resulting zip file
 ZIP_NAME="sparkling-water-$VERSION.zip" 
 ZIP_FILE="$DIST_BUILD_DIR/$ZIP_NAME"
@@ -91,6 +95,9 @@ rsync -rtvW "$SCALADOC_SRC_DIR" "$SCALADOC_DST_DIR"
 # Copy user documentation
 rsync -rtvW "$USERDOC_SRC_DIR" "$USERDOC_DST_DIR"
 
+# Copy templates
+rsync -rtvW "$TEMPLATES_SRC_DIR" "$TEMPLATES_DST_DIR"
+
 # Try to use variables defined in Jenkins first
 GITHASH=${GIT_COMMIT:-$(git rev-parse --verify HEAD)}
 GITBRANCH=${BRANCH_NAME:-$(git rev-parse --verify --abbrev-ref HEAD)}

settings.gradle

@@ -13,7 +13,7 @@ include 'app-streaming'
 include 'package'
 include 'doc'
 include 'extension-stack-trace'
-
+include 'templates'
 // Prefix all projects with sparkling-water name
 rootProject.children.each { project ->
     if (project.name.startsWith("app-")) {

templates/build.gradle

@@ -0,0 +1,34 @@
+apply plugin: 'base'
+description = "Templates for Sparkling Water deployment"
+
+
+/**
+ * We can't use Sphinx substitutions in for example code blocks and other container, so we post-process
+ * the generated files manually
+ */
+task substituteAWS(){
+    doLast {
+        def aws_script = "${project.buildDir.toString()}/aws/install_sparkling_water_${version.replace('-SNAPSHOT', '')}.sh"
+        def contents = file(aws_script).getText('UTF-8')
+        def (first, second, third) = version.replace("-SNAPSHOT", "").tokenize(".")
+        def majorVersion = "${first}.${second}"
+        def minorVersion = third
+        def branchName = version.toString().endsWith("nightly") ? "master" : "rel-${majorVersion}"
+        contents = contents
+                .replaceAll("SUBST_BRANCH_NAME", branchName)
+                .replaceAll("SUBST_MAJOR_VERSION", majorVersion)
+                .replaceAll("SUBST_MINOR_VERSION", minorVersion)
+
+        file(aws_script).write(contents, 'UTF-8')
+    }
+}
+
+
+task copyAWS(type: Copy) {
+    from 'src/aws'
+    into "build/aws"
+    rename "install_sparkling_water.sh", "install_sparkling_water_${version.replace('-SNAPSHOT', '')}.sh"
+}
+
+substituteAWS.dependsOn copyAWS
+build.dependsOn substituteAWS

templates/src/aws/install_sparkling_water.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+set -x -e
+
+
+# AWS EMR bootstrap script
+# for installing Sparkling Water on AWS EMR with Spark
+#
+##############################
+
+## Python installations and libraries needed on the worker roles in order to get PySparkling working
+
+sudo python -m pip install --upgrade pip
+sudo python -m pip install --upgrade colorama
+
+sudo ln -sf /usr/local/bin/pip2.7 /usr/bin/pip
+
+#mkdir -p /home/h2o
+
+sudo pip install -U requests
+sudo pip install -U tabulate
+sudo pip install -U future
+sudo pip install -U six
+
+#Scikit Learn on the nodes
+sudo pip install -U scikit-learn
+
+mkdir -p /home/hadoop/h2o
+cd /home/hadoop/h2o
+
+echo -e "\n Installing sparkling water version SUBST_MAJOR_VERSION.SUBST_MINOR_VERSION "
+
+wget http://h2o-release.s3.amazonaws.com/sparkling-water/SUBST_BRANCH_NAME/SUBST_MINOR_VERSION/sparkling-water-SUBST_MAJOR_VERSION.SUBST_MINOR_VERSION.zip &
+wait
+
+unzip -o sparkling-water-SUBST_MAJOR_VERSION.SUBST_MINOR_VERSION.zip 1> /dev/null &
+wait
+
+export MASTER="yarn-client"
+