Working version of Terraform configs

Signed-off-by: jpolchlo <jpolchlopek@azavea.com>
locationtech · Jan 11, 2018 · d109f28 · d109f28
1 parent 8d4b71c
commit d109f28
Show file tree

Hide file tree

Showing 3 changed files with 36 additions and 10 deletions.
diff --git a/scripts/emr/terraform/bootstrap.sh b/scripts/emr/terraform/bootstrap.sh
@@ -1,5 +1,8 @@
 #!/bin/bash
 
+OAUTH_CLIENT_ID=$1
+OAUTH_CLIENT_SECRET=$2
+
 # Parses a configuration file put in place by EMR to determine the role of this node
 is_master() {
   if [ $(jq '.isMaster' /mnt/var/lib/info/instance.json) = 'true' ]; then
@@ -10,17 +13,18 @@ is_master() {
 }
 
 if is_master; then
-    sudo yum -y update
+    echo "Updating system software ..."
+    sudo yum -y -q update
     curl -sL https://rpm.nodesource.com/setup_6.x | sudo -E bash -
-    sudo yum install -y nodejs
+    sudo yum install -y -q nodejs
 
-    sudo pip-3.4 install jupyter
+    sudo pip-3.4 -q install jupyter
     sudo npm install -g configurable-http-proxy
-    sudo pip-3.4 install jupyterhub
-    sudo pip-3.4 install --upgrade notebook
+    sudo pip-3.4 -q install jupyterhub
+    sudo pip-3.4 -q install --upgrade notebook
 
-    sudo pip-3.4 install sudospawner
-    sudo pip-3.4 install "https://github.com/jupyterhub/oauthenticator/archive/f5e39b1ece62b8d075832054ed3213cc04f85030.zip"
+    sudo pip-3.4 -q install sudospawner
+    sudo pip-3.4 -q install "https://github.com/jupyterhub/oauthenticator/archive/f5e39b1ece62b8d075832054ed3213cc04f85030.zip"
 
     curl -L -o /tmp/jupyter-scala https://raw.githubusercontent.com/jupyter-scala/jupyter-scala/98bac7034f07e3e51d101846953aecbdb7a4bb5d/jupyter-scala
     chmod +x /tmp/jupyter-scala
@@ -58,6 +62,7 @@ user=\$1
 sudo useradd -m -G jupyterhub,hadoop \$user
 sudo -u hdfs hdfs dfs -mkdir /user/\$user
 
+sudo -u \$user /tmp/jupyter-scala
 EOF
     chmod +x /tmp/new_user
     sudo chown root:root /tmp/new_user
@@ -79,4 +84,5 @@ EOF
 
     # Execute
     cd /tmp
+    launch_hub &
 fi
diff --git a/scripts/emr/terraform/emr-spark.tf b/scripts/emr/terraform/emr-spark.tf
@@ -65,11 +65,25 @@ resource "aws_emr_cluster" "emr-spark-cluster" {
   provisioner "file" {
     source      = "bootstrap.sh"
     destination = "/tmp/bootstrap.sh"
+    connection {
+      type        = "ssh"
+      user        = "hadoop"
+      host        = "${aws_emr_cluster.emr-spark-cluster.master_public_dns}"
+      private_key = "${file("${var.pem_path}")}"
+    }
   }
 
-  bootstrap_action {
-    path = "file:///tmp/bootstrap.sh"
-    name = "Install JupyterHub"
+  provisioner "remote-exec" {
+    inline=[
+      "chmod +x /tmp/bootstrap.sh",
+      "/tmp/bootstrap.sh ${var.oauth_client_id} ${var.oauth_client_secret}"
+    ]
+    connection {
+      type        = "ssh"
+      user        = "hadoop"
+      host        = "${aws_emr_cluster.emr-spark-cluster.master_public_dns}"
+      private_key = "${file("${var.pem_path}")}"
+    }
   }
 }
 

diff --git a/scripts/emr/terraform/variables.tf.json b/scripts/emr/terraform/variables.tf.json
@@ -33,5 +33,11 @@
         "default": "anonymous",
         "description": "User name applied to cluster"
     },
+    "oauth_client_id": {
+        "description": "Client ID token for OAuth server"
+    },
+    "oauth_client_secret": {
+        "description": "Client secret token for OAuth server"
+    }
   }
 }