Tableau and EC2 cluster config, ssh key generation, and bug fixes (#106)

**General Updates:** * renamed branch: master->main **EC2 Updates:** * fixed bug for EC2 file inclusion * improved handling of SSH keypairs when connecting to EC2 instances **Tableau Updates:** * add Tableau clustering capability via 'tableau_join.sh' * auto-allow Tableau cluster IP traffic between nodes (TCP ports 8000-9000) * bump Tableau version to '2020.1' * fixed bug in CIDR info not passed from Tableau->EC2 * add sample 'secrets_folder_protection' resource to auto-restrict access to SSH keys and other credentials on Windows/Linux
slalom · Jul 6, 2020 · 05883cc · 05883cc
1 parent 31a0e9d
commit 05883cc
Show file tree

Hide file tree

Showing 33 changed files with 252 additions and 154 deletions.
diff --git a/.github/workflows/github-release-tags.yml b/.github/workflows/github-release-tags.yml
@@ -3,7 +3,7 @@ name: Auto-Tag Release
 on:
   push:
     branches:
-      - master
+      - main
 
 jobs:
   release_new_tag:

diff --git a/autodocs/build.py b/autodocs/build.py
@@ -97,7 +97,7 @@ def build_index(index_type: str, tf_dir: str, output_file: str, overview_desc: s
     }
     git_url_pattern = "git::{git_repo}/{path}?ref={branch}"
     git_repo = "https://github.com/slalom-ggp/dataops-infra"
-    git_branch = "master"
+    git_branch = "main"
 
     toc_str = ""
     for platform_i, platform in enumerate(content_metadata.keys(), start=1):
@@ -201,7 +201,7 @@ def update_module_docs(
                 markdown_text += DOCS_FOOTER.format(
                     src="\n".join(
                         [
-                            "* [{file}]({repo}/tree/master/{dir}/{file})".format(
+                            "* [{file}]({repo}/tree/main/{dir}/{file})".format(
                                 repo=git_repo,
                                 dir=module_path,
                                 file=os.path.basename(tf_file),

diff --git a/autodocs/templates.py b/autodocs/templates.py
@@ -18,7 +18,7 @@
 
 # ```hcl
 # module "{clean_name}" {
-#     source = "git::{git_repo}/{module_path}?ref=master"
+#     source = "git::{git_repo}/{module_path}?ref=main"
 
 #     // ...
 # }

diff --git a/catalog/aws/ml-ops/main.tf b/catalog/aws/ml-ops/main.tf
@@ -120,7 +120,7 @@ EOF
 }
 
 module "step-functions" {
-  #source                  = "git::https://github.com/slalom-ggp/dataops-infra.git//catalog/aws/data-lake?ref=master"
+  #source                  = "git::https://github.com/slalom-ggp/dataops-infra.git//catalog/aws/data-lake?ref=main"
   source      = "../../../components/aws/step-functions"
   name_prefix = var.name_prefix
   writeable_buckets = [

diff --git a/catalog/aws/tableau-server/README.md b/catalog/aws/tableau-server/README.md
diff --git a/catalog/aws/tableau-server/main.tf b/catalog/aws/tableau-server/main.tf
@@ -13,16 +13,22 @@ locals {
   admin_cidr  = var.admin_cidr
   app_cidr    = length(var.app_cidr) == 0 ? local.admin_cidr : var.app_cidr
   ssh_key_dir = pathexpand("~/.ssh")
-  win_files = flatten([
-    fileset(path.module, "resources/win/*"),
-    fileset(path.module, "resources/*"),
-    ["${var.registration_file}:registration.json"]
-  ])
-  lin_files = flatten([
-    fileset(path.module, "resources/lin/*"),
-    fileset(path.module, "resources/*"),
-    ["${var.registration_file}:registration.json"]
-  ])
+  win_files = [
+    for file in flatten([
+      fileset(abspath(path.module), "resources/win/*"),
+      fileset(abspath(path.module), "resources/*"),
+      ["${var.registration_file}::registration.json"]
+    ]) :
+    substr(file, 0, 4) == "http" ? file : "${abspath(path.module)}/${file}"
+  ]
+  lin_files = [
+    for file in flatten([
+      fileset(abspath(path.module), "resources/lin/*"),
+      fileset(abspath(path.module), "resources/*"),
+      ["${var.registration_file}::registration.json"]
+    ]) :
+    substr(file, 0, 4) == "http" ? file : "${abspath(path.module)}/${file}"
+  ]
   tableau_app_ports = {
     "HTTP/HTTPS" = "80"
     "SSL"        = "443"
@@ -35,41 +41,53 @@ locals {
   }
 }
 
-resource "aws_key_pair" "mykey" {
-  count      = var.ssh_public_key_filepath == null ? 0 : 1
-  key_name   = "${local.name_prefix}ec2-keypair"
-  public_key = file(var.ssh_public_key_filepath)
-}
-
 module "windows_tableau_servers" {
-  source                   = "../../../components/aws/ec2"
-  is_windows               = true
-  name_prefix              = "${local.name_prefix}win-"
-  environment              = var.environment
-  resource_tags            = var.resource_tags
-  num_instances            = var.num_windows_instances
-  instance_type            = var.ec2_instance_type
-  instance_storage_gb      = var.ec2_instance_storage_gb
-  ami_owner                = "amazon" # Canonical
-  ami_name_filter          = "Windows_Server-2016-English-Full-Base-*"
-  admin_ports              = merge(local.tableau_admin_ports, { "RDP" : 3389 })
-  app_ports                = local.tableau_app_ports
-  ssh_key_name             = var.ssh_public_key_filepath == null ? "" : aws_key_pair.mykey[0].key_name
-  ssh_private_key_filepath = var.ssh_private_key_filepath == null ? "" : var.ssh_private_key_filepath
+  source        = "../../../components/aws/ec2"
+  name_prefix   = "${local.name_prefix}win-"
+  environment   = var.environment
+  resource_tags = var.resource_tags
+
+  is_windows          = true
+  num_instances       = var.num_windows_instances
+  file_resources      = local.win_files
+  instance_type       = var.ec2_instance_type
+  instance_storage_gb = var.ec2_instance_storage_gb
+  ami_owner           = "amazon" # Canonical
+  ami_name_filter     = "Windows_Server-2016-English-Full-Base-*"
+
+  admin_ports = merge(local.tableau_admin_ports, { "RDP" : 3389 })
+  admin_cidr  = local.admin_cidr
+  app_ports   = local.tableau_app_ports
+  app_cidr    = local.app_cidr
+  cluster_ports = {
+    "Tableau-Server-Cluster" : "8000-9000"
+  }
+
+  ssh_keypair_name         = var.ssh_keypair_name
+  ssh_private_key_filepath = var.ssh_private_key_filepath
 }
 
 module "linux_tableau_servers" {
-  source                   = "../../../components/aws/ec2"
-  name_prefix              = "${local.name_prefix}lin-"
-  environment              = var.environment
-  resource_tags            = var.resource_tags
-  num_instances            = var.num_linux_instances
-  instance_type            = var.ec2_instance_type
-  instance_storage_gb      = var.ec2_instance_storage_gb
-  ami_owner                = "099720109477" # Canonical
-  ami_name_filter          = "ubuntu/images/hvm-ssd/ubuntu-*-18.04-amd64-server-*"
-  admin_ports              = merge(local.tableau_admin_ports, { "SSH" : 22 })
-  app_ports                = local.tableau_app_ports
-  ssh_key_name             = var.ssh_public_key_filepath == null ? "" : aws_key_pair.mykey[0].key_name
-  ssh_private_key_filepath = var.ssh_private_key_filepath == null ? "" : var.ssh_private_key_filepath
+  source        = "../../../components/aws/ec2"
+  name_prefix   = "${local.name_prefix}lin-"
+  environment   = var.environment
+  resource_tags = var.resource_tags
+
+  num_instances       = var.num_linux_instances
+  instance_type       = var.ec2_instance_type
+  instance_storage_gb = var.ec2_instance_storage_gb
+  ami_owner           = "099720109477" # Canonical
+  ami_name_filter     = "ubuntu/images/hvm-ssd/ubuntu-*-18.04-amd64-server-*"
+  file_resources      = local.lin_files
+
+  admin_ports = merge(local.tableau_admin_ports, { "SSH" : 22 })
+  admin_cidr  = local.admin_cidr
+  app_ports   = local.tableau_app_ports
+  app_cidr    = local.app_cidr
+  cluster_ports = {
+    "Tableau-Server-Cluster" : "8000-9000"
+  }
+
+  ssh_keypair_name         = var.ssh_keypair_name
+  ssh_private_key_filepath = var.ssh_private_key_filepath
 }
diff --git a/catalog/aws/tableau-server/outputs.tf b/catalog/aws/tableau-server/outputs.tf
@@ -29,16 +29,11 @@ output "ec2_windows_instance_passwords" {
   description = "The admin passwords for Windows instances (if applicable)."
   value       = module.windows_tableau_servers.windows_instance_passwords
 }
-# output "ssh_key_name" { value = var.num_linux_instances == 0 ? "n/a" : module.linux_tableau_servers[0].key_name }
 # TODO: Detect EC2 Pricing
 # output "ec2_instance_hr_base_price" {
 #   # estimated base price of the (linux) instance type, excluding upcharge for Windows instance and excluding any special pricing or reservation discounts.
 #   value = module.linux_tableau_servers.instance_hr_list_price
 # }
-output "ssh_public_key_filepath" {
-  description = "Local path to public key file for connecting to the server via SSH."
-  value       = var.ssh_public_key_filepath
-}
 output "ssh_private_key_filepath" {
   description = "Local path to private key file for connecting to the server via SSH."
   value       = var.ssh_private_key_filepath

diff --git a/...leau-server/resources/lin/userdata_lin.sh → ...tableau-server/resources/lin/bootstrap.sh b/...leau-server/resources/lin/userdata_lin.sh → ...tableau-server/resources/lin/bootstrap.sh
@@ -8,6 +8,9 @@ mkdir -p /home/ubuntu/tableau/
 LOG_FILE="/home/ubuntu/tableau/setuplog.txt"
 exec > >(tee -a $LOG_FILE) 2>&1 # Log everything to setuplog.txt
 
+export TABLEAU_VERSION_STRING="2020-1-0"
+export TABLEAU_INSTALL_SOURCE="s3://tableau-quickstart/tableau-server-${TABLEAU_VERSION_STRING}_amd64.deb"
+
 echo Configuring SSH banner...
 BANNER="/etc/motd"
 sudo touch $BANNER && chmod 777 $BANNER
@@ -18,6 +21,7 @@ figlet "Welcome to $PROJECT!" >> $BANNER
 set -e
 echo "------------------------------------------------------" >> $BANNER
 echo "Welcome to the Tableau Server at $IP!" >> $BANNER
+echo -e "    EC2 Node Index:      $EC2_NODE_INDEX" >> $BANNER
 echo -e "    TSM Config:          https://$IP:8850/" >> $BANNER
 echo -e "    Tableau Server:      http://$IP/" >> $BANNER
 echo -e "    Default Admin Acct:  tabadmin:tabadmin" >> $BANNER
@@ -35,6 +39,16 @@ echo -e "                         tsm pending-changes apply" >> $BANNER
 echo -e "                         tsm initialize" >> $BANNER
 echo -e "    Configure https:     bash ./ssl_setup.sh bi.mydomain.com" >> $BANNER
 echo "------------------------------------------------------" >> $BANNER
+echo -e "Cluster setup steps:     https://help.tableau.com/current/server-linux/en-us/install_additional_nodes.htm" >> $BANNER
+echo -e "  On Leader Node:      " >> $BANNER
+echo -e "    1. Get bootstrap file:    tsm topology nodes get-bootstrap-file --file ~/bootstrap.sh" >> $BANNER
+echo -e "    2. Print, then copy:      cat ~/bootstrap.sh" >> $BANNER
+echo -e "  On Secondary Nodes:  " >> $BANNER
+echo -e "    3. Create file and paste: nano ~/bootstrap.sh" >> $BANNER
+echo -e "    4. Run join script:       ~/tableau/tableau_join.sh" >> $BANNER
+echo -e "                        OR:   cd /opt/tableau/tableau_server/packages/scripts.*" >> $BANNER
+echo -e "                              sudo ./initialize-tsm --accepteula -a ubuntu -b ~/bootstrap.json" >> $BANNER
+echo "------------------------------------------------------" >> $BANNER
 
 cd /home/ubuntu
 set -v # log verbose
@@ -50,6 +64,10 @@ sudo apt-get update
 sudo apt-get -y install gdebi-core
 sudo apt-get install -y certbot
 
+echo "Configuring locale in the OS..."
+export LANG=en_US.UTF-8
+sudo localectl set-locale LANG=en_US.UTF-8
+
 echo "Installing AWS CLI..."
 snap install aws-cli --classic
 echo "Updating PATH to avoid restart..."
@@ -61,40 +79,39 @@ echo "Showing contents of S3 bucket..."
 $AWS s3 ls s3://tableau-quickstart --no-sign-request
 
 echo "Downloading scripts from S3..."
-mkdir -p /home/ubuntu/tableau
+mkdir -p /home/ubuntu/tableau/installer
 $AWS s3 cp s3://tableau-quickstart/automated-installer tableau/ --no-sign-request
 $AWS s3 cp s3://tableau-quickstart/manageFirewallPorts.py tableau/ --no-sign-request
 $AWS s3 cp s3://tableau-quickstart/SilentInstaller.py tableau/ --no-sign-request
 $AWS s3 cp s3://tableau-quickstart/ScriptedInstaller.py tableau/ --no-sign-request
 
-echo "Downloading installers from S3..."
-$AWS s3 cp s3://tableau-quickstart/tableau-server-2018_amd64.deb tableau/ --no-sign-request
-$AWS s3 cp s3://tableau-quickstart/tableau-server_amd64.deb tableau/ --no-sign-request
+echo "Downloading Postgres installer from S3..."
 $AWS s3 cp s3://tableau-quickstart/tableau-postgresql-odbc_9.5.3_amd64.deb tableau/ --no-sign-request
 
+echo "Downloading Tableau installer from S3..."
+$AWS s3 cp ${TABLEAU_INSTALL_SOURCE} tableau/installer/ --no-sign-request
+
 cd tableau
 
-echo "Installing Tableau Server..."
-sudo gdebi -n tableau-server_amd64.deb
+echo "Showing downloaded installer:"""
+ls installer/* -la
 
-echo "Initializing TSM (Tableau Services Manager)..."
-cd /opt/tableau/tableau_server/packages/scripts.*
-SCRIPTS_DIR=$(pwd)
-export LANG=en_US.UTF-8
-sudo localectl set-locale LANG=en_US.UTF-8
-sudo $SCRIPTS_DIR/initialize-tsm --accepteula -a ubuntu
-source /etc/profile.d/tableau_server.sh  # Updates PATH
-tsm licenses activate -t # Activate 14-day Trial
+echo "Installing Tableau Server..."
+sudo gdebi -n installer/tableau-server-*.deb
 
 echo "Creating 'tabadmin' account..."
 sudo adduser --disabled-password --gecos "Tableau Admin,,," tabadmin
-sudo usermod -G tsmadmin -a tabadmin
 echo "tabadmin:tabadmin" | sudo chpasswd
 
-echo ------------
-echo -e "TSM is initialized and taking requests at: https://$IP:8850/"
-
-echo "Beginning tableau_setup.sh..."
 cd /home/ubuntu/tableau
 sudo chmod 777 *
-./tableau_setup.sh
+if [[ $EC2_NODE_INDEX -eq 0 ]]; then
+    echo "Running on primary node (EC2_NODE_INDEX=$EC2_NODE_INDEX)."
+    echo "Beginning tableau_setup.sh..."
+    ./tableau_setup.sh
+else
+    echo "Running on secondary node (EC2_NODE_INDEX=$EC2_NODE_INDEX)."
+    echo "Skipping tableau_setup.sh..."
+    echo "To join this node to the cluster, generate a file called ~/boostrap.json from TSM on the leader node and then run:"
+    echo " > tableau/tableau_join.sh"
+fi
diff --git a/catalog/aws/tableau-server/resources/lin/tableau_join.sh b/catalog/aws/tableau-server/resources/lin/tableau_join.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+set -ev
+
+exec > >(tee -a /home/ubuntu/tableau_setup_log.txt) 2>&1
+
+echo "Step 1: Initializing TSM with bootstrap.json..."
+source /etc/profile.d/tableau_server.sh  # Update PATH
+set +e
+cd /opt/tableau/tableau_server/packages/scripts.*
+sudo -H ./initialize-tsm --accepteula -a ubuntu -b ~/bootstrap.json
+set -e
diff --git a/catalog/aws/tableau-server/resources/lin/tableau_setup.sh b/catalog/aws/tableau-server/resources/lin/tableau_setup.sh
@@ -4,19 +4,25 @@ set -ev
 exec > >(tee -a /home/ubuntu/tableau_setup_log.txt) 2>&1
 
 echo "Step 1: Initializing TSM (Tableau Services Manager)..."
-cd /opt/tableau/tableau_server/packages/scripts.*
 source /etc/profile.d/tableau_server.sh  # Update PATH
 set +e
-export LANG=en_US.UTF-8
-sudo localectl set-locale LANG=en_US.UTF-8
-sudo -H initialize-tsm --accepteula
+cd /opt/tableau/tableau_server/packages/scripts.*
+sudo -H ./initialize-tsm --accepteula -a ubuntu
 set -e
 
+echo "Adding 'tabadmin' account to 'tsmadmin' group..."
+sudo usermod -G tsmadmin -a tabadmin
+
+echo ------------
+echo -e "TSM is initialized and taking requests at: https://$IP:8850/"
+
 echo "Step 2: Activating Tableau Server (optional)..."
 cd /home/ubuntu/tableau
 export TSM=/opt/tableau/tableau_server/packages/customer*/tsm
-tsm licenses activate -t  # 14-day Trial
-# tsm licenses activate -k <product key>  # License Key
+export TABCMD=/opt/tableau/tableau_server/packages/customer*/tabcmd
+
+$TSM licenses activate -t                  # 14-day Trial
+# $TSM licenses activate -k <product key>  # License Key
 
 echo "Installing SSL for HTTPS..."
 # Uses domain name from www.freenom.com
@@ -25,23 +31,18 @@ echo "Installing SSL for HTTPS..."
 # Ref: https://certbot-dns-route53.readthedocs.io/en/stable/
 
 echo "Step 3: Registering Tableau Server..."
-# Register Tableau
 sudo -H $TSM register --template > registration-template.json
-# nano registration.json
 $TSM register --file registration.json
 
 echo "Step 4: Initializing Tableau Server..."
-# Initialize Tableau
 echo "{\"configEntities\":{\"identityStore\":{\"_type\":\"identityStoreType\",\"type\":\"local\"}}}" > identity_store.json
 $TSM settings import -f identity_store.json
 $TSM pending-changes apply
 $TSM initialize
 
 echo "Step 5: Starting Tableau Server (`date +'%T %Z'`)..."
-# Start Tableau
-
 $TSM start
-tabcmd initialuser --server 'localhost:80' --username 'tabadmin' --password 'tabadmin'
+$TABCMD initialuser --server 'localhost:80' --username 'tabadmin' --password 'tabadmin'
 
 export IP=$(curl ifconfig.me)
 

diff --git a/...eau-server/resources/win/userdata_win.bat → ...ableau-server/resources/win/bootstrap.bat b/...eau-server/resources/win/userdata_win.bat → ...ableau-server/resources/win/bootstrap.bat
diff --git a/catalog/aws/tableau-server/variables.tf b/catalog/aws/tableau-server/variables.tf
@@ -44,7 +44,7 @@ EOF
 }
 variable "ec2_instance_type" {
   description = "Optional. Overrides the Tableau Server instance type."
-  default     = "m4.4xlarge"
+  default     = "m5.4xlarge"
 }
 variable "ec2_instance_storage_gb" {
   description = "The amount of storage to provision on each instance, in GB."
@@ -78,8 +78,8 @@ variable "windows_https_domain" {
   description = "The https domain if the Windows instances should use HTTPS."
   default     = ""
 }
-variable "ssh_public_key_filepath" {
-  description = "Optional. Path to a valid public key for SSH connectivity."
+variable "ssh_keypair_name" {
+  description = "Optional. Name of SSH Keypair in AWS."
   type        = string
   default     = null
 }