Add Dev-Box and Singer Dashboards, improved Singer, EC2, VPC, and Ste…

…p-Functions (#104)

* output a cloudwatch dashboard url
* add `use_private_subnet` support
* revised step-functions iam policy doc
* private subnets working correctly
* revise cli run-task command
* ml-ops: `var.writeable_buckets` in step-function module
* fix tableau server and ec2 ssh key references
* aws/step-functions module now more generic
* new dev-box catalog module
* capability to have custom cidrs for vpc and subnets
* output route tables from env so they can be appended to
* collapse auto-docs generated files in PR
* cloudwatch dashboard titles
* deterministic ecs-task env sort
* updated terraform-docs to v0.9.1

NOTE: 

* VPC subnetting algorithm has been updated/expanded. Please be careful your subnet is not proposed for destruction.
* To leverage a previous version, override the `source` parameter to use a specific version instead of `master`.
* List of published releases (versions) is here: https://github.com/slalom-ggp/dataops-infra/releases

.gitattributes

@@ -3,3 +3,7 @@
 *       text=auto eol=lf
 # Exception for Windows Batch files:
 *.bat   text      eol=crlf
+
+# By default, GitHub PRs will collapse these auto-generated docs:
+catalog/**/README.md linguist-generated
+components/**/README.md linguist-generated

.gitignore

@@ -29,3 +29,4 @@ tfplan
 
 # Other (Python)
 .mypy_cache
+__pycache__

catalog/aws/dev-box/main.tf

@@ -0,0 +1,64 @@
+/*
+* The `dev-box` catalog module deploys an ECS-backed container which can be used to remotely test
+* or develop using the native cloud environment. Applicable use cases include:
+*
+* * Debugging network firewall and routing rules
+* * Debugging components which can only be run from whitelisted IP ranges
+* * Offloading heavy processing from the developer's local laptop
+* * Mitigating network relability issues when working from WiFi or home networks
+*
+*/
+
+data "aws_availability_zones" "az_list" {}
+
+locals {
+  name_prefix = "${var.name_prefix}devbox-"
+  # container_command = ()
+  ssh_public_key_base64 = filebase64(var.ssh_public_key_filepath)
+}
+
+module "ecs_dev_box_cluster" {
+  source        = "../../../components/aws/ecs-cluster"
+  name_prefix   = local.name_prefix
+  environment   = var.environment
+  resource_tags = var.resource_tags
+}
+
+module "ecs_dev_box_task" {
+  # TODO: use for_each to run jobs in parallel when the feature launches
+  # for_each            = var.taps
+  source              = "../../../components/aws/ecs-task"
+  name_prefix         = local.name_prefix
+  environment         = var.environment
+  resource_tags       = var.resource_tags
+  ecs_cluster_name    = module.ecs_dev_box_cluster.ecs_cluster_name
+  container_image     = module.ecr_image.ecr_image_url_and_tag
+  container_ram_gb    = var.container_ram_gb
+  container_num_cores = var.container_num_cores
+  use_private_subnet  = var.use_private_subnet
+  use_fargate         = true
+  always_on           = true
+  environment_vars = merge(var.settings, {
+    SSH_PUBLIC_KEY_BASE64 = local.ssh_public_key_base64
+  })
+  environment_secrets = var.secrets
+  app_ports           = ["22"]
+  admin_ports         = ["22"]
+}
+
+module "ecr_image" {
+  # TODO: use for_each to run jobs in parallel when the feature launches
+  # for_each            = var.taps
+  source               = "../../../components/aws/ecr-image"
+  name_prefix          = local.name_prefix
+  environment          = var.environment
+  resource_tags        = var.resource_tags
+  aws_credentials_file = var.aws_credentials_file
+
+  repository_name   = "devbox"
+  tag               = "latest"
+  source_image_path = "${path.module}/resources"
+  build_args = {
+    source_image = var.source_image
+  }
+}

catalog/aws/dev-box/outputs.tf

@@ -0,0 +1,19 @@
+output "summary" {
+  description = "Summary of resources created by this module."
+  value       = <<EOF
+
+
+Dev Box Summary:
+
+ - ECS Tasks URL:  https://console.aws.amazon.com/ecs/home?region=${var.environment.aws_region}#/clusters/${module.ecs_dev_box_cluster.ecs_cluster_name}/tasks
+ - Logging URL:    ${module.ecs_dev_box_task.ecs_logging_url}
+ - Uploaded image: ${module.ecr_image.ecr_image_url_and_tag}
+ - Connect to remote Dev Box using SSH:
+     ssh -o StrictHostKeyChecking=no -i "${coalesce(var.ssh_private_key_filepath, "n\\a")}" root@<public-ip>
+ - Command to test docker image locally:
+     docker run --rm -it --entrypoint bash ${module.ecr_image.ecr_image_url_and_tag}
+ - Command to host the SSH server locally:
+     docker run --rm -it -e SSH_PUBLIC_KEY_BASE64=${local.ssh_public_key_base64} ${module.ecr_image.ecr_image_url_and_tag}
+
+EOF
+}

catalog/aws/dev-box/resources/Dockerfile

@@ -0,0 +1,33 @@
+ARG source_image=dataopstk/tapdance:mssql-to-snowflake-test--pre
+FROM ${source_image}
+
+# Install SSH Server
+RUN apt-get update \
+    && apt-get install -y \
+    openssh-server \
+    nano \
+    && mkdir -p /var/run/sshd
+
+EXPOSE 22
+
+# Syntax of next command requires bash, not supported in sh
+SHELL ["/bin/bash", "-c"]
+
+RUN echo $'#!/bin/bash \
+    \n\
+    \nif [ -z "$SSH_PUBLIC_KEY_BASE64" ]; then\
+    \n\techo "Missing SSH public key in the SSH_PUBLIC_KEY_BASE64 env variable."\
+    \n\texit 1\
+    \nfi\
+    \nmkdir -p ~/.ssh\
+    \n\
+    \necho "Installing SSH public key from SSH_PUBLIC_KEY env variable..."\
+    \necho $SSH_PUBLIC_KEY_BASE64 | base64 --decode > ~/.ssh/authorized_keys\
+    \nunset SSH_PUBLIC_KEY_BASE64\
+    \n\
+    \necho "Starting the SSH daemon..."\
+    \n/usr/sbin/sshd -D\
+    \n' >> ./bootstrap.sh
+RUN chmod +x ./bootstrap.sh
+
+ENTRYPOINT [ "./bootstrap.sh" ]

catalog/aws/dev-box/variables.tf

@@ -0,0 +1,75 @@
+##############################################
+### Standard variables for all AWS modules ###
+##############################################
+
+variable "name_prefix" {
+  description = "Standard `name_prefix` module input."
+  type        = string
+}
+variable "environment" {
+  description = "Standard `environment` module input."
+  type = object({
+    vpc_id          = string
+    aws_region      = string
+    public_subnets  = list(string)
+    private_subnets = list(string)
+  })
+}
+variable "resource_tags" {
+  description = "Standard `resource_tags` module input."
+  type        = map(string)
+}
+
+########################################
+### Custom variables for this module ###
+########################################
+
+variable "source_image" {
+  description = "Required. The docker image to execute in the container (e.g. 'ubuntu:18.04')."
+  type        = string
+}
+variable "settings" {
+  description = "Map of environment variables."
+  type        = map(string)
+  default     = {}
+}
+variable "secrets" {
+  description = "Map of environment secrets."
+  type        = map(string)
+  default     = {}
+}
+variable "container_entrypoint" {
+  description = "Optional. Override the docker image's entrypoint."
+  default     = null
+}
+variable "container_num_cores" {
+  description = "Optional. Specify the number of cores to use in the container."
+  default     = 0.5
+}
+variable "container_ram_gb" {
+  description = "Optional. Specify the amount of RAM to be available to the container."
+  default     = 1
+}
+variable "use_private_subnet" {
+  description = <<EOF
+If True, tasks will use a private subnet and will require a NAT gateway to pull the docker
+image, and for any outbound traffic. If False, tasks will use a public subnet and will
+not require a NAT gateway.
+EOF
+  type        = bool
+  default     = false
+}
+variable "ssh_public_key_filepath" {
+  description = "Optional. Path to a valid public key for SSH connectivity."
+  type        = string
+  default     = null
+}
+variable "ssh_private_key_filepath" {
+  description = "Optional. Path to a valid public key for SSH connectivity."
+  type        = string
+  default     = null
+}
+variable "aws_credentials_file" {
+  description = "Path to the AWS credentials file, used to ensure that the correct credentials are used during upload of the ECR image."
+  type        = string
+}