Terraform

.gitignore

@@ -127,3 +127,6 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+# Terraform
+.terraform

Dockerfile

@@ -1,18 +1,23 @@
 # Stage 0 - Create from Python3.9.7 image
 FROM python:3.9.7-slim-buster as stage0
 
-# Stage 2 - Create virtual environment and install dependencies
+# Stage 1 - Debian dependencies
 FROM stage0 as stage1
+RUN apt update \
+    && DEBIAN_FRONTEND=noninteractive apt install -y curl zip python3-dev build-essential libhdf5-serial-dev netcdf-bin libnetcdf-dev
+
+# Stage 2 - Create virtual environment and install dependencies
+FROM stage1 as stage2
 COPY requirements.txt /app/requirements.txt
 RUN /usr/local/bin/python3 -m venv /app/env
 RUN /app/env/bin/pip install -r /app/requirements.txt
 
-# Stage 1 - Copy Validation code
-FROM stage1 as stage2
+# Stage 3 - Copy Validation code
+FROM stage2 as stage3
 COPY ./val /app/val/
 
-# Stage 3 - Execute algorithm
-FROM stage2 as stage3
+# Stage 4 - Execute algorithm
+FROM stage3 as stage4
 COPY validation_confluence.py /app/validation_confluence.py
 LABEL version="1.0" \
 	description="Containerized MOI algorithm." \

README.md

@@ -2,8 +2,33 @@
 
 Validation serves as a way to validate results in the Confluence workflow. It compares gage data and produces stats and hydrographs for results that match both in reach identifier and temporally.
 
-# installation
+## installation
 
-# setup
+## setup
 
-# execution
+## execution
+
+## deployment
+
+There is a script to deploy the Docker container image and Terraform AWS infrastructure found in the `deploy` directory.
+
+Script to deploy Terraform and Docker image AWS infrastructure
+
+REQUIRES:
+
+- jq (<https://jqlang.github.io/jq/>)
+- docker (<https://docs.docker.com/desktop/>) > version Docker 1.5
+- AWS CLI (<https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html>)
+- Terraform (<https://developer.hashicorp.com/terraform/tutorials/aws-get-started/install-cli>)
+
+Command line arguments:
+
+[1] registry: Registry URI
+[2] repository: Name of repository to create
+[3] prefix: Prefix to use for AWS resources associated with environment deploying to
+[4] s3_state_bucket: Name of the S3 bucket to store Terraform state in (no need for s3:// prefix)
+[5] profile: Name of profile used to authenticate AWS CLI commands
+
+Example usage: ``./deploy.sh "account-id.dkr.ecr.region.amazonaws.com" "container-image-name" "prefix-for-environment" "s3-state-bucket-name" "confluence-named-profile"`
+
+Note: Run the script from the deploy directory.

deploy/deploy-ecr.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+#
+# Script to deploy a container image to an AWS Lambda Function
+#
+# REQUIRES:
+#   jq (https://jqlang.github.io/jq/)
+#   docker (https://docs.docker.com/desktop/) > version Docker 1.5
+#   AWS CLI (https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html)
+#
+# Command line arguments:
+# [1] registry: Registry URI
+# [2] repository: Name of repository to create
+# [3] prefix: Prefix for environment deploying to
+# [4] profile: Name of profile used to authenticate AWS CLI commands
+# 
+# Example usage: ./deploy-ecr.sh "account-id.dkr.ecr.region.amazonaws.com" "container-image-name" "confluence-dev1" "confluence-named-profile"
+
+REGISTRY=$1
+IMAGE_NAME=$2
+PREFIX=$3
+PROFILE=$4
+
+REPOSITORY=$PREFIX-$IMAGE_NAME
+
+# ECR Repo
+response=$(aws ecr describe-repositories --repository-names "$REPOSITORY" --profile "$PROFILE" 2>&1)
+if [[ $response == *"RepositoryNotFoundException"* ]]; then
+    echo "Respository does not exist. Creating repository: $REPOSITORY."
+    # Create repo
+    response=$(aws ecr create-repository --repository-name "$REPOSITORY" \
+                --image-tag-mutability "MUTABLE" \
+                --image-scanning-configuration scanOnPush=false \
+                --encryption-configuration encryptionType="AES256" \
+                --profile "$PROFILE" )
+
+    # Test if repo was created
+    status=$(echo "$response" | jq '.repository.repositoryName')
+    status="${status%\"}"    # Remove suffix double quote
+    status="${status#\"}"    # Remove prefix double quote
+    if [[ "$status" == "$REPOSITORY" ]]; then
+        echo "Repository was created."
+    else
+        echo "Respository could not be created."
+        echo "Response: $response"
+        exit 1
+    fi
+else
+    repo=$(echo "$response" | jq '.repositories[0].repositoryName')
+    repo="${repo%\"}"    # Remove suffix double quote
+    repo="${repo#\"}"    # Remove prefix double quote
+    echo "Repository exists: '$REPOSITORY' and will not be created."
+fi
+
+# Login
+docker login -u AWS https://$REGISTRY -p $(aws --profile $PROFILE ecr get-login-password --region us-west-2)
+
+# Build
+cd ..
+docker build -t $REGISTRY/$REPOSITORY .
+
+# # Push
+docker push $REGISTRY/$REPOSITORY
+cd deploy

deploy/deploy.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+#
+# Script to deploy Terraform and Docker image AWS infrastructure
+#
+# REQUIRES:
+#   jq (https://jqlang.github.io/jq/)
+#   docker (https://docs.docker.com/desktop/) > version Docker 1.5
+#   AWS CLI (https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html)
+#   Terraform (https://developer.hashicorp.com/terraform/tutorials/aws-get-started/install-cli)
+#
+# Command line arguments:
+# [1] registry: Registry URI
+# [2] repository: Name of repository to create
+# [3] prefix: Prefix to use for AWS resources associated with environment deploying to
+# [4] s3_state_bucket: Name of the S3 bucket to store Terraform state in (no need for s3:// prefix)
+# [5] profile: Name of profile used to authenticate AWS CLI commands
+# 
+# Example usage: ./deploy.sh "account-id.dkr.ecr.region.amazonaws.com" "container-image-name" "prefix-for-environment" "s3-state-bucket-name" "confluence-named-profile" 
+
+REGISTRY=$1
+REPOSITORY=$2
+PREFIX=$3
+S3_STATE=$4
+PROFILE=$5
+
+
+# Deploy Container Image
+./deploy-ecr.sh $REGISTRY $REPOSITORY $PREFIX $PROFILE
+
+# Deploy Terraform
+cd terraform/
+terraform init -reconfigure -backend-config="bucket=$S3_STATE" -backend-config="key=validation.tfstate" -backend-config="region=us-west-2" -backend-config="profile=$PROFILE"
+terraform apply -var-file="conf.tfvars" -auto-approve
+cd ..

deploy/terraform/conf.tfvars

@@ -0,0 +1,3 @@
+environment           = "dev1"
+prefix                = "confluence-dev1"
+profile               = "confluence-dev1"

deploy/terraform/confluence-validation.tf

@@ -0,0 +1,67 @@
+# Job Definition
+resource "aws_batch_job_definition" "generate_batch_jd_validation" {
+  name                  = "${var.prefix}-validation"
+  type                  = "container"
+  container_properties  = <<CONTAINER_PROPERTIES
+  {
+    "image": "${local.account_id}.dkr.ecr.us-west-2.amazonaws.com/${var.prefix}-validation",
+    "image": "${local.account_id}.dkr.ecr.us-west-2.amazonaws.com/${var.prefix}-validation",
+    "executionRoleArn": "${data.aws_iam_role.exe_role.arn}",
+    "jobRoleArn": "${data.aws_iam_role.job_role.arn}",
+    "fargatePlatformConfiguration": { "platformVersion": "LATEST" },
+    "logConfiguration": {
+      "logDriver" : "awslogs",
+      "options": {
+        "awslogs-group" : "${data.aws_cloudwatch_log_group.cw_log_group.name}"
+      }
+    },
+    "resourceRequirements": [
+      {"type": "MEMORY", "value": "2048"},
+      {"type": "VCPU", "value": "1"}
+    ],
+    "mountPoints": [
+      {
+        "sourceVolume": "input",
+        "containerPath": "/mnt/data/input",
+        "readOnly": true
+      },
+      {
+        "sourceVolume": "offline",
+        "containerPath": "/mnt/data/offline",
+        "readOnly": true
+      },
+      {
+        "sourceVolume": "validation",
+        "containerPath": "/mnt/data/output",
+        "readOnly": false
+      }
+    ],
+    "volumes": [
+      {
+        "name": "input",
+        "efsVolumeConfiguration": {
+          "fileSystemId": "${data.aws_efs_file_system.aws_efs_input.file_system_id}",
+          "rootDirectory": "/"
+        }
+      },
+      {
+        "name": "offline",
+        "efsVolumeConfiguration": {
+          "fileSystemId": "${data.aws_efs_file_system.aws_efs_offline.file_system_id}",
+          "rootDirectory": "/"
+        }
+      },
+      {
+        "name": "validation",
+        "efsVolumeConfiguration": {
+          "fileSystemId": "${data.aws_efs_file_system.aws_efs_validation.file_system_id}",
+          "rootDirectory": "/"
+        }
+      }
+    ]
+  }
+  CONTAINER_PROPERTIES
+  platform_capabilities = ["FARGATE"]
+  propagate_tags        = true
+  tags = { "job_definition": "${var.prefix}-validation" }
+}

deploy/terraform/main.tf

@@ -0,0 +1,57 @@
+terraform {
+  backend "s3" {
+    encrypt = true
+  }
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 4.0"
+    }
+  }
+}
+
+# Configure the AWS Provider
+provider "aws" {
+  default_tags {
+    tags = local.default_tags
+  }
+  region  = var.aws_region
+  profile = var.profile
+}
+
+# Data sources
+data "aws_caller_identity" "current" {}
+
+data "aws_cloudwatch_log_group" "cw_log_group" {
+  name = "/aws/batch/job/${var.prefix}-validation/"
+}
+
+data "aws_efs_file_system" "aws_efs_input" {
+  creation_token = "${var.prefix}-input"
+}
+
+data "aws_efs_file_system" "aws_efs_offline" {
+  creation_token = "${var.prefix}-offline"
+}
+
+data "aws_efs_file_system" "aws_efs_validation" {
+  creation_token = "${var.prefix}-validation"
+}
+
+data "aws_iam_role" "job_role" {
+  name = "${var.prefix}-batch-job-role"
+}
+
+data "aws_iam_role" "exe_role" {
+  name = "${var.prefix}-ecs-exe-task-role"
+}
+
+# Local variables
+locals {
+  account_id = data.aws_caller_identity.current.account_id
+  default_tags = length(var.default_tags) == 0 ? {
+    application : var.app_name,
+    environment : var.environment,
+    version : var.app_version
+  } : var.default_tags
+}

deploy/terraform/variables.tf

@@ -0,0 +1,37 @@
+variable "app_name" {
+  type        = string
+  description = "Application name"
+  default     = "generate"
+}
+
+variable "app_version" {
+  type        = number
+  description = "The application version number"
+  default     = 0.1
+}
+
+variable "aws_region" {
+  type        = string
+  description = "AWS region to deploy to"
+  default     = "us-west-2"
+}
+
+variable "default_tags" {
+  type    = map(string)
+  default = {}
+}
+
+variable "environment" {
+  type        = string
+  description = "The environment in which to deploy to"
+}
+
+variable "prefix" {
+  type        = string
+  description = "Prefix to add to all AWS resources as a unique identifier"
+}
+
+variable "profile" {
+  type        = string
+  description = "Named profile to build infrastructure with"
+}