diff --git a/.gitignore b/.gitignore
index b6e4761..c356e15 100644
--- a/.gitignore
+++ b/.gitignore
@@ -127,3 +127,6 @@ dmypy.json
# Pyre type checker
.pyre/
+
+# Terraform
+.terraform
\ No newline at end of file
diff --git a/Dockerfile b/Dockerfile
index e98c800..6112d18 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,18 +1,23 @@
# Stage 0 - Create from Python3.9.7 image
FROM python:3.9.7-slim-buster as stage0
-# Stage 2 - Create virtual environment and install dependencies
+# Stage 1 - Debian dependencies
FROM stage0 as stage1
+RUN apt update \
+ && DEBIAN_FRONTEND=noninteractive apt install -y curl zip python3-dev build-essential libhdf5-serial-dev netcdf-bin libnetcdf-dev
+
+# Stage 2 - Create virtual environment and install dependencies
+FROM stage1 as stage2
COPY requirements.txt /app/requirements.txt
RUN /usr/local/bin/python3 -m venv /app/env
RUN /app/env/bin/pip install -r /app/requirements.txt
-# Stage 1 - Copy Validation code
-FROM stage1 as stage2
+# Stage 3 - Copy Validation code
+FROM stage2 as stage3
COPY ./val /app/val/
-# Stage 3 - Execute algorithm
-FROM stage2 as stage3
+# Stage 4 - Execute algorithm
+FROM stage3 as stage4
COPY validation_confluence.py /app/validation_confluence.py
LABEL version="1.0" \
description="Containerized MOI algorithm." \
diff --git a/README.md b/README.md
index 86d1f72..5190a89 100644
--- a/README.md
+++ b/README.md
@@ -2,8 +2,33 @@
Validation serves as a way to validate results in the Confluence workflow. It compares gage data and produces stats and hydrographs for results that match both in reach identifier and temporally.
-# installation
+## installation
-# setup
+## setup
-# execution
\ No newline at end of file
+## execution
+
+## deployment
+
+There is a script to deploy the Docker container image and Terraform AWS infrastructure found in the `deploy` directory.
+
+Script to deploy Terraform and Docker image AWS infrastructure
+
+REQUIRES:
+
+- jq ()
+- docker () > version Docker 1.5
+- AWS CLI ()
+- Terraform ()
+
+Command line arguments:
+
+[1] registry: Registry URI
+[2] repository: Name of repository to create
+[3] prefix: Prefix to use for AWS resources associated with environment deploying to
+[4] s3_state_bucket: Name of the S3 bucket to store Terraform state in (no need for s3:// prefix)
+[5] profile: Name of profile used to authenticate AWS CLI commands
+
+Example usage: ``./deploy.sh "account-id.dkr.ecr.region.amazonaws.com" "container-image-name" "prefix-for-environment" "s3-state-bucket-name" "confluence-named-profile"`
+
+Note: Run the script from the deploy directory.
diff --git a/deploy/deploy-ecr.sh b/deploy/deploy-ecr.sh
new file mode 100755
index 0000000..3e59970
--- /dev/null
+++ b/deploy/deploy-ecr.sh
@@ -0,0 +1,63 @@
+#!/bin/bash
+#
+# Script to deploy a container image to an AWS Lambda Function
+#
+# REQUIRES:
+# jq (https://jqlang.github.io/jq/)
+# docker (https://docs.docker.com/desktop/) > version Docker 1.5
+# AWS CLI (https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html)
+#
+# Command line arguments:
+# [1] registry: Registry URI
+# [2] repository: Name of repository to create
+# [3] prefix: Prefix for environment deploying to
+# [4] profile: Name of profile used to authenticate AWS CLI commands
+#
+# Example usage: ./deploy-ecr.sh "account-id.dkr.ecr.region.amazonaws.com" "container-image-name" "confluence-dev1" "confluence-named-profile"
+
+REGISTRY=$1
+IMAGE_NAME=$2
+PREFIX=$3
+PROFILE=$4
+
+REPOSITORY=$PREFIX-$IMAGE_NAME
+
+# ECR Repo
+response=$(aws ecr describe-repositories --repository-names "$REPOSITORY" --profile "$PROFILE" 2>&1)
+if [[ $response == *"RepositoryNotFoundException"* ]]; then
+ echo "Respository does not exist. Creating repository: $REPOSITORY."
+ # Create repo
+ response=$(aws ecr create-repository --repository-name "$REPOSITORY" \
+ --image-tag-mutability "MUTABLE" \
+ --image-scanning-configuration scanOnPush=false \
+ --encryption-configuration encryptionType="AES256" \
+ --profile "$PROFILE" )
+
+ # Test if repo was created
+ status=$(echo "$response" | jq '.repository.repositoryName')
+ status="${status%\"}" # Remove suffix double quote
+ status="${status#\"}" # Remove prefix double quote
+ if [[ "$status" == "$REPOSITORY" ]]; then
+ echo "Repository was created."
+ else
+ echo "Respository could not be created."
+ echo "Response: $response"
+ exit 1
+ fi
+else
+ repo=$(echo "$response" | jq '.repositories[0].repositoryName')
+ repo="${repo%\"}" # Remove suffix double quote
+ repo="${repo#\"}" # Remove prefix double quote
+ echo "Repository exists: '$REPOSITORY' and will not be created."
+fi
+
+# Login
+docker login -u AWS https://$REGISTRY -p $(aws --profile $PROFILE ecr get-login-password --region us-west-2)
+
+# Build
+cd ..
+docker build -t $REGISTRY/$REPOSITORY .
+
+# # Push
+docker push $REGISTRY/$REPOSITORY
+cd deploy
diff --git a/deploy/deploy.sh b/deploy/deploy.sh
new file mode 100755
index 0000000..8d1cf65
--- /dev/null
+++ b/deploy/deploy.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+#
+# Script to deploy Terraform and Docker image AWS infrastructure
+#
+# REQUIRES:
+# jq (https://jqlang.github.io/jq/)
+# docker (https://docs.docker.com/desktop/) > version Docker 1.5
+# AWS CLI (https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html)
+# Terraform (https://developer.hashicorp.com/terraform/tutorials/aws-get-started/install-cli)
+#
+# Command line arguments:
+# [1] registry: Registry URI
+# [2] repository: Name of repository to create
+# [3] prefix: Prefix to use for AWS resources associated with environment deploying to
+# [4] s3_state_bucket: Name of the S3 bucket to store Terraform state in (no need for s3:// prefix)
+# [5] profile: Name of profile used to authenticate AWS CLI commands
+#
+# Example usage: ./deploy.sh "account-id.dkr.ecr.region.amazonaws.com" "container-image-name" "prefix-for-environment" "s3-state-bucket-name" "confluence-named-profile"
+
+REGISTRY=$1
+REPOSITORY=$2
+PREFIX=$3
+S3_STATE=$4
+PROFILE=$5
+
+
+# Deploy Container Image
+./deploy-ecr.sh $REGISTRY $REPOSITORY $PREFIX $PROFILE
+
+# Deploy Terraform
+cd terraform/
+terraform init -reconfigure -backend-config="bucket=$S3_STATE" -backend-config="key=validation.tfstate" -backend-config="region=us-west-2" -backend-config="profile=$PROFILE"
+terraform apply -var-file="conf.tfvars" -auto-approve
+cd ..
\ No newline at end of file
diff --git a/deploy/terraform/.terraform.lock.hcl b/deploy/terraform/.terraform.lock.hcl
new file mode 100644
index 0000000..b3d29da
--- /dev/null
+++ b/deploy/terraform/.terraform.lock.hcl
@@ -0,0 +1,26 @@
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/hashicorp/aws" {
+ version = "4.67.0"
+ constraints = "~> 4.0"
+ hashes = [
+ "h1:5Zfo3GfRSWBaXs4TGQNOflr1XaYj6pRnVJLX5VAjFX4=",
+ "h1:dCRc4GqsyfqHEMjgtlM1EympBcgTmcTkWaJmtd91+KA=",
+ "zh:0843017ecc24385f2b45f2c5fce79dc25b258e50d516877b3affee3bef34f060",
+ "zh:19876066cfa60de91834ec569a6448dab8c2518b8a71b5ca870b2444febddac6",
+ "zh:24995686b2ad88c1ffaa242e36eee791fc6070e6144f418048c4ce24d0ba5183",
+ "zh:4a002990b9f4d6d225d82cb2fb8805789ffef791999ee5d9cb1fef579aeff8f1",
+ "zh:559a2b5ace06b878c6de3ecf19b94fbae3512562f7a51e930674b16c2f606e29",
+ "zh:6a07da13b86b9753b95d4d8218f6dae874cf34699bca1470d6effbb4dee7f4b7",
+ "zh:768b3bfd126c3b77dc975c7c0e5db3207e4f9997cf41aa3385c63206242ba043",
+ "zh:7be5177e698d4b547083cc738b977742d70ed68487ce6f49ecd0c94dbf9d1362",
+ "zh:8b562a818915fb0d85959257095251a05c76f3467caa3ba95c583ba5fe043f9b",
+ "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425",
+ "zh:9c385d03a958b54e2afd5279cd8c7cbdd2d6ca5c7d6a333e61092331f38af7cf",
+ "zh:b3ca45f2821a89af417787df8289cb4314b273d29555ad3b2a5ab98bb4816b3b",
+ "zh:da3c317f1db2469615ab40aa6baba63b5643bae7110ff855277a1fb9d8eb4f2c",
+ "zh:dc6430622a8dc5cdab359a8704aec81d3825ea1d305bbb3bbd032b1c6adfae0c",
+ "zh:fac0d2ddeadf9ec53da87922f666e1e73a603a611c57bcbc4b86ac2821619b1d",
+ ]
+}
diff --git a/deploy/terraform/conf.tfvars b/deploy/terraform/conf.tfvars
new file mode 100644
index 0000000..b6e8346
--- /dev/null
+++ b/deploy/terraform/conf.tfvars
@@ -0,0 +1,3 @@
+environment = "dev1"
+prefix = "confluence-dev1"
+profile = "confluence-dev1"
\ No newline at end of file
diff --git a/deploy/terraform/confluence-validation.tf b/deploy/terraform/confluence-validation.tf
new file mode 100644
index 0000000..bae45fa
--- /dev/null
+++ b/deploy/terraform/confluence-validation.tf
@@ -0,0 +1,67 @@
+# Job Definition
+resource "aws_batch_job_definition" "generate_batch_jd_validation" {
+ name = "${var.prefix}-validation"
+ type = "container"
+ container_properties = <1:
+ print('here is index we are workign with ', index)
+ # if its more than one, we take it down to a scalar
+ if len(index[0])>1:
warnings.warn('multiple gages for this reach. Selecting closest meanQ to model')
#pull model q for this reach
modelindex=np.where(self.reach_id == sos['reaches']['reach_id'][:].filled(np.nan))
@@ -178,7 +187,7 @@ def get_gage_q(self, sos, gage_type):
glt.append(len(t[t>0]))
if np.isnan(model_q):
- #when model is nan, choose lingest timeseries
+ #when model is nan, choose longest timeseries
index=np.array(index[np.argmax(np.array(glt))])
if np.size(index)>1:
warnings.warn('model was nan and times are same length')
@@ -190,18 +199,18 @@ def get_gage_q(self, sos, gage_type):
if np.size(index)>1:
warnings.warn('identical mean q values')
index=index[0]
-
+ elif len(index[0]) == 1:
+ index = index[0][0]
-
gage_data = {}
- if np.size(index[0]) != 0:
+ if np.isscalar(index):
if self.run_type == "constrained":
# if constraind check and see if the gage selected at this index is a 0
if gage["CAL"][:][index] == 1:
- warnings.warn('gauge found was calibration.. This is an unconstrained run, so it will not be used for validation')
+ warnings.warn('gauge found was calibration.. This is a constrained run, so it will not be used for validation')
return gage_data
gage_data["type"] = gage_type
gage_data["q"] = gage[f"{gage_type}_q"][index][:].filled(np.nan)
@@ -276,6 +285,7 @@ def is_offline_valid(self, offline_data):
for v in offline_data.values():
if np.count_nonzero(~np.isnan(v)) == 0: invalid += 1
if invalid == self.NUM_ALGOS:
+ print('OFFLINE IS NOT VALID')
return False
else:
return True