diff --git a/.gitignore b/.gitignore index b6e4761..c356e15 100644 --- a/.gitignore +++ b/.gitignore @@ -127,3 +127,6 @@ dmypy.json # Pyre type checker .pyre/ + +# Terraform +.terraform \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index e98c800..6112d18 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,18 +1,23 @@ # Stage 0 - Create from Python3.9.7 image FROM python:3.9.7-slim-buster as stage0 -# Stage 2 - Create virtual environment and install dependencies +# Stage 1 - Debian dependencies FROM stage0 as stage1 +RUN apt update \ + && DEBIAN_FRONTEND=noninteractive apt install -y curl zip python3-dev build-essential libhdf5-serial-dev netcdf-bin libnetcdf-dev + +# Stage 2 - Create virtual environment and install dependencies +FROM stage1 as stage2 COPY requirements.txt /app/requirements.txt RUN /usr/local/bin/python3 -m venv /app/env RUN /app/env/bin/pip install -r /app/requirements.txt -# Stage 1 - Copy Validation code -FROM stage1 as stage2 +# Stage 3 - Copy Validation code +FROM stage2 as stage3 COPY ./val /app/val/ -# Stage 3 - Execute algorithm -FROM stage2 as stage3 +# Stage 4 - Execute algorithm +FROM stage3 as stage4 COPY validation_confluence.py /app/validation_confluence.py LABEL version="1.0" \ description="Containerized MOI algorithm." \ diff --git a/README.md b/README.md index 86d1f72..5190a89 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,33 @@ Validation serves as a way to validate results in the Confluence workflow. It compares gage data and produces stats and hydrographs for results that match both in reach identifier and temporally. -# installation +## installation -# setup +## setup -# execution \ No newline at end of file +## execution + +## deployment + +There is a script to deploy the Docker container image and Terraform AWS infrastructure found in the `deploy` directory. + +Script to deploy Terraform and Docker image AWS infrastructure + +REQUIRES: + +- jq () +- docker () > version Docker 1.5 +- AWS CLI () +- Terraform () + +Command line arguments: + +[1] registry: Registry URI +[2] repository: Name of repository to create +[3] prefix: Prefix to use for AWS resources associated with environment deploying to +[4] s3_state_bucket: Name of the S3 bucket to store Terraform state in (no need for s3:// prefix) +[5] profile: Name of profile used to authenticate AWS CLI commands + +Example usage: ``./deploy.sh "account-id.dkr.ecr.region.amazonaws.com" "container-image-name" "prefix-for-environment" "s3-state-bucket-name" "confluence-named-profile"` + +Note: Run the script from the deploy directory. diff --git a/deploy/deploy-ecr.sh b/deploy/deploy-ecr.sh new file mode 100755 index 0000000..3e59970 --- /dev/null +++ b/deploy/deploy-ecr.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# +# Script to deploy a container image to an AWS Lambda Function +# +# REQUIRES: +# jq (https://jqlang.github.io/jq/) +# docker (https://docs.docker.com/desktop/) > version Docker 1.5 +# AWS CLI (https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) +# +# Command line arguments: +# [1] registry: Registry URI +# [2] repository: Name of repository to create +# [3] prefix: Prefix for environment deploying to +# [4] profile: Name of profile used to authenticate AWS CLI commands +# +# Example usage: ./deploy-ecr.sh "account-id.dkr.ecr.region.amazonaws.com" "container-image-name" "confluence-dev1" "confluence-named-profile" + +REGISTRY=$1 +IMAGE_NAME=$2 +PREFIX=$3 +PROFILE=$4 + +REPOSITORY=$PREFIX-$IMAGE_NAME + +# ECR Repo +response=$(aws ecr describe-repositories --repository-names "$REPOSITORY" --profile "$PROFILE" 2>&1) +if [[ $response == *"RepositoryNotFoundException"* ]]; then + echo "Respository does not exist. Creating repository: $REPOSITORY." + # Create repo + response=$(aws ecr create-repository --repository-name "$REPOSITORY" \ + --image-tag-mutability "MUTABLE" \ + --image-scanning-configuration scanOnPush=false \ + --encryption-configuration encryptionType="AES256" \ + --profile "$PROFILE" ) + + # Test if repo was created + status=$(echo "$response" | jq '.repository.repositoryName') + status="${status%\"}" # Remove suffix double quote + status="${status#\"}" # Remove prefix double quote + if [[ "$status" == "$REPOSITORY" ]]; then + echo "Repository was created." + else + echo "Respository could not be created." + echo "Response: $response" + exit 1 + fi +else + repo=$(echo "$response" | jq '.repositories[0].repositoryName') + repo="${repo%\"}" # Remove suffix double quote + repo="${repo#\"}" # Remove prefix double quote + echo "Repository exists: '$REPOSITORY' and will not be created." +fi + +# Login +docker login -u AWS https://$REGISTRY -p $(aws --profile $PROFILE ecr get-login-password --region us-west-2) + +# Build +cd .. +docker build -t $REGISTRY/$REPOSITORY . + +# # Push +docker push $REGISTRY/$REPOSITORY +cd deploy diff --git a/deploy/deploy.sh b/deploy/deploy.sh new file mode 100755 index 0000000..8d1cf65 --- /dev/null +++ b/deploy/deploy.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# +# Script to deploy Terraform and Docker image AWS infrastructure +# +# REQUIRES: +# jq (https://jqlang.github.io/jq/) +# docker (https://docs.docker.com/desktop/) > version Docker 1.5 +# AWS CLI (https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) +# Terraform (https://developer.hashicorp.com/terraform/tutorials/aws-get-started/install-cli) +# +# Command line arguments: +# [1] registry: Registry URI +# [2] repository: Name of repository to create +# [3] prefix: Prefix to use for AWS resources associated with environment deploying to +# [4] s3_state_bucket: Name of the S3 bucket to store Terraform state in (no need for s3:// prefix) +# [5] profile: Name of profile used to authenticate AWS CLI commands +# +# Example usage: ./deploy.sh "account-id.dkr.ecr.region.amazonaws.com" "container-image-name" "prefix-for-environment" "s3-state-bucket-name" "confluence-named-profile" + +REGISTRY=$1 +REPOSITORY=$2 +PREFIX=$3 +S3_STATE=$4 +PROFILE=$5 + + +# Deploy Container Image +./deploy-ecr.sh $REGISTRY $REPOSITORY $PREFIX $PROFILE + +# Deploy Terraform +cd terraform/ +terraform init -reconfigure -backend-config="bucket=$S3_STATE" -backend-config="key=validation.tfstate" -backend-config="region=us-west-2" -backend-config="profile=$PROFILE" +terraform apply -var-file="conf.tfvars" -auto-approve +cd .. \ No newline at end of file diff --git a/deploy/terraform/.terraform.lock.hcl b/deploy/terraform/.terraform.lock.hcl new file mode 100644 index 0000000..b3d29da --- /dev/null +++ b/deploy/terraform/.terraform.lock.hcl @@ -0,0 +1,26 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "4.67.0" + constraints = "~> 4.0" + hashes = [ + "h1:5Zfo3GfRSWBaXs4TGQNOflr1XaYj6pRnVJLX5VAjFX4=", + "h1:dCRc4GqsyfqHEMjgtlM1EympBcgTmcTkWaJmtd91+KA=", + "zh:0843017ecc24385f2b45f2c5fce79dc25b258e50d516877b3affee3bef34f060", + "zh:19876066cfa60de91834ec569a6448dab8c2518b8a71b5ca870b2444febddac6", + "zh:24995686b2ad88c1ffaa242e36eee791fc6070e6144f418048c4ce24d0ba5183", + "zh:4a002990b9f4d6d225d82cb2fb8805789ffef791999ee5d9cb1fef579aeff8f1", + "zh:559a2b5ace06b878c6de3ecf19b94fbae3512562f7a51e930674b16c2f606e29", + "zh:6a07da13b86b9753b95d4d8218f6dae874cf34699bca1470d6effbb4dee7f4b7", + "zh:768b3bfd126c3b77dc975c7c0e5db3207e4f9997cf41aa3385c63206242ba043", + "zh:7be5177e698d4b547083cc738b977742d70ed68487ce6f49ecd0c94dbf9d1362", + "zh:8b562a818915fb0d85959257095251a05c76f3467caa3ba95c583ba5fe043f9b", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:9c385d03a958b54e2afd5279cd8c7cbdd2d6ca5c7d6a333e61092331f38af7cf", + "zh:b3ca45f2821a89af417787df8289cb4314b273d29555ad3b2a5ab98bb4816b3b", + "zh:da3c317f1db2469615ab40aa6baba63b5643bae7110ff855277a1fb9d8eb4f2c", + "zh:dc6430622a8dc5cdab359a8704aec81d3825ea1d305bbb3bbd032b1c6adfae0c", + "zh:fac0d2ddeadf9ec53da87922f666e1e73a603a611c57bcbc4b86ac2821619b1d", + ] +} diff --git a/deploy/terraform/conf.tfvars b/deploy/terraform/conf.tfvars new file mode 100644 index 0000000..b6e8346 --- /dev/null +++ b/deploy/terraform/conf.tfvars @@ -0,0 +1,3 @@ +environment = "dev1" +prefix = "confluence-dev1" +profile = "confluence-dev1" \ No newline at end of file diff --git a/deploy/terraform/confluence-validation.tf b/deploy/terraform/confluence-validation.tf new file mode 100644 index 0000000..bae45fa --- /dev/null +++ b/deploy/terraform/confluence-validation.tf @@ -0,0 +1,67 @@ +# Job Definition +resource "aws_batch_job_definition" "generate_batch_jd_validation" { + name = "${var.prefix}-validation" + type = "container" + container_properties = <1: + print('here is index we are workign with ', index) + # if its more than one, we take it down to a scalar + if len(index[0])>1: warnings.warn('multiple gages for this reach. Selecting closest meanQ to model') #pull model q for this reach modelindex=np.where(self.reach_id == sos['reaches']['reach_id'][:].filled(np.nan)) @@ -178,7 +187,7 @@ def get_gage_q(self, sos, gage_type): glt.append(len(t[t>0])) if np.isnan(model_q): - #when model is nan, choose lingest timeseries + #when model is nan, choose longest timeseries index=np.array(index[np.argmax(np.array(glt))]) if np.size(index)>1: warnings.warn('model was nan and times are same length') @@ -190,18 +199,18 @@ def get_gage_q(self, sos, gage_type): if np.size(index)>1: warnings.warn('identical mean q values') index=index[0] - + elif len(index[0]) == 1: + index = index[0][0] - gage_data = {} - if np.size(index[0]) != 0: + if np.isscalar(index): if self.run_type == "constrained": # if constraind check and see if the gage selected at this index is a 0 if gage["CAL"][:][index] == 1: - warnings.warn('gauge found was calibration.. This is an unconstrained run, so it will not be used for validation') + warnings.warn('gauge found was calibration.. This is a constrained run, so it will not be used for validation') return gage_data gage_data["type"] = gage_type gage_data["q"] = gage[f"{gage_type}_q"][index][:].filled(np.nan) @@ -276,6 +285,7 @@ def is_offline_valid(self, offline_data): for v in offline_data.values(): if np.count_nonzero(~np.isnan(v)) == 0: invalid += 1 if invalid == self.NUM_ALGOS: + print('OFFLINE IS NOT VALID') return False else: return True