GitHub action

.github/workflows/deploy.yml

@@ -0,0 +1,151 @@
+# Main build pipeline that verifies, builds, and deploys the software
+name: Build and Deploy
+# Events that trigger the workflow
+on:
+  # Trigger based on push to all branches - TODO
+  # push:
+    # branches:
+      # - 'development'
+      # - 'feature/**'
+      # - 'release/**'
+      # - 'main'
+    # tags-ignore:
+      # - '*'
+  # Run workflow manually from the Actions tab
+  workflow_dispatch:
+    inputs:
+        venue:
+            type: choice
+            description: Venue to deploy to
+            options:
+                - DEV1
+                - DEV2
+                - OPS
+
+# Environment variables
+env:
+  APP_NAME_ENV: 'setfinder'
+
+jobs:
+  build:
+    name: Build and Deploy
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+    steps:
+
+      # DEV1 environment variables
+      - name: Set Environment Variables
+        if: github.event.inputs.venue == 'DEV1'
+        run: |
+          echo "TARGET_ENV=DEV1" >> $GITHUB_ENV
+          echo "PREFIX_ENV=confluence-dev1" >> $GITHUB_ENV
+
+      # DEV2 environment variables
+      - name: Set Environment Variables
+        if: github.event.inputs.venue == 'DEV2'
+        run: |
+          echo "TARGET_ENV=DEV2" >> $GITHUB_ENV
+          echo "PREFIX_ENV=confluence-dev2" >> $GITHUB_ENV
+
+      # OPS environment variables
+      - name: Set Environment Variables
+        if: github.event.inputs.venue == 'OPS'
+        run: |
+          echo "TARGET_ENV=OPS" >> $GITHUB_ENV
+          echo "PREFIX_ENV=confluence-ops" >> $GITHUB_ENV
+
+      # Check out GitHub repo
+      - uses: actions/checkout@v4
+
+      # SNYK IAC scan and report - TODO
+      # - name: Run Snyk IAC to test and report
+        # uses: snyk/actions/iac@master
+        # env:
+            # SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+        # with:
+          # command: test
+          # args: >
+              # --org=${{ secrets.SNYK_ORG_ID }}
+              # --severity-threshold=high
+              # --report
+
+      # SNYK Python
+      # - name: Run Snyk Python to test
+        # uses: snyk/actions/python-3.10@master
+        # env:
+            # SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+        # with:
+          # command: test
+          # args: >
+              # --org=${{ secrets.SNYK_ORG_ID }}
+              # --project-name=${{ github.repository }}
+              # --severity-threshold=high
+              # --fail-on=all
+      # - name: Run Snyk Python to report
+        # uses: snyk/actions/python-3.10@master
+        # env:
+          # SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+        # with:
+          # command: monitor
+          # args: >
+            # --org=${{ secrets.SNYK_ORG_ID }}
+            # --project-name=${{ github.repository }}
+
+      # Configure credentials 
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets[format('AWS_ACCESS_KEY_ID_{0}', env.TARGET_ENV)] }}
+          aws-secret-access-key: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.TARGET_ENV)] }}
+          aws-region: us-west-2
+          mask-aws-account-id: true
+
+      # Login and define registry, repository, and tag names
+      - name: Login to AWS ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v2
+        with:
+          mask-password: 'true'
+      - name: Define ECR registry, repository, and image tag names
+        run : |
+          echo "REGISTRY=${{ steps.login-ecr.outputs.registry }}" >> $GITHUB_ENV
+          echo "REPOSITORY=${PREFIX_ENV}-${APP_NAME_ENV}" >> $GITHUB_ENV
+          echo "IMAGE_TAG=latest" >> $GITHUB_ENV
+
+      # Create ECR repository (if it does not exist)
+      - name: Create AWS ECR Repository
+        run: deploy/deploy-ecr.sh $REGISTRY $REPOSITORY
+
+      # Build and push Docker container image
+      - name: Build and Push to AWS ECR
+        run: |
+          docker build -t $REGISTRY/$REPOSITORY:$IMAGE_TAG .
+          docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG
+
+      # Set up Terraform
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@v3
+
+      - name: Define TF_VAR values
+        run: |
+          echo "TF_VAR_environment=$TARGET_ENV" >> $GITHUB_ENV
+          echo "TF_VAR_prefix=$PREFIX_ENV" >> $GITHUB_ENV
+          echo "TF_VAR_api_key"=${{ secrets['API_KEY'] }}" >> $GITHUB_ENV
+          echo "TF_IN_AUTOMATION=true" >> $GITHUB_ENV
+
+      - name: Initialize Terraform
+        working-directory: terraform/
+        run: |
+          terraform init -reconfigure \
+              -backend-config="bucket=${PREFIX_ENV}-tf-state" \
+              -backend-config="key=${APP_NAME_ENV}.tfstate" \
+              -backend-config="region=${AWS_DEFAULT_REGION}"
+
+      - name: Validate Terraform
+        working-directory: terraform/
+        run: terraform validate -no-color
+
+      # Deploy AWS infrastructure
+      - name: Deploy Terraform
+        working-directory: terraform/
+        run: terraform apply -auto-approve

deploy/deploy-ecr.sh

@@ -1,38 +1,34 @@
 #!/bin/bash
 #
-# Script to deploy a container image to an AWS Lambda Function
-#
-# REQUIRES:
-#   jq (https://jqlang.github.io/jq/)
-#   docker (https://docs.docker.com/desktop/) > version Docker 1.5
-#   AWS CLI (https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html)
+# Script to deploy a container image to an AWS ECR.
 #
 # Command line arguments:
 # [1] registry: Registry URI
 # [2] repository: Name of repository to create
-# [3] prefix: Prefix for environment deploying to
-# [4] profile: Name of profile used to authenticate AWS CLI commands
+# [3] local: Whether script is being run locally
 # 
-# Example usage: ./deploy-ecr.sh "account-id.dkr.ecr.region.amazonaws.com" "container-image-name" "confluence-dev1" "confluence-named-profile"
+# Example usage: ./delpoy-ecr.sh "account-id.dkr.ecr.region.amazonaws.com" "docker-container-image"
 
 REGISTRY=$1
-IMAGE_NAME=$2
-PREFIX=$3
-PROFILE=$4
+REPOSITORY=$2
+IS_LOCAL=$3
 
-REPOSITORY=$PREFIX-$IMAGE_NAME
+# Determine if repo exists
+response=$(aws ecr describe-repositories --repository-names "$REPOSITORY" 2>&1)
+repo=$(echo "$response" | jq '.repositories[0].repositoryName')
+repo="${repo%\"}"    # Remove suffix double quote
+repo="${repo#\"}"    # Remove prefix double quote
 
-# ECR Repo
-response=$(aws ecr describe-repositories --repository-names "$REPOSITORY" --profile "$PROFILE" 2>&1)
-if [[ $response == *"RepositoryNotFoundException"* ]]; then
+if [[ "$repo" == "$REPOSITORY" ]]; then
+    echo "Repository exists: '$REPOSITORY' and will not be created."
+else
+    # Creat repo
     echo "Respository does not exist. Creating repository: $REPOSITORY."
-    # Create repo
     response=$(aws ecr create-repository --repository-name "$REPOSITORY" \
                 --image-tag-mutability "MUTABLE" \
                 --image-scanning-configuration scanOnPush=false \
-                --encryption-configuration encryptionType="AES256" \
-                --profile "$PROFILE" )
-
+                --encryption-configuration encryptionType="AES256" )
+
     # Test if repo was created
     status=$(echo "$response" | jq '.repository.repositoryName')
     status="${status%\"}"    # Remove suffix double quote
@@ -41,23 +37,18 @@ if [[ $response == *"RepositoryNotFoundException"* ]]; then
         echo "Repository was created."
     else
         echo "Respository could not be created."
-        echo "Response: $response"
-        exit 1
     fi
-else
-    repo=$(echo "$response" | jq '.repositories[0].repositoryName')
-    repo="${repo%\"}"    # Remove suffix double quote
-    repo="${repo#\"}"    # Remove prefix double quote
-    echo "Repository exists: '$REPOSITORY' and will not be created."
 fi
 
-# Login
-docker login -u AWS https://$REGISTRY -p $(aws --profile $PROFILE ecr get-login-password --region us-west-2)
+if [[ "$IS_LOCAL" == "true" ]]; then
+    # Login
+    docker login -u AWS https://$REGISTRY -p $(aws ecr get-login-password --region us-west-2)
 
-# Build
-cd ..
-docker build -t $REGISTRY/$REPOSITORY .
+    # Build
+    cd ..
+    docker build -t $REGISTRY/$REPOSITORY .
 
-# # Push
-docker push $REGISTRY/$REPOSITORY
-cd deploy
+    # Push
+    docker push $REGISTRY/$REPOSITORY
+    cd deploy
+fi

deploy/deploy.sh

@@ -13,22 +13,28 @@
 # [2] repository: Name of repository to create
 # [3] prefix: Prefix to use for AWS resources associated with environment deploying to
 # [4] s3_state_bucket: Name of the S3 bucket to store Terraform state in (no need for s3:// prefix)
-# [5] profile: Name of profile used to authenticate AWS CLI commands
+# [5] is_local: Whether the script is executing locally (not in GitHub action): "true" or "false"
+#
+# Note, you need to be "logged in" via `aws configure` in order to deploy to AWS
 # 
-# Example usage: ./deploy.sh "account-id.dkr.ecr.region.amazonaws.com" "container-image-name" "prefix-for-environment" "s3-state-bucket-name" "confluence-named-profile" 
+# Example usage: ./deploy.sh "account-id.dkr.ecr.region.amazonaws.com" "container-image-name" "prefix-for-environment" "s3-state-bucket-name" "is-local"
 
 REGISTRY=$1
-REPOSITORY=$2
+NAME=$2
 PREFIX=$3
 S3_STATE=$4
-PROFILE=$5
+IS_LOCAL=$5
 
+REPOSITORY=$PREFIX-$NAME
 
 # Deploy Container Image
-./deploy-ecr.sh $REGISTRY $REPOSITORY $PREFIX $PROFILE
+cd deploy/
+echo "./deploy-ecr.sh $REGISTRY $REPOSITORY $IS_LOCAL"
+./deploy-ecr.sh $REGISTRY $REPOSITORY $IS_LOCAL
+cd ..
 
 # Deploy Terraform
 cd terraform/
-terraform init -reconfigure -backend-config="bucket=$S3_STATE" -backend-config="key=input.tfstate" -backend-config="region=us-west-2" -backend-config="profile=$PROFILE"
-terraform apply -var-file="conf.tfvars" -auto-approve
+terraform init -reconfigure -backend-config="bucket=$S3_STATE" -backend-config="key=$NAME.tfstate" -backend-config="region=us-west-2"
+terraform apply -auto-approve
 cd ..

requirements.txt

@@ -1,32 +1,18 @@
-aiobotocore==2.13.1
-aiohttp==3.9.5
-aioitertools==0.11.0
-aiosignal==1.3.1
-async-timeout==4.0.3
-attrs==23.2.0
-boto3==1.34.131
-botocore==1.34.131
-certifi==2024.7.4
-cftime==1.6.4
-charset-normalizer==3.3.2
-frozenlist==1.4.1
-fsspec==2024.6.1
-idna==3.7
+boto3==1.35.72
+botocore==1.35.72
+certifi==2024.8.30
+cftime==1.6.4.post1
+charset-normalizer==3.4.0
+idna==3.10
 jmespath==1.0.1
-multidict==6.0.5
-netCDF4==1.7.1.post1
-numpy==2.0.0
-pandas==2.2.2
-pyshp==2.3.1
+netCDF4==1.7.2
+numpy==2.1.3
+pandas==2.2.3
 python-dateutil==2.9.0.post0
-pytz==2024.1
+pytz==2024.2
 requests==2.32.3
-s3fs==2024.6.1
-s3transfer==0.10.2
-scipy==1.14.0
+s3transfer==0.10.4
+scipy==1.14.1
 six==1.16.0
-typing_extensions==4.12.2
-tzdata==2024.1
-urllib3==2.2.2
-wrapt==1.16.0
-yarl==1.9.4
+tzdata==2024.2
+urllib3==2.2.3

deploy/terraform/main.tf → terraform/main.tf

@@ -16,7 +16,6 @@ provider "aws" {
     tags = local.default_tags
   }
   region  = var.aws_region
-  profile = var.profile
 }
 
 # Data sources
@@ -43,7 +42,7 @@ locals {
   account_id = data.aws_caller_identity.current.account_id
   default_tags = length(var.default_tags) == 0 ? {
     application : var.app_name,
-    environment : var.environment,
+    environment : lower(var.environment),
     version : var.app_version
   } : var.default_tags
 }

deploy/terraform/variables.tf → terraform/variables.tf

@@ -35,8 +35,3 @@ variable "prefix" {
   type        = string
   description = "Prefix to add to all AWS resources as a unique identifier"
 }
-
-variable "profile" {
-  type        = string
-  description = "Named profile to build infrastructure with"
-}