Main -> Val

.github/build-vars.sh

@@ -13,12 +13,14 @@ set_value() {
   if [ ! -z "${!varname}" ]; then
     echo "Setting $varname"
     echo "${varname}=${!varname}" >> $GITHUB_ENV
+    echo "${varname}=${!varname}" >> $GITHUB_OUTPUT
   fi
 }
 
 set_name() {
   varname=${1}
   echo "BRANCH_SPECIFIC_VARNAME_$varname=${branch_name//-/_}_$varname" >> $GITHUB_ENV
+  echo "BRANCH_SPECIFIC_VARNAME_$varname=${branch_name//-/_}_$varname" >> $GITHUB_OUTPUT
 }
 
 action=${1}

.github/waf-controller.sh

@@ -0,0 +1,132 @@
+#!/usr/bin/env bash
+
+CIRCUIT_BREAKER=10
+AWS_RETRY_ERROR=254
+AWS_THROTTLING_EXCEPTION=252
+#0, 1, 2 are the levels of debug, with 0 being off
+DEBUG=1
+
+set -o pipefail -o nounset -u
+
+case ${1} in
+  append)
+    OP=append
+    ;;
+  set)
+    OP=set
+    ;;
+  *)
+    echo "Error:  unkown operation"
+    echo "Usage: ${0} [append|set]" && exit 1
+    ;;
+esac
+
+shift
+NAME="${1}"
+ID="${2}"
+shift; shift
+RUNNER_CIDRS="${@}"
+
+[[ $DEBUG -ge 1 ]] && echo "Inputs:  NAME ${NAME}, ID ${ID}, RUNNER_CIDRS ${RUNNER_CIDRS}"
+
+#Exponential backoff with jitter
+jitter() {
+  #.5 seconds
+  SHORTEST=50
+  #10 seconds
+  LONGEST=1000
+  DIV=100
+  EXP=$(perl -e "use bigint; print $SHORTEST**$1")
+  MIN=$(($EXP>$LONGEST ? $LONGEST : $EXP))
+  RND=$(shuf -i$SHORTEST-$MIN -n1)
+  perl -e "print $RND/$DIV"
+}
+
+#Attempt to avoid resource contention from the start
+sleep $(jitter $(shuf -i1-10 -n1))
+
+for ((i=1; i <= $CIRCUIT_BREAKER; i++)); do
+  #This loop is ONLY for retrying if the retries exceeded exception is thrown
+  for ((j=1; j <= $CIRCUIT_BREAKER; j++)); do
+    #Read WAF configuration from AWS
+    WAF_CONFIG=$(aws wafv2 get-ip-set --scope CLOUDFRONT --id ${ID} --name ${NAME} 2>&1)
+    CMD_CD=$?
+    [[ $DEBUG -ge 1 ]] && echo "AWS CLI Read Response Code:  ${CMD_CD}"
+    [[ $DEBUG -ge 2 ]] && echo "AWS CLI Read Response:  ${WAF_CONFIG}"
+
+    #If the retries exceeded error code is returned, try again, otherwise exit the loop
+    [[ $CMD_CD -eq $AWS_RETRY_ERROR ]] || break
+
+    SLEEP_FOR=$(jitter ${j})
+    echo "CLI retries exceed.  Waiting for ${SLEEP_FOR} seconds to execute read again...$({j})"
+    sleep ${SLEEP_FOR}
+  done
+
+  #Unable to get the lock tocken and IP set so there isn't any point in attempting the write op
+  [[ $j -ge $CIRCUIT_BREAKER ]] && echo “Attempts to read WAF IPSet exceeded” && sleep $(jitter ${i}) && continue
+
+  #The loop was short circuited with an error code other than 0, so something is wrong
+  [[ $CMD_CD -eq 0 ]] || ( echo "An unexpected read error occurred:  ${CMD_CD}" && exit 2 )
+
+  echo "Read was successful."
+
+  if [ ${OP} == "append" ]; then
+    ##If this is used to whitelist individual ips or cidrs, using an additive approach is what is required
+    #Parse out IP set addresses to array
+    IP_ADDRESSES=($(jq -r '.IPSet.Addresses | .[]' <<< ${WAF_CONFIG}))
+
+    #If CIDR is already present in IP set, eject
+    grep -q $RUNNER_CIDRS <<< ${IP_ADDRESSES}
+    [[ $? -ne 0 ]] || ( echo "CIDR is present in IP Set." && exit 0 )
+
+    #Add runner CIDR to array
+    IP_ADDRESSES+=("$RUNNER_CIDRS")
+  else 
+    ##If this is used to hard set the IP set, just clobber it
+    IP_ADDRESSES=("$RUNNER_CIDRS")
+  fi
+
+  #Stringify IPs
+  STRINGIFIED=$(echo $(IFS=" " ; echo "${IP_ADDRESSES[*]}"))
+  [[ $DEBUG -ge 2 ]] && echo "Ip Addresses:  ${STRINGIFIED}"
+
+  #Parse out optimistic concurrency control token
+  OCC_TOKEN=$(jq -r '.LockToken' <<< ${WAF_CONFIG})
+  [[ $DEBUG -ge 2 ]] && echo "LockToken:  ${OCC_TOKEN}"
+
+  #This loop is ONLY for retrying if the retries exceeded exception is thrown
+  for ((k=1; k <= $CIRCUIT_BREAKER; k++)); do
+    #Write updated WAF configuration to AWS
+    OUTPUT=$(aws wafv2 update-ip-set --scope CLOUDFRONT --id ${ID} --name ${NAME} --lock-token ${OCC_TOKEN} --addresses ${STRINGIFIED} 2>&1)
+    CMD_CD=$?
+    [[ $DEBUG -ge 1 ]] && echo "AWS CLI Write Response Code:  ${CMD_CD}"
+    [[ $DEBUG -ge 2 ]] && echo "AWS CLI Write Response:  ${OUTPUT}"
+
+    #If the retries exceeded error code is returned, try again, otherwise exit the loop
+    [[ $CMD_CD -eq $AWS_RETRY_ERROR ]] || break
+    #If WAFOptimisticLockException error code is returned, exit the loop
+    [[ "$OUTPUT" =~ "WAFOptimisticLockException" ]] && break
+
+    SLEEP_FOR=$(jitter ${k})
+    echo "CLI retries exceed.  Waiting for ${SLEEP_FOR} seconds to execute write again...(${k})"
+    sleep ${SLEEP_FOR}
+  done
+
+  [[ $CMD_CD -ne 0 ]] || break
+  #Still not having success, so try again
+
+  echo "Exit Code:  ${CMD_CD}"
+
+  SLEEP_FOR=$(jitter ${i})
+  echo "Waiting for ${SLEEP_FOR} seconds to execute main loop again...(${i})"
+  sleep ${SLEEP_FOR}
+done
+
+[[ $DEBUG -ge 1 ]] && echo "Attempts to update ip set:  $i"
+
+[[ $i -gt $CIRCUIT_BREAKER ]] && echo “Attempts to update WAF IPSet exceeded, exiting.” && exit 2
+
+echo "Applied the IP Set successfully."
+
+#Things should not have made it this far without being able to successfully write the IP Set
+exit $CMD_CD

.github/workflows/codeql-analysis.yml

@@ -27,7 +27,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Initialize CodeQL
         uses: github/codeql-action/init@v2

.github/workflows/deploy.yml

@@ -21,6 +21,16 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+      - name: 'Setup jq'
+        uses: dcarbone/install-jq-action@v2.1.0
+        with:
+          version: '${{ inputs.version }}'
+          force: '${{ inputs.force }}'
+      - name: 'Check jq'
+        # language=sh
+        run: |
+          which jq
+          jq --version
       - name: set variable values
         run: ./.github/build-vars.sh set_values
         env:
@@ -118,6 +128,10 @@ jobs:
           MSG_MINIMAL: actions url,commit,ref
     outputs:
       application_endpoint: ${{ steps.endpoint.outputs.application_endpoint}}
+      BRANCH_SPECIFIC_VARNAME_AWS_DEFAULT_REGION: ${{ steps.set_names.outputs.BRANCH_SPECIFIC_VARNAME_AWS_DEFAULT_REGION }}
+      BRANCH_SPECIFIC_VARNAME_AWS_OIDC_ROLE_TO_ASSUME: ${{ steps.set_names.outputs.BRANCH_SPECIFIC_VARNAME_AWS_OIDC_ROLE_TO_ASSUME }}
+      # ipset_name: ${{ steps.fetch-ip-set-info.outputs.IPSET_NAME }}
+      # ipset_id: ${{ steps.fetch-ip-set-info.outputs.IPSET_ID }}
 
   e2e-test:
     name: E2E Integration Tests
@@ -127,6 +141,72 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+
+      - name: Configure AWS credentials for GitHub Actions
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets[env.BRANCH_SPECIFIC_VARNAME_AWS_OIDC_ROLE_TO_ASSUME] || secrets.AWS_OIDC_ROLE_TO_ASSUME }}
+          aws-region: ${{ secrets[env.BRANCH_SPECIFIC_VARNAME_AWS_DEFAULT_REGION] || secrets.AWS_DEFAULT_REGION }}
+
+      - name: set branch_name # Some integrations (Dependabot & Snyk) build very long branch names. This is a switch to make long branch names shorter.
+        run: |
+          BRANCH_NAME=$(./.github/setBranchName.sh ${{ github.ref_name }})
+          echo "branch_name=${BRANCH_NAME}" >> $GITHUB_ENV
+
+      - name: set branch specific variable names
+        id: set_names
+        run: ./.github/build-vars.sh set_names
+
+      - name: set variable values
+        id: set_values
+        run: ./.github/build-vars.sh set_values
+        env:
+          AWS_DEFAULT_REGION: ${{ secrets[env.BRANCH_SPECIFIC_VARNAME_AWS_DEFAULT_REGION] || secrets.AWS_DEFAULT_REGION }}
+          AWS_OIDC_ROLE_TO_ASSUME: ${{ secrets[env.BRANCH_SPECIFIC_VARNAME_AWS_OIDC_ROLE_TO_ASSUME] || secrets.AWS_OIDC_ROLE_TO_ASSUME }}
+          STAGE_PREFIX: ${{ secrets.STAGE_PREFIX }}
+
+      - name: Get Github Actions CIDR Blocks
+        id: get-gha-cidrs
+        shell: bash
+        run: |
+          #! /bin/bash
+          GHA_RESP=$(curl --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' https://api.github.com/meta)
+          echo "Response for GHA runner CIDR blocks:  $GHA_RESP"
+          IPV4_CIDR_ARR=($(echo $GHA_RESP | jq -r '.actions | .[]' | grep -v ':'))
+          GHA_CIDRS_IPV4=$(echo $(IFS=" "; echo ${IPV4_CIDR_ARR[*]}))
+          echo "GHA_CIDRS_IPV4=$GHA_CIDRS_IPV4" >> $GITHUB_OUTPUT
+
+      - name: Generate IP Set Name
+        id: gen-ip-set-name
+        run: |
+          #!/bin/bash
+          STAGE_GH_IPSET_NAME=$STAGE_PREFIX$branch_name-gh-ipset
+          echo "Github IP Set name:  $STAGE_GH_IPSET_NAME"
+          echo "STAGE_GH_IPSET_NAME=$STAGE_GH_IPSET_NAME" >> $GITHUB_OUTPUT
+
+      - name: Fetch AWS IP set Metadata
+        id: fetch-ip-set-info
+        run: |
+          #!/bin/bash
+          # Fetch AWS IP set ARNs using AWS CLI and store them in a variable
+          AWS_IP_SET_INFO=$(aws wafv2 list-ip-sets --scope=CLOUDFRONT)
+          # Store the IP set ARNs in an output variable using GITHUB_OUTPUT
+          IPSET_NAME=${{ steps.gen-ip-set-name.outputs.STAGE_GH_IPSET_NAME }}
+          IPSET=$(jq '.IPSets | map(select(.Name == "'${IPSET_NAME}'")) | .[]' <<< ${AWS_IP_SET_INFO})
+          [ -z "$IPSET" ] && echo "IP Set with name ${IPSET_NAME} was not located.  Exiting..." && exit 1
+          echo "IP Set metadata:  ${IPSET}"
+          #Get Values from the IP SET
+          IPSET_ID=$(jq -r '.Id' <<< ${IPSET})
+          echo "IPSET_ARN=$IPSET_ARN" >> $GITHUB_OUTPUT
+          echo "IPSET_NAME=$IPSET_NAME" >> $GITHUB_OUTPUT
+          echo "IPSET_ID=$IPSET_ID" >> $GITHUB_OUTPUT
+
+      - name: Update IP Set
+        id: update-ip-set
+        run: ./.github/waf-controller.sh set ${{ steps.fetch-ip-set-info.outputs.IPSET_NAME }} ${{ steps.fetch-ip-set-info.outputs.IPSET_ID }} ${{ steps.get-gha-cidrs.outputs.GHA_CIDRS_IPV4 }}
+        env:
+          AWS_RETRY_MODE: adaptive
+          AWS_MAX_ATTEMPTS: 10
 
       - name: Run Cypress Tests
         uses: cypress-io/github-action@v5
@@ -153,6 +233,9 @@ jobs:
           path: |
             tests/cypress/screenshots/
             tests/cypress/videos/
+    outputs:
+      ipset_name: ${{ steps.fetch-ip-set-info.outputs.IPSET_NAME }}
+      ipset_id: ${{ steps.fetch-ip-set-info.outputs.IPSET_ID }} 
 
   a11y-tests:
     name: E2E A11y Tests
@@ -187,3 +270,27 @@ jobs:
           path: |
             tests/cypress/screenshots/
             tests/cypress/videos/
+
+  cleanup:
+    name: Delist GHA Runner CIDR Blocks
+    if: ${{ github.ref != 'refs/heads/production' }}
+    runs-on: ubuntu-latest
+    needs:
+      - deploy
+      - a11y-tests
+      - e2e-test
+    env:
+      SLS_DEPRECATION_DISABLE: "*" # Turn off deprecation warnings in the pipeline
+    steps:
+      - uses: actions/checkout@v3
+      - name: Configure AWS credentials for GitHub Actions
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets[env.BRANCH_SPECIFIC_VARNAME_AWS_OIDC_ROLE_TO_ASSUME] || secrets.AWS_OIDC_ROLE_TO_ASSUME }}
+          aws-region: ${{ secrets[env.BRANCH_SPECIFIC_VARNAME_AWS_DEFAULT_REGION] || secrets.AWS_DEFAULT_REGION }}
+      - name: cleanup-ip-list
+        id: reset-ip-set
+        run: ./.github/waf-controller.sh set ${{ needs.e2e-test.outputs.ipset_name }} ${{ needs.e2e-test.outputs.ipset_id }} '[]'
+        env:
+          AWS_RETRY_MODE: adaptive
+          AWS_MAX_ATTEMPTS: 10

.github/workflows/git-secrets.yaml

@@ -4,7 +4,7 @@ jobs:
   gitleaks-scan:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Run gitleaks docker
         uses: docker://zricethezav/gitleaks
         with:

.github/workflows/pull-request.yml

@@ -6,7 +6,7 @@ jobs:
   prchecks:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v3
       - uses: pre-commit/action@v3.0.0
       - uses: actions/setup-node@v3
@@ -17,7 +17,7 @@ jobs:
   assignAuthor:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Assign PR to Creator
         run: |
           if [ "$PR_AUTHOR_TYPE" != "Bot" ]

.github/workflows/scan_security-hub-jira-integration.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: Check out repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v4
         with:

.github/workflows/scan_snyk-jira-integration.yml

@@ -16,7 +16,7 @@ jobs:
     if: github.event_name == 'pull_request'
     steps:
       - name: Check out repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install Snyk and Run Snyk test
         run: |
@@ -31,7 +31,7 @@ jobs:
     if: github.event_name == 'schedule'
     steps:
       - name: Check out repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install Snyk and Run Snyk test
         run: |

package.json

@@ -23,7 +23,8 @@
   "homepage": "https://github.com/Enterprise-CMCS/macpro-mdct-mcr#readme",
   "dependencies": {
     "fast-xml-parser": "4.2.5",
-    "loader-utils": "2.0.3"
+    "loader-utils": "2.0.3",
+    "@enterprise-cmcs/serverless-waf-plugin": "^1.3.2"
   },
   "devDependencies": {
     "@types/dotenv": "^8.2.0",

services/app-api/handlers/templates/fetch.ts

@@ -25,6 +25,6 @@ export const fetchTemplate = handler(async (event, _context) => {
     Expires: 60,
     Key: key,
   };
-  const url = s3Lib.getSignedDownloadUrl(params);
+  const url = await s3Lib.getSignedDownloadUrl(params);
   return { status: StatusCodes.SUCCESS, body: url };
 });