chore: Update CI to move scripts to script folder and ensure CloudWatch logs are deleted before applying

.github/scripts/delete-log-groups.py

@@ -0,0 +1,21 @@
+import os
+import boto3
+
+REGION = os.environ.get('AWS_DEFAULT_REGION', 'us-west-2')
+CLIENT = boto3.client('logs', region_name=REGION)
+
+def delete_log_groups():
+    """Delete all log groups in the region that start with `/aws/eks/`"""
+    response = CLIENT.describe_log_groups(
+        logGroupNamePrefix='/aws/eks/',
+        limit=50
+    )
+
+    for log_group in [log.get('logGroupName') for log in response.get('logGroups', {})]:
+        CLIENT.delete_log_group(
+            logGroupName=log_group
+        )
+
+
+if __name__ == '__main__':
+    delete_log_groups()

.github/scripts/iam-policy-generator.py

@@ -0,0 +1,35 @@
+import json
+import boto3
+import os
+
+iam_actions = []
+s3 = boto3.resource('s3')
+bucket_name = os.getenv('BUCKET_NAME')
+bucket = s3.Bucket(bucket_name)
+bucket_files = [x.key for x in bucket.objects.all()]
+
+# Read all the files from the bucket
+for file in bucket_files:
+    obj = s3.Object(bucket_name, file)
+    f = obj.get()['Body'].read()
+    data = json.loads(f)
+    # Merge all policies actions, keep them unique with 'set'
+    for statement in data['Statement']:
+        iam_actions = list(set(iam_actions + statement['Action']))
+
+# Skeleton IAM policy template , wild card all resources for now.
+template = {
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": [
+            ],
+            "Resource": "*"
+        }
+    ]
+}
+
+# Apply merged actions to the skeleton IAM policy
+template['Statement'][0]['Action'] = sorted(iam_actions)
+print(json.dumps(template, indent=4))

.github/scripts/plan-examples.py

@@ -0,0 +1,30 @@
+import json
+import glob
+import re
+
+
+def get_examples():
+    """
+    Get all Terraform example root directories using their respective `versions.tf`;
+    returning a string formatted json array of the example directories minus those that are excluded
+    """
+    exclude = {
+        'examples/appmesh-mtls',  # excluded until Rout53 is setup
+        'examples/eks-cluster-with-external-dns',  # excluded until Rout53 is setup
+        'examples/fully-private-eks-cluster/vpc',  # skipping until issue #711 is addressed
+        'examples/fully-private-eks-cluster/eks',
+        'examples/fully-private-eks-cluster/add-ons',
+        'examples/ai-ml/ray',  # excluded until #887 is fixed
+    }
+
+    projects = {
+        x.replace('/versions.tf', '')
+        for x in glob.glob('examples/**/versions.tf', recursive=True)
+        if not re.match(r'^.+/_', x)
+    }
+
+    print(json.dumps(list(projects.difference(exclude))))
+
+
+if __name__ == '__main__':
+    get_examples()

.github/workflows/e2e-parallel-destroy.yml

@@ -23,14 +23,16 @@ jobs:
       fail-fast: false
       matrix:
         include:
+          - example_path: examples/agones-game-controller
           - example_path: examples/analytics/emr-on-eks
           - example_path: examples/analytics/spark-k8s-operator
+          - example_path: examples/complete-kubernetes-addons
           - example_path: examples/crossplane
           - example_path: examples/eks-cluster-with-new-vpc
           - example_path: examples/fargate-serverless
-          # - example_path: examples/fully-private-eks-cluster # skipping until issue #711 is addressed
-          - example_path: examples/game-tech/agones-game-controller
           - example_path: examples/gitops/argocd
+          - example_path: examples/grafana-loki
+          - example_path: examples/ipv4-prefix-delegation
           - example_path: examples/ipv6-eks-cluster
           - example_path: examples/karpenter
           - example_path: examples/multi-tenancy-with-teams

.github/workflows/e2e-parallel-full.yml

@@ -27,14 +27,16 @@ jobs:
       fail-fast: false
       matrix:
         include:
+          - example_path: examples/agones-game-controller
           - example_path: examples/analytics/emr-on-eks
           - example_path: examples/analytics/spark-k8s-operator
+          - example_path: examples/complete-kubernetes-addons
           - example_path: examples/crossplane
           - example_path: examples/eks-cluster-with-new-vpc
           - example_path: examples/fargate-serverless
-          # - example_path: examples/fully-private-eks-cluster # skipping until issue #711
-          - example_path: examples/game-tech/agones-game-controller
           - example_path: examples/gitops/argocd
+          - example_path: examples/grafana-loki
+          - example_path: examples/ipv4-prefix-delegation
           - example_path: examples/ipv6-eks-cluster
           - example_path: examples/karpenter
           - example_path: examples/multi-tenancy-with-teams
@@ -56,6 +58,11 @@ jobs:
           role-duration-seconds: 3600
           role-session-name: GithubActions-Session
 
+      - name: Ensure log groups are removed
+        run: |
+          pip3 install boto3
+          python3 .github/workflows/delete-log-groups.py
+
       - name: Iamlive Setup & Run
         run: |
           #!/bin/bash
@@ -139,4 +146,4 @@ jobs:
         id: dirs
         run: |
           pip3 install boto3
-          python3 .github/workflows/iam-policy-generator.py
+          python3 .github/scripts/iam-policy-generator.py

.github/workflows/plan-examples.yml

@@ -29,7 +29,7 @@ jobs:
       - name: Get Terraform directories for evaluation
         id: dirs
         run: |
-          DIRS=$(python3 .github/workflows/plan-examples.py)
+          DIRS=$(python3 .github/scripts/plan-examples.py)
           echo "::set-output name=directories::$DIRS"
 
   plan:

.github/workflows/pre-commit.yaml

@@ -31,7 +31,7 @@ jobs:
 
       - name: Get root directories
         id: dirs
-        uses: clowdhaus/terraform-composite-actions/directories@v1.4.1
+        uses: clowdhaus/terraform-composite-actions/directories@v1.8.0
 
   preCommitMinVersions:
     name: Min TF pre-commit
@@ -70,22 +70,22 @@ jobs:
           restore-keys: ${{ runner.os }}-terraform-
 
       - name: Terraform min/max versions
-        uses: clowdhaus/terraform-min-max@v1.0.7
+        uses: clowdhaus/terraform-min-max@v1.2.0
         if: steps.changes.outputs.src== 'true'
         id: minMax
         with:
           directory: ${{ matrix.directory }}
 
       - name: Pre-commit Terraform ${{ steps.minMax.outputs.minVersion }}
-        uses: clowdhaus/terraform-composite-actions/pre-commit@v1.6.0
+        uses: clowdhaus/terraform-composite-actions/pre-commit@v1.8.0
         # Run only validate pre-commit check on min version supported
         if: ${{ matrix.directory !=  '.' && steps.changes.outputs.src== 'true' }}
         with:
           terraform-version: ${{ steps.minMax.outputs.minVersion }}
           args: 'terraform_validate --color=always --show-diff-on-failure --files ${{ matrix.directory }}/*'
 
       - name: Pre-commit Terraform ${{ steps.minMax.outputs.minVersion }}
-        uses: clowdhaus/terraform-composite-actions/pre-commit@v1.6.0
+        uses: clowdhaus/terraform-composite-actions/pre-commit@v1.8.0
         # Run only validate pre-commit check on min version supported
         if: ${{ matrix.directory ==  '.' && steps.changes.outputs.src== 'true' }}
         with:
@@ -128,11 +128,11 @@ jobs:
 
       - name: Terraform min/max versions
         id: minMax
-        uses: clowdhaus/terraform-min-max@v1.0.7
+        uses: clowdhaus/terraform-min-max@v1.2.0
         if: steps.changes.outputs.src== 'true'
 
       - name: Pre-commit Terraform ${{ steps.minMax.outputs.maxVersion }}
-        uses: clowdhaus/terraform-composite-actions/pre-commit@v1.6.0
+        uses: clowdhaus/terraform-composite-actions/pre-commit@v1.8.0
         if: steps.changes.outputs.src== 'true'
         with:
           terraform-version: ${{ steps.minMax.outputs.maxVersion }}

examples/game-tech/agones-game-controller/main.tf → examples/agones-game-controller/main.tf

@@ -40,7 +40,7 @@ locals {
 #---------------------------------------------------------------
 
 module "eks_blueprints" {
-  source = "../../.."
+  source = "../.."
 
   cluster_name    = local.name
   cluster_version = "1.23"
@@ -88,7 +88,7 @@ module "eks_blueprints" {
 }
 
 module "eks_blueprints_kubernetes_addons" {
-  source = "../../..//modules/kubernetes-addons"
+  source = "../../modules/kubernetes-addons"
 
   eks_cluster_id               = module.eks_blueprints.eks_cluster_id
   eks_cluster_endpoint         = module.eks_blueprints.eks_cluster_endpoint