From 494bfd7e9c82e1b5788164cfa9d089c8f28d82cd Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Mon, 31 Oct 2022 15:30:09 -0400 Subject: [PATCH 1/2] chore: Update CI to move scripts to script folder and ensure CloudWatch logs are deleted before applying --- .github/scripts/delete-log-groups.py | 21 +++++++++++++++ .github/scripts/iam-policy-generator.py | 35 +++++++++++++++++++++++++ .github/scripts/plan-examples.py | 30 +++++++++++++++++++++ .github/workflows/e2e-parallel-full.yml | 7 ++++- .github/workflows/plan-examples.yml | 2 +- .github/workflows/pre-commit.yaml | 12 ++++----- 6 files changed, 99 insertions(+), 8 deletions(-) create mode 100644 .github/scripts/delete-log-groups.py create mode 100644 .github/scripts/iam-policy-generator.py create mode 100644 .github/scripts/plan-examples.py diff --git a/.github/scripts/delete-log-groups.py b/.github/scripts/delete-log-groups.py new file mode 100644 index 0000000000..4935b4a07e --- /dev/null +++ b/.github/scripts/delete-log-groups.py @@ -0,0 +1,21 @@ +import os +import boto3 + +REGION = os.environ.get('AWS_DEFAULT_REGION', 'us-west-2') +CLIENT = boto3.client('logs', region_name=REGION) + +def delete_log_groups(): + """Delete all log groups in the region that start with `/aws/eks/`""" + response = CLIENT.describe_log_groups( + logGroupNamePrefix='/aws/eks/', + limit=50 + ) + + for log_group in [log.get('logGroupName') for log in response.get('logGroups', {})]: + CLIENT.delete_log_group( + logGroupName=log_group + ) + + +if __name__ == '__main__': + delete_log_groups() diff --git a/.github/scripts/iam-policy-generator.py b/.github/scripts/iam-policy-generator.py new file mode 100644 index 0000000000..6b9022d3a7 --- /dev/null +++ b/.github/scripts/iam-policy-generator.py @@ -0,0 +1,35 @@ +import json +import boto3 +import os + +iam_actions = [] +s3 = boto3.resource('s3') +bucket_name = os.getenv('BUCKET_NAME') +bucket = s3.Bucket(bucket_name) +bucket_files = [x.key for x in bucket.objects.all()] + +# Read all the files from the bucket +for file in bucket_files: + obj = s3.Object(bucket_name, file) + f = obj.get()['Body'].read() + data = json.loads(f) + # Merge all policies actions, keep them unique with 'set' + for statement in data['Statement']: + iam_actions = list(set(iam_actions + statement['Action'])) + +# Skeleton IAM policy template , wild card all resources for now. +template = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + ], + "Resource": "*" + } + ] +} + +# Apply merged actions to the skeleton IAM policy +template['Statement'][0]['Action'] = sorted(iam_actions) +print(json.dumps(template, indent=4)) diff --git a/.github/scripts/plan-examples.py b/.github/scripts/plan-examples.py new file mode 100644 index 0000000000..4c24a6bf95 --- /dev/null +++ b/.github/scripts/plan-examples.py @@ -0,0 +1,30 @@ +import json +import glob +import re + + +def get_examples(): + """ + Get all Terraform example root directories using their respective `versions.tf`; + returning a string formatted json array of the example directories minus those that are excluded + """ + exclude = { + 'examples/appmesh-mtls', # excluded until Rout53 is setup + 'examples/eks-cluster-with-external-dns', # excluded until Rout53 is setup + 'examples/fully-private-eks-cluster/vpc', # skipping until issue #711 is addressed + 'examples/fully-private-eks-cluster/eks', + 'examples/fully-private-eks-cluster/add-ons', + 'examples/ai-ml/ray', # excluded until #887 is fixed + } + + projects = { + x.replace('/versions.tf', '') + for x in glob.glob('examples/**/versions.tf', recursive=True) + if not re.match(r'^.+/_', x) + } + + print(json.dumps(list(projects.difference(exclude)))) + + +if __name__ == '__main__': + get_examples() diff --git a/.github/workflows/e2e-parallel-full.yml b/.github/workflows/e2e-parallel-full.yml index a5224fea2d..924e2098e7 100644 --- a/.github/workflows/e2e-parallel-full.yml +++ b/.github/workflows/e2e-parallel-full.yml @@ -56,6 +56,11 @@ jobs: role-duration-seconds: 3600 role-session-name: GithubActions-Session + - name: Ensure log groups are removed + run: | + pip3 install boto3 + python3 .github/workflows/delete-log-groups.py + - name: Iamlive Setup & Run run: | #!/bin/bash @@ -139,4 +144,4 @@ jobs: id: dirs run: | pip3 install boto3 - python3 .github/workflows/iam-policy-generator.py + python3 .github/scripts/iam-policy-generator.py diff --git a/.github/workflows/plan-examples.yml b/.github/workflows/plan-examples.yml index 57eb4405bd..f10f05d16e 100644 --- a/.github/workflows/plan-examples.yml +++ b/.github/workflows/plan-examples.yml @@ -29,7 +29,7 @@ jobs: - name: Get Terraform directories for evaluation id: dirs run: | - DIRS=$(python3 .github/workflows/plan-examples.py) + DIRS=$(python3 .github/scripts/plan-examples.py) echo "::set-output name=directories::$DIRS" plan: diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml index 0fbede9759..8795244f7e 100644 --- a/.github/workflows/pre-commit.yaml +++ b/.github/workflows/pre-commit.yaml @@ -31,7 +31,7 @@ jobs: - name: Get root directories id: dirs - uses: clowdhaus/terraform-composite-actions/directories@v1.4.1 + uses: clowdhaus/terraform-composite-actions/directories@v1.8.0 preCommitMinVersions: name: Min TF pre-commit @@ -70,14 +70,14 @@ jobs: restore-keys: ${{ runner.os }}-terraform- - name: Terraform min/max versions - uses: clowdhaus/terraform-min-max@v1.0.7 + uses: clowdhaus/terraform-min-max@v1.2.0 if: steps.changes.outputs.src== 'true' id: minMax with: directory: ${{ matrix.directory }} - name: Pre-commit Terraform ${{ steps.minMax.outputs.minVersion }} - uses: clowdhaus/terraform-composite-actions/pre-commit@v1.6.0 + uses: clowdhaus/terraform-composite-actions/pre-commit@v1.8.0 # Run only validate pre-commit check on min version supported if: ${{ matrix.directory != '.' && steps.changes.outputs.src== 'true' }} with: @@ -85,7 +85,7 @@ jobs: args: 'terraform_validate --color=always --show-diff-on-failure --files ${{ matrix.directory }}/*' - name: Pre-commit Terraform ${{ steps.minMax.outputs.minVersion }} - uses: clowdhaus/terraform-composite-actions/pre-commit@v1.6.0 + uses: clowdhaus/terraform-composite-actions/pre-commit@v1.8.0 # Run only validate pre-commit check on min version supported if: ${{ matrix.directory == '.' && steps.changes.outputs.src== 'true' }} with: @@ -128,11 +128,11 @@ jobs: - name: Terraform min/max versions id: minMax - uses: clowdhaus/terraform-min-max@v1.0.7 + uses: clowdhaus/terraform-min-max@v1.2.0 if: steps.changes.outputs.src== 'true' - name: Pre-commit Terraform ${{ steps.minMax.outputs.maxVersion }} - uses: clowdhaus/terraform-composite-actions/pre-commit@v1.6.0 + uses: clowdhaus/terraform-composite-actions/pre-commit@v1.8.0 if: steps.changes.outputs.src== 'true' with: terraform-version: ${{ steps.minMax.outputs.maxVersion }} From 42b0a777a18af0d035f4ba6bf8d8392e346a7738 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Mon, 31 Oct 2022 15:52:22 -0400 Subject: [PATCH 2/2] refactor: Rename agones directory path and update e2e examples used --- .github/workflows/e2e-parallel-destroy.yml | 6 ++++-- .github/workflows/e2e-parallel-full.yml | 6 ++++-- examples/{game-tech => }/agones-game-controller/README.md | 0 .../agones-game-controller/helm_values/agones-values.yaml | 0 examples/{game-tech => }/agones-game-controller/main.tf | 4 ++-- examples/{game-tech => }/agones-game-controller/outputs.tf | 0 .../test/sample-game-server/fleet.yaml | 0 .../test/sample-game-server/gameserver.yaml | 0 .../agones-game-controller/test/xonotic/fleet.yaml | 0 .../test/xonotic/fleetautoscaler.yaml | 0 .../agones-game-controller/test/xonotic/gameserver.yaml | 0 .../test/xonotic/gameserverallocator.yaml | 0 .../{game-tech => }/agones-game-controller/variables.tf | 0 examples/{game-tech => }/agones-game-controller/versions.tf | 0 14 files changed, 10 insertions(+), 6 deletions(-) rename examples/{game-tech => }/agones-game-controller/README.md (100%) rename examples/{game-tech => }/agones-game-controller/helm_values/agones-values.yaml (100%) rename examples/{game-tech => }/agones-game-controller/main.tf (98%) rename examples/{game-tech => }/agones-game-controller/outputs.tf (100%) rename examples/{game-tech => }/agones-game-controller/test/sample-game-server/fleet.yaml (100%) rename examples/{game-tech => }/agones-game-controller/test/sample-game-server/gameserver.yaml (100%) rename examples/{game-tech => }/agones-game-controller/test/xonotic/fleet.yaml (100%) rename examples/{game-tech => }/agones-game-controller/test/xonotic/fleetautoscaler.yaml (100%) rename examples/{game-tech => }/agones-game-controller/test/xonotic/gameserver.yaml (100%) rename examples/{game-tech => }/agones-game-controller/test/xonotic/gameserverallocator.yaml (100%) rename examples/{game-tech => }/agones-game-controller/variables.tf (100%) rename examples/{game-tech => }/agones-game-controller/versions.tf (100%) diff --git a/.github/workflows/e2e-parallel-destroy.yml b/.github/workflows/e2e-parallel-destroy.yml index 4f00d3f32b..73e806ee33 100644 --- a/.github/workflows/e2e-parallel-destroy.yml +++ b/.github/workflows/e2e-parallel-destroy.yml @@ -23,14 +23,16 @@ jobs: fail-fast: false matrix: include: + - example_path: examples/agones-game-controller - example_path: examples/analytics/emr-on-eks - example_path: examples/analytics/spark-k8s-operator + - example_path: examples/complete-kubernetes-addons - example_path: examples/crossplane - example_path: examples/eks-cluster-with-new-vpc - example_path: examples/fargate-serverless - # - example_path: examples/fully-private-eks-cluster # skipping until issue #711 is addressed - - example_path: examples/game-tech/agones-game-controller - example_path: examples/gitops/argocd + - example_path: examples/grafana-loki + - example_path: examples/ipv4-prefix-delegation - example_path: examples/ipv6-eks-cluster - example_path: examples/karpenter - example_path: examples/multi-tenancy-with-teams diff --git a/.github/workflows/e2e-parallel-full.yml b/.github/workflows/e2e-parallel-full.yml index 924e2098e7..207527918f 100644 --- a/.github/workflows/e2e-parallel-full.yml +++ b/.github/workflows/e2e-parallel-full.yml @@ -27,14 +27,16 @@ jobs: fail-fast: false matrix: include: + - example_path: examples/agones-game-controller - example_path: examples/analytics/emr-on-eks - example_path: examples/analytics/spark-k8s-operator + - example_path: examples/complete-kubernetes-addons - example_path: examples/crossplane - example_path: examples/eks-cluster-with-new-vpc - example_path: examples/fargate-serverless - # - example_path: examples/fully-private-eks-cluster # skipping until issue #711 - - example_path: examples/game-tech/agones-game-controller - example_path: examples/gitops/argocd + - example_path: examples/grafana-loki + - example_path: examples/ipv4-prefix-delegation - example_path: examples/ipv6-eks-cluster - example_path: examples/karpenter - example_path: examples/multi-tenancy-with-teams diff --git a/examples/game-tech/agones-game-controller/README.md b/examples/agones-game-controller/README.md similarity index 100% rename from examples/game-tech/agones-game-controller/README.md rename to examples/agones-game-controller/README.md diff --git a/examples/game-tech/agones-game-controller/helm_values/agones-values.yaml b/examples/agones-game-controller/helm_values/agones-values.yaml similarity index 100% rename from examples/game-tech/agones-game-controller/helm_values/agones-values.yaml rename to examples/agones-game-controller/helm_values/agones-values.yaml diff --git a/examples/game-tech/agones-game-controller/main.tf b/examples/agones-game-controller/main.tf similarity index 98% rename from examples/game-tech/agones-game-controller/main.tf rename to examples/agones-game-controller/main.tf index 8ec422e715..69510a0688 100644 --- a/examples/game-tech/agones-game-controller/main.tf +++ b/examples/agones-game-controller/main.tf @@ -40,7 +40,7 @@ locals { #--------------------------------------------------------------- module "eks_blueprints" { - source = "../../.." + source = "../.." cluster_name = local.name cluster_version = "1.23" @@ -88,7 +88,7 @@ module "eks_blueprints" { } module "eks_blueprints_kubernetes_addons" { - source = "../../..//modules/kubernetes-addons" + source = "../../modules/kubernetes-addons" eks_cluster_id = module.eks_blueprints.eks_cluster_id eks_cluster_endpoint = module.eks_blueprints.eks_cluster_endpoint diff --git a/examples/game-tech/agones-game-controller/outputs.tf b/examples/agones-game-controller/outputs.tf similarity index 100% rename from examples/game-tech/agones-game-controller/outputs.tf rename to examples/agones-game-controller/outputs.tf diff --git a/examples/game-tech/agones-game-controller/test/sample-game-server/fleet.yaml b/examples/agones-game-controller/test/sample-game-server/fleet.yaml similarity index 100% rename from examples/game-tech/agones-game-controller/test/sample-game-server/fleet.yaml rename to examples/agones-game-controller/test/sample-game-server/fleet.yaml diff --git a/examples/game-tech/agones-game-controller/test/sample-game-server/gameserver.yaml b/examples/agones-game-controller/test/sample-game-server/gameserver.yaml similarity index 100% rename from examples/game-tech/agones-game-controller/test/sample-game-server/gameserver.yaml rename to examples/agones-game-controller/test/sample-game-server/gameserver.yaml diff --git a/examples/game-tech/agones-game-controller/test/xonotic/fleet.yaml b/examples/agones-game-controller/test/xonotic/fleet.yaml similarity index 100% rename from examples/game-tech/agones-game-controller/test/xonotic/fleet.yaml rename to examples/agones-game-controller/test/xonotic/fleet.yaml diff --git a/examples/game-tech/agones-game-controller/test/xonotic/fleetautoscaler.yaml b/examples/agones-game-controller/test/xonotic/fleetautoscaler.yaml similarity index 100% rename from examples/game-tech/agones-game-controller/test/xonotic/fleetautoscaler.yaml rename to examples/agones-game-controller/test/xonotic/fleetautoscaler.yaml diff --git a/examples/game-tech/agones-game-controller/test/xonotic/gameserver.yaml b/examples/agones-game-controller/test/xonotic/gameserver.yaml similarity index 100% rename from examples/game-tech/agones-game-controller/test/xonotic/gameserver.yaml rename to examples/agones-game-controller/test/xonotic/gameserver.yaml diff --git a/examples/game-tech/agones-game-controller/test/xonotic/gameserverallocator.yaml b/examples/agones-game-controller/test/xonotic/gameserverallocator.yaml similarity index 100% rename from examples/game-tech/agones-game-controller/test/xonotic/gameserverallocator.yaml rename to examples/agones-game-controller/test/xonotic/gameserverallocator.yaml diff --git a/examples/game-tech/agones-game-controller/variables.tf b/examples/agones-game-controller/variables.tf similarity index 100% rename from examples/game-tech/agones-game-controller/variables.tf rename to examples/agones-game-controller/variables.tf diff --git a/examples/game-tech/agones-game-controller/versions.tf b/examples/agones-game-controller/versions.tf similarity index 100% rename from examples/game-tech/agones-game-controller/versions.tf rename to examples/agones-game-controller/versions.tf