diff --git a/.github/workflows/pr-title.yml b/.github/workflows/pr-title.yml index 1e50760..6419f3a 100644 --- a/.github/workflows/pr-title.yml +++ b/.github/workflows/pr-title.yml @@ -14,7 +14,7 @@ jobs: steps: # Please look up the latest version from # https://github.com/amannn/action-semantic-pull-request/releases - - uses: amannn/action-semantic-pull-request@v5.5.3 + - uses: amannn/action-semantic-pull-request@v6.1.1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index a19ff83..057b9c4 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -7,8 +7,8 @@ on: - master env: - TERRAFORM_DOCS_VERSION: v0.19.0 - TFLINT_VERSION: v0.53.0 + TERRAFORM_DOCS_VERSION: v0.20.0 + TFLINT_VERSION: v0.59.1 jobs: collectInputs: @@ -18,11 +18,11 @@ jobs: directories: ${{ steps.dirs.outputs.directories }} steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Get root directories id: dirs - uses: clowdhaus/terraform-composite-actions/directories@v1.9.0 + uses: clowdhaus/terraform-composite-actions/directories@v1.14.0 preCommitMinVersions: name: Min TF pre-commit @@ -32,27 +32,49 @@ jobs: matrix: directory: ${{ fromJson(needs.collectInputs.outputs.directories) }} steps: + - name: Install rmz + uses: jaxxstorm/action-install-gh-release@v2.1.0 + with: + repo: SUPERCILEX/fuc + asset-name: x86_64-unknown-linux-gnu-rmz + rename-to: rmz + chmod: 0755 + extension-matching: disable + # https://github.com/orgs/community/discussions/25678#discussioncomment-5242449 - - name: Delete huge unnecessary tools folder + - name: Delete unnecessary files run: | - rm -rf /opt/hostedtoolcache/CodeQL - rm -rf /opt/hostedtoolcache/Java_Temurin-Hotspot_jdk - rm -rf /opt/hostedtoolcache/Ruby - rm -rf /opt/hostedtoolcache/go + formatByteCount() { echo $(numfmt --to=iec-i --suffix=B --padding=7 $1'000'); } + getAvailableSpace() { echo $(df -a $1 | awk 'NR > 1 {avail+=$4} END {print avail}'); } + + BEFORE=$(getAvailableSpace) + + ln -s /opt/hostedtoolcache/SUPERCILEX/x86_64-unknown-linux-gnu-rmz/latest/linux-x64/rmz /usr/local/bin/rmz + rmz -f /opt/hostedtoolcache/CodeQL & + rmz -f /opt/hostedtoolcache/Java_Temurin-Hotspot_jdk & + rmz -f /opt/hostedtoolcache/PyPy & + rmz -f /opt/hostedtoolcache/Ruby & + rmz -f /opt/hostedtoolcache/go & + + wait + + AFTER=$(getAvailableSpace) + SAVED=$((AFTER-BEFORE)) + echo "=> Saved $(formatByteCount $SAVED)" - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Terraform min/max versions id: minMax - uses: clowdhaus/terraform-min-max@v1.3.1 + uses: clowdhaus/terraform-min-max@v2.1.0 with: directory: ${{ matrix.directory }} - name: Pre-commit Terraform ${{ steps.minMax.outputs.minVersion }} # Run only validate pre-commit check on min version supported if: ${{ matrix.directory != '.' }} - uses: clowdhaus/terraform-composite-actions/pre-commit@v1.11.1 + uses: clowdhaus/terraform-composite-actions/pre-commit@v1.14.0 with: terraform-version: ${{ steps.minMax.outputs.minVersion }} tflint-version: ${{ env.TFLINT_VERSION }} @@ -61,7 +83,7 @@ jobs: - name: Pre-commit Terraform ${{ steps.minMax.outputs.minVersion }} # Run only validate pre-commit check on min version supported if: ${{ matrix.directory == '.' }} - uses: clowdhaus/terraform-composite-actions/pre-commit@v1.11.1 + uses: clowdhaus/terraform-composite-actions/pre-commit@v1.14.0 with: terraform-version: ${{ steps.minMax.outputs.minVersion }} tflint-version: ${{ env.TFLINT_VERSION }} @@ -72,26 +94,73 @@ jobs: runs-on: ubuntu-latest needs: collectInputs steps: + - name: Install rmz + uses: jaxxstorm/action-install-gh-release@v2.1.0 + with: + repo: SUPERCILEX/fuc + asset-name: x86_64-unknown-linux-gnu-rmz + rename-to: rmz + chmod: 0755 + extension-matching: disable + # https://github.com/orgs/community/discussions/25678#discussioncomment-5242449 - - name: Delete huge unnecessary tools folder + - name: Delete unnecessary files run: | - rm -rf /opt/hostedtoolcache/CodeQL - rm -rf /opt/hostedtoolcache/Java_Temurin-Hotspot_jdk - rm -rf /opt/hostedtoolcache/Ruby - rm -rf /opt/hostedtoolcache/go + formatByteCount() { echo $(numfmt --to=iec-i --suffix=B --padding=7 $1'000'); } + getAvailableSpace() { echo $(df -a $1 | awk 'NR > 1 {avail+=$4} END {print avail}'); } + + BEFORE=$(getAvailableSpace) + + ln -s /opt/hostedtoolcache/SUPERCILEX/x86_64-unknown-linux-gnu-rmz/latest/linux-x64/rmz /usr/local/bin/rmz + rmz -f /opt/hostedtoolcache/CodeQL & + rmz -f /opt/hostedtoolcache/Java_Temurin-Hotspot_jdk & + rmz -f /opt/hostedtoolcache/PyPy & + rmz -f /opt/hostedtoolcache/Ruby & + rmz -f /opt/hostedtoolcache/go & + sudo rmz -f /usr/local/lib/android & + + if [[ ${{ github.repository }} == terraform-aws-modules/terraform-aws-security-group ]]; then + sudo rmz -f /usr/share/dotnet & + sudo rmz -f /usr/local/.ghcup & + sudo apt-get -qq remove -y 'azure-.*' + sudo apt-get -qq remove -y 'cpp-.*' + sudo apt-get -qq remove -y 'dotnet-runtime-.*' + sudo apt-get -qq remove -y 'google-.*' + sudo apt-get -qq remove -y 'libclang-.*' + sudo apt-get -qq remove -y 'libllvm.*' + sudo apt-get -qq remove -y 'llvm-.*' + sudo apt-get -qq remove -y 'mysql-.*' + sudo apt-get -qq remove -y 'postgresql-.*' + sudo apt-get -qq remove -y 'php.*' + sudo apt-get -qq remove -y 'temurin-.*' + sudo apt-get -qq remove -y kubectl firefox mono-devel + sudo apt-get -qq autoremove -y + sudo apt-get -qq clean + fi + + wait + + AFTER=$(getAvailableSpace) + SAVED=$((AFTER-BEFORE)) + echo "=> Saved $(formatByteCount $SAVED)" - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: ref: ${{ github.event.pull_request.head.ref }} repository: ${{github.event.pull_request.head.repo.full_name}} - name: Terraform min/max versions id: minMax - uses: clowdhaus/terraform-min-max@v1.3.1 + uses: clowdhaus/terraform-min-max@v2.1.0 + + - name: Hide template dir + # Special to this repo, we don't want to check this dir + if: ${{ github.repository == 'terraform-aws-modules/terraform-aws-security-group' }} + run: rm -rf modules/_templates - name: Pre-commit Terraform ${{ steps.minMax.outputs.maxVersion }} - uses: clowdhaus/terraform-composite-actions/pre-commit@v1.11.1 + uses: clowdhaus/terraform-composite-actions/pre-commit@v1.14.0 with: terraform-version: ${{ steps.minMax.outputs.maxVersion }} tflint-version: ${{ env.TFLINT_VERSION }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4a94226..e739b79 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,18 +20,26 @@ jobs: if: github.repository_owner == 'terraform-aws-modules' steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: persist-credentials: false fetch-depth: 0 + - name: Set correct Node.js version + uses: actions/setup-node@v6 + with: + node-version: 24 + + - name: Install dependencies + run: | + npm install \ + @semantic-release/changelog@6.0.3 \ + @semantic-release/git@10.0.1 \ + conventional-changelog-conventionalcommits@9.1.0 + - name: Release - uses: cycjimmy/semantic-release-action@v4 + uses: cycjimmy/semantic-release-action@v5 with: - semantic_version: 23.0.2 - extra_plugins: | - @semantic-release/changelog@6.0.3 - @semantic-release/git@10.0.1 - conventional-changelog-conventionalcommits@7.0.2 + semantic_version: 25.0.0 env: GITHUB_TOKEN: ${{ secrets.SEMANTIC_RELEASE_TOKEN }} diff --git a/.github/workflows/stale-actions.yaml b/.github/workflows/stale-actions.yaml index 6ccd0ed..3e826dc 100644 --- a/.github/workflows/stale-actions.yaml +++ b/.github/workflows/stale-actions.yaml @@ -7,7 +7,7 @@ jobs: stale: runs-on: ubuntu-latest steps: - - uses: actions/stale@v9 + - uses: actions/stale@v10 with: repo-token: ${{ secrets.GITHUB_TOKEN }} # Staling issues and PR's diff --git a/.gitignore b/.gitignore index 788c463..fd39819 100644 --- a/.gitignore +++ b/.gitignore @@ -1,13 +1,13 @@ # Local .terraform directories **/.terraform/* +# Terraform lockfile +.terraform.lock.hcl + # .tfstate files *.tfstate *.tfstate.* -# terraform lockfile -.terraform.lock.hcl - # Crash log files crash.log @@ -15,7 +15,6 @@ crash.log # password, private keys, and other secrets. These should not be part of version # control as they are data points which are potentially sensitive and subject # to change depending on the environment. -# *.tfvars # Ignore override files as they are usually used to override resources locally and so @@ -25,13 +24,16 @@ override.tf.json *_override.tf *_override.tf.json -# Include override files you do wish to add to version control using negated pattern -# -# !example_override.tf - -# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan -# example: *tfplan* - # Ignore CLI configuration files .terraformrc terraform.rc + +# Lambda build artifacts +builds/ +__pycache__/ +*.zip +.tox + +# Local editors/macos files +.DS_Store +.idea diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 481604d..9223e3c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.99.1 + rev: v1.103.0 hooks: - id: terraform_fmt - id: terraform_docs @@ -23,7 +23,7 @@ repos: - '--args=--only=terraform_workspace_remote' - id: terraform_validate - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 + rev: v6.0.0 hooks: - id: check-merge-conflict - id: end-of-file-fixer diff --git a/README.md b/README.md index fa5bb55..e5b2c2a 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ Terraform module which creates AWS EMR resources. [![SWUbanner](https://raw.githubusercontent.com/vshymanskyy/StandWithUkraine/main/banner2-direct.svg)](https://github.com/vshymanskyy/StandWithUkraine/blob/main/docs/README.md) This module supports the creation of: + - EMR clusters using instance fleets or instance groups deployed in public or private subnets - EMR Virtual clusters that run on Amazon EKS - EMR Serverless clusters @@ -13,7 +14,8 @@ This module supports the creation of: - Security group for EMR `service` to support private clusters - IAM roles for autoscaling, EMR `service`, and EC2 instance profiles - :information_source: The appropriate resources have been tagged with `{ "for-use-with-amazon-emr-managed-policies" = true }` to support the use of the recommended IAM policy `"arn:aws:iam::aws:policy/service-role/AmazonEMRServicePolicy_v2"`. Users are required to tag the appropriate VPC resources (VPC and subnets) as needed. See [here](https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-managed-iam-policies.html) for more details regarding v2 of managed EMR policies and their usage requirements. +> [!IMPORTANT] +> The appropriate resources have been tagged with `{ "for-use-with-amazon-emr-managed-policies" = true }` to support the use of the recommended IAM policy `"arn:aws:iam::aws:policy/service-role/AmazonEMRServicePolicy_v2"`. Users are required to tag the appropriate VPC resources (VPC and subnets) as needed. See [here](https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-managed-iam-policies.html) for more details regarding v2 of managed EMR policies and their usage requirements. ## Usage diff --git a/examples/private-cluster/README.md b/examples/private-cluster/README.md index abed116..03e4024 100644 --- a/examples/private-cluster/README.md +++ b/examples/private-cluster/README.md @@ -27,7 +27,7 @@ Note that this example may create resources which will incur monetary charges on | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | +| [terraform](#requirement\_terraform) | >= 1.5.7 | | [aws](#requirement\_aws) | >= 5.83 | ## Providers @@ -43,9 +43,9 @@ Note that this example may create resources which will incur monetary charges on | [emr\_disabled](#module\_emr\_disabled) | ../.. | n/a | | [emr\_instance\_fleet](#module\_emr\_instance\_fleet) | ../.. | n/a | | [emr\_instance\_group](#module\_emr\_instance\_group) | ../.. | n/a | -| [s3\_bucket](#module\_s3\_bucket) | terraform-aws-modules/s3-bucket/aws | ~> 4.0 | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | -| [vpc\_endpoints](#module\_vpc\_endpoints) | terraform-aws-modules/vpc/aws//modules/vpc-endpoints | ~> 5.0 | +| [s3\_bucket](#module\_s3\_bucket) | terraform-aws-modules/s3-bucket/aws | ~> 5.0 | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 6.0 | +| [vpc\_endpoints](#module\_vpc\_endpoints) | terraform-aws-modules/vpc/aws//modules/vpc-endpoints | ~> 6.0 | | [vpc\_endpoints\_sg](#module\_vpc\_endpoints\_sg) | terraform-aws-modules/security-group/aws | ~> 5.0 | ## Resources @@ -62,7 +62,6 @@ Note that this example may create resources which will incur monetary charges on | [aws_iam_policy_document.assume](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.autoscaling](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | -| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | ## Inputs diff --git a/examples/private-cluster/main.tf b/examples/private-cluster/main.tf index 764e5c7..d7653e5 100644 --- a/examples/private-cluster/main.tf +++ b/examples/private-cluster/main.tf @@ -3,13 +3,9 @@ provider "aws" { } data "aws_availability_zones" "available" {} - data "aws_partition" "current" {} - data "aws_caller_identity" "current" {} -data "aws_region" "current" {} - locals { name = replace(basename(path.cwd), "-cluster", "") region = "eu-west-1" @@ -270,7 +266,7 @@ module "emr_disabled" { module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "~> 5.0" + version = "~> 6.0" name = local.name cidr = local.vpc_cidr @@ -294,7 +290,7 @@ module "vpc" { module "vpc_endpoints" { source = "terraform-aws-modules/vpc/aws//modules/vpc-endpoints" - version = "~> 5.0" + version = "~> 6.0" vpc_id = module.vpc.vpc_id security_group_ids = [module.vpc_endpoints_sg.security_group_id] @@ -343,7 +339,7 @@ module "vpc_endpoints_sg" { module "s3_bucket" { source = "terraform-aws-modules/s3-bucket/aws" - version = "~> 4.0" + version = "~> 5.0" bucket_prefix = "${local.name}-" @@ -354,11 +350,6 @@ module "s3_bucket" { attach_deny_insecure_transport_policy = true attach_require_latest_tls_policy = true - block_public_acls = true - block_public_policy = true - ignore_public_acls = true - restrict_public_buckets = true - server_side_encryption_configuration = { rule = { apply_server_side_encryption_by_default = { @@ -427,7 +418,7 @@ data "aws_iam_policy_document" "autoscaling" { condition { test = "ArnLike" variable = "aws:SourceArn" - values = ["arn:aws:elasticmapreduce:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:*"] + values = ["arn:aws:elasticmapreduce:${local.region}:${data.aws_caller_identity.current.account_id}:*"] } } } diff --git a/examples/private-cluster/versions.tf b/examples/private-cluster/versions.tf index e0d6884..5ce9aba 100644 --- a/examples/private-cluster/versions.tf +++ b/examples/private-cluster/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 1.0" + required_version = ">= 1.5.7" required_providers { aws = { diff --git a/examples/public-cluster/README.md b/examples/public-cluster/README.md index 254a5de..453b266 100644 --- a/examples/public-cluster/README.md +++ b/examples/public-cluster/README.md @@ -25,7 +25,7 @@ Note that this example may create resources which will incur monetary charges on | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | +| [terraform](#requirement\_terraform) | >= 1.5.7 | | [aws](#requirement\_aws) | >= 5.83 | ## Providers @@ -40,8 +40,8 @@ Note that this example may create resources which will incur monetary charges on |------|--------|---------| | [emr\_instance\_fleet](#module\_emr\_instance\_fleet) | ../.. | n/a | | [emr\_instance\_group](#module\_emr\_instance\_group) | ../.. | n/a | -| [s3\_bucket](#module\_s3\_bucket) | terraform-aws-modules/s3-bucket/aws | ~> 4.0 | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | +| [s3\_bucket](#module\_s3\_bucket) | terraform-aws-modules/s3-bucket/aws | ~> 5.0 | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 6.0 | ## Resources diff --git a/examples/public-cluster/main.tf b/examples/public-cluster/main.tf index 78eb297..b8fc6a8 100644 --- a/examples/public-cluster/main.tf +++ b/examples/public-cluster/main.tf @@ -247,7 +247,7 @@ module "emr_instance_group" { module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "~> 5.0" + version = "~> 6.0" name = local.name cidr = local.vpc_cidr @@ -270,7 +270,7 @@ module "vpc" { module "s3_bucket" { source = "terraform-aws-modules/s3-bucket/aws" - version = "~> 4.0" + version = "~> 5.0" bucket_prefix = "${local.name}-" @@ -281,11 +281,6 @@ module "s3_bucket" { attach_deny_insecure_transport_policy = true attach_require_latest_tls_policy = true - block_public_acls = true - block_public_policy = true - ignore_public_acls = true - restrict_public_buckets = true - server_side_encryption_configuration = { rule = { apply_server_side_encryption_by_default = { diff --git a/examples/public-cluster/versions.tf b/examples/public-cluster/versions.tf index e0d6884..5ce9aba 100644 --- a/examples/public-cluster/versions.tf +++ b/examples/public-cluster/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 1.0" + required_version = ">= 1.5.7" required_providers { aws = { diff --git a/examples/serverless-cluster/README.md b/examples/serverless-cluster/README.md index a384ab5..acbab05 100644 --- a/examples/serverless-cluster/README.md +++ b/examples/serverless-cluster/README.md @@ -25,7 +25,7 @@ Note that this example may create resources which will incur monetary charges on | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | +| [terraform](#requirement\_terraform) | >= 1.5.7 | | [aws](#requirement\_aws) | >= 5.83 | ## Providers @@ -41,7 +41,7 @@ Note that this example may create resources which will incur monetary charges on | [emr\_serverless\_disabled](#module\_emr\_serverless\_disabled) | ../../modules/serverless | n/a | | [emr\_serverless\_hive](#module\_emr\_serverless\_hive) | ../../modules/serverless | n/a | | [emr\_serverless\_spark](#module\_emr\_serverless\_spark) | ../../modules/serverless | n/a | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 6.0 | ## Resources diff --git a/examples/serverless-cluster/main.tf b/examples/serverless-cluster/main.tf index d3d2846..a440fbf 100644 --- a/examples/serverless-cluster/main.tf +++ b/examples/serverless-cluster/main.tf @@ -137,7 +137,7 @@ module "emr_serverless_disabled" { module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "~> 5.0" + version = "~> 6.0" name = local.name cidr = local.vpc_cidr diff --git a/examples/serverless-cluster/versions.tf b/examples/serverless-cluster/versions.tf index e0d6884..5ce9aba 100644 --- a/examples/serverless-cluster/versions.tf +++ b/examples/serverless-cluster/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 1.0" + required_version = ">= 1.5.7" required_providers { aws = { diff --git a/examples/studio/README.md b/examples/studio/README.md index ca459b9..61dbc7b 100644 --- a/examples/studio/README.md +++ b/examples/studio/README.md @@ -21,7 +21,7 @@ $ terraform apply | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | +| [terraform](#requirement\_terraform) | >= 1.5.7 | | [aws](#requirement\_aws) | >= 5.83 | ## Providers @@ -38,9 +38,9 @@ $ terraform apply | [emr\_studio\_disabled](#module\_emr\_studio\_disabled) | ../../modules/studio | n/a | | [emr\_studio\_iam](#module\_emr\_studio\_iam) | ../../modules/studio | n/a | | [emr\_studio\_sso](#module\_emr\_studio\_sso) | ../../modules/studio | n/a | -| [kms](#module\_kms) | terraform-aws-modules/kms/aws | ~> 2.0 | -| [s3\_bucket](#module\_s3\_bucket) | terraform-aws-modules/s3-bucket/aws | ~> 4.0 | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | +| [kms](#module\_kms) | terraform-aws-modules/kms/aws | ~> 4.0 | +| [s3\_bucket](#module\_s3\_bucket) | terraform-aws-modules/s3-bucket/aws | ~> 5.0 | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 6.0 | ## Resources @@ -49,7 +49,6 @@ $ terraform apply | [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_identitystore_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/identitystore_group) | data source | -| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | | [aws_ssoadmin_instances.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ssoadmin_instances) | data source | ## Inputs diff --git a/examples/studio/main.tf b/examples/studio/main.tf index cd48aba..d20f415 100644 --- a/examples/studio/main.tf +++ b/examples/studio/main.tf @@ -3,11 +3,8 @@ provider "aws" { } data "aws_availability_zones" "available" {} - data "aws_caller_identity" "current" {} -data "aws_region" "current" {} - locals { name = replace(basename(path.cwd), "-cluster", "") region = "eu-west-1" @@ -174,7 +171,7 @@ module "emr_studio_disabled" { module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "~> 5.0" + version = "~> 6.0" name = local.name cidr = local.vpc_cidr @@ -192,7 +189,7 @@ module "vpc" { module "s3_bucket" { source = "terraform-aws-modules/s3-bucket/aws" - version = "~> 4.0" + version = "~> 5.0" bucket_prefix = "${local.name}-" @@ -221,7 +218,7 @@ module "s3_bucket" { module "kms" { source = "terraform-aws-modules/kms/aws" - version = "~> 2.0" + version = "~> 4.0" deletion_window_in_days = 7 description = "KMS key for ${local.name}." @@ -262,7 +259,7 @@ module "kms" { { test = "StringEquals" variable = "kms:ViaService" - values = ["s3.${data.aws_region.current.name}.amazonaws.com"] + values = ["s3.${local.region}.amazonaws.com"] } ] } diff --git a/examples/studio/versions.tf b/examples/studio/versions.tf index e0d6884..5ce9aba 100644 --- a/examples/studio/versions.tf +++ b/examples/studio/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 1.0" + required_version = ">= 1.5.7" required_providers { aws = { diff --git a/examples/virtual-cluster/README.md b/examples/virtual-cluster/README.md index 6853b75..7561917 100644 --- a/examples/virtual-cluster/README.md +++ b/examples/virtual-cluster/README.md @@ -1,14 +1,10 @@ # AWS EMR Virtual Cluster Example -This example shows how to provision a serverless cluster (serverless data plane) using Fargate Profiles to support EMR on EKS virtual clusters. - -There are two Fargate profiles created: -1. `kube-system` to support core Kubernetes components such as CoreDNS -2. `emr-wildcard` which supports any namespaces that begin with `emr-*`; this allows for creating multiple virtual clusters without having to create additional Fargate profiles for each new cluster. +This example shows how to provision a serverless cluster (serverless data plane) using EKS Auto Mode to support EMR on EKS virtual clusters. The resources created by the `virtual-cluster` module include: + - Kubernetes namespace, role, and role binding; existing or externally created namespace and role can be utilized as well -- IAM role for service account (IRSA) used by for job execution. Users can scope access to the appropriate S3 bucket and path via `s3_bucket_arns`, use for both accessing job data as well as writing out results. The bare minimum permissions have been provided for the job execution role; users can provide additional permissions by passing in additional policies to attach to the role via `iam_role_additional_policies` - CloudWatch log group for task execution logs. Log streams are created by the job itself and not via Terraform - EMR managed security group for the virtual cluster - EMR virtual cluster scoped to the namespace created/provided @@ -18,9 +14,9 @@ The resources created by the `virtual-cluster` module include: To run this example you need to execute: ```bash -$ terraform init -$ terraform plan -$ terraform apply +terraform init +terraform plan +terraform apply ``` Note that this example may create resources which will incur monetary charges on your AWS bill. Run `terraform destroy` when you no longer need these resources. @@ -44,7 +40,7 @@ aws emr-containers list-virtual-clusters --region us-west-2 --states ARRESTED \ | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | +| [terraform](#requirement\_terraform) | >= 1.5.7 | | [aws](#requirement\_aws) | >= 5.83 | | [kubernetes](#requirement\_kubernetes) | >= 2.17 | | [null](#requirement\_null) | >= 3.0 | @@ -56,7 +52,6 @@ aws emr-containers list-virtual-clusters --region us-west-2 --states ARRESTED \ |------|---------| | [aws](#provider\_aws) | >= 5.83 | | [null](#provider\_null) | >= 3.0 | -| [time](#provider\_time) | >= 0.7 | ## Modules @@ -65,10 +60,10 @@ aws emr-containers list-virtual-clusters --region us-west-2 --states ARRESTED \ | [complete](#module\_complete) | ../../modules/virtual-cluster | n/a | | [default](#module\_default) | ../../modules/virtual-cluster | n/a | | [disabled](#module\_disabled) | ../../modules/virtual-cluster | n/a | -| [eks](#module\_eks) | terraform-aws-modules/eks/aws | ~> 19.13 | -| [s3\_bucket](#module\_s3\_bucket) | terraform-aws-modules/s3-bucket/aws | ~> 4.0 | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | -| [vpc\_endpoints](#module\_vpc\_endpoints) | terraform-aws-modules/vpc/aws//modules/vpc-endpoints | ~> 5.0 | +| [eks](#module\_eks) | terraform-aws-modules/eks/aws | ~> 21.0 | +| [s3\_bucket](#module\_s3\_bucket) | terraform-aws-modules/s3-bucket/aws | ~> 5.0 | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 6.0 | +| [vpc\_endpoints](#module\_vpc\_endpoints) | terraform-aws-modules/vpc/aws//modules/vpc-endpoints | ~> 6.0 | | [vpc\_endpoints\_sg](#module\_vpc\_endpoints\_sg) | terraform-aws-modules/security-group/aws | ~> 5.0 | ## Resources @@ -77,9 +72,7 @@ aws emr-containers list-virtual-clusters --region us-west-2 --states ARRESTED \ |------|------| | [null_resource.s3_sync](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | | [null_resource.start_job_run](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [time_sleep.coredns](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource | | [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | -| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | ## Inputs diff --git a/examples/virtual-cluster/main.tf b/examples/virtual-cluster/main.tf index ea0013f..1a5a9bd 100644 --- a/examples/virtual-cluster/main.tf +++ b/examples/virtual-cluster/main.tf @@ -14,11 +14,10 @@ provider "kubernetes" { } } -data "aws_caller_identity" "current" {} data "aws_availability_zones" "available" {} locals { - name = replace(basename(path.cwd), "-cluster", "") + name = "virtual-emr" region = "eu-west-1" vpc_cidr = "10.0.0.0/16" @@ -38,11 +37,13 @@ locals { module "complete" { source = "../../modules/virtual-cluster" + eks_cluster_id = module.eks.cluster_name + oidc_provider_arn = module.eks.oidc_provider_arn + name = "emr-custom" create_namespace = true namespace = "emr-custom" - create_iam_role = true s3_bucket_arns = [ module.s3_bucket.s3_bucket_arn, "${module.s3_bucket.s3_bucket_arn}/*" @@ -60,6 +61,15 @@ module "complete" { module "default" { source = "../../modules/virtual-cluster" + eks_cluster_id = module.eks.cluster_name + oidc_provider_arn = module.eks.oidc_provider_arn + + s3_bucket_arns = [ + module.s3_bucket.s3_bucket_arn, + "${module.s3_bucket.s3_bucket_arn}/*" + ] + + name = "emr-default" namespace = "emr-default" tags = local.tags @@ -90,16 +100,6 @@ resource "null_resource" "s3_sync" { } } -resource "time_sleep" "coredns" { - create_duration = "60s" - - # In practice, this generally won't be necessary since the cluster will be provisioned long before jobs are scheduled on the cluster - # However, for this example, its necessary to ensure CoreDNS is ready before we schedule the example job - triggers = { - coredns = module.eks.cluster_addons["coredns"].id - } -} - resource "null_resource" "start_job_run" { provisioner "local-exec" { interpreter = ["/bin/sh", "-c"] @@ -138,10 +138,6 @@ resource "null_resource" "start_job_run" { }' EOT } - - depends_on = [ - time_sleep.coredns - ] } ################################################################################ @@ -150,78 +146,32 @@ resource "null_resource" "start_job_run" { module "eks" { source = "terraform-aws-modules/eks/aws" - version = "~> 19.13" - - cluster_name = local.name - cluster_version = "1.27" - cluster_endpoint_public_access = true - - cluster_addons = { - coredns = { - configuration_values = jsonencode({ - computeType = "Fargate" - # Ensure that the we fully utilize the minimum amount of resources that are supplied by - # Fargate https://docs.aws.amazon.com/eks/latest/userguide/fargate-pod-configuration.html - # Fargate adds 256 MB to each pod's memory reservation for the required Kubernetes - # components (kubelet, kube-proxy, and containerd). Fargate rounds up to the following - # compute configuration that most closely matches the sum of vCPU and memory requests in - # order to ensure pods always have the resources that they need to run. - resources = { - limits = { - cpu = "0.25" - # We are targetting the smallest Task size of 512Mb, so we subtract 256Mb from the - # request/limit to ensure we can fit within that task - memory = "256M" - } - requests = { - cpu = "0.25" - # We are targetting the smallest Task size of 512Mb, so we subtract 256Mb from the - # request/limit to ensure we can fit within that task - memory = "256M" - } - } - }) - } - kube-proxy = {} - vpc-cni = {} + version = "~> 21.0" + + name = local.name + kubernetes_version = "1.33" + endpoint_public_access = true + + enable_cluster_creator_admin_permissions = true + + compute_config = { + enabled = true + node_pools = ["general-purpose", "system"] } vpc_id = module.vpc.vpc_id subnet_ids = module.vpc.private_subnets - # Fargate profiles use the cluster primary security group so these are not utilized - create_cluster_security_group = false - create_node_security_group = false - - manage_aws_auth_configmap = true - aws_auth_roles = [ - { - # Required for EMR on EKS virtual cluster - rolearn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/AWSServiceRoleForAmazonEMRContainers" - username = "emr-containers" - }, - ] - - fargate_profiles = { - emr_wildcard = { - selectors = [ - { namespace = "emr-*" } - ] - } - kube_system = { - name = "kube-system" - selectors = [ - { namespace = "kube-system" } - ] - } - } + # Auto Mode uses the cluster primary security group so these are not utilized + create_security_group = false + create_node_security_group = false tags = local.tags } module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "~> 5.0" + version = "~> 6.0" name = local.name cidr = local.vpc_cidr @@ -246,7 +196,7 @@ module "vpc" { module "vpc_endpoints" { source = "terraform-aws-modules/vpc/aws//modules/vpc-endpoints" - version = "~> 5.0" + version = "~> 6.0" vpc_id = module.vpc.vpc_id security_group_ids = [module.vpc_endpoints_sg.security_group_id] @@ -295,7 +245,7 @@ module "vpc_endpoints_sg" { module "s3_bucket" { source = "terraform-aws-modules/s3-bucket/aws" - version = "~> 4.0" + version = "~> 5.0" bucket_prefix = "${local.name}-" @@ -306,11 +256,6 @@ module "s3_bucket" { attach_deny_insecure_transport_policy = true attach_require_latest_tls_policy = true - block_public_acls = true - block_public_policy = true - ignore_public_acls = true - restrict_public_buckets = true - server_side_encryption_configuration = { rule = { apply_server_side_encryption_by_default = { diff --git a/examples/virtual-cluster/versions.tf b/examples/virtual-cluster/versions.tf index 7aeb5c1..441e4e6 100644 --- a/examples/virtual-cluster/versions.tf +++ b/examples/virtual-cluster/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 1.0" + required_version = ">= 1.5.7" required_providers { aws = { diff --git a/modules/studio/README.md b/modules/studio/README.md index 5773447..b201b46 100644 --- a/modules/studio/README.md +++ b/modules/studio/README.md @@ -32,7 +32,7 @@ module "emr_studio" { } ``` -### IAM Identity Center authentication mode (SSO) +### IAM authentication mode ```hcl module "emr_studio" { diff --git a/modules/virtual-cluster/README.md b/modules/virtual-cluster/README.md index dd6203c..451f58a 100644 --- a/modules/virtual-cluster/README.md +++ b/modules/virtual-cluster/README.md @@ -2,22 +2,6 @@ Terraform module which creates AWS EMR Virtual Cluster resources. -Note: you will need to add the `AWSServiceRoleForAmazonEMRContainers` role to the clusters `aws-auth` configmap under the username of `emr-containers`. See below for reference: - -```hcl -data "aws_caller_identity" "current" {} - -... - aws_auth_roles = [ - { - # Required for EMR on EKS virtual cluster - rolearn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/AWSServiceRoleForAmazonEMRContainers" - username = "emr-containers" - }, - ] -... -``` - ## Usage See [`examples`](https://github.com/terraform-aws-modules/terraform-aws-emr/tree/master/examples) directory for working examples to reference: @@ -57,6 +41,7 @@ module "emr_virtual_cluster" { module "emr_virtual_cluster" { source = "terraform-aws-modules/emr/aws//modules/virtual-cluster" + name = "emr-default" namespace = "emr-default" tags = { diff --git a/modules/virtual-cluster/main.tf b/modules/virtual-cluster/main.tf index 4c54930..a6d0f07 100644 --- a/modules/virtual-cluster/main.tf +++ b/modules/virtual-cluster/main.tf @@ -5,8 +5,8 @@ locals { internal_role_name = try(coalesce(var.role_name, var.name), "") - role_name = var.create_kubernetes_role ? kubernetes_role_v1.this[0].metadata[0].name : local.internal_role_name - namespace = var.create_namespace ? kubernetes_namespace_v1.this[0].metadata[0].name : var.namespace + role_name = var.create_kubernetes_role ? try(kubernetes_role_v1.this[0].metadata[0].name, "") : local.internal_role_name + namespace = var.create_namespace ? try(kubernetes_namespace_v1.this[0].metadata[0].name, "") : var.namespace cloudwatch_log_group_name = coalesce(var.cloudwatch_log_group_name, "/emr-on-eks-logs/emr-workload/${local.namespace}") tags = merge(var.tags, { terraform-aws-modules = "emr" })