From f843e29e88c5b1336c3bbe5429dbe84fd53978c2 Mon Sep 17 00:00:00 2001 From: HectorArchilogic Date: Thu, 11 May 2023 01:20:23 +0200 Subject: [PATCH 1/4] Add ARN created roles as output --- README.md | 276 ++++++++++++++++++++++++++++++++++++++++++++++------- outputs.tf | 10 ++ 2 files changed, 252 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index d3e8dfb..06751a3 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,119 @@ # AWS MWAA Terraform Module -Terraform module which creates AWS MWAA resources and connects them together. +Terraform +module +which +creates +AWS +MWAA +resources +and +connects +them +together. ## How to ### Contribute -When creating PRs from forks, make sure to create them against the `fork_prs` branch. One check, the automated doc generation, -will fail, but we (maintainers) can merge your PR into `fork_prs` and create a new PR with working doc generation from `fork_prs` -against `main`, preserving your commits (so your contribution is visible on GitHub). +When +creating +PRs +from +forks, +make +sure +to +create +them +against +the `fork_prs` +branch. +One +check, +the +automated +doc +generation, +will +fail, +but +we ( +maintainers) +can +merge +your +PR +into `fork_prs` +and +create +a +new +PR +with +working +doc +generation +from `fork_prs` +against `main` +, +preserving +your +commits ( +so +your +contribution +is +visible +on +GitHub) +. + +If +the +fork_prs +branch +does +not +exist, +please +create +an +issue. + +### Use + +Use +this +code +to +create +a +basic +MWAA +environment ( +using +all +default +parameters, +see [Inputs](#inputs)): -If the fork_prs branch does not exist, please create an issue. - -### Use - -Use this code to create a basic MWAA environment (using all default parameters, see [Inputs](#inputs)): ```terraform module "airflow" { source = "idealo/mwaa/aws" version = "x.x.x" - + account_id = "12345679" environment_name = "MyEnvironment" internet_gateway_id = "ig-12345" - private_subnet_cidrs = ["10.0.1.0/24","10.0.2.0/24"] # depending on your vpc ip range - public_subnet_cidrs = ["10.0.3.0/24","10.0.4.0/24"] # depending on your vpc ip range + private_subnet_cidrs = [ + "10.0.1.0/24", + "10.0.2.0/24"] + # depending on your vpc ip range + public_subnet_cidrs = [ + "10.0.3.0/24", + "10.0.4.0/24"] + # depending on your vpc ip range region = "us-west-1" source_bucket_arn = "arn:aws:s3:::MyMwaaBucket" vpc_id = "vpc-12345" @@ -33,7 +122,33 @@ module "airflow" { ### Add permissions to the Airflow execution role -To give additional permissions to your airflow executions role (e.g. elasticmapreduce:CreateJobFlow to start an EMR cluster), create a Policy document containing the permissions you need: +To +give +additional +permissions +to +your +airflow +executions +role ( +e.g. +elasticmapreduce: +CreateJobFlow +to +start +an +EMR +cluster) +, +create +a +Policy +document +containing +the +permissions +you +need: ```terraform data aws_iam_policy_document "additional_execution_policy_doc" { @@ -48,35 +163,124 @@ data aws_iam_policy_document "additional_execution_policy_doc" { } ``` -and pass the document json to the module: +and +pass +the +document +json +to +the +module: + ```terraform module "airflow" { ... - additional_execution_role_policy_document_json = data.aws_iam_policy_document.additional_execution_policy_doc.json - ... +additional_execution_role_policy_document_json = data.aws_iam_policy_document.additional_execution_policy_doc.json +... } ``` ### Add custom plugins -Simply upload the plugins.zip to s3 and pass the relative path inside the MWAA bucket to the `plugins_s3_path` parameter. -If you zip and upload it via terraform, this would look like this: +Simply +upload +the +plugins.zip +to +s3 +and +pass +the +relative +path +inside +the +MWAA +bucket +to +the `plugins_s3_path` +parameter. +If +you +zip +and +upload +it +via +terraform, +this +would +look +like +this: ```terraform module "airflow" { ... - plugins_s3_path = aws_s3_bucket_object.your_plugin.key - ... +plugins_s3_path = aws_s3_bucket_object.your_plugin.key +... } ``` ### Use your own networking config -If you set ``create_networking_config = false`` no subnets, eip, NAT gateway and route tables will be created. -**Be aware that you still need the networking resources to get your environment running, follow the [official documentation](https://docs.aws.amazon.com/mwaa/latest/userguide/vpc-create.html) to create them properly.** +If +you +set ``create_networking_config = false`` +no +subnets, +eip, +NAT +gateway +and +route +tables +will +be +created. +** +Be +aware +that +you +still +need +the +networking +resources +to +get +your +environment +running, +follow +the [official documentation](https://docs.aws.amazon.com/mwaa/latest/userguide/vpc-create.html) +to +create +them +properly.** ### S3 Bucket configuration -MWAA needs a S3 bucket to store the DAG files. Here is a minimal configuration for this S3 bucket: + +MWAA +needs +a +S3 +bucket +to +store +the +DAG +files. +Here +is +a +minimal +configuration +for +this +S3 +bucket: ```terraform resource "aws_s3_bucket" "mwaa" { @@ -92,21 +296,21 @@ resource "aws_s3_bucket_versioning" "mwaa" { resource "aws_s3_bucket_public_access_block" "mwaa" { # required: https://docs.aws.amazon.com/mwaa/latest/userguide/mwaa-s3-bucket.html - bucket = aws_s3_bucket.mwaa.id - block_public_acls = true - block_public_policy = true - ignore_public_acls = true + bucket = aws_s3_bucket.mwaa.id + block_public_acls = true + block_public_policy = true + ignore_public_acls = true restrict_public_buckets = true } ``` - + ## Requirements | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >=1.0.0 | +| [terraform](#requirement\_terraform) | > =1.0.0 | | [aws](#requirement\_aws) | ~> 4.0 | ## Providers @@ -117,7 +321,8 @@ resource "aws_s3_bucket_public_access_block" "mwaa" { ## Modules -No modules. +No +modules. ## Resources @@ -185,9 +390,12 @@ No modules. ## Outputs -| Name | Description | -|------|-------------| -| [mwaa\_arn](#output\_mwaa\_arn) | The arn of the created MWAA environment. | +| Name | Description | +|-------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------| +| [mwaa\_arn](#output\_mwaa\_arn) | The arn of the created MWAA environment. | +| [mwaa\_execution\_role\_arn](#output\_mwaa\_execution\_role\_arn) | The IAM Role arn for MWAA Execution Role. | +| [mwaa\_service\_role\_arn](#output\_mwaa_service\_role\_arn) | The Service Role arn for MWAA environment. | | [mwaa\_nat\_gateway\_public\_ips](#output\_mwaa\_nat\_gateway\_public\_ips) | List of the ips of the nat gateways created by this module. | -| [mwaa\_webserver\_url](#output\_mwaa\_webserver\_url) | The webserver URL of the MWAA Environment. | +| [mwaa\_webserver\_url](#output\_mwaa\_webserver\_url) | The webserver URL of the MWAA Environment. | + diff --git a/outputs.tf b/outputs.tf index e3f9e3c..6b7ab3b 100644 --- a/outputs.tf +++ b/outputs.tf @@ -3,6 +3,16 @@ output "mwaa_arn" { description = "The arn of the created MWAA environment." } +output "mwaa_execution_role_arn" { + value = aws_mwaa_environment.this.execution_role_arn + description = "The IAM Role arn for MWAA Execution Role." +} + +output "mwaa_service_role_arn" { + value = aws_mwaa_environment.this.execution_role_arn + description = "The Service Role arn for MWAA environment." +} + output "mwaa_nat_gateway_public_ips" { value = aws_nat_gateway.this[*].public_ip description = "List of the ips of the nat gateways created by this module." From 421fd389da954f4ec6155ab4847cf07468bc8144 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 10 May 2023 23:23:08 +0000 Subject: [PATCH 2/4] terraform-docs: automated action --- README.md | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 06751a3..752a465 100644 --- a/README.md +++ b/README.md @@ -305,12 +305,11 @@ resource "aws_s3_bucket_public_access_block" "mwaa" { ``` - ## Requirements | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | > =1.0.0 | +| [terraform](#requirement\_terraform) | >=1.0.0 | | [aws](#requirement\_aws) | ~> 4.0 | ## Providers @@ -321,8 +320,7 @@ resource "aws_s3_bucket_public_access_block" "mwaa" { ## Modules -No -modules. +No modules. ## Resources @@ -390,12 +388,11 @@ modules. ## Outputs -| Name | Description | -|-------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------| -| [mwaa\_arn](#output\_mwaa\_arn) | The arn of the created MWAA environment. | -| [mwaa\_execution\_role\_arn](#output\_mwaa\_execution\_role\_arn) | The IAM Role arn for MWAA Execution Role. | -| [mwaa\_service\_role\_arn](#output\_mwaa_service\_role\_arn) | The Service Role arn for MWAA environment. | +| Name | Description | +|------|-------------| +| [mwaa\_arn](#output\_mwaa\_arn) | The arn of the created MWAA environment. | +| [mwaa\_execution\_role\_arn](#output\_mwaa\_execution\_role\_arn) | The IAM Role arn for MWAA Execution Role. | | [mwaa\_nat\_gateway\_public\_ips](#output\_mwaa\_nat\_gateway\_public\_ips) | List of the ips of the nat gateways created by this module. | -| [mwaa\_webserver\_url](#output\_mwaa\_webserver\_url) | The webserver URL of the MWAA Environment. | - +| [mwaa\_service\_role\_arn](#output\_mwaa\_service\_role\_arn) | The Service Role arn for MWAA environment. | +| [mwaa\_webserver\_url](#output\_mwaa\_webserver\_url) | The webserver URL of the MWAA Environment. | From ddeeb82c83f5d85817fc8be04dacf57f74baf0ed Mon Sep 17 00:00:00 2001 From: HectorArchilogic Date: Thu, 11 May 2023 01:24:56 +0200 Subject: [PATCH 3/4] Rollback Readme.md --- README.md | 265 +++++++----------------------------------------------- 1 file changed, 30 insertions(+), 235 deletions(-) diff --git a/README.md b/README.md index 06751a3..f0c16b5 100644 --- a/README.md +++ b/README.md @@ -1,119 +1,30 @@ # AWS MWAA Terraform Module -Terraform -module -which -creates -AWS -MWAA -resources -and -connects -them -together. +Terraform module which creates AWS MWAA resources and connects them together. ## How to ### Contribute -When -creating -PRs -from -forks, -make -sure -to -create -them -against -the `fork_prs` -branch. -One -check, -the -automated -doc -generation, -will -fail, -but -we ( -maintainers) -can -merge -your -PR -into `fork_prs` -and -create -a -new -PR -with -working -doc -generation -from `fork_prs` -against `main` -, -preserving -your -commits ( -so -your -contribution -is -visible -on -GitHub) -. - -If -the -fork_prs -branch -does -not -exist, -please -create -an -issue. - -### Use - -Use -this -code -to -create -a -basic -MWAA -environment ( -using -all -default -parameters, -see [Inputs](#inputs)): +When creating PRs from forks, make sure to create them against the `fork_prs` branch. One check, the automated doc generation, +will fail, but we (maintainers) can merge your PR into `fork_prs` and create a new PR with working doc generation from `fork_prs` +against `main`, preserving your commits (so your contribution is visible on GitHub). +If the fork_prs branch does not exist, please create an issue. + +### Use + +Use this code to create a basic MWAA environment (using all default parameters, see [Inputs](#inputs)): ```terraform module "airflow" { source = "idealo/mwaa/aws" version = "x.x.x" - + account_id = "12345679" environment_name = "MyEnvironment" internet_gateway_id = "ig-12345" - private_subnet_cidrs = [ - "10.0.1.0/24", - "10.0.2.0/24"] - # depending on your vpc ip range - public_subnet_cidrs = [ - "10.0.3.0/24", - "10.0.4.0/24"] - # depending on your vpc ip range + private_subnet_cidrs = ["10.0.1.0/24","10.0.2.0/24"] # depending on your vpc ip range + public_subnet_cidrs = ["10.0.3.0/24","10.0.4.0/24"] # depending on your vpc ip range region = "us-west-1" source_bucket_arn = "arn:aws:s3:::MyMwaaBucket" vpc_id = "vpc-12345" @@ -122,33 +33,7 @@ module "airflow" { ### Add permissions to the Airflow execution role -To -give -additional -permissions -to -your -airflow -executions -role ( -e.g. -elasticmapreduce: -CreateJobFlow -to -start -an -EMR -cluster) -, -create -a -Policy -document -containing -the -permissions -you -need: +To give additional permissions to your airflow executions role (e.g. elasticmapreduce:CreateJobFlow to start an EMR cluster), create a Policy document containing the permissions you need: ```terraform data aws_iam_policy_document "additional_execution_policy_doc" { @@ -163,124 +48,35 @@ data aws_iam_policy_document "additional_execution_policy_doc" { } ``` -and -pass -the -document -json -to -the -module: - +and pass the document json to the module: ```terraform module "airflow" { ... -additional_execution_role_policy_document_json = data.aws_iam_policy_document.additional_execution_policy_doc.json -... + additional_execution_role_policy_document_json = data.aws_iam_policy_document.additional_execution_policy_doc.json + ... } ``` ### Add custom plugins -Simply -upload -the -plugins.zip -to -s3 -and -pass -the -relative -path -inside -the -MWAA -bucket -to -the `plugins_s3_path` -parameter. -If -you -zip -and -upload -it -via -terraform, -this -would -look -like -this: +Simply upload the plugins.zip to s3 and pass the relative path inside the MWAA bucket to the `plugins_s3_path` parameter. +If you zip and upload it via terraform, this would look like this: ```terraform module "airflow" { ... -plugins_s3_path = aws_s3_bucket_object.your_plugin.key -... + plugins_s3_path = aws_s3_bucket_object.your_plugin.key + ... } ``` ### Use your own networking config +If you set ``create_networking_config = false`` no subnets, eip, NAT gateway and route tables will be created. +**Be aware that you still need the networking resources to get your environment running, follow the [official documentation](https://docs.aws.amazon.com/mwaa/latest/userguide/vpc-create.html) to create them properly.** -If -you -set ``create_networking_config = false`` -no -subnets, -eip, -NAT -gateway -and -route -tables -will -be -created. -** -Be -aware -that -you -still -need -the -networking -resources -to -get -your -environment -running, -follow -the [official documentation](https://docs.aws.amazon.com/mwaa/latest/userguide/vpc-create.html) -to -create -them -properly.** ### S3 Bucket configuration - -MWAA -needs -a -S3 -bucket -to -store -the -DAG -files. -Here -is -a -minimal -configuration -for -this -S3 -bucket: +MWAA needs a S3 bucket to store the DAG files. Here is a minimal configuration for this S3 bucket: ```terraform resource "aws_s3_bucket" "mwaa" { @@ -296,21 +92,21 @@ resource "aws_s3_bucket_versioning" "mwaa" { resource "aws_s3_bucket_public_access_block" "mwaa" { # required: https://docs.aws.amazon.com/mwaa/latest/userguide/mwaa-s3-bucket.html - bucket = aws_s3_bucket.mwaa.id - block_public_acls = true - block_public_policy = true - ignore_public_acls = true + bucket = aws_s3_bucket.mwaa.id + block_public_acls = true + block_public_policy = true + ignore_public_acls = true restrict_public_buckets = true } ``` - + ## Requirements | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | > =1.0.0 | +| [terraform](#requirement\_terraform) | >=1.0.0 | | [aws](#requirement\_aws) | ~> 4.0 | ## Providers @@ -321,8 +117,7 @@ resource "aws_s3_bucket_public_access_block" "mwaa" { ## Modules -No -modules. +No modules. ## Resources From 3ff9dc048f2e067acc31352c853d7d5ef9fce09e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 10 May 2023 23:26:48 +0000 Subject: [PATCH 4/4] terraform-docs: automated action --- README.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index f0c16b5..7f65d32 100644 --- a/README.md +++ b/README.md @@ -185,12 +185,11 @@ No modules. ## Outputs -| Name | Description | -|-------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------| -| [mwaa\_arn](#output\_mwaa\_arn) | The arn of the created MWAA environment. | -| [mwaa\_execution\_role\_arn](#output\_mwaa\_execution\_role\_arn) | The IAM Role arn for MWAA Execution Role. | -| [mwaa\_service\_role\_arn](#output\_mwaa_service\_role\_arn) | The Service Role arn for MWAA environment. | +| Name | Description | +|------|-------------| +| [mwaa\_arn](#output\_mwaa\_arn) | The arn of the created MWAA environment. | +| [mwaa\_execution\_role\_arn](#output\_mwaa\_execution\_role\_arn) | The IAM Role arn for MWAA Execution Role. | | [mwaa\_nat\_gateway\_public\_ips](#output\_mwaa\_nat\_gateway\_public\_ips) | List of the ips of the nat gateways created by this module. | -| [mwaa\_webserver\_url](#output\_mwaa\_webserver\_url) | The webserver URL of the MWAA Environment. | - +| [mwaa\_service\_role\_arn](#output\_mwaa\_service\_role\_arn) | The Service Role arn for MWAA environment. | +| [mwaa\_webserver\_url](#output\_mwaa\_webserver\_url) | The webserver URL of the MWAA Environment. |