Merge pull request #14 from binbashar/fix/bucket-replication

Fix/bucket replication
binbashar · Aug 25, 2021 · 01b0617 · 01b0617
2 parents 82350ab + 42a9ed3
commit 01b0617
Show file tree

Hide file tree

Showing 7 changed files with 118 additions and 57 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -6,7 +6,7 @@ jobs:
   #
   test-static-code-and-linting:
     machine:
-      image: ubuntu-1604:202007-01 # Ubuntu 16.04, Docker v19.03.12, Docker Compose v1.26.1
+      image: ubuntu-2004:202107-02 # Ubuntu 20.04, Docker v20.10.7, Docker Compose v1.29.2,
 
       # This job has been blocked because Docker Layer Caching is not available on your plan.
       # Should upgrade if necessary.
@@ -26,31 +26,33 @@ jobs:
             git update-index --assume-unchanged "Makefile"
 
       - run:
-          name: test-terraform-format-and-docs
+          name: test-dependencies
           command: |
             #
             # Install pre-commit
-            pip install pre-commit
+            sudo -H pip3 install pre-commit
             #
             # Install terraform
-            sudo apt-get install unzip
+            sudo apt-get install unzip curl
+            curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo apt-key add -
             wget https://releases.hashicorp.com/terraform/${TERRAFORM_VER}/terraform_${TERRAFORM_VER}_linux_amd64.zip
             unzip terraform_${TERRAFORM_VER}_linux_amd64.zip
             sudo mv terraform /usr/local/bin/
             terraform --version
             #
             # Install terraform-docs
             sudo apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"
-            curl -Lo ./terraform-docs https://github.com/terraform-docs/terraform-docs/releases/download/v0.10.1/terraform-docs-v0.10.1-$(uname | tr '[:upper:]' '[:lower:]')-amd64
+            curl -Lo ./terraform-docs https://github.com/terraform-docs/terraform-docs/releases/download/v0.12.1/terraform-docs-v0.12.1-$(uname | tr '[:upper:]' '[:lower:]')-amd64
             chmod +x ./terraform-docs
             sudo mv ./terraform-docs /usr/local/bin/terraform-docs
             #
-            # Run tests
-            make pre-commit
+      - run:
+          name: test-terraform-format-and-docs
+          command: make pre-commit
 
       - run:
           name: Install awscli
-          command: sudo -H pip install awscli
+          command: sudo -H pip3 install awscli
 
       - run:
           name: Configure awscli
@@ -152,7 +154,7 @@ jobs:
   #
   test-e2e-terratests:
     machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-2004:202107-02 # Ubuntu 20.04, Docker v20.10.7, Docker Compose v1.29.2,
       docker_layer_caching: false
 
     steps:
@@ -170,11 +172,15 @@ jobs:
 
       - run:
           name: Install awscli
-          command: sudo pip install awscli
+          command: sudo -H pip3 install awscli
 
       - run:
           name: Configure awscli
           command: |
+            # AWS credentials dir
+            mkdir --parents /home/circleci/.aws/bb
+            sudo chown -R $USER:$USER /home/circleci/.aws
+
             # AWS defautl awscli profile
             aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID
             aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
@@ -184,8 +190,8 @@ jobs:
             # AWS dev awscli profile
             aws configure set role_arn arn:aws:iam::$AWS_ACCOUNT_ID_DEV:role/DeployMaster --profile $AWS_PROFILE_NAME
             aws configure set source_profile default --profile $AWS_PROFILE_NAME
+
             # moving credentials to specific project folder
-            mkdir --parents /home/circleci/.aws/bb
             cp /home/circleci/.aws/credentials /home/circleci/.aws/bb/credentials
             cp /home/circleci/.aws/config /home/circleci/.aws/bb/config
 
@@ -276,7 +282,7 @@ jobs:
   #
   release-version-with-changelog:
     machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-2004:202107-02 # Ubuntu 20.04, Docker v20.10.7, Docker Compose v1.29.2,
       docker_layer_caching: false
 
     environment:

diff --git a/Makefile b/Makefile
@@ -2,7 +2,7 @@
 SHELL         := /bin/bash
 MAKEFILE_PATH := ./Makefile
 MAKEFILES_DIR := ./@bin/makefiles
-MAKEFILES_VER := v0.1.12
+MAKEFILES_VER := v0.1.33
 PROJECT_SHORT := bb
 
 help:
@@ -20,5 +20,6 @@ init-makefiles: ## initialize makefiles
 
 -include ${MAKEFILES_DIR}/circleci/circleci.mk
 -include ${MAKEFILES_DIR}/release-mgmt/release.mk
+-include ${MAKEFILES_DIR}/terraform14/terraform14-root-context.mk
 -include ${MAKEFILES_DIR}/terraform14/terraform14.mk
 -include ${MAKEFILES_DIR}/terratest14/terratest14.mk
diff --git a/README.md b/README.md
@@ -35,57 +35,81 @@ We have a tfstate S3 Bucket per account
 
 | Name | Version |
 |------|---------|
-| terraform | >= 0.13.2 |
-| aws | ~> 3.0 |
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 0.13.2 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 3.0 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| aws.main\_region | ~> 3.0 |
-| aws.secondary\_region | ~> 3.0 |
+| <a name="provider_aws.main_region"></a> [aws.main\_region](#provider\_aws.main\_region) | ~> 3.0 |
+| <a name="provider_aws.secondary_region"></a> [aws.secondary\_region](#provider\_aws.secondary\_region) | ~> 3.0 |
+| <a name="provider_time"></a> [time](#provider\_time) | n/a |
+
+## Modules
+
+No modules.
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [aws_dynamodb_table.with_server_side_encryption](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/dynamodb_table) | resource |
+| [aws_dynamodb_table.without_server_side_encryption](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/dynamodb_table) | resource |
+| [aws_iam_policy.bucket_replication](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
+| [aws_iam_policy_attachment.bucket_replication](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy_attachment) | resource |
+| [aws_iam_role.bucket_replication](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
+| [aws_s3_bucket.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource |
+| [aws_s3_bucket.replication_bucket](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource |
+| [aws_s3_bucket_policy.bucket_replication](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_policy) | resource |
+| [aws_s3_bucket_policy.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_policy) | resource |
+| [aws_s3_bucket_policy.default-ssl-vpc](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_policy) | resource |
+| [aws_s3_bucket_public_access_block.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_public_access_block) | resource |
+| [time_sleep.wait_30_secs](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource |
+| [aws_iam_policy_document.default-ssl](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
+| [aws_iam_policy_document.default-ssl-vpc](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 
 ## Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
-| acl | The canned ACL to apply to the S3 bucket | `string` | `"private"` | no |
-| additional\_tag\_map | Additional tags for appending to each tag map | `map(string)` | `{}` | no |
-| attributes | Additional attributes (e.g. `state`) | `list(string)` | <pre>[<br>  "state"<br>]</pre> | no |
-| block\_public\_acls | Whether Amazon S3 should block public ACLs for this bucket. | `bool` | `false` | no |
-| block\_public\_policy | Whether Amazon S3 should block public bucket policies for this bucket. | `bool` | `false` | no |
-| bucket\_replication\_enabled | Enable/Disable replica for S3 bucket (for cross region replication purpose) | `bool` | `false` | no |
-| context | Default context to use for passing state between label invocations | `map(string)` | `{}` | no |
-| delimiter | Delimiter to be used between `namespace`, `environment`, `stage`, `name` and `attributes` | `string` | `"-"` | no |
-| enable\_server\_side\_encryption | Enable DynamoDB server-side encryption | `bool` | `true` | no |
-| enforce\_ssl\_requests | Enable/Disable replica for S3 bucket (for cross region replication purpose) | `bool` | `false` | no |
-| enforce\_vpc\_requests | Enable/Disable VPC endpoint for S3 bucket | `bool` | `false` | no |
-| environment | Environment, e.g. 'prod', 'staging', 'dev', 'pre-prod', 'UAT' | `string` | `""` | no |
-| force\_destroy | A boolean that indicates the S3 bucket can be destroyed even if it contains objects. These objects are not recoverable | `bool` | `false` | no |
-| ignore\_public\_acls | Whether Amazon S3 should ignore public ACLs for this bucket. | `bool` | `false` | no |
-| label\_order | The naming order of the id output and Name tag | `list(string)` | `[]` | no |
-| mfa\_delete | A boolean that indicates that versions of S3 objects can only be deleted with MFA. ( Terraform cannot apply changes of this value; https://github.com/terraform-providers/terraform-provider-aws/issues/629 ) | `bool` | `false` | no |
-| name | Solution name, e.g. 'app' or 'jenkins' | `string` | `"terraform"` | no |
-| namespace | Namespace, which could be your organization name or abbreviation, e.g. 'eg' or 'cp' | `string` | `""` | no |
-| read\_capacity | DynamoDB read capacity units | `number` | `5` | no |
-| regex\_replace\_chars | Regex to replace chars with empty string in `namespace`, `environment`, `stage` and `name`. By default only hyphens, letters and digits are allowed, all other chars are removed | `string` | `"/[^a-zA-Z0-9-]/"` | no |
-| restrict\_public\_buckets | Whether Amazon S3 should restrict public bucket policies for this bucket. | `bool` | `false` | no |
-| stage | Stage, e.g. 'prod', 'staging', 'dev', OR 'source', 'build', 'test', 'deploy', 'release' | `string` | `""` | no |
-| tags | Additional tags (e.g. `map('BusinessUnit','XYZ')` | `map(string)` | `{}` | no |
-| vpc\_ids\_list | VPC id to access the S3 bucket vía vpc endpoint. The VPCe must be in the same AWS Region as the bucket. | `list(string)` | `[]` | no |
-| write\_capacity | DynamoDB write capacity units | `number` | `5` | no |
+| <a name="input_acl"></a> [acl](#input\_acl) | The canned ACL to apply to the S3 bucket | `string` | `"private"` | no |
+| <a name="input_additional_tag_map"></a> [additional\_tag\_map](#input\_additional\_tag\_map) | Additional tags for appending to each tag map | `map(string)` | `{}` | no |
+| <a name="input_attributes"></a> [attributes](#input\_attributes) | Additional attributes (e.g. `state`) | `list(string)` | <pre>[<br>  "state"<br>]</pre> | no |
+| <a name="input_block_public_acls"></a> [block\_public\_acls](#input\_block\_public\_acls) | Whether Amazon S3 should block public ACLs for this bucket. | `bool` | `false` | no |
+| <a name="input_block_public_policy"></a> [block\_public\_policy](#input\_block\_public\_policy) | Whether Amazon S3 should block public bucket policies for this bucket. | `bool` | `false` | no |
+| <a name="input_bucket_replication_enabled"></a> [bucket\_replication\_enabled](#input\_bucket\_replication\_enabled) | Enable/Disable replica for S3 bucket (for cross region replication purpose) | `bool` | `false` | no |
+| <a name="input_context"></a> [context](#input\_context) | Default context to use for passing state between label invocations | `map(string)` | `{}` | no |
+| <a name="input_delimiter"></a> [delimiter](#input\_delimiter) | Delimiter to be used between `namespace`, `environment`, `stage`, `name` and `attributes` | `string` | `"-"` | no |
+| <a name="input_enable_server_side_encryption"></a> [enable\_server\_side\_encryption](#input\_enable\_server\_side\_encryption) | Enable DynamoDB server-side encryption | `bool` | `true` | no |
+| <a name="input_enforce_ssl_requests"></a> [enforce\_ssl\_requests](#input\_enforce\_ssl\_requests) | Enable/Disable replica for S3 bucket (for cross region replication purpose) | `bool` | `false` | no |
+| <a name="input_enforce_vpc_requests"></a> [enforce\_vpc\_requests](#input\_enforce\_vpc\_requests) | Enable/Disable VPC endpoint for S3 bucket | `bool` | `false` | no |
+| <a name="input_environment"></a> [environment](#input\_environment) | Environment, e.g. 'prod', 'staging', 'dev', 'pre-prod', 'UAT' | `string` | `""` | no |
+| <a name="input_force_destroy"></a> [force\_destroy](#input\_force\_destroy) | A boolean that indicates the S3 bucket can be destroyed even if it contains objects. These objects are not recoverable | `bool` | `false` | no |
+| <a name="input_ignore_public_acls"></a> [ignore\_public\_acls](#input\_ignore\_public\_acls) | Whether Amazon S3 should ignore public ACLs for this bucket. | `bool` | `false` | no |
+| <a name="input_label_order"></a> [label\_order](#input\_label\_order) | The naming order of the id output and Name tag | `list(string)` | `[]` | no |
+| <a name="input_logging"></a> [logging](#input\_logging) | Bucket access logging configuration. | <pre>object({<br>    bucket_name = string<br>    prefix      = string<br>  })</pre> | `null` | no |
+| <a name="input_mfa_delete"></a> [mfa\_delete](#input\_mfa\_delete) | A boolean that indicates that versions of S3 objects can only be deleted with MFA. ( Terraform cannot apply changes of this value; https://github.com/terraform-providers/terraform-provider-aws/issues/629 ) | `bool` | `false` | no |
+| <a name="input_name"></a> [name](#input\_name) | Solution name, e.g. 'app' or 'jenkins' | `string` | `"terraform"` | no |
+| <a name="input_namespace"></a> [namespace](#input\_namespace) | Namespace, which could be your organization name or abbreviation, e.g. 'eg' or 'cp' | `string` | `""` | no |
+| <a name="input_read_capacity"></a> [read\_capacity](#input\_read\_capacity) | DynamoDB read capacity units | `number` | `5` | no |
+| <a name="input_regex_replace_chars"></a> [regex\_replace\_chars](#input\_regex\_replace\_chars) | Regex to replace chars with empty string in `namespace`, `environment`, `stage` and `name`. By default only hyphens, letters and digits are allowed, all other chars are removed | `string` | `"/[^a-zA-Z0-9-]/"` | no |
+| <a name="input_restrict_public_buckets"></a> [restrict\_public\_buckets](#input\_restrict\_public\_buckets) | Whether Amazon S3 should restrict public bucket policies for this bucket. | `bool` | `false` | no |
+| <a name="input_stage"></a> [stage](#input\_stage) | Stage, e.g. 'prod', 'staging', 'dev', OR 'source', 'build', 'test', 'deploy', 'release' | `string` | `""` | no |
+| <a name="input_tags"></a> [tags](#input\_tags) | Additional tags (e.g. `map('BusinessUnit','XYZ')` | `map(string)` | `{}` | no |
+| <a name="input_vpc_ids_list"></a> [vpc\_ids\_list](#input\_vpc\_ids\_list) | VPC id to access the S3 bucket vía vpc endpoint. The VPCe must be in the same AWS Region as the bucket. | `list(string)` | `[]` | no |
+| <a name="input_write_capacity"></a> [write\_capacity](#input\_write\_capacity) | DynamoDB write capacity units | `number` | `5` | no |
 
 ## Outputs
 
 | Name | Description |
 |------|-------------|
-| dynamodb\_table\_arn | DynamoDB table ARN |
-| dynamodb\_table\_id | DynamoDB table ID |
-| dynamodb\_table\_name | DynamoDB table name |
-| s3\_bucket\_arn | S3 bucket ARN |
-| s3\_bucket\_domain\_name | S3 bucket domain name |
-| s3\_bucket\_id | S3 bucket ID |
-
+| <a name="output_dynamodb_table_arn"></a> [dynamodb\_table\_arn](#output\_dynamodb\_table\_arn) | DynamoDB table ARN |
+| <a name="output_dynamodb_table_id"></a> [dynamodb\_table\_id](#output\_dynamodb\_table\_id) | DynamoDB table ID |
+| <a name="output_dynamodb_table_name"></a> [dynamodb\_table\_name](#output\_dynamodb\_table\_name) | DynamoDB table name |
+| <a name="output_s3_bucket_arn"></a> [s3\_bucket\_arn](#output\_s3\_bucket\_arn) | S3 bucket ARN |
+| <a name="output_s3_bucket_domain_name"></a> [s3\_bucket\_domain\_name](#output\_s3\_bucket\_domain\_name) | S3 bucket domain name |
+| <a name="output_s3_bucket_id"></a> [s3\_bucket\_id](#output\_s3\_bucket\_id) | S3 bucket ID |
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 
 ## Usage

diff --git a/bucket_replication.tf b/bucket_replication.tf
@@ -84,6 +84,8 @@ resource "aws_iam_policy" "bucket_replication" {
   ]
 }
 POLICY
+
+  depends_on = [aws_s3_bucket.replication_bucket, aws_s3_bucket_public_access_block.default, time_sleep.wait_30_secs]
 }
 
 resource "aws_iam_policy_attachment" "bucket_replication" {
@@ -93,6 +95,7 @@ resource "aws_iam_policy_attachment" "bucket_replication" {
   name       = format("%s-%s-%s-role-policy-attachment", var.namespace, var.stage, var.name)
   roles      = [aws_iam_role.bucket_replication[0].name]
   policy_arn = aws_iam_policy.bucket_replication[0].arn
+  depends_on = [aws_s3_bucket.replication_bucket, time_sleep.wait_30_secs]
 }
 
 resource "aws_s3_bucket_policy" "bucket_replication" {
@@ -120,4 +123,7 @@ resource "aws_s3_bucket_policy" "bucket_replication" {
   ]
 }
 POLICY
+
+  depends_on = [aws_s3_bucket.replication_bucket, aws_s3_bucket_public_access_block.default, time_sleep.wait_30_secs]
+
 }
diff --git a/main.tf b/main.tf
@@ -36,11 +36,20 @@ resource "aws_s3_bucket" "default" {
     }
   }
 
+  dynamic "logging" {
+    for_each = var.logging == null ? [] : [1]
+    content {
+      target_bucket = var.logging["bucket_name"]
+      target_prefix = var.logging["prefix"]
+    }
+  }
+
   tags = {
     Terraform   = "true"
     Environment = var.stage
   }
 
+  depends_on = [aws_s3_bucket.replication_bucket]
 }
 
 resource "aws_s3_bucket_public_access_block" "default" {
@@ -50,6 +59,13 @@ resource "aws_s3_bucket_public_access_block" "default" {
   ignore_public_acls      = var.ignore_public_acls
   block_public_policy     = var.block_public_policy
   restrict_public_buckets = var.restrict_public_buckets
+  depends_on              = [aws_s3_bucket.default]
+
+}
+
+resource "time_sleep" "wait_30_secs" {
+  create_duration = "30s"
+  depends_on      = [aws_s3_bucket_public_access_block.default]
 }
 
 resource "aws_dynamodb_table" "with_server_side_encryption" {

diff --git a/policy.tf b/policy.tf
@@ -1,9 +1,10 @@
 resource "aws_s3_bucket_policy" "default" {
   count = var.enforce_ssl_requests ? 1 : 0
 
-  provider = aws.main_region
-  bucket   = aws_s3_bucket.default.id
-  policy   = data.aws_iam_policy_document.default-ssl.json
+  provider   = aws.main_region
+  bucket     = aws_s3_bucket.default.id
+  policy     = data.aws_iam_policy_document.default-ssl.json
+  depends_on = [aws_s3_bucket_public_access_block.default, time_sleep.wait_30_secs]
 }
 
 data "aws_iam_policy_document" "default-ssl" {
@@ -42,11 +43,10 @@ data "aws_iam_policy_document" "default-ssl" {
 resource "aws_s3_bucket_policy" "default-ssl-vpc" {
   count = var.enforce_ssl_requests && var.enforce_vpc_requests && var.vpc_ids_list != [] ? 1 : 0
 
-  provider = aws.main_region
-  bucket   = aws_s3_bucket.default.id
-
-
-  policy = data.aws_iam_policy_document.default-ssl-vpc.json
+  provider   = aws.main_region
+  bucket     = aws_s3_bucket.default.id
+  policy     = data.aws_iam_policy_document.default-ssl-vpc.json
+  depends_on = [aws_s3_bucket_public_access_block.default, time_sleep.wait_30_secs]
 }
 
 data "aws_iam_policy_document" "default-ssl-vpc" {

diff --git a/variables.tf b/variables.tf
@@ -148,3 +148,11 @@ variable "vpc_ids_list" {
   default     = []
 }
 
+variable "logging" {
+  type = object({
+    bucket_name = string
+    prefix      = string
+  })
+  default     = null
+  description = "Bucket access logging configuration."
+}