diff --git a/.gitignore b/.gitignore index 6b09e324..7537901b 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,12 @@ __pycache__/ /.pytest_cache /.mypy_cache +# Terraform specific +.terraform/ +*.tfstate +.terraform.lock.hcl +*.tfvars + # Environments .env .venv diff --git a/README.md b/README.md index ba000525..0c46d996 100644 --- a/README.md +++ b/README.md @@ -3,11 +3,12 @@ [![PyPI version](https://badge.fury.io/py/cid-cmd.svg)](https://badge.fury.io/py/cid-cmd) ## Welcome to Cloud Intelligence Dashboards (CUDOS Framework) automation repository -This repository contains CloudFormation templates and Command Line tool (cid-cmd) for managing various dashboards provided in AWS Well Architected LAB [Cloud Intelligence Dashboards](https://www.wellarchitectedlabs.com/cost/200_labs/200_cloud_intelligence/). +This repository contains CloudFormation templates, Terraform modules, and a Command Line tool (cid-cmd) for managing various dashboards provided in AWS Well Architected LAB [Cloud Intelligence Dashboards](https://www.wellarchitectedlabs.com/cost/200_labs/200_cloud_intelligence/). There are several ways we can manage dashboards: -1. CloudFormation Template (using cid-cmd tool in lambda) -2. Using cid-cmd tool from command line +1. [CloudFormation Template](./cfn-templates/cid-cfn.yml) (using cid-cmd tool in lambda) +2. [Terraform module](./terraform-modules/cid-dashboards/README.md) (wrapper around CloudFormation Template) +3. Using cid-cmd tool from command line We recommend cid-cmd tool via [AWS CloudShell](https://console.aws.amazon.com/cloudshell/home). diff --git a/terraform-modules/cid-dashboards/.terraform-docs.yml b/terraform-modules/cid-dashboards/.terraform-docs.yml new file mode 100644 index 00000000..79772427 --- /dev/null +++ b/terraform-modules/cid-dashboards/.terraform-docs.yml @@ -0,0 +1,12 @@ +output: + file: README.md + mode: inject +formatter: "markdown document" +sections: + hide: + - providers + - modules +sort: + by: required +settings: + anchor: false \ No newline at end of file diff --git a/terraform-modules/cid-dashboards/README.md b/terraform-modules/cid-dashboards/README.md new file mode 100644 index 00000000..2159726d --- /dev/null +++ b/terraform-modules/cid-dashboards/README.md @@ -0,0 +1,164 @@ +# CID Terraform Module: cid-dashboards + +Terraform module to deploy CID dashboards. This module is a wrapper around CloudFormation +to allow you to deploy CID dashboards using your existing Terraform workflows. Under the +hood, the module will deploy a CloudFormation stack which will provision the necessary +resources and a custom Lambda function to create the dashboards using `cid-cmd`. + +## Before You Start + + - Existing S3 bucket to upload the CloudFormation template + - Complete prerequisites in [Before You Start](../../README.md#before-you-start) including CUR and Quicksight setup + +## Example Usage + +```hcl +module "cid_dashboards" { + source = "github.com/aws-samples/aws-cudos-framework-deployment//terraform-modules/cid-dashboards" + + stack_name = "Cloud-Intelligence-Dashboards" + template_bucket = "UPDATEME" + stack_parameters = { + "PrerequisitesQuickSight" = "yes" + "PrerequisitesQuickSightPermissions" = "yes" + "QuickSightUser" = "UPDATEME" + "DeployCUDOSDashboard" = "yes" + "DeployCostIntelligenceDashboard" = "yes" + "DeployKPIDashboard" = "yes" + } +} +``` + +## Version Locking + +For production deployments, you should lock the version of this module to a release tag to better +control when and what updates are made. To specify the release tag to use, append `?ref=VERSION` +to the module source. For example, the following source reference will use the Terraform module +and Cloudformation template from version 0.2.13 of this module: + +``` +source = "github.com/aws-samples/aws-cudos-framework-deployment//terraform-modules/cid-dashboards?ref=0.2.13" +``` + +For a complete list of release tags, visit https://github.com/aws-samples/aws-cudos-framework-deployment/tags. + +## Troubleshooting + +Because this module is primarily a wrapper for CloudFormation, Terraform output may not be sufficient +for debugging if deployment fails. For additional troubleshooting information, refer to the CloudFormation +console for details on stack operation, resources, and error output. Additionally, you can refer to logs +for the custom resource Lambda function "CidCustomDashboardResource"if dashboards fail to deploy. + + +## Requirements + +The following requirements are needed by this module: + +- terraform (>= 1.0) + +- aws (>= 3.0) + +## Resources + +The following resources are used by this module: + +- [aws_cloudformation_stack.cid](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudformation_stack) (resource) +- [aws_s3_object.template](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_object) (resource) +- [aws_s3_bucket.template_bucket](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/s3_bucket) (data source) + +## Required Inputs + +The following input variables are required: + +### stack\_name + +Description: CloudFormation stack name for Cloud Intelligence Dashboards deployment + +Type: `string` + +### stack\_parameters + +Description: CloudFormation stack parameters. For the full list of available parameters, refer to +https://github.com/aws-samples/aws-cudos-framework-deployment/blob/main/cfn-templates/cid-cfn.yml. +For most setups, you will want to set the following parameters: + - PrerequisitesQuickSight: yes/no + - PrerequisitesQuickSightPermissions: yes/no + - QuickSightUser: Existing quicksight user + - QuickSightDataSetRefreshSchedule: Cron expression to refresh spice datasets daily outside of business hours. Default is 4 AM UTC, which should work for most customers in US and EU time zones + - CURBucketPath: Leave as default is if CUR was created with CloudFormation (cur-aggregation.yaml). If it was a manually created CUR, the path entered below must be for the directory that contains the years partition (s3://curbucketname/prefix/curname/curname/). + - OptimizationDataCollectionBucketPath: The S3 path to the bucket created by the Cost Optimization Data Collection Lab. The path will need point to a folder containing /optics-data-collector folder. Required for TAO and Compute Optimizer dashboards. + - DataBuketsKmsKeyArns: Comma-delimited list of KMS key ARNs ("*" is also valid). Include any KMS keys used to encrypt your CUR or Cost Optimization Data S3 data + - DeployCUDOSDashboard: (yes/no, default no) + - DeployCostIntelligenceDashboard: (yes/no, default no) + - DeployKPIDashboard: (yes/no, default no) + - DeployTAODashboard: (yes/no, default no) + - DeployComputeOptimizerDashboard: (yes/no, default no) + +Type: `map(string)` + +### template\_bucket + +Description: S3 bucket where the Cloudformation template will be uploaded. Must already exist and be in the same region as the stack. + +Type: `string` + +## Optional Inputs + +The following input variables are optional (have default values): + +### stack\_iam\_role + +Description: The ARN of an IAM role that AWS CloudFormation assumes to create the stack (default behavior is to use the previous role if available, or current user permissions otherwise). + +Type: `string` + +Default: `null` + +### stack\_notification\_arns + +Description: A list of SNS topic ARNs to publish stack related events. + +Type: `list(string)` + +Default: `[]` + +### stack\_policy\_body + +Description: String containing the stack policy body. Conflicts with stack\_policy\_url. + +Type: `string` + +Default: `null` + +### stack\_policy\_url + +Description: Location of a file containing the stack policy body. Conflicts with stack\_policy\_body. + +Type: `string` + +Default: `null` + +### stack\_tags + +Description: Tag key-value pairs to apply to the stack + +Type: `map(string)` + +Default: `null` + +### template\_key + +Description: Name of the S3 path/key where the Cloudformation template will be created. Defaults to cid-cfn.yml + +Type: `string` + +Default: `"cid-cfn.yml"` + +## Outputs + +The following outputs are exported: + +### stack\_outputs + +Description: CloudFormation stack outputs (map of strings) + \ No newline at end of file diff --git a/terraform-modules/cid-dashboards/main.tf b/terraform-modules/cid-dashboards/main.tf new file mode 100644 index 00000000..0cbceac4 --- /dev/null +++ b/terraform-modules/cid-dashboards/main.tf @@ -0,0 +1,22 @@ +data "aws_s3_bucket" "template_bucket" { + bucket = var.template_bucket +} + +resource "aws_s3_object" "template" { + bucket = data.aws_s3_bucket.template_bucket.bucket + key = var.template_key + source = "${path.module}/../../cfn-templates/cid-cfn.yml" + etag = filemd5("${path.module}/../../cfn-templates/cid-cfn.yml") +} + +resource "aws_cloudformation_stack" "cid" { + name = var.stack_name + template_url = "https://${data.aws_s3_bucket.template_bucket.bucket_regional_domain_name}/${aws_s3_object.template.key}?etag=${aws_s3_object.template.etag}" + capabilities = ["CAPABILITY_NAMED_IAM"] + parameters = var.stack_parameters + iam_role_arn = var.stack_iam_role + policy_body = var.stack_policy_body + policy_url = var.stack_policy_url + notification_arns = var.stack_notification_arns + tags = var.stack_tags +} diff --git a/terraform-modules/cid-dashboards/outputs.tf b/terraform-modules/cid-dashboards/outputs.tf new file mode 100644 index 00000000..2b127963 --- /dev/null +++ b/terraform-modules/cid-dashboards/outputs.tf @@ -0,0 +1,4 @@ +output "stack_outputs" { + description = "CloudFormation stack outputs (map of strings)" + value = aws_cloudformation_stack.cid.outputs +} \ No newline at end of file diff --git a/terraform-modules/cid-dashboards/variables.tf b/terraform-modules/cid-dashboards/variables.tf new file mode 100644 index 00000000..9442c1e8 --- /dev/null +++ b/terraform-modules/cid-dashboards/variables.tf @@ -0,0 +1,66 @@ +variable "stack_name" { + type = string + description = "CloudFormation stack name for Cloud Intelligence Dashboards deployment" +} + +variable "template_bucket" { + type = string + description = "S3 bucket where the Cloudformation template will be uploaded. Must already exist and be in the same region as the stack." +} + +variable "template_key" { + type = string + description = "Name of the S3 path/key where the Cloudformation template will be created. Defaults to cid-cfn.yml" + default = "cid-cfn.yml" +} + +variable "stack_parameters" { + type = map(string) + description = <<-EOF + CloudFormation stack parameters. For the full list of available parameters, refer to + https://github.com/aws-samples/aws-cudos-framework-deployment/blob/main/cfn-templates/cid-cfn.yml. + For most setups, you will want to set the following parameters: + - PrerequisitesQuickSight: yes/no + - PrerequisitesQuickSightPermissions: yes/no + - QuickSightUser: Existing quicksight user + - QuickSightDataSetRefreshSchedule: Cron expression to refresh spice datasets daily outside of business hours. Default is 4 AM UTC, which should work for most customers in US and EU time zones + - CURBucketPath: Leave as default is if CUR was created with CloudFormation (cur-aggregation.yaml). If it was a manually created CUR, the path entered below must be for the directory that contains the years partition (s3://curbucketname/prefix/curname/curname/). + - OptimizationDataCollectionBucketPath: The S3 path to the bucket created by the Cost Optimization Data Collection Lab. The path will need point to a folder containing /optics-data-collector folder. Required for TAO and Compute Optimizer dashboards. + - DataBuketsKmsKeyArns: Comma-delimited list of KMS key ARNs ("*" is also valid). Include any KMS keys used to encrypt your CUR or Cost Optimization Data S3 data + - DeployCUDOSDashboard: (yes/no, default no) + - DeployCostIntelligenceDashboard: (yes/no, default no) + - DeployKPIDashboard: (yes/no, default no) + - DeployTAODashboard: (yes/no, default no) + - DeployComputeOptimizerDashboard: (yes/no, default no) + EOF +} + +variable "stack_tags" { + type = map(string) + description = "Tag key-value pairs to apply to the stack" + default = null +} + +variable "stack_policy_body" { + type = string + description = "String containing the stack policy body. Conflicts with stack_policy_url." + default = null +} + +variable "stack_policy_url" { + type = string + description = "Location of a file containing the stack policy body. Conflicts with stack_policy_body." + default = null +} + +variable "stack_notification_arns" { + type = list(string) + description = "A list of SNS topic ARNs to publish stack related events." + default = [] +} + +variable "stack_iam_role" { + type = string + description = "The ARN of an IAM role that AWS CloudFormation assumes to create the stack (default behavior is to use the previous role if available, or current user permissions otherwise)." + default = null +} \ No newline at end of file diff --git a/terraform-modules/cid-dashboards/versions.tf b/terraform-modules/cid-dashboards/versions.tf new file mode 100644 index 00000000..0bedb0e4 --- /dev/null +++ b/terraform-modules/cid-dashboards/versions.tf @@ -0,0 +1,9 @@ +terraform { + required_version = ">= 1.0" + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 3.0" + } + } +} \ No newline at end of file diff --git a/terraform-modules/terraform-test.bats b/terraform-modules/terraform-test.bats new file mode 100644 index 00000000..61ac24f5 --- /dev/null +++ b/terraform-modules/terraform-test.bats @@ -0,0 +1,102 @@ +#!/bin/bats + +# This is a Bats test file. See https://bats-core.readthedocs.io + +# Run: +# bats $thisfile + +# Debug: +# bats $thisfile --show-output-of-passing-tests + +@test "setup environment" { + + #Vars + export account_id=$(aws sts get-caller-identity --query Account --output text) + export qs_username=$(aws quicksight list-users --aws-account-id $account_id --namespace default --query 'UserList[0].UserName' --output text) + export template_bucket=test-cid-tf-template-$account_id + export cur_bucket=test-cid-tf-cur-$account_id + + # Create a tmp bucket to store CFN template + if aws s3api head-bucket --bucket "$template_bucket" 2>/dev/null; then + echo 'Template Bucket exist' + else + echo 'Creating bucket for template' + aws s3api create-bucket --bucket $template_bucket + fi + + # Create a tmp bucket to store CFN template + if aws s3api head-bucket --bucket "$cur_bucket" 2>/dev/null; then + echo 'CUR Bucket exist' + else + aws s3api create-bucket --bucket $cur_bucket + fi + +} + + +@test "generate terraform manifest" { + + export git_base_url=$(git config --get remote.origin.url | cut -d '@' -f2 | sed s/.git// | sed s@https://@@ | tr : /) + export git_branch=$(git branch --show-current) + + # Create a tf file + cat >main.tf <