Update Terraform code to match CloudFormation

LeanerCloud · Sep 1, 2021 · d9ad5d0 · d9ad5d0
1 parent 1836c81
commit d9ad5d0
Show file tree

Hide file tree

Showing 11 changed files with 208 additions and 284 deletions.
diff --git a/README.md b/README.md
@@ -6,16 +6,27 @@ See [https://github.com/autospotting/autospotting](https://github.com/autospotti
 
 ## Using the terraform-aws-autospotting module
 
-* [Sources](#sources)
-* [Setting variables](#setting-variables)
-* [Using custom binaries](#using-custom-binaries)
-  * [From local file](#from-local-file)
-  * [From S3](#from-s3)
-* [Multiple instances](#multiple-instances)
+- [AutoSpotting](#autospotting)
+  - [Using the terraform-aws-autospotting module](#using-the-terraform-aws-autospotting-module)
+    - [Sources](#sources)
+    - [Setting variables](#setting-variables)
+    - [Multiple installations](#multiple-installations)
+    - [Logs or Troubleshooting](#logs-or-troubleshooting)
 
 ### Sources
 
-This module can be used from the Terraform Registry or directly from this repository. The AutoSpotting binary defaults to the latest nightly build hosted on S3.
+This module can be used from the Terraform Registry or directly from this
+repository.
+
+The installer defaults to the latest stable version of AutoSpotting available on
+the AWS Marketplace. In order to use it you first need to
+[subscribe](https://aws.amazon.com/marketplace/pp/prodview-6uj4pruhgmun6) to
+it.
+
+Alternatively you can build your own image as described in the
+[documentation](https://github.com/AutoSpotting/AutoSpotting/blob/master/CUSTOM_BUILDS.md),
+but this comes without any support unless you're trying to contribute code via pull
+requests.
 
 Using from the Terraform Registry:
 
@@ -36,46 +47,13 @@ module "autospotting" {
 
 Notes:
 
-- The official (nightly) AutoSpotting binary is hosted in the `cloudprowess` S3 bucket in AWS region `us-east-1`. When the source of the Lambda function is an S3 bucket, both the function and the bucket must be in the same region.
-
-  - If you are already using the AWS provider in the `us-east-1` region, no additional configuration is required:
-
-      ```hcl
-      provider "aws" {
-        region  = "us-east-1"
-        profile = "default"
-      }
-
-      module "autospotting" {
-        source = "github.com/autospotting/terraform-aws-autospotting"
-      }
-      ```
-
-  - If you are using the AWS provider in another region, you must either (1) alias another AWS provider in the `us-east-1` region, (2) copy the binary into an S3 bucket in the region you are already using, or (3) provide a local binary. Here, we demonstrate using an alias:
-
-      ```hcl
-      provider "aws" {
-        region  = "eu-central-1"
-        profile = "default"
-      }
-
-      provider "aws" {
-        alias   = "us"
-        region  = "us-east-1"
-        profile = "default"
-      }
-
-      module "autospotting" {
-        source = "github.com/autospotting/terraform-aws-autospotting"
-        providers = {
-          aws = aws.us
-        }
-      }
-      ```
-
-  - The Lambda function can run in any supported region
-
-- New releases of this module only work with Terraform 0.12 or newer.
+- If the first run gives this error message `Error: Provider produced
+  inconsistent final plan`, this is expected because we generate a Dockerfile
+  from Terraform (it's a long story). Just re-run Terraform and it should work
+  on a second run. Sorry about this!
+- At runtime the code will run against all other enabled regions.
+- New releases of this module have been tested with Terraform 1.0 or newer, YMMV
+  with older versions.
 
 ### Setting variables
 
@@ -84,7 +62,6 @@ Available variables are defined in the [variables file](variables.tf). To change
 ```hcl
 module "autospotting" {
   source                                = "github.com/autospotting/terraform-aws-autospotting"
-  autospotting_regions_enabled          = "eu*,us*"
   autospotting_min_on_demand_percentage = "33.3"
   lambda_memory_size                    = 1024
 }
@@ -94,41 +71,11 @@ Or you can pass them in on the command line:
 
 ``` shell
  terraform apply \
-   -var autospotting_regions_enabled="eu*,us*" \
    -var autospotting_min_on_demand_percentage="33.3" \
    -var lambda_memory_size=1024
 ```
 
-### Using custom binaries
-
-The `lambda.zip` file can be generated by building it locally. Further instructions are available in the [AutoSpotting repo](https://github.com/AutoSpotting/AutoSpotting/blob/master/CUSTOM_BUILDS.md).
-
-You can also download the latest official nightly build available [here](https://cloudprowess.s3.amazonaws.com/nightly/lambda.zip).
-
-#### From local file
-
-If you store the file locally:
-
-```hcl
-module "autospotting" {
-  source         = "github.com/autospotting/terraform-aws-autospotting"
-  lambda_zipname = "lambda.zip"
-}
-```
-
-#### From S3
-
-Instead of using a local ZIP file you can refer to the Lambda code in a location in S3:
-
-```hcl
-module "autospotting" {
-  source           = "github.com/autospotting/terraform-aws-autospotting"
-  lambda_s3_bucket = "lambda-releases"
-  lambda_s3_key    = "lambda.zip"
-}
-```
-
-### Multiple instances
+### Multiple installations
 
 You can change the names of the resources terraform will create – or run multiple instances of autospotting that target different ASGs – by using the label variables:
 

diff --git a/examples/main.tf b/examples/main.tf
@@ -1,5 +1,6 @@
 provider "aws" {
-  region = "us-east-1"
+  region = "eu-west-1"
+  # region = "us-east-1"
 }
 
 module "autospotting-test" {
@@ -10,7 +11,9 @@ module "autospotting-test" {
   # all regions. After this value is changed, Terraform apply needs to be
   # executed twice because of the way we generate some Terraform code from
   # Terraform itself.
-  # autospotting_regions_enabled = ["us-east-1", "eu-west-1"]
+  autospotting_regions_enabled                          = ["us-east-1", "eu-west-1"]
+  autospotting_disable_event_based_instance_replacement = true
+  autospotting_ebs_gp2_conversion_threshold             = 100
 }
 
 # This can be used to install a second instance of AutoSpotting with different

diff --git a/main.tf b/main.tf
@@ -22,13 +22,12 @@ module "aws_lambda_function" {
 
   label_context = module.label.context
 
-  lambda_zipname     = var.lambda_zipname
-  lambda_s3_bucket   = var.lambda_s3_bucket
-  lambda_s3_key      = var.lambda_s3_key
-  lambda_runtime     = var.lambda_runtime
-  lambda_timeout     = var.lambda_timeout
-  lambda_memory_size = var.lambda_memory_size
-  lambda_tags        = var.lambda_tags
+  lambda_source_ecr       = var.lambda_source_ecr
+  lambda_source_image     = var.lambda_source_image
+  lambda_source_image_tag = var.lambda_source_image_tag
+  lambda_timeout          = var.lambda_timeout
+  lambda_memory_size      = var.lambda_memory_size
+  lambda_tags             = var.lambda_tags
 
   sqs_fifo_queue_name = "${module.label.id}.fifo"
 

diff --git a/modules/lambda/docker.tf b/modules/lambda/docker.tf
@@ -0,0 +1,48 @@
+
+resource "aws_ecr_repository" "autospotting" {
+  name                 = "autospotting-${module.label.id}"
+  image_tag_mutability = "MUTABLE"
+
+  image_scanning_configuration {
+    scan_on_push = true
+  }
+  timeouts {
+    delete = "2m"
+  }
+}
+
+data "aws_ecr_authorization_token" "source" {
+  registry_id = split(".", var.lambda_source_ecr)[0]
+
+}
+data "aws_ecr_authorization_token" "destination" {}
+
+provider "docker" {
+  registry_auth {
+    address  = split("/", aws_ecr_repository.autospotting.repository_url)[0]
+    username = data.aws_ecr_authorization_token.destination.user_name
+    password = data.aws_ecr_authorization_token.destination.password
+  }
+}
+
+resource "local_file" "Dockerfile" {
+  content  = "FROM ${var.lambda_source_ecr}/${var.lambda_source_image}:${var.lambda_source_image_tag}"
+  filename = "${path.module}/docker/Dockerfile"
+}
+
+resource "docker_registry_image" "destination" {
+  name          = "${aws_ecr_repository.autospotting.repository_url}:${var.lambda_source_image_tag}"
+  keep_remotely = false
+  build {
+    context = "${path.module}/docker"
+
+    auth_config {
+      host_name = var.lambda_source_ecr
+      user_name = data.aws_ecr_authorization_token.source.user_name
+      password  = data.aws_ecr_authorization_token.source.password
+    }
+  }
+  depends_on = [
+    local_file.Dockerfile,
+  ]
+}
diff --git a/modules/lambda/docker/.keep b/modules/lambda/docker/.keep
diff --git a/modules/lambda/docker/Dockerfile b/modules/lambda/docker/Dockerfile
@@ -0,0 +1 @@
+FROM 709825985650.dkr.ecr.us-east-1.amazonaws.com/cloudutil/autospotting:1.0.6
diff --git a/modules/lambda/fargate.tf b/modules/lambda/fargate.tf
@@ -0,0 +1,64 @@
+
+data "aws_availability_zones" "available" {
+  state = "available"
+}
+
+
+module "vpc" {
+  source  = "terraform-aws-modules/vpc/aws"
+  version = "~> 2.21"
+
+  name = "autospotting-${module.label.id}"
+
+  cidr = "10.0.0.0/24"
+
+  azs            = [data.aws_availability_zones.available.names[0], data.aws_availability_zones.available.names[1]]
+  public_subnets = ["10.0.0.0/25", "10.0.0.128/25"]
+
+  enable_nat_gateway   = false
+  enable_dns_hostnames = true
+  enable_dns_support   = true
+  map_public_ip_on_launch = true
+}
+
+resource "aws_ecs_cluster" "autospotting" {
+  name               = "autospotting-${module.label.id}"
+  capacity_providers = ["FARGATE"]
+  default_capacity_provider_strategy {
+    capacity_provider = "FARGATE"
+  }
+}
+
+module "ecs-task-definition" {
+  source  = "umotif-public/ecs-fargate-task-definition/aws"
+  version = "~> 1.0.0"
+
+  enabled              = true
+  name_prefix          = "autospotting-${module.label.id}"
+  task_container_image = "${aws_ecr_repository.autospotting.repository_url}:${var.lambda_source_image_tag}"
+  container_name       = "autospotting-${module.label.id}"
+
+  cloudwatch_log_group_name = aws_cloudwatch_log_group.autospotting.name
+}
+
+resource "aws_cloudwatch_log_group" "autospotting"{
+    name = "autospotting-${module.label.id}"
+    retention_in_days= 7
+}
+
+module "ecs-fargate-scheduled-task" {
+  source  = "umotif-public/ecs-fargate-scheduled-task/aws"
+  version = "~> 1.0.0"
+
+  name_prefix = "test-scheduled-task"
+
+  ecs_cluster_arn = aws_ecs_cluster.autospotting.arn
+
+  task_role_arn      = module.ecs-task-definition.task_role_arn
+  execution_role_arn = module.ecs-task-definition.execution_role_arn
+
+  event_target_task_definition_arn = module.ecs-task-definition.task_definition_arn
+  event_rule_schedule_expression   = "rate(1 hour)"
+  event_target_subnets             = module.vpc.public_subnets
+  event_target_assign_public_ip = true
+  }
diff --git a/modules/lambda/main.tf b/modules/lambda/main.tf
@@ -1,20 +1,27 @@
+terraform {
+  required_providers {
+    docker = {
+      source  = "kreuzwerker/docker"
+      version = "~> 2.0"
+    }
+  }
+}
+
 module "label" {
   source  = "git::https://github.com/cloudposse/terraform-null-label.git?ref=0.21.0"
   context = var.label_context
 }
 
+data "aws_caller_identity" "current" {}
+
 resource "aws_lambda_function" "autospotting" {
-  function_name    = "autospotting-lambda-${module.label.id}"
-  filename         = var.lambda_zipname
-  source_code_hash = var.lambda_zipname == null ? null : filebase64sha256(var.lambda_zipname)
-  s3_bucket        = var.lambda_zipname == null ? var.lambda_s3_bucket : null
-  s3_key           = var.lambda_zipname == null ? var.lambda_s3_key : null
-  role             = aws_iam_role.autospotting_role.arn
-  runtime          = var.lambda_runtime
-  timeout          = var.lambda_timeout
-  handler          = "AutoSpotting"
-  memory_size      = var.lambda_memory_size
-  tags             = merge(var.lambda_tags, module.label.tags)
+  function_name = "autospotting-lambda-${module.label.id}"
+  package_type  = "Image"
+  image_uri     = "${aws_ecr_repository.autospotting.repository_url}:${var.lambda_source_image_tag}"
+  role          = aws_iam_role.autospotting_role.arn
+  timeout       = var.lambda_timeout
+  memory_size   = var.lambda_memory_size
+  tags          = merge(var.lambda_tags, module.label.tags)
 
   environment {
     variables = {
@@ -42,6 +49,9 @@ resource "aws_lambda_function" "autospotting" {
       TERMINATION_NOTIFICATION_ACTION          = var.autospotting_termination_notification_action
     }
   }
+  depends_on = [
+    docker_registry_image.destination,
+  ]
 }
 
 data "aws_iam_policy_document" "lambda_policy" {
@@ -77,6 +87,8 @@ data "aws_iam_policy_document" "autospotting_policy" {
       "autoscaling:SuspendProcesses",
       "autoscaling:TerminateInstanceInAutoScalingGroup",
       "autoscaling:UpdateAutoScalingGroup",
+      "aws-marketplace:MeterUsage",
+      "aws-marketplace:RegisterUsage",
       "cloudformation:Describe*",
       "ec2:CreateTags",
       "ec2:DeleteTags",
@@ -105,14 +117,27 @@ data "aws_iam_policy_document" "autospotting_policy" {
     ]
     resources = [aws_sqs_queue.autospotting_fifo_queue.arn]
   }
+  statement {
+    actions = [
+      "ssm:GetParameter",
+      "ssm:PutParameter",
+    ]
+    resources = ["arn:aws:ssm:us-east-1:${data.aws_caller_identity.current.account_id}:parameter/autospotting-metering"]
+  }
 }
 
-resource "aws_iam_role_policy" "autospotting_policy" {
-  name   = "policy_for_${module.label.id}"
+resource "aws_iam_role_policy" "autospotting_policy_lambda" {
+  name   = "policy_for_lambda_${module.label.id}"
   role   = aws_iam_role.autospotting_role.id
   policy = data.aws_iam_policy_document.autospotting_policy.json
 }
 
+resource "aws_iam_role_policy" "autospotting_policy_fargate" {
+  name   = "policy_for_fargate_${module.label.id}"
+  role   = module.ecs-task-definition.task_role_id
+  policy = data.aws_iam_policy_document.autospotting_policy.json
+}
+
 resource "aws_sqs_queue" "autospotting_fifo_queue" {
   name                        = var.sqs_fifo_queue_name
   content_based_deduplication = true
@@ -124,5 +149,5 @@ resource "aws_sqs_queue" "autospotting_fifo_queue" {
 resource "aws_lambda_event_source_mapping" "autospotting_lambda_event_source_mapping" {
   event_source_arn = aws_sqs_queue.autospotting_fifo_queue.arn
   function_name    = aws_lambda_function.autospotting.arn
-  depends_on       = [aws_iam_role_policy.autospotting_policy]
+  depends_on       = [aws_iam_role_policy.autospotting_policy_lambda]
 }