From 46d60e1893f56dca4636bc5fc626f120d2dd8d93 Mon Sep 17 00:00:00 2001 From: Ryan Deivert Date: Thu, 2 Apr 2020 17:04:24 -0700 Subject: [PATCH] updating tf_lambda module to remove extra resources --- .../modules/tf_lambda/README.md | 12 +-- .../modules/tf_lambda/cloudwatch.tf | 54 +++++------- .../_infrastructure/modules/tf_lambda/iam.tf | 14 ++- .../_infrastructure/modules/tf_lambda/main.tf | 86 ++++--------------- .../modules/tf_lambda/output.tf | 47 ++-------- .../modules/tf_lambda/variables.tf | 14 +-- .../modules/tf_scheduled_queries/lambda.tf | 14 ++- 7 files changed, 65 insertions(+), 176 deletions(-) diff --git a/streamalert_cli/_infrastructure/modules/tf_lambda/README.md b/streamalert_cli/_infrastructure/modules/tf_lambda/README.md index 26eb298b8..e89650166 100644 --- a/streamalert_cli/_infrastructure/modules/tf_lambda/README.md +++ b/streamalert_cli/_infrastructure/modules/tf_lambda/README.md @@ -8,10 +8,10 @@ This Terraform module creates a single AWS Lambda function and its related compo * CloudWatch log group * CloudWatch metric alarms related to Lambda -All StreamAlert Lambda functions will eventually leverage this module. +All StreamAlert Lambda functions should leverage this module. The created IAM role has permission to publish CloudWatch logs and metrics. To add function-specific -permissions, attach/inline them to the created IAM role. +permissions, attach them to the created IAM role. ## Example ```hcl @@ -25,13 +25,11 @@ module "alert_processor" { } // Commonly used optional variables - enabled = true description = "Function Description" memory_size_mb = 128 timeout_sec = 60 vpc_subnet_ids = ["abc"] vpc_security_group_ids = ["id0"] - aliased_version = 1 log_retention_days = 14 alarm_actions = ["SNS_ARN"] errors_alarm_threshold = 1 @@ -57,8 +55,4 @@ data "aws_iam_policy_document" "policy" { For a complete list of available options and their descriptions, see [`variables.tf`](variables.tf). ## Outputs -If your Lambda function is in a VPC, `function_vpc_arn` is the ARN of the generated Lambda -function. Otherwise, it will be `function_no_vpc_arn`. (This split is a workaround for a -[Terraform bug](https://github.com/terraform-providers/terraform-provider-aws/issues/443)). - -This module also exports the `role_arn` and `role_id` for the Lambda execution role. +This module exports the `function_arn` for the Lambda function, along with the `role_arn`, and `role_id` for the Lambda execution role. diff --git a/streamalert_cli/_infrastructure/modules/tf_lambda/cloudwatch.tf b/streamalert_cli/_infrastructure/modules/tf_lambda/cloudwatch.tf index 7fb983f55..1eb179789 100644 --- a/streamalert_cli/_infrastructure/modules/tf_lambda/cloudwatch.tf +++ b/streamalert_cli/_infrastructure/modules/tf_lambda/cloudwatch.tf @@ -3,33 +3,25 @@ // CloudWatch event to trigger Lambda on a regular schedule (if applicable) resource "aws_cloudwatch_event_rule" "invocation_schedule" { - count = var.enabled && local.schedule_enabled ? 1 : 0 - name = "${var.function_name}_schedule" - description = "Invokes ${var.function_name} at ${var.schedule_expression}" + count = local.schedule_enabled ? 1 : 0 + name = "${aws_lambda_function.function.function_name}_schedule" + description = "Invokes ${aws_lambda_function.function.function_name} at ${var.schedule_expression}" schedule_expression = var.schedule_expression tags = local.tags } -resource "aws_cloudwatch_event_target" "invoke_lambda_vpc" { - count = var.enabled && local.schedule_enabled && local.vpc_enabled ? 1 : 0 - rule = aws_cloudwatch_event_rule.invocation_schedule[0].name - arn = aws_lambda_alias.alias_vpc[0].arn - input = jsonencode(var.lambda_input_event) -} - -resource "aws_cloudwatch_event_target" "invoke_lambda_no_vpc" { - count = var.enabled && local.schedule_enabled && false == local.vpc_enabled ? 1 : 0 - rule = aws_cloudwatch_event_rule.invocation_schedule[0].name - arn = aws_lambda_alias.alias_no_vpc[0].arn +resource "aws_cloudwatch_event_target" "invoke_lambda" { + count = local.schedule_enabled ? 1 : 0 + rule = aws_cloudwatch_event_rule.invocation_schedule.name + arn = aws_lambda_alias.alias.arn input = jsonencode(var.lambda_input_event) } // CloudWatch log group with configurable retention, tagging, and metric filters resource "aws_cloudwatch_log_group" "lambda_log_group" { - count = var.enabled ? 1 : 0 - name = "/aws/lambda/${var.function_name}" + name = "/aws/lambda/${aws_lambda_function.function.function_name}" retention_in_days = var.log_retention_days tags = local.tags @@ -38,8 +30,8 @@ resource "aws_cloudwatch_log_group" "lambda_log_group" { // Generic CloudWatch metric alarms related to this function resource "aws_cloudwatch_metric_alarm" "lambda_invocation_errors" { - count = var.enabled && var.errors_alarm_enabled ? 1 : 0 - alarm_name = "${var.function_name}_invocation_errors" + count = var.errors_alarm_enabled ? 1 : 0 + alarm_name = "${aws_lambda_function.function.function_name}_invocation_errors" namespace = "AWS/Lambda" metric_name = "Errors" statistic = "Sum" @@ -47,20 +39,20 @@ resource "aws_cloudwatch_metric_alarm" "lambda_invocation_errors" { threshold = var.errors_alarm_threshold evaluation_periods = var.errors_alarm_evaluation_periods period = var.errors_alarm_period_secs - alarm_description = "StreamAlert Lambda Invocation Errors: ${var.function_name}" + alarm_description = "StreamAlert Lambda Invocation Errors: ${aws_lambda_function.function.function_name}" alarm_actions = var.alarm_actions dimensions = { - FunctionName = var.function_name - Resource = "${var.function_name}:${var.alias_name}" + FunctionName = aws_lambda_function.function.function_name + Resource = "${aws_lambda_function.function.function_name}:${aws_lambda_alias.alias.name}" } tags = local.tags } resource "aws_cloudwatch_metric_alarm" "lambda_throttles" { - count = var.enabled && var.throttles_alarm_enabled ? 1 : 0 - alarm_name = "${var.function_name}_throttles" + count = var.throttles_alarm_enabled ? 1 : 0 + alarm_name = "${aws_lambda_function.function.function_name}_throttles" namespace = "AWS/Lambda" metric_name = "Throttles" statistic = "Sum" @@ -68,20 +60,20 @@ resource "aws_cloudwatch_metric_alarm" "lambda_throttles" { threshold = var.throttles_alarm_threshold evaluation_periods = var.throttles_alarm_evaluation_periods period = var.throttles_alarm_period_secs - alarm_description = "StreamAlert Lambda Throttles: ${var.function_name}" + alarm_description = "StreamAlert Lambda Throttles: ${aws_lambda_function.function.function_name}" alarm_actions = var.alarm_actions dimensions = { - FunctionName = var.function_name - Resource = "${var.function_name}:${var.alias_name}" + FunctionName = aws_lambda_function.function.function_name + Resource = "${aws_lambda_function.function.function_name}:${aws_lambda_alias.alias.name}" } tags = local.tags } resource "aws_cloudwatch_metric_alarm" "streamalert_lambda_iterator_age" { - count = var.enabled && var.iterator_age_alarm_enabled ? 1 : 0 - alarm_name = "${var.function_name}_iterator_age" + count = var.iterator_age_alarm_enabled ? 1 : 0 + alarm_name = "${aws_lambda_function.function.function_name}_iterator_age" namespace = "AWS/Lambda" metric_name = "IteratorAge" statistic = "Maximum" @@ -89,12 +81,12 @@ resource "aws_cloudwatch_metric_alarm" "streamalert_lambda_iterator_age" { threshold = var.iterator_age_alarm_threshold_ms evaluation_periods = var.iterator_age_alarm_evaluation_periods period = var.iterator_age_alarm_period_secs - alarm_description = "StreamAlert Lambda High Iterator Age: ${var.function_name}" + alarm_description = "StreamAlert Lambda High Iterator Age: ${aws_lambda_function.function.function_name}" alarm_actions = var.alarm_actions dimensions = { - FunctionName = var.function_name - Resource = "${var.function_name}:${var.alias_name}" + FunctionName = aws_lambda_function.function.function_name + Resource = "${aws_lambda_function.function.function_name}:${aws_lambda_alias.alias.name}" } tags = local.tags diff --git a/streamalert_cli/_infrastructure/modules/tf_lambda/iam.tf b/streamalert_cli/_infrastructure/modules/tf_lambda/iam.tf index caa220722..1ca886563 100644 --- a/streamalert_cli/_infrastructure/modules/tf_lambda/iam.tf +++ b/streamalert_cli/_infrastructure/modules/tf_lambda/iam.tf @@ -12,24 +12,22 @@ data "aws_iam_policy_document" "lambda_execution_policy" { // Create the execution role for the Lambda function. resource "aws_iam_role" "role" { - count = var.enabled ? 1 : 0 - name = "${var.function_name}_role" + name = "${aws_lambda_function.function.function_name}_role" path = "/streamalert/" assume_role_policy = data.aws_iam_policy_document.lambda_execution_policy.json tags = local.tags } -// Attach write permissions for CloudWatch logs -resource "aws_iam_role_policy_attachment" "logs_metrics_policy" { - count = var.enabled ? 1 : 0 - role = aws_iam_role.role[0].id +// Attach basic Lambda permissions +resource "aws_iam_role_policy_attachment" "lambda_basic_policy" { + role = aws_iam_role.role.id policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" } // Attach VPC policy (if applicable) resource "aws_iam_role_policy_attachment" "vpc_access" { - count = var.enabled && local.vpc_enabled ? 1 : 0 - role = aws_iam_role.role[0].id + count = local.vpc_enabled ? 1 : 0 + role = aws_iam_role.role.id policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" } diff --git a/streamalert_cli/_infrastructure/modules/tf_lambda/main.tf b/streamalert_cli/_infrastructure/modules/tf_lambda/main.tf index 28ed72e59..1ed306c25 100644 --- a/streamalert_cli/_infrastructure/modules/tf_lambda/main.tf +++ b/streamalert_cli/_infrastructure/modules/tf_lambda/main.tf @@ -1,5 +1,4 @@ -// Generic module for any StreamAlert Lambda function. -// TODO - migrate all Lambda functions and Lambda metric alarms to use this module +// Generic module for any StreamAlert Lambda function locals { schedule_enabled = var.schedule_expression != "" @@ -7,13 +6,12 @@ locals { tags = merge(var.default_tags, var.tags) } -// Either the function_vpc or the function_no_vpc resource will be used -resource "aws_lambda_function" "function_vpc" { - count = var.enabled && local.vpc_enabled ? 1 : 0 +// Lambda function, with optional VPC config +resource "aws_lambda_function" "function" { function_name = var.function_name description = var.description runtime = var.runtime - role = aws_iam_role.role[0].arn + role = aws_iam_role.role.arn handler = var.handler memory_size = var.memory_size_mb publish = var.auto_publish_versions @@ -29,76 +27,32 @@ resource "aws_lambda_function" "function_vpc" { variables = var.environment_variables } - // Empty vpc_config lists are theoretically supported, but it actually breaks subsequent deploys: - // https://github.com/terraform-providers/terraform-provider-aws/issues/443 + # Empty values for both of these values will prevent a vpc_config from being used + # https://www.terraform.io/docs/providers/aws/r/lambda_function.html#subnet_ids vpc_config { - security_group_ids = var.vpc_security_group_ids subnet_ids = var.vpc_subnet_ids - } - - tags = local.tags - - // We need VPC access before the function can be created - depends_on = [aws_iam_role_policy_attachment.vpc_access] -} - -resource "aws_lambda_alias" "alias_vpc" { - count = var.enabled && local.vpc_enabled ? 1 : 0 - name = var.alias_name - description = "${var.alias_name} alias for ${var.function_name}" - function_name = var.function_name - function_version = var.aliased_version == "" ? aws_lambda_function.function_vpc[0].version : var.aliased_version - depends_on = [aws_lambda_function.function_vpc] -} - -resource "aws_lambda_function" "function_no_vpc" { - count = var.enabled && false == local.vpc_enabled ? 1 : 0 - function_name = var.function_name - description = var.description - runtime = var.runtime - role = aws_iam_role.role[0].arn - handler = var.handler - memory_size = var.memory_size_mb - publish = var.auto_publish_versions - timeout = var.timeout_sec - - filename = var.filename - source_code_hash = filebase64sha256(var.filename) - - // Maximum number of concurrent executions allowed - reserved_concurrent_executions = var.concurrency_limit - - environment { - variables = var.environment_variables + security_group_ids = var.vpc_security_group_ids } tags = local.tags } -resource "aws_lambda_alias" "alias_no_vpc" { - count = var.enabled && false == local.vpc_enabled ? 1 : 0 +resource "aws_lambda_alias" "alias" { name = var.alias_name - description = "${var.alias_name} alias for ${var.function_name}" - function_name = var.function_name - function_version = var.aliased_version == "" ? aws_lambda_function.function_no_vpc[0].version : var.aliased_version - depends_on = [aws_lambda_function.function_no_vpc] + description = "${var.alias_name} alias for ${aws_lambda_function.function.function_name}" + function_name = aws_lambda_function.function.function_name + function_version = aws_lambda_function.function.version } // Allow Lambda function to be invoked via a CloudWatch event rule (if applicable) resource "aws_lambda_permission" "allow_cloudwatch_invocation" { - count = var.enabled && local.schedule_enabled ? 1 : 0 - statement_id = "AllowExecutionFromCloudWatch_${var.function_name}" + count = local.schedule_enabled ? 1 : 0 + statement_id = "AllowExecutionFromCloudWatch_${aws_lambda_function.function.function_name}" action = "lambda:InvokeFunction" - function_name = var.function_name + function_name = aws_lambda_function.function.function_name principal = "events.amazonaws.com" - source_arn = aws_cloudwatch_event_rule.invocation_schedule[0].arn - qualifier = var.alias_name - - // The alias must be created before we can grant permission to invoke it - depends_on = [ - aws_lambda_alias.alias_vpc, - aws_lambda_alias.alias_no_vpc, - ] + source_arn = aws_cloudwatch_event_rule.invocation_schedule.arn + qualifier = aws_lambda_alias.alias.name } // Lambda Permission: Allow SNS to invoke this function @@ -106,12 +60,8 @@ resource "aws_lambda_permission" "sns_inputs" { count = length(var.input_sns_topics) statement_id = "AllowExecutionFromSNS${count.index}" action = "lambda:InvokeFunction" - function_name = var.function_name + function_name = aws_lambda_function.function.function_name principal = "sns.amazonaws.com" source_arn = element(var.input_sns_topics, count.index) - qualifier = "production" - depends_on = [ - aws_lambda_alias.alias_vpc, - aws_lambda_alias.alias_no_vpc, - ] + qualifier = aws_lambda_alias.alias.name } diff --git a/streamalert_cli/_infrastructure/modules/tf_lambda/output.tf b/streamalert_cli/_infrastructure/modules/tf_lambda/output.tf index f3815b1ab..d49ad6412 100644 --- a/streamalert_cli/_infrastructure/modules/tf_lambda/output.tf +++ b/streamalert_cli/_infrastructure/modules/tf_lambda/output.tf @@ -1,58 +1,27 @@ -// Defined only if the Lambda is in a VPC -output "function_vpc_arn" { - value = join(" ", aws_lambda_function.function_vpc.*.arn) -} - -// Defined only if the Lambda is NOT in a VPC -output "function_no_vpc_arn" { - value = join(" ", aws_lambda_function.function_no_vpc.*.arn) +output "function_arn" { + value = aws_lambda_function.function.arn } output "role_arn" { - value = aws_iam_role.role[0].arn + value = aws_iam_role.role.arn } output "role_id" { - value = aws_iam_role.role[0].id + value = aws_iam_role.role.id } -// Combine the two mutually exclusive lists and export the first element as the function alias output "function_alias" { - value = element( - concat( - aws_lambda_alias.alias_vpc.*.name, - aws_lambda_alias.alias_no_vpc.*.name, - ), - 0, - ) + value = aws_lambda_alias.alias.name } -// Combine the two mutually exclusive lists and export the first element as the function name output "function_name" { - value = element( - concat( - aws_lambda_function.function_vpc.*.function_name, - aws_lambda_function.function_no_vpc.*.function_name, - ), - 0, - ) + value = aws_lambda_function.function.function_name } -// Combine the two mutually exclusive lists and export the first element as the function alias arn output "function_alias_arn" { - value = element( - concat( - aws_lambda_alias.alias_vpc.*.arn, - aws_lambda_alias.alias_no_vpc.*.arn, - ), - 0, - ) + value = aws_lambda_alias.alias.arn } -// Log group name for this Lambda function to enable applying metrics filters output "log_group_name" { - value = element( - concat(aws_cloudwatch_log_group.lambda_log_group.*.name, [""]), - 0, - ) + value = aws_cloudwatch_log_group.lambda_log_group.name } diff --git a/streamalert_cli/_infrastructure/modules/tf_lambda/variables.tf b/streamalert_cli/_infrastructure/modules/tf_lambda/variables.tf index b09e743c1..f8fe8984a 100644 --- a/streamalert_cli/_infrastructure/modules/tf_lambda/variables.tf +++ b/streamalert_cli/_infrastructure/modules/tf_lambda/variables.tf @@ -1,10 +1,3 @@ -// Note: We use this variable because terraform does not support "count" for module resources -// https://github.com/hashicorp/terraform/issues/953 -variable "enabled" { - default = true - description = "If true, the Lambda function and all associated components will be created" -} - variable "function_name" { description = "Name of the Lambda function" } @@ -82,12 +75,7 @@ variable "auto_publish_versions" { variable "alias_name" { default = "production" - description = "An alias with this name is automatically created which points to aliased_version" -} - -variable "aliased_version" { - default = "" - description = "Alias points to this version (or the latest published version if not specified)" + description = "An alias with this name is automatically created which points to the current version" } variable "schedule_expression" { diff --git a/streamalert_cli/_infrastructure/modules/tf_scheduled_queries/lambda.tf b/streamalert_cli/_infrastructure/modules/tf_scheduled_queries/lambda.tf index c4ab32716..f5e82c764 100644 --- a/streamalert_cli/_infrastructure/modules/tf_scheduled_queries/lambda.tf +++ b/streamalert_cli/_infrastructure/modules/tf_scheduled_queries/lambda.tf @@ -1,18 +1,16 @@ module "scheduled_queries_lambda" { source = "../tf_lambda" - enabled = true - function_name = "${var.prefix}_streamalert_scheduled_queries_runner" description = "Lambda function that powers StreamQuery, StreamAlert's scheduled query service" runtime = "python3.7" handler = var.lambda_handler - memory_size_mb = var.lambda_memory - timeout_sec = var.lambda_timeout - filename = var.lambda_filename + memory_size_mb = var.lambda_memory + timeout_sec = var.lambda_timeout + filename = var.lambda_filename - concurrency_limit = var.lambda_concurrency_limit + concurrency_limit = var.lambda_concurrency_limit environment_variables = { REGION = var.region @@ -28,8 +26,8 @@ module "scheduled_queries_lambda" { auto_publish_versions = true - log_retention_days = var.lambda_log_retention_days - alarm_actions = var.lambda_alarm_actions + log_retention_days = var.lambda_log_retention_days + alarm_actions = var.lambda_alarm_actions errors_alarm_enabled = var.lambda_alarms_enabled errors_alarm_evaluation_periods = var.lambda_error_evaluation_periods