Fire priority alert if more than 6 alerts in 12 hours

terraform/envs/general/main.tf

@@ -38,4 +38,5 @@ module "dirsync" {
   LogRetentionDays = var.LogRetentionDays
   SyncFrequency    = "rate(1 hour)"
   SnsArn           = var.SnsArn
+  PrioritySnsArn   = var.PrioritySnsArn
 }

terraform/envs/general/variables.tf

@@ -12,3 +12,8 @@ variable "SnsArn" {
   type    = string
   default = "arn:aws:sns:us-east-2:898906883758:DiscordGeneralAlerts"
 }
+
+variable "PrioritySnsArn" {
+  type    = string
+  default = "arn:aws:sns:us-east-2:898906883758:PriorityDiscordAlerts"
+}

terraform/modules/dirsync/main.tf

@@ -66,7 +66,7 @@ resource "aws_lambda_function" "this" {
   timeout          = 900
   memory_size      = 2048
   source_code_hash = data.archive_file.lambda_code.output_sha256
-  # reserved_concurrent_executions = 1
+  reserved_concurrent_executions = 1
   description = "GSuite Sync Lambda."
   environment {
     variables = {
@@ -132,3 +132,21 @@ resource "aws_cloudwatch_metric_alarm" "no_invocation" {
     FunctionName = local.sync_lambda_name
   }
 }
+
+resource "aws_cloudwatch_metric_alarm" "repeated_errors" {
+  alarm_name          = "${local.sync_lambda_name}-repeated-errors"
+  alarm_description   = "GSuite Directory Sync lambda has had multiple errors in the past 12 hours."
+  namespace           = "AWS/Lambda"
+  metric_name         = "Errors"
+  statistic           = "Sum"
+  period              = "43200"  # 12 hours
+  evaluation_periods  = "1"
+  comparison_operator = "GreaterThanThreshold"
+  threshold           = "6"
+  alarm_actions = [
+    var.PrioritySnsArn
+  ]
+  dimensions = {
+    FunctionName = local.sync_lambda_name
+  }
+}

terraform/modules/dirsync/variables.tf

@@ -23,3 +23,7 @@ variable "SyncFrequency" {
 variable "SnsArn" {
   type = string
 }
+
+variable "PrioritySnsArn" {
+  type = string
+}