🔔 Implement Alert change streams + add SNS destination for webhooks (#…

…117)

#116 

Add a feature that allows users to consume Matano alerts in realtime
using webhooks for security automation use cases. To make this possible,
we need to expose an SNS topic where we publish alerts, and that a user
can create HTTP subscriptions on top of.

**Note:** Alerts in matano are deduplicated and track a series of rule
matches over time, so this integration will actually more of an "**alert
change stream**", where every time an alert is updated with new rule
matches, the user is notified of (a) the latest state of the alert, (b)
a contextual summary of the incoming rule matches that caused the alert
to be republished and (c) a contextual diff that highlights what is
actually new and never before seen.

For use cases like submitting indicators to scanner like VirusTotal, the
`context_diff` feature would allow users to only submit the IOC (e.g.
file hash) once if an alert ends up deduplicating many consecutive rule
matches.

**Alert SNS webhook payload structure**

```jsonc
{
  "updated_alert": {
    "id": "61d3ac78-cedd-434e-b407-ae0b330e1d4a",
    "creation_time": "2023-03-15T06:21:13.687059Z",
    "title": "AWS Root Credentials Change",
    "severity": "medium",
    "severity_icon_url": "https://gist.githubusercontent.com/shaeqahmed/6c38fc5f0c3adb7e1a3fe6c5f78bbc4f/raw/9a12ff8d23592b31f224f9e27503e77b843b075c/apple-sev-medium-icon.png",
    "runbook": "Check the AWS root account activity",
    "false_positives": [],
    "destinations": [
      "slack_my_team",
      "jira_main"
    ],
    "context": {
      "cloud": {
        "account": {
          "id": [
            "2029292292992"
          ]
        },
        "region": [
          "us-west-2"
        ]
      },
      "event": {
        "action": [
          "RunInstances"
        ],
        "outcome": [
          "failure"
        ],
        "provider": [
          "ec2.amazonaws.com"
        ],
        "type": [
          "info"
        ]
      },
      "matano": {
        "table": [
          "aws_cloudtrail"
        ]
      },
      "related": {
        "user": [
          "backup",
          "John Doe"
        ]
      },
      "source": {
        "address": [
          "5.205.62.253"
        ],
        "ip": [
          "5.205.62.253"
        ]
      },
      "user": {
        "id": [
          "AIDA929292929292"
        ],
        "name": [
          "backup"
        ]
      }
    },
    "tables": [
      "aws_cloudtrail"
    ],
    "match_count": 2,
    "update_count": 1,
    "destination_to_alert_info": {
      "slack_my_team": "{\"ts\":\"1678861298.161909\"}"
    }
  },
  "incoming_rule_matches_context": { 
    "cloud": {
      "account": {
        "id": [
          "2929922929292"
        ]
      },
      "region": [
        "us-west-2"
      ]
    },
    "event": {
      "action": [
        "RunInstances"
      ],
      "outcome": [
        "failure"
      ],
      "provider": [
        "ec2.amazonaws.com"
      ],
      "type": [
        "info"
      ]
    },
    "matano": {
      "table": [
        "aws_cloudtrail"
      ]
    },
    "related": {
      "user": [
        "John Doe"
      ]
    },
    "source": {
      "address": [
        "5.205.62.253"
      ],
      "ip": [
        "5.205.62.253"
      ]
    },
    "user": {
      "id": [
        "AIDA929292929292"
      ],
      "name": [
        "John Doe"
      ]
    }
  },
  "context_diff": { // what's new
    "user": {
      "name": [
        "John Doe"
      ]
    },
    "related": {
      "user": [
        "John Doe"
      ]
    }
   } 
}
```

infra/lib/alerting.ts

@@ -17,10 +17,14 @@ interface MatanoAlertingProps {
 }
 
 export class MatanoAlerting extends Construct {
+  ruleMatchesTopic: sns.Topic;
   alertingTopic: sns.Topic;
   constructor(scope: Construct, id: string, props: MatanoAlertingProps) {
     super(scope, id);
 
+    this.ruleMatchesTopic = new sns.Topic(this, "RuleMatchesTopic", {
+      displayName: "MatanoRuleMatchesTopic",
+    });
     this.alertingTopic = new sns.Topic(this, "Topic", {
       displayName: "MatanoAlertingTopic",
     });
@@ -29,14 +33,16 @@ export class MatanoAlerting extends Construct {
       const alertForwarder = new AlertForwarder(this, "Forwarder", {
         integrationsStore: props.integrationsStore,
         alertTrackerTable: props.alertTrackerTable,
-        alertsSnsTopic: this.alertingTopic,
+        ruleMatchesSnsTopic: this.ruleMatchesTopic,
+        alertingSnsTopic: this.alertingTopic,
       });
     }
   }
 }
 
 interface AlertForwarderProps {
-  alertsSnsTopic: sns.Topic;
+  ruleMatchesSnsTopic: sns.Topic;
+  alertingSnsTopic: sns.Topic;
   integrationsStore: IntegrationsStore;
   alertTrackerTable: ddb.Table;
 }
@@ -64,6 +70,7 @@ export class AlertForwarder extends Construct {
       environment: {
         RUST_LOG: "warn,alert_forwarder=info",
         ALERT_TRACKER_TABLE_NAME: props.alertTrackerTable.tableName,
+        ALERTING_SNS_TOPIC_ARN: props.alertingSnsTopic.topicArn,
         DESTINATION_TO_CONFIGURATION_MAP: JSON.stringify(props.integrationsStore?.integrationsInfoMap ?? {}),
         DESTINATION_TO_SECRET_ARN_MAP: JSON.stringify(destinationToSecretArnMap),
       },
@@ -90,11 +97,12 @@ export class AlertForwarder extends Construct {
       visibilityTimeout: cdk.Duration.seconds(Math.max(this.function.timeout!.toSeconds(), 30)),
     });
 
-    props.alertsSnsTopic.addSubscription(
+    props.ruleMatchesSnsTopic.addSubscription(
       new SqsSubscription(this.queue, {
         rawMessageDelivery: true,
       })
     );
+    props.alertingSnsTopic.grantPublish(this.function);
 
     this.function.addEventSource(
       new SqsEventSource(this.queue, {

infra/lib/lake-writer.ts

@@ -10,7 +10,7 @@ interface LakeWriterProps {
   realtimeBucket: s3.IBucket;
   outputBucket: s3.IBucket;
   outputObjectPrefix: string;
-  alertingSnsTopic: sns.Topic;
+  ruleMatchesSnsTopic: sns.Topic;
 }
 
 export class LakeWriter extends Construct {
@@ -43,14 +43,14 @@ export class LakeWriter extends Construct {
         RUST_LOG: "warn,lake_writer=info",
         OUT_BUCKET_NAME: props.outputBucket.bucketName,
         OUT_KEY_PREFIX: props.outputObjectPrefix,
-        ALERTING_SNS_TOPIC_ARN: props.alertingSnsTopic.topicArn,
+        RULE_MATCHES_SNS_TOPIC_ARN: props.ruleMatchesSnsTopic.topicArn,
       },
       timeout: cdk.Duration.seconds(120),
       // prevent concurrency
       reservedConcurrentExecutions: 1,
     });
     props.realtimeBucket.grantRead(this.alertsLakeWriterLambda);
     props.outputBucket.grantReadWrite(this.alertsLakeWriterLambda);
-    props.alertingSnsTopic.grantPublish(this.alertsLakeWriterLambda);
+    props.ruleMatchesSnsTopic.grantPublish(this.alertsLakeWriterLambda);
   }
 }

infra/src/DPMainStack.ts

@@ -82,7 +82,7 @@ export class DPMainStack extends MatanoStack {
 
     const lakeWriter = new LakeWriter(this, "LakeWriter", {
       realtimeBucket: props.realtimeBucket,
-      alertingSnsTopic: matanoAlerting.alertingTopic,
+      ruleMatchesSnsTopic: matanoAlerting.ruleMatchesTopic,
       outputBucket: props.lakeStorageBucket.bucket,
       outputObjectPrefix: "lake",
     });

lib/rust/alert_forwarder/Cargo.toml

@@ -31,6 +31,7 @@ lambda_runtime = "0.7.0"
 aws-config = "0.51.0"
 aws_lambda_events = "0.7.2"
 aws-sdk-sqs = "0.21.0"
+aws-sdk-sns = "0.21.0"
 aws-sdk-dynamodb = "0.21.0"
 serde_dynamo = { version = "4", features = ["aws-sdk-dynamodb+0_21"] }