Merge 1bd30dc into 1ed99e1

stream_alert/__init__.py

@@ -1,2 +1,2 @@
 """StreamAlert version."""
-__version__ = '2.1.5'
+__version__ = '2.1.6'

stream_alert/classifier/clients/firehose.py

@@ -19,7 +19,7 @@
 
 import backoff
 import boto3
-from botocore.exceptions import ClientError, ConnectionClosedError, ConnectionError
+from botocore.exceptions import ClientError, ConnectionError, HTTPClientError
 
 from stream_alert.shared import CLASSIFIER_FUNCTION_NAME as FUNCTION_NAME
 import stream_alert.shared.helpers.boto as boto_helpers
@@ -54,7 +54,7 @@ class FirehoseClient(object):
     DEFAULT_FIREHOSE_PREFIX = 'streamalert_data_{}'
 
     # Exception for which backoff operations should be performed
-    EXCEPTIONS_TO_BACKOFF = (ClientError, ConnectionClosedError, ConnectionError)
+    EXCEPTIONS_TO_BACKOFF = (ClientError, ConnectionError, HTTPClientError)
 
     # Set of enabled log types for firehose, loaded from configs
     _ENABLED_LOGS = dict()
@@ -99,7 +99,7 @@ def _record_batches(cls, records):
             # Check if the max size of the batch has been reached or if the current
             # record will exceed the max batch size and start a new batch
             if ((len(current_batch) == cls.MAX_BATCH_COUNT) or
-                    (current_batch_size + line_len > cls.MAX_BATCH_SIZE)):
+                    (current_batch_size + line_len > cls.MAX_BATCH_SIZE)) and current_batch:
                 yield current_batch[:]
                 current_batch_size = 0
                 del current_batch[:]

stream_alert/classifier/clients/sqs.py

@@ -18,7 +18,7 @@
 
 import backoff
 import boto3
-from botocore.exceptions import ClientError, ConnectionClosedError, ConnectionError
+from botocore.exceptions import ClientError, ConnectionError, HTTPClientError
 
 from stream_alert.shared import CLASSIFIER_FUNCTION_NAME as FUNCTION_NAME
 from stream_alert.shared.helpers import boto
@@ -40,7 +40,7 @@ class SQSClientError(Exception):
 class SQSClient(object):
     """SQSClient for sending batches of classified records to the Rules Engine function"""
     # Exception for which backoff operations should be performed
-    EXCEPTIONS_TO_BACKOFF = (ClientError, ConnectionClosedError, ConnectionError)
+    EXCEPTIONS_TO_BACKOFF = (ClientError, ConnectionError, HTTPClientError)
 
     # Maximum amount of times to retry with backoff
     MAX_BACKOFF_ATTEMPTS = 5
@@ -76,6 +76,7 @@ def _segment_records(cls, records):
             size = len(record) + (1 if idx != record_count and batch else 0)
             if size + 2 > cls.MAX_SIZE:
                 LOGGER.error('Record is too large to send to SQS:\n%s', record)
+                MetricLogger.log_metric(FUNCTION_NAME, MetricLogger.SQS_FAILED_RECORDS, 1)
                 continue
 
             if idx == record_count or size + batch_size >= cls.MAX_SIZE:

stream_alert_cli/terraform/classifier.py

@@ -94,5 +94,8 @@ def generate_classifier(cluster_name, cluster_dict, config):
         environment={
             'CLUSTER': cluster_name,
             'SQS_QUEUE_URL': '${module.globals.classifier_sqs_queue_url}',
-        }
+        },
+        tags={
+            'Cluster': cluster_name
+        },
     )

stream_alert_cli/terraform/generate.py

@@ -89,7 +89,27 @@ def generate_s3_bucket(bucket, logging, **kwargs):
                     'sse_algorithm': sse_algorithm
                 }
             }
-        }
+        },
+        'policy': json.dumps({
+            'Version': '2012-10-17',
+            'Statement': [
+                {
+                    'Sid': 'ForceSSLOnlyAccess',
+                    'Effect': 'Deny',
+                    'Principal': '*',
+                    'Action': 's3:*',
+                    'Resource': [
+                        'arn:aws:s3:::{}/*'.format(bucket),
+                        'arn:aws:s3:::{}'.format(bucket)
+                    ],
+                    'Condition': {
+                        'Bool': {
+                            'aws:SecureTransport': 'false'
+                        }
+                    }
+                }
+            ]
+        })
     }
 
     if sse_algorithm == 'aws:kms':

stream_alert_cli/terraform/lambda_module.py

@@ -52,7 +52,7 @@ def _tf_vpc_config(lambda_config):
 
 
 def generate_lambda(function_name, zip_file, handler, lambda_config, config,
-                    environment=None, input_event=None):
+                    environment=None, input_event=None, tags=None):
     """Generate an instance of the Lambda Terraform module.
 
     Args:
@@ -63,6 +63,7 @@ def generate_lambda(function_name, zip_file, handler, lambda_config, config,
         config (dict): Parsed config from conf/
         environment (dict): Optional environment variables to specify.
             ENABLE_METRICS and LOGGER_LEVEL are included automatically.
+        tags (dict): Optional tags to be added to this Lambda resource.
 
     Example Lambda config:
         {
@@ -117,7 +118,8 @@ def generate_lambda(function_name, zip_file, handler, lambda_config, config,
         'memory_size_mb': lambda_config['memory'],
         'timeout_sec': lambda_config['timeout'],
         'filename': zip_file,
-        'environment_variables': environment_variables
+        'environment_variables': environment_variables,
+        'tags': tags or {},
     }
 
     # Add Classifier input config from the loaded cluster file

terraform/modules/tf_lambda/cloudwatch.tf

@@ -30,9 +30,7 @@ resource "aws_cloudwatch_log_group" "lambda_log_group" {
   name              = "/aws/lambda/${var.function_name}"
   retention_in_days = "${var.log_retention_days}"
 
-  tags {
-    Name = "${var.name_tag}"
-  }
+  tags = "${local.tags}"
 }
 
 // Generic CloudWatch metric alarms related to this function

terraform/modules/tf_lambda/main.tf

@@ -4,6 +4,7 @@
 locals {
   schedule_enabled = "${var.schedule_expression != ""}"
   vpc_enabled      = "${length(var.vpc_subnet_ids) > 0}"
+  tags             = "${merge(var.default_tags, var.tags)}"
 }
 
 // Either the function_vpc or the function_no_vpc resource will be used
@@ -35,9 +36,7 @@ resource "aws_lambda_function" "function_vpc" {
     subnet_ids         = "${var.vpc_subnet_ids}"
   }
 
-  tags {
-    Name = "${var.name_tag}"
-  }
+  tags = "${local.tags}"
 
   // We need VPC access before the function can be created
   depends_on = ["aws_iam_role_policy_attachment.vpc_access"]
@@ -73,9 +72,7 @@ resource "aws_lambda_function" "function_no_vpc" {
     variables = "${var.environment_variables}"
   }
 
-  tags {
-    Name = "${var.name_tag}"
-  }
+  tags = "${local.tags}"
 }
 
 resource "aws_lambda_alias" "alias_no_vpc" {

terraform/modules/tf_lambda/variables.tf

@@ -63,9 +63,20 @@ variable "vpc_security_group_ids" {
   description = "Optional list of security group IDs (for VPC)"
 }
 
-variable "name_tag" {
-  default     = "StreamAlert"
-  description = "The value for the Name cost tag associated with all applicable components"
+variable "default_tags" {
+  type = "map"
+
+  default = {
+    Name = "StreamAlert"
+  }
+
+  description = "The default tags to be associated with all applicable components"
+}
+
+variable "tags" {
+  type        = "map"
+  default     = {}
+  description = "Any dditional tags to be associated with all applicable components"
 }
 
 variable "auto_publish_versions" {

terraform/modules/tf_stream_alert_athena/main.tf

@@ -26,10 +26,39 @@ resource "aws_lambda_function" "athena_partition_refresh" {
   }
 }
 
+// Policy for S3 bucket
+data "aws_iam_policy_document" "athena_results_bucket" {
+  # Force SSL access only
+  statement {
+    sid = "ForceSSLOnlyAccess"
+
+    effect = "Deny"
+
+    principals {
+      type        = "AWS"
+      identifiers = ["*"]
+    }
+
+    actions = ["s3:*"]
+
+    resources = [
+      "arn:aws:s3:::${var.results_bucket}",
+      "arn:aws:s3:::${var.results_bucket}/*",
+    ]
+
+    condition {
+      test     = "Bool"
+      variable = "aws:SecureTransport"
+      values   = ["false"]
+    }
+  }
+}
+
 // S3 Bucket: Athena Query Results and Metastore Bucket
 resource "aws_s3_bucket" "athena_results_bucket" {
   bucket        = "${var.results_bucket}"
   acl           = "private"
+  policy        = "${data.aws_iam_policy_document.athena_results_bucket.json}"
   force_destroy = false
 
   tags {

terraform/modules/tf_stream_alert_cloudtrail/main.tf

@@ -1,6 +1,11 @@
+locals {
+  apply_filter_string    = "{ $$.awsRegion != \"${var.region}\" }"
+  cloudtrail_bucket_name = "${var.prefix}.${var.cluster}.streamalert.cloudtrail"
+}
+
 // KMS key for encrypting CloudTrail logs
 resource "aws_kms_key" "cloudtrail_encryption" {
-  description         = "Encrypt Cloudtrail logs for ${var.prefix}.${var.cluster}.streamalert.cloudtrail"
+  description         = "Encrypt Cloudtrail logs for ${local.cloudtrail_bucket_name}"
   policy              = "${data.aws_iam_policy_document.cloudtrail_encryption.json}"
   enable_key_rotation = true
 }
@@ -86,7 +91,7 @@ resource "aws_kms_alias" "cloudtrail_encryption" {
 // StreamAlert CloudTrail, also sending to CloudWatch Logs group
 resource "aws_cloudtrail" "streamalert" {
   count                         = "${var.send_to_cloudwatch && !var.existing_trail ? 1 : 0}"
-  name                          = "${var.prefix}.${var.cluster}.streamalert.cloudtrail"
+  name                          = "${local.cloudtrail_bucket_name}"
   s3_bucket_name                = "${aws_s3_bucket.cloudtrail_bucket.id}"
   cloud_watch_logs_role_arn     = "${aws_iam_role.cloudtrail_to_cloudwatch_role.arn}"
   cloud_watch_logs_group_arn    = "${aws_cloudwatch_log_group.cloudtrail_logging.arn}"
@@ -113,7 +118,7 @@ resource "aws_cloudtrail" "streamalert" {
 // StreamAlert CloudTrail, not sending to CloudWatch
 resource "aws_cloudtrail" "streamalert_no_cloudwatch" {
   count                         = "${!var.send_to_cloudwatch && !var.existing_trail ? 1 : 0}"
-  name                          = "${var.prefix}.${var.cluster}.streamalert.cloudtrail"
+  name                          = "${local.cloudtrail_bucket_name}"
   s3_bucket_name                = "${aws_s3_bucket.cloudtrail_bucket.id}"
   enable_log_file_validation    = true
   enable_logging                = "${var.enable_logging}"
@@ -204,10 +209,6 @@ data "aws_iam_policy_document" "cloudtrail_to_cloudwatch_create_logs" {
   }
 }
 
-locals {
-  apply_filter_string = "{ $$.awsRegion != \"${var.region}\" }"
-}
-
 // CloudWatch Log Subscription Filter
 //   If we are collecting CloudTrail logs in the 'home region' another way, this allows
 //   for suppression of logs that originated in this region.
@@ -223,7 +224,8 @@ resource "aws_cloudwatch_log_subscription_filter" "cloudtrail_via_cloudwatch" {
 // S3 bucket for CloudTrail output
 resource "aws_s3_bucket" "cloudtrail_bucket" {
   count         = "${var.existing_trail ? 0 : 1}"
-  bucket        = "${var.prefix}.${var.cluster}.streamalert.cloudtrail"
+  bucket        = "${local.cloudtrail_bucket_name}"
+  policy        = "${data.aws_iam_policy_document.cloudtrail_bucket.json}"
   force_destroy = false
 
   versioning {
@@ -232,11 +234,9 @@ resource "aws_s3_bucket" "cloudtrail_bucket" {
 
   logging {
     target_bucket = "${var.s3_logging_bucket}"
-    target_prefix = "${var.prefix}.${var.cluster}.streamalert.cloudtrail/"
+    target_prefix = "${local.cloudtrail_bucket_name}/"
   }
 
-  policy = "${data.aws_iam_policy_document.cloudtrail_bucket.json}"
-
   server_side_encryption_configuration {
     rule {
       apply_server_side_encryption_by_default {
@@ -247,7 +247,7 @@ resource "aws_s3_bucket" "cloudtrail_bucket" {
   }
 
   tags {
-    Name    = "${var.prefix}.${var.cluster}.streamalert.cloudtrail"
+    Name    = "${local.cloudtrail_bucket_name}"
     Cluster = "${var.cluster}"
   }
 }
@@ -263,7 +263,7 @@ data "aws_iam_policy_document" "cloudtrail_bucket" {
     ]
 
     resources = [
-      "arn:aws:s3:::${var.prefix}.${var.cluster}.streamalert.cloudtrail",
+      "arn:aws:s3:::${local.cloudtrail_bucket_name}",
     ]
 
     principals {
@@ -280,7 +280,7 @@ data "aws_iam_policy_document" "cloudtrail_bucket" {
     ]
 
     resources = [
-      "${formatlist("arn:aws:s3:::${var.prefix}.${var.cluster}.streamalert.cloudtrail/AWSLogs/%s/*", var.account_ids)}",
+      "${formatlist("arn:aws:s3:::${local.cloudtrail_bucket_name}/AWSLogs/%s/*", var.account_ids)}",
     ]
 
     principals {
@@ -297,4 +297,29 @@ data "aws_iam_policy_document" "cloudtrail_bucket" {
       ]
     }
   }
+
+  # Force SSL access only
+  statement {
+    sid = "ForceSSLOnlyAccess"
+
+    effect = "Deny"
+
+    principals {
+      type        = "AWS"
+      identifiers = ["*"]
+    }
+
+    actions = ["s3:*"]
+
+    resources = [
+      "arn:aws:s3:::${local.cloudtrail_bucket_name}",
+      "arn:aws:s3:::${local.cloudtrail_bucket_name}/*",
+    ]
+
+    condition {
+      test     = "Bool"
+      variable = "aws:SecureTransport"
+      values   = ["false"]
+    }
+  }
 }

terraform/modules/tf_stream_alert_kinesis_firehose_setup/main.tf

@@ -1,6 +1,35 @@
+// Policy for S3 bucket
+data "aws_iam_policy_document" "stream_alert_data" {
+  # Force SSL access only
+  statement {
+    sid = "ForceSSLOnlyAccess"
+
+    effect = "Deny"
+
+    principals {
+      type        = "AWS"
+      identifiers = ["*"]
+    }
+
+    actions = ["s3:*"]
+
+    resources = [
+      "arn:aws:s3:::${var.s3_bucket_name}",
+      "arn:aws:s3:::${var.s3_bucket_name}/*",
+    ]
+
+    condition {
+      test     = "Bool"
+      variable = "aws:SecureTransport"
+      values   = ["false"]
+    }
+  }
+}
+
 resource "aws_s3_bucket" "stream_alert_data" {
   bucket        = "${var.s3_bucket_name}"
   acl           = "private"
+  policy        = "${data.aws_iam_policy_document.stream_alert_data.json}"
   force_destroy = false
 
   versioning {