Merge 904a759 into 19458d7

.gitignore

@@ -6,11 +6,11 @@ docs/build
 __pycache__
 
 # Terraform files
-*.terraform/
-terraform.tfvars
-terraform/terraform.tfstate*
-terraform/*.zip
-terraform/*.tf.json
+streamalert_cli/_infrastructure/.terraform/
+streamalert_cli/_infrastructure/terraform.tfvars
+streamalert_cli/_infrastructure/terraform.tfstate*
+streamalert_cli/_infrastructure/*.zip
+streamalert_cli/_infrastructure/*.tf.json
 
 # Coveralls repo token
 .coveralls.yml

Vagrantfile

@@ -33,9 +33,6 @@ def configure_python(machine, version)
 end
 
 STREAMALERT_SHELL_ENV = %{
-export AWS_DEFAULT_REGION='#{ENV.fetch('SA_AWS_DEFAULT_REGION', 'Your region here!')}'
-export AWS_ACCESS_KEY_ID='#{ENV.fetch('SA_AWS_ACCESS_KEY_ID', 'Your access key ID here!')}'
-export AWS_SECRET_ACCESS_KEY='#{ENV.fetch('SA_AWS_SECRET_ACCESS_KEY', 'Your secret access key here!')}'
 export SA_EMAIL='#{ENV.fetch('SA_EMAIL', 'example@example.com')}'
 }
 
@@ -114,4 +111,11 @@ Vagrant.configure(2) do |config|
     configure_streamalert(py3)
     final_message(py3)
   end
+
+  config.ssh.forward_env = [
+    'AWS_DEFAULT_REGION',
+    'AWS_ACCESS_KEY_ID',
+    'AWS_SECRET_ACCESS_KEY',
+    'AWS_SESSION_TOKEN'
+  ]
 end

conf/clusters/prod.json

@@ -40,7 +40,8 @@
       "prefix.cluster.sample.bucket": [
         "cloudtrail",
         "carbonblack",
-        "fleet"
+        "fleet",
+        "packetbeat"
       ]
     },
     "sns": {
@@ -84,4 +85,4 @@
       "lambda_alarms_enabled": true
     }
   }
-}
+}

conf/global.json

@@ -10,6 +10,9 @@
     ],
     "rule_locations": [
       "rules"
+    ],
+    "scheduled_query_locations": [
+      "scheduled_queries"
     ]
   },
   "infrastructure": {
@@ -21,7 +24,6 @@
       "use_prefix": true,
       "buffer_interval": 900,
       "buffer_size": 128,
-      "compression_format": "GZIP",
       "enabled": false,
       "enabled_logs": {}
     },

conf/lambda.json

@@ -52,6 +52,7 @@
   },
   "athena_partition_refresh_config": {
     "concurrency_limit": 10,
+    "file_format": null,
     "log_level": "info"
   },
   "classifier_config": {},
@@ -79,7 +80,7 @@
     "timeout": 120
   },
   "rules_engine_config": {
-    "concurrency_limit": 200,
+    "concurrency_limit": 10,
     "enable_custom_metrics": true,
     "log_level": "info",
     "log_retention_days": 14,

conf/outputs.json

@@ -5,6 +5,9 @@
   "aws-s3": {
     "bucket": "aws-s3-bucket"
   },
+  "aws-ses": [
+    "sample-integration"
+  ],
   "aws-sns": {
     "sample-topic": "sample-topic-name"
   },
@@ -25,5 +28,8 @@
   ],
   "slack": [
     "sample-channel"
+  ],
+  "teams": [
+    "sample-webhook"
   ]
 }

conf/scheduled_queries.json

@@ -0,0 +1,24 @@
+{
+  "enabled": false,
+  "config": {
+    "destination_kinesis_stream": "KINESIS_STREAM",
+    "sfn_timeout_secs": 300,
+    "sfn_wait_secs": 30
+  },
+  "packs": {
+    "sample": {
+      "description": "Runs sample queries once per hour",
+      "schedule_expression": "rate(1 hour)"
+    }
+  },
+  "lambda_config": {
+    "log_level": "info",
+    "log_retention_days": 14,
+    "memory": 128,
+    "timeout": 60,
+    "alarms_enabled": false,
+    "error_threshold": 1,
+    "error_period_secs": 3600,
+    "error_evaluation_periods": 2
+  }
+}