official release of v3.1.2 (#1219)

* bumping version to 3.1.2

* fixing bug in scheduled queries and adding unit tests

* fixing bug in default athena results bucket name, dry out code

streamalert/__init__.py

@@ -1,2 +1,2 @@
 """StreamAlert version."""
-__version__ = '3.1.1'
+__version__ = '3.1.2'

streamalert/shared/config.py

@@ -145,7 +145,7 @@ def athena_query_results_bucket(config):
 
     return athena_config.get(
         'results_bucket',
-        '{}.streamalert.athena-results'.format(prefix)
+        '{}-streamalert-athena-results'.format(prefix)
     ).strip()
 
 

streamalert_cli/terraform/athena.py

@@ -14,7 +14,7 @@
 limitations under the License.
 """
 from streamalert.shared import metrics
-from streamalert.shared.config import athena_partition_buckets
+from streamalert.shared.config import athena_partition_buckets, athena_query_results_bucket
 from streamalert_cli.manage_lambda.package import AthenaPackage
 from streamalert_cli.terraform.common import (
     infinitedict,
@@ -40,10 +40,7 @@ def generate_athena(config):
     prefix = config['global']['account']['prefix']
     database = athena_config.get('database_name', '{}_streamalert'.format(prefix))
 
-    results_bucket_name = athena_config.get(
-        'results_bucket',
-        '{}-streamalert-athena-results'.format(prefix)
-    ).strip()
+    results_bucket_name = athena_query_results_bucket(config)
 
     queue_name = athena_config.get(
         'queue_name',

streamalert_cli/terraform/scheduled_queries.py

@@ -37,7 +37,7 @@ def generate_scheduled_queries_module_configuration(config):
     # FIXME (derek.wang) DRY out this code
     results_bucket = athena_query_results_bucket(config)
 
-    athena_s3_buckets = athena_partition_buckets(config)
+    athena_s3_buckets = sorted(athena_partition_buckets(config))
 
     # Copy the config over directly
     scheduled_queries_module = streamquery_config.get('config', {})

tests/unit/conf/scheduled_queries.json

@@ -0,0 +1,24 @@
+{
+  "enabled": true,
+  "config": {
+    "destination_kinesis_stream": "unit-test_stream",
+    "sfn_timeout_secs": 3600,
+    "sfn_wait_secs": 30
+  },
+  "packs": {
+    "hourly": {
+      "description": "Runs all hourly queries. Once per day on :05",
+      "schedule_expression": "cron(5 * * * ? *)"
+    }
+  },
+  "lambda_config": {
+    "log_level": "info",
+    "log_retention_days": 14,
+    "memory": 128,
+    "timeout": 60,
+    "alarms_enabled": true,
+    "error_threshold": 1,
+    "error_period_secs": 3600,
+    "error_evaluation_periods": 2
+  }
+}

tests/unit/streamalert_cli/terraform/test_scheduled_queries.py

@@ -0,0 +1,70 @@
+"""
+Copyright 2017-present Airbnb, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+from nose.tools import assert_equal
+
+from streamalert_cli.config import CLIConfig
+from streamalert_cli.terraform import scheduled_queries
+
+CONFIG = CLIConfig(config_path='tests/unit/conf')
+
+
+def test_generate_scheduled_queries():
+    """CLI - Terraform Generate Scheduled Queries"""
+
+    expected_sq_config = {
+        'module': {
+            'scheduled_queries': {
+                'source': './modules/tf_scheduled_queries',
+                'destination_kinesis_stream': 'unit-test_stream',
+                'sfn_timeout_secs': 3600,
+                'sfn_wait_secs': 30,
+                'prefix': 'unit-test',
+                'account_id': '12345678910',
+                'region': 'us-west-1',
+                'athena_database': 'unit-test_streamalert',
+                'athena_results_bucket': 'unit-test-streamalert-athena-results',
+                'athena_s3_buckets': [
+                    'bucket',
+                    'unit-test-streamalert-data',
+                    'unit-test-streamalerts'
+                ],
+                'lambda_filename': 'scheduled_queries.zip',
+                'lambda_handler': 'streamalert.scheduled_queries.main.handler',
+                'query_packs': [
+                    {
+                        'name': 'hourly',
+                        'schedule_expression': 'cron(5 * * * ? *)',
+                        'description': 'Runs all hourly queries. Once per day on :05'
+                    }
+                ],
+                'lambda_log_level': 'info',
+                'lambda_memory': 128,
+                'lambda_timeout': 60,
+                'lambda_log_retention_days': 14,
+                'lambda_alarms_enabled': True,
+                'lambda_error_threshold': 1,
+                'lambda_error_period_secs': 3600,
+                'lambda_error_evaluation_periods': 2,
+                'lambda_alarm_actions': [
+                    'arn:aws:sns:us-west-1:12345678910:unit-test_streamalert_monitoring'
+                ],
+            }
+        }
+    }
+
+    result = scheduled_queries.generate_scheduled_queries_module_configuration(CONFIG)
+
+    assert_equal(result, expected_sq_config)