From 41da6b5514b4949c04cab90aa85631fa833634d0 Mon Sep 17 00:00:00 2001 From: ryandeivert Date: Wed, 1 Apr 2020 17:04:22 -0700 Subject: [PATCH] official release of v3.1.2 (#1219) * bumping version to 3.1.2 * fixing bug in scheduled queries and adding unit tests * fixing bug in default athena results bucket name, dry out code --- streamalert/__init__.py | 2 +- streamalert/shared/config.py | 2 +- streamalert_cli/terraform/athena.py | 7 +- .../terraform/scheduled_queries.py | 2 +- tests/unit/conf/scheduled_queries.json | 24 +++++++ .../terraform/test_scheduled_queries.py | 70 +++++++++++++++++++ 6 files changed, 99 insertions(+), 8 deletions(-) create mode 100644 tests/unit/conf/scheduled_queries.json create mode 100644 tests/unit/streamalert_cli/terraform/test_scheduled_queries.py diff --git a/streamalert/__init__.py b/streamalert/__init__.py index 49f37a67d..b544ce1d1 100644 --- a/streamalert/__init__.py +++ b/streamalert/__init__.py @@ -1,2 +1,2 @@ """StreamAlert version.""" -__version__ = '3.1.1' +__version__ = '3.1.2' diff --git a/streamalert/shared/config.py b/streamalert/shared/config.py index 150cc5285..c9ddcc892 100644 --- a/streamalert/shared/config.py +++ b/streamalert/shared/config.py @@ -145,7 +145,7 @@ def athena_query_results_bucket(config): return athena_config.get( 'results_bucket', - '{}.streamalert.athena-results'.format(prefix) + '{}-streamalert-athena-results'.format(prefix) ).strip() diff --git a/streamalert_cli/terraform/athena.py b/streamalert_cli/terraform/athena.py index 73f994e10..7e90cd9b2 100644 --- a/streamalert_cli/terraform/athena.py +++ b/streamalert_cli/terraform/athena.py @@ -14,7 +14,7 @@ limitations under the License. """ from streamalert.shared import metrics -from streamalert.shared.config import athena_partition_buckets +from streamalert.shared.config import athena_partition_buckets, athena_query_results_bucket from streamalert_cli.manage_lambda.package import AthenaPackage from streamalert_cli.terraform.common import ( infinitedict, @@ -40,10 +40,7 @@ def generate_athena(config): prefix = config['global']['account']['prefix'] database = athena_config.get('database_name', '{}_streamalert'.format(prefix)) - results_bucket_name = athena_config.get( - 'results_bucket', - '{}-streamalert-athena-results'.format(prefix) - ).strip() + results_bucket_name = athena_query_results_bucket(config) queue_name = athena_config.get( 'queue_name', diff --git a/streamalert_cli/terraform/scheduled_queries.py b/streamalert_cli/terraform/scheduled_queries.py index 6041e80ca..f7ccaf790 100644 --- a/streamalert_cli/terraform/scheduled_queries.py +++ b/streamalert_cli/terraform/scheduled_queries.py @@ -37,7 +37,7 @@ def generate_scheduled_queries_module_configuration(config): # FIXME (derek.wang) DRY out this code results_bucket = athena_query_results_bucket(config) - athena_s3_buckets = athena_partition_buckets(config) + athena_s3_buckets = sorted(athena_partition_buckets(config)) # Copy the config over directly scheduled_queries_module = streamquery_config.get('config', {}) diff --git a/tests/unit/conf/scheduled_queries.json b/tests/unit/conf/scheduled_queries.json new file mode 100644 index 000000000..4b24fc210 --- /dev/null +++ b/tests/unit/conf/scheduled_queries.json @@ -0,0 +1,24 @@ +{ + "enabled": true, + "config": { + "destination_kinesis_stream": "unit-test_stream", + "sfn_timeout_secs": 3600, + "sfn_wait_secs": 30 + }, + "packs": { + "hourly": { + "description": "Runs all hourly queries. Once per day on :05", + "schedule_expression": "cron(5 * * * ? *)" + } + }, + "lambda_config": { + "log_level": "info", + "log_retention_days": 14, + "memory": 128, + "timeout": 60, + "alarms_enabled": true, + "error_threshold": 1, + "error_period_secs": 3600, + "error_evaluation_periods": 2 + } +} \ No newline at end of file diff --git a/tests/unit/streamalert_cli/terraform/test_scheduled_queries.py b/tests/unit/streamalert_cli/terraform/test_scheduled_queries.py new file mode 100644 index 000000000..b1b165975 --- /dev/null +++ b/tests/unit/streamalert_cli/terraform/test_scheduled_queries.py @@ -0,0 +1,70 @@ +""" +Copyright 2017-present Airbnb, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from nose.tools import assert_equal + +from streamalert_cli.config import CLIConfig +from streamalert_cli.terraform import scheduled_queries + +CONFIG = CLIConfig(config_path='tests/unit/conf') + + +def test_generate_scheduled_queries(): + """CLI - Terraform Generate Scheduled Queries""" + + expected_sq_config = { + 'module': { + 'scheduled_queries': { + 'source': './modules/tf_scheduled_queries', + 'destination_kinesis_stream': 'unit-test_stream', + 'sfn_timeout_secs': 3600, + 'sfn_wait_secs': 30, + 'prefix': 'unit-test', + 'account_id': '12345678910', + 'region': 'us-west-1', + 'athena_database': 'unit-test_streamalert', + 'athena_results_bucket': 'unit-test-streamalert-athena-results', + 'athena_s3_buckets': [ + 'bucket', + 'unit-test-streamalert-data', + 'unit-test-streamalerts' + ], + 'lambda_filename': 'scheduled_queries.zip', + 'lambda_handler': 'streamalert.scheduled_queries.main.handler', + 'query_packs': [ + { + 'name': 'hourly', + 'schedule_expression': 'cron(5 * * * ? *)', + 'description': 'Runs all hourly queries. Once per day on :05' + } + ], + 'lambda_log_level': 'info', + 'lambda_memory': 128, + 'lambda_timeout': 60, + 'lambda_log_retention_days': 14, + 'lambda_alarms_enabled': True, + 'lambda_error_threshold': 1, + 'lambda_error_period_secs': 3600, + 'lambda_error_evaluation_periods': 2, + 'lambda_alarm_actions': [ + 'arn:aws:sns:us-west-1:12345678910:unit-test_streamalert_monitoring' + ], + } + } + } + + result = scheduled_queries.generate_scheduled_queries_module_configuration(CONFIG) + + assert_equal(result, expected_sq_config)