Merge 2afa484 into 8edeab6

.gitignore

@@ -6,11 +6,11 @@ docs/build
 __pycache__
 
 # Terraform files
-*.terraform/
-terraform.tfvars
-terraform/terraform.tfstate*
-terraform/*.zip
-terraform/*.tf.json
+streamalert_cli/_infrastructure/.terraform/
+streamalert_cli/_infrastructure/terraform.tfvars
+streamalert_cli/_infrastructure/terraform.tfstate*
+streamalert_cli/_infrastructure/*.zip
+streamalert_cli/_infrastructure/*.tf.json
 
 # Coveralls repo token
 .coveralls.yml

conf/lambda.json

@@ -52,7 +52,7 @@
   },
   "athena_partition_refresh_config": {
     "concurrency_limit": 10,
-    "file_format": null,
+    "file_format": "parquet",
     "log_level": "info"
   },
   "classifier_config": {},

streamalert_cli/__init__.py

@@ -0,0 +1,18 @@
+"""
+Copyright 2017-present Airbnb, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+import os
+
+STREAMALERT_CLI_ROOT = os.path.dirname(os.path.abspath(__file__))

streamalert_cli/helpers.py

@@ -24,6 +24,9 @@
 
 from streamalert.shared.logger import get_logger
 
+from streamalert_cli.terraform import TERRAFORM_FILES_PATH
+
+
 LOGGER = get_logger(__name__)
 
 SCHEMA_TYPE_LOOKUP = {
@@ -49,8 +52,7 @@ def run_command(runner_args, **kwargs):
     """
     default_error_message = "An error occurred while running: {}".format(' '.join(runner_args))
     error_message = kwargs.get('error_message', default_error_message)
-    default_cwd = 'terraform'
-    cwd = kwargs.get('cwd', default_cwd)
+    cwd = kwargs.get('cwd', TERRAFORM_FILES_PATH)
 
     # Add the -force-copy flag for s3 state copying to suppress dialogs that
     # the user must type 'yes' into.

streamalert_cli/manage_lambda/package.py

@@ -21,6 +21,7 @@
 
 from streamalert.shared.logger import get_logger
 from streamalert_cli.helpers import run_command
+from streamalert_cli.terraform import TERRAFORM_FILES_PATH
 
 # Build .zip files in the top-level of the terraform directory
 THIS_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
@@ -85,7 +86,7 @@ def create(self):
 
         # Zip up files
         result = shutil.make_archive(
-            os.path.join(BUILD_DIRECTORY, self.package_name), 'zip', temp_package_path)
+            os.path.join(TERRAFORM_FILES_PATH, self.package_name), 'zip', temp_package_path)
         LOGGER.info('Successfully created %s', os.path.basename(result))
 
         # Remove temp files

streamalert_cli/terraform/__init__.py

@@ -0,0 +1,5 @@
+import os
+
+from streamalert_cli import STREAMALERT_CLI_ROOT
+
+TERRAFORM_FILES_PATH = os.path.join(STREAMALERT_CLI_ROOT, '_infrastructure')

streamalert_cli/terraform/common.py

@@ -15,7 +15,6 @@
 """
 from collections import defaultdict
 
-
 DEFAULT_SNS_MONITORING_TOPIC_SUFFIX = '{}_streamalert_monitoring'
 DEFAULT_S3_LOGGING_BUCKET_SUFFIX = '{}-streamalert-s3-logging'
 DEFAULT_TERRAFORM_STATE_BUCKET_SUFFIX = '{}-streamalert-terraform-state'

streamalert_cli/terraform/generate.py

@@ -22,6 +22,7 @@
 from streamalert.shared.utils import get_database_name, get_data_file_format
 from streamalert_cli.athena.helpers import generate_alerts_table_schema
 from streamalert_cli.helpers import check_credentials
+from streamalert_cli.terraform import TERRAFORM_FILES_PATH
 from streamalert_cli.terraform.common import (
     InvalidClusterName,
     infinitedict,
@@ -375,9 +376,9 @@ def cleanup_old_tf_files():
     """
     Cleanup old *.tf.json files
     """
-    for terraform_file in os.listdir('terraform'):
+    for terraform_file in os.listdir(TERRAFORM_FILES_PATH):
         if fnmatch(terraform_file, '*.tf.json'):
-            os.remove(os.path.join('terraform', terraform_file))
+            os.remove(os.path.join(TERRAFORM_FILES_PATH, terraform_file))
 
 
 class TerraformGenerateCommand(CLICommand):
@@ -414,7 +415,10 @@ def terraform_generate_handler(config, init=False, check_tf=True, check_creds=Tr
 
     # Setup the main.tf.json file
     LOGGER.debug('Generating cluster file: main.tf.json')
-    _create_terraform_module_file(generate_main(config, init=init), 'terraform/main.tf.json')
+    _create_terraform_module_file(
+        generate_main(config, init=init),
+        os.path.join(TERRAFORM_FILES_PATH, 'main.tf.json')
+    )
 
     # Return early during the init process, clusters are not needed yet
     if init:
@@ -433,22 +437,32 @@ def terraform_generate_handler(config, init=False, check_tf=True, check_creds=Tr
                 'An error was generated while creating the %s cluster', cluster)
             return False
 
-        _create_terraform_module_file(cluster_dict, 'terraform/{}.tf.json'.format(cluster))
+        file_name = '{}.tf.json'.format(cluster)
+        _create_terraform_module_file(
+            cluster_dict,
+            os.path.join(TERRAFORM_FILES_PATH, file_name),
+        )
 
     metric_filters = generate_aggregate_cloudwatch_metric_filters(config)
     if metric_filters:
-        _create_terraform_module_file(metric_filters, 'terraform/metric_filters.tf.json')
+        _create_terraform_module_file(
+            metric_filters,
+            os.path.join(TERRAFORM_FILES_PATH, 'metric_filters.tf.json')
+        )
 
     metric_alarms = generate_aggregate_cloudwatch_metric_alarms(config)
     if metric_alarms:
-        _create_terraform_module_file(metric_alarms, 'terraform/metric_alarms.tf.json')
+        _create_terraform_module_file(
+            metric_alarms,
+            os.path.join(TERRAFORM_FILES_PATH, 'metric_alarms.tf.json')
+        )
 
     # Setup Athena
     generate_global_lambda_settings(
         config,
         config_name='athena_partition_refresh_config',
         generate_func=generate_athena,
-        tf_tmp_file='terraform/athena.tf.json',
+        tf_tmp_file=os.path.join(TERRAFORM_FILES_PATH, 'athena.tf.json'),
         message='Removing old Athena Terraform file'
     )
 
@@ -457,7 +471,7 @@ def terraform_generate_handler(config, init=False, check_tf=True, check_creds=Tr
         config,
         config_name='threat_intel_downloader_config',
         generate_func=generate_threat_intel_downloader,
-        tf_tmp_file='terraform/ti_downloader.tf.json',
+        tf_tmp_file=os.path.join(TERRAFORM_FILES_PATH, 'ti_downloader.tf.json'),
         message='Removing old Threat Intel Downloader Terraform file'
     )
 
@@ -466,7 +480,7 @@ def terraform_generate_handler(config, init=False, check_tf=True, check_creds=Tr
         config,
         config_name='rule_promotion_config',
         generate_func=generate_rule_promotion,
-        tf_tmp_file='terraform/rule_promotion.tf.json',
+        tf_tmp_file=os.path.join(TERRAFORM_FILES_PATH, 'rule_promotion.tf.json'),
         message='Removing old Rule Promotion Terraform file'
     )
 
@@ -475,7 +489,7 @@ def terraform_generate_handler(config, init=False, check_tf=True, check_creds=Tr
         config,
         config_name='rules_engine_config',
         generate_func=generate_rules_engine,
-        tf_tmp_file='terraform/rules_engine.tf.json',
+        tf_tmp_file=os.path.join(TERRAFORM_FILES_PATH, 'rules_engine.tf.json'),
         message='Removing old Rules Engine Terraform file'
     )
 
@@ -484,7 +498,7 @@ def terraform_generate_handler(config, init=False, check_tf=True, check_creds=Tr
         config,
         config_name='alert_processor_config',
         generate_func=generate_alert_processor,
-        tf_tmp_file='terraform/alert_processor.tf.json',
+        tf_tmp_file=os.path.join(TERRAFORM_FILES_PATH, 'alert_processor.tf.json'),
         message='Removing old Alert Processor Terraform file'
     )
 
@@ -493,7 +507,7 @@ def terraform_generate_handler(config, init=False, check_tf=True, check_creds=Tr
         config,
         config_name='alert_merger_config',
         generate_func=generate_alert_merger,
-        tf_tmp_file='terraform/alert_merger.tf.json',
+        tf_tmp_file=os.path.join(TERRAFORM_FILES_PATH, 'alert_merger.tf.json'),
         message='Removing old Alert Merger Terraform file'
     )
 
@@ -510,7 +524,7 @@ def _generate_lookup_tables_settings(config):
     """
     Generates .tf.json file for LookupTables
     """
-    tf_file_name = 'terraform/lookup_tables.tf.json'
+    tf_file_name = os.path.join(TERRAFORM_FILES_PATH, 'lookup_tables.tf.json')
 
     if not config['lookup_tables'].get('enabled', False):
         remove_temp_terraform_file(tf_file_name, 'Removing old LookupTables Terraform file')
@@ -572,7 +586,7 @@ def _generate_streamquery_module(config):
     """
     Generates .tf.json file for scheduled queries
     """
-    tf_file_name = 'terraform/scheduled_queries.tf.json'
+    tf_file_name = os.path.join(TERRAFORM_FILES_PATH, 'scheduled_queries.tf.json')
     if not config.get('scheduled_queries', {}).get('enabled', False):
         remove_temp_terraform_file(tf_file_name, 'Removing old scheduled queries Terraform file')
         return

streamalert_cli/terraform/handlers.py

@@ -24,6 +24,7 @@
 from streamalert_cli.athena.handler import create_table, create_log_tables
 from streamalert_cli.helpers import check_credentials, continue_prompt, run_command, tf_runner
 from streamalert_cli.manage_lambda.deploy import deploy
+from streamalert_cli.terraform import TERRAFORM_FILES_PATH
 from streamalert_cli.terraform.generate import terraform_generate_handler
 from streamalert_cli.terraform.helpers import terraform_check
 from streamalert_cli.utils import (
@@ -304,12 +305,13 @@ def _rm_file(path):
                     _rm_file(path)
 
         for tf_file in ['terraform.tfstate', 'terraform.tfstate.backup']:
-            path = 'terraform/{}'.format(tf_file)
+            path = os.path.join(TERRAFORM_FILES_PATH, tf_file)
             _rm_file(path)
 
         # Finally, delete the Terraform directory
-        if os.path.isdir('terraform/.terraform/'):
-            shutil.rmtree('terraform/.terraform/')
+        tf_path = os.path.join(TERRAFORM_FILES_PATH, '.terraform')
+        if os.path.isdir(tf_path):
+            shutil.rmtree(tf_path)
 
         return True