CLI support for extra user supplied terraform files (#1267)

* adding cli arg to supply additional terraform config files

* removing old tf cleanup code since temp path will be used

* cliconfig support for temp tf directory

* updates to tf_runner and run_command for temp tf path

* removing tf clean command since runs are now idempotent

* packaging change for tf temp path

* logic for copying files to tf temp path

* removing init backend option

* cleanup

* fix unit tests

* config support for extra tf files

* doc update for `terraform_files` setting

* unit test for cliconfig terraform files

* fix for init backend outside of generate logic

* update to support supplying static dir for builds

conf/global.json

@@ -5,6 +5,7 @@
     "region": "us-east-1"
   },
   "general": {
+    "terraform_files": [],
     "matcher_locations": [
       "matchers"
     ],

docs/source/config-global.rst

@@ -61,6 +61,9 @@ Configuration
 
   {
     "general": {
+      "terraform_files": [
+        "/absolute/path/to/extra/terraform/file.tf"
+      ],
       "matcher_locations": [
         "matchers"
       ],
@@ -90,6 +93,7 @@ Options
 ``scheduled_query_locations``  Yes            ``["scheduled_queries"]``  List of local paths where ``scheduled_queries`` are defined
 ``publisher_locations``        Yes            ``["publishers"]``         List of local paths where ``publishers`` are defined
 ``third_party_libraries``      No             ``["pathlib2==2.3.5"]``    List of third party dependencies that should be installed via ``pip`` at deployment time. These are libraries needed in rules, custom code, etc that are defined in one of the above settings.
+``terraform_files``            No             ``[]``                     List of local paths to Terraform files that should be included as part of this StreamAlert deployment
 =============================  =============  =========================  ===============
 
 

manage.py

@@ -23,13 +23,17 @@
 
 terraform <cmd>
 """
-from argparse import ArgumentParser, RawDescriptionHelpFormatter
+from argparse import ArgumentParser, FileType, RawDescriptionHelpFormatter
 import sys
 
 from streamalert import __version__ as version
 from streamalert_cli.config import DEFAULT_CONFIG_PATH
 from streamalert_cli.runner import cli_runner, StreamAlertCLICommandRepository
-from streamalert_cli.utils import DirectoryType, generate_subparser
+from streamalert_cli.utils import (
+    DirectoryType,
+    generate_subparser,
+    UniqueSortedFileListAppendAction,
+)
 
 
 def build_parser():
@@ -79,6 +83,29 @@ def build_parser():
         type=DirectoryType()
     )
 
+    parser.add_argument(
+        '-t',
+        '--terraform-file',
+        dest='terraform_files',
+        help=(
+            'Path to one or more additional Terraform configuration '
+            'files to include in this deployment'
+        ),
+        action=UniqueSortedFileListAppendAction,
+        type=FileType('r'),
+        default=[]
+    )
+
+    parser.add_argument(
+        '-b',
+        '--build-directory',
+        help=(
+            'Path to directory to use for building StreamAlert and its infrastructure. '
+            'If no path is provided, a temporary directory will be used.'
+        ),
+        type=str
+    )
+
     # Dynamically generate subparsers, and create a 'commands' block for the prog description
     command_block = []
     subparsers = parser.add_subparsers(dest='command', required=True)

streamalert_cli/config.py

@@ -16,7 +16,9 @@
 import json
 import os
 import re
+import shutil
 import string
+import tempfile
 
 from streamalert.apps import StreamAlertApp
 from streamalert.shared import CLUSTERED_FUNCTIONS, config, metrics
@@ -32,9 +34,11 @@
 class CLIConfig:
     """A class to load, modify, and display the StreamAlertCLI Config"""
 
-    def __init__(self, config_path):
+    def __init__(self, config_path, extra_terraform_files=None, build_directory=None):
         self.config_path = config_path
         self.config = config.load_config(config_path)
+        self._terraform_files = extra_terraform_files or []
+        self.build_directory = self._setup_build_directory(build_directory)
 
     def __repr__(self):
         return str(self.config)
@@ -58,6 +62,32 @@ def clusters(self):
         """Return list of cluster configuration keys"""
         return list(self.config['clusters'].keys())
 
+    @property
+    def terraform_files(self):
+        """Return set of terraform files to include with this deployment"""
+        return set(self._terraform_files).union(
+            self.config['global']['general'].get('terraform_files', [])
+        )
+
+    @staticmethod
+    def _setup_build_directory(directory):
+        """Create the directory to be used for building infrastructure
+
+        Args:
+            directory (str): Optional path to directory to create
+
+        Returns:
+            str: Path to directory that will be used
+        """
+        if not directory:
+            temp_dir = tempfile.TemporaryDirectory(prefix='streamalert_build-')
+            directory = temp_dir.name
+
+        if os.path.exists(directory):
+            shutil.rmtree(directory)
+
+        return directory
+
     def set_prefix(self, prefix):
         """Set the Org Prefix in Global settings"""
         if not isinstance(prefix, str):

streamalert_cli/helpers.py

@@ -24,8 +24,6 @@
 
 from streamalert.shared.logger import get_logger
 
-from streamalert_cli.terraform import TERRAFORM_FILES_PATH
-
 
 LOGGER = get_logger(__name__)
 
@@ -39,7 +37,7 @@
 }
 
 
-def run_command(runner_args, **kwargs):
+def run_command(runner_args, cwd='./', **kwargs):
     """Helper function to run commands with error handling.
 
     Args:
@@ -52,7 +50,6 @@ def run_command(runner_args, **kwargs):
     """
     default_error_message = "An error occurred while running: {}".format(' '.join(runner_args))
     error_message = kwargs.get('error_message', default_error_message)
-    cwd = kwargs.get('cwd', TERRAFORM_FILES_PATH)
 
     # Add the -force-copy flag for s3 state copying to suppress dialogs that
     # the user must type 'yes' into.
@@ -98,12 +95,13 @@ def continue_prompt(message=None):
     return response == 'yes'
 
 
-def tf_runner(action='apply', refresh=True, auto_approve=False, targets=None):
+def tf_runner(config, action='apply', refresh=True, auto_approve=False, targets=None):
     """Terraform wrapper to build StreamAlert infrastructure.
 
     Resolves modules with `terraform get` before continuing.
 
     Args:
+        config (CLIConfig): Loaded StreamAlert config
         action (str): Terraform action ('apply' or 'destroy').
         refresh (bool): If True, Terraform will refresh its state before applying the change.
         auto_approve (bool): If True, Terraform will *not* prompt the user for approval.
@@ -113,8 +111,12 @@ def tf_runner(action='apply', refresh=True, auto_approve=False, targets=None):
     Returns:
         bool: True if the terraform command was successful
     """
+    LOGGER.info('Initializing StreamAlert')
+    if not run_command(['terraform', 'init'], cwd=config.build_directory):
+        return False
+
     LOGGER.debug('Resolving Terraform modules')
-    if not run_command(['terraform', 'get'], quiet=True):
+    if not run_command(['terraform', 'get'], cwd=config.build_directory, quiet=True):
         return False
 
     tf_command = ['terraform', action, '-refresh={}'.format(str(refresh).lower())]
@@ -130,7 +132,7 @@ def tf_runner(action='apply', refresh=True, auto_approve=False, targets=None):
     if targets:
         tf_command.extend('-target={}'.format(x) for x in targets)
 
-    return run_command(tf_command)
+    return run_command(tf_command, cwd=config.build_directory)
 
 
 def check_credentials():

streamalert_cli/kinesis/handler.py

@@ -83,6 +83,7 @@ def handler(cls, options, config):
             return False
 
         return tf_runner(
+            config,
             action='apply',
             targets=[
                 'module.{}_{}'.format('kinesis_events', cluster) for cluster in config.clusters()

streamalert_cli/manage_lambda/deploy.py

@@ -124,7 +124,7 @@ def deploy(config, functions, clusters=None):
     LOGGER.debug('Applying terraform targets: %s', ', '.join(sorted(deploy_targets)))
 
     # Terraform applies the new package and publishes a new version
-    return helpers.tf_runner(targets=deploy_targets)
+    return helpers.tf_runner(config, targets=deploy_targets)
 
 
 def _update_rule_table(options, config):

streamalert_cli/manage_lambda/package.py

@@ -19,7 +19,6 @@
 
 from streamalert.shared.logger import get_logger
 from streamalert_cli.helpers import run_command
-from streamalert_cli.terraform import TERRAFORM_FILES_PATH
 
 LOGGER = get_logger(__name__)
 
@@ -29,7 +28,7 @@ class LambdaPackage:
     # The name of the directory to package and basename of the generated .zip file
     PACKAGE_NAME = 'streamalert'
 
-    # The configurable items for user specified files
+    # The configurable items for user specified files to include in deployment pacakge
     CONFIG_EXTRAS = {
         'matcher_locations',
         'rule_locations',
@@ -86,7 +85,7 @@ def create(self):
         # Zip it all up
         # Build these in the top-level of the terraform directory as streamalert.zip
         result = shutil.make_archive(
-            os.path.join(TERRAFORM_FILES_PATH, self.PACKAGE_NAME),
+            os.path.join(self.config.build_directory, self.PACKAGE_NAME),
             'zip',
             self.temp_package_path
         )

streamalert_cli/runner.py

@@ -34,7 +34,6 @@
 from streamalert_cli.terraform.generate import TerraformGenerateCommand
 from streamalert_cli.terraform.handlers import (
     TerraformBuildCommand,
-    TerraformCleanCommand,
     TerraformDestroyCommand,
     TerraformInitCommand,
     TerraformListTargetsCommand,
@@ -59,7 +58,7 @@ def cli_runner(args):
     Returns:
         bool: False if errors occurred, True otherwise
     """
-    config = CLIConfig(args.config_dir)
+    config = CLIConfig(args.config_dir, args.terraform_files, args.build_directory)
 
     set_logger_levels(args.debug)
 
@@ -93,7 +92,6 @@ def register_all(cls):
             'app': AppCommand,
             'athena': AthenaCommand,
             'build': TerraformBuildCommand,
-            'clean': TerraformCleanCommand,
             'configure': ConfigureCommand,
             'create-alarm': CreateMetricAlarmCommand,
             'create-cluster-alarm': CreateClusterMetricAlarmCommand,

streamalert_cli/terraform/generate.py

@@ -13,9 +13,9 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
-from fnmatch import fnmatch
 import json
 import os
+import shutil
 
 from streamalert.shared.config import ConfigError, firehose_alerts_bucket
 from streamalert.shared.logger import get_logger
@@ -373,14 +373,6 @@ def generate_cluster(config, cluster_name):
     return cluster_dict
 
 
-def cleanup_old_tf_files():
-    """
-    Cleanup old *.tf.json files
-    """
-    for terraform_file in os.listdir(TERRAFORM_FILES_PATH):
-        if fnmatch(terraform_file, '*.tf.json'):
-            os.remove(os.path.join(TERRAFORM_FILES_PATH, terraform_file))
-
 
 class TerraformGenerateCommand(CLICommand):
     description = 'Generate Terraform files from JSON cluster files'
@@ -394,6 +386,28 @@ def handler(cls, options, config):
         return terraform_generate_handler(config, check_creds=False)
 
 
+def _copy_terraform_files(config):
+    """Copy all packaged terraform files and terraform files provided by the user to temp
+
+    Args:
+        config (CLIConfig): Loaded StreamAlert config
+    """
+    # Copy the packaged terraform files to temp
+    # Currently this ignores *.tf.json and *.zip files, in the instance that these
+    # exist in current deployments. This can be removed in a future release.
+    shutil.copytree(
+        TERRAFORM_FILES_PATH,
+        config.build_directory,
+        ignore=shutil.ignore_patterns('*.tf.json', '*.zip')  # TODO: remove this eventually
+    )
+
+    # Copy any additional user provided terraform files to temp
+    for item in config.terraform_files:
+        shutil.copy2(item, config.build_directory)
+
+    LOGGER.info('Copied Terraform configuration to \'%s\'', config.build_directory)
+
+
 def terraform_generate_handler(config, init=False, check_tf=True, check_creds=True):
     """Generate all Terraform plans for the configured clusters.
 
@@ -412,13 +426,13 @@ def terraform_generate_handler(config, init=False, check_tf=True, check_creds=Tr
     if check_tf and not terraform_check():
         return False
 
-    cleanup_old_tf_files()
+    _copy_terraform_files(config)
 
     # Setup the main.tf.json file
     LOGGER.debug('Generating cluster file: main.tf.json')
     _create_terraform_module_file(
         generate_main(config, init=init),
-        os.path.join(TERRAFORM_FILES_PATH, 'main.tf.json')
+        os.path.join(config.build_directory, 'main.tf.json')
     )
 
     # Setup Artifact Extractor if it is enabled.
@@ -451,21 +465,21 @@ def terraform_generate_handler(config, init=False, check_tf=True, check_creds=Tr
         file_name = '{}.tf.json'.format(cluster)
         _create_terraform_module_file(
             cluster_dict,
-            os.path.join(TERRAFORM_FILES_PATH, file_name),
+            os.path.join(config.build_directory, file_name),
         )
 
     metric_filters = generate_aggregate_cloudwatch_metric_filters(config)
     if metric_filters:
         _create_terraform_module_file(
             metric_filters,
-            os.path.join(TERRAFORM_FILES_PATH, 'metric_filters.tf.json')
+            os.path.join(config.build_directory, 'metric_filters.tf.json')
         )
 
     metric_alarms = generate_aggregate_cloudwatch_metric_alarms(config)
     if metric_alarms:
         _create_terraform_module_file(
             metric_alarms,
-            os.path.join(TERRAFORM_FILES_PATH, 'metric_alarms.tf.json')
+            os.path.join(config.build_directory, 'metric_alarms.tf.json')
         )
 
     # Setup Threat Intel Downloader Lambda function if it is enabled
@@ -531,7 +545,7 @@ def _generate_lookup_tables_settings(config):
     """
     Generates .tf.json file for LookupTables
     """
-    tf_file_name = os.path.join(TERRAFORM_FILES_PATH, 'lookup_tables.tf.json')
+    tf_file_name = os.path.join(config.build_directory, 'lookup_tables.tf.json')
 
     if not config['lookup_tables'].get('enabled', False):
         remove_temp_terraform_file(tf_file_name)
@@ -594,7 +608,7 @@ def _generate_streamquery_module(config):
     """
     Generates .tf.json file for scheduled queries
     """
-    tf_file_name = os.path.join(TERRAFORM_FILES_PATH, 'scheduled_queries.tf.json')
+    tf_file_name = os.path.join(config.build_directory, 'scheduled_queries.tf.json')
     if not config.get('scheduled_queries', {}).get('enabled', False):
         remove_temp_terraform_file(tf_file_name)
         return
@@ -637,7 +651,7 @@ def generate_global_lambda_settings(
             )
             raise ConfigError(message)
 
-    tf_tmp_file = os.path.join(TERRAFORM_FILES_PATH, '{}.tf.json'.format(tf_tmp_file_name))
+    tf_tmp_file = os.path.join(config.build_directory, '{}.tf.json'.format(tf_tmp_file_name))
 
     if required and conf_name not in config['lambda']:
         message = 'Required configuration missing in lambda.json: {}'.format(conf_name)