Merge branch 'release-3-3-0' into move_artifact_extractor_logic_to_cl…

…assifier

streamalert_cli/config.py

@@ -82,6 +82,9 @@ def _setup_build_directory(directory):
         if not directory:
             temp_dir = tempfile.TemporaryDirectory(prefix='streamalert_build-')
             directory = temp_dir.name
+            # Calling cleanup here to remove this directory so shutil can recreate it
+            # Without calling this here, an exception is raised when tempfile garbage collects
+            temp_dir.cleanup()
 
         if os.path.exists(directory):
             shutil.rmtree(directory)

streamalert_cli/helpers.py

@@ -95,46 +95,6 @@ def continue_prompt(message=None):
     return response == 'yes'
 
 
-def tf_runner(config, action='apply', refresh=True, auto_approve=False, targets=None):
-    """Terraform wrapper to build StreamAlert infrastructure.
-
-    Resolves modules with `terraform get` before continuing.
-
-    Args:
-        config (CLIConfig): Loaded StreamAlert config
-        action (str): Terraform action ('apply' or 'destroy').
-        refresh (bool): If True, Terraform will refresh its state before applying the change.
-        auto_approve (bool): If True, Terraform will *not* prompt the user for approval.
-        targets (list): Optional list of affected targets.
-            If not specified, Terraform will run against all of its resources.
-
-    Returns:
-        bool: True if the terraform command was successful
-    """
-    LOGGER.info('Initializing StreamAlert')
-    if not run_command(['terraform', 'init'], cwd=config.build_directory):
-        return False
-
-    LOGGER.debug('Resolving Terraform modules')
-    if not run_command(['terraform', 'get'], cwd=config.build_directory, quiet=True):
-        return False
-
-    tf_command = ['terraform', action, '-refresh={}'.format(str(refresh).lower())]
-
-    if action == 'destroy':
-        # Terraform destroy has a '-force' flag instead of '-auto-approve'
-        LOGGER.info('Destroying infrastructure')
-        tf_command.append('-force={}'.format(str(auto_approve).lower()))
-    else:
-        LOGGER.info('%s changes', 'Applying' if auto_approve else 'Planning')
-        tf_command.append('-auto-approve={}'.format(str(auto_approve).lower()))
-
-    if targets:
-        tf_command.extend('-target={}'.format(x) for x in targets)
-
-    return run_command(tf_command, cwd=config.build_directory)
-
-
 def check_credentials():
     """Check for valid AWS credentials in environment variables
 

streamalert_cli/kinesis/handler.py

@@ -14,8 +14,8 @@
 limitations under the License.
 """
 from streamalert.shared.logger import get_logger
-from streamalert_cli.helpers import tf_runner
 from streamalert_cli.terraform.generate import terraform_generate_handler
+from streamalert_cli.terraform.helpers import terraform_runner
 from streamalert_cli.utils import CLICommand, set_parser_epilog, add_clusters_arg
 
 LOGGER = get_logger(__name__)
@@ -82,9 +82,8 @@ def handler(cls, options, config):
         if not terraform_generate_handler(config):
             return False
 
-        return tf_runner(
+        return terraform_runner(
             config,
-            action='apply',
             targets=[
                 'module.{}_{}'.format('kinesis_events', cluster) for cluster in config.clusters()
             ]

streamalert_cli/manage_lambda/deploy.py

@@ -15,9 +15,8 @@
 """
 from streamalert.shared import rule_table
 from streamalert.shared.logger import get_logger
-from streamalert_cli import helpers
-from streamalert_cli.manage_lambda import package
 from streamalert_cli.terraform.generate import terraform_generate_handler
+from streamalert_cli.terraform.helpers import terraform_runner
 from streamalert_cli.utils import (
     add_default_lambda_args,
     CLICommand,
@@ -111,11 +110,6 @@ def deploy(config, functions, clusters=None):
     """
     LOGGER.info('Deploying: %s', ', '.join(sorted(functions)))
 
-    deployment_package = package.LambdaPackage(config)
-    package_path = deployment_package.create()
-    if not package_path:
-        return False
-
     # Terraform apply only to the module which contains our lambda functions
     clusters = clusters or config.clusters()
 
@@ -124,7 +118,7 @@ def deploy(config, functions, clusters=None):
     LOGGER.debug('Applying terraform targets: %s', ', '.join(sorted(deploy_targets)))
 
     # Terraform applies the new package and publishes a new version
-    return helpers.tf_runner(config, targets=deploy_targets)
+    return terraform_runner(config, targets=deploy_targets, build_pkg=True)
 
 
 def _update_rule_table(options, config):

streamalert_cli/terraform/handlers.py

@@ -21,10 +21,10 @@
 from streamalert.shared.logger import get_logger
 from streamalert.shared.utils import get_data_file_format
 from streamalert_cli.athena.handler import create_table, create_log_tables
-from streamalert_cli.helpers import check_credentials, continue_prompt, run_command, tf_runner
+from streamalert_cli.helpers import check_credentials, continue_prompt, run_command
 from streamalert_cli.manage_lambda.deploy import deploy
 from streamalert_cli.terraform.generate import terraform_generate_handler
-from streamalert_cli.terraform.helpers import terraform_check
+from streamalert_cli.terraform.helpers import terraform_check, terraform_runner
 from streamalert_cli.utils import (
     add_clusters_arg,
     CLICommand,
@@ -79,7 +79,7 @@ def handler(cls, options, config):
         if config['global']['infrastructure'].get('firehose', {}).get('enabled'):
             init_targets.append('aws_s3_bucket.streamalert_data')
 
-        if not tf_runner(config, targets=init_targets):
+        if not terraform_runner(config, targets=init_targets):
             LOGGER.error('An error occurred while running StreamAlert init')
             return False
 
@@ -115,7 +115,7 @@ def handler(cls, options, config):
                 return
 
         LOGGER.info('Building remaining infrastructure')
-        return tf_runner(config, refresh=False)
+        return terraform_runner(config, refresh=False)
 
 
 class TerraformBuildCommand(CLICommand):
@@ -161,7 +161,13 @@ def handler(cls, options, config):
         if not valid:
             return False
 
-        return tf_runner(config, targets=target_modules if target_modules else None)
+        build_pkg = any('lambda' in target for target in target_modules)
+
+        return terraform_runner(
+            config,
+            targets=target_modules if target_modules else None,
+            build_pkg=build_pkg,
+        )
 
 
 class TerraformDestroyCommand(CLICommand):
@@ -211,9 +217,9 @@ def handler(cls, options, config):
             if not valid:
                 return False
 
-            return tf_runner(
+            return terraform_runner(
                 config,
-                action='destroy',
+                destroy=True,
                 auto_approve=True,
                 targets=target_modules if target_modules else None
             )
@@ -229,7 +235,7 @@ def handler(cls, options, config):
             return False
 
         # Destroy all of the infrastructure
-        return tf_runner(config, action='destroy', auto_approve=True)
+        return terraform_runner(config, destroy=True, auto_approve=True)
 
 
 class TerraformListTargetsCommand(CLICommand):

streamalert_cli/terraform/helpers.py

@@ -14,10 +14,69 @@
 """
 from streamalert.shared.logger import get_logger
 from streamalert_cli.helpers import run_command
+from streamalert_cli.manage_lambda import package
 
 LOGGER = get_logger(__name__)
 
 
+def terraform_runner(
+        config,
+        refresh=True,
+        auto_approve=False,
+        targets=None,
+        destroy=False,
+        build_pkg=False):
+    """Terraform wrapper to build StreamAlert infrastructure.
+
+    Resolves modules with `terraform get` before continuing.
+
+    Args:
+        config (CLIConfig): Loaded StreamAlert config
+        action (str): Terraform action ('apply' or 'destroy').
+        refresh (bool): If True, Terraform will refresh its state before applying the change.
+        auto_approve (bool): If True, Terraform will *not* prompt the user for approval.
+        targets (list): Optional list of affected targets.
+            If not specified, Terraform will run against all of its resources.
+
+    Returns:
+        bool: True if the terraform command was successful
+    """
+    LOGGER.info('Initializing StreamAlert')
+    if not run_command(['terraform', 'init'], cwd=config.build_directory):
+        return False
+
+    LOGGER.debug('Resolving Terraform modules')
+    if not run_command(['terraform', 'get'], cwd=config.build_directory, quiet=True):
+        return False
+
+    tf_command = ['terraform']
+
+    if destroy:
+        tf_command.append('destroy')
+        # Terraform destroy has a '-force' flag instead of '-auto-approve'
+        LOGGER.info('Destroying infrastructure')
+        tf_command.append('-force={}'.format(str(auto_approve).lower()))
+    else:
+        tf_command.append('apply')
+        LOGGER.info('%s changes', 'Applying' if auto_approve else 'Planning')
+        tf_command.append('-auto-approve={}'.format(str(auto_approve).lower()))
+
+    tf_command.append('-refresh={}'.format(str(refresh).lower()))
+
+    if targets:
+        tf_command.extend('-target={}'.format(x) for x in targets)
+
+    # Building of the deployment package should happen if Lambda is
+    # an explict target or there are no targets at all
+    if build_pkg or not targets:
+        deployment_package = package.LambdaPackage(config)
+        package_path = deployment_package.create()
+        if not package_path:
+            return False
+
+    return run_command(tf_command, cwd=config.build_directory)
+
+
 def terraform_check():
     """Verify that Terraform is configured correctly