diff --git a/streamalert_cli/config.py b/streamalert_cli/config.py index f3029728b..6c353778c 100644 --- a/streamalert_cli/config.py +++ b/streamalert_cli/config.py @@ -82,6 +82,9 @@ def _setup_build_directory(directory): if not directory: temp_dir = tempfile.TemporaryDirectory(prefix='streamalert_build-') directory = temp_dir.name + # Calling cleanup here to remove this directory so shutil can recreate it + # Without calling this here, an exception is raised when tempfile garbage collects + temp_dir.cleanup() if os.path.exists(directory): shutil.rmtree(directory) diff --git a/streamalert_cli/helpers.py b/streamalert_cli/helpers.py index b5c9339bc..839178c13 100644 --- a/streamalert_cli/helpers.py +++ b/streamalert_cli/helpers.py @@ -95,46 +95,6 @@ def continue_prompt(message=None): return response == 'yes' -def tf_runner(config, action='apply', refresh=True, auto_approve=False, targets=None): - """Terraform wrapper to build StreamAlert infrastructure. - - Resolves modules with `terraform get` before continuing. - - Args: - config (CLIConfig): Loaded StreamAlert config - action (str): Terraform action ('apply' or 'destroy'). - refresh (bool): If True, Terraform will refresh its state before applying the change. - auto_approve (bool): If True, Terraform will *not* prompt the user for approval. - targets (list): Optional list of affected targets. - If not specified, Terraform will run against all of its resources. - - Returns: - bool: True if the terraform command was successful - """ - LOGGER.info('Initializing StreamAlert') - if not run_command(['terraform', 'init'], cwd=config.build_directory): - return False - - LOGGER.debug('Resolving Terraform modules') - if not run_command(['terraform', 'get'], cwd=config.build_directory, quiet=True): - return False - - tf_command = ['terraform', action, '-refresh={}'.format(str(refresh).lower())] - - if action == 'destroy': - # Terraform destroy has a '-force' flag instead of '-auto-approve' - LOGGER.info('Destroying infrastructure') - tf_command.append('-force={}'.format(str(auto_approve).lower())) - else: - LOGGER.info('%s changes', 'Applying' if auto_approve else 'Planning') - tf_command.append('-auto-approve={}'.format(str(auto_approve).lower())) - - if targets: - tf_command.extend('-target={}'.format(x) for x in targets) - - return run_command(tf_command, cwd=config.build_directory) - - def check_credentials(): """Check for valid AWS credentials in environment variables diff --git a/streamalert_cli/kinesis/handler.py b/streamalert_cli/kinesis/handler.py index 80d02d3e0..5324e95a4 100644 --- a/streamalert_cli/kinesis/handler.py +++ b/streamalert_cli/kinesis/handler.py @@ -14,8 +14,8 @@ limitations under the License. """ from streamalert.shared.logger import get_logger -from streamalert_cli.helpers import tf_runner from streamalert_cli.terraform.generate import terraform_generate_handler +from streamalert_cli.terraform.helpers import terraform_runner from streamalert_cli.utils import CLICommand, set_parser_epilog, add_clusters_arg LOGGER = get_logger(__name__) @@ -82,9 +82,8 @@ def handler(cls, options, config): if not terraform_generate_handler(config): return False - return tf_runner( + return terraform_runner( config, - action='apply', targets=[ 'module.{}_{}'.format('kinesis_events', cluster) for cluster in config.clusters() ] diff --git a/streamalert_cli/manage_lambda/deploy.py b/streamalert_cli/manage_lambda/deploy.py index 1e584f2c8..f6cad23b0 100644 --- a/streamalert_cli/manage_lambda/deploy.py +++ b/streamalert_cli/manage_lambda/deploy.py @@ -15,9 +15,8 @@ """ from streamalert.shared import rule_table from streamalert.shared.logger import get_logger -from streamalert_cli import helpers -from streamalert_cli.manage_lambda import package from streamalert_cli.terraform.generate import terraform_generate_handler +from streamalert_cli.terraform.helpers import terraform_runner from streamalert_cli.utils import ( add_default_lambda_args, CLICommand, @@ -111,11 +110,6 @@ def deploy(config, functions, clusters=None): """ LOGGER.info('Deploying: %s', ', '.join(sorted(functions))) - deployment_package = package.LambdaPackage(config) - package_path = deployment_package.create() - if not package_path: - return False - # Terraform apply only to the module which contains our lambda functions clusters = clusters or config.clusters() @@ -124,7 +118,7 @@ def deploy(config, functions, clusters=None): LOGGER.debug('Applying terraform targets: %s', ', '.join(sorted(deploy_targets))) # Terraform applies the new package and publishes a new version - return helpers.tf_runner(config, targets=deploy_targets) + return terraform_runner(config, targets=deploy_targets, build_pkg=True) def _update_rule_table(options, config): diff --git a/streamalert_cli/terraform/handlers.py b/streamalert_cli/terraform/handlers.py index e96426dee..1963843cd 100644 --- a/streamalert_cli/terraform/handlers.py +++ b/streamalert_cli/terraform/handlers.py @@ -21,10 +21,10 @@ from streamalert.shared.logger import get_logger from streamalert.shared.utils import get_data_file_format from streamalert_cli.athena.handler import create_table, create_log_tables -from streamalert_cli.helpers import check_credentials, continue_prompt, run_command, tf_runner +from streamalert_cli.helpers import check_credentials, continue_prompt, run_command from streamalert_cli.manage_lambda.deploy import deploy from streamalert_cli.terraform.generate import terraform_generate_handler -from streamalert_cli.terraform.helpers import terraform_check +from streamalert_cli.terraform.helpers import terraform_check, terraform_runner from streamalert_cli.utils import ( add_clusters_arg, CLICommand, @@ -79,7 +79,7 @@ def handler(cls, options, config): if config['global']['infrastructure'].get('firehose', {}).get('enabled'): init_targets.append('aws_s3_bucket.streamalert_data') - if not tf_runner(config, targets=init_targets): + if not terraform_runner(config, targets=init_targets): LOGGER.error('An error occurred while running StreamAlert init') return False @@ -115,7 +115,7 @@ def handler(cls, options, config): return LOGGER.info('Building remaining infrastructure') - return tf_runner(config, refresh=False) + return terraform_runner(config, refresh=False) class TerraformBuildCommand(CLICommand): @@ -161,7 +161,13 @@ def handler(cls, options, config): if not valid: return False - return tf_runner(config, targets=target_modules if target_modules else None) + build_pkg = any('lambda' in target for target in target_modules) + + return terraform_runner( + config, + targets=target_modules if target_modules else None, + build_pkg=build_pkg, + ) class TerraformDestroyCommand(CLICommand): @@ -211,9 +217,9 @@ def handler(cls, options, config): if not valid: return False - return tf_runner( + return terraform_runner( config, - action='destroy', + destroy=True, auto_approve=True, targets=target_modules if target_modules else None ) @@ -229,7 +235,7 @@ def handler(cls, options, config): return False # Destroy all of the infrastructure - return tf_runner(config, action='destroy', auto_approve=True) + return terraform_runner(config, destroy=True, auto_approve=True) class TerraformListTargetsCommand(CLICommand): diff --git a/streamalert_cli/terraform/helpers.py b/streamalert_cli/terraform/helpers.py index 536e3dd93..8af3cdd63 100644 --- a/streamalert_cli/terraform/helpers.py +++ b/streamalert_cli/terraform/helpers.py @@ -14,10 +14,69 @@ """ from streamalert.shared.logger import get_logger from streamalert_cli.helpers import run_command +from streamalert_cli.manage_lambda import package LOGGER = get_logger(__name__) +def terraform_runner( + config, + refresh=True, + auto_approve=False, + targets=None, + destroy=False, + build_pkg=False): + """Terraform wrapper to build StreamAlert infrastructure. + + Resolves modules with `terraform get` before continuing. + + Args: + config (CLIConfig): Loaded StreamAlert config + action (str): Terraform action ('apply' or 'destroy'). + refresh (bool): If True, Terraform will refresh its state before applying the change. + auto_approve (bool): If True, Terraform will *not* prompt the user for approval. + targets (list): Optional list of affected targets. + If not specified, Terraform will run against all of its resources. + + Returns: + bool: True if the terraform command was successful + """ + LOGGER.info('Initializing StreamAlert') + if not run_command(['terraform', 'init'], cwd=config.build_directory): + return False + + LOGGER.debug('Resolving Terraform modules') + if not run_command(['terraform', 'get'], cwd=config.build_directory, quiet=True): + return False + + tf_command = ['terraform'] + + if destroy: + tf_command.append('destroy') + # Terraform destroy has a '-force' flag instead of '-auto-approve' + LOGGER.info('Destroying infrastructure') + tf_command.append('-force={}'.format(str(auto_approve).lower())) + else: + tf_command.append('apply') + LOGGER.info('%s changes', 'Applying' if auto_approve else 'Planning') + tf_command.append('-auto-approve={}'.format(str(auto_approve).lower())) + + tf_command.append('-refresh={}'.format(str(refresh).lower())) + + if targets: + tf_command.extend('-target={}'.format(x) for x in targets) + + # Building of the deployment package should happen if Lambda is + # an explict target or there are no targets at all + if build_pkg or not targets: + deployment_package = package.LambdaPackage(config) + package_path = deployment_package.create() + if not package_path: + return False + + return run_command(tf_command, cwd=config.build_directory) + + def terraform_check(): """Verify that Terraform is configured correctly