diff --git a/streamalert/_vendored/boxsdk[jwt]==2.6.1_dependencies.zip b/streamalert/_vendored/boxsdk[jwt]==2.6.1_dependencies.zip deleted file mode 100644 index 64c717559..000000000 Binary files a/streamalert/_vendored/boxsdk[jwt]==2.6.1_dependencies.zip and /dev/null differ diff --git a/streamalert/_vendored/README.rst b/streamalert_cli/_infrastructure/modules/tf_globals/lambda_layers/README.rst similarity index 87% rename from streamalert/_vendored/README.rst rename to streamalert_cli/_infrastructure/modules/tf_globals/lambda_layers/README.rst index 97933b85e..bcc987b7f 100644 --- a/streamalert/_vendored/README.rst +++ b/streamalert_cli/_infrastructure/modules/tf_globals/lambda_layers/README.rst @@ -1,6 +1,10 @@ How to Update Precompiled Dependencies ###################################### +For dependencies included in zips to be usable with Lambda Layers, files must reside within a ``python`` directory. See the +AWS `documentation `_ +for more information. + Building Dependencies Using EC2 =============================== @@ -36,10 +40,10 @@ On EC2 Instance $ pip install --upgrade pip setuptools # Make a temp build directory and temp pip install directory - $ mkdir $HOME/build_temp $HOME/pip_temp + $ mkdir -p $HOME/build_temp $HOME/pip_temp/python # Install all of the dependencies to this directory - $ pip install boxsdk[jwt]==2.6.1 --build $HOME/build_temp/ --target $HOME/pip_temp + $ pip install boxsdk[jwt]==2.6.1 --build $HOME/build_temp/ --target $HOME/pip_temp/python # Replace the `boxsdk[jwt]==2.6.1` below with the desired package & version # For example, the following would update the aliyun dependencies: @@ -104,8 +108,8 @@ SSH and Build Dependencies # upgrade pip and setuptools if neccessary $ pip install --upgrade pip setuptools - $ mkdir $HOME/build_temp $HOME/pip_temp - $ pip install boxsdk[jwt]==2.6.1 --build $HOME/build_temp/ --target $HOME/pip_temp + $ mkdir -p $HOME/build_temp $HOME/pip_temp/python + $ pip install boxsdk[jwt]==2.6.1 --build $HOME/build_temp/ --target $HOME/pip_temp/python # Replace the `boxsdk[jwt]==2.6.1` below with the desired package & version # For example, the following would update the aliyun dependencies: @@ -126,7 +130,7 @@ Copy the `pip.zip` file from the virtual machine to the local host. .. code-block:: bash - $ cp pip.zip /vagrant/streamalert/_vendored/boxsdk[jwt]==2.6.1_dependencies.zip + $ cp pip.zip /vagrant/streamalert_cli/_infrastructure/modules/tf_globals/lambda_layers/boxsdk[jwt]==2.6.1_dependencies.zip $ exit # exit the session diff --git a/streamalert/_vendored/aliyun-python-sdk-actiontrail==2.0.0_dependencies.zip b/streamalert_cli/_infrastructure/modules/tf_globals/lambda_layers/aliyun-python-sdk-actiontrail==2.0.0_dependencies.zip similarity index 58% rename from streamalert/_vendored/aliyun-python-sdk-actiontrail==2.0.0_dependencies.zip rename to streamalert_cli/_infrastructure/modules/tf_globals/lambda_layers/aliyun-python-sdk-actiontrail==2.0.0_dependencies.zip index 228f6c7c2..4df5cf412 100644 Binary files a/streamalert/_vendored/aliyun-python-sdk-actiontrail==2.0.0_dependencies.zip and b/streamalert_cli/_infrastructure/modules/tf_globals/lambda_layers/aliyun-python-sdk-actiontrail==2.0.0_dependencies.zip differ diff --git a/streamalert_cli/_infrastructure/modules/tf_globals/lambda_layers/boxsdk[jwt]==2.6.1_dependencies.zip b/streamalert_cli/_infrastructure/modules/tf_globals/lambda_layers/boxsdk[jwt]==2.6.1_dependencies.zip new file mode 100644 index 000000000..6bc44b940 Binary files /dev/null and b/streamalert_cli/_infrastructure/modules/tf_globals/lambda_layers/boxsdk[jwt]==2.6.1_dependencies.zip differ diff --git a/streamalert_cli/_infrastructure/modules/tf_globals/main.tf b/streamalert_cli/_infrastructure/modules/tf_globals/main.tf index 300eec537..5d84e04be 100644 --- a/streamalert_cli/_infrastructure/modules/tf_globals/main.tf +++ b/streamalert_cli/_infrastructure/modules/tf_globals/main.tf @@ -71,3 +71,15 @@ resource "aws_dynamodb_table" "rules_table" { Name = "StreamAlert" } } + +resource "aws_lambda_layer_version" "aliyun_dependencies" { + filename = "${path.module}/lambda_layers/aliyun-python-sdk-actiontrail==2.0.0_dependencies.zip" + layer_name = "aliyun" + compatible_runtimes = ["python3.7"] +} + +resource "aws_lambda_layer_version" "box_dependencies" { + filename = "${path.module}/lambda_layers/boxsdk[jwt]==2.6.1_dependencies.zip" + layer_name = "box" + compatible_runtimes = ["python3.7"] +} diff --git a/streamalert_cli/_infrastructure/modules/tf_globals/output.tf b/streamalert_cli/_infrastructure/modules/tf_globals/output.tf index 16e370da5..3653901a6 100644 --- a/streamalert_cli/_infrastructure/modules/tf_globals/output.tf +++ b/streamalert_cli/_infrastructure/modules/tf_globals/output.tf @@ -13,3 +13,10 @@ output "classifier_sqs_queue_arn" { output "classifier_sqs_sse_kms_key_arn" { value = module.classifier_queue.sqs_sse_kms_key_arn } + +output "lamdba_layer_arns" { + value = [ + aws_lambda_layer_version.aliyun_dependencies.arn, + aws_lambda_layer_version.box_dependencies.arn, + ] +} diff --git a/streamalert_cli/_infrastructure/modules/tf_lambda/main.tf b/streamalert_cli/_infrastructure/modules/tf_lambda/main.tf index 658ec12cb..28fbaf51e 100644 --- a/streamalert_cli/_infrastructure/modules/tf_lambda/main.tf +++ b/streamalert_cli/_infrastructure/modules/tf_lambda/main.tf @@ -34,6 +34,8 @@ resource "aws_lambda_function" "function" { security_group_ids = var.vpc_security_group_ids } + layers = var.layers + tags = local.tags } diff --git a/streamalert_cli/_infrastructure/modules/tf_lambda/variables.tf b/streamalert_cli/_infrastructure/modules/tf_lambda/variables.tf index 8f72a4072..a24e1c81a 100644 --- a/streamalert_cli/_infrastructure/modules/tf_lambda/variables.tf +++ b/streamalert_cli/_infrastructure/modules/tf_lambda/variables.tf @@ -12,6 +12,12 @@ variable "runtime" { description = "Function runtime environment" } +variable "layers" { + type = list(string) + default = [] + description = "List of Lambda Layer ARNs to use with this function" +} + variable "handler" { description = "Entry point for the function" } diff --git a/streamalert_cli/manage_lambda/package.py b/streamalert_cli/manage_lambda/package.py index c0651e197..16d56bb61 100644 --- a/streamalert_cli/manage_lambda/package.py +++ b/streamalert_cli/manage_lambda/package.py @@ -16,7 +16,6 @@ import os import shutil import tempfile -import zipfile from streamalert.shared.logger import get_logger from streamalert_cli.helpers import run_command @@ -54,11 +53,6 @@ class LambdaPackage: 'pymsteams==0.1.12', } - PRECOMPILED_LIBS = { # Default precompiled dependencies - 'aliyun-python-sdk-actiontrail==2.0.0', - 'boxsdk[jwt]==2.6.1', - } - def __init__(self, config): self.config = config self.temp_package_path = os.path.join(tempfile.gettempdir(), self.package_name) @@ -78,7 +72,7 @@ def create(self): shutil.rmtree(self.temp_package_path) # Copy all of the default package files - self._copy_files(self.DEFAULT_PACKAGE_FILES, ignores={'*dependencies.zip'}) + self._copy_files(self.DEFAULT_PACKAGE_FILES) # Copy in any user-specified files self._copy_user_config_files() @@ -87,11 +81,6 @@ def create(self): LOGGER.error('Failed to install necessary libraries') return False - # Extract any precompiled libs for this package - if not self._extract_precompiled_libs(): - LOGGER.error('Failed to extract precompiled libraries') - return False - # Zip it all up # Build these in the top-level of the terraform directory as streamalert.zip result = shutil.make_archive( @@ -119,41 +108,6 @@ def _copy_files(self, paths, ignores=None): kwargs = {'ignore': shutil.ignore_patterns(*ignores)} if ignores else dict() shutil.copytree(path, os.path.join(self.temp_package_path, path), **kwargs) - def _extract_precompiled_libs(self): - """Extract any precompiled libraries into the deployment package folder - - Returns: - bool: True if precompiled libs were extracted successfully, False if some are missing - """ - vendored_dir = os.path.join('streamalert', '_vendored') - found_libs = set() - for zipped_file in os.listdir(vendored_dir): - # Skip files that are not explicitly deps - if not zipped_file.endswith('_dependencies.zip'): - continue - - value = zipped_file.rstrip('_dependencies.zip') - if value not in self.PRECOMPILED_LIBS: - LOGGER.error( - 'Found precompiled libraries not included in PRECOMPILED_LIBS: %s', value - ) - return False - - LOGGER.info('Extracting precompiled library: %s', zipped_file) - - # Copy the contents of the dependency zip to the package directory - with zipfile.ZipFile(os.path.join(vendored_dir, zipped_file), 'r') as libs_file: - libs_file.extractall(self.temp_package_path) - - found_libs.add(value) - - diff = self.PRECOMPILED_LIBS.difference(found_libs) - if diff: - LOGGER.error('Missing required precompiled libraries: %s', ', '.join(diff)) - return False - - return True - def _resolve_libraries(self): """Install all libraries into the deployment package folder diff --git a/streamalert_cli/terraform/apps.py b/streamalert_cli/terraform/apps.py index 1c292e866..4b8736c8e 100644 --- a/streamalert_cli/terraform/apps.py +++ b/streamalert_cli/terraform/apps.py @@ -59,5 +59,6 @@ def generate_apps(cluster_name, cluster_dict, config): 'streamalert.apps.main.handler', config['clusters'][cluster_name]['modules']['streamalert_apps'][function_name], config, - input_event=app_config + input_event=app_config, + include_layers=True, ) diff --git a/streamalert_cli/terraform/lambda_module.py b/streamalert_cli/terraform/lambda_module.py index 63d98cbcd..9c074307f 100644 --- a/streamalert_cli/terraform/lambda_module.py +++ b/streamalert_cli/terraform/lambda_module.py @@ -52,7 +52,7 @@ def _tf_vpc_config(lambda_config): def generate_lambda(function_name, handler, lambda_config, config, environment=None, - input_event=None, tags=None, zip_file=None): + input_event=None, tags=None, **kwargs): """Generate an instance of the Lambda Terraform module. Args: @@ -63,6 +63,9 @@ def generate_lambda(function_name, handler, lambda_config, config, environment=N environment (dict): Optional environment variables to specify. ENABLE_METRICS and LOGGER_LEVEL are included automatically. tags (dict): Optional tags to be added to this Lambda resource. + + Keyword Args: + include_layers (bool): Optionally include the default Lambda Layers (default: False) zip_file (str): Optional name for the .zip of deployment package (default: streamalert.zip) Example Lambda config: @@ -121,9 +124,12 @@ def generate_lambda(function_name, handler, lambda_config, config, environment=N 'tags': tags or {}, } + if kwargs.get('include_layers', False): + lambda_module['layers'] = '${module.globals.lamdba_layer_arns}' + # The lambda module defaults to using the 'streamalert.zip' file that is created - if zip_file: - lambda_module['filename'] = zip_file + if kwargs.get('zip_file'): + lambda_module['filename'] = kwargs.get('zip_file') # Add Classifier input config from the loaded cluster file input_config = lambda_config.get('inputs')