Merge ffbdd8c into b4f5984

constraints.txt

@@ -0,0 +1,2 @@
+# botocore requires a version of docutils < 0.16, but sphinx-rtd-theme's requirement of >=0.12 breaks this
+docutils<0.16

requirements-top-level.txt

@@ -1,3 +1,4 @@
+-c constraints.txt  # remove if using pip freeze
 aliyun-python-sdk-core==2.13.5
 aliyun-python-sdk-actiontrail==2.0.0
 autoflake

requirements.txt

@@ -1,120 +1,119 @@
 aliyun-python-sdk-core==2.13.5
 aliyun-python-sdk-actiontrail==2.0.0
 autoflake==1.3.1
-autopep8==1.4.4
-backoff==1.8.1
+autopep8==1.5.3
+backoff==1.10.0
 bandit==1.6.2
-boto3==1.10.7
-boxsdk==2.6.1
-cbapi==1.5.4
-coverage==4.5.4
-coveralls==1.11.1
-google-api-python-client==1.7.11
-jmespath==0.9.4
+boto3==1.14.29
+boxsdk==2.9.0
+cbapi==1.7.1
+coverage==5.2.1
+coveralls==2.1.1
+google-api-python-client==1.10.0
+jmespath==0.10.0
 jsonlines==1.2.0
-mock==3.0.5
-moto==1.3.13
-netaddr==0.7.19
+mock==4.0.2
+moto==1.3.14
+netaddr==0.8.0
 nose==1.3.7
-nose-timer==0.7.5
+nose-timer==1.0.0
 pathlib2==2.3.5
-policyuniverse==1.3.2.1
-pyfakefs==3.6.1
+policyuniverse==1.3.2.3
+pyfakefs==4.1.0
 pylint==2.3.1
-requests==2.22.0
-Sphinx==2.2.1
-sphinx-rtd-theme==0.4.3
-yapf==0.28.0
+pymsteams==0.1.13
+requests==2.24.0
+Sphinx==3.1.2
+sphinx-rtd-theme==0.5.0
+yapf==0.30.0
 ## The following requirements were added by pip freeze:
 alabaster==0.7.12
-aliyun-python-sdk-core-v3==2.13.10
-astroid==2.3.2
-atomicwrites==1.3.0
+aliyun-python-sdk-core-v3==2.13.11
+astroid==2.4.2
 attrdict==2.0.1
 attrs==19.3.0
-aws-sam-translator==1.15.1
-aws-xray-sdk==2.4.2
-Babel==2.7.0
+aws-sam-translator==1.25.0
+aws-xray-sdk==2.6.0
+Babel==2.8.0
 boto==2.49.0
-botocore==1.13.7
-cachetools==3.1.1
-certifi==2019.9.11
-cffi==1.13.1
-cfn-lint==0.24.6
+botocore==1.17.29
+cachetools==4.1.1
+certifi==2020.6.20
+cffi==1.14.1
+cfn-lint==0.34.0
 chardet==3.0.4
-cryptography==2.8
-DateTime==4.3
-decorator==4.4.1
-docker==4.1.0
+cryptography==3.0
+decorator==4.4.2
+docker==4.2.2
 docopt==0.6.2
 docutils==0.15.2
-ecdsa==0.13.3
+ecdsa==0.15
 future==0.18.2
-gitdb2==2.0.6
-GitPython==3.0.4
-google-auth==1.6.3
-google-auth-httplib2==0.0.3
-httplib2==0.14.0
+gitdb==4.0.5
+GitPython==3.1.7
+google-api-core==1.22.0
+google-auth==1.19.2
+google-auth-httplib2==0.0.4
+googleapis-common-protos==1.52.0
+httplib2==0.18.1
 idna==2.8
-imagesize==1.1.0
-importlib-metadata==0.23
+imagesize==1.2.0
+importlib-metadata==1.7.0
 isort==4.3.21
-Jinja2==2.10.3
+Jinja2==2.11.2
 jsondiff==1.1.2
-jsonpatch==1.24
-jsonpickle==1.2
+jsonpatch==1.26
+jsonpickle==1.4.1
 jsonpointer==2.0
-jsonschema==3.1.1
+jsonschema==3.2.0
+junit-xml==1.9
 lazy-object-proxy==1.4.3
 MarkupSafe==1.1.1
 mccabe==0.6.1
-more-itertools==7.2.0
-packaging==19.2
-pbr==5.4.3
+networkx==2.4
+packaging==20.4
+pbr==5.4.5
 pika==1.1.0
-pluggy==0.13.0
-prompt-toolkit==2.0.10
-protobuf==3.10.0
-py==1.8.0
-pyasn1==0.4.7
-pyasn1-modules==0.2.7
-pycodestyle==2.5.0
-pycparser==2.19
-pyflakes==2.1.1
-Pygments==2.4.2
+prompt-toolkit==3.0.5
+protobuf==3.12.2
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+pycodestyle==2.6.0
+pycparser==2.20
+pycryptodome==3.9.8
+pyflakes==2.2.0
+Pygments==2.6.1
 PyJWT==1.7.1
-pymsteams==0.1.12
-pyparsing==2.4.2
-pyrsistent==0.15.5
-pytest==5.0.0
-python-dateutil==2.8.0
-python-jose==3.0.1
-pytz==2019.3
-PyYAML==5.1.2
+pyparsing==2.4.7
+pyrsistent==0.16.0
+python-dateutil==2.8.1
+python-jose==3.1.0
+pytz==2020.1
+PyYAML==5.3.1
 requests-toolbelt==0.9.1
-responses==0.10.6
-rsa==4.0
-s3transfer==0.2.1
-six==1.12.0
-smmap2==2.0.5
+responses==0.10.15
+rsa==4.6
+s3transfer==0.3.3
+six==1.15.0
+smmap==3.0.4
 snowballstemmer==2.0.0
 solrq==1.1.1
-sphinxcontrib-applehelp==1.0.1
-sphinxcontrib-devhelp==1.0.1
-sphinxcontrib-htmlhelp==1.0.2
+sphinxcontrib-applehelp==1.0.2
+sphinxcontrib-devhelp==1.0.2
+sphinxcontrib-htmlhelp==1.0.3
 sphinxcontrib-jsmath==1.0.1
-sphinxcontrib-qthelp==1.0.2
-sphinxcontrib-serializinghtml==1.1.3
+sphinxcontrib-qthelp==1.0.3
+sphinxcontrib-serializinghtml==1.1.4
 sshpubkeys==3.1.0
-stevedore==1.31.0
-typed-ast==1.4.0
-uritemplate==3.0.0
-urllib3==1.25.6
-validators==0.14.0
-wcwidth==0.1.7
-websocket-client==0.56.0
-Werkzeug==0.16.0
-wrapt==1.11.2
+stevedore==3.2.0
+toml==0.10.1
+typed-ast==1.4.1
+uritemplate==3.0.1
+urllib3==1.25.10
+validators==0.16.0
+wcwidth==0.2.5
+websocket-client==0.57.0
+Werkzeug==1.0.1
+wrapt==1.12.1
 xmltodict==0.12.0
-zipp==0.6.0
-zope.interface==4.6.0
+zipp==3.1.0

streamalert/shared/helpers/aws_api_client.py

@@ -44,9 +44,10 @@ def encrypt(plaintext_data, region, key_alias):
             ClientError
         """
         try:
-            key_id = 'alias/{}'.format(key_alias)
+            if not key_alias.startswith('alias/'):
+                key_alias = 'alias/{}'.format(key_alias)
             client = boto3.client('kms', config=default_config(region=region))
-            response = client.encrypt(KeyId=key_id, Plaintext=plaintext_data)
+            response = client.encrypt(KeyId=key_alias, Plaintext=plaintext_data)
             return response['CiphertextBlob']
         except ClientError:
             LOGGER.error('An error occurred during KMS encryption')

streamalert_cli/_infrastructure/modules/tf_globals/lambda_layers/README.rst

@@ -102,14 +102,14 @@ SSH and Build Dependencies
   # make sure you create virtual environment with python3.7
   $ which python3.7
 
-  # Create and source venv
-  $ python3.7 -m venv venv && source venv/bin/activate
+  # Create and use venv
+  $ mkvirtualenv --python=$(which python3.7) venv
 
   # upgrade pip and setuptools if neccessary
   $ pip install --upgrade pip setuptools
 
   $ mkdir -p $HOME/build_temp $HOME/pip_temp/python
-  $ pip install boxsdk[jwt]==2.6.1 --build $HOME/build_temp/ --target $HOME/pip_temp/python
+  $ pip install boxsdk[jwt]==2.9.0 --build $HOME/build_temp/ --target $HOME/pip_temp/python
 
   # Replace the `boxsdk[jwt]==2.6.1` below with the desired package & version
   # For example, the following would update the aliyun dependencies:

streamalert_cli/_infrastructure/modules/tf_globals/main.tf

@@ -79,7 +79,7 @@ resource "aws_lambda_layer_version" "aliyun_dependencies" {
 }
 
 resource "aws_lambda_layer_version" "box_dependencies" {
-  filename            = "${path.module}/lambda_layers/boxsdk[jwt]==2.6.1_dependencies.zip"
+  filename            = "${path.module}/lambda_layers/boxsdk[jwt]==2.9.0_dependencies.zip"
   layer_name          = "box"
   compatible_runtimes = ["python3.7"]
 }

streamalert_cli/manage_lambda/package.py

@@ -38,15 +38,15 @@ class LambdaPackage:
 
     # Define a package dict to support pinning versions across all subclasses
     REQUIRED_LIBS = {
-        'backoff==1.8.1',
-        'boto3==1.10.6',
-        'cbapi==1.5.4',
-        'google-api-python-client==1.7.11',
-        'jmespath==0.9.4',
+        'backoff==1.10.0',
+        'boto3==1.14.29',
+        'cbapi==1.7.1',
+        'google-api-python-client==1.10.0',
+        'jmespath==0.10.0',
         'jsonlines==1.2.0',
-        'netaddr==0.7.19',
-        'requests==2.22.0',
-        'pymsteams==0.1.12',
+        'netaddr==0.8.0',
+        'requests==2.24.0',
+        'pymsteams==0.1.13',
     }
 
     def __init__(self, config):

tests/scripts/update_reqs.sh

@@ -1,5 +1,5 @@
 #! /bin/bash
-pip install -r requirements-top-level.txt --upgrade --force-reinstall
+pip install -r requirements-top-level.txt --upgrade --force-reinstall --no-cache-dir
 pip freeze -r requirements-top-level.txt > requirements.txt
 
 echo "Please also update library versions in streamalert_cli/manage_lambda/package.py"

tests/unit/streamalert/alert_processor/helpers.py

@@ -15,6 +15,8 @@
 """
 import random
 
+import boto3
+
 from streamalert.alert_processor.outputs.credentials.provider import LocalFileDriver
 from streamalert.shared.alert import Alert
 from streamalert.shared.helpers.aws_api_client import AwsKms, AwsSsm
@@ -81,6 +83,15 @@ def remove_temp_secrets():
     LocalFileDriver.clear()
 
 
+def setup_mock_kms(region, alias):
+    client = boto3.client('kms', region_name=region)
+    response = client.create_key()
+    client.create_alias(
+        AliasName=alias,
+        TargetKeyId=response['KeyMetadata']['KeyId']
+    )
+
+
 def encrypt_with_kms(data, region, alias):
     """Encrypt the given data with KMS."""
     return AwsKms.encrypt(data, region=region, key_alias=alias)