Merge a2c9dda into 50aaea9

lambda_functions/analyzer/README.rst

@@ -10,9 +10,10 @@ Many libraries used by BinaryAlert are natively compiled, and must therefore be
 Amazon Linux AMI in order to run in Lambda. This has already been done for you:
 ``dependencies.zip`` contains the following pre-built libraries:
 
+- `cryptography <https://cryptography.io>`_ (v2.3)
 - `UPX <https://github.com/upx/upx>`_ (v3.94)
-- `yara-python <https://github.com/VirusTotal/yara-python>`_ (v3.7.0)
-    - `yara <https://github.com/VirusTotal/yara>`_ (v3.7.1)
+- `yara-python <https://github.com/VirusTotal/yara-python>`_ (v3.8.0)
+    - `yara <https://github.com/VirusTotal/yara>`_ (v3.8.0)
 - `yextend <https://github.com/BayshoreNetworks/yextend>`_ (v1.6)
     - `pdftotext <https://poppler.freedesktop.org/>`_ (v0.26.5)
 
@@ -24,37 +25,49 @@ and install the dependencies as follows:
 
     # Install requirements
     sudo yum update
-    sudo yum install autoconf automake bzip2-devel gcc64 gcc64-c++ libarchive-devel libtool \
-        libuuid-devel openssl-devel pcre-devel poppler-utils python36 python36-devel zlib-devel
+    sudo yum install autoconf automake bzip2-devel gcc64 gcc64-c++ libarchive-devel libffi-devel \
+        libtool libuuid-devel openssl-devel pcre-devel poppler-utils python36 python36-devel zlib-devel
     sudo pip install nose
 
     # Compile YARA
-    wget https://github.com/VirusTotal/yara/archive/v3.7.1.tar.gz
-    tar -xzf v3.7.1.tar.gz
-    cd yara-3.7.1
+    wget https://github.com/VirusTotal/yara/archive/v3.8.0.tar.gz
+    tar -xzf v3.8.0.tar.gz
+    cd yara-3.8.0
     ./bootstrap.sh
     ./configure
     make
     make check  # Run unit tests
     sudo make install
 
-    # Install yara-python
+    # Install cryptography and yara-python
     cd ~
     mkdir pip
-    pip-3.6 install yara-python -t pip
+    pip-3.6 install cryptography yara-python -t pip
 
     # Compile yextend
     wget https://github.com/BayshoreNetworks/yextend/archive/1.6.tar.gz
     tar -xvzf 1.6.tar.gz
     cd yextend-1.6
-    # Manually: modify main.cpp, line 473 to hardcode the yara version to 3.7
+    # Manually: modify main.cpp, line 473 to hardcode the yara version to 3.8
     ./build.sh
     make unittests  # Run unit tests
 
-    # Gather YARA files
+    # Clean cryptography files
+    cd ~/pip
+    rm -r *.dist-info *.egg-info
+    find . -name __pycache__ | xargs rm -r
+    mv _cffi_backend.cpython-36m-x86_64-linux-gnu.so _cffi_backend.so
+    cd cryptography/hazmat/bindings
+    mv _constant_time.abi3.so _constant_time.so
+    mv _openssl.abi3.so _openssl.so
+    mv _padding.abi3.so _padding.so
+
+    # Gather pip files
     cd ~
     mkdir lambda
-    cp pip/yara.cpython-36m-x86_64-linux-gnu.so lambda/yara.so
+    cp pip/.libs_cffi_backend/* lambda
+    cp -r pip/* lambda
+    mv lambda/yara.cpython-36m-x86_64-linux-gnu.so lambda/yara.so
     wget https://raw.githubusercontent.com/VirusTotal/yara/master/COPYING -O lambda/YARA_LICENSE
     wget https://raw.githubusercontent.com/VirusTotal/yara-python/master/LICENSE -O lambda/YARA_PYTHON_LICENSE
 
@@ -95,4 +108,4 @@ and install the dependencies as follows:
     zip -r dependencies.zip *
 
 
-Then ``scp`` the new zipfile to replace the one in the repo.
+Then ``scp`` the ``dependencies.zip`` package to replace the one in the repo.

lambda_functions/analyzer/analyzer_aws_lib.py

@@ -138,7 +138,7 @@ def put_metric_data(num_yara_rules: int, binaries: List[BinaryInfo]) -> None:
     CLOUDWATCH.put_metric_data(Namespace='BinaryAlert', MetricData=metric_data)
 
 
-class DynamoMatchTable(object):
+class DynamoMatchTable:
     """Saves YARA match information into a Dynamo table.
 
     The table uses a composite key:
@@ -193,8 +193,8 @@ def _most_recent_item(self, sha: str) -> Optional[Tuple[int, Set[str], Set[str],
             if len(most_recent_items) >= 2:
                 previous_s3_objects = set(most_recent_items[1]['S3Objects'])
             return analyzer_version, matched_rules, s3_objects, previous_s3_objects
-        else:
-            return None
+
+        return None
 
     @staticmethod
     def _replace_empty_strings(data: Dict[str, str]) -> Dict[str, str]:

lambda_functions/analyzer/binary_info.py

@@ -19,7 +19,7 @@
     from yara_analyzer import YaraAnalyzer, YaraMatch  # type: ignore
 
 
-class BinaryInfo(object):
+class BinaryInfo:
     """Organizes the analysis of a single binary blob in S3."""
 
     def __init__(self, bucket_name: str, object_key: str, yara_analyzer: YaraAnalyzer) -> None:

lambda_functions/analyzer/yara_analyzer.py

@@ -62,7 +62,7 @@ def _convert_yextend_to_yara_match(yextend_json: Dict[str, Any]) -> List[YaraMat
     return matches
 
 
-class YaraAnalyzer(object):
+class YaraAnalyzer:
     """Encapsulates YARA analysis and matching functions."""
 
     def __init__(self, compiled_rules_file: str) -> None:

lambda_functions/batcher/main.py

@@ -23,7 +23,7 @@
 SQS_MAX_MESSAGES_PER_BATCH = 10
 
 
-class SQSMessage(object):
+class SQSMessage:
     """Encapsulates a single SQS message (which will contain multiple S3 keys)."""
 
     def __init__(self, msg_id: int) -> None:
@@ -70,7 +70,7 @@ def reset(self) -> None:
         self._keys = []
 
 
-class SQSBatcher(object):
+class SQSBatcher:
     """Collect groups of S3 keys and batch them into as few SQS requests as possible."""
 
     def __init__(self, queue_url: str, objects_per_message: int) -> None:
@@ -142,7 +142,7 @@ def finalize(self) -> None:
             self._send_batch()
 
 
-class S3BucketEnumerator(object):
+class S3BucketEnumerator:
     """Enumerates all of the S3 objects in a given bucket."""
 
     def __init__(self, bucket_name: str, prefix: Optional[str],
@@ -164,7 +164,7 @@ def __init__(self, bucket_name: str, prefix: Optional[str],
         self.finished = False  # Have we finished enumerating all of the S3 bucket?
 
     @property
-    def continuation_token(self) -> str:
+    def continuation_token(self) -> Optional[str]:
         return self.kwargs.get('ContinuationToken')
 
     def next_page(self) -> List[str]:

lambda_functions/downloader/main.py

@@ -107,7 +107,7 @@ def _process_md5(md5: str) -> bool:
         binary = CARBON_BLACK.select(Binary, md5)
         download_path = _download_from_carbon_black(binary)
         metadata = _build_metadata(binary)
-        _upload_to_s3(binary.md5, download_path, metadata)
+        _upload_to_s3(binary.md5, download_path, metadata)  # pylint: disable=no-member
         return True
     except (BotoCoreError, ObjectNotFoundError, ServerError, zipfile.BadZipFile):
         LOGGER.exception('Error downloading %s', md5)

manage.py

@@ -65,7 +65,7 @@ def _get_input(prompt: str, default_value: str) -> str:
     return input(prompt).strip().lower() or default_value
 
 
-class BinaryAlertConfig(object):
+class BinaryAlertConfig:
     """Wrapper around reading, validating, and updating the terraform.tfvars config file."""
     # Expected configuration value formats.
     VALID_AWS_ACCOUNT_ID_FORMAT = r'\d{12}'
@@ -349,7 +349,7 @@ def save(self) -> None:
             config_file.write(raw_config)
 
 
-class Manager(object):
+class Manager:
     """BinaryAlert management utility."""
 
     def __init__(self) -> None:

requirements.txt

@@ -1,29 +1,29 @@
-alabaster==0.7.10
+alabaster==0.7.11
 asn1crypto==0.24.0
-astroid==1.6.3
+astroid==2.0.2
 attrdict==2.0.0
 aws-xray-sdk==0.95
-Babel==2.5.3
+Babel==2.6.0
 bandit==1.4.0
-boto==2.48.0
-boto3==1.7.10
-botocore==1.10.10
-cachetools==2.0.1
+boto==2.49.0
+boto3==1.7.71
+botocore==1.10.71
+cachetools==2.1.0
 cbapi==1.3.6
 certifi==2018.4.16
 cffi==1.11.5
 chardet==3.0.4
 cookies==2.2.1
 coverage==4.5.1
 coveralls==1.3.0
-cryptography==2.2.2
-docker==3.3.0
-docker-pycreds==0.2.3
+cryptography==2.3
+docker==3.4.1
+docker-pycreds==0.3.0
 docopt==0.6.2
 docutils==0.14
-gitdb2==2.0.3
-GitPython==2.1.9
-idna==2.6
+gitdb2==2.0.4
+GitPython==2.1.11
+idna==2.7
 imagesize==1.0.0
 isort==4.3.4
 Jinja2==2.10
@@ -35,38 +35,40 @@ MarkupSafe==1.0
 mccabe==0.6.1
 mock==2.0.0
 moto==1.3.3
-mypy==0.590
+mypy==0.620
 packaging==17.1
-pbr==4.0.2
-pika==0.11.2
+pbr==4.2.0
+pika==0.12.0
 ply==3.10
-prompt-toolkit==1.0.15
-protobuf==3.5.2.post1
+prompt-toolkit==2.0.4
+protobuf==3.6.0
 pyaml==17.12.1
 pycparser==2.18
-pyfakefs==3.4.1
+pyfakefs==3.4.3
 Pygments==2.2.0
 pyhcl==0.3.10
-pylint==1.8.4
+pylint==2.1.1
+pyOpenSSL==18.0.0
 pyparsing==2.2.0
 python-dateutil==2.6.1
-pytz==2018.4
+pytz==2018.5
 PyYAML==3.13
-requests==2.18.4
+requests==2.19.1
 responses==0.9.0
 s3transfer==0.1.13
 six==1.11.0
-smmap2==2.0.3
+smmap2==2.0.4
 snowballstemmer==1.2.1
-Sphinx==1.7.4
-sphinx-rtd-theme==0.3.0
-sphinxcontrib-websupport==1.0.1
-stevedore==1.28.0
+Sphinx==1.7.6
+sphinx-rtd-theme==0.4.1
+sphinxcontrib-websupport==1.1.0
+stevedore==1.29.0
 typed-ast==1.1.0
-urllib3==1.22
+typing==3.6.4
+urllib3==1.23
 wcwidth==0.1.7
-websocket-client==0.47.0
+websocket-client==0.48.0
 Werkzeug==0.14.1
 wrapt==1.10.11
 xmltodict==0.11.0
-yara-python==3.7.0
+yara-python==3.8.0

requirements_top_level.txt

@@ -11,4 +11,4 @@ pylint
 python-dateutil==2.6.1
 sphinx
 sphinx-rtd-theme
-yara-python==3.7.0
+yara-python==3.8.0

tests/common.py

@@ -1,7 +1,7 @@
 """Utilities common to several different unit tests."""
 
 
-class MockLambdaContext(object):
+class MockLambdaContext():
     """http://docs.aws.amazon.com/lambda/latest/dg/python-context-object.html"""
     def __init__(self, function_version: int = 1, time_limit_ms: int = 30000,
                  decrement_ms: int = 10000) -> None:

tests/lambda_functions/analyzer/main_test.py

@@ -31,7 +31,7 @@
 TEST_CONTEXT = common.MockLambdaContext(LAMBDA_VERSION)
 
 
-class MockS3Object(object):
+class MockS3Object:
     """Simple mock for boto3.resource('s3').Object"""
     def __init__(self, bucket_name, object_key):
         self.name = bucket_name

tests/lambda_functions/analyzer/yara_mocks.py

@@ -27,7 +27,7 @@
 """
 
 
-class YaraRulesMock(object):
+class YaraRulesMock:
     """A wrapper around Yara.Rules which redirects .match() to open files with Python's open()."""
 
     def __init__(self, yara_rules_object):

tests/lambda_functions/batcher/main_test.py

@@ -207,7 +207,7 @@ def mock_list(**kwargs):
 
     def test_batcher_re_invoke(self):
         """If the batcher runs out of time, it has to re-invoke itself."""
-        class MockEnumerator(object):
+        class MockEnumerator:
             """Simple mock for S3BucketEnumerator which never finishes."""
             def __init__(self, *args):  # pylint: disable=unused-argument
                 self.continuation_token = 'test-continuation-token'

tests/lambda_functions/build_test.py

@@ -56,6 +56,7 @@ def test_build_analyzer(self, mock_print: mock.MagicMock):
                 'compiled_yara_rules.bin',
 
                 # Natively compiled binaries
+                'cryptography/',
                 'libarchive.so.13',
                 'libs/',
                 'libs/bayshore_file_type_detect.o',
@@ -89,7 +90,8 @@ def test_build_analyzer(self, mock_print: mock.MagicMock):
                 'YARA_LICENSE',
                 'YARA_PYTHON_LICENSE',
                 'YEXTEND_LICENSE'
-            }
+            },
+            subset=True
         )
         mock_print.assert_called_once()
 

tests/lambda_functions/downloader/main_test.py

@@ -10,10 +10,10 @@
 from pyfakefs import fake_filesystem_unittest
 
 
-class MockBinary(object):
+class MockBinary:
     """Mock for cbapi.response.models.Binary."""
 
-    class MockVirusTotal(object):
+    class MockVirusTotal:
         """Mock for cbapi.response.models.VirusTotal."""
 
         def __init__(self, score: int = 0) -> None:

tests/manage_test.py

@@ -19,17 +19,17 @@ def _mock_input(prompt: str) -> str:
     # pylint: disable=too-many-return-statements
     if prompt.startswith('AWS Account'):
         return '111122223333'
-    elif prompt.startswith('AWS Region'):
+    if prompt.startswith('AWS Region'):
         return 'us-west-2'
-    elif prompt.startswith('Unique name prefix'):
+    if prompt.startswith('Unique name prefix'):
         return ' NEW_NAME_PREFIX '  # Spaces and case shouldn't matter.
-    elif prompt.startswith('Enable the CarbonBlack downloader'):
+    if prompt.startswith('Enable the CarbonBlack downloader'):
         return 'yes'
-    elif prompt.startswith('CarbonBlack URL'):
+    if prompt.startswith('CarbonBlack URL'):
         return 'https://new-example.com'
-    elif prompt.startswith('Change the CarbonBlack API token'):
+    if prompt.startswith('Change the CarbonBlack API token'):
         return 'yes'
-    elif prompt.startswith('Delete all S3 objects'):
+    if prompt.startswith('Delete all S3 objects'):
         return 'yes'
     return 'UNKNOWN'