Upgrade requirements and include yextend (#79)

docs/source/getting-started.rst

@@ -15,12 +15,12 @@ BinaryAlert can be deployed from any MacOS/Linux environment (and likely Windows
   $ sudo apt-get install python3.6  # Ubuntu 16+
   $ python3 --version  # Should show 3.6.x
 
-2. Install the latest version of `Terraform <https://www.terraform.io/downloads.html>`_:
+2. Install `Terraform <https://www.terraform.io/downloads.html>`_ v0.11.X:
 
 .. code-block:: bash
 
   $ brew install terraform  # MacOS Homebrew
-  $ terraform --version  # Must be v0.10.4+
+  $ terraform --version  # Must be v0.11.X
 
 3. Install `virtualenv <https://virtualenv.pypa.io/en/stable/installation>`_:
 

lambda_functions/analyzer/README.rst

@@ -4,38 +4,61 @@ This Lambda function is the core of BinaryAlert. Each invocation downloads one o
 S3, scans them against all available YARA rules, and forwards any matches to Dynamo and SNS.
 
 
-Updating YARA-Python
---------------------
-The `yara-python <https://github.com/VirusTotal/yara-python>`_ library is natively compiled.
-It must therefore be pre-built on an Amazon Linux AMI in order to run in Lambda.
-This has already been done for you: ``yara_python_3.6.3.zip`` contains the
-pre-built ``yara_python`` library (v3.6.3) for the Lambda environment and is automatically bundled
-on every deploy.
+Updating YARA Binaries
+----------------------
+The YARA libaries used by BinaryAlert are natively compiled, and must therefore be pre-built on an
+Amazon Linux AMI in order to run in Lambda. This has already been done for you:
+``yara3.7.0_yextend1.5.zip`` contains the pre-built
+`yara_python <https://github.com/VirusTotal/yara-python>`_ (v3.7.0) and
+`yextend <https://github.com/BayshoreNetworks/yextend>`_ (v1.5) libraries for the Lambda environment
+and is automatically bundled with every deploy.
 
 If, however, you need to update or re-create the zipfile, SSH to an EC2 instance running the
 `AWS Lambda AMI <http://docs.aws.amazon.com/lambda/latest/dg/current-supported-versions.html>`_
-and install ``yara-python``:
+and install ``yara-python`` and ``yextend`` as follows:
 
 .. code-block:: bash
 
-    $ sudo su
-    # yum update
-    # yum install gcc openssl-devel.x86_64 python35-devel.x86_64 python35-pip.noarch
-    # python3
-        >>> import pip
-        >>> pip.main(['install', '--upgrade', 'pip'])
-        >>> exit()
-    # python3
-        >>> import pip
-        >>> pip.main(['install', 'yara-python', '--target', '.'])
-        >>> exit()
-    # mv yara.cpython-35m-x86_64-linux-gnu.so yara.so
-    # cp /usr/lib64/libpython3.5m.so.1.0 .
-    # zip -r yara_python_VERSION.zip *
-
-Then replace ``yara_python_3.6.3.zip`` in the repo.
-
-Some notes:
-
-- Python3.6 is not currently available in the public Lambda AMI. You can either manually install Python3.6 from source or (what's done here) include the required Python3.5 bytecode in the zipfile.
-- The openssl development libraries are required to support the "hash" module.
+    # Install requirements
+    sudo yum update
+    sudo yum install autconf automake gcc gcc-c++ libarchive-devel libtool libuuid-devel \
+        openssl-devel pcre-devel poppler-utils python36 python36-devel zlib-devel
+    sudo pip install nose
+
+    # Install YARA
+    wget https://github.com/VirusTotal/yara/archive/v3.7.0.tar.gz
+    tar -xvzf v3.7.0.tar.gz
+    cd yara-3.7.0
+    ./bootstrap.sh
+    ./configure
+    make
+    make check  # Run unit tests
+    sudo make install
+
+    # Install yara-python
+    cd ~
+    mkdir pip
+    pip-3.6 install yara-python -t pip
+
+    # Compile yextend
+    cd ~
+    wget https://github.com/BayshoreNetworks/yextend/archive/1.5.tar.gz
+    tar -xvzf 1.5.tar.gz
+    cd yextend-1.5
+    # Modify main.cpp, line 177 to hardcode the yara version to 3.7
+    ./build.sh
+    make unittests  # Run unit tests
+
+    # Gather binaries and build zipfile
+    cd ~
+    mkdir lambda
+    cp pip/yara.cpython-36m-x86_64-linux-gnu.so lambda/yara.so
+    cp yextend-1.5/yextend lambda
+    cp /usr/lib64/libarchive.so.13 lambda
+    cp /usr/lib64/liblzo2.so.2 lambda
+    cp /usr/local/lib/libyara.so.3 lambda
+    cd lambda
+    zip yara3.7.0_yextend1.5.zip *
+
+
+Then ``scp`` the newzipfile to replace the one in the repo.

lambda_functions/build.py

@@ -13,7 +13,7 @@
 LAMBDA_DIR = os.path.dirname(os.path.realpath(__file__))
 
 ANALYZE_SOURCE = os.path.join(LAMBDA_DIR, 'analyzer')
-ANALYZE_DEPENDENCIES = os.path.join(ANALYZE_SOURCE, 'yara_python_3.6.3.zip')
+ANALYZE_DEPENDENCIES = os.path.join(ANALYZE_SOURCE, 'yara3.7.0_yextend1.5.zip')
 ANALYZE_ZIPFILE = 'lambda_analyzer'
 
 BATCH_SOURCE = os.path.join(LAMBDA_DIR, 'batcher', 'main.py')
@@ -23,7 +23,7 @@
 DISPATCH_ZIPFILE = 'lambda_dispatcher'
 
 DOWNLOAD_SOURCE = os.path.join(LAMBDA_DIR, 'downloader', 'main.py')
-DOWNLOAD_DEPENDENCIES = os.path.join(LAMBDA_DIR, 'downloader', 'cbapi_1.3.2.zip')
+DOWNLOAD_DEPENDENCIES = os.path.join(LAMBDA_DIR, 'downloader', 'cbapi_1.3.4.zip')
 DOWNLOAD_ZIPFILE = 'lambda_downloader'
 
 

lambda_functions/downloader/README.rst

@@ -7,5 +7,21 @@ For more information, see the `documentation <https://binaryalert.io/uploading-f
 
 Cbapi Pip Dependency
 --------------------
-The ``cbapi`` library works best when pre-built on the Lambda AMI. Follow the same instructions given
-in the `analyzer README <../analyzer/README.rst>`_ to upgrade ``cbapi_1.3.2.zip`` when needed.
+The ``cbapi`` library needs to be pre-built on the AWS Lambda AMI. ``cbapi_1.3.4.zip`` is already
+included in the repo for you, but if you need to upgrade it or rebuild it, SSH to an EC2 instance
+running the
+`AWS Lambda AMI <http://docs.aws.amazon.com/lambda/latest/dg/current-supported-versions.html>`_
+and install ``cbapi`` as follows:
+
+.. code-block:: bash
+
+    # Install requirements
+    sudo yum update
+    sudo yum install gcc python36
+
+    # Install cbapi and build the zipfile
+    pip-3.6 install cbapi -t ~
+    rm -r *dist-info *egg-info  # Remove unnecessary package information
+    zip -r cbapi.zip *
+
+Then you need only ``scp`` the new zip file to replace ``cbapi_1.3.4.zip``

lambda_functions/downloader/copy_all.py

@@ -107,7 +107,7 @@ def copy_all_binaries() -> None:
 
     # Create process communication queues.
     tasks = JoinableQueue(MAX_TASK_QUEUE_SIZE)  # CopyTasks to execute.
-    failures = Queue()  # A list of MD5s which failed to copy.
+    failures: Queue = Queue()  # A list of MD5s which failed to copy.
 
     # Start the consumer processes.
     LOGGER.info('Start %d consumers', NUM_CONSUMERS)

manage.py

@@ -188,6 +188,7 @@ def _encrypt_cb_api_token(self) -> None:
         # The same key will be used by the downloader to decrypt the API token at runtime.
         print('Terraforming KMS key...')
         os.chdir(TERRAFORM_DIR)
+        subprocess.check_call(['terraform', 'get'])
         subprocess.check_call(
             ['terraform', 'apply', '-target={}'.format(CB_KMS_ALIAS_TERRAFORM_ID)]
         )

requirements.txt

@@ -1,52 +1,48 @@
 alabaster==0.7.10
-asn1crypto==0.22.0
 astroid==1.5.3
 attrdict==2.0.0
-Babel==2.5.0
+Babel==2.5.1
 backoff==1.4.3
 boto3==1.4.7
-botocore==1.7.9
+botocore==1.7.47
 cachetools==2.0.1
-cbapi==1.3.2
-certifi==2017.7.27.1
-cffi==1.10.0
+cbapi==1.3.4
+certifi==2017.11.5
 chardet==3.0.4
-coverage==4.4.1
+coverage==4.4.2
 coveralls==1.2.0
-cryptography==2.0.3
 docopt==0.6.2
 docutils==0.14
 idna==2.6
 imagesize==0.7.1
 isort==4.2.15
-Jinja2==2.9.6
+Jinja2==2.10
 jmespath==0.9.3
 lazy-object-proxy==1.3.1
 MarkupSafe==1.0
 mccabe==0.6.1
-mypy==0.521
+mypy==0.550
 pika==0.11.0
-ply==3.9
+ply==3.10
 prompt-toolkit==1.0.15
-protobuf==3.4.0
-pycparser==2.18
-pyfakefs==3.2
+protobuf==3.5.0
+psutil==5.4.1
+pyfakefs==3.3
 Pygments==2.2.0
-pyhcl==0.3.8
-pylint==1.7.2
-pyOpenSSL==17.2.0
+pyhcl==0.3.9
+pylint==1.7.4
 python-dateutil==2.6.1
-pytz==2017.2
+pytz==2017.3
 PyYAML==3.12
 requests==2.18.4
 s3transfer==0.1.11
-six==1.10.0
+six==1.11.0
 snowballstemmer==1.2.1
-Sphinx==1.6.3
+Sphinx==1.6.5
 sphinx-rtd-theme==0.2.4
 sphinxcontrib-websupport==1.0.1
-typed-ast==1.0.4
+typed-ast==1.1.0
 urllib3==1.22
 wcwidth==0.1.7
 wrapt==1.10.11
-yara-python==3.6.3
+yara-python==3.7.0

requirements_top_level.txt

@@ -1,12 +1,12 @@
 backoff
 boto3
-cbapi==1.3.2
+cbapi==1.3.4
 coverage
 coveralls
-mypy==0.521
+mypy==0.550
 pyfakefs
 pyhcl
 pylint
 sphinx
 sphinx-rtd-theme
-yara-python==3.6.3
+yara-python==3.7.0

terraform/main.tf

@@ -1,10 +1,8 @@
 terraform {
-  // 0.10.4 is required for locals to work correctly across multiple files
-  required_version = "~> 0.10.4"
+  required_version = "~> 0.11.0"
 }
 
 provider "aws" {
-  // 0.1.4 required for aws_cloudwatch_dashboard
-  version = "~> 0.1.4"
+  version = "~> 1.3.1"
   region  = "${var.aws_region}"
 }

terraform/modules/lambda/outputs.tf

@@ -1,23 +1,26 @@
+# All outputs are conditionally defined based on whether the underlying resources exist (count > 0)
+# https://www.terraform.io/upgrade-guides/0-11.html#referencing-attributes-from-resources-with-count-0
+
 output "alias_arn" {
-  value = "${aws_lambda_alias.production_alias.arn}"
+  value = "${element(concat(aws_lambda_alias.production_alias.*.arn, list("")), 0)}"
 }
 
 output "alias_name" {
-  value = "${aws_lambda_alias.production_alias.name}"
+  value = "${element(concat(aws_lambda_alias.production_alias.*.name, list("")), 0)}"
 }
 
 output "function_arn" {
-  value = "${aws_lambda_function.function.arn}"
+  value = "${element(concat(aws_lambda_function.function.*.arn, list("")), 0)}"
 }
 
 output "function_name" {
-  value = "${aws_lambda_function.function.function_name}"
+  value = "${element(concat(aws_lambda_function.function.*.function_name, list("")), 0)}"
 }
 
 output "log_group_name" {
-  value = "${aws_cloudwatch_log_group.lambda_log_group.name}"
+  value = "${element(concat(aws_cloudwatch_log_group.lambda_log_group.*.name, list("")), 0)}"
 }
 
 output "role_id" {
-  value = "${aws_iam_role.role.id}"
+  value = "${element(concat(aws_iam_role.role.*.id, list("")), 0)}"
 }

terraform/terraform.tfvars

@@ -65,16 +65,16 @@ lambda_batch_memory_mb = 128
 
 // How often the Lambda dispatcher will be invoked.
 // To ensure only one dispatcher is running, this rate should be > the lambda dispatch timeout.
-lambda_dispatch_frequency_minutes = 1
+lambda_dispatch_frequency_minutes = 2
 
 // Maximum number of analyzers that can be asynchronously invoked during one dispatcher run.
 // Higher values allow for more throughtput, but if too many analyzers are invoked too quickly,
 // Lambda invocations may be throttled.
-lambda_dispatch_limit = 100
+lambda_dispatch_limit = 500
 
 // Memory and time limits for the dispatching function.
 lambda_dispatch_memory_mb = 128
-lambda_dispatch_timeout_sec = 40
+lambda_dispatch_timeout_sec = 115
 
 // Memory and time limits for the analyzer functions.
 lambda_analyze_memory_mb = 1024

tests/lambda_functions/analyzer/main_test.py

@@ -60,13 +60,8 @@ def setUp(self):
         self.maxDiff = None  # pylint: disable=invalid-name
 
         # Set up the fake filesystem.
-        real_tempdir = tempfile.gettempdir()
         self.setUpPyfakefs()
         os.makedirs(os.path.dirname(COMPILED_RULES_FILEPATH))
-        os.makedirs(real_tempdir)
-        if not os.path.exists(tempfile.gettempdir()):
-            # Temp directory in pyfakefs may look different than in the real fs.
-            os.makedirs(tempfile.gettempdir())
         yara_mocks.save_test_yara_rules(COMPILED_RULES_FILEPATH)
 
         # Set environment variables.

tests/lambda_functions/build_test.py

@@ -44,23 +44,24 @@ def test_build_analyzer(self, mock_print: mock.MagicMock):
         self._verify_filenames(
             os.path.join(self._tempdir, build.ANALYZE_ZIPFILE + '.zip'),
             {
-                'yara_python-3.6.3.egg-info/',
+                # Python source files
                 '__init__.py',
                 'analyzer_aws_lib.py',
                 'binary_info.py',
                 'common.py',
-                'compiled_yara_rules.bin',
                 'file_hash.py',
-                'libpython3.5m.so.1.0',
                 'main.py',
-                'yara.so',
                 'yara_analyzer.py',
-                'yara_python-3.6.3.egg-info/dependency_links.txt',
-                'yara_python-3.6.3.egg-info/installed-files.txt',
-                'yara_python-3.6.3.egg-info/not-zip-safe',
-                'yara_python-3.6.3.egg-info/PKG-INFO',
-                'yara_python-3.6.3.egg-info/SOURCES.txt',
-                'yara_python-3.6.3.egg-info/top_level.txt'
+
+                # Compiled rules file
+                'compiled_yara_rules.bin',
+
+                # Natively compiled binaries
+                'libarchive.so.13',
+                'liblzo2.so.2',
+                'libyara.so.3',
+                'yara.so',
+                'yextend'
             }
         )
         mock_print.assert_called_once()

tests/lambda_functions/downloader/main_test.py

@@ -3,7 +3,6 @@
 import base64
 import io
 import os
-import tempfile
 from unittest import mock
 
 import boto3
@@ -41,9 +40,7 @@ def setUp(self):
         os.environ['TARGET_S3_BUCKET'] = 'test-s3-bucket'
 
         # Setup fake filesystem.
-        temp_dir = tempfile.gettempdir()  # Get real temp directory before starting fake filesystem.
         self.setUpPyfakefs()
-        os.makedirs(temp_dir)
 
         # Create a mock binary.
         self._binary = MockBinary(

tests/manage_test.py

@@ -248,7 +248,10 @@ def test_encrypt_cb_api_token(
             mock.call('Terraforming KMS key...'),
             mock.call('Encrypting API token...')
         ])
-        mock_subprocess.assert_called_once()
+        mock_subprocess.assert_has_calls([
+            mock.call(['terraform', 'get']),
+            mock.call(['terraform', 'apply', '-target={}'.format(manage.CB_KMS_ALIAS_TERRAFORM_ID)])
+        ])
 
 
 class ManagerTest(FakeFilesystemBase):