Merge ac3257a into 7d198a3

.github/CONTRIBUTING.rst

@@ -4,10 +4,10 @@ Contributing
 Installing Dependencies
 -----------------------
 
-Make sure you are running Python 2.7::
+Make sure you are running Python 3.7::
 
   $ python -V
-  Python 2.7.x
+  Python 3.7.x
 
 Install:
 
@@ -51,12 +51,12 @@ Make changes, add features, or fix bugs.
 When writing commit messages, make sure to prefix with one of the following tags::
 
   [docs]              # changes to StreamAlert documentation
-  [cli]               # stream_alert_cli changes
+  [cli]               # streamalert_cli changes
   [terraform]         # terraform changes
-  [core]              # changes with core stream_alert classes used across lambda functions
+  [core]              # changes with core streamalert classes used across lambda functions
   [testing]           # changes with testing infrastructure or processes
   [setup]             # StreamAlert development setup changes
-  [config]            # stream_alert config changes
+  [config]            # streamalert config changes
 
 The first line of your commit message should be short.  Use newlines to explain further::
 
@@ -127,7 +127,7 @@ If you are adding features to existing classes with tests, you must add test cas
 Integration Testing
 ~~~~~~~~~~~~~~~~~~~
 
-To verify StreamAlert works from end-to-end, locally, follow the testing instructions `here <https://streamalert.io/rules.html>`_.
+To verify StreamAlert works from end-to-end, locally, follow the testing instructions `here <https://streamalert.io/en/stable/testing.html#running-tests>`_.
 
 Pull Request
 ------------
@@ -147,5 +147,3 @@ In your pull request body, use the following template::
   * summary of changes 2
 
 All pull requests must pass continuous integration tests (nosetests) and receive a code review from one of the maintainers.
-
-After your branch has been accepted, you are free to merge into ``master``!

.gitignore

@@ -3,6 +3,7 @@ docs/build
 
 # Compiled Python files
 *.pyc
+__pycache__
 
 # Terraform files
 *.terraform/
@@ -20,5 +21,9 @@ terraform/*.tf.json
 # virtualenv files in root of repo
 /venv
 
-/vagrant
+# Vagrant VM artifacts
+ubuntu*.log
+/.vagrant
+
+# Coverage artifacts
 /htmlcov

.pylintrc

@@ -271,7 +271,7 @@ contextmanager-decorators=contextlib.contextmanager
 # List of members which are set dynamically and missed by pylint inference
 # system, and so shouldn't trigger E1101 when accessed. Python regular
 # expressions are accepted.
-generated-members=
+generated-members=.*boto3\.resource.*|.*Table.*
 
 # Tells whether missing members accessed in mixin class should be ignored. A
 # mixin class is detected if its name ends with "mixin" (case insensitive).

.travis.yml

@@ -1,16 +1,17 @@
 language: python
-dist: trusty
+dist: xenial
 cache: pip
 sudo: required
 python:
-  - 2.7
+  - 3.7
 before_install:
   - sudo rm -f /etc/boto.cfg
 install:
   - pip install -r requirements.txt
 script:
   - ./tests/scripts/unit_tests.sh
   - ./manage.py test rules
+  - ./manage.py test classifier
   - sphinx-build -W docs/source docs/build
   - ./tests/scripts/pylint.sh
   - bandit --ini setup.cfg -r .

AUTHORS.rst

@@ -1,22 +1,34 @@
-Core Members:
+Current Maintainers:
+
+* `Ryan Deivert <https://github.com/ryandeivert>`_
+* `Chunyong Lin <https://github.com/chunyong-lin>`_
+* `Blake Motl <https://github.com/blakemotl>`_
+* `Derek Wang <https://github.com/Ryxias>`_
+
+Former Maintainers:
 
-* `Jack Naglieri <https://www.github.com/jacknagz>`_
-* `Mime Frame <https://www.github.com/mime-frame>`_
 * `Jamie Brim <https://www.github.com/strcrzy>`_
 * `Austin Byers <https://github.com/austinbyers>`_
-* `Chunyong Lin <https://github.com/chunyong-lin>`_
-* `Ryan Deivert <https://github.com/ryandeivert>`_
+* `Mime Frame <https://www.github.com/mime-frame>`_
+* `Jack Naglieri <https://www.github.com/jacknagz>`_
 
 Contributors, in no particular order:
 
-* `Javier Marcos <https://github.com/javuto>`_
-* `Xinghong Fang <https://github.com/javefang>`_
-* `Scott Piper <https://github.com/0xdabbad00>`_
-* `Jeremy Stott <https://github.com/stoggi>`_
-* `Armen Tashjian <https://github.com/armtash>`_
-* `Patrick O'Doherty <https://github.com/patrickod>`_
-* `Daimon Greaves <https://github.com/fusionrace>`_
-* `Joseph Bussing <https://github.com/jbussing>`_
-* `Garret Reece <https://github.com/GarretReece>`_
-* `Jordan Harband <https://github.com/ljharb>`_
-* `clippy <https://github.com/securityclippy>`_
+* `javuto <https://github.com/javuto>`_
+* `javefang <https://github.com/javefang>`_
+* `0xdabbad00 <https://github.com/0xdabbad00>`_
+* `stoggi <https://github.com/stoggi>`_
+* `armtash <https://github.com/armtash>`_
+* `patrickod <https://github.com/patrickod>`_
+* `fusionrace <https://github.com/fusionrace>`_
+* `jbussing <https://github.com/jbussing>`_
+* `GarretReece <https://github.com/GarretReece>`_
+* `ljharb <https://github.com/ljharb>`_
+* `securityclippy <https://github.com/securityclippy>`_
+* `jack1902 <https://github.com/jack1902>`_
+* `gavinelder <https://github.com/gavinelder>`_
+* `geremyCohen <https://github.com/geremyCohen>`_
+* `catpham <https://github.com/catpham>`_
+* `ricardf-cmp <https://github.com/ricardf-cmp>`_
+* `scoders <https://github.com/scoders>`_
+* `stoggi <https://github.com/stoggi>`_

README.rst

@@ -35,5 +35,5 @@ Resources
 -----------
 
 * `Blog Post <https://medium.com/@airbnbeng/e8619e3e5043>`_
-* `User Guide <https://streamalert.readthedocs.io/>`_
+* `User Guide <https://streamalert.io/>`_
 * `Slack <https://streamalert.herokuapp.com>`_ (unofficial)

Vagrantfile

@@ -1,3 +1,117 @@
+def patch_libssl(machine)
+  # This fixes the interactive prompt when updating libssl
+  machine.vm.provision :shell,
+                       inline: "sudo apt-get update -y -qq && "\
+                         "sudo dpkg-reconfigure libc6 && "\
+                         "export DEBIAN_FRONTEND=noninteractive && "\
+                         "sudo -E apt-get -q --option \"Dpkg::Options::=--force-confold\" --assume-yes install libssl1.1"
+end
+
+def configure_python(machine, version)
+  # Install the virtualenv and virtualenvwrapper dependencies
+  machine.vm.provision :shell,
+                   path: "vagrant/cli/python-virtualenvwrapper/install.sh",
+                   # We need root to update & install the packages
+                   privileged: true,
+                   env: {
+                     PYTHON_VERSION: "python#{version.to_s}"
+                   }
+
+  # Configure the default vagrant user bash session with the virtualenv
+  machine.vm.provision :shell,
+                   path: "vagrant/cli/python-virtualenvwrapper/configure.sh",
+                   # Install this to the vagrant user (unprivileged default)
+                   privileged: false,
+                   # Reset the terminal session so changes are sourced in
+                   # subsequent shells
+                   reset: true,
+                   # Provide the shell script with the version of Python to
+                   # install.
+                   env: {
+                     PYTHON_VERSION: "python#{version.to_s}"
+                   }
+end
+
+STREAMALERT_SHELL_ENV = %{
+export AWS_DEFAULT_REGION='#{ENV.fetch('SA_AWS_DEFAULT_REGION', 'Your region here!')}'
+export AWS_ACCESS_KEY_ID='#{ENV.fetch('SA_AWS_ACCESS_KEY_ID', 'Your access key ID here!')}'
+export AWS_SECRET_ACCESS_KEY='#{ENV.fetch('SA_AWS_SECRET_ACCESS_KEY', 'Your secret access key here!')}'
+export SA_EMAIL='#{ENV.fetch('SA_EMAIL', 'example@example.com')}'
+}
+
+def configure_streamalert(machine)
+  # Install streamalert and it's dependencies
+  # NOTE: The `aws` cli tool is installed as a dependency, thus it is
+  # available once streamalert is installed.
+  machine.vm.provision :shell,
+                   path: "vagrant/cli/streamalert/install.sh",
+                   # Install this to the vagrant user (unprivileged default)
+                   privileged: true
+
+  # Configure streamalert with required environment variables
+  machine.vm.provision :shell,
+                   # Append the environment variables to the .bashrc for
+                   # the vagrant user (unprivileged default)
+                   inline: "echo \"#{STREAMALERT_SHELL_ENV}\" >> ~/.bashrc",
+                   # Install this to the vagrant user (unprivileged default)
+                   privileged: false
+
+  # Configure streamalert once the environment variables have been defined
+  machine.vm.provision :shell,
+                   path: "vagrant/cli/streamalert/configure.sh",
+                   privileged: false
+end
+
+TERRAFORM_VERSION = ENV.fetch('SA_TERRAFORM_VERSION', '0.12.9')
+def configure_terraform(machine)
+  # Install terraform with the specified version.
+  machine.vm.provision :shell,
+                       path: "vagrant/cli/terraform/install.sh",
+                       # Provide the shell script with the version of terraform
+                       # to install.
+                       env: {
+                         TERRAFORM_VERSION: TERRAFORM_VERSION
+                       }
+end
+
+FINAL_MESSAGE = %{
+Your local environment has been created! To provision the remote infrastructure,
+execute "yes | $PROJECT_ROOT/manage.py init". To verify the infrastructure was
+created correctly, run "aws s3 ls | grep streamalert".
+
+The following lines were appended to the vagrant (default) user's
+~/.bashrc:
+
+#{STREAMALERT_SHELL_ENV}
+
+}
+
+def final_message(machine)
+  # Output the final message for easy copy/paste of next steps. We scope it
+  # in the vm via inline shell provisioner so it makes it easy to grep for
+  # the last N machine output lines.
+  machine.vm.provision :shell,
+                   inline: "cat << EOF #{FINAL_MESSAGE}\nEOF",
+                   # We don't need root to echo
+                   privileged: false
+end
+
 Vagrant.configure(2) do |config|
-  config.vm.box = "bento/centos-7.2"
+  config.vm.box = "ubuntu/bionic64"
+
+  config.vm.define :development_py2, autostart: false do |py2|
+    patch_libssl(py2)
+    configure_python(py2, 2.7)
+    configure_terraform(py2)
+    configure_streamalert(py2)
+    final_message(py2)
+  end
+
+  config.vm.define :development_py3, primary: true do |py3|
+    patch_libssl(py3)
+    configure_python(py3, 3.7)
+    configure_terraform(py3)
+    configure_streamalert(py3)
+    final_message(py3)
+  end
 end

conf/clusters/prod.json

@@ -1,42 +1,87 @@
 {
   "id": "prod",
+  "classifier_config": {
+    "enable_custom_metrics": true,
+    "inputs": {
+      "aws-sns": []
+    },
+    "log_level": "info",
+    "log_retention_days": 14,
+    "memory": 128,
+    "metric_alarms": {
+      "errors": {
+        "enabled": true,
+        "evaluation_periods": 1,
+        "period_secs": 120,
+        "threshold": 0
+      },
+      "throttles": {
+        "enabled": true,
+        "evaluation_periods": 1,
+        "period_secs": 120,
+        "threshold": 0
+      }
+    },
+    "timeout": 60,
+    "vpc_config": {
+      "security_group_ids": [],
+      "subnet_ids": []
+    }
+  },
+  "data_sources": {
+    "kinesis": {
+      "prefix_cluster1_streamalert": [
+        "cloudwatch",
+        "ghe",
+        "osquery"
+      ]
+    },
+    "s3": {
+      "prefix.cluster.sample.bucket": [
+        "cloudtrail",
+        "carbonblack",
+        "fleet"
+      ]
+    },
+    "sns": {
+      "prefix_cluster_sample_topic": [
+        "binaryalert",
+        "trendmicro"
+      ]
+    },
+    "streamalert_app": {
+      "prefix_cluster_box_admin_events_sm-app-name_app": [
+        "box"
+      ],
+      "prefix_cluster_duo_admin_sm-app-name_app": [
+        "duo"
+      ],
+      "prefix_cluster_duo_auth_sm-app-name_app": [
+        "duo"
+      ],
+      "prefix_cluster_gsuite_admin_sm-app-name_app": [
+        "gsuite"
+      ],
+      "prefix_cluster_onelogin-events-app-name_app": [
+        "onelogin"
+      ],
+      "prefix_cluster_slack_access_sm-app-name_app": [
+        "slack"
+      ],
+      "prefix_cluster_slack_integration_sm-app-name_app": [
+        "slack"
+      ],
+      "prefix_cluster_aliyun_actiontrail_sm-app-name_app": [
+        "aliyun"
+      ]
+    }
+  },
+  "enable_threat_intel": false,
   "modules": {
     "cloudwatch_monitoring": {
       "enabled": true,
       "kinesis_alarms_enabled": false,
       "lambda_alarms_enabled": true
-    },
-    "stream_alert": {
-      "classifier_config": {
-        "enable_custom_metrics": true,
-        "inputs": {
-          "aws-sns": []
-        },
-        "log_level": "info",
-        "log_retention_days": 14,
-        "memory": 128,
-        "metric_alarms": {
-          "errors": {
-            "enabled": true,
-            "evaluation_periods": 1,
-            "period_secs": 120,
-            "threshold": 0
-          },
-          "throttles": {
-            "enabled": true,
-            "evaluation_periods": 1,
-            "period_secs": 120,
-            "threshold": 0
-          }
-        },
-        "timeout": 60,
-        "vpc_config": {
-          "security_group_ids": [],
-          "subnet_ids": []
-        }
-      },
-      "enable_threat_intel": false
     }
-  },
-  "region": "us-east-1"
+  }
 }