Initial commit of streamingphish.

.gitignore

@@ -0,0 +1,93 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# IPython Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# dotenv
+.env
+
+# virtualenv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+
+# Rope project settings
+.ropeproject
+
+# vi(m)
+*.swp
+

cli/Dockerfile

@@ -0,0 +1,29 @@
+# Build from base image and install prereqs from repo.
+FROM ubuntu:16.04
+RUN apt-get update; apt-get -y upgrade
+RUN apt-get install -y python3-pip
+RUN pip3 install --upgrade pip setuptools
+
+# Install framework requirements.
+ADD requirements.txt /tmp/requirements.txt
+RUN pip3 install -r /tmp/requirements.txt
+
+# Make project directories in /opt/streamingphish and copy framework files.
+ADD config/config.yaml /opt/streamingphish/config/
+
+# Temporarily add streamingphish source.
+ADD streamingphish/ /tmp/source/
+WORKDIR /tmp/source/
+
+# Run unit tests.
+RUN find . -name '*.pyc' -delete
+RUN pytest -s --cov=streamingphish
+
+# Install.
+RUN python3 setup.py install
+RUN rm -rf /tmp/source
+
+# Small hack, lots of debate in docker compose forums for how to address this.
+# I only want service available on demand - not as a daemon during initialization.
+WORKDIR /opt/streamingphish/
+ENTRYPOINT ["sh", "-c", "sleep infinity"]

cli/config/config.yaml

@@ -0,0 +1,32 @@
+certstream:
+  colors: true
+  include_issuer_ca_name: true
+  include_log_source: false
+  include_root_ca_name: false
+  include_seen_timestamp: false
+classifier:
+  active: final
+data:
+  benign_dir: /opt/streamingphish/training_data/benign/
+  fqdn_keywords_dir: /opt/streamingphish/training_data/fqdn_keywords/
+  keywords_dir: /opt/streamingphish/training_data/keywords/
+  malicious_dir: /opt/streamingphish/training_data/malicious/
+  similarity_words_dir: /opt/streamingphish/training_data/similarity_words/
+  targeted_brands_dir: /opt/streamingphish/training_data/targeted_brands/
+  tld_dir: /opt/streamingphish/training_data/tlds/
+logging:
+  enabled: true
+  path: /opt/streamingphish/predictions/
+logging_tiers:
+  high:
+    color: red
+    threshold: 0.9
+  low:
+    color: cyan
+    threshold: 0.6
+  suspicious:
+    color: yellow
+    threshold: 0.75
+system:
+  log_path: /opt/streamingphish/system/
+version: 1

cli/requirements.txt

@@ -0,0 +1,33 @@
+attrs==17.4.0
+certifi==2018.1.18
+certstream==1.9
+chardet==3.0.4
+coverage==4.5.1
+idna==2.6
+mock==2.0.0
+mongomock==3.9.0
+more-itertools==4.1.0
+numpy==1.14.2
+pandas==0.22.0
+pbr==4.0.2
+pluggy==0.6.0
+py==1.5.3
+pymongo==3.6.1
+pytest==3.5.0
+pytest-cov==2.5.1
+python-dateutil==2.7.0
+python-Levenshtein==0.12.0
+pytz==2018.3
+PyYAML==3.12
+requests==2.18.4
+requests-file==1.4.3
+scikit-learn==0.19.1
+scipy==1.0.0
+sentinels==1.0.0
+six==1.11.0
+sklearn==0.0
+termcolor==1.1.0
+tldextract==2.2.0
+tqdm==4.19.8
+urllib3==1.22
+websocket-client==0.47.0

cli/streamingphish/setup.py

@@ -0,0 +1,47 @@
+#!/usr/bin/python3
+
+"""StreamingPhish software installer"""
+
+import os
+from setuptools import setup, find_packages
+
+# TODO read from README.
+long_description = """
+streamingphish is a utility that uses supervised machine learning to detect phishing domains from the Certificate Transparency log network. The firehose of domain names and SSL certificates are made available thanks to the certstream network (certstream.calidog.io).
+
+As a prototype and educational utility, this package also includes a Jupyter notebook to help explain each step of the supervised machine learning lifecycle.
+"""
+
+# Function to get non-data files.
+# Data and Example Files
+def get_files(dir_name):
+    """
+    Takes a directory name and returns a list of all files inside of it.
+    """
+    return [(os.path.join('.', d), [os.path.join(d, f) for f in files]) for d, _, files in os.walk(dir_name)]
+
+setup(
+    name = 'streamingphish',
+    version = '0.4',
+    url = 'https://github.com/wesleyraptor/streamingphish/',
+    author = 'Wes Connell',
+    author_email = 'wes@raptorlabs.io',
+    description = 'streamingphish is a utility that uses machine learning to identify phishing domains.',
+    long_description = long_description,
+    packages = find_packages(),
+    include_package_data=True,
+    data_files=get_files('data') + get_files('config'),
+    entry_points = {
+        'console_scripts': ['streamingphish=streamingphish.__main__:main'],
+    },
+    license='MIT License',
+    classifiers = [
+        "Development Status :: 4 - Alpha",
+        "Environment :: Console",
+        "License :: OSI Approved :: MIT License",
+        "Programming Language :: Python :: 3",
+        "Natural Language :: English",
+        "Topic :: Scientific/Engineering :: Information Analysis"
+    ],
+    keywords='phishing certstream machine learning'
+)

cli/streamingphish/streamingphish/__main__.py

@@ -0,0 +1,8 @@
+#!/usr/bin/python3
+from .cli import PhishCLI
+
+def main():
+    PhishCLI()
+
+if __name__ == '__main__':
+    main()