Merge branch 'develop' into AL-datasource-to-dbschema

README.md

@@ -80,6 +80,34 @@ Options:
                                 "warning", "error", and "critical". [default: info]
 ```
 
+## Collect P&E Source Data ##
+
+- Add database and data source credentials to src/pe_reports/data/config.ini
+
+```console
+Usage:
+  pe-source DATA_SOURCE [--log-level=LEVEL] [--orgs=ORG_LIST] [--cybersix-methods=METHODS]
+
+Arguments:
+  DATA_SOURCE                       Source to collect data from. Valid values are "cybersixgill",
+                                    "dnstwist", "hibp", and "shodan".
+Options:
+  -h --help                         Show this message.
+  -v --version                      Show version information.
+  -l --log-level=LEVEL              If specified, then the log level will be set to
+                                    the specified value.  Valid values are "debug", "info",
+                                    "warning", "error", and "critical". [default: info]
+  -o --orgs=ORG_LIST                A comma-separated list of orgs to collect data for.
+                                    If not specified, data will be collected for all
+                                    orgs in the pe database. Orgs in the list must match the
+                                    IDs in the cyhy-db. E.g. DHS,DHS_ICE,DOC
+                                    [default: all]
+  -csg --cybersix-methods=METHODS   A comma-separated list of cybersixgill methods.
+                                    If not specified, all will run. Valid values are "alerts",
+                                    "credentials", "mentions", "topCVEs". E.g. alerts,mentions.
+                                    [default: all]
+```
+
 ## Contributing ##
 
 We welcome contributions!  Please see [`CONTRIBUTING.md`](CONTRIBUTING.md) for details.

setup.py

@@ -85,33 +85,35 @@ def get_version(version_file):
     packages=find_packages(where="src"),
     package_dir={"": "src"},
     package_data={
-        "pe_reports": ["data/shell/*.pptx", "data/*.ini"],
         "pe_mailer": ["data/*"],
+        "pe_reports": ["data/shell/*.pptx", "data/*.ini"],
+        "pe_source": ["data/*"],
     },
     py_modules=[splitext(basename(path))[0] for path in glob("src/*.py")],
     include_package_data=True,
     install_requires=[
-        "boto3",
-        "botocore",
-        "chevron",
-        "docopt",
-        "glob2",
-        "importlib-resources",
-        "matplotlib",
+        "boto3 == 1.21.10",
+        "botocore == 1.24.10",
+        "chevron == 0.14.0",
+        "docopt == 0.6.2",
+        "glob2 == 0.7",
+        "importlib-resources == 5.4.0",
+        "matplotlib == 3.3.4",
         "mongo-db-from-config@http://github.com/cisagov/mongo-db-from-config/tarball/develop",
-        "openpyxl",
-        "pandas",
-        "psycopg2",
-        "psycopg2-binary",
-        "pymongo",
-        "pymupdf",
-        "pytest-cov",
-        "python-pptx",
-        "pyyaml",
-        "schema",
-        "setuptools >= 24.2.0",
-        "types-PyYAML",
-        "xhtml2pdf",
+        "openpyxl == 3.0.9",
+        "pandas == 1.1.5",
+        "psycopg2 == 2.9.3",
+        "psycopg2-binary == 2.9.3",
+        "pymongo == 4.0.1",
+        "pymupdf == 1.19.0",
+        "pytest-cov == 3.0.0",
+        "python-pptx == 0.6.21",
+        "pyyaml == 6.0",
+        "reportlab == 3.6.6",
+        "schema == 0.7.5",
+        "setuptools == 58.1.0",
+        "types-PyYAML == 6.0.4",
+        "xhtml2pdf == 0.2.5",
     ],
     extras_require={
         "test": [
@@ -131,8 +133,9 @@ def get_version(version_file):
     # Conveniently allows one to run the CLI tool as `pe-reports` or 'pe-mailer'
     entry_points={
         "console_scripts": [
-            "pe-reports = pe_reports.report_generator:main",
             "pe-mailer = pe_mailer.email_reports:main",
+            "pe-reports = pe_reports.report_generator:main",
+            "pe-source = pe_source.pe_scripts:main",
         ]
     },
 )

src/pe_reports/data/database.ini

@@ -4,3 +4,25 @@ database=
 user=
 password=
 port=
+
+[blocklist]
+[dehashed]
+[dnstwist]
+[hibp]
+
+[postgresql]
+host=
+database=
+user=
+password=
+port=
+
+[shodan]
+key=
+
+[sixgill]
+client_id=
+client_secret=
+
+[rapid7]
+[whoisxml]

src/pe_source/__init__.py

@@ -0,0 +1,8 @@
+"""The pe_source library."""
+# We disable a Flake8 check for "Module imported but unused (F401)" here because
+# although this import is not directly used, it populates the value
+# package_name.__version__, which is used to get version information about this
+# Python package.
+from ._version import __version__  # noqa: F401
+
+__all__ = ["cybersixgill"]

src/pe_source/__main__.py

@@ -0,0 +1,5 @@
+"""Code to run if this package is used as a Python module."""
+
+from .pe_scripts import main
+
+main()

src/pe_source/_version.py

@@ -0,0 +1,2 @@
+"""This file defines the version of this module."""
+__version__ = "1.0.0"