Adding requirements-all and pre-commit hook to generate this file

.pre-commit-config.yaml

@@ -26,3 +26,11 @@ repos:
           - --extend-ignore=E0401,E501
           - --max-line-length=90
           - --exclude=tests,test*.py
+  - repo: local
+    hooks:
+    - id: check_reqs_all
+      name: check_reqs_all
+      entry: python -m tools.create_reqs_all
+      pass_filenames: False
+      language: python
+      types: [python]

requirements-all.txt

@@ -0,0 +1,47 @@
+attrs>=18.2.0
+azure-common>=1.1.18
+azure-core>=1.2.2
+azure-identity==1.4.0
+azure-keyvault-secrets>=4.0.0
+azure-mgmt-compute>=4.6.2
+azure-mgmt-core>=1.2.1
+azure-mgmt-keyvault>=2.0.0
+azure-mgmt-monitor>=1.0.1
+azure-mgmt-network>=2.7.0
+azure-mgmt-resource>=2.2.0
+azure-mgmt-subscription>=0.2.0
+azure-storage-blob>=12.5.0
+bokeh>=1.4.0
+cryptography>=3.1
+deprecated>=1.2.4
+dnspython>=2.0.0
+folium>=0.9.0
+geoip2>=2.9.0
+html5lib  # req by pandas for scraping
+ipwhois>=1.1.0
+ipython>=7.1.1
+ipywidgets>=7.4.2
+keyring>=13.2.1
+Kqlmagic>=0.1.106
+lxml
+matplotlib>=3.0.0
+msrest>=0.6.0
+msrestazure>=0.6.0
+nest_asyncio>=1.4.0
+networkx>=2.2
+numpy>=1.15.4  # pandas
+pandas>=0.25.0
+python-dateutil>=2.8.1  # pandas
+pytz>=2019.2  # pandas
+pyyaml>=3.13
+requests>=2.21.1
+scikit-learn>=0.20.2
+scipy>=1.1.0
+setuptools>=40.6.3
+splunk-sdk>=1.6.0
+statsmodels>=0.11.1
+tldextract>=2.2.2
+tqdm>=4.36.1
+urllib3>=1.23
+vt-graph-api>=1.0.1
+vt-py>=0.6.1

tests/test_pkg_imports.py

@@ -12,7 +12,7 @@
 import pytest
 import pytest_check as check
 
-from tools.toollib.import_analyzer import analyze_imports
+from tools.toollib.import_analyzer import analyze_imports, get_extras_from_setup
 
 PKG_ROOT = "."
 PKG_NAME = "msticpy"
@@ -26,36 +26,13 @@
 @pytest.fixture(scope="module")
 def extras_from_setup():
     """Read extras packages from setup.py."""
-    setup_py = Path(PKG_ROOT) / "setup.py"
+    return get_extras_from_setup(PKG_ROOT, extra="all")
 
-    setup_txt = None
-    with open(setup_py, "+r") as f_handle:
-        setup_txt = f_handle.read()
 
-    srch_txt = "setuptools.setup("
-    repl_txt = [
-        "def fake_setup(*args, **kwargs):" "    pass",
-        "",
-        "fake_setup(",
-    ]
-    setup_txt = setup_txt.replace(srch_txt, "\n".join(repl_txt))
-
-    neut_setup_py = Path(PKG_ROOT) / "msticpy/neut_setup.py"
-    try:
-        with open(neut_setup_py, "+w") as f_handle:
-            f_handle.writelines(setup_txt)
-
-        setup_mod = importlib.import_module("msticpy.neut_setup", "msticpy")
-        return getattr(setup_mod, "EXTRAS").get("all")
-    finally:
-        neut_setup_py.unlink()
-
-
-def test_missing_pkgs_req(extras_from_setup):
+def test_missing_pkgs_req():
     """Check for packages used in code but not in requirements.txt."""
-    extras = extras_from_setup
     mod_imports = analyze_imports(
-        package_root=PKG_ROOT, package_name=PKG_NAME, req_file=REQS_FILE, extras=extras
+        package_root=PKG_ROOT, package_name=PKG_NAME, req_file=REQS_FILE
     )
     import_errs = {v for s in mod_imports.values() for v in s.unknown}
     print("re module path:", re.__file__)
@@ -84,33 +61,13 @@ def test_conda_reqs(extras_from_setup):
     conda_reqs_file = Path(PKG_ROOT) / "conda/conda-reqs.txt"
     conda_reqs_pip_file = Path(PKG_ROOT) / "conda/conda-reqs-pip.txt"
 
-    main_reqs_dict = {}
-    with open(str(main_reqs_file), "r") as f_hdl:
-        reqs = f_hdl.readlines()
-        lines = [line for line in reqs if not line.strip().startswith("#")]
-        for item in [re.split(REQS_OP_RGX, line) for line in lines]:
-            main_reqs_dict[item[0].strip()] = item[1].strip() if len(item) > 1 else None
+    main_reqs_dict = _get_reqs_from_file(main_reqs_file)
     # Add extras
     for item in [re.split(REQS_OP_RGX, line) for line in extras_from_setup]:
         main_reqs_dict[item[0].strip()] = item[1].strip() if len(item) > 1 else None
 
-    conda_reqs_dict = {}
-    with open(str(conda_reqs_file), "r") as f_hdl:
-        reqs = f_hdl.readlines()
-        lines = [line for line in reqs if not line.strip().startswith("#")]
-        for item in [re.split(REQS_OP_RGX, line) for line in lines]:
-            conda_reqs_dict[item[0].strip()] = (
-                item[1].strip() if len(item) > 1 else None
-            )
-
-    conda_reqs_pip_dict = {}
-    with open(str(conda_reqs_pip_file), "r") as f_hdl:
-        reqs = f_hdl.readlines()
-        lines = [line for line in reqs if not line.strip().startswith("#")]
-        for item in [re.split(REQS_OP_RGX, line) for line in lines]:
-            conda_reqs_pip_dict[item[0].strip()] = (
-                item[1].strip() if len(item) > 1 else None
-            )
+    conda_reqs_dict = _get_reqs_from_file(conda_reqs_file)
+    conda_reqs_pip_dict = _get_reqs_from_file(conda_reqs_pip_file)
 
     for key, val in main_reqs_dict.items():
         print(f"Checking {key} in conda-reqs.txt", bool(key in conda_reqs_dict))
@@ -152,3 +109,15 @@ def test_conda_reqs(extras_from_setup):
     if conda_reqs_pip_dict:
         print("Extra items found in conda-reqs-pip.txt", conda_reqs_dict)
     check.is_false(conda_reqs_pip_dict, "no extra items in conda-reqs-pip.txt")
+
+
+def _get_reqs_from_file(reqs_file):
+    conda_reqs_dict = {}
+    with open(str(reqs_file), "r") as f_hdl:
+        reqs = f_hdl.readlines()
+        lines = [line for line in reqs if not line.strip().startswith("#")]
+        for item in [re.split(REQS_OP_RGX, line) for line in lines]:
+            conda_reqs_dict[item[0].strip()] = (
+                item[1].strip() if len(item) > 1 else None
+            )
+    return conda_reqs_dict

tools/analyze_imports.py

@@ -5,9 +5,13 @@
 # --------------------------------------------------------------------------
 """Python file import analyzer."""
 import argparse
+import sys
 
-from toollib import VERSION
-from toollib.import_analyzer import analyze_imports
+sys.path.append("./tools")
+
+# pylint: disable=wrong-import-position
+from toollib import VERSION  # noqa: E402
+from toollib.import_analyzer import analyze_imports  # noqa: E402
 
 __version__ = VERSION
 __author__ = "Ian Hellen"
@@ -83,19 +87,19 @@ def _print_single_module(mod_name, imps, p_args):
     print(mod_name)
     if p_args.internal and imps.internal:
         print("internal imports:", end=" ")
-        print(imps.internal if imps.internal else "none")
+        print(imps.internal or "none")
     if p_args.stdlib and imps.standard:
         print("std lib imports:", end=" ")
-        print(imps.standard if imps.standard else "none")
+        print(imps.standard or "none")
     if p_args.reqs and imps.setup_reqs:
         print(f"external imports listed in {p_args.req_file}:", end=" ")
-        print(imps.setup_reqs if imps.setup_reqs else "none")
+        print(imps.setup_reqs or "none")
     if p_args.missing and imps.missing_reqs:
         print("missing imports (used but not in requirements):", end=" ")
-        print(imps.missing_reqs if imps.missing_reqs else "none")
+        print(imps.missing_reqs or "none")
     if p_args.unknown and imps.unknown:
         print("unknown imports:", end=" ")
-        print(imps.unknown if imps.unknown else "none")
+        print(imps.unknown or "none")
 
 
 def _print_all_imports(mod_imports, p_args):

tools/config2kv.py

@@ -145,9 +145,10 @@ def _prompt_yn(mssg, confirm):
 
 
 def _add_secrets_to_vault(vault_name, secrets, confirm, **kwargs):
+    print("Vault management requires authentication")
+    kv_mgmt = BHKeyVaultMgmtClient(**kwargs)
+    vault_uri = None
     try:
-        print("Vault management requires authentication")
-        kv_mgmt = BHKeyVaultMgmtClient(**kwargs)
         vault_uri = kv_mgmt.get_vault_uri(vault_name)
         print(f"Vault {vault_name} found.")
     except CloudError:

tools/create_reqs_all.py

@@ -0,0 +1,175 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+"""Requirements file writer from setup.py extras."""
+import argparse
+import difflib
+from importlib import import_module
+from pathlib import Path
+import sys
+from typing import List
+
+
+VERSION = "1.0.0"
+
+__version__ = VERSION
+__author__ = "Ian Hellen"
+
+
+def _add_script_args():
+    parser = argparse.ArgumentParser(
+        description=f"Package imports analyer. v.{VERSION}"
+    )
+    parser.add_argument(
+        "--out",
+        "-o",
+        default="./requirements-all.txt",
+        required=False,
+        help="Path of output file",
+    )
+    parser.add_argument(
+        "--setup",
+        "-s",
+        default="./setup.py",
+        required=False,
+        help="Path of setup.py to process.",
+    )
+    parser.add_argument(
+        "--extra",
+        "-e",
+        default="all",
+        required=False,
+        help="Name of extra to use.",
+    )
+    parser.add_argument(
+        "--diff",
+        "-d",
+        required=False,
+        action="store_true",
+        help="Print diffs, don't write file.",
+    )
+    parser.add_argument(
+        "--print",
+        "-p",
+        required=False,
+        action="store_true",
+        help="Print new requirements, don't write file.",
+    )
+    return parser
+
+
+def _get_current_reqs_all(app_args):
+    current_reqs = Path(app_args.out)
+    if current_reqs.is_file():
+        curr_text = current_reqs.read_text()
+        return sorted(
+            req
+            for req in curr_text.split("\n")
+            if req.strip() and not req.strip().startswith("#")
+        )
+    return []
+
+
+def _compare_reqs(new, current):
+    return list(
+        difflib.context_diff(
+            sorted(new, key=str.casefold),
+            sorted(current, key=str.casefold),
+            fromfile="Corrected",
+            tofile="Current",
+        )
+    )
+
+
+def _write_requirements(app_args, extras: List[str]):
+    Path(app_args.out).write_text("\n".join(extras))
+
+
+def _get_extras_from_setup(
+    package_root: str,
+    setup_py: str = "setup.py",
+    extra: str = "all",
+    include_base: bool = False,
+) -> List[str]:
+    """
+    Return list of extras from setup.py.
+
+    Parameters
+    ----------
+    package_root : str
+        The root folder of the package
+    setup_py : str, optional
+        The name of the setup file to process, by default "setup.py"
+    extra : str, optiona
+        The name of the extra to return, by default "all"
+    include_base : bool, optional
+        If True include install_requires, by default False
+
+    Returns
+    -------
+    List[str]
+        List of package requirements.
+
+    Notes
+    -----
+    Duplicated from tools/toollib/import_analyzer.py
+
+    """
+    setup_py = str(Path(package_root) / setup_py)
+
+    setup_txt = None
+    with open(setup_py, "+r") as f_handle:
+        setup_txt = f_handle.read()
+
+    srch_txt = "setuptools.setup("
+    repl_txt = [
+        "def fake_setup(*args, **kwargs):",
+        "    pass",
+        "",
+        "fake_setup(",
+    ]
+    setup_txt = setup_txt.replace(srch_txt, "\n".join(repl_txt))
+
+    neut_setup_py = Path(package_root) / "neut_setup.py"
+    try:
+        with open(neut_setup_py, "+w") as f_handle:
+            f_handle.writelines(setup_txt)
+
+        setup_mod = import_module("neut_setup")
+        extras = getattr(setup_mod, "EXTRAS").get(extra)
+        if include_base:
+            base_install = getattr(setup_mod, "INSTALL_REQUIRES")
+            extras.extend(
+                [req.strip() for req in base_install if not req.strip().startswith("#")]
+            )
+        return sorted(list(set(extras)), key=str.casefold)
+    finally:
+        neut_setup_py.unlink()
+
+
+# pylint: disable=invalid-name
+if __name__ == "__main__":
+    arg_parser = _add_script_args()
+    args = arg_parser.parse_args()
+
+    extra_reqs = _get_extras_from_setup(
+        package_root=str(Path(args.setup).parent),
+        setup_py=str(Path(args.setup).name),
+        extra="all",
+        include_base=True,
+    )
+
+    if args.print:
+        print("\n".join(extra_reqs))
+        sys.exit(0)
+
+    existing_reqs = _get_current_reqs_all(args)
+    diff_reqs = _compare_reqs(new=extra_reqs, current=existing_reqs)
+    if diff_reqs:
+        if args.diff:
+            print("\n".join(diff.strip() for diff in diff_reqs))
+        else:
+            _write_requirements(app_args=args, extras=extra_reqs)
+        sys.exit(1)