Fixing/constraining some azure dependencies for Sphinx (#68)

* Fixing/constraining some azure dependencies for Sphinx

* Adding py script for comparing requirements files.

* mypy detected error in vtlookup.py

* Fixing naming and import issue

conda/conda-reqs-pip.txt

@@ -5,7 +5,7 @@ azure-keyvault-secrets>=4.0.0
 azure-mgmt-compute>=4.6.2
 azure-mgmt-keyvault>=2.0.0
 azure-mgmt-monitor>=0.5.2
-azure-mgmt-resource>=2.2.0
+azure-mgmt-resource>=2.2.0,<=9.0.0
 azure-mgmt-subscription>=0.2.0
 geoip2>=2.9.0
 ipwhois>=1.1.0

conda/conda-reqs.txt

@@ -2,7 +2,7 @@ adal
 attrs
 azure-common
 azure-mgmt-network
-azure-mgmt-resource
+azure-mgmt-resource>=2.2.0,<=9.0.0
 beautifulsoup4
 bokeh
 cryptography

msticpy/sectools/tld_seed.txt

@@ -47,6 +47,7 @@ ALLY
 ALSACE
 ALSTOM
 AM
+AMAZON
 AMERICANEXPRESS
 AMERICANFAMILY
 AMEX
@@ -391,7 +392,6 @@ ERNI
 ES
 ESQ
 ESTATE
-ESURANCE
 ET
 ETISALAT
 EU
@@ -1070,7 +1070,6 @@ SCHULE
 SCHWARZ
 SCIENCE
 SCJOHNSON
-SCOR
 SCOT
 SD
 SE
@@ -1375,6 +1374,7 @@ XN--BCK1B9A5DRE4C
 XN--C1AVG
 XN--C2BR7G
 XN--CCK2B3B
+XN--CCKWCXETD
 XN--CG4BKI
 XN--CLCHC0EA0B2G2A9GCD
 XN--CZR694B
@@ -1410,6 +1410,7 @@ XN--IO0A7I
 XN--J1AEF
 XN--J1AMH
 XN--J6W193G
+XN--JLQ480N2RG
 XN--JLQ61U9W7B
 XN--JVR189M
 XN--KCRX77D1X4A

msticpy/sectools/vtlookup.py

@@ -317,7 +317,9 @@ def lookup_ioc(self, observable: str, ioc_type: str, output: str = "dict") -> An
             )
 
         if self._VT_TYPE_MAP[ioc_type] not in self._VT_API_TYPES:
-            vt_types = {k for k, val in self.ioc_vt_type_mapping if val is not None}
+            vt_types = {
+                k for k, val in self.ioc_vt_type_mapping.items() if val is not None
+            }
             err = "IoC Type {} is recognized by VirusTotal. Valid types are [{}]".format(
                 ioc_type, ", ".join(vt_types)
             )

requirements.txt

@@ -9,7 +9,7 @@ azure-mgmt-compute>=4.6.2
 azure-mgmt-keyvault>=2.0.0
 azure-mgmt-monitor>=0.5.2
 azure-mgmt-network>=2.7.0
-azure-mgmt-resource>=2.2.0
+azure-mgmt-resource>=2.2.0,<=9.0.0
 azure-mgmt-subscription>=0.2.0
 beautifulsoup4>=4.8.0
 bokeh>=1.4.0

setup.py

@@ -21,7 +21,7 @@
     "azure-mgmt-keyvault>=2.0.0",
     "azure-mgmt-monitor>=0.5.2",
     "azure-mgmt-network>=2.7.0",
-    "azure-mgmt-resource>=2.2.0",
+    "azure-mgmt-resource>=2.2.0,<=9.0.0",
     "azure-mgmt-subscription>=0.2.0",
     "beautifulsoup4>=4.8.0",
     "bokeh>=1.4.0",

tools/comp_reqs.py

@@ -0,0 +1,61 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+"""Compare two requirements files."""
+import argparse
+import re
+from packaging.version import parse
+from packaging.specifiers import SpecifierSet
+
+
+def _add_script_args(description):
+    parser = argparse.ArgumentParser(
+        description=description, formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+    parser.add_argument("file1", help="First requirements file to compare.")
+    parser.add_argument("file2", help="Second requirements file to compare.")
+    return parser
+
+
+def _parse_line(line):
+    req_regex = r"(?P<pkg>[^=<>]+)(?P<op>[=<>]*)(?P<ver>.*)"
+
+    match = re.search(req_regex, line)
+    if match:
+        return match.groups()
+    return None, None, None
+
+
+def _parse_reqs(file):
+    with open(file, "r") as file_reqs:
+        req_lines = file_reqs.readlines()
+    p_lines = [_parse_line(line) for line in req_lines]
+    return {pkg: (op, ver) for pkg, op, ver in p_lines}
+
+
+# pylint: disable=invalid-name
+if __name__ == "__main__":
+    arg_parser = _add_script_args(description=__doc__)
+    args = arg_parser.parse_args()
+
+    f1_dict = _parse_reqs(args.file1)
+    f2_dict = _parse_reqs(args.file2)
+
+    f1_only = {pkg for pkg in f1_dict if pkg not in f2_dict}
+    f2_only = {pkg for pkg in f2_dict if pkg not in f1_dict}
+
+    both = set(f1_dict) - f1_only
+
+    not_compat = []
+    compat = []
+    for pkg in both:
+        v2 = parse(f2_dict[pkg][1])
+        spec1 = SpecifierSet(f1_dict[pkg][0] + f1_dict[pkg][1])
+        if v2 in spec1:
+            compat.append(f"Compatible: {pkg}, {f1_dict[pkg]}, {f2_dict[pkg]}")
+        else:
+            not_compat.append(f"Not compatible: {pkg}, {f1_dict[pkg]}, {f2_dict[pkg]}")
+    print("\n".join(compat))
+    print("\n".join(not_compat))