Merge pull request #619 from microsoft/ianhelle/build-break-file-issu…

…e-2023-01-26

Removing prospector from CI build

.github/workflows/python-package.yml

@@ -55,12 +55,11 @@ jobs:
       - name: Install test dependencies
         run: |
           if [ -f requirements-dev.txt ]; then
-            python -m pip install prospector
             python -m pip install -r requirements-dev.txt
           else
-            python -m pip install flake8 black bandit mypy lxml pylint types-attrs
+            echo "Missing requirements-dev.txt. Installing minimal requirements for testing."
             python -m pip install pytest pytest-cov pytest-xdist pytest-check aiohttp nbconvert jupyter_contrib_nbextensions
-            python -m pip install Pygments respx pytest-xdist markdown beautifulsoup4 Pillow async-cache
+            python -m pip install Pygments respx pytest-xdist markdown beautifulsoup4 Pillow async-cache lxml
           fi
           python -m pip install "pandas>=1.3.0" "pygeohash>=1.2.0"
       - name: Prepare test dummy data
@@ -123,10 +122,15 @@ jobs:
           python -m pip install -e .
       - name: Install test dependencies
         run: |
-          python -m pip install flake8\<6 black bandit mypy lxml pylint types-attrs
+          if [ -f requirements-dev.txt ]; then
+            python -m pip install -r requirements-dev.txt
+          else
+            echo "Missing requirements-dev.txt. Installing minimal requirements for testing."
+            python -m pip install flake8 black bandit mypy pylint types-attrs pydocstyle pyroma
+          fi
       - name: black
         run: |
-          black --diff --check --exclude venv .
+          black --diff --check --exclude venv msticpy
         if: ${{ always() }}
       - name: flake8
         run: |
@@ -169,12 +173,13 @@ jobs:
         run: |
           flake8 --max-line-length=90 --exclude=tests* . --ignore=E501,W503 --jobs=auto
         if: ${{ always() }}
-      - name: prospector
+      - name: pydocstyle
+        run: |
+          pydocstyle --convention=numpy msticpy
+        if: ${{ always() }}
+      - name: pyroma
         run: |
-          # install this separately since it uses older versions of pylint/isort
-          # chain running to installation since this sometimes fails
-          # Temporarily fixing Prospector to 1.7.7 - due to bug wrt to PyRoma - https://github.com/PyCQA/prospector/issues/562
-          python -m pip install prospector[with_pyroma]==1.7.7 && prospector --ignore-paths tests --without-tool pylint
+          pyroma --min 10 .
         if: ${{ always() }}
   check_status:
     runs-on: ubuntu-latest

.pre-commit-config.yaml

@@ -8,19 +8,19 @@ repos:
       - id: trailing-whitespace
         args: [--markdown-linebreak-ext=md]
   - repo: https://github.com/ambv/black
-    rev: 22.3.0
+    rev: 23.1.0
     hooks:
       - id: black
         language: python
   - repo: https://github.com/PyCQA/pylint
-    rev: v2.14.1
+    rev: v2.16.1
     hooks:
       - id: pylint
         args:
           - --disable=duplicate-code,import-error
           - --ignore-patterns=test_
-  - repo: https://gitlab.com/pycqa/flake8
-    rev: 3.9.2
+  - repo: https://github.com/pycqa/flake8
+    rev: 6.0.0
     hooks:
       - id: flake8
         args:
@@ -35,6 +35,12 @@ repos:
         args:
           - --profile
           - black
+  - repo: https://github.com/pycqa/pydocstyle
+    rev: 6.3.0
+    hooks:
+      - id: pydocstyle
+        args:
+          - --convention=numpy
   - repo: local
     hooks:
       - id: check_reqs_all

azure-pipelines.yml

@@ -130,78 +130,6 @@ stages:
         codeCoverageTool: Cobertura
         summaryFileLocation: '$(System.DefaultWorkingDirectory)/coverage.xml'
         reportDirectory: '$(System.DefaultWorkingDirectory)/htmlcov'
-# - stage: PythonLinters
-#   displayName: Python linters and security checkers
-#   dependsOn:
-#   - PythonTests
-#   condition: succeeded('PythonTests')
-#   variables:
-#     # This is run explicitly, so does not need to be auto-injected
-#     skipComponentGovernanceDetection: true
-#   jobs:
-#     - job: PythonLintersJob
-#       pool:
-#         vmImage: windows-latest
-#       variables:
-#         python.version: '3.6'
-#       steps:
-#       - task: Cache@2
-#         displayName: Cache pip packages
-#         continueOnError: True
-#         inputs:
-#           key: python | "$(Agent.OS)"
-#           path: $(PIP_CACHE_DIR)
-#           restoreKeys: python
-#       - script: |
-#           python -m pip install --upgrade pip wheel setuptools==56.0.0
-#           python -m pip install -r requirements-all.txt
-#           python -m pip install -e .
-#         displayName: 'Install package and dependencies'
-#       - script: |
-#           python -m pip install --upgrade pylint pylint_junit
-#           pylint msticpy -j 0 --disable=duplicate-code --disable=E1135,E1101,E1133 --output-format=pylint_junit.JUnitReporter
-#         displayName: Pylint
-#         continueOnError: true
-#         condition: succeededOrFailed()
-#       - script: |
-#           python -m pip install --upgrade black
-#           black --diff --check --exclude venv .
-#         displayName: Black
-#         continueOnError: true
-#         condition: succeededOrFailed()
-#       - script: |
-#           python -m pip install --upgrade bandit
-#           bandit -x tests -r -s B303,B404,B603,B607 msticpy
-#         displayName: bandit
-#         continueOnError: true
-#         condition: succeededOrFailed()
-#       - script: |
-#           python -m pip install --upgrade flake8
-#           flake8 --max-line-length=90 --exclude=tests* . --ignore=E501,W503 --jobs=auto
-#         displayName: flake8
-#         continueOnError: true
-#         condition: succeededOrFailed()
-#       - script: |
-#           python -m pip install mypy lxml types-attrs
-#           mypy --ignore-missing-imports --follow-imports=silent --show-column-numbers --junit-xml junit/mypy-test-results.xml msticpy
-#         displayName: mypy
-#         continueOnError: true
-#         condition: succeededOrFailed()
-#         env:
-#           tools_dir: ${AGENT_TOOLSDIRECTORY}
-#       - script: |
-#           python -m pip install prospector[with_pyroma]
-#           prospector --ignore-paths tests --without-tool pylint
-#         displayName: prospector
-#         continueOnError: true
-#         condition: succeededOrFailed()
-#         env:
-#           tools_dir: ${AGENT_TOOLSDIRECTORY}
-#       - task: PublishTestResults@2
-#         displayName: 'Publish Test Results'
-#         inputs:
-#           testResultsFiles: '**/*-results.xml'
-#           testRunTitle: 'Python $(python.version) $(variables.imageName)'
 - stage: SecurityChecks
   # Pipeline
   displayName: Azure pipeline, security and component governance tools

conda/conda-reqs-dev.txt

@@ -15,7 +15,6 @@ pandas>=1.4.0
 pep8-naming>=0.10.0
 pep8>=1.7.1
 pipreqs>=0.4.9
-prospector>=1.3.1
 pycodestyle>=2.6.0
 pydocstyle>=6.0.0
 pyflakes>=2.2.0

msticpy/common/exceptions.py

@@ -26,15 +26,15 @@ def _get_config(setting_path: str):
 
 
 # Standard exception types
-class MsticpyException(Exception):
+class MsticpyException(Exception):  # noqa: N818
     """Default exception class for msticpy."""
 
 
-class MsticpyConfigException(MsticpyException):
+class MsticpyConfigError(MsticpyException):
     """Configuration exception class for msticpy."""
 
 
-class MsticpyResourceException(MsticpyException):
+class MsticpyResourceError(MsticpyException):
     """Exception class for resource errors."""
 
 

msticpy/config/mp_config_control.py

@@ -211,7 +211,6 @@ def _create_ctrl_dict(self, config_dict):
         """Create a blank control dictionary from settings."""
         ctrl_dict = config_dict.copy()
         for name, val in ctrl_dict.items():
-
             if (
                 isinstance(val, dict)
                 and STORE_KEYVAULT not in val

msticpy/context/azure/azure_data.py

@@ -27,7 +27,7 @@
     MsticpyAzureConfigError,
     MsticpyImportExtraError,
     MsticpyNotConnectedError,
-    MsticpyResourceException,
+    MsticpyResourceError,
 )
 
 try:
@@ -547,7 +547,7 @@ def _get_api(  # noqa: MC0001
                 namespace = resource_id.split("/")[6]
                 service = resource_id.split("/")[7]
             except IndexError as idx_err:
-                raise MsticpyResourceException(
+                raise MsticpyResourceError(
                     "Provided Resource ID isn't in the correct format.",
                     "It should look like:",
                     "/subscriptions/SUB_ID/resourceGroups/RESOURCE_GROUP/"
@@ -559,7 +559,7 @@ def _get_api(  # noqa: MC0001
                 namespace = resource_provider.split("/")[0]
                 service = resource_provider.split("/")[1]
             except IndexError as idx_err:
-                raise MsticpyResourceException(
+                raise MsticpyResourceError(
                     "Provided Resource Provider isn't in the correct format.",
                     "It should look like: NAMESPACE/SERVICE_NAME",
                 ) from idx_err
@@ -586,7 +586,7 @@ def _get_api(  # noqa: MC0001
 
         # Get first API version that isn't in preview
         if not resource_types:
-            raise MsticpyResourceException("Resource provider not found")
+            raise MsticpyResourceError("Resource provider not found")
 
         api_version = [
             v

msticpy/context/geoip.py

@@ -50,7 +50,7 @@
 __author__ = "Ian Hellen"
 
 
-class GeoIPDatabaseException(Exception):
+class GeoIPDatabaseError(Exception):
     """Exception when GeoIP database cannot be found."""
 
 

msticpy/context/http_provider.py

@@ -22,7 +22,7 @@
 from attr import Factory
 
 from .._version import VERSION
-from ..common.exceptions import MsticpyConfigException
+from ..common.exceptions import MsticpyConfigError
 from ..common.pkg_config import get_http_timeout
 from ..common.utility import mp_ua_header
 from .lookup_result import LookupStatus
@@ -149,7 +149,7 @@ def __init__(self, **kwargs):
 
         if missing_params:
             param_list = ", ".join(f"'{param}'" for param in missing_params)
-            raise MsticpyConfigException(
+            raise MsticpyConfigError(
                 f"Parameter values missing for Provider '{self.__class__.__name__}'",
                 f"Missing parameters are: {param_list}",
             )

msticpy/context/ip_utils.py

@@ -291,7 +291,7 @@ def get_whois_info(
             if show_progress:
                 print(".", end="")
             return whois_result  # type: ignore
-        except (MsticpyException) as err:
+        except MsticpyException as err:
             return f"Error during lookup of {ip_str} {type(err)}", {}
     return f"No ASN Information for IP type: {ip_type}", {}
 

msticpy/context/lookup.py

@@ -23,7 +23,7 @@
 from tqdm.auto import tqdm
 
 from .._version import VERSION
-from ..common.exceptions import MsticpyConfigException, MsticpyUserConfigError
+from ..common.exceptions import MsticpyConfigError, MsticpyUserConfigError
 from ..common.provider_settings import get_provider_settings, reload_settings
 from ..common.utility import export, is_ipython
 from ..vis.ti_browser import browse_results
@@ -97,7 +97,6 @@ def __init__(self, providers: Optional[List[str]] = None, **kwargs):
 
         primary_providers = kwargs.pop("primary_providers", None)
         if primary_providers:
-
             for prov in primary_providers:
                 self.add_provider(prov, primary=True)
         secondary_providers = kwargs.pop("secondary_providers", None)
@@ -687,7 +686,7 @@ def _load_providers(self, **kwargs):
             # instantiate class sending args from settings to init
             try:
                 provider_instance = provider_class(**(settings.args))
-            except MsticpyConfigException as mp_ex:
+            except MsticpyConfigError as mp_ex:
                 # If the TI Provider didn't load, raise an exception
                 raise MsticpyUserConfigError(
                     f"Could not load Provider {provider_name}",

msticpy/context/tiproviders/kql_base.py

@@ -23,7 +23,7 @@
 import pandas as pd
 
 from ..._version import VERSION
-from ...common.exceptions import MsticpyConfigException
+from ...common.exceptions import MsticpyConfigError
 from ...common.utility import export
 from ...common.wsconfig import WorkspaceConfig
 from ...data import QueryProvider
@@ -62,7 +62,7 @@ def __init__(self, **kwargs):
             )
 
         if not self._query_provider:
-            raise MsticpyConfigException("Query provider for KQL could not be created.")
+            raise MsticpyConfigError("Query provider for KQL could not be created.")
 
     @property
     def _connected(self):
@@ -321,7 +321,6 @@ def _get_query_and_params(
         query_type: str = None,
         **kwargs,
     ) -> Tuple[Callable, Dict[str, Any]]:
-
         ioc_key = f"{ioc_type}-{query_type}" if query_type else ioc_type
         query_def = self._QUERIES.get(ioc_key, None)
         if not query_def: