Merge pull request #171 from DomainTools/2.5

Release v.2.5

Author: jbabac

.github/workflows/test-build-publish.yml

@@ -29,6 +29,38 @@ jobs:
         export TOX_SKIP_MISSING_INTERPRETERS="False";
         tox -e py
 
+  e2e-tests:
+    runs-on: ubuntu-latest
+    env:
+      MITM_BASIC_AUTH_CONTAINER_NAME: e2e_test_mitm_basic_auth
+      MITM_CUSTOM_CERT_CONTAINER_NAME: e2e_test_mitm_custom_cert
+      DOCKER_NETWORK_NAME: e2e_test_docker_network
+      TEST_USER: integrations_testing
+      TEST_KEY: ${{ secrets.TEST_KEY }}
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e .
+          pip install -r ./requirements/development.txt
+
+      - name: Setup E2E environment
+        run: |
+          sh ./tests/e2e/scripts/setup_e2e.sh
+
+      - name: Run E2E tests
+        run: |
+          python -m pytest -s --capture=sys -v --cov=domaintools tests/e2e
+
+      - name: Cleanup E2E environment
+        if: '!cancelled()'
+        run: |
+          sh ./tests/e2e/scripts/cleanup_e2e.sh
+
   # run only in main and in pull request to `main` and in publish release
   release-build:
     if: |

README.md

@@ -270,4 +270,44 @@ results = api.nod(sessionID="my-session-id", after=-7200)
 
 for partial_result in results.response() # generator that holds NOD feeds data for the past 2 hours and is expected to request multiple times
     # do things to partial_result
-```
+```
+
+
+Running E2E Tests Locally
+===================
+For now, e2e tests only covers proxy and ssl testing. We are expected to broaden our e2e tests to other scenarios moving forward.
+To add more e2e tests, put these in the `../tests/e2e` folder.
+
+## Preparation
+- Create virtual environment.
+    ```bash
+        python3 -m venv venv
+    ```
+
+- Activate virtual environment
+    ```bash
+        source venv/bin/activate
+    ```
+
+- Install dependencies.
+    ```bash
+        pip install -r requirements/development.txt
+    ```
+
+- From the python_api project root directory, install the package.
+    ```bash
+        pip install -e .
+    ```
+
+- Export api credentials to use.
+    ```bash
+        export TEST_USER=<user-key>
+        export TEST_KEY=<api-key>
+    ```
+
+## Run the end-to-end test script
+- Before running the test, be sure that docker is running.
+- Execute the e2e test script .
+    ```bash
+        sh tests/e2e/scripts/test_e2e_runner.sh
+    ```

VERSION

@@ -1 +1 @@
-2.4.1
+2.5.0

domaintools/_version.py

@@ -20,4 +20,4 @@
 
 """
 
-current = "2.4.1"
+current = "2.5.0"

domaintools/api.py

@@ -1,8 +1,10 @@
 from datetime import datetime, timedelta, timezone
 from hashlib import sha1, sha256
 from hmac import new as hmac
+from typing import Union
 
 import re
+import ssl
 
 from domaintools.constants import Endpoint, ENDPOINT_TO_SOURCE_MAP, FEEDS_PRODUCTS_LIST, OutputFormat
 from domaintools._version import current as version
@@ -76,7 +78,7 @@ def __init__(
         self.username = username
         self.key = key
         self.https = https
-        self.verify_ssl = verify_ssl
+        self.verify_ssl = self._get_ssl_default_context(verify_ssl)
         self.rate_limit = rate_limit
         self.proxy_url = proxy_url
         self.extra_request_params = {}
@@ -92,6 +94,9 @@ def __init__(
         if proxy_url and not isinstance(proxy_url, str):
             raise Exception("Proxy URL must be a string. For example: '127.0.0.1:8888'")
 
+    def _get_ssl_default_context(self, verify_ssl: Union[str, bool]):
+        return ssl.create_default_context(cafile=verify_ssl) if isinstance(verify_ssl, str) else verify_ssl
+
     def _build_api_url(self, api_url=None, api_port=None):
         """Build the API url based on the given url and port. Defaults to `https://api.domaintools.com`"""
         rest_api_url = "https://api.domaintools.com"
@@ -1187,3 +1192,65 @@ def noh(self, **kwargs) -> FeedsResults:
             cls=FeedsResults,
             **kwargs,
         )
+
+    def realtime_domain_risk(self, **kwargs) -> FeedsResults:
+        """Returns back list of the realtime domain risk feed.
+        Contains realtime domain risk information for apex-level domains, regardless of observed traffic.
+
+        domain: str: Filter for an exact domain or a substring contained within a domain by prefixing or suffixing your substring with "*". Check the documentation for examples
+
+        before: str: Filter for records before the given time value inclusive or time offset relative to now
+
+        after: str: Filter for records after the given time value inclusive or time offset relative to now
+
+        headers: bool: Use in combination with Accept: text/csv headers to control if headers are sent or not
+
+        sessionID: str: A custom string to distinguish between different sessions
+
+        top: int: Limit the number of results to the top N, where N is the value of this parameter.
+        """
+        validate_feeds_parameters(kwargs)
+        endpoint = kwargs.pop("endpoint", Endpoint.FEED.value)
+        source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value
+        if endpoint == Endpoint.DOWNLOAD.value or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value:
+            # headers param is allowed only in Feed API and CSV format
+            kwargs.pop("headers", None)
+
+        return self._results(
+            f"domain-risk-({source})",
+            f"v1/{endpoint}/domainrisk/",
+            response_path=(),
+            cls=FeedsResults,
+            **kwargs,
+        )
+
+    def domainhotlist(self, **kwargs) -> FeedsResults:
+        """Returns back list of domain hotlist feed.
+        Contains high-risk, apex-level domains that are observed by DomainTools' global sensor network to be active within 24 hours.
+
+        domain: str: Filter for an exact domain or a substring contained within a domain by prefixing or suffixing your substring with "*". Check the documentation for examples
+
+        before: str: Filter for records before the given time value inclusive or time offset relative to now
+
+        after: str: Filter for records after the given time value inclusive or time offset relative to now
+
+        headers: bool: Use in combination with Accept: text/csv headers to control if headers are sent or not
+
+        sessionID: str: A custom string to distinguish between different sessions
+
+        top: int: Limit the number of results to the top N, where N is the value of this parameter.
+        """
+        validate_feeds_parameters(kwargs)
+        endpoint = kwargs.pop("endpoint", Endpoint.FEED.value)
+        source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value
+        if endpoint == Endpoint.DOWNLOAD.value or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value:
+            # headers param is allowed only in Feed API and CSV format
+            kwargs.pop("headers", None)
+
+        return self._results(
+            f"domain-hotlist-feed-({source})",
+            f"v1/{endpoint}/domainhotlist/",
+            response_path=(),
+            cls=FeedsResults,
+            **kwargs,
+        )

domaintools/cli/commands/feeds.py

@@ -394,3 +394,163 @@ def feeds_noh(
     ),
 ):
     DTCLICommand.run(name=c.FEEDS_NOH, params=ctx.params)
+
+
+@dt_cli.command(
+    name=c.FEEDS_DOMAINHOTLIST,
+    help=get_cli_helptext_by_name(command_name=c.FEEDS_DOMAINHOTLIST),
+)
+def feeds_domainhotlist(
+    ctx: typer.Context,
+    user: str = typer.Option(None, "-u", "--user", help="Domaintools API Username."),
+    key: str = typer.Option(None, "-k", "--key", help="DomainTools API key"),
+    creds_file: str = typer.Option(
+        "~/.dtapi",
+        "-c",
+        "--credfile",
+        help="Optional file with API username and API key, one per line.",
+    ),
+    no_verify_ssl: bool = typer.Option(
+        False,
+        "--no-verify-ssl",
+        help="Skip verification of SSL certificate when making HTTPs API calls",
+    ),
+    no_sign_api_key: bool = typer.Option(
+        False,
+        "--no-sign-api-key",
+        help="Skip signing of api key",
+    ),
+    header_authentication: bool = typer.Option(
+        True,
+        "--no-header-auth",
+        help="Don't use header authentication",
+    ),
+    output_format: str = typer.Option(
+        "jsonl",
+        "-f",
+        "--format",
+        help=f"Output format in [{OutputFormat.JSONL.value}, {OutputFormat.CSV.value}]",
+        callback=DTCLICommand.validate_feeds_format_input,
+    ),
+    endpoint: str = typer.Option(
+        Endpoint.FEED.value,
+        "-e",
+        "--endpoint",
+        help=f"Valid endpoints: [{Endpoint.FEED.value}, {Endpoint.DOWNLOAD.value}]",
+        callback=DTCLICommand.validate_endpoint_input,
+    ),
+    sessionID: str = typer.Option(
+        None,
+        "--session-id",
+        help="Unique identifier for the session",
+    ),
+    after: str = typer.Option(
+        None,
+        "--after",
+        help="Start of the time window, relative to the current time in seconds, for which data will be provided",
+        callback=DTCLICommand.validate_after_or_before_input,
+    ),
+    before: str = typer.Option(
+        None,
+        "--before",
+        help="The end of the query window in seconds, relative to the current time, inclusive",
+        callback=DTCLICommand.validate_after_or_before_input,
+    ),
+    domain: str = typer.Option(
+        None,
+        "-d",
+        "--domain",
+        help="A string value used to filter feed results",
+    ),
+    headers: bool = typer.Option(
+        False,
+        "--headers",
+        help="Adds a header to the first line of response when text/csv is set in header parameters",
+    ),
+    top: str = typer.Option(
+        None,
+        "--top",
+        help="Number of results to return in the response payload. This is ignored in download endpoint",
+    ),
+):
+    DTCLICommand.run(name=c.FEEDS_DOMAINHOTLIST, params=ctx.params)
+
+
+@dt_cli.command(
+    name=c.FEEDS_REALTIME_DOMAIN_RISK,
+    help=get_cli_helptext_by_name(command_name=c.FEEDS_REALTIME_DOMAIN_RISK),
+)
+def feeds_realtime_domain_risk(
+    ctx: typer.Context,
+    user: str = typer.Option(None, "-u", "--user", help="Domaintools API Username."),
+    key: str = typer.Option(None, "-k", "--key", help="DomainTools API key"),
+    creds_file: str = typer.Option(
+        "~/.dtapi",
+        "-c",
+        "--credfile",
+        help="Optional file with API username and API key, one per line.",
+    ),
+    no_verify_ssl: bool = typer.Option(
+        False,
+        "--no-verify-ssl",
+        help="Skip verification of SSL certificate when making HTTPs API calls",
+    ),
+    no_sign_api_key: bool = typer.Option(
+        False,
+        "--no-sign-api-key",
+        help="Skip signing of api key",
+    ),
+    header_authentication: bool = typer.Option(
+        True,
+        "--no-header-auth",
+        help="Don't use header authentication",
+    ),
+    output_format: str = typer.Option(
+        "jsonl",
+        "-f",
+        "--format",
+        help=f"Output format in [{OutputFormat.JSONL.value}, {OutputFormat.CSV.value}]",
+        callback=DTCLICommand.validate_feeds_format_input,
+    ),
+    endpoint: str = typer.Option(
+        Endpoint.FEED.value,
+        "-e",
+        "--endpoint",
+        help=f"Valid endpoints: [{Endpoint.FEED.value}, {Endpoint.DOWNLOAD.value}]",
+        callback=DTCLICommand.validate_endpoint_input,
+    ),
+    sessionID: str = typer.Option(
+        None,
+        "--session-id",
+        help="Unique identifier for the session",
+    ),
+    after: str = typer.Option(
+        None,
+        "--after",
+        help="Start of the time window, relative to the current time in seconds, for which data will be provided",
+        callback=DTCLICommand.validate_after_or_before_input,
+    ),
+    before: str = typer.Option(
+        None,
+        "--before",
+        help="The end of the query window in seconds, relative to the current time, inclusive",
+        callback=DTCLICommand.validate_after_or_before_input,
+    ),
+    domain: str = typer.Option(
+        None,
+        "-d",
+        "--domain",
+        help="A string value used to filter feed results",
+    ),
+    headers: bool = typer.Option(
+        False,
+        "--headers",
+        help="Adds a header to the first line of response when text/csv is set in header parameters",
+    ),
+    top: str = typer.Option(
+        None,
+        "--top",
+        help="Number of results to return in the response payload. This is ignored in download endpoint",
+    ),
+):
+    DTCLICommand.run(name=c.FEEDS_REALTIME_DOMAIN_RISK, params=ctx.params)

domaintools/cli/constants.py

@@ -43,5 +43,7 @@
 FEEDS_NAD = "nad"
 FEEDS_NOD = "nod"
 FEEDS_NOH = "noh"
+FEEDS_DOMAINHOTLIST = "domainhotlist"
 FEEDS_DOMAINRDAP = "domainrdap"
 FEEDS_DOMAINDISCOVERY = "domaindiscovery"
+FEEDS_REALTIME_DOMAIN_RISK = "realtime_domain_risk"

domaintools/cli/utils.py

@@ -64,8 +64,10 @@ def _iris_investigate_helptext():
     c.FEEDS_NAD: "Returns back newly active domains feed.",
     c.FEEDS_NOD: "Returns back newly observed domains feed.",
     c.FEEDS_NOH: "Returns back newly observed hosts feed.",
+    c.FEEDS_DOMAINHOTLIST: "Returns domaint hotlist feed.",
     c.FEEDS_DOMAINRDAP: "Returns changes to global domain registration information, populated by the Registration Data Access Protocol (RDAP).",
     c.FEEDS_DOMAINDISCOVERY: "Returns new domains as they are either discovered in domain registration information, observed by our global sensor network, or reported by trusted third parties.",
+    c.FEEDS_REALTIME_DOMAIN_RISK: "Returns realtime domain risk information for apex-level domains, regardless of observed traffic.",
 }
 
 

domaintools/constants.py

@@ -30,8 +30,12 @@ class OutputFormat(Enum):
     "newly-observed-domains-feed-(s3)",
     "newly-observed-hosts-feed-(api)",
     "newly-observed-hosts-feed-(s3)",
+    "domain-hotlist-feed-(api)",
+    "domain-hotlist-feed-(s3)",
     "domain-registration-data-access-protocol-feed-(api)",
     "domain-registration-data-access-protocol-feed-(s3)",
+    "domain-risk-feed-(api)",
+    "domain-risk-feed-(s3)",
     "real-time-domain-discovery-feed-(api)",
     "real-time-domain-discovery-feed-(s3)",
 ]

requirements/development.txt

@@ -7,3 +7,4 @@ ipython==8.1.1
 wheel==0.37.1
 python-coveralls==2.9.3
 vcrpy==4.1.1
+urllib3==1.26.15