IDEV-2563: Add IP Feeds to DT Python Wrapper

CHANGELOG.md

@@ -1,5 +1,68 @@
 # Changelog
 
+### 2.9.0
+- [NEW] Add support for querying real-time IP feeds (`iphotlist` and `iprisk`).
+
+### 2.8.1
+- [FIX] Update python wrapper with a patch that allows the wrapper to proceed with Iris Enrich call even if `account_information` returns a 503 error only (rate limit error).
+
+### 2.8.0
+- [NEW] Add IrisQL Support in `iris_investigate` function.
+- [NEW] Add `domain_history` command/function in the wrapper.
+
+### 2.7.4
+- [FIX] Handle validation error for none value arguments.
+
+### 2.7.3
+- [FIX] Issue on missing `risk_score` when passing it on `iris_investigate` function.
+
+### 2.7.2
+- [FIX] Issue on importing OpenAPI spec.
+
+### 2.7.0
+- [FIX] `iris_investigate` API parity updates (update help documentation)
+- [FIX] `iris_investigate` `active` should raise an error as it requires a boolean not a string
+
+### 2.6.0
+- [NEW] Implement streaming request for RTTF endpoints to handle the feeds properly and yield the feed line by line and not storing all of it in memory.
+- [UPDATE] Test cases.
+- [FIX] `available_api_calls` function for RTTF products
+
+### 2.5.3
+- [UPDATE] Add IrisQL Support in `iris_investigate` function.
+- [UPDATE] Change default Feeds authentication behavior
+
+### 2.5.2
+- [FIX] Total count bug for Iris Investigate and Iris Enrich results.
+
+### 2.5.1
+- [FIX] Fix for bug found in `realtime_domain_risk` endpoint. Changed product name from `domain-risk-<source>` to `domain-risk-feed-<source>`
+
+### 2.5.0
+- [NEW] Add support for Real Time Domain Risk Feed
+- [NEW] Add support for Domain Hotlist Feed
+- [NEW] Add e2e tests for proxy and ssl
+- [UPDATE] Integrate e2e tests in CI pipeline.
+- [FIX] Bugs found in `/domainrisk` and `/domainhotlist` endpoints
+
+### 2.4.1
+- [UPDATE] Remove support for MD5 based signing.
+
+### 2.4.0
+- [NEW] Integrate the NOH Feed to be supportable with the Python Wrapper.
+- [NEW] Improve worfklow to automate publishing the package to PyPI.
+- [UPDATE] Remove PhishEye.
+- [FIX] Improvements on help texts.
+- [FIX] Pegged httpxdependency to v.0.28.1 to prevent proxy key error.
+
+### 2.3.0
+- [NEW] Integrate the Domain RDAP Feed to be supportable with the Python Wrapper.
+- [NEW] Integrate the Domain Discovery Feed to be supportable with the Python Wrapper.
+- [UPDATE] Enhancements to RTUF endpoints to support the following: download API, header authentication, csv format.
+- [UPDATE] Processing of Iterative Response (HTTP 206) from RTUF endpoints.
+- [UPDATE] Help Text and Information Using New Documentation.
+- [FIX] Simplification of using the new `proxy` param of httpx.Client. Before we’re using proxy mounts equivalent which we used `proxies` but this was deprecated on httpx v.0.28.x and onward causing errors in the python_wrapper
+
 ### 2.0.0
 - [NEW] Modernize package - migrate package settings to pyproject.toml
 - [NEW] Migrate CLI wrapper to use `typer` library. (CLI comes now with new interface.)
@@ -11,12 +74,11 @@
     - Filtering of results based on `updated_after` field.
     - Filtering of results based on a missing field. (include_domains_with_missing_field` or `exclude_domains_with_missing_field`).
 - [NEW] Add support on removing/stripping colon in when passing a value in `--ssl_hash` in `iris_investigate` cli command.
-- [UPDATE] replace use of upcoming deprecated `datetime.uctnow()` to `timezone.utc`
+- [UPDATE] replace use of upcoming deprecated `datetime.utcnow()` to `timezone.utc`
 - [UPDATE] Improve help text in CLI commands.
 - [UPDATE] Remove `dateparser` dependency and use native python `datetime` library.
 - [FIX] Fix error in `-o` or `--out-file` parameter.
 
-
 ### 1.0.1
 
 - Adds support for the hourly query limit on the Account API endpoint

domaintools/api.py

@@ -1412,3 +1412,67 @@ def domainhotlist(self, **kwargs) -> FeedsResults:
             cls=FeedsResults,
             **kwargs,
         )
+
+    def iphotlist(self, **kwargs) -> FeedsResults:
+        """Returns back list of ip hotlist feed.
+        Captures IP addresses that meet strict criteria for both risk level and recent activity, making it ideal for immediate blocking and threat response.
+
+        before: str: Filter for records before the given time value inclusive or time offset relative to now
+
+        after: str: Filter for records after the given time value inclusive or time offset relative to now
+
+        headers: bool: Use in combination with Accept: text/csv headers to control if headers are sent or not
+
+        sessionID: str: A custom string to distinguish between different sessions
+
+        fromBeginning: bool: Requires a sessionID. When used with a new session ID, returns the first hour of data in the time window (rather than the last). Returns an error if the session ID already exists
+        """
+        validate_feeds_parameters(kwargs)
+        endpoint = kwargs.pop("endpoint", Endpoint.FEED.value)
+        source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value
+        if (
+            endpoint == Endpoint.DOWNLOAD.value
+            or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value
+        ):
+            # headers param is allowed only in Feed API and CSV format
+            kwargs.pop("headers", None)
+
+        return self._results(
+            f"real-time-ip-hotlist-({source})",
+            f"v1/{endpoint}/iphotlist/",
+            response_path=(),
+            cls=FeedsResults,
+            **kwargs,
+        )
+
+    def iprisk(self, **kwargs) -> FeedsResults:
+        """Returns back list of domain hotlist feed.
+        Captures all IP addresses that actively host one or more domains, providing risk assessment and enrichment data for each IP address.
+
+        before: str: Filter for records before the given time value inclusive or time offset relative to now
+
+        after: str: Filter for records after the given time value inclusive or time offset relative to now
+
+        headers: bool: Use in combination with Accept: text/csv headers to control if headers are sent or not
+
+        sessionID: str: A custom string to distinguish between different sessions
+
+        fromBeginning: bool: Requires a sessionID. When used with a new session ID, returns the first hour of data in the time window (rather than the last). Returns an error if the session ID already exists
+        """
+        validate_feeds_parameters(kwargs)
+        endpoint = kwargs.pop("endpoint", Endpoint.FEED.value)
+        source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value
+        if (
+            endpoint == Endpoint.DOWNLOAD.value
+            or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value
+        ):
+            # headers param is allowed only in Feed API and CSV format
+            kwargs.pop("headers", None)
+
+        return self._results(
+            f"real-time-ip-risk-({source})",
+            f"v1/{endpoint}/iprisk/",
+            response_path=(),
+            cls=FeedsResults,
+            **kwargs,
+        )

domaintools/cli/commands/feeds.py

@@ -554,3 +554,153 @@ def feeds_realtime_domain_risk(
     ),
 ):
     DTCLICommand.run(name=c.FEEDS_REALTIME_DOMAIN_RISK, params=ctx.params)
+
+
+@dt_cli.command(
+    name=c.FEEDS_IPHOTLIST,
+    help=get_cli_helptext_by_name(command_name=c.FEEDS_IPHOTLIST),
+)
+def feeds_iphotlist(
+    ctx: typer.Context,
+    user: str = typer.Option(None, "-u", "--user", help="Domaintools API Username."),
+    key: str = typer.Option(None, "-k", "--key", help="DomainTools API key"),
+    creds_file: str = typer.Option(
+        "~/.dtapi",
+        "-c",
+        "--credfile",
+        help="Optional file with API username and API key, one per line.",
+    ),
+    no_verify_ssl: bool = typer.Option(
+        False,
+        "--no-verify-ssl",
+        help="Skip verification of SSL certificate when making HTTPs API calls",
+    ),
+    no_sign_api_key: bool = typer.Option(
+        False,
+        "--no-sign-api-key",
+        help="Skip signing of api key",
+    ),
+    no_header_authentication: bool = typer.Option(
+        False,
+        "--no-header-auth",
+        help="Don't use header authentication",
+    ),
+    output_format: str = typer.Option(
+        "jsonl",
+        "-f",
+        "--format",
+        help=f"Output format in [{OutputFormat.JSONL.value}, {OutputFormat.CSV.value}]",
+        callback=DTCLICommand.validate_feeds_format_input,
+    ),
+    endpoint: str = typer.Option(
+        Endpoint.FEED.value,
+        "-e",
+        "--endpoint",
+        help=f"Valid endpoints: [{Endpoint.FEED.value}, {Endpoint.DOWNLOAD.value}]",
+        callback=DTCLICommand.validate_endpoint_input,
+    ),
+    sessionID: str = typer.Option(
+        None,
+        "--session-id",
+        help="Unique identifier for the session",
+    ),
+    after: str = typer.Option(
+        None,
+        "--after",
+        help="Start of the time window, relative to the current time in seconds, for which data will be provided",
+        callback=DTCLICommand.validate_after_or_before_input,
+    ),
+    before: str = typer.Option(
+        None,
+        "--before",
+        help="The end of the query window in seconds, relative to the current time, inclusive",
+        callback=DTCLICommand.validate_after_or_before_input,
+    ),
+    fromBeginning: bool = typer.Option(
+        None,
+        "-fb",
+        "--frombeginning",
+        help="Requires a sessionID. When used with a new session ID, returns the first hour of data in the time window (rather than the last). Returns an error if the session ID already exists",
+    ),
+    headers: bool = typer.Option(
+        False,
+        "--headers",
+        help="Adds a header to the first line of response when text/csv is set in header parameters",
+    ),
+):
+    DTCLICommand.run(name=c.FEEDS_IPHOTLIST, params=ctx.params)
+
+
+@dt_cli.command(
+    name=c.FEEDS_IPRISK,
+    help=get_cli_helptext_by_name(command_name=c.FEEDS_IPRISK),
+)
+def feeds_iprisk(
+    ctx: typer.Context,
+    user: str = typer.Option(None, "-u", "--user", help="Domaintools API Username."),
+    key: str = typer.Option(None, "-k", "--key", help="DomainTools API key"),
+    creds_file: str = typer.Option(
+        "~/.dtapi",
+        "-c",
+        "--credfile",
+        help="Optional file with API username and API key, one per line.",
+    ),
+    no_verify_ssl: bool = typer.Option(
+        False,
+        "--no-verify-ssl",
+        help="Skip verification of SSL certificate when making HTTPs API calls",
+    ),
+    no_sign_api_key: bool = typer.Option(
+        False,
+        "--no-sign-api-key",
+        help="Skip signing of api key",
+    ),
+    no_header_authentication: bool = typer.Option(
+        False,
+        "--no-header-auth",
+        help="Don't use header authentication",
+    ),
+    output_format: str = typer.Option(
+        "jsonl",
+        "-f",
+        "--format",
+        help=f"Output format in [{OutputFormat.JSONL.value}, {OutputFormat.CSV.value}]",
+        callback=DTCLICommand.validate_feeds_format_input,
+    ),
+    endpoint: str = typer.Option(
+        Endpoint.FEED.value,
+        "-e",
+        "--endpoint",
+        help=f"Valid endpoints: [{Endpoint.FEED.value}, {Endpoint.DOWNLOAD.value}]",
+        callback=DTCLICommand.validate_endpoint_input,
+    ),
+    sessionID: str = typer.Option(
+        None,
+        "--session-id",
+        help="Unique identifier for the session",
+    ),
+    after: str = typer.Option(
+        None,
+        "--after",
+        help="Start of the time window, relative to the current time in seconds, for which data will be provided",
+        callback=DTCLICommand.validate_after_or_before_input,
+    ),
+    before: str = typer.Option(
+        None,
+        "--before",
+        help="The end of the query window in seconds, relative to the current time, inclusive",
+        callback=DTCLICommand.validate_after_or_before_input,
+    ),
+    fromBeginning: bool = typer.Option(
+        None,
+        "-fb",
+        "--frombeginning",
+        help="Requires a sessionID. When used with a new session ID, returns the first hour of data in the time window (rather than the last). Returns an error if the session ID already exists",
+    ),
+    headers: bool = typer.Option(
+        False,
+        "--headers",
+        help="Adds a header to the first line of response when text/csv is set in header parameters",
+    ),
+):
+    DTCLICommand.run(name=c.FEEDS_IPRISK, params=ctx.params)

domaintools/cli/constants.py

@@ -48,3 +48,5 @@
 FEEDS_DOMAINRDAP = "domainrdap"
 FEEDS_DOMAINDISCOVERY = "domaindiscovery"
 FEEDS_REALTIME_DOMAIN_RISK = "realtime_domain_risk"
+FEEDS_IPHOTLIST = "iphotlist"
+FEEDS_IPRISK = "iprisk"

domaintools/cli/utils.py

@@ -65,10 +65,12 @@ def _iris_investigate_helptext():
     c.FEEDS_NAD: "Returns back newly active domains feed.",
     c.FEEDS_NOD: "Returns back newly observed domains feed.",
     c.FEEDS_NOH: "Returns back newly observed hosts feed.",
-    c.FEEDS_DOMAINHOTLIST: "Returns domaint hotlist feed.",
+    c.FEEDS_DOMAINHOTLIST: "Returns domain hotlist feed.",
     c.FEEDS_DOMAINRDAP: "Returns changes to global domain registration information, populated by the Registration Data Access Protocol (RDAP).",
     c.FEEDS_DOMAINDISCOVERY: "Returns new domains as they are either discovered in domain registration information, observed by our global sensor network, or reported by trusted third parties.",
     c.FEEDS_REALTIME_DOMAIN_RISK: "Returns realtime domain risk information for apex-level domains, regardless of observed traffic.",
+    c.FEEDS_IPHOTLIST: "Returns ip hotlist feed.",
+    c.FEEDS_IPRISK: "Returns ip risk feed.",
 }
 
 

domaintools/constants.py

@@ -38,6 +38,10 @@ class OutputFormat(Enum):
     "real-time-domain-risk-(s3)",
     "real-time-domain-discovery-feed-(api)",
     "real-time-domain-discovery-feed-(s3)",
+    "real-time-ip-hotlist-(api)",
+    "real-time-ip-hotlist-(s3)",
+    "real-time-ip-risk-(api)",
+    "real-time-ip-risk-(s3)",
 ]
 
 RTTF_PRODUCTS_CMD_MAPPING = {
@@ -55,6 +59,10 @@ class OutputFormat(Enum):
     "real-time-domain-risk-(s3)": "realtime_domain_risk",
     "real-time-domain-discovery-feed-(api)": "domaindiscovery",
     "real-time-domain-discovery-feed-(s3)": "domaindiscovery",
+    "real-time-ip-hotlist-(api)": "iphotlist",
+    "real-time-ip-hotlist-(s3)": "iphotlist",
+    "real-time-ip-risk-(api)": "iprisk",
+    "real-time-ip-risk-(s3)": "iprisk",
 }
 
 SPECS_MAPPING = {