diff --git a/domaintools/api.py b/domaintools/api.py index 50e034f..26433a4 100644 --- a/domaintools/api.py +++ b/domaintools/api.py @@ -3,6 +3,7 @@ from hmac import new as hmac import re +from domaintools.constants import Endpoint, ENDPOINT_TO_SOURCE_MAP, OutputFormat from domaintools._version import current as version from domaintools.results import ( GroupedIterable, @@ -18,6 +19,8 @@ filter_by_field, DTResultFilter, ) +from domaintools.utils import validate_feeds_parameters + AVAILABLE_KEY_SIGN_HASHES = ["sha1", "sha256", "md5"] @@ -1088,15 +1091,29 @@ def nad(self, **kwargs): def domainrdap(self, **kwargs): """Returns changes to global domain registration information, populated by the Registration Data Access Protocol (RDAP)""" - sessionID = kwargs.get("sessionID") - after = kwargs.get("after") - before = kwargs.get("before") - if not (sessionID or after or before): - raise ValueError("sessionID or after or before must be defined") + validate_feeds_parameters(kwargs) + endpoint = kwargs.pop("endpoint", Endpoint.FEED.value) + source = ENDPOINT_TO_SOURCE_MAP.get(endpoint) + + return self._results( + f"domain-registration-data-access-protocol-feed-({source.value})", + f"v1/{endpoint}/domainrdap/", + response_path=(), + **kwargs, + ) + + def domaindiscovery(self, **kwargs): + """Returns new domains as they are either discovered in domain registration information, observed by our global sensor network, or reported by trusted third parties""" + validate_feeds_parameters(kwargs) + endpoint = kwargs.pop("endpoint", Endpoint.FEED.value) + source = ENDPOINT_TO_SOURCE_MAP.get(endpoint) + if endpoint == Endpoint.DOWNLOAD.value or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value: + # headers param is allowed only in Feed API and CSV format + kwargs.pop("headers", None) return self._results( - "domain-registration-data-access-protocol-feed-(api)", - "v1/feed/domainrdap/", + f"real-time-domain-discovery-feed-({source.value})", + f"v1/{endpoint}/domaindiscovery/", response_path=(), **kwargs, ) diff --git a/domaintools/base_results.py b/domaintools/base_results.py index e83c390..bc8f0b0 100644 --- a/domaintools/base_results.py +++ b/domaintools/base_results.py @@ -4,8 +4,12 @@ import re import time import logging + +from copy import deepcopy from datetime import datetime +from httpx import Client +from domaintools.constants import OutputFormat, HEADER_ACCEPT_KEY_CSV_FORMAT from domaintools.exceptions import ( BadRequestException, InternalServerErrorException, @@ -18,7 +22,6 @@ ) from domaintools.utils import get_feeds_products_list -from httpx import Client try: # pragma: no cover from collections.abc import MutableMapping, MutableSequence @@ -90,6 +93,18 @@ def _make_request(self): patch_data = self.kwargs.copy() patch_data.update(self.api.extra_request_params) return session.patch(url=self.url, json=patch_data) + elif self.product in get_feeds_products_list(): + parameters = deepcopy(self.kwargs) + parameters.pop("output_format", None) + parameters.pop( + "format", None + ) # For some unknownn reasons, even if "format" is not included in the cli params for feeds endpoint, it is being populated thus we need to remove it. Happens only if using CLI. + headers = {} + if self.kwargs.get("output_format", OutputFormat.JSONL.value) == OutputFormat.CSV.value: + parameters["headers"] = int(bool(self.kwargs.get("headers", False))) + headers["accept"] = HEADER_ACCEPT_KEY_CSV_FORMAT + + return session.get(url=self.url, params=parameters, headers=headers, **self.api.extra_request_params) else: return session.get(url=self.url, params=self.kwargs, **self.api.extra_request_params) @@ -259,6 +274,32 @@ def json(self): **self.kwargs, ) + @property + def jsonl(self): + self.kwargs.pop("format", None) + return self.__class__( + format="jsonl", + product=self.product, + url=self.url, + items_path=self.items_path, + response_path=self.response_path, + api=self.api, + **self.kwargs, + ) + + @property + def csv(self): + self.kwargs.pop("format", None) + return self.__class__( + format="csv", + product=self.product, + url=self.url, + items_path=self.items_path, + response_path=self.response_path, + api=self.api, + **self.kwargs, + ) + @property def xml(self): self.kwargs.pop("format", None) diff --git a/domaintools/cli/api.py b/domaintools/cli/api.py index c713116..a1c08a9 100644 --- a/domaintools/cli/api.py +++ b/domaintools/cli/api.py @@ -8,6 +8,7 @@ from typing import Optional, Dict, Tuple from rich.progress import Progress, SpinnerColumn, TextColumn +from domaintools.constants import Endpoint, OutputFormat from domaintools.api import API from domaintools.exceptions import ServiceException from domaintools.cli.utils import get_file_extension @@ -32,6 +33,20 @@ def validate_format_input(value: str): raise typer.BadParameter(f"{value} is not in available formats: {VALID_FORMATS}") return value + @staticmethod + def validate_feeds_format_input(value: str): + VALID_FEEDS_FORMATS = ("jsonl", "csv") + if value not in VALID_FEEDS_FORMATS: + raise typer.BadParameter(f"{value} is not in available formats: {VALID_FEEDS_FORMATS}") + return value + + @staticmethod + def validate_endpoint_input(value: str): + VALID_ENDPOINTS = (Endpoint.FEED.value, Endpoint.DOWNLOAD.value) + if value not in VALID_ENDPOINTS: + raise typer.BadParameter(f"{value} is not in available endpoints: {VALID_ENDPOINTS}") + return value + @staticmethod def validate_after_or_before_input(value: str): if value is None or value.replace("-", "").isdigit(): @@ -152,7 +167,13 @@ def run(cls, name: str, params: Optional[Dict] = {}, **kwargs): """ try: rate_limit = params.pop("rate_limit", False) - response_format = params.pop("format", "json") + response_format = ( + params.pop("format", "json") + if params.get("format", None) + else params.get( + "output_format", OutputFormat.JSONL.value + ) # Using output_format for RTUF endpoints to separate from other endpoints. This will be needed further along the process + ) out_file = params.pop("out_file", sys.stdout) verify_ssl = params.pop("no_verify_ssl", False) always_sign_api_key = params.pop("no_sign_api_key", False) diff --git a/domaintools/cli/commands/feeds.py b/domaintools/cli/commands/feeds.py index 230446a..5f5be45 100644 --- a/domaintools/cli/commands/feeds.py +++ b/domaintools/cli/commands/feeds.py @@ -6,6 +6,7 @@ from domaintools.cli.api import DTCLICommand from domaintools.cli.utils import get_cli_helptext_by_name from domaintools.cli import constants as c +from domaintools.constants import Endpoint, OutputFormat @dt_cli.command( @@ -158,6 +159,13 @@ def feeds_domainrdap( "--no-sign-api-key", help="Skip signing of api key", ), + endpoint: str = typer.Option( + Endpoint.FEED.value, + "-e", + "--endpoint", + help=f"Valid endpoints: [{Endpoint.FEED.value}, {Endpoint.DOWNLOAD.value}]", + callback=DTCLICommand.validate_endpoint_input, + ), sessionID: str = typer.Option( None, "--session-id", @@ -188,3 +196,78 @@ def feeds_domainrdap( ), ): DTCLICommand.run(name=c.FEEDS_DOMAINRDAP, params=ctx.params) + + +@dt_cli.command( + name=c.FEEDS_DOMAINDISCOVERY, + help=get_cli_helptext_by_name(command_name=c.FEEDS_DOMAINDISCOVERY), +) +def feeds_domaindiscovery( + ctx: typer.Context, + user: str = typer.Option(None, "-u", "--user", help="Domaintools API Username."), + key: str = typer.Option(None, "-k", "--key", help="DomainTools API key"), + creds_file: str = typer.Option( + "~/.dtapi", + "-c", + "--credfile", + help="Optional file with API username and API key, one per line.", + ), + no_verify_ssl: bool = typer.Option( + False, + "--no-verify-ssl", + help="Skip verification of SSL certificate when making HTTPs API calls", + ), + no_sign_api_key: bool = typer.Option( + False, + "--no-sign-api-key", + help="Skip signing of api key", + ), + output_format: str = typer.Option( + "jsonl", + "-f", + "--format", + help=f"Output format in [{OutputFormat.JSONL.value}, {OutputFormat.CSV.value}]", + callback=DTCLICommand.validate_feeds_format_input, + ), + endpoint: str = typer.Option( + Endpoint.FEED.value, + "-e", + "--endpoint", + help=f"Valid endpoints: [{Endpoint.FEED.value}, {Endpoint.DOWNLOAD.value}]", + callback=DTCLICommand.validate_endpoint_input, + ), + sessionID: str = typer.Option( + None, + "--session-id", + help="Unique identifier for the session", + ), + after: str = typer.Option( + None, + "--after", + help="Start of the time window, relative to the current time in seconds, for which data will be provided", + callback=DTCLICommand.validate_after_or_before_input, + ), + before: str = typer.Option( + None, + "--before", + help="The end of the query window in seconds, relative to the current time, inclusive", + callback=DTCLICommand.validate_after_or_before_input, + ), + domain: str = typer.Option( + None, + "-d", + "--domain", + help="A string value used to filter feed results", + ), + headers: bool = typer.Option( + False, + "--headers", + help="Adds a header to the first line of response when text/csv is set in header parameters", + ), + top: str = typer.Option( + None, + "--top", + help="Number of results to return in the response payload. This is ignored in download endpoint", + ), +): + DTCLICommand.run(name=c.FEEDS_DOMAINDISCOVERY, params=ctx.params) diff --git a/domaintools/cli/constants.py b/domaintools/cli/constants.py index 1162764..636fb6f 100644 --- a/domaintools/cli/constants.py +++ b/domaintools/cli/constants.py @@ -47,3 +47,4 @@ FEEDS_NAD = "nad" FEEDS_NOD = "nod" FEEDS_DOMAINRDAP = "domainrdap" +FEEDS_DOMAINDISCOVERY = "domaindiscovery" diff --git a/domaintools/constants.py b/domaintools/constants.py new file mode 100644 index 0000000..829229a --- /dev/null +++ b/domaintools/constants.py @@ -0,0 +1,24 @@ +from enum import Enum + + +class Endpoint(Enum): + FEED = "feed" + DOWNLOAD = "download" + + +class Source(Enum): + API = "api" + S3 = "s3" + + +class OutputFormat(Enum): + JSONL = "jsonl" + CSV = "csv" + + +HEADER_ACCEPT_KEY_CSV_FORMAT = "text/csv" + +ENDPOINT_TO_SOURCE_MAP = { + Endpoint.FEED.value: Source.API, + Endpoint.DOWNLOAD.value: Source.S3, +} diff --git a/domaintools/utils.py b/domaintools/utils.py index 01c9aa1..9ee38c3 100644 --- a/domaintools/utils.py +++ b/domaintools/utils.py @@ -1,7 +1,8 @@ from datetime import datetime - from typing import Optional +from domaintools.constants import Endpoint, OutputFormat + import re @@ -176,4 +177,19 @@ def get_feeds_products_list(): "newly-active-domains-feed-(api)", "newly-observed-domains-feed-(api)", "domain-registration-data-access-protocol-feed-(api)", + "domain-registration-data-access-protocol-feed-(s3)", + "real-time-domain-discovery-feed-(api)", + "real-time-domain-discovery-feed-(s3)", ] + + +def validate_feeds_parameters(params): + sessionID = params.get("sessionID") + after = params.get("after") + before = params.get("before") + if not (sessionID or after or before): + raise ValueError("sessionID or after or before must be defined") + + format = params.get("output_format") + if params.get("endpoint") == Endpoint.DOWNLOAD.value and format == OutputFormat.CSV.value: + raise ValueError(f"{format} format is not available in {Endpoint.DOWNLOAD.value} API.") diff --git a/domaintools_async/__init__.py b/domaintools_async/__init__.py index d852457..90075ea 100644 --- a/domaintools_async/__init__.py +++ b/domaintools_async/__init__.py @@ -1,11 +1,14 @@ """Adds async capabilities to the base product object""" import asyncio + +from copy import deepcopy from httpx import AsyncClient from domaintools.base_results import Results - -from domaintools.exceptions import ServiceUnavailableException, ServiceException +from domaintools.constants import OutputFormat, HEADER_ACCEPT_KEY_CSV_FORMAT +from domaintools.exceptions import ServiceUnavailableException +from domaintools.utils import get_feeds_products_list class _AIter(object): @@ -49,6 +52,17 @@ async def _make_async_request(self, session): patch_data = self.kwargs.copy() patch_data.update(self.api.extra_request_params) results = await session.patch(url=self.url, json=patch_data) + elif self.product in get_feeds_products_list(): + parameters = deepcopy(self.kwargs) + parameters.pop("output_format", None) + parameters.pop( + "format", None + ) # For some unknownn reasons, even if "format" is not included in the cli params for feeds endpoint, it is being populated thus we need to remove it. Happens only if using CLI. + headers = {} + if self.kwargs.get("output_format", OutputFormat.JSONL.value) == OutputFormat.CSV.value: + parameters["headers"] = int(bool(self.kwargs.get("headers", False))) + headers["accept"] = HEADER_ACCEPT_KEY_CSV_FORMAT + results = await session.get(url=self.url, params=parameters, headers=headers, **self.api.extra_request_params) else: results = await session.get(url=self.url, params=self.kwargs, **self.api.extra_request_params) if results: diff --git a/tests/fixtures/vcr/test_domain_discovery_feed.yaml b/tests/fixtures/vcr/test_domain_discovery_feed.yaml new file mode 100644 index 0000000..9ad2181 --- /dev/null +++ b/tests/fixtures/vcr/test_domain_discovery_feed.yaml @@ -0,0 +1,342 @@ +interactions: +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + host: + - api.domaintools.com + user-agent: + - python-httpx/0.28.1 + method: GET + uri: https://api.domaintools.com/v1/feed/domaindiscovery/?after=-60&app_name=python_wrapper&app_version=2.2.0 + response: + body: + string: '{"timestamp":"2025-01-24T15:47:29Z","domain":"3lbrasil.com.br"} + + {"timestamp":"2025-01-24T15:47:29Z","domain":"webmeet-google.com"} + + {"timestamp":"2025-01-24T15:47:29Z","domain":"lightschain.club"} + + {"timestamp":"2025-01-24T15:47:29Z","domain":"thinkricch.com"} + + {"timestamp":"2025-01-24T15:47:29Z","domain":"ecodisolos.it"} + + {"timestamp":"2025-01-24T15:47:29Z","domain":"centralexclusivacliente.com"} + + {"timestamp":"2025-01-24T15:47:29Z","domain":"protextify.com"} + + {"timestamp":"2025-01-24T15:47:29Z","domain":"uppt.com"} + + {"timestamp":"2025-01-24T15:47:30Z","domain":"jrlittlebaby.us"} + + {"timestamp":"2025-01-24T15:47:30Z","domain":"wwwrvu.com"} + + {"timestamp":"2025-01-24T15:47:30Z","domain":"especialistaenpensionesacevedo.com"} + + {"timestamp":"2025-01-24T15:47:30Z","domain":"zidyn.com"} + + {"timestamp":"2025-01-24T15:47:31Z","domain":"ph-electronicsandaccessories.com"} + + {"timestamp":"2025-01-24T15:47:31Z","domain":"darosadvogado.com.br"} + + {"timestamp":"2025-01-24T15:47:31Z","domain":"canprovideunique.com"} + + {"timestamp":"2025-01-24T15:47:31Z","domain":"xz7058.cc"} + + {"timestamp":"2025-01-24T15:47:31Z","domain":"logicservices.fr"} + + {"timestamp":"2025-01-24T15:47:31Z","domain":"authorandcompany.co"} + + {"timestamp":"2025-01-24T15:47:31Z","domain":"illuminateimpact.org"} + + {"timestamp":"2025-01-24T15:47:32Z","domain":"chopcentral.shop"} + + {"timestamp":"2025-01-24T15:47:32Z","domain":"online-advertising-58183.bond"} + + {"timestamp":"2025-01-24T15:47:32Z","domain":"xilftentv.com"} + + {"timestamp":"2025-01-24T15:47:32Z","domain":"netresultspickleball.com"} + + {"timestamp":"2025-01-24T15:47:32Z","domain":"vroomthreads.de"} + + {"timestamp":"2025-01-24T15:47:33Z","domain":"geavers.us"} + + {"timestamp":"2025-01-24T15:47:33Z","domain":"basefamilyoffice.com.br"} + + {"timestamp":"2025-01-24T15:47:33Z","domain":"ms-o365.com"} + + {"timestamp":"2025-01-24T15:47:33Z","domain":"teresamarietc.com"} + + {"timestamp":"2025-01-24T15:47:34Z","domain":"tripodtravelers.com"} + + {"timestamp":"2025-01-24T15:47:34Z","domain":"bheos.club"} + + {"timestamp":"2025-01-24T15:47:35Z","domain":"cocacann.pl"} + + {"timestamp":"2025-01-24T15:47:35Z","domain":"butik-online.com"} + + {"timestamp":"2025-01-24T15:47:36Z","domain":"libanoconstrutora.com.br"} + + {"timestamp":"2025-01-24T15:47:36Z","domain":"xz9674.cc"} + + {"timestamp":"2025-01-24T15:47:36Z","domain":"dmitriybull.com"} + + {"timestamp":"2025-01-24T15:47:36Z","domain":"new-construction-south-austin243mo.de"} + + {"timestamp":"2025-01-24T15:47:36Z","domain":"vyvyd.co"} + + {"timestamp":"2025-01-24T15:47:37Z","domain":"chathamsprinklers.com"} + + {"timestamp":"2025-01-24T15:47:38Z","domain":"musikwerkstatt-dressler.de"} + + {"timestamp":"2025-01-24T15:47:38Z","domain":"sacredsite-uranai.com"} + + {"timestamp":"2025-01-24T15:47:38Z","domain":"saglampays.com"} + + {"timestamp":"2025-01-24T15:47:38Z","domain":"sagalrealstate.com"} + + {"timestamp":"2025-01-24T15:47:38Z","domain":"warehouse-services-53214.bond"} + + {"timestamp":"2025-01-24T15:47:38Z","domain":"b-br-bradesconetacessoempresas.com"} + + {"timestamp":"2025-01-24T15:47:38Z","domain":"hg9300rr.cc"} + + {"timestamp":"2025-01-24T15:47:38Z","domain":"saglamepara.com"} + + {"timestamp":"2025-01-24T15:47:38Z","domain":"ada4u6.shop"} + + {"timestamp":"2025-01-24T15:47:38Z","domain":"admitech.com"} + + {"timestamp":"2025-01-24T15:47:38Z","domain":"aetherialbuilds.com"} + + {"timestamp":"2025-01-24T15:47:38Z","domain":"hg9300gg.cc"} + + {"timestamp":"2025-01-24T15:47:38Z","domain":"xn--zufalsmehrheiten-ruc.de"} + + {"timestamp":"2025-01-24T15:47:38Z","domain":"safezoneaitech.com"} + + {"timestamp":"2025-01-24T15:47:38Z","domain":"thecourtsociety.net"} + + {"timestamp":"2025-01-24T15:47:38Z","domain":"authorcompany.co"} + + {"timestamp":"2025-01-24T15:47:38Z","domain":"aliexprsrcoin.lol"} + + {"timestamp":"2025-01-24T15:47:38Z","domain":"setbrasill.com.br"} + + {"timestamp":"2025-01-24T15:47:38Z","domain":"doublegvital.com"} + + {"timestamp":"2025-01-24T15:47:38Z","domain":"tele-branchenportal.de"} + + {"timestamp":"2025-01-24T15:47:38Z","domain":"cimbribested.shop"} + + {"timestamp":"2025-01-24T15:47:38Z","domain":"nickielorum.shop"} + + {"timestamp":"2025-01-24T15:47:38Z","domain":"cafedazizi.com.br"} + + {"timestamp":"2025-01-24T15:47:39Z","domain":"promluxurywears.com"} + + {"timestamp":"2025-01-24T15:47:39Z","domain":"xn--zufalsmehrheit-4jc.de"} + + {"timestamp":"2025-01-24T15:47:40Z","domain":"safwaadmin.com"} + + {"timestamp":"2025-01-24T15:47:40Z","domain":"crownglow.se"} + + {"timestamp":"2025-01-24T15:47:41Z","domain":"73637.plus"} + + {"timestamp":"2025-01-24T15:47:42Z","domain":"bullypettides.com"} + + {"timestamp":"2025-01-24T15:47:44Z","domain":"clickwellpathway.com"} + + {"timestamp":"2025-01-24T15:47:44Z","domain":"sunwiredeurope.com"} + + {"timestamp":"2025-01-24T15:47:45Z","domain":"mariaeduardavinicius.com.br"} + + {"timestamp":"2025-01-24T15:47:45Z","domain":"saffronwaldenroofingexperts.com"} + + {"timestamp":"2025-01-24T15:47:45Z","domain":"hausverkauf-in-heidelberg.de"} + + {"timestamp":"2025-01-24T15:47:46Z","domain":"steel-metal-roofing-repair-pro.live"} + + {"timestamp":"2025-01-24T15:47:46Z","domain":"camsc.top"} + + {"timestamp":"2025-01-24T15:47:47Z","domain":"stagecoach-buses.com"} + + {"timestamp":"2025-01-24T15:47:47Z","domain":"infonexus.com.ng"} + + {"timestamp":"2025-01-24T15:47:47Z","domain":"software-de-gestion-de-leads-de-ventas43.de"} + + {"timestamp":"2025-01-24T15:47:56Z","domain":"buergerportal-braunschweig.de"} + + {"timestamp":"2025-01-24T15:48:00Z","domain":"letreiroacm.com.br"} + + {"timestamp":"2025-01-24T15:48:02Z","domain":"sagenio.com"} + + {"timestamp":"2025-01-24T15:48:02Z","domain":"fireme.to"} + + {"timestamp":"2025-01-24T15:48:03Z","domain":"accessnextperimeter.com"} + + {"timestamp":"2025-01-24T15:48:04Z","domain":"eternalmind.org"} + + {"timestamp":"2025-01-24T15:48:04Z","domain":"climproff.ru"} + + {"timestamp":"2025-01-24T15:48:04Z","domain":"dotnet.ltd"} + + {"timestamp":"2025-01-24T15:48:05Z","domain":"moneycat.xyz"} + + {"timestamp":"2025-01-24T15:48:05Z","domain":"luftreinhaltungssysteme664562.icu"} + + {"timestamp":"2025-01-24T15:48:05Z","domain":"customfactory.shop"} + + {"timestamp":"2025-01-24T15:48:05Z","domain":"26375.plus"} + + {"timestamp":"2025-01-24T15:48:05Z","domain":"craonfuneraire.fr"} + + {"timestamp":"2025-01-24T15:48:05Z","domain":"57623.plus"} + + {"timestamp":"2025-01-24T15:48:05Z","domain":"otorrinoalbacete.com"} + + {"timestamp":"2025-01-24T15:48:06Z","domain":"kurumsalio.com"} + + {"timestamp":"2025-01-24T15:48:06Z","domain":"paroisses-olivet45.fr"} + + {"timestamp":"2025-01-24T15:48:06Z","domain":"alfonsopropertyservices.com"} + + {"timestamp":"2025-01-24T15:48:07Z","domain":"decomplexifying.com"} + + {"timestamp":"2025-01-24T15:48:07Z","domain":"go-healthy.my.id"} + + {"timestamp":"2025-01-24T15:48:07Z","domain":"ergonomia.se"} + + {"timestamp":"2025-01-24T15:48:08Z","domain":"aajee.ir"} + + {"timestamp":"2025-01-24T15:48:08Z","domain":"alquimiaindumentaria.com.ar"} + + {"timestamp":"2025-01-24T15:48:08Z","domain":"platinumprestigemotors.co.uk"} + + {"timestamp":"2025-01-24T15:48:09Z","domain":"80311.pictures"} + + {"timestamp":"2025-01-24T15:48:09Z","domain":"talkaboutmk.com"} + + {"timestamp":"2025-01-24T15:48:10Z","domain":"62326.plus"} + + {"timestamp":"2025-01-24T15:48:11Z","domain":"participantstobetterunders.com"} + + {"timestamp":"2025-01-24T15:48:11Z","domain":"cleardot.in"} + + {"timestamp":"2025-01-24T15:48:12Z","domain":"blackeaglelakehouse.co.za"} + + {"timestamp":"2025-01-24T15:48:12Z","domain":"32425.plus"} + + {"timestamp":"2025-01-24T15:48:13Z","domain":"parkplatzbau.ch"} + + {"timestamp":"2025-01-24T15:48:13Z","domain":"onestopmentor.com"} + + {"timestamp":"2025-01-24T15:48:14Z","domain":"gmclean.eu"} + + {"timestamp":"2025-01-24T15:48:14Z","domain":"lotusbook9id.com"} + + {"timestamp":"2025-01-24T15:48:15Z","domain":"pinsrecipes.com"} + + {"timestamp":"2025-01-24T15:48:16Z","domain":"naturalhome.store"} + + {"timestamp":"2025-01-24T15:48:16Z","domain":"siweddingservices.com"} + + {"timestamp":"2025-01-24T15:48:17Z","domain":"digitaldevices.ai"} + + {"timestamp":"2025-01-24T15:48:18Z","domain":"xjsyjt.cn"} + + {"timestamp":"2025-01-24T15:48:18Z","domain":"wahlrecgt.de"} + + {"timestamp":"2025-01-24T15:48:18Z","domain":"trubrexapatch.com"} + + {"timestamp":"2025-01-24T15:48:19Z","domain":"carmengarcia.eu"} + + {"timestamp":"2025-01-24T15:48:19Z","domain":"walldecorzone.net"} + + {"timestamp":"2025-01-24T15:48:21Z","domain":"chrismerrealty.com"} + + {"timestamp":"2025-01-24T15:48:21Z","domain":"bfeiv.club"} + + {"timestamp":"2025-01-24T15:48:21Z","domain":"growingkidsschool.com"} + + {"timestamp":"2025-01-24T15:48:22Z","domain":"annank.com"} + + {"timestamp":"2025-01-24T15:48:22Z","domain":"connexense.com"} + + {"timestamp":"2025-01-24T15:48:22Z","domain":"lotte4dpizh.com"} + + {"timestamp":"2025-01-24T15:48:23Z","domain":"abdelfattahcontractors.com"} + + {"timestamp":"2025-01-24T15:48:23Z","domain":"theploughinnscarborough.com"} + + {"timestamp":"2025-01-24T15:48:23Z","domain":"giftback.sk"} + + {"timestamp":"2025-01-24T15:48:24Z","domain":"noelbrandconsulting.org"} + + {"timestamp":"2025-01-24T15:48:24Z","domain":"province-du-zondoma.my.id"} + + {"timestamp":"2025-01-24T15:48:24Z","domain":"enoc2027.com"} + + {"timestamp":"2025-01-24T15:48:24Z","domain":"contentedgepro.com"} + + {"timestamp":"2025-01-24T15:48:25Z","domain":"super-premios-spinners.com"} + + {"timestamp":"2025-01-24T15:48:25Z","domain":"kearch.co.kr"} + + {"timestamp":"2025-01-24T15:48:25Z","domain":"xn--tempreo-zxa.com"} + + {"timestamp":"2025-01-24T15:48:25Z","domain":"truealchemy.org"} + + {"timestamp":"2025-01-24T15:48:25Z","domain":"abzbedshop.com"} + + {"timestamp":"2025-01-24T15:48:25Z","domain":"spenceph.com"} + + {"timestamp":"2025-01-24T15:48:25Z","domain":"coheteserviceexpressllc.com"} + + {"timestamp":"2025-01-24T15:48:26Z","domain":"chistyye-bory.biz.id"} + + {"timestamp":"2025-01-24T15:48:26Z","domain":"thedsgn.co"} + + {"timestamp":"2025-01-24T15:48:26Z","domain":"medicaldigitusdeus.com"} + + {"timestamp":"2025-01-24T15:48:26Z","domain":"aspssecurite.fr"} + + {"timestamp":"2025-01-24T15:48:26Z","domain":"local-produc-team.fr"} + + {"timestamp":"2025-01-24T15:48:27Z","domain":"seoselin.com"} + + {"timestamp":"2025-01-24T15:48:28Z","domain":"svry.fr"} + + ' + headers: + Cache-Control: + - no-store, no-cache, must-revalidate + Content-Security-Policy: + - 'default-src * data: blob: ''unsafe-eval'' ''unsafe-inline''' + Content-Type: + - application/x-ndjson + Date: + - Fri, 24 Jan 2025 15:48:29 GMT + Expires: + - Thu, 19 Nov 1981 08:52:00 GMT + Pragma: + - no-cache + Set-Cookie: + - dtsession=hd7lpn4ebs9o1t99dqcshe7dvums6hm0onohopah96ahegh6tpo5jccdb3ulknrllbi6hdovd32v25cjfbsr60fmpscgulbff156ofp; + expires=Sun, 23-Feb-2025 15:48:29 GMT; Max-Age=2592000; path=/; domain=.domaintools.com; + secure; HttpOnly + Strict-Transport-Security: + - max-age=31536000; includeSubDomains + Transfer-Encoding: + - chunked + X-Envoy-Upstream-Service-Time: + - '9' + status: + code: 200 + message: OK +version: 1 diff --git a/tests/test_api.py b/tests/test_api.py index 516bdc2..5ec7546 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -577,3 +577,19 @@ def test_domainrdap_feed(): assert "domain" in feed_result["parsed_record"]["parsed_fields"] assert "emails" in feed_result["parsed_record"]["parsed_fields"] assert "contacts" in feed_result["parsed_record"]["parsed_fields"] + + +@vcr.use_cassette +def test_domain_discovery_feed(): + results = api.domaindiscovery(after="-60") + response = results.response() + rows = response.strip().split("\n") + + assert response is not None + assert results.status == 200 + assert len(rows) >= 1 + + for row in rows: + feed_result = json.loads(row) + assert "timestamp" in feed_result.keys() + assert "domain" in feed_result.keys()