Skip to content

Commit 78c93ed

Browse files
jbabacjbabac
authored
Merge pull request #178 from DomainTools/IDEV-2273-support-header-auth-for-iris-endpoints
IDEV-2272 and IDEV-2273: support header auth for iris endpoints and change default Feeds auth behavior
2 parents b0cbb15 + ba03168 commit 78c93ed

19 files changed

+23942
-121945
lines changed

‎README.md‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -215,25 +215,25 @@ Please see the [supported versions](https://github.com/DomainTools/python_api/ra
215215
for the DomainTools Python support policy.
216216

217217

218-
Real-Time Threat Intelligence Feeds
218+
Real-Time Threat Feeds
219219
===================
220220

221-
Real-Time Threat Intelligence Feeds provide data on the different stages of the domain lifecycle: from first-observed in the wild, to newly re-activated after a period of quiet. Access current feed data in real-time or retrieve historical feed data through separate APIs.
221+
Real-Time Threat Feeds provide data on the different stages of the domain lifecycle: from first-observed in the wild, to newly re-activated after a period of quiet. Access current feed data in real-time or retrieve historical feed data through separate APIs.
222222

223223
Custom parameters aside from the common `GET` Request parameters:
224224
- `endpoint` (choose either `download` or `feed` API endpoint - default is `feed`)
225225
```python
226-
api = API(USERNAME, KEY, always_sign_api_key=False)
226+
api = API(USERNAME, KEY)
227227
api.nod(endpoint="feed", **kwargs)
228228
```
229229
- `header_authentication`: by default, we're using API Header Authentication. Set this False if you want to use API Key and Secret Authentication. Apparently, you can't use API Header Authentication for `download` endpoints so this will be defaulted to `False` even without explicitly setting it.
230230
```python
231-
api = API(USERNAME, KEY, always_sign_api_key=False)
232-
api.nod(header_authentication=False, **kwargs)
231+
api = API(USERNAME, KEY, header_authentication=False)
232+
api.nod(**kwargs)
233233
```
234234
- `output_format`: (choose either `csv` or `jsonl` - default is `jsonl`). Cannot be used in `domainrdap` feeds. Additionally, `csv` is not available for `download` endpoints.
235235
```python
236-
api = API(USERNAME, KEY, always_sign_api_key=False)
236+
api = API(USERNAME, KEY)
237237
api.nod(output_format="csv", **kwargs)
238238
```
239239

@@ -254,7 +254,7 @@ Since we may dealing with large feeds datasets, the python wrapper uses `generat
254254
```python
255255
from domaintools import API
256256

257-
api = API(USERNAME, KEY, always_sign_api_key=False)
257+
api = API(USERNAME, KEY)
258258
results = api.nod(sessionID="my-session-id", after=-60)
259259

260260
for result in results.response() # generator that holds NOD feeds data for the past 60 seconds and is expected to request only once
@@ -265,7 +265,7 @@ for result in results.response() # generator that holds NOD feeds data for the p
265265
```python
266266
from domaintools import API
267267

268-
api = API(USERNAME, KEY, always_sign_api_key=False)
268+
api = API(USERNAME, KEY)
269269
results = api.nod(sessionID="my-session-id", after=-7200)
270270

271271
for partial_result in results.response() # generator that holds NOD feeds data for the past 2 hours and is expected to request multiple times

‎domaintools/api.py‎

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
import re
77
import ssl
88

9-
from domaintools.constants import Endpoint, ENDPOINT_TO_SOURCE_MAP, FEEDS_PRODUCTS_LIST, OutputFormat
9+
from domaintools.constants import Endpoint, ENDPOINT_TO_SOURCE_MAP, RTTF_PRODUCTS_LIST, OutputFormat
1010
from domaintools._version import current as version
1111
from domaintools.results import (
1212
GroupedIterable,
@@ -63,7 +63,8 @@ def __init__(
6363
verify_ssl=True,
6464
rate_limit=True,
6565
proxy_url=None,
66-
always_sign_api_key=True,
66+
always_sign_api_key=None,
67+
header_authentication=None,
6768
key_sign_hash="sha256",
6869
app_name="python_wrapper",
6970
app_version=version,
@@ -83,6 +84,7 @@ def __init__(
8384
self.proxy_url = proxy_url
8485
self.extra_request_params = {}
8586
self.always_sign_api_key = always_sign_api_key
87+
self.header_authentication = header_authentication
8688
self.key_sign_hash = key_sign_hash
8789
self.default_parameters["app_name"] = app_name
8890
self.default_parameters["app_version"] = app_version
@@ -129,19 +131,27 @@ def _results(self, product, path, cls=Results, **kwargs):
129131
uri = "/".join((self._rest_api_url, path.lstrip("/")))
130132
parameters = self.default_parameters.copy()
131133
parameters["api_username"] = self.username
132-
header_authentication = kwargs.pop("header_authentication", True) # Used only by Real-Time Threat Intelligence Feeds endpoints for now
133-
self.handle_api_key(product, path, parameters, header_authentication)
134+
is_rttf_product = product in RTTF_PRODUCTS_LIST
135+
self._handle_api_key_parameters(is_rttf_product)
136+
self.handle_api_key(is_rttf_product, path, parameters)
134137
parameters.update({key: str(value).lower() if value in (True, False) else value for key, value in kwargs.items() if value is not None})
135138

136139
return cls(self, product, uri, **parameters)
137140

138-
def handle_api_key(self, product, path, parameters, header_authentication):
141+
def _handle_api_key_parameters(self, is_rttf_product):
142+
if self.always_sign_api_key is None:
143+
self.always_sign_api_key = not is_rttf_product
144+
145+
if self.header_authentication is None:
146+
self.header_authentication = is_rttf_product
147+
148+
def handle_api_key(self, is_rttf_product, path, parameters):
139149
if self.https and not self.always_sign_api_key:
140-
if product in FEEDS_PRODUCTS_LIST and header_authentication:
141-
parameters["X-Api-Key"] = self.key
142-
else:
143-
parameters["api_key"] = self.key
150+
parameters["api_key"] = self.key
144151
else:
152+
if is_rttf_product:
153+
# As per requirement in IDEV-2272, raise this error when the user explicitly sets signing of API key for RTTF endpoints
154+
raise ValueError("Real Time Threat Feeds do not support signed API keys.")
145155
if self.key_sign_hash and self.key_sign_hash in AVAILABLE_KEY_SIGN_HASHES:
146156
signing_hash = eval(self.key_sign_hash)
147157
else:

‎domaintools/base_results.py‎

Lines changed: 23 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
from datetime import datetime
1010
from httpx import Client
1111

12-
from domaintools.constants import FEEDS_PRODUCTS_LIST, OutputFormat, HEADER_ACCEPT_KEY_CSV_FORMAT
12+
from domaintools.constants import RTTF_PRODUCTS_LIST, OutputFormat, HEADER_ACCEPT_KEY_CSV_FORMAT
1313
from domaintools.exceptions import (
1414
BadRequestException,
1515
InternalServerErrorException,
@@ -75,45 +75,45 @@ def _wait_time(self):
7575

7676
return wait_for
7777

78-
def _get_session_params(self):
79-
parameters = deepcopy(self.kwargs)
80-
parameters.pop("output_format", None)
81-
parameters.pop(
82-
"format", None
83-
) # For some unknownn reasons, even if "format" is not included in the cli params for feeds endpoint, it is being populated thus we need to remove it. Happens only if using CLI.
78+
def _get_session_params_and_headers(self):
8479
headers = {}
85-
if self.kwargs.get("output_format", OutputFormat.JSONL.value) == OutputFormat.CSV.value:
86-
parameters["headers"] = int(bool(self.kwargs.get("headers", False)))
87-
headers["accept"] = HEADER_ACCEPT_KEY_CSV_FORMAT
88-
89-
header_api_key = parameters.pop("X-Api-Key", None)
90-
if header_api_key:
91-
headers["X-Api-Key"] = header_api_key
80+
parameters = deepcopy(self.kwargs)
81+
is_rttf_product = self.product in RTTF_PRODUCTS_LIST
82+
if is_rttf_product:
83+
parameters.pop("output_format", None)
84+
parameters.pop(
85+
"format", None
86+
) # For some unknownn reasons, even if "format" is not included in the cli params for feeds endpoint, it is being populated thus we need to remove it. Happens only if using CLI.
87+
if self.kwargs.get("output_format", OutputFormat.JSONL.value) == OutputFormat.CSV.value:
88+
parameters["headers"] = int(bool(self.kwargs.get("headers", False)))
89+
headers["accept"] = HEADER_ACCEPT_KEY_CSV_FORMAT
90+
91+
if self.api.header_authentication:
92+
header_key_for_api_key = "X-Api-Key" if is_rttf_product else "X-API-Key"
93+
headers[header_key_for_api_key] = parameters.pop("api_key", None)
9294

9395
return {"parameters": parameters, "headers": headers}
9496

9597
def _make_request(self):
9698

9799
with Client(verify=self.api.verify_ssl, proxy=self.api.proxy_url, timeout=None) as session:
100+
session_params_and_headers = self._get_session_params_and_headers()
101+
headers = session_params_and_headers.get("headers")
98102
if self.product in [
99103
"iris-investigate",
100104
"iris-enrich",
101105
"iris-detect-escalate-domains",
102106
]:
103107
post_data = self.kwargs.copy()
104108
post_data.update(self.api.extra_request_params)
105-
return session.post(url=self.url, data=post_data)
109+
return session.post(url=self.url, data=post_data, headers=headers)
106110
elif self.product in ["iris-detect-manage-watchlist-domains"]:
107111
patch_data = self.kwargs.copy()
108112
patch_data.update(self.api.extra_request_params)
109-
return session.patch(url=self.url, json=patch_data)
110-
elif self.product in FEEDS_PRODUCTS_LIST:
111-
session_params = self._get_session_params()
112-
parameters = session_params.get("parameters")
113-
headers = session_params.get("headers")
114-
return session.get(url=self.url, params=parameters, headers=headers, **self.api.extra_request_params)
113+
return session.patch(url=self.url, json=patch_data, headers=headers)
115114
else:
116-
return session.get(url=self.url, params=self.kwargs, **self.api.extra_request_params)
115+
parameters = session_params_and_headers.get("parameters")
116+
return session.get(url=self.url, params=parameters, headers=headers, **self.api.extra_request_params)
117117

118118
def _get_results(self):
119119
wait_for = self._wait_time()
@@ -170,7 +170,7 @@ def status(self):
170170

171171
def setStatus(self, code, response=None):
172172
self._status = code
173-
if code == 200 or (self.product in FEEDS_PRODUCTS_LIST and code == 206):
173+
if code == 200 or (self.product in RTTF_PRODUCTS_LIST and code == 206):
174174
return
175175

176176
reason = None

‎domaintools/cli/api.py‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
from rich.progress import Progress, SpinnerColumn, TextColumn
1010

1111
from domaintools.api import API
12-
from domaintools.constants import Endpoint, FEEDS_PRODUCTS_LIST, OutputFormat
12+
from domaintools.constants import Endpoint, RTTF_PRODUCTS_LIST, OutputFormat
1313
from domaintools.cli.utils import get_file_extension
1414
from domaintools.exceptions import ServiceException
1515
from domaintools._version import current as version
@@ -110,7 +110,7 @@ def args_to_dict(*args) -> Dict:
110110
def _get_formatted_output(cls, cmd_name: str, response, out_format: str = "json"):
111111
if cmd_name in ("available_api_calls",):
112112
return "\n".join(response)
113-
if response.product in FEEDS_PRODUCTS_LIST:
113+
if response.product in RTTF_PRODUCTS_LIST:
114114
return "\n".join([data for data in response.response()])
115115
return str(getattr(response, out_format) if out_format != "list" else response.as_list())
116116

@@ -180,6 +180,7 @@ def run(cls, name: str, params: Optional[Dict] = {}, **kwargs):
180180
out_file = params.pop("out_file", sys.stdout)
181181
verify_ssl = params.pop("no_verify_ssl", False)
182182
always_sign_api_key = params.pop("no_sign_api_key", False)
183+
header_authentication = params.pop("no_header_authentication", False)
183184
source = None
184185

185186
if "src_file" in params:
@@ -214,6 +215,7 @@ def run(cls, name: str, params: Optional[Dict] = {}, **kwargs):
214215
verify_ssl=verify_ssl,
215216
rate_limit=rate_limit,
216217
always_sign_api_key=always_sign_api_key,
218+
header_authentication=header_authentication,
217219
)
218220
dt_api_func = getattr(dt_api, name)
219221

@@ -229,7 +231,7 @@ def run(cls, name: str, params: Optional[Dict] = {}, **kwargs):
229231

230232
if isinstance(out_file, _io.TextIOWrapper):
231233
# use rich `print` command to prettify the ouput in sys.stdout
232-
if response.product in FEEDS_PRODUCTS_LIST:
234+
if response.product in RTTF_PRODUCTS_LIST:
233235
print(output)
234236
else:
235237
print(response)

‎domaintools/cli/commands/feeds.py‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,8 @@ def feeds_nad(
3232
"--no-sign-api-key",
3333
help="Skip signing of api key",
3434
),
35-
header_authentication: bool = typer.Option(
36-
True,
35+
no_header_authentication: bool = typer.Option(
36+
False,
3737
"--no-header-auth",
3838
help="Don't use header authentication",
3939
),
@@ -112,8 +112,8 @@ def feeds_nod(
112112
"--no-sign-api-key",
113113
help="Skip signing of api key",
114114
),
115-
header_authentication: bool = typer.Option(
116-
True,
115+
no_header_authentication: bool = typer.Option(
116+
False,
117117
"--no-header-auth",
118118
help="Don't use header authentication",
119119
),
@@ -192,8 +192,8 @@ def feeds_domainrdap(
192192
"--no-sign-api-key",
193193
help="Skip signing of api key",
194194
),
195-
header_authentication: bool = typer.Option(
196-
True,
195+
no_header_authentication: bool = typer.Option(
196+
False,
197197
"--no-header-auth",
198198
help="Don't use header authentication",
199199
),
@@ -260,8 +260,8 @@ def feeds_domaindiscovery(
260260
"--no-sign-api-key",
261261
help="Skip signing of api key",
262262
),
263-
header_authentication: bool = typer.Option(
264-
True,
263+
no_header_authentication: bool = typer.Option(
264+
False,
265265
"--no-header-auth",
266266
help="Don't use header authentication",
267267
),
@@ -340,8 +340,8 @@ def feeds_noh(
340340
"--no-sign-api-key",
341341
help="Skip signing of api key",
342342
),
343-
header_authentication: bool = typer.Option(
344-
True,
343+
no_header_authentication: bool = typer.Option(
344+
False,
345345
"--no-header-auth",
346346
help="Don't use header authentication",
347347
),
@@ -420,8 +420,8 @@ def feeds_domainhotlist(
420420
"--no-sign-api-key",
421421
help="Skip signing of api key",
422422
),
423-
header_authentication: bool = typer.Option(
424-
True,
423+
no_header_authentication: bool = typer.Option(
424+
False,
425425
"--no-header-auth",
426426
help="Don't use header authentication",
427427
),
@@ -500,8 +500,8 @@ def feeds_realtime_domain_risk(
500500
"--no-sign-api-key",
501501
help="Skip signing of api key",
502502
),
503-
header_authentication: bool = typer.Option(
504-
True,
503+
no_header_authentication: bool = typer.Option(
504+
False,
505505
"--no-header-auth",
506506
help="Don't use header authentication",
507507
),

0 commit comments

Comments
 (0)