From 16f18177afe0318f39be39503ab015d7a068bb13 Mon Sep 17 00:00:00 2001 From: EyalPintzov <91007713+eyalpalo@users.noreply.github.com> Date: Thu, 21 Sep 2023 10:19:29 +0300 Subject: [PATCH] XSUP-27936 problem with regex (#29613) * failed test * fix * rn * rn * unit test * ut * validations * fixed test and docker * fix * validation --- Packs/Whois/.secrets-ignore | 5 +- Packs/Whois/Integrations/Whois/Whois.py | 7 +- Packs/Whois/Integrations/Whois/Whois.yml | 18 +-- Packs/Whois/Integrations/Whois/Whois_test.py | 75 +++++------- .../Whois/test_data/whois_response.txt | 108 ++++++++++++++++++ .../Whois/test_data/whois_response_text.txt | 78 +++++++++++++ Packs/Whois/ReleaseNotes/1_5_3.md | 8 ++ Packs/Whois/pack_metadata.json | 2 +- 8 files changed, 237 insertions(+), 64 deletions(-) create mode 100644 Packs/Whois/Integrations/Whois/test_data/whois_response.txt create mode 100644 Packs/Whois/Integrations/Whois/test_data/whois_response_text.txt create mode 100644 Packs/Whois/ReleaseNotes/1_5_3.md diff --git a/Packs/Whois/.secrets-ignore b/Packs/Whois/.secrets-ignore index 14744ae8b5b8..6e02c7b310bb 100644 --- a/Packs/Whois/.secrets-ignore +++ b/Packs/Whois/.secrets-ignore @@ -54,4 +54,7 @@ http://cscdbs.com http://www.apnic.net http://rdap.apnic.net helpdesk@apnic.net -https://www.apnic.net \ No newline at end of file +https://www.apnic.net +test@test.net +http://www.test.net +test@test.fr \ No newline at end of file diff --git a/Packs/Whois/Integrations/Whois/Whois.py b/Packs/Whois/Integrations/Whois/Whois.py index 90e9fcbb9816..c4b5c4b0ad1d 100644 --- a/Packs/Whois/Integrations/Whois/Whois.py +++ b/Packs/Whois/Integrations/Whois/Whois.py @@ -7658,12 +7658,7 @@ def preprocess_regex(regex): # nic.ir, individual - this is a nasty one. "nic-hdl:\s+(?P.+)\norg:\s+(?P.+)\n(?:e-mail:\s+(?P.+)\n)?(?:address:\s+(?P.+?)(?:,+ (?P.+?)(?:,+ (?P.+?)(?:,+ (?P.+?)(?:,+ (?P.+?)(?:,+ (?P.+?)(?:,+ (?P.+?))?)?)?)?)?)?, (?P.+), (?P.+), (?P.+)\n)?(?:phone:\s+(?P.+)\n)?(?:fax-no:\s+(?P.+)\n)?", # nic.ir, organization - "nic-hdl:\s*(?P.+)\ntype:\s*(?P.+)\ncontact:\s*(?P.+)\n(?:.+\n)*?(?:address:\s*(?P.+)\naddress:\s*(?P.+)\naddress:\s*(?P.+)\naddress:\s*(?P.+)\n)?(?:phone:\s*(?P.+)\n)?(?:fax-no:\s*(?P.+)\n)?(?:.+\n)*?(?:e-mail:\s*(?P.+)\n)?(?:.+\n)*?changed:\s*(?P.*}).*\n", - # AFNIC madness without country field - "nic-hdl:\s*(?P.+)\ntype:\s*(?P.+)\ncontact:\s*(?P.+)\n(?:.+\n)*?(?:address:\s*(?P.+)\n)?(?:address:\s*(?P.+)\n)?(?:address:\s*(?P.+)\n)?(?:phone:\s*(?P.+)\n)?(?:fax-no:\s*(?P.+)\n)?(?:.+\n)*?(?:e-mail:\s*(?P.+)\n)?(?:.+\n)*?changed:\s*(?P.*).*\n", - # AFNIC madness any country -at all- - "nic-hdl:\s*(?P.+)\ntype:\s*(?P.+)\ncontact:\s*(?P.+)\n(?:.+\n)*?(?:address:\s*(?P.+)\n)?(?:address:\s*(?P.+)\n)?(?:address:\s*(?P.+)\n)?(?:address:\s*(?P.+)\n)?country:\s*(?P.+)\n(?:phone:\s*(?P.+)\n)?(?:fax-no:\s*(?P.+)\n)?(?:.+\n)*?(?:e-mail:\s*(?P.+)\n)?(?:.+\n)*?changed:\s*(?P.+).*\n", - # AFNIC madness with country field + "nic-hdl:[ ]*(?P.*?)\ntype:[ ]*(?P.*)\ncontact:[ ]*(?P.*?)\n(?:.*\n)*?(?:(?:address:[ ]*(?P.*?)\n)(?:address:[ ]*(?P.*?)\n)?(?:address:[ ]*(?P.*)\n)?(?:address:[ ]*(?P.*)\n)?(?:country:[ ]*(?P.*?)\n)?)(?:phone:[ ]*(?P.*?)\n)?(?:fax-no:[ ]*(?P.*?)\n)?(?:.*\n)*?(?:e-mail:[ ]*(?P.*?)\n)?registrar:[ ]*(?P.*?)\n(?:.*?\n)*?(?:changed:[ ]*(?P.*?)\n)?" ] organization_regexes = ( diff --git a/Packs/Whois/Integrations/Whois/Whois.yml b/Packs/Whois/Integrations/Whois/Whois.yml index 4f217ce319f2..036adffffbd0 100644 --- a/Packs/Whois/Integrations/Whois/Whois.yml +++ b/Packs/Whois/Integrations/Whois/Whois.yml @@ -160,10 +160,10 @@ script: description: The organization of the domain administrator. type: string - contextPath: Domain.Whois.Administrator.postalcode - description: The postal code of the domain administrator + description: The postal code of the domain administrator. type: string - contextPath: Domain.Whois.Administrator.street - description: The street of the domain admin + description: The street of the domain admin. type: string - contextPath: Domain.Whois.Administrator.phone description: The phone number of the domain administrator. @@ -313,10 +313,10 @@ script: description: The organization of the domain administrator. type: string - contextPath: Domain.Whois.Administrator.postalcode - description: The postal code of the domain administrator + description: The postal code of the domain administrator. type: string - contextPath: Domain.Whois.Administrator.street - description: The street of the domain admin + description: The street of the domain admin. type: string - contextPath: Domain.Whois.Administrator.phone description: The phone number of the domain administrator. @@ -484,7 +484,7 @@ script: description: ASN allocation date in ISO 8601 format. type: Date - contextPath: Whois.IP.asn_description - description: The ASN description + description: The ASN description. type: string - contextPath: Whois.IP.asn_registry description: ASN assigned regional internet registry. @@ -547,10 +547,10 @@ script: description: The RIR classification of a registered network. type: string - contextPath: Whois.IP.query - description: The IP address + description: The IP address. type: string - contextPath: IP.Address - description: IP address + description: IP address. type: string - contextPath: IP.ASN description: 'The autonomous system name for the IP address, for example: "AS8948".' @@ -565,7 +565,7 @@ script: description: Indicators that are associated with the IP. type: string - contextPath: IP.feed_related_indicators.type - description: The type of the indicators that are associated with the IP + description: The type of the indicators that are associated with the IP. type: string - contextPath: DBotScore.Indicator description: The indicator that was tested. @@ -582,7 +582,7 @@ script: - contextPath: DBotScore.Reliability description: Reliability of the source providing the intelligence data. type: String - dockerimage: demisto/ippysocks-py3:1.0.0.72626 + dockerimage: demisto/ippysocks-py3:1.0.0.74506 runonce: false script: '-' subtype: python3 diff --git a/Packs/Whois/Integrations/Whois/Whois_test.py b/Packs/Whois/Integrations/Whois/Whois_test.py index a042772262fd..b3b33457b7df 100644 --- a/Packs/Whois/Integrations/Whois/Whois_test.py +++ b/Packs/Whois/Integrations/Whois/Whois_test.py @@ -2,6 +2,7 @@ import pickle import Whois import demistomock as demisto +import pathlib import pytest import subprocess import time @@ -340,61 +341,24 @@ def test_whois_with_verbose(args, expected_res, mocker: MockerFixture): def test_parse_nic_contact(): - data = ["%%\n%% This is the AFNIC Whois server.\n%%\n%% complete date format : YYYY-MM-DDThh:mm:ssZ\n%% short date " - "format : DD/MM\n%% version : FRNIC-2.5\n%%\n%% Rights restricted by copyright.\n%% See " - "https://www.afnic.fr/en/products-and-services/services/whois/whois-special-notice/\n%%\n%% Use '-h' option" - "to obtain more information about this service.\n%%\n%% [1111 REQUEST] >> google.fr\n%%\n%% RL " - "Net [##########] - RL IP [#########.]\n%%\n\ndomain: google.fr\nstatus: ACTIVE\nhold: " - "NO\nholder-c: GIHU100-FRNIC\nadmin-c: GIHU101-FRNIC\ntech-c: MI3669-FRNIC\nzone-c: " - "NFC1-FRNIC\nnsl-id: NSL4386-FRNIC\nregistrar: MARKMONITOR Inc.\nExpiry Date: 2022-12-30T17:16" - ":48Z\ncreated: 2000-07-26T22:00:00Z\nlast-update: 2022-08-17T16:39:47Z\nsource: FRNIC\n\nns-list:" - " NSL4386-FRNIC\nnserver: ns1.google.com\nnserver: ns2.google.com\nnserver: ns3.google.com\n" - "nserver: ns4.google.com\nsource: FRNIC\n\nregistrar: MARKMONITOR Inc.\ntype: Isp Option " - "\naddress: 2150 S. Bonito Way, Suite 150\naddress: ID 83642 MERIDIAN\ncountry: US\n" - "phone: +1 208 389 5740\nfax-no: +1 208 389 5771\ne-mail: registry.admin@markmonitor.com\n" - "website: http://www.markmonitor.com\nanonymous: NO\nregistered: 2002-01-10T12:00:00Z\nsource: " - "FRNIC\n\nnic-hdl: GIHU100-FRNIC\ntype: ORGANIZATION\ncontact: Google Ireland Holdings " - "Unlimited Company\naddress: Google Ireland Holdings Unlimited Company\naddress: 70 Sir John " - "Rogerson's Quay\naddress: 2 Dublin\naddress: Dublin\ncountry: IE\nphone: " - "+353.14361000\ne-mail: dns-admin@google.com\nregistrar: MARKMONITOR Inc.\nchanged: " - " 2018-03-02T18:03:31Z nic.fr\nanonymous: NO\nobsoleted: NO\neligstatus: not identified\n" - "reachstatus: not identified\nsource: FRNIC\n\nnic-hdl: GIHU101-FRNIC\ntype: ORGANIZATION" - "\ncontact: Google Ireland Holdings Unlimited Company\naddress: 70 Sir John Rogerson's Quay\n" - "address: 2 Dublin\ncountry: IE\nphone: +353.14361000\ne-mail: dns-admin@google.com\n" - "registrar: MARKMONITOR Inc.\nchanged: 2018-03-02T17:52:06Z nic.fr\nanonymous: NO\nobsoleted: " - " NO\neligstatus: not identified\nreachmedia: email\nreachstatus: ok\nreachsource: REGISTRAR\nreachdate: " - "2018-03-02T17:52:06Z\nsource: FRNIC\n\nnic-hdl: MI3669-FRNIC\ntype: ORGANIZATION\ncontact:" - "MarkMonitor Inc.\naddress: 2150 S. Bonito Way, Suite 150\naddress: 83642 Meridian\naddress: " - "ID\ncountry: US\nphone: +1.2083895740\nfax-no: +1.2083895771\ne-mail: " - "ccops@markmonitor" - ".com\nregistrar: MARKMONITOR Inc.\nchanged: 2021-10-05T15:17:57Z nic.fr\nanonymous: NO\n" - "obsoleted: NO\neligstatus: ok\neligsource: REGISTRAR\neligdate: 2021-10-05T15:17:56Z\nreachmedia: " - "email\nreachstatus: ok\nreachsource: REGISTRAR\nreachdate: 2021-10-05T15:17:56Z\nsource: FRNIC\n\n"] + with open('./test_data/whois_response.txt') as f: + data = [f.read()] res = Whois.parse_nic_contact(data) expected = [{'handle': 'GIHU100-FRNIC', 'type': 'ORGANIZATION', 'name': 'Google Ireland Holdings Unlimited Company', 'street1': 'Google Ireland Holdings Unlimited Company', 'street2': "70 Sir John Rogerson's Quay", - 'street3': '2 Dublin', 'phone': None, 'fax': None, 'email': None, - 'changedate': '2018-03-02T18:03:31Z nic.fr'}, - {'handle': 'GIHU101-FRNIC', 'type': 'ORGANIZATION', 'name': 'Google Ireland Holdings Unlimited Company', - 'street1': "70 Sir John Rogerson's Quay", 'street2': '2 Dublin', 'street3': None, 'phone': None, - 'fax': None, 'email': None, 'changedate': '2018-03-02T17:52:06Z nic.fr'}, + 'street3': '2 Dublin', 'country': 'IE', 'phone': '+353.14361000', 'fax': None, 'email': 'email@google.com', + 'changedate': '2022-10-15T05:41:14.918179Z', 'registrar': 'MARKMONITOR Inc.', 'street4': None, }, {'handle': 'MI3669-FRNIC', 'type': 'ORGANIZATION', 'name': 'MarkMonitor Inc.', - 'street1': '2150 S. Bonito Way, Suite 150', 'street2': '83642 Meridian', 'street3': 'ID', - 'phone': None, 'fax': None, 'email': None, 'changedate': '2021-10-05T15:17:57Z nic.fr'}, - {'handle': 'GIHU100-FRNIC', 'type': 'ORGANIZATION', 'name': 'Google Ireland Holdings Unlimited Company', - 'street1': 'Google Ireland Holdings Unlimited Company', 'street2': "70 Sir John Rogerson's Quay", - 'street3': '2 Dublin', 'street4': 'Dublin', 'country': 'IE', 'phone': '+353.14361000', 'fax': None, - 'email': 'dns-admin@google.com', 'changedate': '2018-03-02T18:03:31Z nic.fr'}, + 'street1': '2150 S. Bonito Way, Suite 150', 'street2': '83642 Meridian', 'street3': None, 'street4': None, + 'phone': '+1.2083895740', 'fax': '+1.2083895771', 'email': 'email@markmonitor.com', + 'changedate': '2023-09-07T07:32:23.899353Z', 'country': 'US', 'registrar': 'MARKMONITOR Inc.'}, {'handle': 'GIHU101-FRNIC', 'type': 'ORGANIZATION', 'name': 'Google Ireland Holdings Unlimited Company', 'street1': "70 Sir John Rogerson's Quay", 'street2': '2 Dublin', 'street3': None, 'street4': None, - 'country': 'IE', 'phone': '+353.14361000', 'fax': None, 'email': 'dns-admin@google.com', - 'changedate': '2018-03-02T17:52:06Z nic.fr'}, - {'handle': 'MI3669-FRNIC', 'type': 'ORGANIZATION', 'name': 'MarkMonitor Inc.', - 'street1': '2150 S. Bonito Way, Suite 150', 'street2': '83642 Meridian', 'street3': 'ID', - 'street4': None, 'country': 'US', 'phone': '+1.2083895740', 'fax': '+1.2083895771', - 'email': 'ccops@markmonitor.com', 'changedate': '2021-10-05T15:17:57Z nic.fr'}] + 'phone': '+353.14361000', 'fax': None, 'email': 'email@google.com', 'changedate': None, 'country': 'IE', + 'registrar': 'MARKMONITOR Inc.'}] + assert res == expected @@ -669,3 +633,20 @@ def test_domain_command(args: dict[str, Any], expected_res, mocker: MockerFixtur reliability='B - Usually reliable' ) assert len(result) == expected_res + + +def test_parse_nic_contact_new_regex(): + """ + Given: + - Data fetched from the API. + When: + - calling the whois/domain command. + Then: + - validate that the data extracted without timeout. + """ + from Whois import parse_nic_contact + data = pathlib.Path('test_data/whois_response_text.txt').read_text() + res = parse_nic_contact([data]) + assert len(res) == 2 + assert any(entry.get('email') == 'test@test.net' for entry in res) + assert any(entry.get('country') == 'TEST' for entry in res) diff --git a/Packs/Whois/Integrations/Whois/test_data/whois_response.txt b/Packs/Whois/Integrations/Whois/test_data/whois_response.txt new file mode 100644 index 000000000000..b98886017e48 --- /dev/null +++ b/Packs/Whois/Integrations/Whois/test_data/whois_response.txt @@ -0,0 +1,108 @@ +%% +%% This is the AFNIC Whois server. +%% +%% complete date format: YYYY-MM-DDThh:mm:ssZ +%% +%% Rights restricted by copyright. +%% See https://www.afnic.fr/en/domain-names-and-support/everything-there-is-to-know-about-domain-names/find-a-domain-name-or-a-holder-using-whois/ +%% +%% + +domain: google.fr +status: ACTIVE +eppstatus: serverUpdateProhibited +eppstatus: serverTransferProhibited +eppstatus: serverDeleteProhibited +eppstatus: serverRecoverProhibited +hold: NO +holder-c: GIHU100-FRNIC +admin-c: GIHU101-FRNIC +tech-c: MI3669-FRNIC +registrar: MARKMONITOR Inc. +Expiry Date: 2023-12-30T17:16:48Z +created: 2000-07-26T22:00:00Z +last-update: 2022-12-03T09:40:42.40624Z +source: FRNIC + +nserver: ns1.google.com +nserver: ns2.google.com +nserver: ns3.google.com +nserver: ns4.google.com +source: FRNIC + +registrar: MARKMONITOR Inc. +address: 2150 S. Bonito Way, Suite 150 +address: ID 83642 MERIDIAN +country: US +phone: +1.2083895740 +fax-no: +1.2083895771 +e-mail: email@markmonitor.com +website: http://www.markmonitor.com +anonymous: No +registered: 2002-01-07T00:00:00Z +source: FRNIC + +nic-hdl: GIHU100-FRNIC +type: ORGANIZATION +contact: Google Ireland Holdings Unlimited Company +address: Google Ireland Holdings Unlimited Company +address: 70 Sir John Rogerson's Quay +address: 2 Dublin +country: IE +phone: +353.14361000 +e-mail: email@google.com +registrar: MARKMONITOR Inc. +changed: 2022-10-15T05:41:14.918179Z +anonymous: NO +obsoleted: NO +eppstatus: serverUpdateProhibited +eppstatus: associated +eligstatus: not identified +reachstatus: not identified +source: FRNIC + +nic-hdl: MI3669-FRNIC +type: ORGANIZATION +contact: MarkMonitor Inc. +address: 2150 S. Bonito Way, Suite 150 +address: 83642 Meridian +country: US +phone: +1.2083895740 +fax-no: +1.2083895771 +e-mail: email@markmonitor.com +registrar: MARKMONITOR Inc. +changed: 2023-09-07T07:32:23.899353Z +anonymous: NO +obsoleted: NO +eppstatus: associated +eppstatus: active +eligstatus: ok +eligsource: REGISTRAR +eligdate: 2021-10-05T00:00:00Z +reachstatus: ok +reachmedia: email +reachsource: REGISTRAR +reachdate: 2021-10-05T00:00:00Z +source: FRNIC + +nic-hdl: GIHU101-FRNIC +type: ORGANIZATION +contact: Google Ireland Holdings Unlimited Company +address: 70 Sir John Rogerson's Quay +address: 2 Dublin +country: IE +phone: +353.14361000 +e-mail: email@google.com +registrar: MARKMONITOR Inc. +anonymous: NO +obsoleted: NO +eppstatus: associated +eppstatus: active +eligstatus: not identified +reachstatus: ok +reachmedia: email +reachsource: REGISTRAR +reachdate: 2018-03-02T00:00:00Z +source: FRNIC + +>>> WHOIS request date: 2023-09-13T09:36:56.921026Z <<< \ No newline at end of file diff --git a/Packs/Whois/Integrations/Whois/test_data/whois_response_text.txt b/Packs/Whois/Integrations/Whois/test_data/whois_response_text.txt new file mode 100644 index 000000000000..9497a354f590 --- /dev/null +++ b/Packs/Whois/Integrations/Whois/test_data/whois_response_text.txt @@ -0,0 +1,78 @@ +%% +%% This is the AFNIC Whois server. +%% +%% complete date format: YYYY-MM-DDThh:mm:ssZ +%% +%% Rights restricted by copyright. +%% See test +%% +%% + +domain: test.fr +status: TEST +eppstatus: TEST +hold: TEST +holder-c: TEST +admin-c: TEST +tech-c: TEST +registrar: TEST +Expiry Date: 2026-06-09T14:25:21.617352Z +created: 2026-06-09T14:25:21.638904Z +last-update: 2027-06-16T13:38:17.476151Z +source: TEST + +nserver: test.com +nserver: test.com +source: TEST + +registrar: TEST +address: TEST 1 +address: TEST +country: TEST +phone: +12.1234 +fax-no: +12.1234 +e-mail: test@test.net +website: http://www.test.net/tld/fr +anonymous: TEST +registered: 2006-07-24T00:00:00Z +source: TEST + +nic-hdl: TEST +type: TEST +contact: TEST +address: TEST +address: 9 TEST +address: 1234 TEST +country: TEST +phone: +12.1234 +e-mail: test@test.fr +registrar: TEST +anonymous: TEST +obsoleted: TEST +eppstatus: TEST +eppstatus: TEST +eligstatus: TEST +reachstatus: TEST +source: TEST + +nic-hdl: TEST +type: TEST +contact: TEST +address: TEST +address: TEST 1 +address: TEST +country: TEST +phone: +12.1234 +e-mail: test@test.net +registrar: TEST +changed: 2023-09-09T22:59:54.000448Z +anonymous: TEST +obsoleted: TEST +eppstatus: TEST +eppstatus: TEST +eligstatus: TEST +reachstatus: TEST +source: TEST + +>>> WHOIS request date: 2023-09-11T11:59:51.886322Z <<< + diff --git a/Packs/Whois/ReleaseNotes/1_5_3.md b/Packs/Whois/ReleaseNotes/1_5_3.md new file mode 100644 index 000000000000..5384da4d5f13 --- /dev/null +++ b/Packs/Whois/ReleaseNotes/1_5_3.md @@ -0,0 +1,8 @@ + +#### Integrations + +##### Whois +- Updated the Docker image to: *demisto/ippysocks-py3:1.0.0.74506*. + +- Fixed an issue where extraction on some domains caused timeout issue. + diff --git a/Packs/Whois/pack_metadata.json b/Packs/Whois/pack_metadata.json index 717a7094b5c0..053f83ce2718 100644 --- a/Packs/Whois/pack_metadata.json +++ b/Packs/Whois/pack_metadata.json @@ -2,7 +2,7 @@ "name": "Whois", "description": "This Content Pack helps you run Whois commands as playbook tasks or real-time actions within Cortex XSOAR to obtain valuable domain metadata.", "support": "xsoar", - "currentVersion": "1.5.2", + "currentVersion": "1.5.3", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "",