plugin: add draft for shodan parser

sner/plugin/shodan/parser.py

@@ -0,0 +1,74 @@
+# This file is part of sner4 project governed by MIT license, see the LICENSE.txt file.
+"""
+parsers to import from agent outputs to storage
+"""
+
+import json
+import sys
+from pathlib import Path
+from pprint import pprint
+
+from sner.server.parser import ParsedItemsDb, ParserBase
+
+
+class ParserModule(ParserBase):  # pylint: disable=too-few-public-methods
+    """nmap xml output parser"""
+
+    NAMELEN = 100
+
+    @classmethod
+    def parse_path(cls, path):
+        """parse data from path"""
+
+        pidb = ParsedItemsDb()
+        return cls._parse_data(Path(path).read_text(encoding='utf-8'), pidb)
+
+    @classmethod
+    def _parse_data(cls, data, pidb):
+        """parse raw string data"""
+
+        data = list(map(json.loads, data.splitlines()))
+
+        for ihost in data:
+            # parse host
+            host_data = {}
+            if ihost['hostnames']:
+                host_data['hostnames'] = list(set(ihost['hostnames']))
+                if not host_data.get('hostname'):
+                    host_data['hostname'] = host_data['hostnames'][0]
+
+            pidb.upsert_host(ihost['ip_str'], **host_data)
+
+            # parse services
+            for iservice in ihost['data']:
+                service_data = {
+                    'state': 'open:shodan',
+                    'import_time': iservice['timestamp']
+                }
+                pidb.upsert_service(ihost['ip_str'], iservice['transport'], iservice['port'], **service_data)
+
+                for key, value in iservice.items():
+                    if key in ['ip', 'ip_str', 'transport', 'timestamp', 'hash', '_shodan', 'port']:
+                        continue
+                    if not value:
+                        continue
+                    pidb.upsert_note(ihost['ip_str'], f'shodan.{key}', iservice['transport'], iservice['port'], data=json.dumps(value))
+
+                for vulnid, vuln_data in iservice.get('vulns', {}).items():
+                    pidb.upsert_vuln(
+                        ihost['ip_str'],
+                        f'{vulnid} {vuln_data["summary"][:cls.NAMELEN]}',
+                        f'shodan.{vulnid}',
+                        iservice['transport'],
+                        iservice['port'],
+                        descr=vuln_data['summary'],
+                        data=json.dumps(vuln_data),
+                        refs=[f'URL-{x}' for x in vuln_data['references']],
+                        severity='unknown',
+                    )
+
+        return pidb
+
+
+if __name__ == '__main__':  # pragma: no cover
+    pprint(ParserModule.parse_path(sys.argv[1]).__dict__)

tests/plugin/shodan/test_parser.py

@@ -0,0 +1,18 @@
+# This file is part of sner4 project governed by MIT license, see the LICENSE.txt file.
+"""
+shodan output parser tests
+"""
+
+from sner.plugin.shodan.parser import ParserModule
+
+
+def test_parse_path():
+    """check basic parse_path impl"""
+
+    expected_hosts = ['127.6.1.1']
+    expected_vuln = 'shodan.CVE-2019-20372'
+
+    pidb = ParserModule.parse_path('tests/server/data/parser-shodan.jsonlines')
+
+    assert [x.address for x in pidb.hosts] == expected_hosts
+    assert expected_vuln in [x.xtype for x in pidb.vulns]

tests/server/data/parser-shodan.jsonlines

@@ -0,0 +1 @@
+{"region_code": "10", "tags": ["self-signed", "eol-product"], "ip": 2131099905, "area_code": null, "domains": [], "hostnames": ["dummy.hostname.test"], "country_code": "US", "org": "dummy org string", "data": [{"product": "ciscoSystems", "hash": 1679078035, "os": null, "transport": "udp", "timestamp": "2023-11-13T08:10:29.397104", "isp": "dummy org string", "snmp": {"versions": [3], "engineid_format": "mac", "engine_boots": 0, "engineid_data": "aa:bb:cc:dd:ee:ff", "enterprise": 9, "engine_time": "0:00:00"}, "_shodan": {"region": "na", "module": "snmp_v3", "ptr": true, "options": {}, "id": "44d1bc2e-7bea-4d34-bb4d-0c07619fc6b5", "crawler": "1e4769fde0b9dbe1f84ba3a427cb5b0e415246c1"}, "asn": "AS65551", "hostnames": [], "ip": 2131099905, "domains": [], "org": "dummy org string", "data": "SNMP:\n  Versions:\n    3\n  Engineid Format: mac\n  Engine Boots: 0\n  Engineid Data: 7c:ad:4f:6f:72:98\n  Enterprise: 9\n  Engine Time: 0:00:00", "port": 161, "opts": {"raw": "3062020103300f02024a69020300ffe3040100020103041b3019040b80000009037cad4f6f7298020100020100040004000400302f040b80000009037cad4f6f72980400a81e020237f002010002010030123010060a2b060106030f0101040041024330"}, "ip_str": "127.6.1.1"}, {"asn": "AS65551", "hash": 0, "os": null, "timestamp": "2023-11-13T02:19:50.295358", "isp": "dummy org string", "transport": "tcp", "_shodan": {"region": "eu", "module": "auto", "ptr": true, "options": {}, "id": "5692eab1-7f47-4f34-a0f8-7c9ec8258d18", "crawler": "f0a69a808ee1e7ed3468b22330f65929e8e0cde5"}, "hostnames": [], "ip": 2131099905, "domains": [], "org": "dummy org string", "data": "", "port": 179, "opts": {}, "ip_str": "127.6.1.1"}, {"ip": 2131099905, "http": {"status": 401, "robots_hash": null, "redirects": [], "securitytxt": null, "title": "401 Authorization Required", "sitemap_hash": null, "robots": null, "server": "nginx/1.7.10", "headers_hash": -707165912, "host": "127.6.1.1", "html": "<html>\r\n<head><title>401 Authorization Required</title></head>\r\n<body bgcolor=\"white\">\r\n<center><h1>401 Authorization Required</h1></center>\r\n<hr><center>nginx/1.7.10</center>\r\n</body>\r\n</html>\r\n<!-- a padding to disable MSIE and Chrome friendly error page -->\r\n<!-- a padding to disable MSIE and Chrome friendly error page -->\r\n<!-- a padding to disable MSIE and Chrome friendly error page -->\r\n<!-- a padding to disable MSIE and Chrome friendly error page -->\r\n<!-- a padding to disable MSIE and Chrome friendly error page -->\r\n<!-- a padding to disable MSIE and Chrome friendly error page -->\r\n", "location": "/", "components": {}, "securitytxt_hash": null, "sitemap": null, "html_hash": -2081071606}, "port": 443, "transport": "tcp", "version": "1.7.10", "vulns": {"CVE-2019-20372": {"verified": false, "references": ["https://github.com/kubernetes/ingress-nginx/pull/4859", "https://bertjwregeer.keybase.pub/2019-12-10%20-%20error_page%20request%20smuggling.pdf", "https://duo.com/docs/dng-notes#version-1.5.4-january-2020", "http://nginx.org/en/CHANGES", "https://github.com/nginx/nginx/commit/c1be55f97211d38b69ac0c2027e6812ab8b1b94e", "https://usn.ubuntu.com/4235-1/", "https://usn.ubuntu.com/4235-2/", "https://security.netapp.com/advisory/ntap-20200127-0003/", "http://lists.opensuse.org/opensuse-security-announce/2020-02/msg00013.html", "https://support.apple.com/kb/HT212818", "http://seclists.org/fulldisclosure/2021/Sep/36"], "cvss": 4.3, "summary": "NGINX before 1.17.7, with certain error_page configurations, allows HTTP request smuggling, as demonstrated by the ability of an attacker to read unauthorized web pages in environments where NGINX is being fronted by a load balancer."}, "CVE-2017-7529": {"verified": false, "references": ["http://mailman.nginx.org/pipermail/nginx-announce/2017/000200.html", "http://www.securityfocus.com/bid/99534", "http://www.securitytracker.com/id/1039238", "https://puppet.com/security/cve/cve-2017-7529", "https://access.redhat.com/errata/RHSA-2017:2538", "https://support.apple.com/kb/HT212818", "http://seclists.org/fulldisclosure/2021/Sep/36"], "cvss": 5.0, "summary": "Nginx versions since 0.5.6 up to and including 1.13.2 are vulnerable to integer overflow vulnerability in nginx range filter module resulting into leak of potentially sensitive information triggered by specially crafted request."}, "CVE-2016-4450": {"verified": false, "references": ["http://mailman.nginx.org/pipermail/nginx-announce/2016/000179.html", "http://www.ubuntu.com/usn/USN-2991-1", "http://www.debian.org/security/2016/dsa-3592", "http://www.securitytracker.com/id/1036019", "http://www.securityfocus.com/bid/90967", "https://security.gentoo.org/glsa/201606-06", "https://access.redhat.com/errata/RHSA-2016:1425"], "cvss": 5.0, "summary": "os/unix/ngx_files.c in nginx before 1.10.1 and 1.11.x before 1.11.1 allows remote attackers to cause a denial of service (NULL pointer dereference and worker process crash) via a crafted request, involving writing a client request body to a temporary file."}, "CVE-2017-20005": {"verified": false, "references": ["https://github.com/nginx/nginx/commit/b900cc28fcbb4cf5a32ab62f80b59292e1c85b4b", "https://trac.nginx.org/nginx/ticket/1368", "https://github.com/nginx/nginx/commit/0206ebe76f748bb39d9de4dd4b3fce777fdfdccf", "http://nginx.org/en/CHANGES", "https://lists.debian.org/debian-lts-announce/2021/06/msg00009.html", "https://security.netapp.com/advisory/ntap-20210805-0006/"], "cvss": 7.5, "summary": "NGINX before 1.13.6 has a buffer overflow for years that exceed four digits, as demonstrated by a file with a modification date in 1969 that causes an integer overflow (or a false modification date far in the future), when encountered by the autoindex module."}, "CVE-2018-16845": {"verified": false, "references": ["https://bugzilla.redhat.com/show_bug.cgi?id=CVE-2018-16845", "http://mailman.nginx.org/pipermail/nginx-announce/2018/000221.html", "https://usn.ubuntu.com/3812-1/", "http://www.securitytracker.com/id/1042039", "https://www.debian.org/security/2018/dsa-4335", "https://lists.debian.org/debian-lts-announce/2018/11/msg00010.html", "http://www.securityfocus.com/bid/105868", "https://access.redhat.com/errata/RHSA-2018:3653", "https://access.redhat.com/errata/RHSA-2018:3652", "https://access.redhat.com/errata/RHSA-2018:3681", "https://access.redhat.com/errata/RHSA-2018:3680", "http://lists.opensuse.org/opensuse-security-announce/2019-09/msg00035.html", "https://support.apple.com/kb/HT212818", "http://seclists.org/fulldisclosure/2021/Sep/36"], "cvss": 5.8, "summary": "nginx before versions 1.15.6, 1.14.1 has a vulnerability in the ngx_http_mp4_module, which might allow an attacker to cause infinite loop in a worker process, cause a worker process crash, or might result in worker process memory disclosure by using a specially crafted mp4 file. The issue only affects nginx if it is built with the ngx_http_mp4_module (the module is not built by default) and the .mp4. directive is used in the configuration file. Further, the attack is only possible if an attacker is able to trigger processing of a specially crafted mp4 file with the ngx_http_mp4_module."}, "CVE-2016-1247": {"verified": false, "references": ["http://www.debian.org/security/2016/dsa-3701", "https://legalhackers.com/advisories/Nginx-Exploit-Deb-Root-PrivEsc-CVE-2016-1247.html", "http://www.securityfocus.com/bid/93903", "http://www.ubuntu.com/usn/USN-3114-1", "https://www.exploit-db.com/exploits/40768/", "http://www.securitytracker.com/id/1037104", "http://packetstormsecurity.com/files/139750/Nginx-Debian-Based-Distros-Root-Privilege-Escalation.html", "http://seclists.org/fulldisclosure/2016/Nov/78", "https://security.gentoo.org/glsa/201701-22", "http://seclists.org/fulldisclosure/2017/Jan/33", "https://www.youtube.com/watch?v=aTswN1k1fQs", "http://www.securityfocus.com/archive/1/539796/100/0/threaded", "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/P3WOO7E5R2HT5XVOIOFPEFALILVOWZUF/", "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/ESTIADC7BDB6VTH4JAP6C6OCW2CQ4NHP/", "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/CBIZEKHBOCKO7FUMCO4X53ENMWU5OYFX/"], "cvss": 7.2, "summary": "The nginx package before 1.6.2-5+deb8u3 on Debian jessie, the nginx packages before 1.4.6-1ubuntu3.6 on Ubuntu 14.04 LTS, before 1.10.0-0ubuntu0.16.04.3 on Ubuntu 16.04 LTS, and before 1.10.1-0ubuntu1.1 on Ubuntu 16.10, and the nginx ebuild before 1.10.2-r3 on Gentoo allow local users with access to the web server user account to gain root privileges via a symlink attack on the error log."}, "CVE-2016-0747": {"verified": false, "references": ["http://lists.opensuse.org/opensuse-updates/2016-02/msg00042.html", "https://bugzilla.redhat.com/show_bug.cgi?id=1302589", "http://www.ubuntu.com/usn/USN-2892-1", "http://mailman.nginx.org/pipermail/nginx/2016-January/049700.html", "http://www.debian.org/security/2016/dsa-3473", "https://bto.bluecoat.com/security-advisory/sa115", "http://www.securitytracker.com/id/1034869", "https://security.gentoo.org/glsa/201606-06", "https://access.redhat.com/errata/RHSA-2016:1425", "https://support.apple.com/kb/HT212818", "http://seclists.org/fulldisclosure/2021/Sep/36"], "cvss": 5.0, "summary": "The resolver in nginx before 1.8.1 and 1.9.x before 1.9.10 does not properly limit CNAME resolution, which allows remote attackers to cause a denial of service (worker process resource consumption) via vectors related to arbitrary name resolution."}, "CVE-2016-0746": {"verified": false, "references": ["http://lists.opensuse.org/opensuse-updates/2016-02/msg00042.html", "https://bugzilla.redhat.com/show_bug.cgi?id=1302588", "http://www.ubuntu.com/usn/USN-2892-1", "http://mailman.nginx.org/pipermail/nginx/2016-January/049700.html", "http://www.debian.org/security/2016/dsa-3473", "https://bto.bluecoat.com/security-advisory/sa115", "http://www.securitytracker.com/id/1034869", "https://security.gentoo.org/glsa/201606-06", "https://access.redhat.com/errata/RHSA-2016:1425", "https://support.apple.com/kb/HT212818", "http://seclists.org/fulldisclosure/2021/Sep/36"], "cvss": 7.5, "summary": "Use-after-free vulnerability in the resolver in nginx 0.6.18 through 1.8.0 and 1.9.x before 1.9.10 allows remote attackers to cause a denial of service (worker process crash) or possibly have unspecified other impact via a crafted DNS response related to CNAME response processing."}, "CVE-2021-3618": {"verified": false, "references": ["https://bugzilla.redhat.com/show_bug.cgi?id=1975623", "https://alpaca-attack.com/", "https://lists.debian.org/debian-lts-announce/2022/11/msg00031.html"], "cvss": 5.8, "summary": "ALPACA is an application layer protocol content confusion attack, exploiting TLS servers implementing different protocols but using compatible certificates, such as multi-domain or wildcard certificates. A MiTM attacker having access to victim's traffic at the TCP/IP layer can redirect traffic from one subdomain to another, resulting in a valid TLS session. This breaks the authentication of TLS and cross-protocol attacks may be possible where the behavior of one protocol service may compromise the other at the application layer."}, "CVE-2021-23017": {"verified": false, "references": ["http://mailman.nginx.org/pipermail/nginx-announce/2021/000300.html", "https://support.f5.com/csp/article/K12331123,", "https://lists.apache.org/thread.html/r6fc5c57b38e93e36213e9a18c8a4e5dbd5ced1c7e57f08a1735975ba@%3Cnotifications.apisix.apache.org%3E", "https://lists.apache.org/thread.html/rf318aeeb4d7a3a312734780b47de83cefb7e6995da0b2cae5c28675c@%3Cnotifications.apisix.apache.org%3E", "https://lists.apache.org/thread.html/r37e6b2165f7c910d8e15fd54f4697857619ad2625f56583802004009@%3Cnotifications.apisix.apache.org%3E", "https://lists.apache.org/thread.html/rf232eecd47fdc44520192810560303073cefd684b321f85e311bad31@%3Cnotifications.apisix.apache.org%3E", "https://lists.apache.org/thread.html/r4d4966221ca399ce948ef34884652265729d7d9ef8179c78d7f17e7f@%3Cnotifications.apisix.apache.org%3E", "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/7SFVYHC7OXTEO4SMBWXDVK6E5IMEYMEE/", "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/GNKOP2JR5L7KCIZTJRZDCUPJTUONMC5I/", "https://security.netapp.com/advisory/ntap-20210708-0006/", "https://www.oracle.com/security-alerts/cpuoct2021.html", "https://www.oracle.com/security-alerts/cpujan2022.html", "https://www.oracle.com/security-alerts/cpuapr2022.html", "http://packetstormsecurity.com/files/167720/Nginx-1.20.0-Denial-Of-Service.html"], "cvss": 6.8, "summary": "A security issue in nginx resolver was identified, which might allow an attacker who is able to forge UDP packets from the DNS server to cause 1-byte memory overwrite, resulting in worker process crash or potential other impact."}, "CVE-2016-0742": {"verified": false, "references": ["http://lists.opensuse.org/opensuse-updates/2016-02/msg00042.html", "http://www.ubuntu.com/usn/USN-2892-1", "https://bugzilla.redhat.com/show_bug.cgi?id=1302587", "http://mailman.nginx.org/pipermail/nginx/2016-January/049700.html", "http://www.debian.org/security/2016/dsa-3473", "https://bto.bluecoat.com/security-advisory/sa115", "http://www.securitytracker.com/id/1034869", "https://security.gentoo.org/glsa/201606-06", "https://access.redhat.com/errata/RHSA-2016:1425", "https://support.apple.com/kb/HT212818", "http://seclists.org/fulldisclosure/2021/Sep/36"], "cvss": 5.0, "summary": "The resolver in nginx before 1.8.1 and 1.9.x before 1.9.10 allows remote attackers to cause a denial of service (invalid pointer dereference and worker process crash) via a crafted UDP DNS response."}}, "product": "nginx", "hash": 277944797, "tags": ["self-signed", "eol-product"], "timestamp": "2023-11-12T23:09:49.335090", "hostnames": [], "org": "dummy org string", "data": "HTTP/1.1 401 Unauthorized\r\nServer: nginx/1.7.10\r\nDate: Sun, 12 Nov 2023 23:09:49 GMT\r\nContent-Type: text/html; charset=UTF-8\r\nTransfer-Encoding: chunked\r\nConnection: keep-alive\r\nWWW-Authenticate: Basic realm=\"Secure Zone\"\r\nStrict-Transport-Security: max-age=31536000; includeSubDomains\r\nX-Frame-Options: SAMEORIGIN\r\nContent-Security-Policy: block-all-mixed-content; base-uri 'self'; default-src 'self'; script-src 'self' 'nonce-Lp9UV30PZQhP8w5FNPc4d81YafCOL8dR'; style-src 'self' 'nonce-Lp9UV30PZQhP8w5FNPc4d81YafCOL8dR'; img-src 'self'; connect-src 'self'; font-src 'self'; object-src 'self'; media-src 'self'; form-action 'self'; frame-ancestors 'self';\r\n\r\n", "asn": "AS65551", "cpe23": ["cpe:2.3:a:f5:nginx:1.7.10"], "isp": "dummy org string", "cpe": ["cpe:/a:f5:nginx:1.7.10"], "domains": [], "ip_str": "127.6.1.1", "os": null, "_shodan": {"region": "na", "module": "https", "ptr": true, "options": {}, "id": "16f2ff5c-ff41-424d-992f-c3901c01a267", "crawler": "eb6a3b0c4c23bbec821e9a56c850ef9b7c5c7f4c"}, "opts": {"vulns": [], "heartbleed": "2023/11/12 23:10:10 127.6.1.1:443 - SAFE\n"}}], "asn": "AS65551", "isp": "dummy org string", "last_update": "2023-11-13T08:10:29.397104", "vulns": ["CVE-2017-7529", "CVE-2016-4450", "CVE-2017-20005", "CVE-2021-23017", "CVE-2016-0742", "CVE-2016-1247", "CVE-2019-20372", "CVE-2016-0746", "CVE-2021-3618", "CVE-2016-0747", "CVE-2018-16845"], "country_name": "Czechia", "ip_str": "127.6.1.1", "os": null, "ports": [161, 179, 443]}