Merge pull request #53 from F5-Labs/development

Fixed some Wappalyzer and Geolocation lookup issues

cryptonice/gethttp.py

@@ -82,8 +82,11 @@ def get_http(ip_address, hostname, int_port, usetls, http_pages, force_redirect)
                 else:
                     str_location = split_location(res.getheader('Location'))
                     str_protocol = str_location[0]
-                    str_host = str_location[1]
-                    str_path = str_location[2]
+                    try:
+                        str_host = str_location[1]
+                        str_path = str_location[2]
+                    except:
+                        pass
             else:
                 pass
                 # print(f'{int_redirect}: Finished. Status = {int_status}')
@@ -286,30 +289,32 @@ def get_http(ip_address, hostname, int_port, usetls, http_pages, force_redirect)
 
 
     #### Wappalyzer build #####
+    try:
+        webpage = {}
 
-    webpage = {}
-
-    webpage['url'] = str_host + str_path
-    webpage['headers'] = res.headers
-    webpage['response'] = str(pagebody)
-    webpage['html'] = BeautifulSoup(str(pagebody), 'html.parser')
-    webpage['scripts'] = [script['src'] for script in webpage['html'].findAll('script', src=True)]
-    webpage['metatags'] = {meta['name'].lower(): meta['content']
-        for meta in webpage['html'].findAll('meta', attrs=dict(name=True, content=True))}
+        webpage['url'] = str_host + str_path
+        webpage['headers'] = res.headers
+        webpage['response'] = str(pagebody)
+        webpage['html'] = BeautifulSoup(str(pagebody), 'html.parser')
+        webpage['scripts'] = [script['src'] for script in webpage['html'].findAll('script', src=True)]
+        webpage['metatags'] = {meta['name'].lower(): meta['content']
+            for meta in webpage['html'].findAll('meta', attrs=dict(name=True, content=True))}
 
-    conn.close()
+        conn.close()
 
-    page = {}
-    page['scripts'] = webpage['scripts']
-    page['metatags'] = webpage['metatags']
+        page = {}
+        page['scripts'] = webpage['scripts']
+        page['metatags'] = webpage['metatags']
 
-    wapped = {}
-    elements = wappalyze(webpage)
-    for x in elements.items():
-        wapped.update({str(x[0]): x[1]})
+        wapped = {}
+        elements = wappalyze(webpage)
+        for x in elements.items():
+            wapped.update({str(x[0]): x[1]})
 
-    connection_data.update({'Components': wapped})
-    connection_data.update({'Page': page})
+        connection_data.update({'Components': wapped})
+        connection_data.update({'Page': page})
+    except:
+        pass
 
     return [str_host, str_path, b_httptohttps], connection_data
 

cryptonice/scanner.py

@@ -3,7 +3,11 @@
 
 import socket, ipaddress
 
-from .getgeo import getlocation
+try:
+    from .getgeo import getlocation
+except ImportError:
+    pass
+
 
 from .output import writeToJSONFile, print_to_console
 from .gettls import tls_scan
@@ -200,7 +204,10 @@ def scanner_driver(input_data):
                 # Lookup geolocation using Maxmind database
                 # NOTE: This is not enabled by default for public users of Cryptonice
                 if geolocation:
-                    geo_data = getlocation(ip_address)
+                    try:
+                        geo_data = getlocation(ip_address)
+                    except:
+                        print('You must have the Maxmind GeoIP 2 module installed to make use of geolocation lookups')
                 ###########
 
                 if 'TLS' in str(input_data['scans']).upper():

feedburner.com.json

@@ -0,0 +1 @@
+{"scan_metadata": {"cryptonice_version": "1.4.1.6", "job_id": "Test scan", "hostname": "feedburner.com", "port": 443, "node_name": "CHR-ML-00038111", "site_pos": 0, "http_to_https": false, "status": "Successful", "start": "2021-07-23 14:29:14.575297", "end": "2021-07-23 14:29:56.167491"}, "http": {"Connection": {"hostname": "accounts.google.com", "path": "/ServiceLogin?passive=1209600&continue=https%3A%2F%2Faccounts.google.com%2F&followup=https%3A%2F%2Faccounts.google.com%2F"}, "Headers": {"Access-Control-Allow-Origin": null, "Access-Control-Allow-Credentials": null, "Access-Control-Expose-Headers": null, "Access-Control-Max-Age": null, "Access-Control-Allow-Methods": null, "Access-Control-Allow-Headers": null, "Allow": null, "Alt-Svc": {"h3": ":443", "ma": "2592000", "v": "46,43"}, "Content-Encoding": null, "Content-Language": null, "Content-Length": null, "Content-Location": null, "Content-Type": "text/html; charset=UTF-8", "ETag": null, "Location": null, "Origin": null, "Public-Key-Pins": null, "Server": "GSE", "Strict-Transport-Security": "max-age=31536000; includeSubDomains", "Transfer-Encoding": "chunked", "Tk": null, "Upgrade": null, "Via": null, "WWW-Authenticate": null, "X-Frame-Options": "DENY", "Content-Security-Policy": "script-src 'report-sample' 'nonce-MbMItwXvMMDH6dD48aTNFw' 'unsafe-inline' 'unsafe-eval';object-src 'none';base-uri 'self';report-uri /cspreport", "X-Content-Security-Policy": null, "X-WebKit-CSP": null, "X-Powered-By": null, "X-XSS-Protection": "1; mode=block"}, "Cookies": {"cookie_1": {"__Host-GAPS": "1:MU3wucAxVQ3fgxgAM5eHLIK_fiX8Gw:pkxLP0uXAbTBR01s;Path=/;Expires=23-Jul-2023 13:29:36 GMT;Secure;HttpOnly;Priority=HIGH"}}, "Page": {"scripts": [], "metatags": {"viewport": "width=300, initial-scale=1", "google-site-verification": "LrdTUW9psUAMbh4Ia074-BPEVmcpBxF6Gwf0MSgQXZs"}}, "Components": {"Web servers": ["OpenGSE"], "Programming languages": ["Java"]}}, "http2": {"http2": true}, "tls": {"hostname": "accounts.google.com", "ip_address": "142.250.200.13", "cipher_suite_supported": "TLS_AES_256_GCM_SHA384", "client_authorization_requirement": "DISABLED", "highest_tls_version_supported": "TLS_1_3", "cert_recommendations": {}, "certificate_info": {"leaf_certificate_has_must_staple_extension": false, "leaf_certificate_is_ev": false, "received_chain_has_valid_order": true, "received_chain_has_contains_root": false, "leaf_certificate_signed_certificate_timestamps_count": 2, "leaf_certificate_subject_matches_hostname": true, "ocsp_response": "", "ocsp_response_is_trusted": null, "certificate_0": {"common_name": "accounts.google.com", "issuer_name": "GTS CA 1O1", "serial_number": "42513433863708116934742442472729213865", "fingerprint": "1195a4a444f701686f495fd3febd940423ef270874e2f124068ab6a4ce70ab5c", "public_key_algorithm": "EllipticCurve", "curve_algorithm": "secp256r1", "public_key_size": 256, "valid_from": "2021-06-28 04:15:03", "valid_until": "2021-09-20 04:15:02", "days_left": 58, "signature_algorithm": "sha256", "subject_alt_names": ["accounts.google.com", "*.partner.android.com"], "certificate_errors": {"cert_trusted": true, "hostname_matches": true}}, "certificate_1": {"common_name": "GTS CA 1O1", "issuer_name": "GlobalSign", "serial_number": "149699596615803609916394524856", "fingerprint": "95c074e35902a14abd9d19afb6e7f80e669ff8e2363270539d963613f04aaa21", "public_key_algorithm": "RSA", "public_key_size": 2048, "valid_from": "2017-06-15 00:00:42", "valid_until": "2021-12-15 00:00:42", "days_left": 144, "signature_algorithm": "sha256", "subject_alt_names": [], "certificate_errors": {"cert_trusted": true, "hostname_matches": true}}}, "ssl_2_0": {"preferred_cipher_suite": null, "accepted_ssl_2_0_cipher_suites": []}, "ssl_3_0": {"preferred_cipher_suite": null, "accepted_ssl_3_0_cipher_suites": []}, "tls_1_0": {"preferred_cipher_suite": null, "accepted_tls_1_0_cipher_suites": ["TLS_RSA_WITH_AES_256_CBC_SHA", "TLS_RSA_WITH_AES_128_CBC_SHA", "TLS_RSA_WITH_3DES_EDE_CBC_SHA", "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA", "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA"]}, "tls_1_1": {"preferred_cipher_suite": null, "accepted_tls_1_1_cipher_suites": ["TLS_RSA_WITH_AES_256_CBC_SHA", "TLS_RSA_WITH_AES_128_CBC_SHA", "TLS_RSA_WITH_3DES_EDE_CBC_SHA", "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA", "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA"]}, "tls_1_2": {"preferred_cipher_suite": null, "accepted_tls_1_2_cipher_suites": ["TLS_RSA_WITH_AES_256_GCM_SHA384", "TLS_RSA_WITH_AES_256_CBC_SHA", "TLS_RSA_WITH_AES_128_GCM_SHA256", "TLS_RSA_WITH_AES_128_CBC_SHA", "TLS_RSA_WITH_3DES_EDE_CBC_SHA", "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256", "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA", "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA", "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256"]}, "tls_1_3": {"preferred_cipher_suite": null, "accepted_tls_1_3_cipher_suites": ["TLS_CHACHA20_POLY1305_SHA256", "TLS_AES_256_GCM_SHA384", "TLS_AES_128_GCM_SHA256"]}, "tests": {"compression_supported": false, "accepts_early_data": true, "http_headers": {"strict_transport_security_header": {"preload": false, "include_subdomains": true, "max_age": 31536000}}}, "scan_information": {"tls_scan_start": "2021-07-23 14:29:38.242531", "tls_scan_end": "2021-07-23 14:29:52.901345", "scan_parameters": ["certificate_info", "ssl_2_0_cipher_suites", "ssl_3_0_cipher_suites", "tls_1_0_cipher_suites", "tls_1_1_cipher_suites", "tls_1_2_cipher_suites", "tls_1_3_cipher_suites", "tls_compression", "tls_1_3_early_data", "http_headers"], "commands_with_errors": {}}, "tls_recommendations": {"HIGH - TLSv1.0": "Major browsers are disabling TLS 1.0 imminently. Carefully monitor if clients still use this protocol. ", "HIGH - 3DES": "The 3DES symmetric cipher is vulnerable to the Sweet32 attack", "HIGH - TLSv1.1": "Major browsers are disabling this TLS 1.1 immenently. Carefully monitor if clients still use this protocol. "}, "pwnedkeys": {"pwned": false}, "jarm": {"fingerprint": "27d40d40d29d40d1dc42d43d00041d4689ee210389f4f6b4b5b1b93f92252d"}}, "dns": {"Connection": "feedburner.com", "dns_recommendations": {}, "records": {"A": ["142.250.187.206"], "CAA": ["0 issue \"pki.goog\""], "TXT": [], "MX": ["0 smtp.google.com."]}}, "geo": {}}

nike.com.json

@@ -0,0 +1 @@
+{"scan_metadata": {"cryptonice_version": "1.4.1.6", "job_id": "Test scan", "hostname": "nike.com", "port": 443, "node_name": "CHR-ML-00038111", "site_pos": 0, "http_to_https": true, "status": "Successful", "start": "2021-07-23 14:10:17.983825", "end": "2021-07-23 14:10:37.630514"}, "http": {"Connection": {"hostname": "www.nike.com", "path": "/"}, "Headers": {"Access-Control-Allow-Origin": null, "Access-Control-Allow-Credentials": null, "Access-Control-Expose-Headers": null, "Access-Control-Max-Age": null, "Access-Control-Allow-Methods": null, "Access-Control-Allow-Headers": null, "Allow": null, "Content-Encoding": null, "Content-Language": null, "Content-Length": "262", "Content-Location": null, "Content-Type": "text/html", "ETag": null, "Location": null, "Origin": null, "Public-Key-Pins": null, "Server": "AkamaiGHost", "Strict-Transport-Security": null, "Transfer-Encoding": null, "Tk": null, "Upgrade": null, "Via": null, "WWW-Authenticate": null, "X-Frame-Options": null, "Content-Security-Policy": null, "X-Content-Security-Policy": null, "X-WebKit-CSP": null, "X-Powered-By": null, "X-XSS-Protection": null}, "Cookies": {"cookie_1": {"AnalysisUserId": "95.101.143.151.85071627045822190", "expires": "31-Dec-2038 23:59:59 GMT", "path": "/", "domain": ".nike.com"}, "cookie_2": {"geoloc": "cc=GB,rc=EN,tp=vhigh,tz=GMT,la=51.50,lo=-0.12", "path": "/", "domain": ".nike.com"}, "cookie_3": {"anonymousId": "68B3ACFBE29894076C206E68406135FD", "expires": "06-Aug-2021 13:10:22 GMT"}}, "Page": {"scripts": [], "metatags": {}}, "Components": {}}, "http2": {"http2": true}, "tls": {"hostname": "www.nike.com", "ip_address": "96.16.108.133", "cipher_suite_supported": "TLS_AES_256_GCM_SHA384", "client_authorization_requirement": "DISABLED", "highest_tls_version_supported": "TLS_1_3", "cert_recommendations": {}, "certificate_info": {"leaf_certificate_has_must_staple_extension": false, "leaf_certificate_is_ev": true, "received_chain_has_valid_order": true, "received_chain_has_contains_root": false, "leaf_certificate_signed_certificate_timestamps_count": 3, "leaf_certificate_subject_matches_hostname": true, "ocsp_response": "", "ocsp_response_is_trusted": true, "certificate_0": {"common_name": "www.nike.com", "issuer_name": "DigiCert ECC Extended Validation Server CA", "serial_number": "19352687275285415863728324306145802667", "fingerprint": "16ec5099814d93ef7e3f998d1a0d782d2d3af0cd92c3526cc990d54632e9fd9c", "public_key_algorithm": "EllipticCurve", "curve_algorithm": "secp256r1", "public_key_size": 256, "valid_from": "2020-04-16 00:00:00", "valid_until": "2021-10-05 12:00:00", "days_left": 73, "signature_algorithm": "sha384", "subject_alt_names": ["www.nike.com", "nike.com", "busca.nike.com.br", "jobs.nike.com", "niketeam.nike.com", "www.nike.com.hk", "store.nike.com", "img.nike.com.hk", "unite.nike.com.br", "softlaunch.nike.com.br", "m.nike.com", "secure-store.nike.com", "www.nike.com.br", "elite.nike.com", "news.nike.com", "help.nike.com", "about.nike.com", "m.nike.com.hk", "communityimpact.nike.com", "web.nike.com"], "certificate_errors": {"cert_trusted": true, "hostname_matches": true}}}, "ssl_2_0": {"preferred_cipher_suite": null, "accepted_ssl_2_0_cipher_suites": []}, "ssl_3_0": {"preferred_cipher_suite": null, "accepted_ssl_3_0_cipher_suites": []}, "tls_1_0": {"preferred_cipher_suite": null, "accepted_tls_1_0_cipher_suites": []}, "tls_1_1": {"preferred_cipher_suite": null, "accepted_tls_1_1_cipher_suites": []}, "tls_1_2": {"preferred_cipher_suite": null, "accepted_tls_1_2_cipher_suites": ["TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256", "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256", "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256"]}, "tls_1_3": {"preferred_cipher_suite": null, "accepted_tls_1_3_cipher_suites": ["TLS_CHACHA20_POLY1305_SHA256", "TLS_AES_256_GCM_SHA384", "TLS_AES_128_GCM_SHA256"]}, "tests": {"compression_supported": false, "accepts_early_data": false, "http_headers": {}}, "scan_information": {"tls_scan_start": "2021-07-23 14:10:22.404144", "tls_scan_end": "2021-07-23 14:10:34.004153", "scan_parameters": ["certificate_info", "ssl_2_0_cipher_suites", "ssl_3_0_cipher_suites", "tls_1_0_cipher_suites", "tls_1_1_cipher_suites", "tls_1_2_cipher_suites", "tls_1_3_cipher_suites", "tls_compression", "tls_1_3_early_data", "http_headers"], "commands_with_errors": {}}, "tls_recommendations": {}, "pwnedkeys": {"pwned": false}, "jarm": {"fingerprint": "29d29d00029d29d00041d41d00041d69337e5f535144f26f5d7e01b189f9d0"}}, "dns": {"Connection": "nike.com", "dns_recommendations": {"Low - CAA": "Consider creating DNS CAA records to prevent accidental or malicious certificate issuance."}, "records": {"A": ["52.85.104.7", "52.85.104.34", "52.85.104.56", "52.85.104.30"], "CAA": [], "TXT": ["\"mongodb-site-verification=POJ3Ib5m6oOyqX91ZrlgcbAFVKwRhMml\"", "\"scnv-verification=03fe4bf81654621682a5c049867fccb4:4fa32facf3797aa643beb4108a6c956c:d986a0a5e1ca4ceb8941e8d9ed54eb11\"", "\"soa header update\"", "\"stripe-verification=c814f4607c0171c39868805fb1ae7723a2d91d54b0c3149eb8ec46aad7a7be6f\"", "\"v=spf1 include:spf.protection.outlook.com include:spf.nike.com ip4:146.197.185.243 ip4:146.197.185.244 ip4:146.197.27.210 ip4:146.197.27.211 ip4:146.197.27.212 ~all\"", "\"validate post vanity\"", "\"validation and verification zone\"", "\"6IHSYqvkFCDiMubO/Czv+XdVcUWb8weKG9uyqn9Kh0u3RZwA+xx73W/f+iQrSciT+peqOK3yNcx1jsIqsHgY+g==\"", "\"MS=ms93873368\"", "\"Validation no ext\"", "\"ZOOM_verify_sCcWC00VT72Xd3cKn3ByDQ\"", "\"_globalsign-domain-verification=oTFxDWw1mdzzP-Zd45wwC6Ifc6nSakWOmRQSYD_XHz\"", "\"adobe-idp-site-verification=6f949b65-8b97-4bb1-be7a-6088c7b96a98\"", "\"amazonses:guDcMedRx3D3yaitJGO/aV+NzhugJRiKDDdKbUXqbLM=\"", "\"apple-domain-verification=FIPflKu9G6kEqyab\"", "\"cisco-ci-domain-verification=7ab30713f81d07c44eea6bc370f3a51f2ca7d6c0477a8db10974320ec4bef38f\"", "\"cloudhealth=a8cf6829-a2b8-4f9f-9945-f71659ede7ff\"", "\"facebook-domain-verification=9qrf9w3jjvoc7r3ry2mv6fds4qd9jd\"", "\"github-verification=9rQmuxzyEyx4yq7LjueTPBmTFPNf9xcMKDBpDkkw\"", "\"google-site-verification=0tcT2lcGC5FZjgDR2BU_t4PwfRpG5TdMC3W7Uq8tleI\"", "\"google-site-verification=AqwT-_WzscdFOiacOoMpw-aMjwRDmJLuT9cHH3zFfkw\"", "\"google-site-verification=JjQcml7P2JjvkTRFwUtkeEGy6-Pehs-zJ3YC2bVcCp4\""], "MX": ["10 mxa-001b6002.gslb.pphosted.com.", "10 mxb-001b6002.gslb.pphosted.com."]}}}

sample_scan.json

@@ -15,7 +15,8 @@
 	"force_redirect": true,
 	"print_out": true,
 	"generate_json": true,
-	"json_path" : "./results",
+	"json_path" : "./",
 	"recommendations": false,
-	"targets": ["1rx.io", "nike.com", "www.nike.com", "f5.com","www.f5labs.com"]
+	"geolocation": true,
+	"targets": ["feedburner.com"]
 }