Merge pull request #48 from F5-Labs/development

Development
F5-Labs · Jul 14, 2021 · 65f82cc · 65f82cc
2 parents 88e9b82 + 44b0384
commit 65f82cc
Show file tree

Hide file tree

Showing 18 changed files with 122 additions and 103 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,5 +1,5 @@
 FROM python:3.7
-LABEL version="1.4.1"
+LABEL version="1.4.1.1"
 LABEL maintainer="f5labs@f5.com"
 RUN pip3 install pycurl
 RUN pip3 install cryptonice

diff --git a/awssetup.py b/awssetup.py
@@ -55,11 +55,10 @@ def __init__(self, script, targetName):  # type: ignore
     package_data={'cryptonice': ['technologies.json', './maxmind/*.mmdb']},
     install_requires=["sslyze>=4.0.0",
                       "dnspython>=2.0.0",
-                      "http-client>=0.1.21",
                       "urllib3>=1.22",
                       "ipaddress>=1.0.22",
                       "pathlib~=1.0.1",
-                      "bs4>=0.0.1",
+                      "beautifulsoup4>=4.9.0",
                       "regex>=2020.5.14",
                       "geoip2"],
     entry_points={

diff --git a/cryptonice/__init__.py b/cryptonice/__init__.py
@@ -2,4 +2,4 @@
 # __init__.py
 
 # Version of cryptonice package
-__version__ = "1.4.1"
+__version__ = "1.4.1.1"
diff --git a/cryptonice/__pycache__/__init__.cpython-38.pyc b/cryptonice/__pycache__/__init__.cpython-38.pyc
diff --git a/cryptonice/__pycache__/__init__.cpython-39.pyc b/cryptonice/__pycache__/__init__.cpython-39.pyc
diff --git a/cryptonice/__pycache__/gethttp.cpython-38.pyc b/cryptonice/__pycache__/gethttp.cpython-38.pyc
diff --git a/cryptonice/__pycache__/pwnedkeys.cpython-38.pyc b/cryptonice/__pycache__/pwnedkeys.cpython-38.pyc
diff --git a/cryptonice/__pycache__/scanner.cpython-38.pyc b/cryptonice/__pycache__/scanner.cpython-38.pyc
diff --git a/cryptonice/__pycache__/scanner.cpython-39.pyc b/cryptonice/__pycache__/scanner.cpython-39.pyc
diff --git a/cryptonice/__pycache__/wappalyze.cpython-38.pyc b/cryptonice/__pycache__/wappalyze.cpython-38.pyc
diff --git a/cryptonice/__pycache__/wappalyze.cpython-39.pyc b/cryptonice/__pycache__/wappalyze.cpython-39.pyc
diff --git a/cryptonice/gethttp.py b/cryptonice/gethttp.py
@@ -3,13 +3,11 @@
 
 import http.client
 import ssl
-
-# Wappalyzer Requirements
 import json
 import re
-from .wappalyze import wappalyze
-# import warnings
 import pkg_resources
+
+from .wappalyze import wappalyze
 from bs4 import BeautifulSoup
 
 
@@ -118,10 +116,9 @@ def get_http(ip_address, hostname, int_port, usetls, http_pages, force_redirect)
         # print(f'{int_redirect}: Checking {str_host} at {str_path}')
 
         if usetls:
-            # print(f'Attempting HTTPS connection to {ip_address} using SNI of {str_host}')
+            print(f'Attempting HTTPS connection to {ip_address} using SNI of {str_host}')
             try:
-                conn = http.client.HTTPSConnection(str_host, int_port, timeout=5,
-                                                   context=ssl._create_unverified_context())
+                conn = http.client.HTTPSConnection(str_host, int_port, timeout=5, context=ssl._create_unverified_context())
                 conn.request("GET", str_path)
                 res = conn.getresponse()
                 pagebody = res.read()
@@ -160,6 +157,17 @@ def get_http(ip_address, hostname, int_port, usetls, http_pages, force_redirect)
                     str_host = str_location[1]
                     str_path = str_location[2]
 
+                #Occassionally a redirect will include a port number. We need to strip this from the host but
+                #we can use it to update the int_port variable just in case we get redirected to a non-standard HTTPS port.
+                int_colon = str_host.find(":")
+                if int_colon > 0:
+                    int_port = int(str_host[int_colon+1:len(str_host)])
+                    str_host = str_host[0:str_host.find(":")]
+
+                # DEBUG
+                print(f'{int_redirect}: Found new location at {str_host} with path {str_path} on port {int_port}')
+
+
                 # Some redirects will not specify a new domain name
                 # This prevents us having an empty host if only a new path is specified
                 if str_host == "":

diff --git a/cryptonice/pwnedkeys.py b/cryptonice/pwnedkeys.py
@@ -3,14 +3,10 @@
 
 import http.client
 import ssl
-
-# import warnings
 import pkg_resources
-from bs4 import BeautifulSoup
 
 
 def check_key(cert_fingerprint):
-
     connection = http.client.HTTPSConnection("v1.pwnedkeys.com", 443, timeout=5, context=ssl._create_unverified_context())
     connection.request("GET", "/8d296b2f6c9e3f8913c7ac3689c5ce116943c62ab6640a759926895a2137a79c")
     response = connection.getresponse()

diff --git a/cryptonice/scanner.py b/cryptonice/scanner.py
@@ -111,112 +111,130 @@ def scanner_driver(input_data):
         # We can also check DNS regardless of open ports since it's an independent protocol
         # At this point there should be no WWW. prefix or path, but it's a good idea to check and scrub it anyway
 
+        goodToGo = False
+
         # First check if our target is actually a valid IP address...
         # If it is, set the IP address to the 'hostname',
         # if not, perform a DNS lookup
         try:
             ipaddress.ip_address(hostname)
             # If we have a valid IP, skip the DNS lookup...
             ip_address = hostname
+            goodToGo = True
             print(f'{hostname} is already a valid IP')
         except ValueError:
             # Determine if we are only using DNS to get an IP address, or whether we should query for all records
-            if 'DNS' in input_data['scans'] or 'dns' in input_data['scans']:
+            if 'DNS' in str(input_data['scans']).upper():
                 dns_data = get_dns(hostname, True)
             else:
                 dns_data = get_dns(hostname, False)
 
             if dns_data:
-                ip_address = dns_data.get('records').get('A')[0]  # get first IP in list
-                print(f'{hostname} resolves to {ip_address}')
-        #########################################################################################################
+                try:
+                    ip_address = dns_data.get('records').get('A')[0]  # get first IP in list
+                    goodToGo = True
+                    print(f'{hostname} resolves to {ip_address}')
+                except:
+                    str_error = "ERROR: Unable to resolve " + hostname + "(domain does not exist or may not have any A records)"
+                    dns_data = {str_error}
+                    print(str_error)
 
+        #########################################################################################################
 
-        ############
-        # Lookup geolocation using Maxmind database
-        # NOTE: This is not enabled by default for public users of Cryptonice
-        if geolocation:
-            geo_data = getlocation(ip_address)
-        ###########
-
+        #Assume we want to follow redirects unless the input parameter says otherwise
+        force_redirect = True
 
-        str_host = hostname  # will allow data to be outputted even if port is closed
-        # Now we begin the proper scans based on the port we've been asked to connect to
-        target_portopen, target_tlsopen = port_open(ip_address, port)
+        #All subsequent scans are now dependant on the goodToGo variable which is based on having a valid IP or performing a successful DNS lookup
+        if goodToGo:
+            ############
+            # Lookup geolocation using Maxmind database
+            # NOTE: This is not enabled by default for public users of Cryptonice
+            if geolocation:
+                geo_data = getlocation(ip_address)
+            ###########
 
-        try:
-            force_redirect = input_data['force_redirect']
-        except KeyError:
-            force_redirect = True
 
-        if target_portopen:
-            print(f'{ip_address}:{port}: OPEN')
-            print(f'TLS is available: {target_tlsopen}')
-            # First check for redirects on whatever port we were told to scan
-            # ...but only if we're connecting to port 80 or 443
-            # if port == 80 or port == 443:
-            #     # Now we also pass in whether TLS is available so we know whether to try TLS regardless of the port
-            #     redirection_results = redirect_hostname(hostname, port, target_tlsopen)
+            str_host = hostname  # will allow data to be outputted even if port is closed
+            # Now we begin the proper scans based on the port we've been asked to connect to
+            target_portopen, target_tlsopen = port_open(ip_address, port)
 
             try:
-                http_body = input_data['http_body']  # boolean variable for HTTP pages info
+                force_redirect = input_data['force_redirect']
             except KeyError:
-                http_body = False
-
-            redirection_results, http_data = get_http(ip_address, host_sni, port, target_tlsopen, http_body,
-                                                      force_redirect)
-
-            if redirection_results == 'ERROR: Connection failed':
-                str_host = hostname  # default to original hostname if redirects failed
-                str_path = '/'  # default path
-                b_httptohttps = False
-            else:
-                str_host = redirection_results[0]  # updated hostname
-                str_path = redirection_results[1]  # updated path
-                b_httptohttps = redirection_results[2]  # updated http to https redirect
-
-            if 'TLS' in input_data['scans'] or 'tls' in input_data['scans']:
-                if target_tlsopen:
-                    # List to hold desired ScanCommands for later
-                    commands_to_run = []
-                    # Read in command list
-                    try:
-                        tls_params = input_data['tls_params']
-                        # If the TLS parameters are blank but the TLS scan option is present, then assume a default set of scans to run
-                        if len(tls_params) == 0:
-                            commands_to_run = tls_defaults
-                        else:
-                            for param in tls_params:
-                                # If the tls_params value in the JSON input file is 'all' then apply every TLS scan function automatically
-                                if (param.upper() == "ALL"):
-                                    commands_to_run = tls_command_list
-                                else:
-                                    if param in tls_command_list:
-                                        commands_to_run.append(str(param))
-
-                        tls_data = tls_scan(ip_address, str_host, commands_to_run, port)
-
-                    except KeyError:
-                        tls_data = "No TLS scan parameters provided"
+                force_redirect = True
+
+            if target_portopen:
+                print(f'{ip_address}:{port}: OPEN')
+                print(f'TLS is available: {target_tlsopen}')
+                # First check for redirects on whatever port we were told to scan
+                # ...but only if we're connecting to port 80 or 443
+                # if port == 80 or port == 443:
+                #     # Now we also pass in whether TLS is available so we know whether to try TLS regardless of the port
+                #     redirection_results = redirect_hostname(hostname, port, target_tlsopen)
+
+                try:
+                    http_body = input_data['http_body']  # boolean variable for HTTP pages info
+                except KeyError:
+                    http_body = False
+
+                redirection_results, http_data = get_http(ip_address, host_sni, port, target_tlsopen, http_body,
+                                                          force_redirect)
+
+                if redirection_results == 'ERROR: Connection failed':
+                    str_host = hostname  # default to original hostname if redirects failed
+                    str_path = '/'  # default path
+                    b_httptohttps = False
                 else:
-                    tls_data = {'ERROR': 'Could not perform TLS handshake'}
-
-
-            if 'PWNED' in input_data['scans']:
-                cert_fingerprint = tls_data['certificate_info']['certificate_0']['fingerprint']
-                pwned_data = check_key(cert_fingerprint)
+                    str_host = redirection_results[0]  # updated hostname
+                    str_path = redirection_results[1]  # updated path
+                    b_httptohttps = redirection_results[2]  # updated http to https redirect
+
+                if 'TLS' in input_data['scans'] or 'tls' in input_data['scans']:
+                    if target_tlsopen:
+                        # List to hold desired ScanCommands for later
+                        commands_to_run = []
+                        # Read in command list
+                        try:
+                            tls_params = input_data['tls_params']
+                            # If the TLS parameters are blank but the TLS scan option is present, then assume a default set of scans to run
+                            if len(tls_params) == 0:
+                                commands_to_run = tls_defaults
+                            else:
+                                for param in tls_params:
+                                    # If the tls_params value in the JSON input file is 'all' then apply every TLS scan function automatically
+                                    if (param.upper() == "ALL"):
+                                        commands_to_run = tls_command_list
+                                    else:
+                                        if param in tls_command_list:
+                                            commands_to_run.append(str(param))
+
+                            tls_data = tls_scan(ip_address, str_host, commands_to_run, port)
+
+                        except KeyError:
+                            tls_data = "No TLS scan parameters provided"
+                    else:
+                        tls_data = {'ERROR': 'Could not perform TLS handshake'}
+
+
+                if 'PWNED' in input_data['scans']:
+                    cert_fingerprint = tls_data['certificate_info']['certificate_0']['fingerprint']
+                    pwned_data = check_key(cert_fingerprint)
+
+                if 'HTTP2' in input_data['scans'] or 'http2' in input_data['scans']:
+                    http2_data = check_http2(host_path, port)
+
+                #if 'JARM' in input_data['scans'] or 'jarm' in input_data['scans']:
+                jarm_data = check_jarm(host_path, port)
+
+                metadata.update({'http_to_https': b_httptohttps})
+                metadata.update({'status': "Successful"})
+            else:
+                metadata.update({'status': "Failed"})
+                print(f"{hostname}:{port} is closed")
 
-            if 'HTTP2' in input_data['scans'] or 'http2' in input_data['scans']:
-                http2_data = check_http2(host_path, port)
+        #End of goodToGo IF block
 
-            #if 'JARM' in input_data['scans'] or 'jarm' in input_data['scans']:
-            jarm_data = check_jarm(host_path, port)
 
-            metadata.update({'http_to_https': b_httptohttps})
-            metadata.update({'status': "Successful"})
-        else:
-            metadata.update({'status': "Failed"})
-            print(f"{hostname}:{port} is closed")
 
         end_time = datetime.today()
         metadata.update({'start': start_time.__str__()})
@@ -236,26 +254,29 @@ def scanner_driver(input_data):
             tls_data.update({'jarm': jarm_data})
         if tls_data:
             scan_data.update({'tls': tls_data})
-        if 'DNS' in input_data['scans'] or 'dns' in input_data['scans']:
+        if dns_data:
             scan_data.update({'dns': dns_data})
         if geolocation:
             scan_data.update({'geo': geo_data})
 
 
         if input_data['print_out']:
-            print_to_console(str_host, scan_data, b_httptohttps, force_redirect)
+            print_to_console(hostname, scan_data, b_httptohttps, force_redirect)
 
         print('\nScans complete')
         print('-------------------------------------')
         print(f'Total run time: {end_time - start_time}')
 
+
+
         if input_data['generate_json']:
             try:
                 pathToJson = input_data['json_path']
             except:
                 pathToJson = "./"
             writeToJSONFile(hostname, pathToJson, scan_data)
 
+
     return scan_data, hostname