Merge pull request #54 from F5-Labs/development

Wappalyzer fixes
F5-Labs · Jul 29, 2021 · 4238762 · 4238762
2 parents fe45cff + 5d4cb4c
commit 4238762
Show file tree

Hide file tree

Showing 20 changed files with 9,656 additions and 1,163 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,5 +1,5 @@
 FROM python:3.7
-LABEL version="1.4.1.6"
+LABEL version="1.4.2.1"
 LABEL maintainer="f5labs@f5.com"
 RUN pip3 install pycurl
 RUN pip3 install cryptonice

diff --git a/cryptonice/__init__.py b/cryptonice/__init__.py
@@ -2,4 +2,4 @@
 # __init__.py
 
 # Version of cryptonice package
-__version__ = "1.4.1.6"
+__version__ = "1.4.2.1"
diff --git a/cryptonice/__pycache__/__init__.cpython-38.pyc b/cryptonice/__pycache__/__init__.cpython-38.pyc
diff --git a/cryptonice/__pycache__/checkport.cpython-38.pyc b/cryptonice/__pycache__/checkport.cpython-38.pyc
diff --git a/cryptonice/__pycache__/getdns.cpython-38.pyc b/cryptonice/__pycache__/getdns.cpython-38.pyc
diff --git a/cryptonice/__pycache__/getgeo.cpython-38.pyc b/cryptonice/__pycache__/getgeo.cpython-38.pyc
diff --git a/cryptonice/__pycache__/gethttp.cpython-38.pyc b/cryptonice/__pycache__/gethttp.cpython-38.pyc
diff --git a/cryptonice/__pycache__/gethttp2.cpython-38.pyc b/cryptonice/__pycache__/gethttp2.cpython-38.pyc
diff --git a/cryptonice/__pycache__/gettls.cpython-38.pyc b/cryptonice/__pycache__/gettls.cpython-38.pyc
diff --git a/cryptonice/__pycache__/jarm.cpython-38.pyc b/cryptonice/__pycache__/jarm.cpython-38.pyc
diff --git a/cryptonice/__pycache__/pwnedkeys.cpython-38.pyc b/cryptonice/__pycache__/pwnedkeys.cpython-38.pyc
diff --git a/cryptonice/__pycache__/scanner.cpython-38.pyc b/cryptonice/__pycache__/scanner.cpython-38.pyc
diff --git a/cryptonice/__pycache__/wappalyze.cpython-38.pyc b/cryptonice/__pycache__/wappalyze.cpython-38.pyc
diff --git a/cryptonice/gethttp.py b/cryptonice/gethttp.py
@@ -136,6 +136,7 @@ def get_http(ip_address, hostname, int_port, usetls, http_pages, force_redirect)
                 conn = http.client.HTTPConnection(ip_address, int_port, timeout=5)
                 conn.request('GET', str_path, headers={"Host": hostname})
                 res = conn.getresponse()
+                pagebody = res.read()
                 conn.close()
             except:
                 return [str_host, str_path, b_httptohttps], []
@@ -285,36 +286,36 @@ def get_http(ip_address, hostname, int_port, usetls, http_pages, force_redirect)
 
     # Include page data in output if requested
     if http_pages:
-        connection_data.update({'Page': pagebody})
+        connection_data.update({'HTMLbody': pagebody})
 
 
     #### Wappalyzer build #####
-    try:
-        webpage = {}
-
-        webpage['url'] = str_host + str_path
-        webpage['headers'] = res.headers
-        webpage['response'] = str(pagebody)
-        webpage['html'] = BeautifulSoup(str(pagebody), 'html.parser')
-        webpage['scripts'] = [script['src'] for script in webpage['html'].findAll('script', src=True)]
-        webpage['metatags'] = {meta['name'].lower(): meta['content']
-            for meta in webpage['html'].findAll('meta', attrs=dict(name=True, content=True))}
-
-        conn.close()
-
-        page = {}
-        page['scripts'] = webpage['scripts']
-        page['metatags'] = webpage['metatags']
-
-        wapped = {}
-        elements = wappalyze(webpage)
-        for x in elements.items():
-            wapped.update({str(x[0]): x[1]})
-
-        connection_data.update({'Components': wapped})
-        connection_data.update({'Page': page})
-    except:
-        pass
+    #try:
+    webpage = {}
+
+    webpage['url'] = str_host + str_path
+    webpage['headers'] = res.headers
+    webpage['response'] = str(pagebody)
+    webpage['html'] = BeautifulSoup(str(pagebody), 'html.parser')
+    webpage['scripts'] = [script['src'] for script in webpage['html'].findAll('script', src=True)]
+    webpage['metatags'] = {meta['name'].lower(): meta['content']
+        for meta in webpage['html'].findAll('meta', attrs=dict(name=True, content=True))}
+
+    conn.close()
+
+    page = {}
+    page['scripts'] = webpage['scripts']
+    page['metatags'] = webpage['metatags']
+
+    wapped = {}
+    elements = wappalyze(webpage)
+    for x in elements.items():
+        wapped.update({str(x[0]): x[1]})
+
+    connection_data.update({'Components': wapped})
+    connection_data.update({'Page': page})
+    #except:
+        #pass
 
     return [str_host, str_path, b_httptohttps], connection_data
 

diff --git a/cryptonice/scanner.py b/cryptonice/scanner.py
@@ -139,8 +139,8 @@ def scanner_driver(input_data):
                     goodToGo = True
                     print(f'{hostname} resolves to {ip_address}')
                 except:
-                    str_error = "ERROR: Unable to resolve " + hostname + "(domain does not exist or may not have any A records)"
-                    dns_data = {str_error}
+                    str_error = "Unable to resolve " + hostname + "(domain does not exist or may not have any A records)"
+                    dns_data = {'Error:' : str_error}
                     print(str_error)
 
         #########################################################################################################
@@ -193,8 +193,8 @@ def scanner_driver(input_data):
                             goodToGo = True
                             print(f'{str_host} resolves to {ip_address}')
                         except:
-                            str_error = "ERROR: Unable to resolve " + str_host + "(domain does not exist or may not have any A records)"
-                            new_dns_data = {str_error}
+                            str_error = "Unable to resolve " + str_host + "(domain does not exist or may not have any A records)"
+                            new_dns_data = {'Error': str_error}
                             print(str_error)
 
                     # Recheck redirection on hew host