Merge pull request #40 from F5-Labs/development

Development

Dockerfile

@@ -1,5 +1,5 @@
 FROM python:3.7
-LABEL version="1.3.8"
+LABEL version="1.4.0.1"
 LABEL maintainer="f5labs@f5.com"
 RUN pip3 install pycurl
 RUN pip3 install cryptonice

build/lib/cryptonice/checkport.py

@@ -0,0 +1,37 @@
+# cryptonice
+# checkport.py
+
+import socket
+import http.client
+import ssl
+
+
+def port_open(hostname, port):
+    #print('\nOpen port checks')
+    #print('------------------------------')
+    """
+    Check status of a port and return simple True/False
+    Also check for TLS handshake
+    """
+    open_port = False
+    open_tls = True  # change to False later
+
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+        sock_result = sock.connect_ex((hostname, port))
+
+        if sock_result == 0:
+            open_port = True
+            try:
+                conn = http.client.HTTPSConnection(hostname, port, timeout=3, context=ssl._create_unverified_context())
+                conn.request("GET", "/")
+                res = conn.getresponse()
+                conn.close()
+                open_tls = True
+            except:
+                pass
+
+    return open_port, open_tls
+
+
+if __name__ == "__main__":
+    port_open()

build/lib/cryptonice/getdns.py

@@ -0,0 +1,80 @@
+# cryptonice
+# getdns.py
+
+import dns.resolver
+
+
+def getDNSRecord(hostname, record_type):
+    record_list = []
+    got_record = False
+    #testdomain = dns.name.from_text(hostname)
+    #print(testdomain.labels)
+
+    try:
+        while not got_record:
+            '''
+            1. Lookup record type
+            2. Detect if SOA exists
+            3. Update hostname to target of SOA
+            4. Iterate with this new hostname
+            '''
+            answer = dns.resolver.resolve(hostname, record_type, raise_on_no_answer=False)
+
+            if answer.rrset is None:
+                result = answer.response.authority[0].to_text()
+                if "SOA" in result:
+                    # Check for SOA's pointing to the same hostname and exit loop, if so
+                    if result.split('. ')[0] == hostname:
+                        got_record = True
+                    else:
+                        hostname = result.split('. ')[0]
+            else:
+                for ipval in answer:
+                    record_list.append(ipval.to_text())
+                    got_record = True
+    except:
+        record_list = []
+
+    return record_list
+
+
+def get_dns(hostname, all_checks):
+    print(f'Analyzing DNS data for {hostname}')
+    connection_data = {}
+    host_data = {}
+    dns_data = {}
+    dns_recommendations = {}
+
+    host_data.update({'hostname': hostname})
+    connection_data.update({'Connection': hostname})
+
+    # DEBUG
+    dns_data.update({'A': getDNSRecord(hostname, 'A')})
+
+    if all_checks:
+        # Certain DNS records, such as CAA, are usually only present when querying the 'root' domain
+        # THIS WON'T WORK FOR DOMAINS THAT HAVE A SUB-DOMAIN AND USE CNAME RECORDS TO DIRECT USERS TO ANOTHER DOMAIN
+        root_domain = hostname.replace('www.', '')  # make sure domain name does not have any prefix
+        try:
+            root_host = root_domain.split('/', 1)[0]  # will remove a path if it exists
+        except:
+            root_host = root_domain
+
+        print(f'Fetching additional records for {root_host}')
+
+        dns_caa = getDNSRecord(root_host, 'CAA')
+        if len(dns_caa) == 0:
+            dns_recommendations.update({'Low - CAA': 'Consider creating DNS CAA records to prevent accidental or malicious certificate issuance.'})
+
+        dns_data.update({'CAA': dns_caa})
+        dns_data.update({'TXT': getDNSRecord(root_host, 'TXT')})
+        dns_data.update({'MX': getDNSRecord(root_host, 'MX')})
+
+    connection_data.update({'dns_recommendations': dns_recommendations})
+    connection_data.update({'records': dns_data})
+
+    return connection_data
+
+
+if __name__ == "__main__":
+    get_dns()

build/lib/cryptonice/getentropy.py

@@ -0,0 +1,71 @@
+#for entropy
+import numpy as np
+from scipy.stats import entropy
+from math import log, e
+import pandas as pd
+
+#for url parsing
+from urllib.parse import urlparse
+
+
+def splitdomain(url):
+    parse_object = urlparse(url)
+    base = parse_object.netloc
+    path = parse_object.path
+    scheme = parse_object.scheme
+
+    return base, path, scheme
+
+
+def percentage(part, whole):
+    return 100 * float(part)/float(whole)
+
+
+def entropy(labels, base=2):
+    """ Computes entropy of label distribution. """
+
+    n_labels = len(labels)
+
+    if n_labels <= 1:
+        return 0
+
+    value,counts = np.unique(labels, return_counts=True)
+    probs = counts / n_labels
+    n_classes = np.count_nonzero(probs)
+
+    if n_classes <= 1:
+        return 0
+
+    ent = 0.
+
+    # Compute entropy
+    base = e if base is None else base
+    for i in probs:
+        ent -= i * log(i, base)
+
+    return ent
+
+
+
+print("Starting...")
+
+
+
+# Step 2. Open the list of domains, line by line, in to a dictionary object
+import csv
+
+with open('Phishstats.csv') as csv_file:
+    csv_reader = csv.reader(csv_file, delimiter=',')
+    line_count = 0
+    for line in csv_reader:
+        score = line[1]
+        url = line[2]
+        ip = line[3]
+        domain, uri, scheme = splitdomain(url)
+        ent = entropy(list(domain))
+
+        print(f"{domain} score = {score}, entropy = {ent}")
+
+
+
+print("Finished.")