From 7d2f49b1b03940cdced34cd26eb77a28bf1b5e2b Mon Sep 17 00:00:00 2001
From: Eric Mill <eric@konklone.com>
Date: Tue, 31 Oct 2017 18:06:21 -0400
Subject: [PATCH 01/18] local timeout that works on sslyze, but doesn't clean
 up processes

---
 requirements.txt   |  1 +
 scanners/sslyze.py | 34 +++++++++++++++++++++++++---------
 2 files changed, 26 insertions(+), 9 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 6d7bea46..63326e1b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,6 +6,7 @@ pyyaml
 # to support sslyze scanner
 sslyze
 cryptography
+timeout-decorator
 
 # to support censys gatherer
 censys
diff --git a/scanners/sslyze.py b/scanners/sslyze.py
index 2202a556..9367dcf4 100644
--- a/scanners/sslyze.py
+++ b/scanners/sslyze.py
@@ -2,6 +2,8 @@
 from scanners import utils
 import os
 
+import timeout_decorator
+
 import json
 import cryptography
 import cryptography.hazmat.backends.openssl
@@ -18,6 +20,14 @@
 
 command = os.environ.get("SSLYZE_PATH", "sslyze")
 
+# This timeout is enforced in this file, in Python, not in sslyze.
+timeout = 1
+
+
+# Blocked off the scan command to use a forcible timeout
+@timeout_decorator.timeout(timeout, use_signals=False)
+def actual_scan(scan_args):
+    return utils.scan(scan_args)
 
 def scan(domain, options):
     logging.debug("[%s][sslyze]" % domain)
@@ -59,15 +69,21 @@ def scan(domain, options):
 
         # This is --regular minus --heartbleed
         # See: https://github.com/nabla-c0d3/sslyze/issues/217
-        raw_response = utils.scan([
-            command,
-            "--sslv2", "--sslv3", "--tlsv1", "--tlsv1_1", "--tlsv1_2",
-            "--reneg", "--resum", "--certinfo",
-            "--http_get", "--hide_rejected_ciphers",
-            "--compression", "--openssl_ccs",
-            "--fallback", "--quiet",
-            scan_domain, "--json_out=%s" % cache_json
-        ])
+        raw_response = None
+
+        try:
+            raw_response = actual_scan([
+                command,
+                "--sslv2", "--sslv3", "--tlsv1", "--tlsv1_1", "--tlsv1_2",
+                "--reneg", "--resum", "--certinfo",
+                "--http_get", "--hide_rejected_ciphers",
+                "--compression", "--openssl_ccs",
+                "--fallback", "--quiet",
+                scan_domain, "--json_out=%s" % cache_json
+            ])
+        except timeout_decorator.timeout_decorator.TimeoutError:
+            # logging.warn(utils.format_last_exception())
+            logging.warn("\tTimeout error (%is) running sslyze." % timeout)
 
         if raw_response is None:
             # TODO: save standard invalid JSON data...?

From 1ea4f91c56d2594fad5cdbd4555d06ea1372e40f Mon Sep 17 00:00:00 2001
From: Eric Mill <eric@konklone.com>
Date: Tue, 31 Oct 2017 18:23:07 -0400
Subject: [PATCH 02/18] sane production timeout value

---
 scanners/sslyze.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scanners/sslyze.py b/scanners/sslyze.py
index 9367dcf4..48fae813 100644
--- a/scanners/sslyze.py
+++ b/scanners/sslyze.py
@@ -21,7 +21,7 @@
 command = os.environ.get("SSLYZE_PATH", "sslyze")
 
 # This timeout is enforced in this file, in Python, not in sslyze.
-timeout = 1
+timeout = 20
 
 
 # Blocked off the scan command to use a forcible timeout

From edd2ba226b5454e725aee2ed022f94f7108e0d08 Mon Sep 17 00:00:00 2001
From: Eric Mill <eric@konklone.com>
Date: Tue, 31 Oct 2017 21:52:18 -0400
Subject: [PATCH 03/18] move to using sslyze' Python API instead of its JSON
 output

---
 scanners/sslyze.py | 261 ++++++++++++++++++++++++++-------------------
 1 file changed, 151 insertions(+), 110 deletions(-)

diff --git a/scanners/sslyze.py b/scanners/sslyze.py
index 48fae813..412436d1 100644
--- a/scanners/sslyze.py
+++ b/scanners/sslyze.py
@@ -4,9 +4,15 @@
 
 import timeout_decorator
 
+import sslyze
+from sslyze.concurrent_scanner import ConcurrentScanner, PluginRaisedExceptionScanResult
+from sslyze.plugins.openssl_cipher_suites_plugin import Tlsv10ScanCommand, Tlsv11ScanCommand, Tlsv12ScanCommand, Sslv20ScanCommand, Sslv30ScanCommand
+from sslyze.plugins.certificate_info_plugin import CertificateInfoScanCommand
+
 import json
 import cryptography
 import cryptography.hazmat.backends.openssl
+from cryptography.hazmat.primitives.serialization import Encoding
 from cryptography.hazmat.primitives.asymmetric import ec, dsa, rsa
 
 ###
@@ -67,39 +73,24 @@ def scan(domain, options):
         # use scan_domain (possibly www-prefixed) to do actual scan
         logging.debug("\t %s %s" % (command, scan_domain))
 
-        # This is --regular minus --heartbleed
-        # See: https://github.com/nabla-c0d3/sslyze/issues/217
         raw_response = None
 
         try:
-            raw_response = actual_scan([
-                command,
-                "--sslv2", "--sslv3", "--tlsv1", "--tlsv1_1", "--tlsv1_2",
-                "--reneg", "--resum", "--certinfo",
-                "--http_get", "--hide_rejected_ciphers",
-                "--compression", "--openssl_ccs",
-                "--fallback", "--quiet",
-                scan_domain, "--json_out=%s" % cache_json
-            ])
+            data = run_sslyze(scan_domain)
         except timeout_decorator.timeout_decorator.TimeoutError:
             # logging.warn(utils.format_last_exception())
             logging.warn("\tTimeout error (%is) running sslyze." % timeout)
 
-        if raw_response is None:
+        if data is None:
             # TODO: save standard invalid JSON data...?
             utils.write(utils.invalid({}), cache_json)
             logging.warn("\tBad news scanning, sorry!")
             return None
 
-        raw_json = utils.scan(["cat", cache_json])
-        if not raw_json:
-            logging.warn("\tBad news reading JSON, sorry!")
-            return None
-
+        # not so raw...
+        raw_json = utils.json_for(data)
         utils.write(raw_json, cache_json)
 
-    data = parse_sslyze(raw_json)
-
     if data is None:
         logging.warn("\tNo valid target for scanning, couldn't connect.")
         return None
@@ -145,33 +136,69 @@ def scan(domain, options):
     "Errors"
 ]
 
+
 # Get the relevant fields out of sslyze's JSON format.
 #
 # Certificate PEM data must be separately parsed using
 # the Python cryptography module.
-#
-# If we were using the sslyze Python API, this would be
-# done for us automatically, but serializing the results
-# to disk for caching would be prohibitively complex.
 
+def run_sslyze(hostname):
+    server_info, scanner = init_sslyze(hostname)
+
+    logging.debug("\t Running scans...")
+
+    # Initialize commands and result containers
+    sslv2, sslv3, tlsv1, tlsv1_1, tlsv1_2 = None, None, None, None, None
+
+    # Queue them all up
+    run_command(Sslv20ScanCommand(), server_info, scanner)
+    run_command(Sslv30ScanCommand(), server_info, scanner)
+    run_command(Tlsv10ScanCommand(), server_info, scanner)
+    run_command(Tlsv11ScanCommand(), server_info, scanner)
+    run_command(Tlsv12ScanCommand(), server_info, scanner)
+    run_command(CertificateInfoScanCommand(), server_info, scanner)
 
-def parse_sslyze(raw_json):
+    # Reassign them back to predictable places after they're all done
+    was_error = False
+    for result in scanner.get_results():
+        try:
+            if isinstance(result, PluginRaisedExceptionScanResult):
+                logging.warn(u'Scan command failed: {}'.format(result.as_text()))
+                return None
 
-    data = json.loads(raw_json)
+            if type(result.scan_command) == Sslv20ScanCommand:
+                sslv2 = result
+            elif type(result.scan_command) == Sslv30ScanCommand:
+                sslv3 = result
+            elif type(result.scan_command) == Tlsv10ScanCommand:
+                tlsv1 = result
+            elif type(result.scan_command) == Tlsv11ScanCommand:
+                tlsv1_1 = result
+            elif type(result.scan_command) == Tlsv12ScanCommand:
+                tlsv1_2 = result
+            elif type(result.scan_command) == CertificateInfoScanCommand:
+                certs = result
+            else:
+                logging.warn("\tCouldn't match scan result with command! %s" % result)
+                was_error = True
 
-    # 1. Isolate first successful scanned IP.
-    if len(data['accepted_targets']) == 0:
+        except Exception as err:
+            logging.warn("\t Exception inside async scanner result processing.")
+            was_error = True
+            utils.notify(err)
+
+    # There was an error during async processing.
+    if was_error:
         return None
-    target = data['accepted_targets'][0]['commands_results']
 
-    # Protocol version support.
+    # Parse the results into a dict, which will also be cached as JSON.
     data = {
         'protocols': {
-            'sslv2': supported_protocol(target, 'sslv2'),
-            'sslv3': supported_protocol(target, 'sslv3'),
-            'tlsv1.0': supported_protocol(target, 'tlsv1'),
-            'tlsv1.1': supported_protocol(target, 'tlsv1_1'),
-            'tlsv1.2': supported_protocol(target, 'tlsv1_2')
+            'sslv2': supported_protocol(sslv2),
+            'sslv3': supported_protocol(sslv3),
+            'tlsv1.0': supported_protocol(tlsv1),
+            'tlsv1.1': supported_protocol(tlsv1_1),
+            'tlsv1.2': supported_protocol(tlsv1_2)
         },
 
         'config': {},
@@ -181,19 +208,12 @@ def parse_sslyze(raw_json):
         'errors': None
     }
 
-    # TODO: Whether OCSP stapling is enabled.
-    # Relevant fields: https://nabla-c0d3.github.io/sslyze/documentation/available-scan-commands.html#sslyze.plugins.certificate_info_plugin.CertificateInfoScanResult.ocsp_response
-
-    # ocsp = target.select_one('ocspStapling')
-    # if ocsp:
-    #     data['config']['ocsp_stapling'] = (ocsp["isSupported"] == 'True')
-
     accepted_ciphers = (
-        target['sslv2'].get("accepted_cipher_list", []) +
-        target['sslv3'].get("accepted_cipher_list", []) +
-        target['tlsv1'].get("accepted_cipher_list", []) +
-        target['tlsv1_1'].get("accepted_cipher_list", []) +
-        target['tlsv1_2'].get("accepted_cipher_list", [])
+        (sslv2.accepted_cipher_list or []) +
+        (sslv3.accepted_cipher_list or []) +
+        (tlsv1.accepted_cipher_list or []) +
+        (tlsv1_1.accepted_cipher_list or []) +
+        (tlsv1_2.accepted_cipher_list or [])
     )
 
     if len(accepted_ciphers) > 0:
@@ -207,7 +227,7 @@ def parse_sslyze(raw_json):
         any_3des = False
 
         for cipher in accepted_ciphers:
-            name = cipher["openssl_name"]
+            name = cipher.openssl_name
             if "RC4" in name:
                 any_rc4 = True
             else:
@@ -230,8 +250,8 @@ def parse_sslyze(raw_json):
         # Find the weakest available DH group size, if any are available.
         weakest_dh = 1234567890  # nonsense maximum
         for cipher in accepted_ciphers:
-            if cipher.get('dh_info', None) is not None:
-                size = int(cipher['dh_info']['GroupSize'])
+            if cipher.dh_info is not None:
+                size = int(cipher.dh_info['GroupSize'])
                 if size < weakest_dh:
                     weakest_dh = size
 
@@ -240,71 +260,65 @@ def parse_sslyze(raw_json):
 
         data['config']['weakest_dh'] = weakest_dh
 
-    # If there was an exception parsing the certificate, catch it before fetching cert info.
-    if False:
-        data['errors'] = "TODO"
-
-    else:
+    # Served chain.
+    served_chain = certs.certificate_chain
 
-        # Served chain.
-        served_chain = target['certinfo']['certificate_chain']
+    # Constructed chain may not be there if it didn't validate.
+    constructed_chain = certs.verified_certificate_chain
 
-        # Constructed chain may not be there if it didn't validate.
-        constructed_chain = target['certinfo']['verified_certificate_chain']
+    highest_served = parse_cert(served_chain[-1])
+    issuer = cert_issuer_name(highest_served)
 
-        highest_served = parse_cert(served_chain[-1])
-        issuer = cert_issuer_name(highest_served)
+    if issuer:
+        data['certs']['served_issuer'] = issuer
+    else:
+        data['certs']['served_issuer'] = "(None found)"
 
+    if (constructed_chain and (len(constructed_chain) > 0)):
+        highest_constructed = parse_cert(constructed_chain[-1])
+        issuer = cert_issuer_name(highest_constructed)
         if issuer:
-            data['certs']['served_issuer'] = issuer
+            data['certs']['constructed_issuer'] = issuer
         else:
-            data['certs']['served_issuer'] = "(None found)"
+            data['certs']['constructed_issuer'] = "(None constructed)"
 
-        if (constructed_chain and (len(constructed_chain) > 0)):
-            highest_constructed = parse_cert(constructed_chain[-1])
-            issuer = cert_issuer_name(highest_constructed)
-            if issuer:
-                data['certs']['constructed_issuer'] = issuer
-            else:
-                data['certs']['constructed_issuer'] = "(None constructed)"
-
-        leaf = parse_cert(served_chain[0])
-        leaf_key = leaf.public_key()
+    leaf = parse_cert(served_chain[0])
+    leaf_key = leaf.public_key()
 
-        if hasattr(leaf_key, "key_size"):
-            data['certs']['key_length'] = leaf_key.key_size
-        elif hasattr(leaf_key, "curve"):
-            data['certs']['key_length'] = leaf_key.curve.key_size
-        else:
-            data['certs']['key_length'] = None
-
-        if isinstance(leaf_key, rsa.RSAPublicKey):
-            leaf_key_type = "RSA"
-        elif isinstance(leaf_key, dsa.DSAPublicKey):
-            leaf_key_type = "DSA"
-        elif isinstance(leaf_key, ec.EllipticCurvePublicKey):
-            leaf_key_type = "ECDSA"
-        else:
-            leaf_key_type == str(leaf_key.__class__)
+    if hasattr(leaf_key, "key_size"):
+        data['certs']['key_length'] = leaf_key.key_size
+    elif hasattr(leaf_key, "curve"):
+        data['certs']['key_length'] = leaf_key.curve.key_size
+    else:
+        data['certs']['key_length'] = None
+
+    if isinstance(leaf_key, rsa.RSAPublicKey):
+        leaf_key_type = "RSA"
+    elif isinstance(leaf_key, dsa.DSAPublicKey):
+        leaf_key_type = "DSA"
+    elif isinstance(leaf_key, ec.EllipticCurvePublicKey):
+        leaf_key_type = "ECDSA"
+    else:
+        leaf_key_type == str(leaf_key.__class__)
 
-        data['certs']['key_type'] = leaf_key_type
+    data['certs']['key_type'] = leaf_key_type
 
-        # Signature of the leaf certificate only.
-        data['certs']['leaf_signature'] = leaf.signature_hash_algorithm.name
+    # Signature of the leaf certificate only.
+    data['certs']['leaf_signature'] = leaf.signature_hash_algorithm.name
 
-        # Beginning and expiration dates of the leaf certificate
-        data['certs']['not_before'] = leaf.not_valid_before
-        data['certs']['not_after'] = leaf.not_valid_after
+    # Beginning and expiration dates of the leaf certificate
+    data['certs']['not_before'] = leaf.not_valid_before
+    data['certs']['not_after'] = leaf.not_valid_after
 
-        any_sha1_served = False
-        for cert in served_chain:
-            if parse_cert(cert).signature_hash_algorithm.name == "sha1":
-                any_sha1_served = True
+    any_sha1_served = False
+    for cert in served_chain:
+        if parse_cert(cert).signature_hash_algorithm.name == "sha1":
+            any_sha1_served = True
 
-        data['certs']['any_sha1_served'] = any_sha1_served
+    data['certs']['any_sha1_served'] = any_sha1_served
 
-        if data['certs'].get('constructed_issuer'):
-            data['certs']['any_sha1_constructed'] = target['certinfo']['has_sha1_in_certificate_chain']
+    if data['certs'].get('constructed_issuer'):
+        data['certs']['any_sha1_constructed'] = certs.has_sha1_in_certificate_chain
 
     return data
 
@@ -313,7 +327,7 @@ def parse_sslyze(raw_json):
 # the cryptography module to parse its PEM contents.
 def parse_cert(cert):
     backend = cryptography.hazmat.backends.openssl.backend
-    pem_bytes = cert['as_pem'].encode('utf-8')
+    pem_bytes = cert.public_bytes(Encoding.PEM).decode('ascii').encode('utf-8')
     return cryptography.x509.load_pem_x509_certificate(pem_bytes, backend)
 
 
@@ -327,13 +341,40 @@ def cert_issuer_name(parsed):
         return None
     return attrs[0].value
 
+# Given CipherSuiteScanResult, whether the protocol is supported
+def supported_protocol(result):
+    return (len(result.accepted_cipher_list) > 0)
+
+
+## SSlyze boilerplate
+
+def init_sslyze(hostname):
+    try:
+        server_info = sslyze.server_connectivity.ServerConnectivityInfo(hostname=hostname, port=443)
+    except Exception as err:
+        utils.notify(err)
+        logging.warn("\tUnknown exception when initializing server connectivity info.")
+        return None
+
+    try:
+        server_info.test_connectivity_to_server()
+    except sslyze.server_connectivity.ServerConnectivityError as err:
+        logging.warn("\tServer connectivity not established.")
+        return None
+    except Exception as err:
+        utils.notify(err)
+        logging.warn("\tUnknown exception when performing server connectivity info.")
+        return None
+
+    scanner = ConcurrentScanner()
+
+    return server_info, scanner
+
+def run_command(command, server_info, scanner):
+    try:
+        return scanner.queue_scan_command(server_info, command)
+    except Exception as err:
+        utils.notify(err)
+        logging.warn("Unknown exception running sslyze command.")
+        return None
 
-# examines whether the protocol version turned out ot be supported
-def supported_protocol(target, protocol):
-    if target[protocol].get("error_message", None) is not None:
-        logging.debug("Error connecting to %s: %s" % (protocol, target[protocol]["error_message"]))
-        return False
-    elif target[protocol].get("accepted_cipher_list", None) is None:
-        return False
-    else:
-        return (len(target[protocol]["accepted_cipher_list"]) > 0)

From 6cc8b81a0a0091c42222541bdb1d20b47ee4b027 Mon Sep 17 00:00:00 2001
From: Eric Mill <eric@konklone.com>
Date: Tue, 31 Oct 2017 21:56:48 -0400
Subject: [PATCH 04/18] remove timeout for now

---
 scanners/sslyze.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/scanners/sslyze.py b/scanners/sslyze.py
index 412436d1..f1199a6e 100644
--- a/scanners/sslyze.py
+++ b/scanners/sslyze.py
@@ -30,11 +30,6 @@
 timeout = 20
 
 
-# Blocked off the scan command to use a forcible timeout
-@timeout_decorator.timeout(timeout, use_signals=False)
-def actual_scan(scan_args):
-    return utils.scan(scan_args)
-
 def scan(domain, options):
     logging.debug("[%s][sslyze]" % domain)
 
@@ -76,6 +71,9 @@ def scan(domain, options):
         raw_response = None
 
         try:
+            # TODO: timeout not actually enforced, due to issues
+            # with multiprocessing.
+            # If we have to, we can try single-threading to enforce a timeout.
             data = run_sslyze(scan_domain)
         except timeout_decorator.timeout_decorator.TimeoutError:
             # logging.warn(utils.format_last_exception())

From e9b599db4ef02fdda715b38f4f8f6657f88030fe Mon Sep 17 00:00:00 2001
From: Eric Mill <eric@konklone.com>
Date: Tue, 31 Oct 2017 22:21:57 -0400
Subject: [PATCH 05/18] fix flake8 errors

---
 scanners/sslyze.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/scanners/sslyze.py b/scanners/sslyze.py
index f1199a6e..2483b109 100644
--- a/scanners/sslyze.py
+++ b/scanners/sslyze.py
@@ -68,8 +68,6 @@ def scan(domain, options):
         # use scan_domain (possibly www-prefixed) to do actual scan
         logging.debug("\t %s %s" % (command, scan_domain))
 
-        raw_response = None
-
         try:
             # TODO: timeout not actually enforced, due to issues
             # with multiprocessing.
@@ -339,13 +337,13 @@ def cert_issuer_name(parsed):
         return None
     return attrs[0].value
 
+
 # Given CipherSuiteScanResult, whether the protocol is supported
 def supported_protocol(result):
     return (len(result.accepted_cipher_list) > 0)
 
 
-## SSlyze boilerplate
-
+# SSlyze initialization boilerplate
 def init_sslyze(hostname):
     try:
         server_info = sslyze.server_connectivity.ServerConnectivityInfo(hostname=hostname, port=443)
@@ -368,6 +366,7 @@ def init_sslyze(hostname):
 
     return server_info, scanner
 
+
 def run_command(command, server_info, scanner):
     try:
         return scanner.queue_scan_command(server_info, command)
@@ -375,4 +374,3 @@ def run_command(command, server_info, scanner):
         utils.notify(err)
         logging.warn("Unknown exception running sslyze command.")
         return None
-

From 9849380cfcafc3ff12d5b952e547b76a7e42317b Mon Sep 17 00:00:00 2001
From: Eric Mill <eric@konklone.com>
Date: Sat, 4 Nov 2017 17:51:29 -0400
Subject: [PATCH 06/18] add serial mode for sslyze

---
 scanners/sslyze.py | 157 +++++++++++++++++++++++++++++----------------
 1 file changed, 102 insertions(+), 55 deletions(-)

diff --git a/scanners/sslyze.py b/scanners/sslyze.py
index 2483b109..cedda242 100644
--- a/scanners/sslyze.py
+++ b/scanners/sslyze.py
@@ -5,6 +5,7 @@
 import timeout_decorator
 
 import sslyze
+from sslyze.synchronous_scanner import SynchronousScanner
 from sslyze.concurrent_scanner import ConcurrentScanner, PluginRaisedExceptionScanResult
 from sslyze.plugins.openssl_cipher_suites_plugin import Tlsv10ScanCommand, Tlsv11ScanCommand, Tlsv12ScanCommand, Sslv20ScanCommand, Sslv30ScanCommand
 from sslyze.plugins.certificate_info_plugin import CertificateInfoScanCommand
@@ -22,6 +23,11 @@
 #
 # If data exists for a domain from `pshtt`, will check results
 # and only process domains with valid HTTPS, or broken chains.
+#
+# Supported options:
+#
+# --sslyze-serial - If set, will use a synchronous (single-threaded
+#   in-process) scanner. Defaults to false.
 ###
 
 command = os.environ.get("SSLYZE_PATH", "sslyze")
@@ -72,7 +78,7 @@ def scan(domain, options):
             # TODO: timeout not actually enforced, due to issues
             # with multiprocessing.
             # If we have to, we can try single-threading to enforce a timeout.
-            data = run_sslyze(scan_domain)
+            data = run_sslyze(scan_domain, options)
         except timeout_decorator.timeout_decorator.TimeoutError:
             # logging.warn(utils.format_last_exception())
             logging.warn("\tTimeout error (%is) running sslyze." % timeout)
@@ -138,54 +144,17 @@ def scan(domain, options):
 # Certificate PEM data must be separately parsed using
 # the Python cryptography module.
 
-def run_sslyze(hostname):
-    server_info, scanner = init_sslyze(hostname)
-
-    logging.debug("\t Running scans...")
-
-    # Initialize commands and result containers
-    sslv2, sslv3, tlsv1, tlsv1_1, tlsv1_2 = None, None, None, None, None
-
-    # Queue them all up
-    run_command(Sslv20ScanCommand(), server_info, scanner)
-    run_command(Sslv30ScanCommand(), server_info, scanner)
-    run_command(Tlsv10ScanCommand(), server_info, scanner)
-    run_command(Tlsv11ScanCommand(), server_info, scanner)
-    run_command(Tlsv12ScanCommand(), server_info, scanner)
-    run_command(CertificateInfoScanCommand(), server_info, scanner)
-
-    # Reassign them back to predictable places after they're all done
-    was_error = False
-    for result in scanner.get_results():
-        try:
-            if isinstance(result, PluginRaisedExceptionScanResult):
-                logging.warn(u'Scan command failed: {}'.format(result.as_text()))
-                return None
-
-            if type(result.scan_command) == Sslv20ScanCommand:
-                sslv2 = result
-            elif type(result.scan_command) == Sslv30ScanCommand:
-                sslv3 = result
-            elif type(result.scan_command) == Tlsv10ScanCommand:
-                tlsv1 = result
-            elif type(result.scan_command) == Tlsv11ScanCommand:
-                tlsv1_1 = result
-            elif type(result.scan_command) == Tlsv12ScanCommand:
-                tlsv1_2 = result
-            elif type(result.scan_command) == CertificateInfoScanCommand:
-                certs = result
-            else:
-                logging.warn("\tCouldn't match scan result with command! %s" % result)
-                was_error = True
+def run_sslyze(hostname, options):
+    sync = options.get("sslyze-serial", False)
 
-        except Exception as err:
-            logging.warn("\t Exception inside async scanner result processing.")
-            was_error = True
-            utils.notify(err)
+    # Initialize either a synchronous or concurrent scanner.
+    server_info, scanner = init_sslyze(hostname, sync)
 
-    # There was an error during async processing.
-    if was_error:
-        return None
+    # Whether sync or concurrent, get responses for all scans.
+    if sync:
+        sslv2, sslv3, tlsv1, tlsv1_1, tlsv1_2, certs = scan_serial(scanner, server_info)
+    else:
+        sslv2, sslv3, tlsv1, tlsv1_1, tlsv1_2, certs = scan_parallel(scanner, server_info)
 
     # Parse the results into a dict, which will also be cached as JSON.
     data = {
@@ -344,7 +313,7 @@ def supported_protocol(result):
 
 
 # SSlyze initialization boilerplate
-def init_sslyze(hostname):
+def init_sslyze(hostname, sync=False):
     try:
         server_info = sslyze.server_connectivity.ServerConnectivityInfo(hostname=hostname, port=443)
     except Exception as err:
@@ -362,15 +331,93 @@ def init_sslyze(hostname):
         logging.warn("\tUnknown exception when performing server connectivity info.")
         return None
 
-    scanner = ConcurrentScanner()
+    if sync:
+        scanner = SynchronousScanner()
+    else:
+        scanner = ConcurrentScanner()
 
     return server_info, scanner
 
+# Run each scan in-process, one at a time.
+# Takes longer, but no multi-process funny business.
+def scan_serial(scanner, server_info):
+    logging.debug("\tRunning scans in serial.")
+    logging.debug("\t\tSSLv2 scan.")
+    sslv2 = scanner.run_scan_command(server_info, Sslv20ScanCommand())
+    logging.debug("\t\tSSLv3 scan.")
+    sslv3 = scanner.run_scan_command(server_info, Sslv30ScanCommand())
+    logging.debug("\t\tTLSv1.0 scan.")
+    tlsv1 = scanner.run_scan_command(server_info, Tlsv10ScanCommand())
+    logging.debug("\t\tTLSv1.1 scan.")
+    tlsv1_1 = scanner.run_scan_command(server_info, Tlsv11ScanCommand())
+    logging.debug("\t\tTLSv1.2 scan.")
+    tlsv1_2 = scanner.run_scan_command(server_info, Tlsv12ScanCommand())
+    logging.debug("\t\tCertificate information scan.")
+    certs = scanner.run_scan_command(server_info, CertificateInfoScanCommand())
+    logging.debug("\tDone scanning.")
+
+    return sslv2, sslv3, tlsv1, tlsv1_1, tlsv1_2, certs
+
+# Run each scan in parallel, using multi-processing.
+# Faster, but can generate many processes.
+def scan_parallel(scanner, server_info):
+    logging.debug("\tRunning scans in parallel.")
+
+    def queue(command):
+        try:
+            return scanner.queue_scan_command(server_info, command)
+        except Exception as err:
+            utils.notify(err)
+            logging.warn("Unknown exception queueing sslyze command.")
+            return None
 
-def run_command(command, server_info, scanner):
-    try:
-        return scanner.queue_scan_command(server_info, command)
-    except Exception as err:
-        utils.notify(err)
-        logging.warn("Unknown exception running sslyze command.")
+
+    # Initialize commands and result containers
+    sslv2, sslv3, tlsv1, tlsv1_1, tlsv1_2, certs = None, None, None, None, None, None
+
+    # Queue them all up
+    queue(Sslv20ScanCommand())
+    queue(Sslv30ScanCommand())
+    queue(Tlsv10ScanCommand())
+    queue(Tlsv11ScanCommand())
+    queue(Tlsv12ScanCommand())
+    queue(CertificateInfoScanCommand())
+
+    # Reassign them back to predictable places after they're all done
+    was_error = False
+    for result in scanner.get_results():
+        try:
+            if isinstance(result, PluginRaisedExceptionScanResult):
+                logging.warn(u'Scan command failed: {}'.format(result.as_text()))
+                return None
+
+            if type(result.scan_command) == Sslv20ScanCommand:
+                sslv2 = result
+            elif type(result.scan_command) == Sslv30ScanCommand:
+                sslv3 = result
+            elif type(result.scan_command) == Tlsv10ScanCommand:
+                tlsv1 = result
+            elif type(result.scan_command) == Tlsv11ScanCommand:
+                tlsv1_1 = result
+            elif type(result.scan_command) == Tlsv12ScanCommand:
+                tlsv1_2 = result
+            elif type(result.scan_command) == CertificateInfoScanCommand:
+                certs = result
+            else:
+                logging.warn("\tCouldn't match scan result with command! %s" % result)
+                was_error = True
+
+        except Exception as err:
+            logging.warn("\t Exception inside async scanner result processing.")
+            was_error = True
+            utils.notify(err)
+
+    # There was an error during async processing.
+    if was_error:
         return None
+
+    logging.debug("\tDone scanning.")
+
+    return sslv2, sslv3, tlsv1, tlsv1_1, tlsv1_2, certs
+
+

From 14dc67e8aed4dd275b739e5846c49e84fcc0b3c8 Mon Sep 17 00:00:00 2001
From: Eric Mill <eric@konklone.com>
Date: Sat, 4 Nov 2017 18:05:06 -0400
Subject: [PATCH 07/18] server connectivity errors can happen during
 initialization too

---
 scanners/sslyze.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scanners/sslyze.py b/scanners/sslyze.py
index cedda242..ed1618d8 100644
--- a/scanners/sslyze.py
+++ b/scanners/sslyze.py
@@ -316,6 +316,9 @@ def supported_protocol(result):
 def init_sslyze(hostname, sync=False):
     try:
         server_info = sslyze.server_connectivity.ServerConnectivityInfo(hostname=hostname, port=443)
+    except sslyze.server_connectivity.ServerConnectivityError as error:
+        logging.warn("\tServer connectivity not established during initialization.")
+        return None
     except Exception as err:
         utils.notify(err)
         logging.warn("\tUnknown exception when initializing server connectivity info.")
@@ -324,7 +327,7 @@ def init_sslyze(hostname, sync=False):
     try:
         server_info.test_connectivity_to_server()
     except sslyze.server_connectivity.ServerConnectivityError as err:
-        logging.warn("\tServer connectivity not established.")
+        logging.warn("\tServer connectivity not established during test.")
         return None
     except Exception as err:
         utils.notify(err)

From 81c4687cd8f21677f86aca7238ff377633713681 Mon Sep 17 00:00:00 2001
From: Eric Mill <eric@konklone.com>
Date: Sat, 4 Nov 2017 21:41:07 -0400
Subject: [PATCH 08/18] fix flake8 errors

---
 scanners/sslyze.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/scanners/sslyze.py b/scanners/sslyze.py
index ed1618d8..47b3be18 100644
--- a/scanners/sslyze.py
+++ b/scanners/sslyze.py
@@ -341,6 +341,7 @@ def init_sslyze(hostname, sync=False):
 
     return server_info, scanner
 
+
 # Run each scan in-process, one at a time.
 # Takes longer, but no multi-process funny business.
 def scan_serial(scanner, server_info):
@@ -361,6 +362,7 @@ def scan_serial(scanner, server_info):
 
     return sslv2, sslv3, tlsv1, tlsv1_1, tlsv1_2, certs
 
+
 # Run each scan in parallel, using multi-processing.
 # Faster, but can generate many processes.
 def scan_parallel(scanner, server_info):
@@ -374,7 +376,6 @@ def queue(command):
             logging.warn("Unknown exception queueing sslyze command.")
             return None
 
-
     # Initialize commands and result containers
     sslv2, sslv3, tlsv1, tlsv1_1, tlsv1_2, certs = None, None, None, None, None, None
 
@@ -422,5 +423,3 @@ def queue(command):
     logging.debug("\tDone scanning.")
 
     return sslv2, sslv3, tlsv1, tlsv1_1, tlsv1_2, certs
-
-

From a56ab8d8098877d2bedb238ffcc1cc9b4dddb6cf Mon Sep 17 00:00:00 2001
From: Eric Mill <eric@konklone.com>
Date: Sat, 4 Nov 2017 21:57:42 -0400
Subject: [PATCH 09/18] remove timeout code

---
 scanners/sslyze.py | 15 +--------------
 1 file changed, 1 insertion(+), 14 deletions(-)

diff --git a/scanners/sslyze.py b/scanners/sslyze.py
index 47b3be18..744d6f4b 100644
--- a/scanners/sslyze.py
+++ b/scanners/sslyze.py
@@ -2,8 +2,6 @@
 from scanners import utils
 import os
 
-import timeout_decorator
-
 import sslyze
 from sslyze.synchronous_scanner import SynchronousScanner
 from sslyze.concurrent_scanner import ConcurrentScanner, PluginRaisedExceptionScanResult
@@ -32,10 +30,6 @@
 
 command = os.environ.get("SSLYZE_PATH", "sslyze")
 
-# This timeout is enforced in this file, in Python, not in sslyze.
-timeout = 20
-
-
 def scan(domain, options):
     logging.debug("[%s][sslyze]" % domain)
 
@@ -74,14 +68,7 @@ def scan(domain, options):
         # use scan_domain (possibly www-prefixed) to do actual scan
         logging.debug("\t %s %s" % (command, scan_domain))
 
-        try:
-            # TODO: timeout not actually enforced, due to issues
-            # with multiprocessing.
-            # If we have to, we can try single-threading to enforce a timeout.
-            data = run_sslyze(scan_domain, options)
-        except timeout_decorator.timeout_decorator.TimeoutError:
-            # logging.warn(utils.format_last_exception())
-            logging.warn("\tTimeout error (%is) running sslyze." % timeout)
+        data = run_sslyze(scan_domain, options)
 
         if data is None:
             # TODO: save standard invalid JSON data...?

From 9d0e6ab40e8fd45fc05f32ec873b644ad838bb67 Mon Sep 17 00:00:00 2001
From: Shane Frasier <jeremy.frasier@beta.dhs.gov>
Date: Sat, 4 Nov 2017 22:39:40 -0400
Subject: [PATCH 10/18] * I noticed that, although python was being installed
 with pyenv, the   python that was being used was the one from Ubuntu.  I
 removed the   Ubuntu python and corrected teh PATH so that the pyenv python
 is   used. * I added lines to install GDB and set it up so that the py-bt,  
 py-list, etc. commands will work when the pyenv python is used.

---
 Dockerfile | 35 ++++++++++++++++++++++++-----------
 1 file changed, 24 insertions(+), 11 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index f26d558e..354d6374 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -2,7 +2,7 @@
 
 # USAGE
 
-FROM      ubuntu:14.04.4
+FROM      ubuntu:16.04
 MAINTAINER V. David Zvenyach <vladlen.zvenyach@gsa.gov>
 
 ###
@@ -34,7 +34,6 @@ RUN \
       unzip \
       wget \
       zlib1g-dev \
-
       autoconf \
       automake \
       bison \
@@ -46,7 +45,6 @@ RUN \
       libtool \
       pkg-config \
       sqlite3 \
-
       # Additional dependencies for python-build
       libbz2-dev \
       llvm \
@@ -57,10 +55,8 @@ RUN apt-get install \
       --yes \
       --no-install-recommends \
       --no-install-suggests \
-    nodejs \
-      npm \
-      python3-dev \
-      python3-pip
+      nodejs \
+      npm
 
     # Clean up packages.
 RUN apt-get clean \
@@ -81,12 +77,29 @@ RUN wget ${PYENV_REPO}/archive/v${PYENV_RELEASE}.zip \
   && mv $PYENV_ROOT/pyenv-$PYENV_RELEASE/* $PYENV_ROOT/ \
   && rm -r $PYENV_ROOT/pyenv-$PYENV_RELEASE
 
-ENV PATH $PYENV_ROOT/bin:$PATH
-
+#
+# Uncomment these lines if you just want to install python...
+#
+# ENV PATH $PYENV_ROOT/bin:$PYENV_ROOT/versions/${PYENV_PYTHON_VERSION}/bin:$PATH
+# RUN echo 'eval "$(pyenv init -)"' >> /etc/profile \
+#     && eval "$(pyenv init -)" \
+#     && pyenv install $PYENV_PYTHON_VERSION \
+#     && pyenv local ${PYENV_PYTHON_VERSION}
+
+#
+# ...uncomment these lines if you want to also debug python code in GDB
+#
+ENV PATH $PYENV_ROOT/bin:$PYENV_ROOT/versions/${PYENV_PYTHON_VERSION}-debug/bin:$PATH
 RUN echo 'eval "$(pyenv init -)"' >> /etc/profile \
     && eval "$(pyenv init -)" \
-    && pyenv install $PYENV_PYTHON_VERSION \
-    && pyenv local $PYENV_PYTHON_VERSION
+    && pyenv install --debug --keep $PYENV_PYTHON_VERSION \
+    && pyenv local ${PYENV_PYTHON_VERSION}-debug
+RUN ln -s /opt/pyenv/sources/${PYENV_PYTHON_VERSION}-debug/Python-${PYENV_PYTHON_VERSION}/python-gdb.py /opt/pyenv/versions/${PYENV_PYTHON_VERSION}-debug/bin/python3.6-gdb.py
+RUN ln -s /opt/pyenv/sources/${PYENV_PYTHON_VERSION}-debug/Python-${PYENV_PYTHON_VERSION}/python-gdb.py /opt/pyenv/versions/${PYENV_PYTHON_VERSION}-debug/bin/python3-gdb.py
+RUN ln -s /opt/pyenv/sources/${PYENV_PYTHON_VERSION}-debug/Python-${PYENV_PYTHON_VERSION}/python-gdb.py /opt/pyenv/versions/${PYENV_PYTHON_VERSION}-debug/bin/python-gdb.py
+RUN apt-get -qq update && \
+    apt-get -qq --yes --no-install-recommends --no-install-suggests install gdb
+RUN echo add-auto-load-safe-path /opt/pyenv/sources/${PYENV_PYTHON_VERSION}-debug/Python-${PYENV_PYTHON_VERSION}/ >> etc/gdb/gdbinit
 
 COPY requirements.txt requirements.txt
 RUN pip3 install --upgrade pip

From 18ed5632157edb2306e7bb912313055e09dccee1 Mon Sep 17 00:00:00 2001
From: Eric Mill <eric@konklone.com>
Date: Sat, 4 Nov 2017 23:38:10 -0400
Subject: [PATCH 11/18] disable cert checking for now

---
 scanners/sslyze.py | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/scanners/sslyze.py b/scanners/sslyze.py
index 744d6f4b..47ed079a 100644
--- a/scanners/sslyze.py
+++ b/scanners/sslyze.py
@@ -212,6 +212,15 @@ def run_sslyze(hostname, options):
 
         data['config']['weakest_dh'] = weakest_dh
 
+    # TODO: do_cert?
+    # data['certs'] = analyze_certs(certs)
+
+    return data
+
+
+def analyze_certs(certs):
+    data = {'certs': {}}
+
     # Served chain.
     served_chain = certs.certificate_chain
 
@@ -272,8 +281,7 @@ def run_sslyze(hostname, options):
     if data['certs'].get('constructed_issuer'):
         data['certs']['any_sha1_constructed'] = certs.has_sha1_in_certificate_chain
 
-    return data
-
+    return data['certs']
 
 # Given the cert sub-obj from the sslyze JSON, use
 # the cryptography module to parse its PEM contents.
@@ -343,8 +351,12 @@ def scan_serial(scanner, server_info):
     tlsv1_1 = scanner.run_scan_command(server_info, Tlsv11ScanCommand())
     logging.debug("\t\tTLSv1.2 scan.")
     tlsv1_2 = scanner.run_scan_command(server_info, Tlsv12ScanCommand())
-    logging.debug("\t\tCertificate information scan.")
-    certs = scanner.run_scan_command(server_info, CertificateInfoScanCommand())
+
+    # TODO: do_cert?
+    # logging.debug("\t\tCertificate information scan.")
+    # certs = scanner.run_scan_command(server_info, CertificateInfoScanCommand())
+    certs = None
+
     logging.debug("\tDone scanning.")
 
     return sslv2, sslv3, tlsv1, tlsv1_1, tlsv1_2, certs
@@ -372,7 +384,8 @@ def queue(command):
     queue(Tlsv10ScanCommand())
     queue(Tlsv11ScanCommand())
     queue(Tlsv12ScanCommand())
-    queue(CertificateInfoScanCommand())
+    # TODO: do_cert?
+    # queue(CertificateInfoScanCommand())
 
     # Reassign them back to predictable places after they're all done
     was_error = False

From 06edd6da27539c394cf7d55f5d63ae8ac54aa9b3 Mon Sep 17 00:00:00 2001
From: Eric Mill <eric@konklone.com>
Date: Sat, 4 Nov 2017 23:52:06 -0400
Subject: [PATCH 12/18] fix unicode errors when building publicsuffixlist
 dependency

---
 Dockerfile | 17 ++++++-----------
 1 file changed, 6 insertions(+), 11 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 354d6374..556c22cd 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -125,16 +125,6 @@ ENV PATH /go/bin:$PATH
 # Node
 RUN ln -s /usr/bin/nodejs /usr/bin/node
 
-###
-# ssllabs-scan
-
-RUN mkdir -p /go/src /go/bin \
-      && chmod -R 777 /go
-RUN go get github.com/ssllabs/ssllabs-scan
-RUN cd /go/src/github.com/ssllabs/ssllabs-scan/ \
-      && git checkout stable \
-      && go install
-ENV SSLLABS_PATH /go/bin/ssllabs-scan
 
 ###
 # phantomas
@@ -149,7 +139,12 @@ RUN npm install \
 ###
 # pshtt
 
-RUN pip3 install pshtt==0.2.1
+RUN apt-get install -qq --yes locales
+RUN locale-gen en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
+RUN pip3 install pshtt
 
 
 ###

From 0aeb37f53104c11bf8b3b87157ceb5d8b0a6cad8 Mon Sep 17 00:00:00 2001
From: Eric Mill <eric@konklone.com>
Date: Sun, 5 Nov 2017 00:21:55 -0400
Subject: [PATCH 13/18] adjust placement of copy in docker build

---
 Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 556c22cd..0ed74d48 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -153,8 +153,6 @@ RUN pip3 install pshtt
 ENV SCANNER_HOME /home/scanner
 RUN mkdir $SCANNER_HOME
 
-COPY . $SCANNER_HOME
-
 RUN groupadd -r scanner \
   && useradd -r -c "Scanner user" -g scanner scanner \
   && chown -R scanner:scanner ${SCANNER_HOME}
@@ -169,3 +167,5 @@ WORKDIR $SCANNER_HOME
 VOLUME /data
 
 ENTRYPOINT ["./scan_wrap.sh"]
+
+COPY . $SCANNER_HOME

From d2de308629c16c4279a745429deeb4a6a455eed5 Mon Sep 17 00:00:00 2001
From: Eric Mill <eric@konklone.com>
Date: Sun, 5 Nov 2017 00:22:04 -0400
Subject: [PATCH 14/18] always close files

---
 scanners/pshtt.py  | 2 +-
 scanners/sslyze.py | 2 +-
 scanners/utils.py  | 8 +++++++-
 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/scanners/pshtt.py b/scanners/pshtt.py
index daffb55a..99d856f3 100644
--- a/scanners/pshtt.py
+++ b/scanners/pshtt.py
@@ -50,7 +50,7 @@ def scan(domain, options):
 
     if (force is False) and (os.path.exists(cache_pshtt)):
         logging.debug("\tCached.")
-        raw = open(cache_pshtt).read()
+        raw = utils.read(cache_pshtt)
         data = json.loads(raw)
         if (data.__class__ is dict) and data.get('invalid'):
             return None
diff --git a/scanners/sslyze.py b/scanners/sslyze.py
index 47ed079a..6c6dedf3 100644
--- a/scanners/sslyze.py
+++ b/scanners/sslyze.py
@@ -55,7 +55,7 @@ def scan(domain, options):
 
     if (force is False) and (os.path.exists(cache_json)):
         logging.debug("\tCached.")
-        raw_json = open(cache_json).read()
+        raw_json = utils.read(cache_json)
         try:
             data = json.loads(raw_json)
             if (data.__class__ is dict) and data.get('invalid'):
diff --git a/scanners/utils.py b/scanners/utils.py
index ba068d83..739a9c48 100644
--- a/scanners/utils.py
+++ b/scanners/utils.py
@@ -111,6 +111,12 @@ def write(content, destination, binary=False):
     f.close()
 
 
+def read(source):
+    with open(source) as f:
+        contents = f.read()
+    return contents
+
+
 def report_dir():
     return options().get("output", "./")
 
@@ -191,7 +197,7 @@ def cache_single(filename):
 def data_for(domain, operation):
     path = cache_path(domain, operation)
     if os.path.exists(path):
-        raw = open(path).read()
+        raw = read(path)
         data = json.loads(raw)
         if isinstance(data, dict) and (data.get('invalid', False)):
             return None

From bd83897eb1796a7cda6ddaffe28f9a08283b3a3c Mon Sep 17 00:00:00 2001
From: Eric Mill <eric@konklone.com>
Date: Sun, 5 Nov 2017 00:40:34 -0400
Subject: [PATCH 15/18] add network timeout

---
 scanners/sslyze.py | 65 ++++++++++++++++++++++++++++------------------
 1 file changed, 40 insertions(+), 25 deletions(-)

diff --git a/scanners/sslyze.py b/scanners/sslyze.py
index 6c6dedf3..f3d9e3d4 100644
--- a/scanners/sslyze.py
+++ b/scanners/sslyze.py
@@ -28,6 +28,10 @@
 #   in-process) scanner. Defaults to false.
 ###
 
+# Number of seconds to wait during sslyze connection check.
+# Not much patience here, and very willing to move on.
+network_timeout = 5
+
 command = os.environ.get("SSLYZE_PATH", "sslyze")
 
 def scan(domain, options):
@@ -86,9 +90,9 @@ def scan(domain, options):
 
     yield [
         scan_domain,
-        data['protocols']['sslv2'], data['protocols']['sslv3'],
-        data['protocols']['tlsv1.0'], data['protocols']['tlsv1.1'],
-        data['protocols']['tlsv1.2'],
+        data['protocols'].get('sslv2'), data['protocols'].get('sslv3'),
+        data['protocols'].get('tlsv1.0'), data['protocols'].get('tlsv1.1'),
+        data['protocols'].get('tlsv1.2'),
 
         data['config'].get('any_dhe'), data['config'].get('all_dhe'),
         data['config'].get('weakest_dh'),
@@ -132,10 +136,25 @@ def scan(domain, options):
 # the Python cryptography module.
 
 def run_sslyze(hostname, options):
+    # Parse the results into a dict, which will also be cached as JSON.
+    data = {
+        'protocols': {},
+
+        'config': {},
+
+        'certs': {},
+
+        'errors': None
+    }
+
     sync = options.get("sslyze-serial", False)
 
     # Initialize either a synchronous or concurrent scanner.
-    server_info, scanner = init_sslyze(hostname, sync)
+    server_info, scanner = init_sslyze(hostname, options, sync=sync)
+
+    if server_info == None:
+        data['errors'] = "Connectivity not established."
+        return data
 
     # Whether sync or concurrent, get responses for all scans.
     if sync:
@@ -143,21 +162,12 @@ def run_sslyze(hostname, options):
     else:
         sslv2, sslv3, tlsv1, tlsv1_1, tlsv1_2, certs = scan_parallel(scanner, server_info)
 
-    # Parse the results into a dict, which will also be cached as JSON.
-    data = {
-        'protocols': {
-            'sslv2': supported_protocol(sslv2),
-            'sslv3': supported_protocol(sslv3),
-            'tlsv1.0': supported_protocol(tlsv1),
-            'tlsv1.1': supported_protocol(tlsv1_1),
-            'tlsv1.2': supported_protocol(tlsv1_2)
-        },
-
-        'config': {},
-
-        'certs': {},
-
-        'errors': None
+    data['protocols'] = {
+        'sslv2': supported_protocol(sslv2),
+        'sslv3': supported_protocol(sslv3),
+        'tlsv1.0': supported_protocol(tlsv1),
+        'tlsv1.1': supported_protocol(tlsv1_1),
+        'tlsv1.2': supported_protocol(tlsv1_2)
     }
 
     accepted_ciphers = (
@@ -308,26 +318,31 @@ def supported_protocol(result):
 
 
 # SSlyze initialization boilerplate
-def init_sslyze(hostname, sync=False):
+def init_sslyze(hostname, options, sync=False):
+    global network_timeout
+
+    network_timeout = int(options.get("network_timeout", network_timeout))
+
     try:
         server_info = sslyze.server_connectivity.ServerConnectivityInfo(hostname=hostname, port=443)
     except sslyze.server_connectivity.ServerConnectivityError as error:
         logging.warn("\tServer connectivity not established during initialization.")
-        return None
+        return None, None
     except Exception as err:
         utils.notify(err)
         logging.warn("\tUnknown exception when initializing server connectivity info.")
-        return None
+        return None, None
 
     try:
-        server_info.test_connectivity_to_server()
+        logging.debug("\tTesting connectivity with timeout of %is." % network_timeout)
+        server_info.test_connectivity_to_server(network_timeout=network_timeout)
     except sslyze.server_connectivity.ServerConnectivityError as err:
         logging.warn("\tServer connectivity not established during test.")
-        return None
+        return None, None
     except Exception as err:
         utils.notify(err)
         logging.warn("\tUnknown exception when performing server connectivity info.")
-        return None
+        return None, None
 
     if sync:
         scanner = SynchronousScanner()

From b21f54ce00ece723da89f1c82b9a332a9c4dcee5 Mon Sep 17 00:00:00 2001
From: Eric Mill <eric@konklone.com>
Date: Sun, 5 Nov 2017 00:52:14 -0400
Subject: [PATCH 16/18] opt in to cert info gathering with --sslyze-certs

---
 scanners/sslyze.py | 28 ++++++++++++++++------------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/scanners/sslyze.py b/scanners/sslyze.py
index f3d9e3d4..8aa83ec7 100644
--- a/scanners/sslyze.py
+++ b/scanners/sslyze.py
@@ -158,9 +158,9 @@ def run_sslyze(hostname, options):
 
     # Whether sync or concurrent, get responses for all scans.
     if sync:
-        sslv2, sslv3, tlsv1, tlsv1_1, tlsv1_2, certs = scan_serial(scanner, server_info)
+        sslv2, sslv3, tlsv1, tlsv1_1, tlsv1_2, certs = scan_serial(scanner, server_info, options)
     else:
-        sslv2, sslv3, tlsv1, tlsv1_1, tlsv1_2, certs = scan_parallel(scanner, server_info)
+        sslv2, sslv3, tlsv1, tlsv1_1, tlsv1_2, certs = scan_parallel(scanner, server_info, options)
 
     data['protocols'] = {
         'sslv2': supported_protocol(sslv2),
@@ -222,8 +222,8 @@ def run_sslyze(hostname, options):
 
         data['config']['weakest_dh'] = weakest_dh
 
-    # TODO: do_cert?
-    # data['certs'] = analyze_certs(certs)
+    if certs:
+        data['certs'] = analyze_certs(certs)
 
     return data
 
@@ -354,7 +354,7 @@ def init_sslyze(hostname, options, sync=False):
 
 # Run each scan in-process, one at a time.
 # Takes longer, but no multi-process funny business.
-def scan_serial(scanner, server_info):
+def scan_serial(scanner, server_info, options):
     logging.debug("\tRunning scans in serial.")
     logging.debug("\t\tSSLv2 scan.")
     sslv2 = scanner.run_scan_command(server_info, Sslv20ScanCommand())
@@ -367,10 +367,12 @@ def scan_serial(scanner, server_info):
     logging.debug("\t\tTLSv1.2 scan.")
     tlsv1_2 = scanner.run_scan_command(server_info, Tlsv12ScanCommand())
 
-    # TODO: do_cert?
-    # logging.debug("\t\tCertificate information scan.")
-    # certs = scanner.run_scan_command(server_info, CertificateInfoScanCommand())
-    certs = None
+    # Default to cert info off
+    if options.get("sslyze-certs"):
+        logging.debug("\t\tCertificate information scan.")
+        certs = scanner.run_scan_command(server_info, CertificateInfoScanCommand())
+    else:
+        certs = None
 
     logging.debug("\tDone scanning.")
 
@@ -379,7 +381,7 @@ def scan_serial(scanner, server_info):
 
 # Run each scan in parallel, using multi-processing.
 # Faster, but can generate many processes.
-def scan_parallel(scanner, server_info):
+def scan_parallel(scanner, server_info, options):
     logging.debug("\tRunning scans in parallel.")
 
     def queue(command):
@@ -399,8 +401,10 @@ def queue(command):
     queue(Tlsv10ScanCommand())
     queue(Tlsv11ScanCommand())
     queue(Tlsv12ScanCommand())
-    # TODO: do_cert?
-    # queue(CertificateInfoScanCommand())
+
+    # Default to cert info off
+    if options.get("sslyze-certs"):
+        queue(CertificateInfoScanCommand())
 
     # Reassign them back to predictable places after they're all done
     was_error = False

From 0d3c1d41eaeddb0bfd0c7807a5feb80a2407ce96 Mon Sep 17 00:00:00 2001
From: Eric Mill <eric@konklone.com>
Date: Sun, 5 Nov 2017 01:03:18 -0400
Subject: [PATCH 17/18] switch cert info to default on

---
 scanners/sslyze.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scanners/sslyze.py b/scanners/sslyze.py
index 8aa83ec7..e31f2b27 100644
--- a/scanners/sslyze.py
+++ b/scanners/sslyze.py
@@ -367,8 +367,8 @@ def scan_serial(scanner, server_info, options):
     logging.debug("\t\tTLSv1.2 scan.")
     tlsv1_2 = scanner.run_scan_command(server_info, Tlsv12ScanCommand())
 
-    # Default to cert info off
-    if options.get("sslyze-certs"):
+    # Default to cert info on
+    if options.get("sslyze-no-certs", False) == False:
         logging.debug("\t\tCertificate information scan.")
         certs = scanner.run_scan_command(server_info, CertificateInfoScanCommand())
     else:
@@ -402,8 +402,8 @@ def queue(command):
     queue(Tlsv11ScanCommand())
     queue(Tlsv12ScanCommand())
 
-    # Default to cert info off
-    if options.get("sslyze-certs"):
+    # Default to cert info on.
+    if options.get("sslyze-no-certs", False) == False:
         queue(CertificateInfoScanCommand())
 
     # Reassign them back to predictable places after they're all done

From 449d80ea879f6cabd60de3dff2b52385c8497e35 Mon Sep 17 00:00:00 2001
From: Eric Mill <eric@konklone.com>
Date: Sun, 5 Nov 2017 01:04:06 -0400
Subject: [PATCH 18/18] fix flake8 warnings

---
 scanners/sslyze.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/scanners/sslyze.py b/scanners/sslyze.py
index e31f2b27..3e39c7e1 100644
--- a/scanners/sslyze.py
+++ b/scanners/sslyze.py
@@ -34,6 +34,7 @@
 
 command = os.environ.get("SSLYZE_PATH", "sslyze")
 
+
 def scan(domain, options):
     logging.debug("[%s][sslyze]" % domain)
 
@@ -152,7 +153,7 @@ def run_sslyze(hostname, options):
     # Initialize either a synchronous or concurrent scanner.
     server_info, scanner = init_sslyze(hostname, options, sync=sync)
 
-    if server_info == None:
+    if server_info is None:
         data['errors'] = "Connectivity not established."
         return data
 
@@ -293,6 +294,7 @@ def analyze_certs(certs):
 
     return data['certs']
 
+
 # Given the cert sub-obj from the sslyze JSON, use
 # the cryptography module to parse its PEM contents.
 def parse_cert(cert):
@@ -368,7 +370,7 @@ def scan_serial(scanner, server_info, options):
     tlsv1_2 = scanner.run_scan_command(server_info, Tlsv12ScanCommand())
 
     # Default to cert info on
-    if options.get("sslyze-no-certs", False) == False:
+    if options.get("sslyze-no-certs", False) is False:
         logging.debug("\t\tCertificate information scan.")
         certs = scanner.run_scan_command(server_info, CertificateInfoScanCommand())
     else:
@@ -403,7 +405,7 @@ def queue(command):
     queue(Tlsv12ScanCommand())
 
     # Default to cert info on.
-    if options.get("sslyze-no-certs", False) == False:
+    if options.get("sslyze-no-certs", False) is False:
         queue(CertificateInfoScanCommand())
 
     # Reassign them back to predictable places after they're all done