diff --git a/CHANGES.rst b/CHANGES.rst index 5a3db87..b99675d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,5 +1,8 @@ Changelog ========= +* next release + * Replaced pyOpenSSL with cryptography + * Added parsing of subjectAltName X509 extension * 1.1.11 * Added first version of WiFi results * 1.1.10 diff --git a/README.rst b/README.rst index fa2dfb4..5470e9a 100644 --- a/README.rst +++ b/README.rst @@ -36,8 +36,7 @@ Troubleshooting Some setups (like MacOS) have trouble with building the dependencies required for reading SSL certificates. If you don't care about SSL stuff and only want to -use sagan to say, parse traceroute or DNS results, then you can tell the -installer to skip building ``pyOpenSSL`` by doing the following: +use sagan to say, parse traceroute or DNS results, then you can do the following: .. code:: bash @@ -175,9 +174,10 @@ What it requires As you might have guessed, with all of this magic going on under the hood, there are a few dependencies: -- `pyOpenSSL`_ (Optional: see "Troubleshooting" above) +- `cryptography`_ (Optional: see "Troubleshooting" above) - `python-dateutil`_ - `pytz`_ +- `IPy`_ Additionally, we recommend that you also install `ujson`_ as it will speed up the JSON-decoding step considerably, and `sphinx`_ if you intend to build the @@ -229,9 +229,9 @@ But why "`Sagan`_"? The RIPE Atlas team decided to name all of its modules after explorers, and what better name for a parser than that of the man who spent decades reaching out to the public about the wonders of the cosmos? -.. _pyOpenSSL: https://pypi.python.org/pypi/pyOpenSSL .. _python-dateutil: https://pypi.python.org/pypi/python-dateutil .. _pytz: https://pypi.python.org/pypi/pytz +.. _IPy: https://pypi.python.org/pypi/IPy/ .. _ujson: https://pypi.python.org/pypi/ujson .. _sphinx: https://pypi.python.org/pypi/Sphinx .. _Read the Docs: http://ripe-atlas-sagan.readthedocs.org/en/latest/ diff --git a/docs/installation.rst b/docs/installation.rst index 42d4f6a..8b1104a 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -11,7 +11,7 @@ Requirements As you might have guessed, with all of the magic going on under the hood, there are a few dependencies: -* `pyOpenSSL`_ +* `cryptography`_ * `python-dateutil`_ * `pytz`_ @@ -19,7 +19,7 @@ Additionally, we recommend that you also install `ujson`_ as it will speed up the JSON-decoding step considerably, and `sphinx`_ if you intend to build the documentation files for offline use. -.. _pyOpenSSL: https://pypi.python.org/pypi/pyOpenSSL/ +.. _cryptography: https://pypi.python.org/pypi/cryptography .. _python-dateutil: https://pypi.python.org/pypi/python-dateutil/ .. _pytz: https://pypi.python.org/pypi/pytz/ .. _ujson: https://pypi.python.org/pypi/ujson/ @@ -75,11 +75,12 @@ Troubleshooting Some setups (like MacOS) have trouble with building the dependencies required for reading SSL certificates. If you don't care about SSL stuff and only want -to use sagan to say, parse traceroute or DNS results, then you can tell the -installer to skip building ``pyOpenSSL`` by doing the following:: +to use sagan to say, parse traceroute or DNS results, then you can do the following: $ SAGAN_WITHOUT_SSL=1 pip install ripe.atlas.sagan +More information can also be found `here`_. + If you *do* care about SSL and have to use a Mac, then `this issue`_ will likely be of assistance. Essentially, you will need to uninstall Xcode (if it's installed already), then attempt to use ``gcc``. This will trigger the OS to @@ -88,4 +89,5 @@ when that's finished, install Sagan with this command: $ CFLAGS="-I/usr/include" pip install ripe.atlas.sagan +.. _here: https://cryptography.io/en/latest/installation/ .. _this issue: https://github.com/RIPE-NCC/ripe.atlas.sagan/issues/52 diff --git a/docs/types.rst b/docs/types.rst index c3e16bd..89a2043 100644 --- a/docs/types.rst +++ b/docs/types.rst @@ -696,6 +696,7 @@ checksum_md5 str The md5 checksum checksum_sha1 str The sha1 checksum checksum_sha256 str The sha256 checksum has_expired bool Set to ``True`` if the certificate is no longer valid +extensions dict Parsed extensions. For now it can only be subjectAltName, which is a list of names contained in the SAN extension, if that exists. ===================== ======== =================================================================================== diff --git a/ripe/atlas/sagan/ssl.py b/ripe/atlas/sagan/ssl.py index 44ea8cc..c281127 100644 --- a/ripe/atlas/sagan/ssl.py +++ b/ripe/atlas/sagan/ssl.py @@ -15,28 +15,30 @@ import logging import pytz -import re +import codecs from datetime import datetime -from dateutil.relativedelta import relativedelta try: - import OpenSSL + from cryptography import x509 + from cryptography.hazmat.backends import openssl + from cryptography.hazmat.primitives import hashes except ImportError: logging.warning( - "pyOpenSSL is not installed, without it you cannot parse SSL " + "cryptography module is not installed, without it you cannot parse SSL " "certificate measurement results" ) from .base import Result, ResultParseError, ParsingDict -class Certificate(ParsingDict): +OID_COUNTRY = "2.5.4.6" +OID_ORG = "2.5.4.10" +OID_COMMON_NAME = "2.5.4.3" +EXT_SAN = "subjectAltName" - TIME_FORMAT = "%Y%m%d%H%M%SZ" - TIME_REGEX = re.compile( - "(\d\d\d\d)(\d\d)(\d\d)(\d\d)(\d\d)(\d\d)(\+|\-)(\d\d)(\d\d)" - ) + +class Certificate(ParsingDict): def __init__(self, data, **kwargs): @@ -49,6 +51,7 @@ def __init__(self, data, **kwargs): self.issuer_cn = None self.issuer_o = None self.issuer_c = None + self.valid_from = None self.valid_until = None @@ -58,32 +61,59 @@ def __init__(self, data, **kwargs): self.has_expired = None - # Clean up the certificate data and use OpenSSL to parse it - x509 = OpenSSL.crypto.load_certificate( - OpenSSL.crypto.FILETYPE_PEM, - data.replace("\\/", "/").replace("\n\n", "\n") - ) - subject = dict(x509.get_subject().get_components()) - issuer = dict(x509.get_issuer().get_components()) + self.extensions = {} + + cert = x509.load_pem_x509_certificate(data.encode("ascii"), openssl.backend) - if x509 and subject and issuer: + if cert: + self.checksum_md5 = self._colonify(cert.fingerprint(hashes.MD5())) + self.checksum_sha1 = self._colonify(cert.fingerprint(hashes.SHA1())) + self.checksum_sha256 = self._colonify(cert.fingerprint(hashes.SHA256())) - self.subject_cn = self._string_from_dict_or_none(subject, b"CN") - self.subject_o = self._string_from_dict_or_none(subject, b"O") - self.subject_c = self._string_from_dict_or_none(subject, b"C") - self.issuer_cn = self._string_from_dict_or_none(issuer, b"CN") - self.issuer_o = self._string_from_dict_or_none(issuer, b"O") - self.issuer_c = self._string_from_dict_or_none(issuer, b"C") + self.valid_from = pytz.utc.localize(cert.not_valid_before) + self.valid_until = pytz.utc.localize(cert.not_valid_after) - self.checksum_md5 = x509.digest("md5").decode() - self.checksum_sha1 = x509.digest("sha1").decode() - self.checksum_sha256 = x509.digest("sha256").decode() + self.has_expired = self._has_expired() - self.has_expired = bool(x509.has_expired()) + self._add_extensions(cert) - self.valid_from = None - self.valid_until = None - self._process_validation_times(x509) + if cert and cert.subject: + self.subject_cn, self.subject_o, self.subject_c = \ + self._parse_x509_name(cert.subject) + + if cert and cert.issuer: + self.issuer_cn, self.issuer_o, self.issuer_c = \ + self._parse_x509_name(cert.issuer) + + def _add_extensions(self, cert): + for ext in cert.extensions: + if ext.oid._name == EXT_SAN: + self.extensions[EXT_SAN] = [] + for san in ext.value: + self.extensions[EXT_SAN].append(san.value) + + @staticmethod + def _colonify(bytes): + hex = codecs.getencoder("hex_codec")(bytes)[0].decode("ascii").upper() + return ":".join(a+b for a,b in zip(hex[::2], hex[1::2])) + + @staticmethod + def _parse_x509_name(name): + cn = None + o = None + c = None + for attr in name: + if attr.oid.dotted_string == OID_COUNTRY: + c = attr.value + elif attr.oid.dotted_string == OID_ORG: + o = attr.value + elif attr.oid.dotted_string == OID_COMMON_NAME: + cn = attr.value + return cn, o, c + + def _has_expired(self): + now = pytz.utc.localize(datetime.utcnow()) + return self.valid_from <= now <= self.valid_until @property def cn(self): @@ -113,72 +143,6 @@ def country(self): def checksum(self): return self.checksum_sha256 - def _process_validation_times(self, x509): - """ - PyOpenSSL uses a kooky date format that *usually* parses out quite - easily but on the off chance that it's not in UTC, a lot of work needs - to be done. - """ - - valid_from = x509.get_notBefore() - valid_until = x509.get_notAfter() - - try: - self.valid_from = pytz.UTC.localize(datetime.strptime( - valid_from.decode(), - self.TIME_FORMAT - )) - except ValueError: - self.valid_from = self._process_nonstandard_time(valid_from) - - try: - self.valid_until = pytz.UTC.localize(datetime.strptime( - valid_until.decode(), - self.TIME_FORMAT - )) - except ValueError: - self.valid_until = self._process_nonstandard_time(valid_until) - - def _process_nonstandard_time(self, string): - """ - In addition to `YYYYMMDDhhmmssZ`, PyOpenSSL can also use timestamps - in `YYYYMMDDhhmmss+hhmm` or `YYYYMMDDhhmmss-hhmm`. - """ - - match = re.match(self.TIME_REGEX, string) - - if not match: - raise ResultParseError( - "Unrecognised time format: {s}".format(s=string) - ) - - r = datetime( - int(match.group(1)), - int(match.group(2)), - int(match.group(3)), - int(match.group(4)), - int(match.group(5)), - int(match.group(6)), - 0, - pytz.UTC - ) - delta = relativedelta( - hours=int(match.group(8)), - minutes=int(match.group(9)) - ) - if match.group(7) == "-": - return r - delta - return r + delta - - @staticmethod - def _string_from_dict_or_none(dictionary, key): - """ - Created to make nice with the Python3 problem. - """ - if key not in dictionary: - return None - return dictionary[key].decode("UTF-8") - class Alert(ParsingDict): diff --git a/setup.py b/setup.py index 1386080..3b90db4 100644 --- a/setup.py +++ b/setup.py @@ -12,9 +12,8 @@ tests_require = ["nose"] -# pyOpenSSL support is flaky on some systems (I'm looking at you Apple) if "SAGAN_WITHOUT_SSL" not in os.environ: - install_requires.append("pyOpenSSL") + install_requires.append("cryptography") # Allow setup.py to be run from any path os.chdir(os.path.normpath(os.path.join(os.path.abspath(__file__), os.pardir))) diff --git a/tests/ssl.py b/tests/ssl.py index 38b76c7..cc9b493 100644 --- a/tests/ssl.py +++ b/tests/ssl.py @@ -386,3 +386,9 @@ def test_alert(): result = Result.get('{"af":4,"cert":["-----BEGIN CERTIFICATE-----\\nMIIFBTCCAu2gAwIBAgIDDLHHMA0GCSqGSIb3DQEBBQUAMHkxEDAOBgNVBAoTB1Jv\\nb3QgQ0ExHjAcBgNVBAsTFWh0dHA6Ly93d3cuY2FjZXJ0Lm9yZzEiMCAGA1UEAxMZ\\nQ0EgQ2VydCBTaWduaW5nIEF1dGhvcml0eTEhMB8GCSqGSIb3DQEJARYSc3VwcG9y\\ndEBjYWNlcnQub3JnMB4XDTEzMDEwNjE0MDA1NVoXDTEzMDcwNTE0MDA1NVowGDEW\\nMBQGA1UEAxQNKi5wcmV0aWNhbC5lZTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC\\nAQoCggEBAMS+vX7gA8TvzFwxryFFRj1OyQjnW88GvfMuGhKJopalG1EB103oRsxi\\nMcXqwFZUicpqLKHW4lCHcRuhpKoZp8EOILnRAJRKFOjgIrcHQ02Xn4Lf/ewl601h\\n5qxqt1keU1P8j+u9m7zZN+vOoNlEKZ5SnZhysAAYqr/XIt1WY2cji/4GxjF+q1OH\\nIl5zddkIfnE52UbREKKlIakfFdj/c6GXqqsP2QTmm4x2HitCD964tZ06fA9BitQj\\nnnBXNhtm2MCuBIPBSq0/C7LREwmfnqxCFqE7iqEPNIQ2IT2D4Gh4c+nIZHqYKvCV\\nP3zh3aUaBj1o5Lo83IDdXCKAIiQRFMkCAwEAAaOB9jCB8zAMBgNVHRMBAf8EAjAA\\nMA4GA1UdDwEB/wQEAwIDqDA0BgNVHSUELTArBggrBgEFBQcDAgYIKwYBBQUHAwEG\\nCWCGSAGG+EIEAQYKKwYBBAGCNwoDAzAzBggrBgEFBQcBAQQnMCUwIwYIKwYBBQUH\\nMAGGF2h0dHA6Ly9vY3NwLmNhY2VydC5vcmcvMDEGA1UdHwQqMCgwJqAkoCKGIGh0\\ndHA6Ly9jcmwuY2FjZXJ0Lm9yZy9yZXZva2UuY3JsMDUGA1UdEQQuMCyCDSoucHJl\\ndGljYWwuZWWgGwYIKwYBBQUHCAWgDwwNKi5wcmV0aWNhbC5lZTANBgkqhkiG9w0B\\nAQUFAAOCAgEAycddS/c47eo0WVrFxpvCIJdfvn7CYdTPpXNSg0kjapkSjYuAkcmq\\nsrScUUGMBe6tfkmkdPTuNKwRVYNJ1Wi9EYaMvJ3CVw6x9O5mgktmu0ogbIXsivwI\\nTSzGDMWcb9Of85e/ALWpK0cWIugtWO0d6v3qMWfxlYfAaYu49pttOJQOjbXAAhfR\\njE5VOcDaIlWChG48jLAyCLsMwHlyLw8D5Myb9MfTs1XxgLESO9ZTSqGEqJw+BwTJ\\nstHk/oCHo9FL/Xv5MmFcNaTpqbB60duYJ+DLLX1QiRRfLJ18G7wEiEAm6H9egupQ\\nL9MhQQLJ4o60xTrCnpqGTXTSR16jiTm70bDB0+SU3xTpNwCzdigH6ceKbPIr0cO6\\no0ump598e2JNCPsXIc+XcbLDDFgYrlnl3YnK3J+K3LC7SWPMsYdDfe+Im880tNuW\\nOlnOCDpP8408KqCp4xm0DMznmThUM34/Ia+o8Q3NWNBfuaOsJ9aA+FmgobJhih9e\\nUr9x3ByRQXcW5Cs/AMtCikKWVPsx+IA5eeyt+1i+dKBWksO40B3ADsq1O5DRYYRa\\n+dwqdX/jduqZjbyHuFH04q28j4zVDviUBQEa9UQoDM3c82dILDjbYtZ+T28sPMTa\\nbMZdcMur9E+ovrS58lIKGCvDEPSUDXHzr0tpb4A13TTnxW6pclqUyJk=\\n-----END CERTIFICATE-----"],"dst_addr":"80.79.115.54","dst_name":"pretical.ee","dst_port":"https","from":"77.95.64.18","fw":4480,"method":"SSL","msm_id":1006864,"prb_id":517,"src_addr":"77.95.64.18","timestamp":1362454627,"type":"sslcert","ver":"3.0"}') assert(result.alert is None) assert(result.is_error is False) + +def test_san_extension(): + result = Result.get('{"af":4,"cert":["-----BEGIN CERTIFICATE-----\nMIIH4jCCBsqgAwIBAgIIFaqhpQEYRXIwDQYJKoZIhvcNAQELBQAwSTELMAkGA1UE\nBhMCVVMxEzARBgNVBAoTCkdvb2dsZSBJbmMxJTAjBgNVBAMTHEdvb2dsZSBJbnRl\ncm5ldCBBdXRob3JpdHkgRzIwHhcNMTcwMzE2MDkzNzQyWhcNMTcwNjA4MDg1NDAw\nWjBmMQswCQYDVQQGEwJVUzETMBEGA1UECAwKQ2FsaWZvcm5pYTEWMBQGA1UEBwwN\nTW91bnRhaW4gVmlldzETMBEGA1UECgwKR29vZ2xlIEluYzEVMBMGA1UEAwwMKi5n\nb29nbGUuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAjgPs3rpA\ntF2jQzXrVQ8x33EVHB3OIpj3GcwVf8U9qcPce0XuG97fHInb20U9Uw1b45ecNRtn\nWLUw14/7+F4cvFJXHHsYaoUdBoeSJAcOy8ktgxvIEMk82KJwJlzWA7X7B459Fy1U\nr8Dvu6dNFzhtu8eJs8bFOMJ/Wczjh8tylKXyWNMpotTbvAG3rGH+8fttmGXnztTB\n3dwxxf6SEL6m4XGH7POxwH9+AKzIwV9PrkU4JM5U2YsGPHf6ao/w27gPVpO5sh3g\nP9J/3jf8lXNwPZWSLCK5C2i7kz12ohaD7jlipVyw4nYLcEFPs27LwzjYa/YFU8VZ\nreIcbazBmDsqBwIDAQABo4IErzCCBKswHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsG\nAQUFBwMCMIIDewYDVR0RBIIDcjCCA26CDCouZ29vZ2xlLmNvbYINKi5hbmRyb2lk\nLmNvbYIWKi5hcHBlbmdpbmUuZ29vZ2xlLmNvbYISKi5jbG91ZC5nb29nbGUuY29t\ngg4qLmdjcC5ndnQyLmNvbYIWKi5nb29nbGUtYW5hbHl0aWNzLmNvbYILKi5nb29n\nbGUuY2GCCyouZ29vZ2xlLmNsgg4qLmdvb2dsZS5jby5pboIOKi5nb29nbGUuY28u\nanCCDiouZ29vZ2xlLmNvLnVrgg8qLmdvb2dsZS5jb20uYXKCDyouZ29vZ2xlLmNv\nbS5hdYIPKi5nb29nbGUuY29tLmJygg8qLmdvb2dsZS5jb20uY2+CDyouZ29vZ2xl\nLmNvbS5teIIPKi5nb29nbGUuY29tLnRygg8qLmdvb2dsZS5jb20udm6CCyouZ29v\nZ2xlLmRlggsqLmdvb2dsZS5lc4ILKi5nb29nbGUuZnKCCyouZ29vZ2xlLmh1ggsq\nLmdvb2dsZS5pdIILKi5nb29nbGUubmyCCyouZ29vZ2xlLnBsggsqLmdvb2dsZS5w\ndIISKi5nb29nbGVhZGFwaXMuY29tgg8qLmdvb2dsZWFwaXMuY26CFCouZ29vZ2xl\nY29tbWVyY2UuY29tghEqLmdvb2dsZXZpZGVvLmNvbYIMKi5nc3RhdGljLmNugg0q\nLmdzdGF0aWMuY29tggoqLmd2dDEuY29tggoqLmd2dDIuY29tghQqLm1ldHJpYy5n\nc3RhdGljLmNvbYIMKi51cmNoaW4uY29tghAqLnVybC5nb29nbGUuY29tghYqLnlv\ndXR1YmUtbm9jb29raWUuY29tgg0qLnlvdXR1YmUuY29tghYqLnlvdXR1YmVlZHVj\nYXRpb24uY29tggsqLnl0aW1nLmNvbYIaYW5kcm9pZC5jbGllbnRzLmdvb2dsZS5j\nb22CC2FuZHJvaWQuY29tghtkZXZlbG9wZXIuYW5kcm9pZC5nb29nbGUuY26CBGcu\nY2+CBmdvby5nbIIUZ29vZ2xlLWFuYWx5dGljcy5jb22CCmdvb2dsZS5jb22CEmdv\nb2dsZWNvbW1lcmNlLmNvbYIKdXJjaGluLmNvbYIKd3d3Lmdvby5nbIIIeW91dHUu\nYmWCC3lvdXR1YmUuY29tghR5b3V0dWJlZWR1Y2F0aW9uLmNvbTBoBggrBgEFBQcB\nAQRcMFowKwYIKwYBBQUHMAKGH2h0dHA6Ly9wa2kuZ29vZ2xlLmNvbS9HSUFHMi5j\ncnQwKwYIKwYBBQUHMAGGH2h0dHA6Ly9jbGllbnRzMS5nb29nbGUuY29tL29jc3Aw\nHQYDVR0OBBYEFHRy1woLF5IqQVubJZ5ZvXAjaJ0aMAwGA1UdEwEB/wQCMAAwHwYD\nVR0jBBgwFoAUSt0GFhu89mi1dvWBtrtiGrpagS8wIQYDVR0gBBowGDAMBgorBgEE\nAdZ5AgUBMAgGBmeBDAECAjAwBgNVHR8EKTAnMCWgI6Ahhh9odHRwOi8vcGtpLmdv\nb2dsZS5jb20vR0lBRzIuY3JsMA0GCSqGSIb3DQEBCwUAA4IBAQAsoPR1jJz2adkK\nTVXpGse/M3l+xKgmuZHpXzXkAiqE9wcsxXxCU3dEUmPBYYGRTqODNkOh9AMyGzIL\nfrYh/zY9rhqJ2B26OunmxKFF9BmwRi2rp+Ksvg/+27F57Hyaq2phSaR8E7hRZcYR\nYqCaNA5e1hialuB1g58mAvs38jxxV4bQhKzCKkBOxolhYbUEBEV4mQ14ODdSvAB0\n8L1dMjk3+LEDB/hWdtpOOhtMbSPa1u7xJeM/Ip7+GV47lS3V6rUALDKz4ASNk8ih\nX0ZmxPA1rabqNFutG8L/4HK2/ffO4bKEkHEdOQXC9B17n1x65fbLUbweDPDAzaow\nrum/OChG\n-----END CERTIFICATE-----"],"dst_addr":"193.0.6.158","dst_name":"atlas.ripe.net","dst_port":"443","from":"86.82.178.27","fw":4760,"lts":133,"method":"TLS","msm_id":14002,"msm_name":"SSLCert","prb_id":10951,"rt":51.558465,"src_addr":"192.168.180.22","timestamp":1490659208,"ttc":14.88238,"type":"sslcert","ver":"1.2"}') + ext = result.certificates[0].extensions + assert(ext and len(ext['subjectAltName'])==54) +