From 2abc94661a16f1a4c5bc18e9c48870f2db458eac Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Mon, 11 Dec 2017 20:25:09 +0100 Subject: [PATCH 01/21] Use josepy instead of acme.jose. (#5203) --- acme/acme/challenges.py | 2 +- acme/acme/challenges_test.py | 2 +- acme/acme/client.py | 4 +- acme/acme/client_test.py | 2 +- acme/acme/crypto_util_test.py | 2 +- acme/acme/errors.py | 2 +- acme/acme/fields.py | 3 +- acme/acme/fields_test.py | 3 +- acme/acme/jose/__init__.py | 82 --- acme/acme/jose/b64.py | 61 --- acme/acme/jose/b64_test.py | 77 --- acme/acme/jose/errors.py | 35 -- acme/acme/jose/errors_test.py | 17 - acme/acme/jose/interfaces.py | 216 -------- acme/acme/jose/interfaces_test.py | 114 ---- acme/acme/jose/json_util.py | 485 ------------------ acme/acme/jose/json_util_test.py | 381 -------------- acme/acme/jose/jwa.py | 180 ------- acme/acme/jose/jwa_test.py | 104 ---- acme/acme/jose/jwk.py | 281 ---------- acme/acme/jose/jwk_test.py | 191 ------- acme/acme/jose/jws.py | 433 ---------------- acme/acme/jose/jws_test.py | 239 --------- acme/acme/jose/util.py | 226 -------- acme/acme/jose/util_test.py | 199 ------- acme/acme/jws.py | 6 +- acme/acme/jws_test.py | 3 +- acme/acme/messages.py | 9 +- acme/acme/messages_test.py | 2 +- acme/acme/standalone_test.py | 2 +- acme/acme/test_util.py | 3 +- acme/docs/api/jose.rst | 9 +- acme/docs/api/jose/base64.rst | 5 - acme/docs/api/jose/errors.rst | 5 - acme/docs/api/jose/interfaces.rst | 5 - acme/docs/api/jose/json_util.rst | 5 - acme/docs/api/jose/jwa.rst | 5 - acme/docs/api/jose/jwk.rst | 5 - acme/docs/api/jose/jws.rst | 5 - acme/docs/api/jose/util.rst | 5 - acme/docs/conf.py | 1 + acme/examples/example_client.py | 2 +- acme/setup.py | 7 +- certbot-apache/certbot_apache/tests/util.py | 3 +- .../certbot_compatibility_test/util.py | 3 +- certbot-nginx/certbot_nginx/tests/util.py | 3 +- certbot/account.py | 2 +- certbot/achallenges.py | 3 +- certbot/client.py | 2 +- certbot/crypto_util.py | 4 +- certbot/main.py | 2 +- certbot/plugins/common.py | 2 +- certbot/plugins/common_test.py | 2 +- certbot/plugins/dns_test_common.py | 2 +- certbot/plugins/dns_test_common_lexicon.py | 2 +- certbot/plugins/standalone_test.py | 2 +- certbot/plugins/webroot_test.py | 2 +- certbot/tests/account_test.py | 2 +- certbot/tests/acme_util.py | 2 +- certbot/tests/client_test.py | 2 +- certbot/tests/display/ops_test.py | 2 +- certbot/tests/main_test.py | 3 +- certbot/tests/util.py | 3 +- tools/deactivate.py | 2 +- 64 files changed, 53 insertions(+), 3422 deletions(-) delete mode 100644 acme/acme/jose/__init__.py delete mode 100644 acme/acme/jose/b64.py delete mode 100644 acme/acme/jose/b64_test.py delete mode 100644 acme/acme/jose/errors.py delete mode 100644 acme/acme/jose/errors_test.py delete mode 100644 acme/acme/jose/interfaces.py delete mode 100644 acme/acme/jose/interfaces_test.py delete mode 100644 acme/acme/jose/json_util.py delete mode 100644 acme/acme/jose/json_util_test.py delete mode 100644 acme/acme/jose/jwa.py delete mode 100644 acme/acme/jose/jwa_test.py delete mode 100644 acme/acme/jose/jwk.py delete mode 100644 acme/acme/jose/jwk_test.py delete mode 100644 acme/acme/jose/jws.py delete mode 100644 acme/acme/jose/jws_test.py delete mode 100644 acme/acme/jose/util.py delete mode 100644 acme/acme/jose/util_test.py delete mode 100644 acme/docs/api/jose/base64.rst delete mode 100644 acme/docs/api/jose/errors.rst delete mode 100644 acme/docs/api/jose/interfaces.rst delete mode 100644 acme/docs/api/jose/json_util.rst delete mode 100644 acme/docs/api/jose/jwa.rst delete mode 100644 acme/docs/api/jose/jwk.rst delete mode 100644 acme/docs/api/jose/jws.rst delete mode 100644 acme/docs/api/jose/util.rst diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 14641af108a..96997297b91 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -6,13 +6,13 @@ import socket from cryptography.hazmat.primitives import hashes # type: ignore +import josepy as jose import OpenSSL import requests from acme import errors from acme import crypto_util from acme import fields -from acme import jose logger = logging.getLogger(__name__) diff --git a/acme/acme/challenges_test.py b/acme/acme/challenges_test.py index 49e79010243..834d569aaf5 100644 --- a/acme/acme/challenges_test.py +++ b/acme/acme/challenges_test.py @@ -1,6 +1,7 @@ """Tests for acme.challenges.""" import unittest +import josepy as jose import mock import OpenSSL import requests @@ -8,7 +9,6 @@ from six.moves.urllib import parse as urllib_parse # pylint: disable=import-error from acme import errors -from acme import jose from acme import test_util CERT = test_util.load_comparable_cert('cert.pem') diff --git a/acme/acme/client.py b/acme/acme/client.py index 2e07d34d70c..dc5efbe86f8 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -10,13 +10,13 @@ import six from six.moves import http_client # pylint: disable=import-error +import josepy as jose import OpenSSL import re import requests import sys from acme import errors -from acme import jose from acme import jws from acme import messages @@ -408,7 +408,7 @@ def _get_cert(self, uri): :param str uri: URI of certificate :returns: tuple of the form - (response, :class:`acme.jose.ComparableX509`) + (response, :class:`josepy.util.ComparableX509`) :rtype: tuple """ diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 4bd762865e3..84620fc994f 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -5,12 +5,12 @@ from six.moves import http_client # pylint: disable=import-error +import josepy as jose import mock import requests from acme import challenges from acme import errors -from acme import jose from acme import jws as acme_jws from acme import messages from acme import messages_test diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index da433c5a2cf..1d7f83ccf22 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -8,10 +8,10 @@ import six from six.moves import socketserver #type: ignore # pylint: disable=import-error +import josepy as jose import OpenSSL from acme import errors -from acme import jose from acme import test_util diff --git a/acme/acme/errors.py b/acme/acme/errors.py index 9d991fd75d2..de5f9d1f419 100644 --- a/acme/acme/errors.py +++ b/acme/acme/errors.py @@ -1,5 +1,5 @@ """ACME errors.""" -from acme.jose import errors as jose_errors +from josepy import errors as jose_errors class Error(Exception): diff --git a/acme/acme/fields.py b/acme/acme/fields.py index 12d09acf48b..d7ec78403ed 100644 --- a/acme/acme/fields.py +++ b/acme/acme/fields.py @@ -1,10 +1,9 @@ """ACME JSON fields.""" import logging +import josepy as jose import pyrfc3339 -from acme import jose - logger = logging.getLogger(__name__) diff --git a/acme/acme/fields_test.py b/acme/acme/fields_test.py index de852b6fa23..69dde8b89bc 100644 --- a/acme/acme/fields_test.py +++ b/acme/acme/fields_test.py @@ -2,10 +2,9 @@ import datetime import unittest +import josepy as jose import pytz -from acme import jose - class FixedTest(unittest.TestCase): """Tests for acme.fields.Fixed.""" diff --git a/acme/acme/jose/__init__.py b/acme/acme/jose/__init__.py deleted file mode 100644 index 9116bc43339..00000000000 --- a/acme/acme/jose/__init__.py +++ /dev/null @@ -1,82 +0,0 @@ -"""Javascript Object Signing and Encryption (jose). - -This package is a Python implementation of the standards developed by -IETF `Javascript Object Signing and Encryption (Active WG)`_, in -particular the following RFCs: - - - `JSON Web Algorithms (JWA)`_ - - `JSON Web Key (JWK)`_ - - `JSON Web Signature (JWS)`_ - - -.. _`Javascript Object Signing and Encryption (Active WG)`: - https://tools.ietf.org/wg/jose/ - -.. _`JSON Web Algorithms (JWA)`: - https://datatracker.ietf.org/doc/draft-ietf-jose-json-web-algorithms/ - -.. _`JSON Web Key (JWK)`: - https://datatracker.ietf.org/doc/draft-ietf-jose-json-web-key/ - -.. _`JSON Web Signature (JWS)`: - https://datatracker.ietf.org/doc/draft-ietf-jose-json-web-signature/ - -""" -from acme.jose.b64 import ( - b64decode, - b64encode, -) - -from acme.jose.errors import ( - DeserializationError, - SerializationError, - Error, - UnrecognizedTypeError, -) - -from acme.jose.interfaces import JSONDeSerializable - -from acme.jose.json_util import ( - Field, - JSONObjectWithFields, - TypedJSONObjectWithFields, - decode_b64jose, - decode_cert, - decode_csr, - decode_hex16, - encode_b64jose, - encode_cert, - encode_csr, - encode_hex16, -) - -from acme.jose.jwa import ( - HS256, - HS384, - HS512, - JWASignature, - PS256, - PS384, - PS512, - RS256, - RS384, - RS512, -) - -from acme.jose.jwk import ( - JWK, - JWKRSA, -) - -from acme.jose.jws import ( - Header, - JWS, - Signature, -) - -from acme.jose.util import ( - ComparableX509, - ComparableKey, - ComparableRSAKey, - ImmutableMap, -) diff --git a/acme/acme/jose/b64.py b/acme/acme/jose/b64.py deleted file mode 100644 index cf79aa8200e..00000000000 --- a/acme/acme/jose/b64.py +++ /dev/null @@ -1,61 +0,0 @@ -"""JOSE Base64. - -`JOSE Base64`_ is defined as: - - - URL-safe Base64 - - padding stripped - - -.. _`JOSE Base64`: - https://tools.ietf.org/html/draft-ietf-jose-json-web-signature-37#appendix-C - -.. Do NOT try to call this module "base64", as it will "shadow" the - standard library. - -""" -import base64 - -import six - - -def b64encode(data): - """JOSE Base64 encode. - - :param data: Data to be encoded. - :type data: `bytes` - - :returns: JOSE Base64 string. - :rtype: bytes - - :raises TypeError: if `data` is of incorrect type - - """ - if not isinstance(data, six.binary_type): - raise TypeError('argument should be {0}'.format(six.binary_type)) - return base64.urlsafe_b64encode(data).rstrip(b'=') - - -def b64decode(data): - """JOSE Base64 decode. - - :param data: Base64 string to be decoded. If it's unicode, then - only ASCII characters are allowed. - :type data: `bytes` or `unicode` - - :returns: Decoded data. - :rtype: bytes - - :raises TypeError: if input is of incorrect type - :raises ValueError: if input is unicode with non-ASCII characters - - """ - if isinstance(data, six.string_types): - try: - data = data.encode('ascii') - except UnicodeEncodeError: - raise ValueError( - 'unicode argument should contain only ASCII characters') - elif not isinstance(data, six.binary_type): - raise TypeError('argument should be a str or unicode') - - return base64.urlsafe_b64decode(data + b'=' * (4 - (len(data) % 4))) diff --git a/acme/acme/jose/b64_test.py b/acme/acme/jose/b64_test.py deleted file mode 100644 index cbabe225171..00000000000 --- a/acme/acme/jose/b64_test.py +++ /dev/null @@ -1,77 +0,0 @@ -"""Tests for acme.jose.b64.""" -import unittest - -import six - - -# https://en.wikipedia.org/wiki/Base64#Examples -B64_PADDING_EXAMPLES = { - b'any carnal pleasure.': (b'YW55IGNhcm5hbCBwbGVhc3VyZS4', b'='), - b'any carnal pleasure': (b'YW55IGNhcm5hbCBwbGVhc3VyZQ', b'=='), - b'any carnal pleasur': (b'YW55IGNhcm5hbCBwbGVhc3Vy', b''), - b'any carnal pleasu': (b'YW55IGNhcm5hbCBwbGVhc3U', b'='), - b'any carnal pleas': (b'YW55IGNhcm5hbCBwbGVhcw', b'=='), -} - - -B64_URL_UNSAFE_EXAMPLES = { - six.int2byte(251) + six.int2byte(239): b'--8', - six.int2byte(255) * 2: b'__8', -} - - -class B64EncodeTest(unittest.TestCase): - """Tests for acme.jose.b64.b64encode.""" - - @classmethod - def _call(cls, data): - from acme.jose.b64 import b64encode - return b64encode(data) - - def test_empty(self): - self.assertEqual(self._call(b''), b'') - - def test_unsafe_url(self): - for text, b64 in six.iteritems(B64_URL_UNSAFE_EXAMPLES): - self.assertEqual(self._call(text), b64) - - def test_different_paddings(self): - for text, (b64, _) in six.iteritems(B64_PADDING_EXAMPLES): - self.assertEqual(self._call(text), b64) - - def test_unicode_fails_with_type_error(self): - self.assertRaises(TypeError, self._call, u'some unicode') - - -class B64DecodeTest(unittest.TestCase): - """Tests for acme.jose.b64.b64decode.""" - - @classmethod - def _call(cls, data): - from acme.jose.b64 import b64decode - return b64decode(data) - - def test_unsafe_url(self): - for text, b64 in six.iteritems(B64_URL_UNSAFE_EXAMPLES): - self.assertEqual(self._call(b64), text) - - def test_input_without_padding(self): - for text, (b64, _) in six.iteritems(B64_PADDING_EXAMPLES): - self.assertEqual(self._call(b64), text) - - def test_input_with_padding(self): - for text, (b64, pad) in six.iteritems(B64_PADDING_EXAMPLES): - self.assertEqual(self._call(b64 + pad), text) - - def test_unicode_with_ascii(self): - self.assertEqual(self._call(u'YQ'), b'a') - - def test_non_ascii_unicode_fails(self): - self.assertRaises(ValueError, self._call, u'\u0105') - - def test_type_error_no_unicode_or_bytes(self): - self.assertRaises(TypeError, self._call, object()) - - -if __name__ == '__main__': - unittest.main() # pragma: no cover diff --git a/acme/acme/jose/errors.py b/acme/acme/jose/errors.py deleted file mode 100644 index 74c9443e1ec..00000000000 --- a/acme/acme/jose/errors.py +++ /dev/null @@ -1,35 +0,0 @@ -"""JOSE errors.""" - - -class Error(Exception): - """Generic JOSE Error.""" - - -class DeserializationError(Error): - """JSON deserialization error.""" - - def __str__(self): - return "Deserialization error: {0}".format( - super(DeserializationError, self).__str__()) - - -class SerializationError(Error): - """JSON serialization error.""" - - -class UnrecognizedTypeError(DeserializationError): - """Unrecognized type error. - - :ivar str typ: The unrecognized type of the JSON object. - :ivar jobj: Full JSON object. - - """ - - def __init__(self, typ, jobj): - self.typ = typ - self.jobj = jobj - super(UnrecognizedTypeError, self).__init__(str(self)) - - def __str__(self): - return '{0} was not recognized, full message: {1}'.format( - self.typ, self.jobj) diff --git a/acme/acme/jose/errors_test.py b/acme/acme/jose/errors_test.py deleted file mode 100644 index 919980920c0..00000000000 --- a/acme/acme/jose/errors_test.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Tests for acme.jose.errors.""" -import unittest - - -class UnrecognizedTypeErrorTest(unittest.TestCase): - def setUp(self): - from acme.jose.errors import UnrecognizedTypeError - self.error = UnrecognizedTypeError('foo', {'type': 'foo'}) - - def test_str(self): - self.assertEqual( - "foo was not recognized, full message: {'type': 'foo'}", - str(self.error)) - - -if __name__ == '__main__': - unittest.main() # pragma: no cover diff --git a/acme/acme/jose/interfaces.py b/acme/acme/jose/interfaces.py deleted file mode 100644 index f841848b380..00000000000 --- a/acme/acme/jose/interfaces.py +++ /dev/null @@ -1,216 +0,0 @@ -"""JOSE interfaces.""" -import abc -import collections -import json - -import six - -from acme.jose import errors -from acme.jose import util - -# pylint: disable=no-self-argument,no-method-argument,no-init,inherit-non-class -# pylint: disable=too-few-public-methods - - -@six.add_metaclass(abc.ABCMeta) -class JSONDeSerializable(object): - # pylint: disable=too-few-public-methods - """Interface for (de)serializable JSON objects. - - Please recall, that standard Python library implements - :class:`json.JSONEncoder` and :class:`json.JSONDecoder` that perform - translations based on respective :ref:`conversion tables - ` that look pretty much like the one below (for - complete tables see relevant Python documentation): - - .. _conversion-table: - - ====== ====== - JSON Python - ====== ====== - object dict - ... ... - ====== ====== - - While the above **conversion table** is about translation of JSON - documents to/from the basic Python types only, - :class:`JSONDeSerializable` introduces the following two concepts: - - serialization - Turning an arbitrary Python object into Python object that can - be encoded into a JSON document. **Full serialization** produces - a Python object composed of only basic types as required by the - :ref:`conversion table `. **Partial - serialization** (accomplished by :meth:`to_partial_json`) - produces a Python object that might also be built from other - :class:`JSONDeSerializable` objects. - - deserialization - Turning a decoded Python object (necessarily one of the basic - types as required by the :ref:`conversion table - `) into an arbitrary Python object. - - Serialization produces **serialized object** ("partially serialized - object" or "fully serialized object" for partial and full - serialization respectively) and deserialization produces - **deserialized object**, both usually denoted in the source code as - ``jobj``. - - Wording in the official Python documentation might be confusing - after reading the above, but in the light of those definitions, one - can view :meth:`json.JSONDecoder.decode` as decoder and - deserializer of basic types, :meth:`json.JSONEncoder.default` as - serializer of basic types, :meth:`json.JSONEncoder.encode` as - serializer and encoder of basic types. - - One could extend :mod:`json` to support arbitrary object - (de)serialization either by: - - - overriding :meth:`json.JSONDecoder.decode` and - :meth:`json.JSONEncoder.default` in subclasses - - - or passing ``object_hook`` argument (or ``object_hook_pairs``) - to :func:`json.load`/:func:`json.loads` or ``default`` argument - for :func:`json.dump`/:func:`json.dumps`. - - Interestingly, ``default`` is required to perform only partial - serialization, as :func:`json.dumps` applies ``default`` - recursively. This is the idea behind making :meth:`to_partial_json` - produce only partial serialization, while providing custom - :meth:`json_dumps` that dumps with ``default`` set to - :meth:`json_dump_default`. - - To make further documentation a bit more concrete, please, consider - the following imaginatory implementation example:: - - class Foo(JSONDeSerializable): - def to_partial_json(self): - return 'foo' - - @classmethod - def from_json(cls, jobj): - return Foo() - - class Bar(JSONDeSerializable): - def to_partial_json(self): - return [Foo(), Foo()] - - @classmethod - def from_json(cls, jobj): - return Bar() - - """ - - @abc.abstractmethod - def to_partial_json(self): # pragma: no cover - """Partially serialize. - - Following the example, **partial serialization** means the following:: - - assert isinstance(Bar().to_partial_json()[0], Foo) - assert isinstance(Bar().to_partial_json()[1], Foo) - - # in particular... - assert Bar().to_partial_json() != ['foo', 'foo'] - - :raises acme.jose.errors.SerializationError: - in case of any serialization error. - :returns: Partially serializable object. - - """ - raise NotImplementedError() - - def to_json(self): - """Fully serialize. - - Again, following the example from before, **full serialization** - means the following:: - - assert Bar().to_json() == ['foo', 'foo'] - - :raises acme.jose.errors.SerializationError: - in case of any serialization error. - :returns: Fully serialized object. - - """ - def _serialize(obj): - if isinstance(obj, JSONDeSerializable): - return _serialize(obj.to_partial_json()) - if isinstance(obj, six.string_types): # strings are Sequence - return obj - elif isinstance(obj, list): - return [_serialize(subobj) for subobj in obj] - elif isinstance(obj, collections.Sequence): - # default to tuple, otherwise Mapping could get - # unhashable list - return tuple(_serialize(subobj) for subobj in obj) - elif isinstance(obj, collections.Mapping): - return dict((_serialize(key), _serialize(value)) - for key, value in six.iteritems(obj)) - else: - return obj - - return _serialize(self) - - @util.abstractclassmethod - def from_json(cls, jobj): # pylint: disable=unused-argument - """Deserialize a decoded JSON document. - - :param jobj: Python object, composed of only other basic data - types, as decoded from JSON document. Not necessarily - :class:`dict` (as decoded from "JSON object" document). - - :raises acme.jose.errors.DeserializationError: - if decoding was unsuccessful, e.g. in case of unparseable - X509 certificate, or wrong padding in JOSE base64 encoded - string, etc. - - """ - # TypeError: Can't instantiate abstract class with - # abstract methods from_json, to_partial_json - return cls() # pylint: disable=abstract-class-instantiated - - @classmethod - def json_loads(cls, json_string): - """Deserialize from JSON document string.""" - try: - loads = json.loads(json_string) - except ValueError as error: - raise errors.DeserializationError(error) - return cls.from_json(loads) - - def json_dumps(self, **kwargs): - """Dump to JSON string using proper serializer. - - :returns: JSON document string. - :rtype: str - - """ - return json.dumps(self, default=self.json_dump_default, **kwargs) - - def json_dumps_pretty(self): - """Dump the object to pretty JSON document string. - - :rtype: str - - """ - return self.json_dumps(sort_keys=True, indent=4, separators=(',', ': ')) - - @classmethod - def json_dump_default(cls, python_object): - """Serialize Python object. - - This function is meant to be passed as ``default`` to - :func:`json.dump` or :func:`json.dumps`. They call - ``default(python_object)`` only for non-basic Python types, so - this function necessarily raises :class:`TypeError` if - ``python_object`` is not an instance of - :class:`IJSONSerializable`. - - Please read the class docstring for more information. - - """ - if isinstance(python_object, JSONDeSerializable): - return python_object.to_partial_json() - else: # this branch is necessary, cannot just "return" - raise TypeError(repr(python_object) + ' is not JSON serializable') diff --git a/acme/acme/jose/interfaces_test.py b/acme/acme/jose/interfaces_test.py deleted file mode 100644 index cf98ff37110..00000000000 --- a/acme/acme/jose/interfaces_test.py +++ /dev/null @@ -1,114 +0,0 @@ -"""Tests for acme.jose.interfaces.""" -import unittest - - -class JSONDeSerializableTest(unittest.TestCase): - # pylint: disable=too-many-instance-attributes - - def setUp(self): - from acme.jose.interfaces import JSONDeSerializable - - # pylint: disable=missing-docstring,invalid-name - - class Basic(JSONDeSerializable): - def __init__(self, v): - self.v = v - - def to_partial_json(self): - return self.v - - @classmethod - def from_json(cls, jobj): - return cls(jobj) - - class Sequence(JSONDeSerializable): - def __init__(self, x, y): - self.x = x - self.y = y - - def to_partial_json(self): - return [self.x, self.y] - - @classmethod - def from_json(cls, jobj): - return cls( - Basic.from_json(jobj[0]), Basic.from_json(jobj[1])) - - class Mapping(JSONDeSerializable): - def __init__(self, x, y): - self.x = x - self.y = y - - def to_partial_json(self): - return {self.x: self.y} - - @classmethod - def from_json(cls, jobj): - pass # pragma: no cover - - self.basic1 = Basic('foo1') - self.basic2 = Basic('foo2') - self.seq = Sequence(self.basic1, self.basic2) - self.mapping = Mapping(self.basic1, self.basic2) - self.nested = Basic([[self.basic1]]) - self.tuple = Basic(('foo',)) - - # pylint: disable=invalid-name - self.Basic = Basic - self.Sequence = Sequence - self.Mapping = Mapping - - def test_to_json_sequence(self): - self.assertEqual(self.seq.to_json(), ['foo1', 'foo2']) - - def test_to_json_mapping(self): - self.assertEqual(self.mapping.to_json(), {'foo1': 'foo2'}) - - def test_to_json_other(self): - mock_value = object() - self.assertTrue(self.Basic(mock_value).to_json() is mock_value) - - def test_to_json_nested(self): - self.assertEqual(self.nested.to_json(), [['foo1']]) - - def test_to_json(self): - self.assertEqual(self.tuple.to_json(), (('foo', ))) - - def test_from_json_not_implemented(self): - from acme.jose.interfaces import JSONDeSerializable - self.assertRaises(TypeError, JSONDeSerializable.from_json, 'xxx') - - def test_json_loads(self): - seq = self.Sequence.json_loads('["foo1", "foo2"]') - self.assertTrue(isinstance(seq, self.Sequence)) - self.assertTrue(isinstance(seq.x, self.Basic)) - self.assertTrue(isinstance(seq.y, self.Basic)) - self.assertEqual(seq.x.v, 'foo1') - self.assertEqual(seq.y.v, 'foo2') - - def test_json_dumps(self): - self.assertEqual('["foo1", "foo2"]', self.seq.json_dumps()) - - def test_json_dumps_pretty(self): - self.assertEqual(self.seq.json_dumps_pretty(), - '[\n "foo1",\n "foo2"\n]') - - def test_json_dump_default(self): - from acme.jose.interfaces import JSONDeSerializable - - self.assertEqual( - 'foo1', JSONDeSerializable.json_dump_default(self.basic1)) - - jobj = JSONDeSerializable.json_dump_default(self.seq) - self.assertEqual(len(jobj), 2) - self.assertTrue(jobj[0] is self.basic1) - self.assertTrue(jobj[1] is self.basic2) - - def test_json_dump_default_type_error(self): - from acme.jose.interfaces import JSONDeSerializable - self.assertRaises( - TypeError, JSONDeSerializable.json_dump_default, object()) - - -if __name__ == '__main__': - unittest.main() # pragma: no cover diff --git a/acme/acme/jose/json_util.py b/acme/acme/jose/json_util.py deleted file mode 100644 index 4baadda5eea..00000000000 --- a/acme/acme/jose/json_util.py +++ /dev/null @@ -1,485 +0,0 @@ -"""JSON (de)serialization framework. - -The framework presented here is somewhat based on `Go's "json" package`_ -(especially the ``omitempty`` functionality). - -.. _`Go's "json" package`: http://golang.org/pkg/encoding/json/ - -""" -import abc -import binascii -import logging - -import OpenSSL -import six - -from acme.jose import b64 -from acme.jose import errors -from acme.jose import interfaces -from acme.jose import util - - -logger = logging.getLogger(__name__) - - -class Field(object): - """JSON object field. - - :class:`Field` is meant to be used together with - :class:`JSONObjectWithFields`. - - ``encoder`` (``decoder``) is a callable that accepts a single - parameter, i.e. a value to be encoded (decoded), and returns the - serialized (deserialized) value. In case of errors it should raise - :class:`~acme.jose.errors.SerializationError` - (:class:`~acme.jose.errors.DeserializationError`). - - Note, that ``decoder`` should perform partial serialization only. - - :ivar str json_name: Name of the field when encoded to JSON. - :ivar default: Default value (used when not present in JSON object). - :ivar bool omitempty: If ``True`` and the field value is empty, then - it will not be included in the serialized JSON object, and - ``default`` will be used for deserialization. Otherwise, if ``False``, - field is considered as required, value will always be included in the - serialized JSON objected, and it must also be present when - deserializing. - - """ - __slots__ = ('json_name', 'default', 'omitempty', 'fdec', 'fenc') - - def __init__(self, json_name, default=None, omitempty=False, - decoder=None, encoder=None): - # pylint: disable=too-many-arguments - self.json_name = json_name - self.default = default - self.omitempty = omitempty - - self.fdec = self.default_decoder if decoder is None else decoder - self.fenc = self.default_encoder if encoder is None else encoder - - @classmethod - def _empty(cls, value): - """Is the provided value considered "empty" for this field? - - This is useful for subclasses that might want to override the - definition of being empty, e.g. for some more exotic data types. - - """ - return not isinstance(value, bool) and not value - - def omit(self, value): - """Omit the value in output?""" - return self._empty(value) and self.omitempty - - def _update_params(self, **kwargs): - current = dict(json_name=self.json_name, default=self.default, - omitempty=self.omitempty, - decoder=self.fdec, encoder=self.fenc) - current.update(kwargs) - return type(self)(**current) # pylint: disable=star-args - - def decoder(self, fdec): - """Descriptor to change the decoder on JSON object field.""" - return self._update_params(decoder=fdec) - - def encoder(self, fenc): - """Descriptor to change the encoder on JSON object field.""" - return self._update_params(encoder=fenc) - - def decode(self, value): - """Decode a value, optionally with context JSON object.""" - return self.fdec(value) - - def encode(self, value): - """Encode a value, optionally with context JSON object.""" - return self.fenc(value) - - @classmethod - def default_decoder(cls, value): - """Default decoder. - - Recursively deserialize into immutable types ( - :class:`acme.jose.util.frozendict` instead of - :func:`dict`, :func:`tuple` instead of :func:`list`). - - """ - # bases cases for different types returned by json.loads - if isinstance(value, list): - return tuple(cls.default_decoder(subvalue) for subvalue in value) - elif isinstance(value, dict): - return util.frozendict( - dict((cls.default_decoder(key), cls.default_decoder(value)) - for key, value in six.iteritems(value))) - else: # integer or string - return value - - @classmethod - def default_encoder(cls, value): - """Default (passthrough) encoder.""" - # field.to_partial_json() is no good as encoder has to do partial - # serialization only - return value - - -class JSONObjectWithFieldsMeta(abc.ABCMeta): - """Metaclass for :class:`JSONObjectWithFields` and its subclasses. - - It makes sure that, for any class ``cls`` with ``__metaclass__`` - set to ``JSONObjectWithFieldsMeta``: - - 1. All fields (attributes of type :class:`Field`) in the class - definition are moved to the ``cls._fields`` dictionary, where - keys are field attribute names and values are fields themselves. - - 2. ``cls.__slots__`` is extended by all field attribute names - (i.e. not :attr:`Field.json_name`). Original ``cls.__slots__`` - are stored in ``cls._orig_slots``. - - In a consequence, for a field attribute name ``some_field``, - ``cls.some_field`` will be a slot descriptor and not an instance - of :class:`Field`. For example:: - - some_field = Field('someField', default=()) - - class Foo(object): - __metaclass__ = JSONObjectWithFieldsMeta - __slots__ = ('baz',) - some_field = some_field - - assert Foo.__slots__ == ('some_field', 'baz') - assert Foo._orig_slots == () - assert Foo.some_field is not Field - - assert Foo._fields.keys() == ['some_field'] - assert Foo._fields['some_field'] is some_field - - As an implementation note, this metaclass inherits from - :class:`abc.ABCMeta` (and not the usual :class:`type`) to mitigate - the metaclass conflict (:class:`ImmutableMap` and - :class:`JSONDeSerializable`, parents of :class:`JSONObjectWithFields`, - use :class:`abc.ABCMeta` as its metaclass). - - """ - - def __new__(mcs, name, bases, dikt): - fields = {} - - for base in bases: - fields.update(getattr(base, '_fields', {})) - # Do not reorder, this class might override fields from base classes! - for key, value in tuple(six.iteritems(dikt)): - # not six.iterkeys() (in-place edit!) - if isinstance(value, Field): - fields[key] = dikt.pop(key) - - dikt['_orig_slots'] = dikt.get('__slots__', ()) - dikt['__slots__'] = tuple( - list(dikt['_orig_slots']) + list(six.iterkeys(fields))) - dikt['_fields'] = fields - - return abc.ABCMeta.__new__(mcs, name, bases, dikt) - - -@six.add_metaclass(JSONObjectWithFieldsMeta) -class JSONObjectWithFields(util.ImmutableMap, interfaces.JSONDeSerializable): - # pylint: disable=too-few-public-methods - """JSON object with fields. - - Example:: - - class Foo(JSONObjectWithFields): - bar = Field('Bar') - empty = Field('Empty', omitempty=True) - - @bar.encoder - def bar(value): - return value + 'bar' - - @bar.decoder - def bar(value): - if not value.endswith('bar'): - raise errors.DeserializationError('No bar suffix!') - return value[:-3] - - assert Foo(bar='baz').to_partial_json() == {'Bar': 'bazbar'} - assert Foo.from_json({'Bar': 'bazbar'}) == Foo(bar='baz') - assert (Foo.from_json({'Bar': 'bazbar', 'Empty': '!'}) - == Foo(bar='baz', empty='!')) - assert Foo(bar='baz').bar == 'baz' - - """ - - @classmethod - def _defaults(cls): - """Get default fields values.""" - return dict([(slot, field.default) for slot, field - in six.iteritems(cls._fields)]) - - def __init__(self, **kwargs): - # pylint: disable=star-args - super(JSONObjectWithFields, self).__init__( - **(dict(self._defaults(), **kwargs))) - - def encode(self, name): - """Encode a single field. - - :param str name: Name of the field to be encoded. - - :raises errors.SerializationError: if field cannot be serialized - :raises errors.Error: if field could not be found - - """ - try: - field = self._fields[name] - except KeyError: - raise errors.Error("Field not found: {0}".format(name)) - - return field.encode(getattr(self, name)) - - def fields_to_partial_json(self): - """Serialize fields to JSON.""" - jobj = {} - omitted = set() - for slot, field in six.iteritems(self._fields): - value = getattr(self, slot) - - if field.omit(value): - omitted.add((slot, value)) - else: - try: - jobj[field.json_name] = field.encode(value) - except errors.SerializationError as error: - raise errors.SerializationError( - 'Could not encode {0} ({1}): {2}'.format( - slot, value, error)) - return jobj - - def to_partial_json(self): - return self.fields_to_partial_json() - - @classmethod - def _check_required(cls, jobj): - missing = set() - for _, field in six.iteritems(cls._fields): - if not field.omitempty and field.json_name not in jobj: - missing.add(field.json_name) - - if missing: - raise errors.DeserializationError( - 'The following fields are required: {0}'.format( - ','.join(missing))) - - @classmethod - def fields_from_json(cls, jobj): - """Deserialize fields from JSON.""" - cls._check_required(jobj) - fields = {} - for slot, field in six.iteritems(cls._fields): - if field.json_name not in jobj and field.omitempty: - fields[slot] = field.default - else: - value = jobj[field.json_name] - try: - fields[slot] = field.decode(value) - except errors.DeserializationError as error: - raise errors.DeserializationError( - 'Could not decode {0!r} ({1!r}): {2}'.format( - slot, value, error)) - return fields - - @classmethod - def from_json(cls, jobj): - return cls(**cls.fields_from_json(jobj)) - - -def encode_b64jose(data): - """Encode JOSE Base-64 field. - - :param bytes data: - :rtype: `unicode` - - """ - # b64encode produces ASCII characters only - return b64.b64encode(data).decode('ascii') - - -def decode_b64jose(data, size=None, minimum=False): - """Decode JOSE Base-64 field. - - :param unicode data: - :param int size: Required length (after decoding). - :param bool minimum: If ``True``, then `size` will be treated as - minimum required length, as opposed to exact equality. - - :rtype: bytes - - """ - error_cls = TypeError if six.PY2 else binascii.Error - try: - decoded = b64.b64decode(data.encode()) - except error_cls as error: - raise errors.DeserializationError(error) - - if size is not None and ((not minimum and len(decoded) != size) or - (minimum and len(decoded) < size)): - raise errors.DeserializationError( - "Expected at least or exactly {0} bytes".format(size)) - - return decoded - - -def encode_hex16(value): - """Hexlify. - - :param bytes value: - :rtype: unicode - - """ - return binascii.hexlify(value).decode() - - -def decode_hex16(value, size=None, minimum=False): - """Decode hexlified field. - - :param unicode value: - :param int size: Required length (after decoding). - :param bool minimum: If ``True``, then `size` will be treated as - minimum required length, as opposed to exact equality. - - :rtype: bytes - - """ - value = value.encode() - if size is not None and ((not minimum and len(value) != size * 2) or - (minimum and len(value) < size * 2)): - raise errors.DeserializationError() - error_cls = TypeError if six.PY2 else binascii.Error - try: - return binascii.unhexlify(value) - except error_cls as error: - raise errors.DeserializationError(error) - - -def encode_cert(cert): - """Encode certificate as JOSE Base-64 DER. - - :type cert: `OpenSSL.crypto.X509` wrapped in `.ComparableX509` - :rtype: unicode - - """ - return encode_b64jose(OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_ASN1, cert.wrapped)) - - -def decode_cert(b64der): - """Decode JOSE Base-64 DER-encoded certificate. - - :param unicode b64der: - :rtype: `OpenSSL.crypto.X509` wrapped in `.ComparableX509` - - """ - try: - return util.ComparableX509(OpenSSL.crypto.load_certificate( - OpenSSL.crypto.FILETYPE_ASN1, decode_b64jose(b64der))) - except OpenSSL.crypto.Error as error: - raise errors.DeserializationError(error) - - -def encode_csr(csr): - """Encode CSR as JOSE Base-64 DER. - - :type csr: `OpenSSL.crypto.X509Req` wrapped in `.ComparableX509` - :rtype: unicode - - """ - return encode_b64jose(OpenSSL.crypto.dump_certificate_request( - OpenSSL.crypto.FILETYPE_ASN1, csr.wrapped)) - - -def decode_csr(b64der): - """Decode JOSE Base-64 DER-encoded CSR. - - :param unicode b64der: - :rtype: `OpenSSL.crypto.X509Req` wrapped in `.ComparableX509` - - """ - try: - return util.ComparableX509(OpenSSL.crypto.load_certificate_request( - OpenSSL.crypto.FILETYPE_ASN1, decode_b64jose(b64der))) - except OpenSSL.crypto.Error as error: - raise errors.DeserializationError(error) - - -class TypedJSONObjectWithFields(JSONObjectWithFields): - """JSON object with type.""" - - typ = NotImplemented - """Type of the object. Subclasses must override.""" - - type_field_name = "type" - """Field name used to distinguish different object types. - - Subclasses will probably have to override this. - - """ - - TYPES = NotImplemented - """Types registered for JSON deserialization""" - - @classmethod - def register(cls, type_cls, typ=None): - """Register class for JSON deserialization.""" - typ = type_cls.typ if typ is None else typ - cls.TYPES[typ] = type_cls - return type_cls - - @classmethod - def get_type_cls(cls, jobj): - """Get the registered class for ``jobj``.""" - if cls in six.itervalues(cls.TYPES): - if cls.type_field_name not in jobj: - raise errors.DeserializationError( - "Missing type field ({0})".format(cls.type_field_name)) - # cls is already registered type_cls, force to use it - # so that, e.g Revocation.from_json(jobj) fails if - # jobj["type"] != "revocation". - return cls - - if not isinstance(jobj, dict): - raise errors.DeserializationError( - "{0} is not a dictionary object".format(jobj)) - try: - typ = jobj[cls.type_field_name] - except KeyError: - raise errors.DeserializationError("missing type field") - - try: - return cls.TYPES[typ] - except KeyError: - raise errors.UnrecognizedTypeError(typ, jobj) - - def to_partial_json(self): - """Get JSON serializable object. - - :returns: Serializable JSON object representing ACME typed object. - :meth:`validate` will almost certainly not work, due to reasons - explained in :class:`acme.interfaces.IJSONSerializable`. - :rtype: dict - - """ - jobj = self.fields_to_partial_json() - jobj[self.type_field_name] = self.typ - return jobj - - @classmethod - def from_json(cls, jobj): - """Deserialize ACME object from valid JSON object. - - :raises acme.errors.UnrecognizedTypeError: if type - of the ACME object has not been registered. - - """ - # make sure subclasses don't cause infinite recursive from_json calls - type_cls = cls.get_type_cls(jobj) - return type_cls(**type_cls.fields_from_json(jobj)) diff --git a/acme/acme/jose/json_util_test.py b/acme/acme/jose/json_util_test.py deleted file mode 100644 index 25e36211eec..00000000000 --- a/acme/acme/jose/json_util_test.py +++ /dev/null @@ -1,381 +0,0 @@ -"""Tests for acme.jose.json_util.""" -import itertools -import unittest - -import mock -import six - -from acme import test_util - -from acme.jose import errors -from acme.jose import interfaces -from acme.jose import util - - -CERT = test_util.load_comparable_cert('cert.pem') -CSR = test_util.load_comparable_csr('csr.pem') - - -class FieldTest(unittest.TestCase): - """Tests for acme.jose.json_util.Field.""" - - def test_no_omit_boolean(self): - from acme.jose.json_util import Field - for default, omitempty, value in itertools.product( - [True, False], [True, False], [True, False]): - self.assertFalse( - Field("foo", default=default, omitempty=omitempty).omit(value)) - - def test_descriptors(self): - mock_value = mock.MagicMock() - - # pylint: disable=missing-docstring - - def decoder(unused_value): - return 'd' - - def encoder(unused_value): - return 'e' - - from acme.jose.json_util import Field - field = Field('foo') - - field = field.encoder(encoder) - self.assertEqual('e', field.encode(mock_value)) - - field = field.decoder(decoder) - self.assertEqual('e', field.encode(mock_value)) - self.assertEqual('d', field.decode(mock_value)) - - def test_default_encoder_is_partial(self): - class MockField(interfaces.JSONDeSerializable): - # pylint: disable=missing-docstring - def to_partial_json(self): - return 'foo' # pragma: no cover - - @classmethod - def from_json(cls, jobj): - pass # pragma: no cover - mock_field = MockField() - - from acme.jose.json_util import Field - self.assertTrue(Field.default_encoder(mock_field) is mock_field) - # in particular... - self.assertNotEqual('foo', Field.default_encoder(mock_field)) - - def test_default_encoder_passthrough(self): - mock_value = mock.MagicMock() - from acme.jose.json_util import Field - self.assertTrue(Field.default_encoder(mock_value) is mock_value) - - def test_default_decoder_list_to_tuple(self): - from acme.jose.json_util import Field - self.assertEqual((1, 2, 3), Field.default_decoder([1, 2, 3])) - - def test_default_decoder_dict_to_frozendict(self): - from acme.jose.json_util import Field - obj = Field.default_decoder({'x': 2}) - self.assertTrue(isinstance(obj, util.frozendict)) - self.assertEqual(obj, util.frozendict(x=2)) - - def test_default_decoder_passthrough(self): - mock_value = mock.MagicMock() - from acme.jose.json_util import Field - self.assertTrue(Field.default_decoder(mock_value) is mock_value) - - -class JSONObjectWithFieldsMetaTest(unittest.TestCase): - """Tests for acme.jose.json_util.JSONObjectWithFieldsMeta.""" - - def setUp(self): - from acme.jose.json_util import Field - from acme.jose.json_util import JSONObjectWithFieldsMeta - self.field = Field('Baz') - self.field2 = Field('Baz2') - # pylint: disable=invalid-name,missing-docstring,too-few-public-methods - # pylint: disable=blacklisted-name - - @six.add_metaclass(JSONObjectWithFieldsMeta) - class A(object): - __slots__ = ('bar',) - baz = self.field - - class B(A): - pass - - class C(A): - baz = self.field2 - - self.a_cls = A - self.b_cls = B - self.c_cls = C - - def test_fields(self): - # pylint: disable=protected-access,no-member - self.assertEqual({'baz': self.field}, self.a_cls._fields) - self.assertEqual({'baz': self.field}, self.b_cls._fields) - - def test_fields_inheritance(self): - # pylint: disable=protected-access,no-member - self.assertEqual({'baz': self.field2}, self.c_cls._fields) - - def test_slots(self): - self.assertEqual(('bar', 'baz'), self.a_cls.__slots__) - self.assertEqual(('baz',), self.b_cls.__slots__) - - def test_orig_slots(self): - # pylint: disable=protected-access,no-member - self.assertEqual(('bar',), self.a_cls._orig_slots) - self.assertEqual((), self.b_cls._orig_slots) - - -class JSONObjectWithFieldsTest(unittest.TestCase): - """Tests for acme.jose.json_util.JSONObjectWithFields.""" - # pylint: disable=protected-access - - def setUp(self): - from acme.jose.json_util import JSONObjectWithFields - from acme.jose.json_util import Field - - class MockJSONObjectWithFields(JSONObjectWithFields): - # pylint: disable=invalid-name,missing-docstring,no-self-argument - # pylint: disable=too-few-public-methods - x = Field('x', omitempty=True, - encoder=(lambda x: x * 2), - decoder=(lambda x: x / 2)) - y = Field('y') - z = Field('Z') # on purpose uppercase - - @y.encoder - def y(value): - if value == 500: - raise errors.SerializationError() - return value - - @y.decoder - def y(value): - if value == 500: - raise errors.DeserializationError() - return value - - # pylint: disable=invalid-name - self.MockJSONObjectWithFields = MockJSONObjectWithFields - self.mock = MockJSONObjectWithFields(x=None, y=2, z=3) - - def test_init_defaults(self): - self.assertEqual(self.mock, self.MockJSONObjectWithFields(y=2, z=3)) - - def test_encode(self): - self.assertEqual(10, self.MockJSONObjectWithFields( - x=5, y=0, z=0).encode("x")) - - def test_encode_wrong_field(self): - self.assertRaises(errors.Error, self.mock.encode, 'foo') - - def test_encode_serialization_error_passthrough(self): - self.assertRaises( - errors.SerializationError, - self.MockJSONObjectWithFields(y=500, z=None).encode, "y") - - def test_fields_to_partial_json_omits_empty(self): - self.assertEqual(self.mock.fields_to_partial_json(), {'y': 2, 'Z': 3}) - - def test_fields_from_json_fills_default_for_empty(self): - self.assertEqual( - {'x': None, 'y': 2, 'z': 3}, - self.MockJSONObjectWithFields.fields_from_json({'y': 2, 'Z': 3})) - - def test_fields_from_json_fails_on_missing(self): - self.assertRaises( - errors.DeserializationError, - self.MockJSONObjectWithFields.fields_from_json, {'y': 0}) - self.assertRaises( - errors.DeserializationError, - self.MockJSONObjectWithFields.fields_from_json, {'Z': 0}) - self.assertRaises( - errors.DeserializationError, - self.MockJSONObjectWithFields.fields_from_json, {'x': 0, 'y': 0}) - self.assertRaises( - errors.DeserializationError, - self.MockJSONObjectWithFields.fields_from_json, {'x': 0, 'Z': 0}) - - def test_fields_to_partial_json_encoder(self): - self.assertEqual( - self.MockJSONObjectWithFields(x=1, y=2, z=3).to_partial_json(), - {'x': 2, 'y': 2, 'Z': 3}) - - def test_fields_from_json_decoder(self): - self.assertEqual( - {'x': 2, 'y': 2, 'z': 3}, - self.MockJSONObjectWithFields.fields_from_json( - {'x': 4, 'y': 2, 'Z': 3})) - - def test_fields_to_partial_json_error_passthrough(self): - self.assertRaises( - errors.SerializationError, self.MockJSONObjectWithFields( - x=1, y=500, z=3).to_partial_json) - - def test_fields_from_json_error_passthrough(self): - self.assertRaises( - errors.DeserializationError, - self.MockJSONObjectWithFields.from_json, - {'x': 4, 'y': 500, 'Z': 3}) - - -class DeEncodersTest(unittest.TestCase): - def setUp(self): - self.b64_cert = ( - u'MIIB3jCCAYigAwIBAgICBTkwDQYJKoZIhvcNAQELBQAwdzELMAkGA1UEBhM' - u'CVVMxETAPBgNVBAgMCE1pY2hpZ2FuMRIwEAYDVQQHDAlBbm4gQXJib3IxKz' - u'ApBgNVBAoMIlVuaXZlcnNpdHkgb2YgTWljaGlnYW4gYW5kIHRoZSBFRkYxF' - u'DASBgNVBAMMC2V4YW1wbGUuY29tMB4XDTE0MTIxMTIyMzQ0NVoXDTE0MTIx' - u'ODIyMzQ0NVowdzELMAkGA1UEBhMCVVMxETAPBgNVBAgMCE1pY2hpZ2FuMRI' - u'wEAYDVQQHDAlBbm4gQXJib3IxKzApBgNVBAoMIlVuaXZlcnNpdHkgb2YgTW' - u'ljaGlnYW4gYW5kIHRoZSBFRkYxFDASBgNVBAMMC2V4YW1wbGUuY29tMFwwD' - u'QYJKoZIhvcNAQEBBQADSwAwSAJBAKx1c7RR7R_drnBSQ_zfx1vQLHUbFLh1' - u'AQQQ5R8DZUXd36efNK79vukFhN9HFoHZiUvOjm0c-pVE6K-EdE_twuUCAwE' - u'AATANBgkqhkiG9w0BAQsFAANBAC24z0IdwIVKSlntksllvr6zJepBH5fMnd' - u'fk3XJp10jT6VE-14KNtjh02a56GoraAvJAT5_H67E8GvJ_ocNnB_o' - ) - self.b64_csr = ( - u'MIIBXTCCAQcCAQAweTELMAkGA1UEBhMCVVMxETAPBgNVBAgMCE1pY2hpZ2F' - u'uMRIwEAYDVQQHDAlBbm4gQXJib3IxDDAKBgNVBAoMA0VGRjEfMB0GA1UECw' - u'wWVW5pdmVyc2l0eSBvZiBNaWNoaWdhbjEUMBIGA1UEAwwLZXhhbXBsZS5jb' - u'20wXDANBgkqhkiG9w0BAQEFAANLADBIAkEArHVztFHtH92ucFJD_N_HW9As' - u'dRsUuHUBBBDlHwNlRd3fp580rv2-6QWE30cWgdmJS86ObRz6lUTor4R0T-3' - u'C5QIDAQABoCkwJwYJKoZIhvcNAQkOMRowGDAWBgNVHREEDzANggtleGFtcG' - u'xlLmNvbTANBgkqhkiG9w0BAQsFAANBAHJH_O6BtC9aGzEVCMGOZ7z9iIRHW' - u'Szr9x_bOzn7hLwsbXPAgO1QxEwL-X-4g20Gn9XBE1N9W6HCIEut2d8wACg' - ) - - def test_encode_b64jose(self): - from acme.jose.json_util import encode_b64jose - encoded = encode_b64jose(b'x') - self.assertTrue(isinstance(encoded, six.string_types)) - self.assertEqual(u'eA', encoded) - - def test_decode_b64jose(self): - from acme.jose.json_util import decode_b64jose - decoded = decode_b64jose(u'eA') - self.assertTrue(isinstance(decoded, six.binary_type)) - self.assertEqual(b'x', decoded) - - def test_decode_b64jose_padding_error(self): - from acme.jose.json_util import decode_b64jose - self.assertRaises(errors.DeserializationError, decode_b64jose, u'x') - - def test_decode_b64jose_size(self): - from acme.jose.json_util import decode_b64jose - self.assertEqual(b'foo', decode_b64jose(u'Zm9v', size=3)) - self.assertRaises( - errors.DeserializationError, decode_b64jose, u'Zm9v', size=2) - self.assertRaises( - errors.DeserializationError, decode_b64jose, u'Zm9v', size=4) - - def test_decode_b64jose_minimum_size(self): - from acme.jose.json_util import decode_b64jose - self.assertEqual(b'foo', decode_b64jose(u'Zm9v', size=3, minimum=True)) - self.assertEqual(b'foo', decode_b64jose(u'Zm9v', size=2, minimum=True)) - self.assertRaises(errors.DeserializationError, decode_b64jose, - u'Zm9v', size=4, minimum=True) - - def test_encode_hex16(self): - from acme.jose.json_util import encode_hex16 - encoded = encode_hex16(b'foo') - self.assertEqual(u'666f6f', encoded) - self.assertTrue(isinstance(encoded, six.string_types)) - - def test_decode_hex16(self): - from acme.jose.json_util import decode_hex16 - decoded = decode_hex16(u'666f6f') - self.assertEqual(b'foo', decoded) - self.assertTrue(isinstance(decoded, six.binary_type)) - - def test_decode_hex16_minimum_size(self): - from acme.jose.json_util import decode_hex16 - self.assertEqual(b'foo', decode_hex16(u'666f6f', size=3, minimum=True)) - self.assertEqual(b'foo', decode_hex16(u'666f6f', size=2, minimum=True)) - self.assertRaises(errors.DeserializationError, decode_hex16, - u'666f6f', size=4, minimum=True) - - def test_decode_hex16_odd_length(self): - from acme.jose.json_util import decode_hex16 - self.assertRaises(errors.DeserializationError, decode_hex16, u'x') - - def test_encode_cert(self): - from acme.jose.json_util import encode_cert - self.assertEqual(self.b64_cert, encode_cert(CERT)) - - def test_decode_cert(self): - from acme.jose.json_util import decode_cert - cert = decode_cert(self.b64_cert) - self.assertTrue(isinstance(cert, util.ComparableX509)) - self.assertEqual(cert, CERT) - self.assertRaises(errors.DeserializationError, decode_cert, u'') - - def test_encode_csr(self): - from acme.jose.json_util import encode_csr - self.assertEqual(self.b64_csr, encode_csr(CSR)) - - def test_decode_csr(self): - from acme.jose.json_util import decode_csr - csr = decode_csr(self.b64_csr) - self.assertTrue(isinstance(csr, util.ComparableX509)) - self.assertEqual(csr, CSR) - self.assertRaises(errors.DeserializationError, decode_csr, u'') - - -class TypedJSONObjectWithFieldsTest(unittest.TestCase): - - def setUp(self): - from acme.jose.json_util import TypedJSONObjectWithFields - - # pylint: disable=missing-docstring,abstract-method - # pylint: disable=too-few-public-methods - - class MockParentTypedJSONObjectWithFields(TypedJSONObjectWithFields): - TYPES = {} - type_field_name = 'type' - - @MockParentTypedJSONObjectWithFields.register - class MockTypedJSONObjectWithFields( - MockParentTypedJSONObjectWithFields): - typ = 'test' - __slots__ = ('foo',) - - @classmethod - def fields_from_json(cls, jobj): - return {'foo': jobj['foo']} - - def fields_to_partial_json(self): - return {'foo': self.foo} - - self.parent_cls = MockParentTypedJSONObjectWithFields - self.msg = MockTypedJSONObjectWithFields(foo='bar') - - def test_to_partial_json(self): - self.assertEqual(self.msg.to_partial_json(), { - 'type': 'test', - 'foo': 'bar', - }) - - def test_from_json_non_dict_fails(self): - for value in [[], (), 5, "asd"]: # all possible input types - self.assertRaises( - errors.DeserializationError, self.parent_cls.from_json, value) - - def test_from_json_dict_no_type_fails(self): - self.assertRaises( - errors.DeserializationError, self.parent_cls.from_json, {}) - - def test_from_json_unknown_type_fails(self): - self.assertRaises(errors.UnrecognizedTypeError, - self.parent_cls.from_json, {'type': 'bar'}) - - def test_from_json_returns_obj(self): - self.assertEqual({'foo': 'bar'}, self.parent_cls.from_json( - {'type': 'test', 'foo': 'bar'})) - - -if __name__ == '__main__': - unittest.main() # pragma: no cover diff --git a/acme/acme/jose/jwa.py b/acme/acme/jose/jwa.py deleted file mode 100644 index 9b682ecabbe..00000000000 --- a/acme/acme/jose/jwa.py +++ /dev/null @@ -1,180 +0,0 @@ -"""JSON Web Algorithm. - -https://tools.ietf.org/html/draft-ietf-jose-json-web-algorithms-40 - -""" -import abc -import collections -import logging - -import cryptography.exceptions -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import hashes # type: ignore -from cryptography.hazmat.primitives import hmac # type: ignore -from cryptography.hazmat.primitives.asymmetric import padding # type: ignore - -from acme.jose import errors -from acme.jose import interfaces -from acme.jose import jwk - - -logger = logging.getLogger(__name__) - - -class JWA(interfaces.JSONDeSerializable): # pylint: disable=abstract-method - # pylint: disable=too-few-public-methods - # for some reason disable=abstract-method has to be on the line - # above... - """JSON Web Algorithm.""" - - -class JWASignature(JWA, collections.Hashable): # type: ignore - """JSON Web Signature Algorithm.""" - SIGNATURES = {} # type: dict - - def __init__(self, name): - self.name = name - - def __eq__(self, other): - if not isinstance(other, JWASignature): - return NotImplemented - return self.name == other.name - - def __hash__(self): - return hash((self.__class__, self.name)) - - def __ne__(self, other): - return not self == other - - @classmethod - def register(cls, signature_cls): - """Register class for JSON deserialization.""" - cls.SIGNATURES[signature_cls.name] = signature_cls - return signature_cls - - def to_partial_json(self): - return self.name - - @classmethod - def from_json(cls, jobj): - return cls.SIGNATURES[jobj] - - @abc.abstractmethod - def sign(self, key, msg): # pragma: no cover - """Sign the ``msg`` using ``key``.""" - raise NotImplementedError() - - @abc.abstractmethod - def verify(self, key, msg, sig): # pragma: no cover - """Verify the ``msg` and ``sig`` using ``key``.""" - raise NotImplementedError() - - def __repr__(self): - return self.name - - -class _JWAHS(JWASignature): - - kty = jwk.JWKOct - - def __init__(self, name, hash_): - super(_JWAHS, self).__init__(name) - self.hash = hash_() - - def sign(self, key, msg): - signer = hmac.HMAC(key, self.hash, backend=default_backend()) - signer.update(msg) - return signer.finalize() - - def verify(self, key, msg, sig): - verifier = hmac.HMAC(key, self.hash, backend=default_backend()) - verifier.update(msg) - try: - verifier.verify(sig) - except cryptography.exceptions.InvalidSignature as error: - logger.debug(error, exc_info=True) - return False - else: - return True - - -class _JWARSA(object): - - kty = jwk.JWKRSA - padding = NotImplemented - hash = NotImplemented - - def sign(self, key, msg): - """Sign the ``msg`` using ``key``.""" - try: - signer = key.signer(self.padding, self.hash) - except AttributeError as error: - logger.debug(error, exc_info=True) - raise errors.Error("Public key cannot be used for signing") - except ValueError as error: # digest too large - logger.debug(error, exc_info=True) - raise errors.Error(str(error)) - signer.update(msg) - try: - return signer.finalize() - except ValueError as error: - logger.debug(error, exc_info=True) - raise errors.Error(str(error)) - - def verify(self, key, msg, sig): - """Verify the ``msg` and ``sig`` using ``key``.""" - verifier = key.verifier(sig, self.padding, self.hash) - verifier.update(msg) - try: - verifier.verify() - except cryptography.exceptions.InvalidSignature as error: - logger.debug(error, exc_info=True) - return False - else: - return True - - -class _JWARS(_JWARSA, JWASignature): - - def __init__(self, name, hash_): - super(_JWARS, self).__init__(name) - self.padding = padding.PKCS1v15() - self.hash = hash_() - - -class _JWAPS(_JWARSA, JWASignature): - - def __init__(self, name, hash_): - super(_JWAPS, self).__init__(name) - self.padding = padding.PSS( - mgf=padding.MGF1(hash_()), - salt_length=padding.PSS.MAX_LENGTH) - self.hash = hash_() - - -class _JWAES(JWASignature): # pylint: disable=abstract-class-not-used - - # TODO: implement ES signatures - - def sign(self, key, msg): # pragma: no cover - raise NotImplementedError() - - def verify(self, key, msg, sig): # pragma: no cover - raise NotImplementedError() - - -HS256 = JWASignature.register(_JWAHS('HS256', hashes.SHA256)) -HS384 = JWASignature.register(_JWAHS('HS384', hashes.SHA384)) -HS512 = JWASignature.register(_JWAHS('HS512', hashes.SHA512)) - -RS256 = JWASignature.register(_JWARS('RS256', hashes.SHA256)) -RS384 = JWASignature.register(_JWARS('RS384', hashes.SHA384)) -RS512 = JWASignature.register(_JWARS('RS512', hashes.SHA512)) - -PS256 = JWASignature.register(_JWAPS('PS256', hashes.SHA256)) -PS384 = JWASignature.register(_JWAPS('PS384', hashes.SHA384)) -PS512 = JWASignature.register(_JWAPS('PS512', hashes.SHA512)) - -ES256 = JWASignature.register(_JWAES('ES256')) -ES384 = JWASignature.register(_JWAES('ES384')) -ES512 = JWASignature.register(_JWAES('ES512')) diff --git a/acme/acme/jose/jwa_test.py b/acme/acme/jose/jwa_test.py deleted file mode 100644 index 3328d083ab9..00000000000 --- a/acme/acme/jose/jwa_test.py +++ /dev/null @@ -1,104 +0,0 @@ -"""Tests for acme.jose.jwa.""" -import unittest - -from acme import test_util - -from acme.jose import errors - - -RSA256_KEY = test_util.load_rsa_private_key('rsa256_key.pem') -RSA512_KEY = test_util.load_rsa_private_key('rsa512_key.pem') -RSA1024_KEY = test_util.load_rsa_private_key('rsa1024_key.pem') - - -class JWASignatureTest(unittest.TestCase): - """Tests for acme.jose.jwa.JWASignature.""" - - def setUp(self): - from acme.jose.jwa import JWASignature - - class MockSig(JWASignature): - # pylint: disable=missing-docstring,too-few-public-methods - # pylint: disable=abstract-class-not-used - def sign(self, key, msg): - raise NotImplementedError() # pragma: no cover - - def verify(self, key, msg, sig): - raise NotImplementedError() # pragma: no cover - - # pylint: disable=invalid-name - self.Sig1 = MockSig('Sig1') - self.Sig2 = MockSig('Sig2') - - def test_eq(self): - self.assertEqual(self.Sig1, self.Sig1) - - def test_ne(self): - self.assertNotEqual(self.Sig1, self.Sig2) - - def test_ne_other_type(self): - self.assertNotEqual(self.Sig1, 5) - - def test_repr(self): - self.assertEqual('Sig1', repr(self.Sig1)) - self.assertEqual('Sig2', repr(self.Sig2)) - - def test_to_partial_json(self): - self.assertEqual(self.Sig1.to_partial_json(), 'Sig1') - self.assertEqual(self.Sig2.to_partial_json(), 'Sig2') - - def test_from_json(self): - from acme.jose.jwa import JWASignature - from acme.jose.jwa import RS256 - self.assertTrue(JWASignature.from_json('RS256') is RS256) - - -class JWAHSTest(unittest.TestCase): # pylint: disable=too-few-public-methods - - def test_it(self): - from acme.jose.jwa import HS256 - sig = ( - b"\xceR\xea\xcd\x94\xab\xcf\xfb\xe0\xacA.:\x1a'\x08i\xe2\xc4" - b"\r\x85+\x0e\x85\xaeUZ\xd4\xb3\x97zO" - ) - self.assertEqual(HS256.sign(b'some key', b'foo'), sig) - self.assertTrue(HS256.verify(b'some key', b'foo', sig) is True) - self.assertTrue(HS256.verify(b'some key', b'foo', sig + b'!') is False) - - -class JWARSTest(unittest.TestCase): - - def test_sign_no_private_part(self): - from acme.jose.jwa import RS256 - self.assertRaises( - errors.Error, RS256.sign, RSA512_KEY.public_key(), b'foo') - - def test_sign_key_too_small(self): - from acme.jose.jwa import RS256 - from acme.jose.jwa import PS256 - self.assertRaises(errors.Error, RS256.sign, RSA256_KEY, b'foo') - self.assertRaises(errors.Error, PS256.sign, RSA256_KEY, b'foo') - - def test_rs(self): - from acme.jose.jwa import RS256 - sig = ( - b'|\xc6\xb2\xa4\xab(\x87\x99\xfa*:\xea\xf8\xa0N&}\x9f\x0f\xc0O' - b'\xc6t\xa3\xe6\xfa\xbb"\x15Y\x80Y\xe0\x81\xb8\x88)\xba\x0c\x9c' - b'\xa4\x99\x1e\x19&\xd8\xc7\x99S\x97\xfc\x85\x0cOV\xe6\x07\x99' - b'\xd2\xb9.>}\xfd' - ) - self.assertEqual(RS256.sign(RSA512_KEY, b'foo'), sig) - self.assertTrue(RS256.verify(RSA512_KEY.public_key(), b'foo', sig)) - self.assertFalse(RS256.verify( - RSA512_KEY.public_key(), b'foo', sig + b'!')) - - def test_ps(self): - from acme.jose.jwa import PS256 - sig = PS256.sign(RSA1024_KEY, b'foo') - self.assertTrue(PS256.verify(RSA1024_KEY.public_key(), b'foo', sig)) - self.assertFalse(PS256.verify( - RSA1024_KEY.public_key(), b'foo', sig + b'!')) - - -if __name__ == '__main__': - unittest.main() # pragma: no cover diff --git a/acme/acme/jose/jwk.py b/acme/acme/jose/jwk.py deleted file mode 100644 index 54423f6709f..00000000000 --- a/acme/acme/jose/jwk.py +++ /dev/null @@ -1,281 +0,0 @@ -"""JSON Web Key.""" -import abc -import binascii -import json -import logging - -import cryptography.exceptions -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import hashes # type: ignore -from cryptography.hazmat.primitives import serialization -from cryptography.hazmat.primitives.asymmetric import ec # type: ignore -from cryptography.hazmat.primitives.asymmetric import rsa - -import six - -from acme.jose import errors -from acme.jose import json_util -from acme.jose import util - - -logger = logging.getLogger(__name__) - - -class JWK(json_util.TypedJSONObjectWithFields): - # pylint: disable=too-few-public-methods - """JSON Web Key.""" - type_field_name = 'kty' - TYPES = {} # type: dict - cryptography_key_types = () # type: tuple - """Subclasses should override.""" - - required = NotImplemented - """Required members of public key's representation as defined by JWK/JWA.""" - - _thumbprint_json_dumps_params = { - # "no whitespace or line breaks before or after any syntactic - # elements" - 'indent': None, - 'separators': (',', ':'), - # "members ordered lexicographically by the Unicode [UNICODE] - # code points of the member names" - 'sort_keys': True, - } - - def thumbprint(self, hash_function=hashes.SHA256): - """Compute JWK Thumbprint. - - https://tools.ietf.org/html/rfc7638 - - :returns bytes: - - """ - digest = hashes.Hash(hash_function(), backend=default_backend()) - digest.update(json.dumps( - dict((k, v) for k, v in six.iteritems(self.to_json()) - if k in self.required), - **self._thumbprint_json_dumps_params).encode()) - return digest.finalize() - - @abc.abstractmethod - def public_key(self): # pragma: no cover - """Generate JWK with public key. - - For symmetric cryptosystems, this would return ``self``. - - """ - raise NotImplementedError() - - @classmethod - def _load_cryptography_key(cls, data, password=None, backend=None): - backend = default_backend() if backend is None else backend - exceptions = {} - - # private key? - for loader in (serialization.load_pem_private_key, - serialization.load_der_private_key): - try: - return loader(data, password, backend) - except (ValueError, TypeError, - cryptography.exceptions.UnsupportedAlgorithm) as error: - exceptions[loader] = error - - # public key? - for loader in (serialization.load_pem_public_key, - serialization.load_der_public_key): - try: - return loader(data, backend) - except (ValueError, - cryptography.exceptions.UnsupportedAlgorithm) as error: - exceptions[loader] = error - - # no luck - raise errors.Error('Unable to deserialize key: {0}'.format(exceptions)) - - @classmethod - def load(cls, data, password=None, backend=None): - """Load serialized key as JWK. - - :param str data: Public or private key serialized as PEM or DER. - :param str password: Optional password. - :param backend: A `.PEMSerializationBackend` and - `.DERSerializationBackend` provider. - - :raises errors.Error: if unable to deserialize, or unsupported - JWK algorithm - - :returns: JWK of an appropriate type. - :rtype: `JWK` - - """ - try: - key = cls._load_cryptography_key(data, password, backend) - except errors.Error as error: - logger.debug('Loading symmetric key, asymmetric failed: %s', error) - return JWKOct(key=data) - - if cls.typ is not NotImplemented and not isinstance( - key, cls.cryptography_key_types): - raise errors.Error('Unable to deserialize {0} into {1}'.format( - key.__class__, cls.__class__)) - for jwk_cls in six.itervalues(cls.TYPES): - if isinstance(key, jwk_cls.cryptography_key_types): - return jwk_cls(key=key) - raise errors.Error('Unsupported algorithm: {0}'.format(key.__class__)) - - -@JWK.register -class JWKES(JWK): # pragma: no cover - # pylint: disable=abstract-class-not-used - """ES JWK. - - .. warning:: This is not yet implemented! - - """ - typ = 'ES' - cryptography_key_types = ( - ec.EllipticCurvePublicKey, ec.EllipticCurvePrivateKey) - required = ('crv', JWK.type_field_name, 'x', 'y') - - def fields_to_partial_json(self): - raise NotImplementedError() - - @classmethod - def fields_from_json(cls, jobj): - raise NotImplementedError() - - def public_key(self): - raise NotImplementedError() - - -@JWK.register -class JWKOct(JWK): - """Symmetric JWK.""" - typ = 'oct' - __slots__ = ('key',) - required = ('k', JWK.type_field_name) - - def fields_to_partial_json(self): - # TODO: An "alg" member SHOULD also be present to identify the - # algorithm intended to be used with the key, unless the - # application uses another means or convention to determine - # the algorithm used. - return {'k': json_util.encode_b64jose(self.key)} - - @classmethod - def fields_from_json(cls, jobj): - return cls(key=json_util.decode_b64jose(jobj['k'])) - - def public_key(self): - return self - - -@JWK.register -class JWKRSA(JWK): - """RSA JWK. - - :ivar key: `cryptography.hazmat.primitives.rsa.RSAPrivateKey` - or `cryptography.hazmat.primitives.rsa.RSAPublicKey` wrapped - in `.ComparableRSAKey` - - """ - typ = 'RSA' - cryptography_key_types = (rsa.RSAPublicKey, rsa.RSAPrivateKey) - __slots__ = ('key',) - required = ('e', JWK.type_field_name, 'n') - - def __init__(self, *args, **kwargs): - if 'key' in kwargs and not isinstance( - kwargs['key'], util.ComparableRSAKey): - kwargs['key'] = util.ComparableRSAKey(kwargs['key']) - super(JWKRSA, self).__init__(*args, **kwargs) - - @classmethod - def _encode_param(cls, data): - """Encode Base64urlUInt. - - :type data: long - :rtype: unicode - - """ - def _leading_zeros(arg): - if len(arg) % 2: - return '0' + arg - return arg - - return json_util.encode_b64jose(binascii.unhexlify( - _leading_zeros(hex(data)[2:].rstrip('L')))) - - @classmethod - def _decode_param(cls, data): - """Decode Base64urlUInt.""" - try: - return int(binascii.hexlify(json_util.decode_b64jose(data)), 16) - except ValueError: # invalid literal for long() with base 16 - raise errors.DeserializationError() - - def public_key(self): - return type(self)(key=self.key.public_key()) - - @classmethod - def fields_from_json(cls, jobj): - # pylint: disable=invalid-name - n, e = (cls._decode_param(jobj[x]) for x in ('n', 'e')) - public_numbers = rsa.RSAPublicNumbers(e=e, n=n) - if 'd' not in jobj: # public key - key = public_numbers.public_key(default_backend()) - else: # private key - d = cls._decode_param(jobj['d']) - if ('p' in jobj or 'q' in jobj or 'dp' in jobj or - 'dq' in jobj or 'qi' in jobj or 'oth' in jobj): - # "If the producer includes any of the other private - # key parameters, then all of the others MUST be - # present, with the exception of "oth", which MUST - # only be present when more than two prime factors - # were used." - p, q, dp, dq, qi, = all_params = tuple( - jobj.get(x) for x in ('p', 'q', 'dp', 'dq', 'qi')) - if tuple(param for param in all_params if param is None): - raise errors.Error( - 'Some private parameters are missing: {0}'.format( - all_params)) - p, q, dp, dq, qi = tuple( - cls._decode_param(x) for x in all_params) - - # TODO: check for oth - else: - # cryptography>=0.8 - p, q = rsa.rsa_recover_prime_factors(n, e, d) - dp = rsa.rsa_crt_dmp1(d, p) - dq = rsa.rsa_crt_dmq1(d, q) - qi = rsa.rsa_crt_iqmp(p, q) - - key = rsa.RSAPrivateNumbers( - p, q, d, dp, dq, qi, public_numbers).private_key( - default_backend()) - - return cls(key=key) - - def fields_to_partial_json(self): - # pylint: disable=protected-access - if isinstance(self.key._wrapped, rsa.RSAPublicKey): - numbers = self.key.public_numbers() - params = { - 'n': numbers.n, - 'e': numbers.e, - } - else: # rsa.RSAPrivateKey - private = self.key.private_numbers() - public = self.key.public_key().public_numbers() - params = { - 'n': public.n, - 'e': public.e, - 'd': private.d, - 'p': private.p, - 'q': private.q, - 'dp': private.dmp1, - 'dq': private.dmq1, - 'qi': private.iqmp, - } - return dict((key, self._encode_param(value)) - for key, value in six.iteritems(params)) diff --git a/acme/acme/jose/jwk_test.py b/acme/acme/jose/jwk_test.py deleted file mode 100644 index eea5793bfa6..00000000000 --- a/acme/acme/jose/jwk_test.py +++ /dev/null @@ -1,191 +0,0 @@ -"""Tests for acme.jose.jwk.""" -import binascii -import unittest - -from acme import test_util - -from acme.jose import errors -from acme.jose import json_util -from acme.jose import util - - -DSA_PEM = test_util.load_vector('dsa512_key.pem') -RSA256_KEY = test_util.load_rsa_private_key('rsa256_key.pem') -RSA512_KEY = test_util.load_rsa_private_key('rsa512_key.pem') - - -class JWKTest(unittest.TestCase): - """Tests for acme.jose.jwk.JWK.""" - - def test_load(self): - from acme.jose.jwk import JWK - self.assertRaises(errors.Error, JWK.load, DSA_PEM) - - def test_load_subclass_wrong_type(self): - from acme.jose.jwk import JWKRSA - self.assertRaises(errors.Error, JWKRSA.load, DSA_PEM) - - -class JWKTestBaseMixin(object): - """Mixin test for JWK subclass tests.""" - - thumbprint = NotImplemented - - def test_thumbprint_private(self): - self.assertEqual(self.thumbprint, self.jwk.thumbprint()) - - def test_thumbprint_public(self): - self.assertEqual(self.thumbprint, self.jwk.public_key().thumbprint()) - - -class JWKOctTest(unittest.TestCase, JWKTestBaseMixin): - """Tests for acme.jose.jwk.JWKOct.""" - - thumbprint = (b"\xf3\xe7\xbe\xa8`\xd2\xdap\xe9}\x9c\xce>" - b"\xd0\xfcI\xbe\xcd\x92'\xd4o\x0e\xf41\xea" - b"\x8e(\x8a\xb2i\x1c") - - def setUp(self): - from acme.jose.jwk import JWKOct - self.jwk = JWKOct(key=b'foo') - self.jobj = {'kty': 'oct', 'k': json_util.encode_b64jose(b'foo')} - - def test_to_partial_json(self): - self.assertEqual(self.jwk.to_partial_json(), self.jobj) - - def test_from_json(self): - from acme.jose.jwk import JWKOct - self.assertEqual(self.jwk, JWKOct.from_json(self.jobj)) - - def test_from_json_hashable(self): - from acme.jose.jwk import JWKOct - hash(JWKOct.from_json(self.jobj)) - - def test_load(self): - from acme.jose.jwk import JWKOct - self.assertEqual(self.jwk, JWKOct.load(b'foo')) - - def test_public_key(self): - self.assertTrue(self.jwk.public_key() is self.jwk) - - -class JWKRSATest(unittest.TestCase, JWKTestBaseMixin): - """Tests for acme.jose.jwk.JWKRSA.""" - # pylint: disable=too-many-instance-attributes - - thumbprint = (b'\x83K\xdc#3\x98\xca\x98\xed\xcb\x80\x80<\x0c' - b'\xf0\x95\xb9H\xb2*l\xbd$\xe5&|O\x91\xd4 \xb0Y') - - def setUp(self): - from acme.jose.jwk import JWKRSA - self.jwk256 = JWKRSA(key=RSA256_KEY.public_key()) - self.jwk256json = { - 'kty': 'RSA', - 'e': 'AQAB', - 'n': 'm2Fylv-Uz7trgTW8EBHP3FQSMeZs2GNQ6VRo1sIVJEk', - } - # pylint: disable=protected-access - self.jwk256_not_comparable = JWKRSA( - key=RSA256_KEY.public_key()._wrapped) - self.jwk512 = JWKRSA(key=RSA512_KEY.public_key()) - self.jwk512json = { - 'kty': 'RSA', - 'e': 'AQAB', - 'n': 'rHVztFHtH92ucFJD_N_HW9AsdRsUuHUBBBDlHwNlRd3fp5' - '80rv2-6QWE30cWgdmJS86ObRz6lUTor4R0T-3C5Q', - } - self.private = JWKRSA(key=RSA256_KEY) - self.private_json_small = self.jwk256json.copy() - self.private_json_small['d'] = ( - 'lPQED_EPTV0UIBfNI3KP2d9Jlrc2mrMllmf946bu-CE') - self.private_json = self.jwk256json.copy() - self.private_json.update({ - 'd': 'lPQED_EPTV0UIBfNI3KP2d9Jlrc2mrMllmf946bu-CE', - 'p': 'zUVNZn4lLLBD1R6NE8TKNQ', - 'q': 'wcfKfc7kl5jfqXArCRSURQ', - 'dp': 'CWJFq43QvT5Bm5iN8n1okQ', - 'dq': 'bHh2u7etM8LKKCF2pY2UdQ', - 'qi': 'oi45cEkbVoJjAbnQpFY87Q', - }) - self.jwk = self.private - - def test_init_auto_comparable(self): - self.assertTrue(isinstance( - self.jwk256_not_comparable.key, util.ComparableRSAKey)) - self.assertEqual(self.jwk256, self.jwk256_not_comparable) - - def test_encode_param_zero(self): - from acme.jose.jwk import JWKRSA - # pylint: disable=protected-access - # TODO: move encode/decode _param to separate class - self.assertEqual('AA', JWKRSA._encode_param(0)) - - def test_equals(self): - self.assertEqual(self.jwk256, self.jwk256) - self.assertEqual(self.jwk512, self.jwk512) - - def test_not_equals(self): - self.assertNotEqual(self.jwk256, self.jwk512) - self.assertNotEqual(self.jwk512, self.jwk256) - - def test_load(self): - from acme.jose.jwk import JWKRSA - self.assertEqual(self.private, JWKRSA.load( - test_util.load_vector('rsa256_key.pem'))) - - def test_public_key(self): - self.assertEqual(self.jwk256, self.private.public_key()) - - def test_to_partial_json(self): - self.assertEqual(self.jwk256.to_partial_json(), self.jwk256json) - self.assertEqual(self.jwk512.to_partial_json(), self.jwk512json) - self.assertEqual(self.private.to_partial_json(), self.private_json) - - def test_from_json(self): - from acme.jose.jwk import JWK - self.assertEqual( - self.jwk256, JWK.from_json(self.jwk256json)) - self.assertEqual( - self.jwk512, JWK.from_json(self.jwk512json)) - self.assertEqual(self.private, JWK.from_json(self.private_json)) - - def test_from_json_private_small(self): - from acme.jose.jwk import JWK - self.assertEqual(self.private, JWK.from_json(self.private_json_small)) - - def test_from_json_missing_one_additional(self): - from acme.jose.jwk import JWK - del self.private_json['q'] - self.assertRaises(errors.Error, JWK.from_json, self.private_json) - - def test_from_json_hashable(self): - from acme.jose.jwk import JWK - hash(JWK.from_json(self.jwk256json)) - - def test_from_json_non_schema_errors(self): - # valid against schema, but still failing - from acme.jose.jwk import JWK - self.assertRaises(errors.DeserializationError, JWK.from_json, - {'kty': 'RSA', 'e': 'AQAB', 'n': ''}) - self.assertRaises(errors.DeserializationError, JWK.from_json, - {'kty': 'RSA', 'e': 'AQAB', 'n': '1'}) - - def test_thumbprint_go_jose(self): - # https://github.com/square/go-jose/blob/4ddd71883fa547d37fbf598071f04512d8bafee3/jwk.go#L155 - # https://github.com/square/go-jose/blob/4ddd71883fa547d37fbf598071f04512d8bafee3/jwk_test.go#L331-L344 - # https://github.com/square/go-jose/blob/4ddd71883fa547d37fbf598071f04512d8bafee3/jwk_test.go#L384 - from acme.jose.jwk import JWKRSA - key = JWKRSA.json_loads("""{ - "kty": "RSA", - "kid": "bilbo.baggins@hobbiton.example", - "use": "sig", - "n": "n4EPtAOCc9AlkeQHPzHStgAbgs7bTZLwUBZdR8_KuKPEHLd4rHVTeT-O-XV2jRojdNhxJWTDvNd7nqQ0VEiZQHz_AJmSCpMaJMRBSFKrKb2wqVwGU_NsYOYL-QtiWN2lbzcEe6XC0dApr5ydQLrHqkHHig3RBordaZ6Aj-oBHqFEHYpPe7Tpe-OfVfHd1E6cS6M1FZcD1NNLYD5lFHpPI9bTwJlsde3uhGqC0ZCuEHg8lhzwOHrtIQbS0FVbb9k3-tVTU4fg_3L_vniUFAKwuCLqKnS2BYwdq_mzSnbLY7h_qixoR7jig3__kRhuaxwUkRz5iaiQkqgc5gHdrNP5zw", - "e": "AQAB" -}""") - self.assertEqual( - binascii.hexlify(key.thumbprint()), - b"f63838e96077ad1fc01c3f8405774dedc0641f558ebb4b40dccf5f9b6d66a932") - - -if __name__ == '__main__': - unittest.main() # pragma: no cover diff --git a/acme/acme/jose/jws.py b/acme/acme/jose/jws.py deleted file mode 100644 index 5f446e4b130..00000000000 --- a/acme/acme/jose/jws.py +++ /dev/null @@ -1,433 +0,0 @@ -"""JOSE Web Signature.""" -import argparse -import base64 -import sys - -import OpenSSL -import six - -from acme.jose import b64 -from acme.jose import errors -from acme.jose import json_util -from acme.jose import jwa -from acme.jose import jwk -from acme.jose import util - - -class MediaType(object): - """MediaType field encoder/decoder.""" - - PREFIX = 'application/' - """MIME Media Type and Content Type prefix.""" - - @classmethod - def decode(cls, value): - """Decoder.""" - # 4.1.10 - if '/' not in value: - if ';' in value: - raise errors.DeserializationError('Unexpected semi-colon') - return cls.PREFIX + value - return value - - @classmethod - def encode(cls, value): - """Encoder.""" - # 4.1.10 - if ';' not in value: - assert value.startswith(cls.PREFIX) - return value[len(cls.PREFIX):] - return value - - -class Header(json_util.JSONObjectWithFields): - """JOSE Header. - - .. warning:: This class supports **only** Registered Header - Parameter Names (as defined in section 4.1 of the - protocol). If you need Public Header Parameter Names (4.2) - or Private Header Parameter Names (4.3), you must subclass - and override :meth:`from_json` and :meth:`to_partial_json` - appropriately. - - .. warning:: This class does not support any extensions through - the "crit" (Critical) Header Parameter (4.1.11) and as a - conforming implementation, :meth:`from_json` treats its - occurrence as an error. Please subclass if you seek for - a different behaviour. - - :ivar x5tS256: "x5t#S256" - :ivar str typ: MIME Media Type, inc. :const:`MediaType.PREFIX`. - :ivar str cty: Content-Type, inc. :const:`MediaType.PREFIX`. - - """ - alg = json_util.Field( - 'alg', decoder=jwa.JWASignature.from_json, omitempty=True) - jku = json_util.Field('jku', omitempty=True) - jwk = json_util.Field('jwk', decoder=jwk.JWK.from_json, omitempty=True) - kid = json_util.Field('kid', omitempty=True) - x5u = json_util.Field('x5u', omitempty=True) - x5c = json_util.Field('x5c', omitempty=True, default=()) - x5t = json_util.Field( - 'x5t', decoder=json_util.decode_b64jose, omitempty=True) - x5tS256 = json_util.Field( - 'x5t#S256', decoder=json_util.decode_b64jose, omitempty=True) - typ = json_util.Field('typ', encoder=MediaType.encode, - decoder=MediaType.decode, omitempty=True) - cty = json_util.Field('cty', encoder=MediaType.encode, - decoder=MediaType.decode, omitempty=True) - crit = json_util.Field('crit', omitempty=True, default=()) - - def not_omitted(self): - """Fields that would not be omitted in the JSON object.""" - return dict((name, getattr(self, name)) - for name, field in six.iteritems(self._fields) - if not field.omit(getattr(self, name))) - - def __add__(self, other): - if not isinstance(other, type(self)): - raise TypeError('Header cannot be added to: {0}'.format( - type(other))) - - not_omitted_self = self.not_omitted() - not_omitted_other = other.not_omitted() - - if set(not_omitted_self).intersection(not_omitted_other): - raise TypeError('Addition of overlapping headers not defined') - - not_omitted_self.update(not_omitted_other) - return type(self)(**not_omitted_self) # pylint: disable=star-args - - def find_key(self): - """Find key based on header. - - .. todo:: Supports only "jwk" header parameter lookup. - - :returns: (Public) key found in the header. - :rtype: .JWK - - :raises acme.jose.errors.Error: if key could not be found - - """ - if self.jwk is None: - raise errors.Error('No key found') - return self.jwk - - @crit.decoder - def crit(unused_value): - # pylint: disable=missing-docstring,no-self-argument,no-self-use - raise errors.DeserializationError( - '"crit" is not supported, please subclass') - - # x5c does NOT use JOSE Base64 (4.1.6) - - @x5c.encoder # type: ignore - def x5c(value): # pylint: disable=missing-docstring,no-self-argument - return [base64.b64encode(OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_ASN1, cert.wrapped)) for cert in value] - - @x5c.decoder # type: ignore - def x5c(value): # pylint: disable=missing-docstring,no-self-argument - try: - return tuple(util.ComparableX509(OpenSSL.crypto.load_certificate( - OpenSSL.crypto.FILETYPE_ASN1, - base64.b64decode(cert))) for cert in value) - except OpenSSL.crypto.Error as error: - raise errors.DeserializationError(error) - - -class Signature(json_util.JSONObjectWithFields): - """JWS Signature. - - :ivar combined: Combined Header (protected and unprotected, - :class:`Header`). - :ivar unicode protected: JWS protected header (Jose Base-64 decoded). - :ivar header: JWS Unprotected Header (:class:`Header`). - :ivar str signature: The signature. - - """ - header_cls = Header - - __slots__ = ('combined',) - protected = json_util.Field('protected', omitempty=True, default='') - header = json_util.Field( - 'header', omitempty=True, default=header_cls(), - decoder=header_cls.from_json) - signature = json_util.Field( - 'signature', decoder=json_util.decode_b64jose, - encoder=json_util.encode_b64jose) - - @protected.encoder # type: ignore - def protected(value): # pylint: disable=missing-docstring,no-self-argument - # wrong type guess (Signature, not bytes) | pylint: disable=no-member - return json_util.encode_b64jose(value.encode('utf-8')) - - @protected.decoder # type: ignore - def protected(value): # pylint: disable=missing-docstring,no-self-argument - return json_util.decode_b64jose(value).decode('utf-8') - - def __init__(self, **kwargs): - if 'combined' not in kwargs: - kwargs = self._with_combined(kwargs) - super(Signature, self).__init__(**kwargs) - assert self.combined.alg is not None - - @classmethod - def _with_combined(cls, kwargs): - assert 'combined' not in kwargs - header = kwargs.get('header', cls._fields['header'].default) - protected = kwargs.get('protected', cls._fields['protected'].default) - - if protected: - combined = header + cls.header_cls.json_loads(protected) - else: - combined = header - - kwargs['combined'] = combined - return kwargs - - @classmethod - def _msg(cls, protected, payload): - return (b64.b64encode(protected.encode('utf-8')) + b'.' + - b64.b64encode(payload)) - - def verify(self, payload, key=None): - """Verify. - - :param JWK key: Key used for verification. - - """ - key = self.combined.find_key() if key is None else key - return self.combined.alg.verify( - key=key.key, sig=self.signature, - msg=self._msg(self.protected, payload)) - - @classmethod - def sign(cls, payload, key, alg, include_jwk=True, - protect=frozenset(), **kwargs): - """Sign. - - :param JWK key: Key for signature. - - """ - assert isinstance(key, alg.kty) - - header_params = kwargs - header_params['alg'] = alg - if include_jwk: - header_params['jwk'] = key.public_key() - - assert set(header_params).issubset(cls.header_cls._fields) - assert protect.issubset(cls.header_cls._fields) - - protected_params = {} - for header in protect: - if header in header_params: - protected_params[header] = header_params.pop(header) - if protected_params: - # pylint: disable=star-args - protected = cls.header_cls(**protected_params).json_dumps() - else: - protected = '' - - header = cls.header_cls(**header_params) # pylint: disable=star-args - signature = alg.sign(key.key, cls._msg(protected, payload)) - - return cls(protected=protected, header=header, signature=signature) - - def fields_to_partial_json(self): - fields = super(Signature, self).fields_to_partial_json() - if not fields['header'].not_omitted(): - del fields['header'] - return fields - - @classmethod - def fields_from_json(cls, jobj): - fields = super(Signature, cls).fields_from_json(jobj) - fields_with_combined = cls._with_combined(fields) - if 'alg' not in fields_with_combined['combined'].not_omitted(): - raise errors.DeserializationError('alg not present') - return fields_with_combined - - -class JWS(json_util.JSONObjectWithFields): - """JSON Web Signature. - - :ivar str payload: JWS Payload. - :ivar str signature: JWS Signatures. - - """ - __slots__ = ('payload', 'signatures') - - signature_cls = Signature - - def verify(self, key=None): - """Verify.""" - return all(sig.verify(self.payload, key) for sig in self.signatures) - - @classmethod - def sign(cls, payload, **kwargs): - """Sign.""" - return cls(payload=payload, signatures=( - cls.signature_cls.sign(payload=payload, **kwargs),)) - - @property - def signature(self): - """Get a singleton signature. - - :rtype: `signature_cls` - - """ - assert len(self.signatures) == 1 - return self.signatures[0] - - def to_compact(self): - """Compact serialization. - - :rtype: bytes - - """ - assert len(self.signatures) == 1 - - assert 'alg' not in self.signature.header.not_omitted() - # ... it must be in protected - - return ( - b64.b64encode(self.signature.protected.encode('utf-8')) + - b'.' + - b64.b64encode(self.payload) + - b'.' + - b64.b64encode(self.signature.signature)) - - @classmethod - def from_compact(cls, compact): - """Compact deserialization. - - :param bytes compact: - - """ - try: - protected, payload, signature = compact.split(b'.') - except ValueError: - raise errors.DeserializationError( - 'Compact JWS serialization should comprise of exactly' - ' 3 dot-separated components') - - sig = cls.signature_cls( - protected=b64.b64decode(protected).decode('utf-8'), - signature=b64.b64decode(signature)) - return cls(payload=b64.b64decode(payload), signatures=(sig,)) - - def to_partial_json(self, flat=True): # pylint: disable=arguments-differ - assert self.signatures - payload = json_util.encode_b64jose(self.payload) - - if flat and len(self.signatures) == 1: - ret = self.signatures[0].to_partial_json() - ret['payload'] = payload - return ret - else: - return { - 'payload': payload, - 'signatures': self.signatures, - } - - @classmethod - def from_json(cls, jobj): - if 'signature' in jobj and 'signatures' in jobj: - raise errors.DeserializationError('Flat mixed with non-flat') - elif 'signature' in jobj: # flat - return cls(payload=json_util.decode_b64jose(jobj.pop('payload')), - signatures=(cls.signature_cls.from_json(jobj),)) - else: - return cls(payload=json_util.decode_b64jose(jobj['payload']), - signatures=tuple(cls.signature_cls.from_json(sig) - for sig in jobj['signatures'])) - - -class CLI(object): - """JWS CLI.""" - - @classmethod - def sign(cls, args): - """Sign.""" - key = args.alg.kty.load(args.key.read()) - args.key.close() - if args.protect is None: - args.protect = [] - if args.compact: - args.protect.append('alg') - - sig = JWS.sign(payload=sys.stdin.read().encode(), key=key, alg=args.alg, - protect=set(args.protect)) - - if args.compact: - six.print_(sig.to_compact().decode('utf-8')) - else: # JSON - six.print_(sig.json_dumps_pretty()) - - @classmethod - def verify(cls, args): - """Verify.""" - if args.compact: - sig = JWS.from_compact(sys.stdin.read().encode()) - else: # JSON - try: - sig = JWS.json_loads(sys.stdin.read()) - except errors.Error as error: - six.print_(error) - return -1 - - if args.key is not None: - assert args.kty is not None - key = args.kty.load(args.key.read()).public_key() - args.key.close() - else: - key = None - - sys.stdout.write(sig.payload) - return not sig.verify(key=key) - - @classmethod - def _alg_type(cls, arg): - return jwa.JWASignature.from_json(arg) - - @classmethod - def _header_type(cls, arg): - assert arg in Signature.header_cls._fields - return arg - - @classmethod - def _kty_type(cls, arg): - assert arg in jwk.JWK.TYPES - return jwk.JWK.TYPES[arg] - - @classmethod - def run(cls, args=sys.argv[1:]): - """Parse arguments and sign/verify.""" - parser = argparse.ArgumentParser() - parser.add_argument('--compact', action='store_true') - - subparsers = parser.add_subparsers() - parser_sign = subparsers.add_parser('sign') - parser_sign.set_defaults(func=cls.sign) - parser_sign.add_argument( - '-k', '--key', type=argparse.FileType('rb'), required=True) - parser_sign.add_argument( - '-a', '--alg', type=cls._alg_type, default=jwa.RS256) - parser_sign.add_argument( - '-p', '--protect', action='append', type=cls._header_type) - - parser_verify = subparsers.add_parser('verify') - parser_verify.set_defaults(func=cls.verify) - parser_verify.add_argument( - '-k', '--key', type=argparse.FileType('rb'), required=False) - parser_verify.add_argument( - '--kty', type=cls._kty_type, required=False) - - parsed = parser.parse_args(args) - return parsed.func(parsed) - - -if __name__ == '__main__': - exit(CLI.run()) # pragma: no cover diff --git a/acme/acme/jose/jws_test.py b/acme/acme/jose/jws_test.py deleted file mode 100644 index ec91f6a1bf3..00000000000 --- a/acme/acme/jose/jws_test.py +++ /dev/null @@ -1,239 +0,0 @@ -"""Tests for acme.jose.jws.""" -import base64 -import unittest - -import mock -import OpenSSL - -from acme import test_util - -from acme.jose import errors -from acme.jose import json_util -from acme.jose import jwa -from acme.jose import jwk - - -CERT = test_util.load_comparable_cert('cert.pem') -KEY = jwk.JWKRSA.load(test_util.load_vector('rsa512_key.pem')) - - -class MediaTypeTest(unittest.TestCase): - """Tests for acme.jose.jws.MediaType.""" - - def test_decode(self): - from acme.jose.jws import MediaType - self.assertEqual('application/app', MediaType.decode('application/app')) - self.assertEqual('application/app', MediaType.decode('app')) - self.assertRaises( - errors.DeserializationError, MediaType.decode, 'app;foo') - - def test_encode(self): - from acme.jose.jws import MediaType - self.assertEqual('app', MediaType.encode('application/app')) - self.assertEqual('application/app;foo', - MediaType.encode('application/app;foo')) - - -class HeaderTest(unittest.TestCase): - """Tests for acme.jose.jws.Header.""" - - def setUp(self): - from acme.jose.jws import Header - self.header1 = Header(jwk='foo') - self.header2 = Header(jwk='bar') - self.crit = Header(crit=('a', 'b')) - self.empty = Header() - - def test_add_non_empty(self): - from acme.jose.jws import Header - self.assertEqual(Header(jwk='foo', crit=('a', 'b')), - self.header1 + self.crit) - - def test_add_empty(self): - self.assertEqual(self.header1, self.header1 + self.empty) - self.assertEqual(self.header1, self.empty + self.header1) - - def test_add_overlapping_error(self): - self.assertRaises(TypeError, self.header1.__add__, self.header2) - - def test_add_wrong_type_error(self): - self.assertRaises(TypeError, self.header1.__add__, 'xxx') - - def test_crit_decode_always_errors(self): - from acme.jose.jws import Header - self.assertRaises(errors.DeserializationError, Header.from_json, - {'crit': ['a', 'b']}) - - def test_x5c_decoding(self): - from acme.jose.jws import Header - header = Header(x5c=(CERT, CERT)) - jobj = header.to_partial_json() - cert_asn1 = OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_ASN1, CERT.wrapped) - cert_b64 = base64.b64encode(cert_asn1) - self.assertEqual(jobj, {'x5c': [cert_b64, cert_b64]}) - self.assertEqual(header, Header.from_json(jobj)) - jobj['x5c'][0] = base64.b64encode(b'xxx' + cert_asn1) - self.assertRaises(errors.DeserializationError, Header.from_json, jobj) - - def test_find_key(self): - self.assertEqual('foo', self.header1.find_key()) - self.assertEqual('bar', self.header2.find_key()) - self.assertRaises(errors.Error, self.crit.find_key) - - -class SignatureTest(unittest.TestCase): - """Tests for acme.jose.jws.Signature.""" - - def test_from_json(self): - from acme.jose.jws import Header - from acme.jose.jws import Signature - self.assertEqual( - Signature(signature=b'foo', header=Header(alg=jwa.RS256)), - Signature.from_json( - {'signature': 'Zm9v', 'header': {'alg': 'RS256'}})) - - def test_from_json_no_alg_error(self): - from acme.jose.jws import Signature - self.assertRaises(errors.DeserializationError, - Signature.from_json, {'signature': 'foo'}) - - -class JWSTest(unittest.TestCase): - """Tests for acme.jose.jws.JWS.""" - - def setUp(self): - self.privkey = KEY - self.pubkey = self.privkey.public_key() - - from acme.jose.jws import JWS - self.unprotected = JWS.sign( - payload=b'foo', key=self.privkey, alg=jwa.RS256) - self.protected = JWS.sign( - payload=b'foo', key=self.privkey, alg=jwa.RS256, - protect=frozenset(['jwk', 'alg'])) - self.mixed = JWS.sign( - payload=b'foo', key=self.privkey, alg=jwa.RS256, - protect=frozenset(['alg'])) - - def test_pubkey_jwk(self): - self.assertEqual(self.unprotected.signature.combined.jwk, self.pubkey) - self.assertEqual(self.protected.signature.combined.jwk, self.pubkey) - self.assertEqual(self.mixed.signature.combined.jwk, self.pubkey) - - def test_sign_unprotected(self): - self.assertTrue(self.unprotected.verify()) - - def test_sign_protected(self): - self.assertTrue(self.protected.verify()) - - def test_sign_mixed(self): - self.assertTrue(self.mixed.verify()) - - def test_compact_lost_unprotected(self): - compact = self.mixed.to_compact() - self.assertEqual( - b'eyJhbGciOiAiUlMyNTYifQ.Zm9v.OHdxFVj73l5LpxbFp1AmYX4yJM0Pyb' - b'_893n1zQjpim_eLS5J1F61lkvrCrCDErTEJnBGOGesJ72M7b6Ve1cAJA', - compact) - - from acme.jose.jws import JWS - mixed = JWS.from_compact(compact) - - self.assertNotEqual(self.mixed, mixed) - self.assertEqual( - set(['alg']), set(mixed.signature.combined.not_omitted())) - - def test_from_compact_missing_components(self): - from acme.jose.jws import JWS - self.assertRaises(errors.DeserializationError, JWS.from_compact, b'.') - - def test_json_omitempty(self): - protected_jobj = self.protected.to_partial_json(flat=True) - unprotected_jobj = self.unprotected.to_partial_json(flat=True) - - self.assertTrue('protected' not in unprotected_jobj) - self.assertTrue('header' not in protected_jobj) - - unprotected_jobj['header'] = unprotected_jobj['header'].to_json() - - from acme.jose.jws import JWS - self.assertEqual(JWS.from_json(protected_jobj), self.protected) - self.assertEqual(JWS.from_json(unprotected_jobj), self.unprotected) - - def test_json_flat(self): - jobj_to = { - 'signature': json_util.encode_b64jose( - self.mixed.signature.signature), - 'payload': json_util.encode_b64jose(b'foo'), - 'header': self.mixed.signature.header, - 'protected': json_util.encode_b64jose( - self.mixed.signature.protected.encode('utf-8')), - } - jobj_from = jobj_to.copy() - jobj_from['header'] = jobj_from['header'].to_json() - - self.assertEqual(self.mixed.to_partial_json(flat=True), jobj_to) - from acme.jose.jws import JWS - self.assertEqual(self.mixed, JWS.from_json(jobj_from)) - - def test_json_not_flat(self): - jobj_to = { - 'signatures': (self.mixed.signature,), - 'payload': json_util.encode_b64jose(b'foo'), - } - jobj_from = jobj_to.copy() - jobj_from['signatures'] = [jobj_to['signatures'][0].to_json()] - - self.assertEqual(self.mixed.to_partial_json(flat=False), jobj_to) - from acme.jose.jws import JWS - self.assertEqual(self.mixed, JWS.from_json(jobj_from)) - - def test_from_json_mixed_flat(self): - from acme.jose.jws import JWS - self.assertRaises(errors.DeserializationError, JWS.from_json, - {'signatures': (), 'signature': 'foo'}) - - def test_from_json_hashable(self): - from acme.jose.jws import JWS - hash(JWS.from_json(self.mixed.to_json())) - - -class CLITest(unittest.TestCase): - - def setUp(self): - self.key_path = test_util.vector_path('rsa512_key.pem') - - def test_unverified(self): - from acme.jose.jws import CLI - with mock.patch('sys.stdin') as sin: - sin.read.return_value = '{"payload": "foo", "signature": "xxx"}' - with mock.patch('sys.stdout'): - self.assertEqual(-1, CLI.run(['verify'])) - - def test_json(self): - from acme.jose.jws import CLI - - with mock.patch('sys.stdin') as sin: - sin.read.return_value = 'foo' - with mock.patch('sys.stdout') as sout: - CLI.run(['sign', '-k', self.key_path, '-a', 'RS256', - '-p', 'jwk']) - sin.read.return_value = sout.write.mock_calls[0][1][0] - self.assertEqual(0, CLI.run(['verify'])) - - def test_compact(self): - from acme.jose.jws import CLI - - with mock.patch('sys.stdin') as sin: - sin.read.return_value = 'foo' - with mock.patch('sys.stdout') as sout: - CLI.run(['--compact', 'sign', '-k', self.key_path]) - sin.read.return_value = sout.write.mock_calls[0][1][0] - self.assertEqual(0, CLI.run([ - '--compact', 'verify', '--kty', 'RSA', - '-k', self.key_path])) - - -if __name__ == '__main__': - unittest.main() # pragma: no cover diff --git a/acme/acme/jose/util.py b/acme/acme/jose/util.py deleted file mode 100644 index 26b7e0c5a69..00000000000 --- a/acme/acme/jose/util.py +++ /dev/null @@ -1,226 +0,0 @@ -"""JOSE utilities.""" -import collections - -from cryptography.hazmat.primitives.asymmetric import rsa -import OpenSSL -import six - - -class abstractclassmethod(classmethod): - # pylint: disable=invalid-name,too-few-public-methods - """Descriptor for an abstract classmethod. - - It augments the :mod:`abc` framework with an abstract - classmethod. This is implemented as :class:`abc.abstractclassmethod` - in the standard Python library starting with version 3.2. - - This particular implementation, allegedly based on Python 3.3 source - code, is stolen from - http://stackoverflow.com/questions/11217878/python-2-7-combine-abc-abstractmethod-and-classmethod. - - """ - __isabstractmethod__ = True - - def __init__(self, target): - target.__isabstractmethod__ = True - super(abstractclassmethod, self).__init__(target) - - -class ComparableX509(object): # pylint: disable=too-few-public-methods - """Wrapper for OpenSSL.crypto.X509** objects that supports __eq__. - - :ivar wrapped: Wrapped certificate or certificate request. - :type wrapped: `OpenSSL.crypto.X509` or `OpenSSL.crypto.X509Req`. - - """ - def __init__(self, wrapped): - assert isinstance(wrapped, OpenSSL.crypto.X509) or isinstance( - wrapped, OpenSSL.crypto.X509Req) - self.wrapped = wrapped - - def __getattr__(self, name): - return getattr(self.wrapped, name) - - def _dump(self, filetype=OpenSSL.crypto.FILETYPE_ASN1): - """Dumps the object into a buffer with the specified encoding. - - :param int filetype: The desired encoding. Should be one of - `OpenSSL.crypto.FILETYPE_ASN1`, - `OpenSSL.crypto.FILETYPE_PEM`, or - `OpenSSL.crypto.FILETYPE_TEXT`. - - :returns: Encoded X509 object. - :rtype: str - - """ - if isinstance(self.wrapped, OpenSSL.crypto.X509): - func = OpenSSL.crypto.dump_certificate - else: # assert in __init__ makes sure this is X509Req - func = OpenSSL.crypto.dump_certificate_request - return func(filetype, self.wrapped) - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return NotImplemented - # pylint: disable=protected-access - return self._dump() == other._dump() - - def __hash__(self): - return hash((self.__class__, self._dump())) - - def __ne__(self, other): - return not self == other - - def __repr__(self): - return '<{0}({1!r})>'.format(self.__class__.__name__, self.wrapped) - - -class ComparableKey(object): # pylint: disable=too-few-public-methods - """Comparable wrapper for `cryptography` keys. - - See https://github.com/pyca/cryptography/issues/2122. - - """ - __hash__ = NotImplemented - - def __init__(self, wrapped): - self._wrapped = wrapped - - def __getattr__(self, name): - return getattr(self._wrapped, name) - - def __eq__(self, other): - # pylint: disable=protected-access - if (not isinstance(other, self.__class__) or - self._wrapped.__class__ is not other._wrapped.__class__): - return NotImplemented - elif hasattr(self._wrapped, 'private_numbers'): - return self.private_numbers() == other.private_numbers() - elif hasattr(self._wrapped, 'public_numbers'): - return self.public_numbers() == other.public_numbers() - else: - return NotImplemented - - def __ne__(self, other): - return not self == other - - def __repr__(self): - return '<{0}({1!r})>'.format(self.__class__.__name__, self._wrapped) - - def public_key(self): - """Get wrapped public key.""" - return self.__class__(self._wrapped.public_key()) - - -class ComparableRSAKey(ComparableKey): # pylint: disable=too-few-public-methods - """Wrapper for `cryptography` RSA keys. - - Wraps around: - - `cryptography.hazmat.primitives.asymmetric.RSAPrivateKey` - - `cryptography.hazmat.primitives.asymmetric.RSAPublicKey` - - """ - - def __hash__(self): - # public_numbers() hasn't got stable hash! - # https://github.com/pyca/cryptography/issues/2143 - if isinstance(self._wrapped, rsa.RSAPrivateKeyWithSerialization): - priv = self.private_numbers() - pub = priv.public_numbers - return hash((self.__class__, priv.p, priv.q, priv.dmp1, - priv.dmq1, priv.iqmp, pub.n, pub.e)) - elif isinstance(self._wrapped, rsa.RSAPublicKeyWithSerialization): - pub = self.public_numbers() - return hash((self.__class__, pub.n, pub.e)) - - -class ImmutableMap(collections.Mapping, collections.Hashable): # type: ignore - # pylint: disable=too-few-public-methods - """Immutable key to value mapping with attribute access.""" - - __slots__ = () - """Must be overridden in subclasses.""" - - def __init__(self, **kwargs): - if set(kwargs) != set(self.__slots__): - raise TypeError( - '__init__() takes exactly the following arguments: {0} ' - '({1} given)'.format(', '.join(self.__slots__), - ', '.join(kwargs) if kwargs else 'none')) - for slot in self.__slots__: - object.__setattr__(self, slot, kwargs.pop(slot)) - - def update(self, **kwargs): - """Return updated map.""" - items = dict(self) - items.update(kwargs) - return type(self)(**items) # pylint: disable=star-args - - def __getitem__(self, key): - try: - return getattr(self, key) - except AttributeError: - raise KeyError(key) - - def __iter__(self): - return iter(self.__slots__) - - def __len__(self): - return len(self.__slots__) - - def __hash__(self): - return hash(tuple(getattr(self, slot) for slot in self.__slots__)) - - def __setattr__(self, name, value): - raise AttributeError("can't set attribute") - - def __repr__(self): - return '{0}({1})'.format(self.__class__.__name__, ', '.join( - '{0}={1!r}'.format(key, value) - for key, value in six.iteritems(self))) - - -class frozendict(collections.Mapping, collections.Hashable): # type: ignore - # pylint: disable=invalid-name,too-few-public-methods - """Frozen dictionary.""" - __slots__ = ('_items', '_keys') - - def __init__(self, *args, **kwargs): - if kwargs and not args: - items = dict(kwargs) - elif len(args) == 1 and isinstance(args[0], collections.Mapping): - items = args[0] - else: - raise TypeError() - # TODO: support generators/iterators - - object.__setattr__(self, '_items', items) - object.__setattr__(self, '_keys', tuple(sorted(six.iterkeys(items)))) - - def __getitem__(self, key): - return self._items[key] - - def __iter__(self): - return iter(self._keys) - - def __len__(self): - return len(self._items) - - def _sorted_items(self): - return tuple((key, self[key]) for key in self._keys) - - def __hash__(self): - return hash(self._sorted_items()) - - def __getattr__(self, name): - try: - return self._items[name] - except KeyError: - raise AttributeError(name) - - def __setattr__(self, name, value): - raise AttributeError("can't set attribute") - - def __repr__(self): - return 'frozendict({0})'.format(', '.join('{0}={1!r}'.format( - key, value) for key, value in self._sorted_items())) diff --git a/acme/acme/jose/util_test.py b/acme/acme/jose/util_test.py deleted file mode 100644 index 0038a6cc14e..00000000000 --- a/acme/acme/jose/util_test.py +++ /dev/null @@ -1,199 +0,0 @@ -"""Tests for acme.jose.util.""" -import functools -import unittest - -import six - -from acme import test_util - - -class ComparableX509Test(unittest.TestCase): - """Tests for acme.jose.util.ComparableX509.""" - - def setUp(self): - # test_util.load_comparable_{csr,cert} return ComparableX509 - self.req1 = test_util.load_comparable_csr('csr.pem') - self.req2 = test_util.load_comparable_csr('csr.pem') - self.req_other = test_util.load_comparable_csr('csr-san.pem') - - self.cert1 = test_util.load_comparable_cert('cert.pem') - self.cert2 = test_util.load_comparable_cert('cert.pem') - self.cert_other = test_util.load_comparable_cert('cert-san.pem') - - def test_getattr_proxy(self): - self.assertTrue(self.cert1.has_expired()) - - def test_eq(self): - self.assertEqual(self.req1, self.req2) - self.assertEqual(self.cert1, self.cert2) - - def test_ne(self): - self.assertNotEqual(self.req1, self.req_other) - self.assertNotEqual(self.cert1, self.cert_other) - - def test_ne_wrong_types(self): - self.assertNotEqual(self.req1, 5) - self.assertNotEqual(self.cert1, 5) - - def test_hash(self): - self.assertEqual(hash(self.req1), hash(self.req2)) - self.assertNotEqual(hash(self.req1), hash(self.req_other)) - - self.assertEqual(hash(self.cert1), hash(self.cert2)) - self.assertNotEqual(hash(self.cert1), hash(self.cert_other)) - - def test_repr(self): - for x509 in self.req1, self.cert1: - self.assertEqual(repr(x509), - ''.format(x509.wrapped)) - - -class ComparableRSAKeyTest(unittest.TestCase): - """Tests for acme.jose.util.ComparableRSAKey.""" - - def setUp(self): - # test_utl.load_rsa_private_key return ComparableRSAKey - self.key = test_util.load_rsa_private_key('rsa256_key.pem') - self.key_same = test_util.load_rsa_private_key('rsa256_key.pem') - self.key2 = test_util.load_rsa_private_key('rsa512_key.pem') - - def test_getattr_proxy(self): - self.assertEqual(256, self.key.key_size) - - def test_eq(self): - self.assertEqual(self.key, self.key_same) - - def test_ne(self): - self.assertNotEqual(self.key, self.key2) - - def test_ne_different_types(self): - self.assertNotEqual(self.key, 5) - - def test_ne_not_wrapped(self): - # pylint: disable=protected-access - self.assertNotEqual(self.key, self.key_same._wrapped) - - def test_ne_no_serialization(self): - from acme.jose.util import ComparableRSAKey - self.assertNotEqual(ComparableRSAKey(5), ComparableRSAKey(5)) - - def test_hash(self): - self.assertTrue(isinstance(hash(self.key), int)) - self.assertEqual(hash(self.key), hash(self.key_same)) - self.assertNotEqual(hash(self.key), hash(self.key2)) - - def test_repr(self): - self.assertTrue(repr(self.key).startswith( - '=0.6) # rsa_recover_prime_factors (>=0.8) 'cryptography>=0.8', + # formerly known as acme.jose: + 'josepy>=1.0.0', # Connection.set_tlsext_host_name (>=0.13) 'mock', 'PyOpenSSL>=0.13', @@ -74,10 +76,5 @@ 'dev': dev_extras, 'docs': docs_extras, }, - entry_points={ - 'console_scripts': [ - 'jws = acme.jose.jws:CLI.run', - ], - }, test_suite='acme', ) diff --git a/certbot-apache/certbot_apache/tests/util.py b/certbot-apache/certbot_apache/tests/util.py index 2405110c53f..ca667465c26 100644 --- a/certbot-apache/certbot_apache/tests/util.py +++ b/certbot-apache/certbot_apache/tests/util.py @@ -5,11 +5,10 @@ import unittest import augeas +import josepy as jose import mock import zope.component -from acme import jose - from certbot.display import util as display_util from certbot.plugins import common diff --git a/certbot-compatibility-test/certbot_compatibility_test/util.py b/certbot-compatibility-test/certbot_compatibility_test/util.py index af951aa6a99..4155944bd5b 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/util.py +++ b/certbot-compatibility-test/certbot_compatibility_test/util.py @@ -6,7 +6,8 @@ import shutil import tarfile -from acme import jose +import josepy as jose + from acme import test_util from certbot import constants diff --git a/certbot-nginx/certbot_nginx/tests/util.py b/certbot-nginx/certbot_nginx/tests/util.py index 6e1b0d8ff58..7b32d8e82fa 100644 --- a/certbot-nginx/certbot_nginx/tests/util.py +++ b/certbot-nginx/certbot_nginx/tests/util.py @@ -5,11 +5,10 @@ import tempfile import unittest +import josepy as jose import mock import zope.component -from acme import jose - from certbot import configuration from certbot.tests import util as test_util diff --git a/certbot/account.py b/certbot/account.py index 389f9679127..41e9800974b 100644 --- a/certbot/account.py +++ b/certbot/account.py @@ -7,13 +7,13 @@ import socket from cryptography.hazmat.primitives import serialization +import josepy as jose import pyrfc3339 import pytz import six import zope.component from acme import fields as acme_fields -from acme import jose from acme import messages from certbot import errors diff --git a/certbot/achallenges.py b/certbot/achallenges.py index f39bb4cec00..6535a6b630d 100644 --- a/certbot/achallenges.py +++ b/certbot/achallenges.py @@ -19,8 +19,9 @@ """ import logging +import josepy as jose + from acme import challenges -from acme import jose logger = logging.getLogger(__name__) diff --git a/certbot/client.py b/certbot/client.py index ed70fda711a..b735421f5a2 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -5,13 +5,13 @@ from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.asymmetric import rsa +import josepy as jose import OpenSSL import zope.component from acme import client as acme_client from acme import crypto_util as acme_crypto_util from acme import errors as acme_errors -from acme import jose from acme import messages import certbot diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index 112ef7c852e..3ae16529d3f 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -14,9 +14,9 @@ import zope.component from cryptography.hazmat.backends import default_backend from cryptography import x509 +import josepy as jose from acme import crypto_util as acme_crypto_util -from acme import jose from certbot import errors from certbot import interfaces @@ -368,7 +368,7 @@ def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM): """Dump certificate chain into a bundle. :param list chain: List of `OpenSSL.crypto.X509` (or wrapped in - `acme.jose.ComparableX509`). + :class:`josepy.util.ComparableX509`). """ # XXX: returns empty string when no chain is available, which diff --git a/certbot/main.py b/certbot/main.py index 72af7fbba9a..1c6432fd986 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -6,9 +6,9 @@ import sys import configobj +import josepy as jose import zope.component -from acme import jose from acme import errors as acme_errors import certbot diff --git a/certbot/plugins/common.py b/certbot/plugins/common.py index 420d15679d3..002d2f22530 100644 --- a/certbot/plugins/common.py +++ b/certbot/plugins/common.py @@ -9,7 +9,7 @@ import pkg_resources import zope.interface -from acme.jose import util as jose_util +from josepy import util as jose_util from certbot import constants from certbot import crypto_util diff --git a/certbot/plugins/common_test.py b/certbot/plugins/common_test.py index 8ce68bbb5ae..1a1ca7dcb80 100644 --- a/certbot/plugins/common_test.py +++ b/certbot/plugins/common_test.py @@ -5,11 +5,11 @@ import tempfile import unittest +import josepy as jose import mock import OpenSSL from acme import challenges -from acme import jose from certbot import achallenges from certbot import crypto_util diff --git a/certbot/plugins/dns_test_common.py b/certbot/plugins/dns_test_common.py index d8cd2940445..54b656b20ac 100644 --- a/certbot/plugins/dns_test_common.py +++ b/certbot/plugins/dns_test_common.py @@ -3,10 +3,10 @@ import os import configobj +import josepy as jose import mock import six from acme import challenges -from acme import jose from certbot import achallenges from certbot.tests import acme_util diff --git a/certbot/plugins/dns_test_common_lexicon.py b/certbot/plugins/dns_test_common_lexicon.py index f9c5735e80e..a221cf1bf0b 100644 --- a/certbot/plugins/dns_test_common_lexicon.py +++ b/certbot/plugins/dns_test_common_lexicon.py @@ -1,7 +1,7 @@ """Base test class for DNS authenticators built on Lexicon.""" +import josepy as jose import mock -from acme import jose from requests.exceptions import HTTPError, RequestException from certbot import errors diff --git a/certbot/plugins/standalone_test.py b/certbot/plugins/standalone_test.py index 1ae731e429e..5227bc59e39 100644 --- a/certbot/plugins/standalone_test.py +++ b/certbot/plugins/standalone_test.py @@ -3,11 +3,11 @@ import socket import unittest +import josepy as jose import mock import six from acme import challenges -from acme import jose from certbot import achallenges from certbot import errors diff --git a/certbot/plugins/webroot_test.py b/certbot/plugins/webroot_test.py index 92160bdfa39..36e2ffba6ee 100644 --- a/certbot/plugins/webroot_test.py +++ b/certbot/plugins/webroot_test.py @@ -10,11 +10,11 @@ import tempfile import unittest +import josepy as jose import mock import six from acme import challenges -from acme import jose from certbot import achallenges from certbot import errors diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index 7245ad6a14f..8ebda56afba 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -6,10 +6,10 @@ import stat import unittest +import josepy as jose import mock import pytz -from acme import jose from acme import messages from certbot import errors diff --git a/certbot/tests/acme_util.py b/certbot/tests/acme_util.py index f0549666a37..53a2f214aed 100644 --- a/certbot/tests/acme_util.py +++ b/certbot/tests/acme_util.py @@ -1,10 +1,10 @@ """ACME utilities for testing.""" import datetime +import josepy as jose import six from acme import challenges -from acme import jose from acme import messages from certbot import auth_handler diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 09c4a50ca95..204f46323a9 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -4,11 +4,11 @@ import tempfile import unittest +import josepy as jose import OpenSSL import mock from acme import errors as acme_errors -from acme import jose from certbot import account from certbot import errors diff --git a/certbot/tests/display/ops_test.py b/certbot/tests/display/ops_test.py index cb0fb32e35f..57d82f8392f 100644 --- a/certbot/tests/display/ops_test.py +++ b/certbot/tests/display/ops_test.py @@ -4,10 +4,10 @@ import sys import unittest +import josepy as jose import mock import zope.component -from acme import jose from acme import messages from certbot import account diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 1f690df261e..04b71dcc7fe 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -11,11 +11,10 @@ import datetime import pytz +import josepy as jose import six from six.moves import reload_module # pylint: disable=import-error -from acme import jose - from certbot import account from certbot import cli from certbot import constants diff --git a/certbot/tests/util.py b/certbot/tests/util.py index c43b44522f9..ddd4a1aecb6 100644 --- a/certbot/tests/util.py +++ b/certbot/tests/util.py @@ -14,11 +14,10 @@ from cryptography.hazmat.primitives import serialization import mock import OpenSSL +import josepy as jose import six from six.moves import reload_module # pylint: disable=import-error -from acme import jose - from certbot import constants from certbot import interfaces from certbot import storage diff --git a/tools/deactivate.py b/tools/deactivate.py index 5facc8436da..d43b845523f 100644 --- a/tools/deactivate.py +++ b/tools/deactivate.py @@ -18,10 +18,10 @@ from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.primitives import serialization +import josepy as jose from acme import client as acme_client from acme import errors as acme_errors -from acme import jose from acme import messages DIRECTORY = os.getenv('DIRECTORY', 'http://localhost:4000/directory') From 0e92d4ea98e44bfc9f1797269c6998195dea5f8a Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Mon, 11 Dec 2017 21:50:56 +0200 Subject: [PATCH 02/21] Parse variables without whitespace separator correctly in CentOS family of distributions (#5318) --- certbot-apache/certbot_apache/apache_util.py | 4 ++++ certbot-apache/certbot_apache/tests/centos_test.py | 2 ++ .../tests/testdata/centos7_apache/apache/sysconfig/httpd | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/certbot-apache/certbot_apache/apache_util.py b/certbot-apache/certbot_apache/apache_util.py index b4a24f137a6..f03c9da8793 100644 --- a/certbot-apache/certbot_apache/apache_util.py +++ b/certbot-apache/certbot_apache/apache_util.py @@ -93,4 +93,8 @@ def parse_define_file(filepath, varname): if v == "-D" and len(a_opts) >= i+2: var_parts = a_opts[i+1].partition("=") return_vars[var_parts[0]] = var_parts[2] + elif len(v) > 2 and v.startswith("-D"): + # Found var with no whitespace separator + var_parts = v[2:].partition("=") + return_vars[var_parts[0]] = var_parts[2] return return_vars diff --git a/certbot-apache/certbot_apache/tests/centos_test.py b/certbot-apache/certbot_apache/tests/centos_test.py index 7ca47a4d56f..d7a2a2fd922 100644 --- a/certbot-apache/certbot_apache/tests/centos_test.py +++ b/certbot-apache/certbot_apache/tests/centos_test.py @@ -118,6 +118,8 @@ def test_get_sysconfig_vars(self, mock_cfg): self.assertTrue("mock_define_too" in self.config.parser.variables.keys()) self.assertTrue("mock_value" in self.config.parser.variables.keys()) self.assertEqual("TRUE", self.config.parser.variables["mock_value"]) + self.assertTrue("MOCK_NOSEP" in self.config.parser.variables.keys()) + self.assertEqual("NOSEP_VAL", self.config.parser.variables["NOSEP_TWO"]) if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/sysconfig/httpd b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/sysconfig/httpd index 0bf6b176c67..4bcb300c29b 100644 --- a/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/sysconfig/httpd +++ b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/sysconfig/httpd @@ -14,7 +14,7 @@ # To pass additional options (for instance, -D definitions) to the # httpd binary at startup, set OPTIONS here. # -OPTIONS="-D mock_define -D mock_define_too -D mock_value=TRUE" +OPTIONS="-D mock_define -D mock_define_too -D mock_value=TRUE -DMOCK_NOSEP -DNOSEP_TWO=NOSEP_VAL" # # This setting ensures the httpd process is started in the "C" locale From 1b6005cc61f8b977af1bc5513994b4815280dd74 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 14 Dec 2017 18:15:42 -0800 Subject: [PATCH 03/21] Pin josepy in letsencrypt-auto (#5321) * pin josepy in le-auto * Put pinned versions in sorted order --- letsencrypt-auto-source/letsencrypt-auto | 11 +++++++---- .../pieces/dependency-requirements.txt | 11 +++++++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 8d2e8a6b63a..93e3e7b83bd 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -983,9 +983,16 @@ idna==2.5 \ ipaddress==1.0.16 \ --hash=sha256:935712800ce4760701d89ad677666cd52691fd2f6f0b340c8b4239a3c17988a5 \ --hash=sha256:5a3182b322a706525c46282ca6f064d27a02cffbd449f9f47416f1dc96aa71b0 +josepy==1.0.1 \ + --hash=sha256:354a3513038a38bbcd27c97b7c68a8f3dfaff0a135b20a92c6db4cc4ea72915e \ + --hash=sha256:9f48b88ca37f0244238b1cc77723989f7c54f7b90b2eee6294390bacfe870acc linecache2==1.0.0 \ --hash=sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef \ --hash=sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c +# Using an older version of mock here prevents regressions of #5276. +mock==1.3.0 \ + --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ + --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 ordereddict==1.1 \ --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f packaging==16.8 \ @@ -1062,10 +1069,6 @@ zope.interface==4.1.3 \ --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \ --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \ --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392 -# Using an older version of mock here prevents regressions of #5276. -mock==1.3.0 \ - --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ - --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 # Contains the requirements for the letsencrypt package. # diff --git a/letsencrypt-auto-source/pieces/dependency-requirements.txt b/letsencrypt-auto-source/pieces/dependency-requirements.txt index dec7ae7d065..0e2cec984a8 100644 --- a/letsencrypt-auto-source/pieces/dependency-requirements.txt +++ b/letsencrypt-auto-source/pieces/dependency-requirements.txt @@ -105,9 +105,16 @@ idna==2.5 \ ipaddress==1.0.16 \ --hash=sha256:935712800ce4760701d89ad677666cd52691fd2f6f0b340c8b4239a3c17988a5 \ --hash=sha256:5a3182b322a706525c46282ca6f064d27a02cffbd449f9f47416f1dc96aa71b0 +josepy==1.0.1 \ + --hash=sha256:354a3513038a38bbcd27c97b7c68a8f3dfaff0a135b20a92c6db4cc4ea72915e \ + --hash=sha256:9f48b88ca37f0244238b1cc77723989f7c54f7b90b2eee6294390bacfe870acc linecache2==1.0.0 \ --hash=sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef \ --hash=sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c +# Using an older version of mock here prevents regressions of #5276. +mock==1.3.0 \ + --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ + --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 ordereddict==1.1 \ --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f packaging==16.8 \ @@ -184,7 +191,3 @@ zope.interface==4.1.3 \ --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \ --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \ --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392 -# Using an older version of mock here prevents regressions of #5276. -mock==1.3.0 \ - --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ - --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 From a1aea021e7a587ea9396b2ebbfcfaec10411ab86 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 18 Dec 2017 12:31:36 -0800 Subject: [PATCH 04/21] Pin dependencies in oldest tests (#5316) * Add tools/merge_requirements.py * Revert "Fix oldest tests by pinning Google DNS deps (#5000)" This reverts commit f68fba2be2fc342dd72deaaf048ab79e5a8fc2be. * Add tools/oldest_constraints.txt * Remove oldest constraints from tox.ini * Rename dev constraints file * Update tools/pip_install.sh * Update install_and_test.sh * Fix pip_install.sh * Don't cat when you can cp * Add ng-httpsclient to dev constraints for oldest tests * Bump tested setuptools version * Update dev_constraints comment * Better document oldest dependencies * test against oldest versions we say we require * Update dev constraints * Properly handle empty lines * Update constraints gen in pip_install * Remove duplicated zope.component * Reduce pyasn1-modules dependency * Remove blank line * pin back google-api-python-client * pin back uritemplate * pin josepy for oldest tests * Undo changes to install_and_test.sh * Update install_and_test.sh description * use split instead of partition --- ...ip_constraints.txt => dev_constraints.txt} | 25 ++++---- tools/install_and_test.sh | 5 +- tools/merge_requirements.py | 61 +++++++++++++++++++ tools/oldest_constraints.txt | 51 ++++++++++++++++ tools/pip_install.sh | 31 ++++++---- tox.ini | 36 +---------- 6 files changed, 151 insertions(+), 58 deletions(-) rename tools/{pip_constraints.txt => dev_constraints.txt} (71%) create mode 100755 tools/merge_requirements.py create mode 100644 tools/oldest_constraints.txt diff --git a/tools/pip_constraints.txt b/tools/dev_constraints.txt similarity index 71% rename from tools/pip_constraints.txt rename to tools/dev_constraints.txt index cacec37d667..afc362ff8b5 100644 --- a/tools/pip_constraints.txt +++ b/tools/dev_constraints.txt @@ -1,16 +1,15 @@ # Specifies Python package versions for packages not specified in -# letsencrypt-auto's requirements file. We should avoid listing packages in -# both places because if both files are used as constraints for the same pip -# invocation, some constraints may be ignored due to pip's lack of dependency -# resolution. +# letsencrypt-auto's requirements file. alabaster==0.7.10 apipkg==1.4 +asn1crypto==0.22.0 astroid==1.3.5 +attrs==17.3.0 Babel==2.5.1 backports.shutil-get-terminal-size==1.0.0 boto3==1.4.7 botocore==1.7.41 -cloudflare==1.8.1 +cloudflare==1.5.1 coverage==4.4.2 decorator==4.1.2 dns-lexicon==2.1.14 @@ -19,7 +18,7 @@ docutils==0.14 execnet==1.5.0 future==0.16.0 futures==3.1.1 -google-api-python-client==1.6.4 +google-api-python-client==1.5 httplib2==0.10.3 imagesize==0.7.1 ipdb==0.10.3 @@ -27,20 +26,22 @@ ipython==5.5.0 ipython-genutils==0.2.0 Jinja2==2.9.6 jmespath==0.9.3 +josepy==1.0.1 +logger==1.4 logilab-common==1.4.1 MarkupSafe==1.0 -oauth2client==4.1.2 +ndg-httpsclient==0.3.2 +oauth2client==2.0.0 pathlib2==2.3.0 pexpect==4.2.1 pickleshare==0.7.4 -pkg-resources==0.0.0 pkginfo==1.4.1 pluggy==0.5.2 prompt-toolkit==1.0.15 ptyprocess==0.5.2 py==1.4.34 -pyasn1==0.3.7 -pyasn1-modules==0.1.5 +pyasn1==0.1.9 +pyasn1-modules==0.0.10 Pygments==2.2.0 pylint==1.4.2 pytest==3.2.5 @@ -48,7 +49,7 @@ pytest-cov==2.5.1 pytest-forked==0.2 pytest-xdist==1.20.1 python-dateutil==2.6.1 -python-digitalocean==1.12 +python-digitalocean==1.11 PyYAML==3.12 repoze.sphinx.autointerface==0.8 requests-file==1.4.2 @@ -65,6 +66,6 @@ tox==2.9.1 tqdm==4.19.4 traitlets==4.3.2 twine==1.9.1 -uritemplate==3.0.0 +uritemplate==0.6 virtualenv==15.1.0 wcwidth==0.1.7 diff --git a/tools/install_and_test.sh b/tools/install_and_test.sh index d57f0974ea3..25b6d548abe 100755 --- a/tools/install_and_test.sh +++ b/tools/install_and_test.sh @@ -2,8 +2,9 @@ # pip installs the requested packages in editable mode and runs unit tests on # them. Each package is installed and tested in the order they are provided # before the script moves on to the next package. If CERTBOT_NO_PIN is set not -# set to 1, packages are installed using certbot-auto's requirements file as -# constraints. +# set to 1, packages are installed using pinned versions of all of our +# dependencies. See pip_install.sh for more information on the versions pinned +# to. if [ "$CERTBOT_NO_PIN" = 1 ]; then pip_install="pip install -q -e" diff --git a/tools/merge_requirements.py b/tools/merge_requirements.py new file mode 100755 index 00000000000..c8fb9535182 --- /dev/null +++ b/tools/merge_requirements.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +"""Merges multiple Python requirements files into one file. + +Requirements files specified later take precedence over earlier ones. Only +simple SomeProject==1.2.3 format is currently supported. + +""" + +from __future__ import print_function + +import sys + + +def read_file(file_path): + """Reads in a Python requirements file. + + :param str file_path: path to requirements file + + :returns: mapping from a project to its pinned version + :rtype: dict + + """ + d = {} + with open(file_path) as f: + for line in f: + line = line.strip() + if line and not line.startswith('#'): + project, version = line.split('==') + if not version: + raise ValueError("Unexpected syntax '{0}'".format(line)) + d[project] = version + return d + + +def print_requirements(requirements): + """Prints requirements to stdout. + + :param dict requirements: mapping from a project to its pinned version + + """ + print('\n'.join('{0}=={1}'.format(k, v) + for k, v in sorted(requirements.items()))) + + +def merge_requirements_files(*files): + """Merges multiple requirements files together and prints the result. + + Requirement files specified later in the list take precedence over earlier + files. + + :param tuple files: paths to requirements files + + """ + d = {} + for f in files: + d.update(read_file(f)) + print_requirements(d) + + +if __name__ == '__main__': + merge_requirements_files(*sys.argv[1:]) diff --git a/tools/oldest_constraints.txt b/tools/oldest_constraints.txt new file mode 100644 index 00000000000..de2b83ad8b1 --- /dev/null +++ b/tools/oldest_constraints.txt @@ -0,0 +1,51 @@ +# This file contains the oldest versions of our dependencies we say we require +# in our packages or versions we need to support to maintain compatibility with +# the versions included in the various Linux distros where we are packaged. + +# CentOS/RHEL 7 EPEL constraints +cffi==1.6.0 +chardet==2.2.1 +configobj==4.7.2 +ipaddress==1.0.16 +mock==1.0.1 +ndg-httpsclient==0.3.2 +ply==3.4 +pyasn1==0.1.9 +pycparser==2.14 +pyOpenSSL==0.13.1 +pyparsing==1.5.6 +pyRFC3339==1.0 +python-augeas==0.5.0 +six==1.9.0 +# setuptools 0.9.8 is the actual version packaged, but some other dependencies +# in this file require setuptools>=1.0 and there are no relevant changes for us +# between these versions. +setuptools==1.0.0 +urllib3==1.10.2 +zope.component==4.1.0 +zope.event==4.0.3 +zope.interface==4.0.5 + +# Debian Jessie Backports constraints +PyICU==1.8 +colorama==0.3.2 +enum34==1.0.3 +html5lib==0.999 +idna==2.0 +pbr==1.8.0 +pytz==2012rc0 + +# Our setup.py constraints +cloudflare==1.5.1 +cryptography==1.2.0 +google-api-python-client==1.5 +oauth2client==2.0 +parsedatetime==1.3 +pyparsing==1.5.5 +python-digitalocean==1.11 +requests[security]==2.4.1 + +# Ubuntu Xenial constraints +ConfigArgParse==0.10.0 +funcsigs==0.4 +zope.hookable==4.0.4 diff --git a/tools/pip_install.sh b/tools/pip_install.sh index fafd58e5490..d2aae4a43ac 100755 --- a/tools/pip_install.sh +++ b/tools/pip_install.sh @@ -1,17 +1,26 @@ -#!/bin/sh -e -# pip installs packages using pinned package versions +#!/bin/bash -e +# pip installs packages using pinned package versions. If CERTBOT_OLDEST is set +# to 1, a combination of tools/oldest_constraints.txt and +# tools/dev_constraints.txt is used, otherwise, a combination of certbot-auto's +# requirements file and tools/dev_constraints.txt is used. The other file +# always takes precedence over tools/dev_constraints.txt. # get the root of the Certbot repo -my_path=$("$(dirname $0)/readlink.py" $0) -repo_root=$(dirname $(dirname $my_path)) -requirements="$repo_root/letsencrypt-auto-source/pieces/dependency-requirements.txt" -certbot_auto_constraints=$(mktemp) -trap "rm -f $certbot_auto_constraints" EXIT -# extract pinned requirements without hashes -sed -n -e 's/^\([^[:space:]]*==[^[:space:]]*\).*$/\1/p' $requirements > $certbot_auto_constraints -dev_constraints="$(dirname $my_path)/pip_constraints.txt" +tools_dir=$(dirname $("$(dirname $0)/readlink.py" $0)) +dev_constraints="$tools_dir/dev_constraints.txt" +merge_reqs="$tools_dir/merge_requirements.py" +test_constraints=$(mktemp) +trap "rm -f $test_constraints" EXIT + +if [ "$CERTBOT_OLDEST" = 1 ]; then + cp "$tools_dir/oldest_constraints.txt" "$test_constraints" +else + repo_root=$(dirname "$tools_dir") + certbot_requirements="$repo_root/letsencrypt-auto-source/pieces/dependency-requirements.txt" + sed -n -e 's/^\([^[:space:]]*==[^[:space:]]*\).*$/\1/p' "$certbot_requirements" > "$test_constraints" +fi set -x # install the requested packages using the pinned requirements as constraints -pip install -q --constraint $certbot_auto_constraints --constraint $dev_constraints "$@" +pip install -q --constraint <("$merge_reqs" "$dev_constraints" "$test_constraints") "$@" diff --git a/tox.ini b/tox.ini index bb421daa536..6ebf681ede0 100644 --- a/tox.ini +++ b/tox.ini @@ -11,9 +11,8 @@ envlist = modification,py{26,33,34,35,36},cover,lint pip_install = {toxinidir}/tools/pip_install_editable.sh # pip installs the requested packages in editable mode and runs unit tests on # them. Each package is installed and tested in the order they are provided -# before the script moves on to the next package. If CERTBOT_NO_PIN is set not -# set to 1, packages are installed using certbot-auto's requirements file as -# constraints. +# before the script moves on to the next package. All dependencies are pinned +# to a specific version for increased stability for developers. install_and_test = {toxinidir}/tools/install_and_test.sh py26_packages = acme[dev] \ @@ -82,36 +81,7 @@ commands = {[testenv]commands} setenv = {[testenv]setenv} - CERTBOT_NO_PIN=1 -deps = - PyOpenSSL==0.13 - cffi==1.5.2 - configargparse==0.10.0 - configargparse==0.10.0 - configobj==4.7.2 - cryptography==1.2.3 - enum34==0.9.23 - google-api-python-client==1.5 - idna==2.0 - ipaddress==1.0.16 - mock==1.0.1 - ndg-httpsclient==0.3.2 - oauth2client==2.0 - parsedatetime==1.4 - pyasn1-modules==0.0.5 - pyasn1==0.1.9 - pyparsing==1.5.6 - pyrfc3339==1.0 - pytest==3.2.5 - python-augeas==0.4.1 - pytz==2012c - requests[security]==2.6.0 - setuptools==0.9.8 - six==1.9.0 - urllib3==1.10 - zope.component==4.0.2 - zope.event==4.0.1 - zope.interface==4.0.5 + CERTBOT_OLDEST=1 [testenv:py27_install] basepython = python2.7 From d6b11fea722ab71584a2bd50cb731a5f67b0e375 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 19 Dec 2017 16:16:45 -0800 Subject: [PATCH 05/21] More pip dependency resolution workarounds (#5339) * remove pyopenssl and six deps * remove outdated tox.ini dep requirement --- setup.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index ee108c51485..ce505a62ef4 100644 --- a/setup.py +++ b/setup.py @@ -30,10 +30,9 @@ def read_file(filename, encoding='utf8'): changes = read_file(os.path.join(here, 'CHANGES.rst')) version = meta['version'] -# Please update tox.ini when modifying dependency version requirements -# This package relies on requests, however, it isn't specified here to avoid -# masking the more specific request requirements in acme. See -# https://github.com/pypa/pip/issues/988 for more info. +# This package relies on PyOpenSSL, requests, and six, however, it isn't +# specified here to avoid masking the more specific request requirements in +# acme. See https://github.com/pypa/pip/issues/988 for more info. install_requires = [ 'acme=={0}'.format(version), # We technically need ConfigArgParse 0.10.0 for Python 2.6 support, but @@ -44,13 +43,11 @@ def read_file(filename, encoding='utf8'): 'cryptography>=1.2', # load_pem_x509_certificate 'mock', 'parsedatetime>=1.3', # Calendar.parseDT - 'PyOpenSSL', 'pyrfc3339', 'pytz', # For pkg_resources. >=1.0 so pip resolves it to a version cryptography # will tolerate; see #2599: 'setuptools>=1.0', - 'six', 'zope.component', 'zope.interface', ] From ed2168aaa8c8a7e1bef449e60167b53d501d173a Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 21 Dec 2017 16:55:21 -0800 Subject: [PATCH 06/21] Fix auto_tests on systems with new bootstrappers (#5348) --- letsencrypt-auto-source/tests/auto_test.py | 30 +++++++++++++++++----- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/letsencrypt-auto-source/tests/auto_test.py b/letsencrypt-auto-source/tests/auto_test.py index 2fa03105d75..156466c82df 100644 --- a/letsencrypt-auto-source/tests/auto_test.py +++ b/letsencrypt-auto-source/tests/auto_test.py @@ -30,6 +30,10 @@ def tests_dir(): from build import build as build_le_auto +BOOTSTRAP_FILENAME = 'certbot-auto-bootstrap-version.txt' +"""Name of the file where certbot-auto saves its bootstrap version.""" + + class RequestHandler(BaseHTTPRequestHandler): """An HTTPS request handler which is quiet and serves a specific folder.""" @@ -296,17 +300,31 @@ def test_successes(self): def test_phase2_upgrade(self): """Test a phase-2 upgrade without a phase-1 upgrade.""" - with temp_paths() as (le_auto_path, venv_dir): - resources = {'certbot/json': dumps({'releases': {'99.9.9': None}}), - 'v99.9.9/letsencrypt-auto': self.NEW_LE_AUTO, - 'v99.9.9/letsencrypt-auto.sig': self.NEW_LE_AUTO_SIG} - with serving(resources) as base_url: + resources = {'certbot/json': dumps({'releases': {'99.9.9': None}}), + 'v99.9.9/letsencrypt-auto': self.NEW_LE_AUTO, + 'v99.9.9/letsencrypt-auto.sig': self.NEW_LE_AUTO_SIG} + with serving(resources) as base_url: + pip_find_links=join(tests_dir(), 'fake-letsencrypt', 'dist') + with temp_paths() as (le_auto_path, venv_dir): + install_le_auto(self.NEW_LE_AUTO, le_auto_path) + + # Create venv saving the correct bootstrap script version + out, err = run_le_auto(le_auto_path, venv_dir, base_url, + PIP_FIND_LINKS=pip_find_links) + self.assertFalse('Upgrading certbot-auto ' in out) + self.assertTrue('Creating virtual environment...' in out) + with open(join(venv_dir, BOOTSTRAP_FILENAME)) as f: + bootstrap_version = f.read() + + # Create a new venv with an old letsencrypt version + with temp_paths() as (le_auto_path, venv_dir): venv_bin = join(venv_dir, 'bin') makedirs(venv_bin) set_le_script_version(venv_dir, '0.0.1') + with open(join(venv_dir, BOOTSTRAP_FILENAME), 'w') as f: + f.write(bootstrap_version) install_le_auto(self.NEW_LE_AUTO, le_auto_path) - pip_find_links=join(tests_dir(), 'fake-letsencrypt', 'dist') out, err = run_le_auto(le_auto_path, venv_dir, base_url, PIP_FIND_LINKS=pip_find_links) From 5388842e5b3868e29caf545fb771a23e7fce4143 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 3 Jan 2018 17:49:22 -0800 Subject: [PATCH 07/21] Fix pytest on macOS in Travis (#5360) * Add tools/pytest.sh * pass TRAVIS through in tox.ini * Use tools/pytest.sh to run pytest * Add quiet to pytest.ini * ignore pytest cache --- .gitignore | 3 +++ pytest.ini | 2 ++ tools/install_and_test.sh | 2 +- tools/pytest.sh | 15 +++++++++++++++ tox.cover.sh | 3 ++- tox.ini | 9 +++++++++ 6 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 pytest.ini create mode 100755 tools/pytest.sh diff --git a/.gitignore b/.gitignore index b63e40d1cbc..e018cf938db 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,6 @@ tests/letstest/*.pem tests/letstest/venv/ .venv + +# pytest cache +/.cache diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000000..b64550cb72a --- /dev/null +++ b/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +addopts = --quiet diff --git a/tools/install_and_test.sh b/tools/install_and_test.sh index 25b6d548abe..0d39e0594bf 100755 --- a/tools/install_and_test.sh +++ b/tools/install_and_test.sh @@ -23,5 +23,5 @@ for requirement in "$@" ; do # See https://travis-ci.org/certbot/certbot/jobs/308774157#L1333. pkg=$(echo "$pkg" | tr - _) fi - pytest --numprocesses auto --quiet --pyargs $pkg + "$(dirname $0)/pytest.sh" --pyargs $pkg done diff --git a/tools/pytest.sh b/tools/pytest.sh new file mode 100755 index 00000000000..8e3619d5dbb --- /dev/null +++ b/tools/pytest.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Runs pytest with the provided arguments, adding --numprocesses to the command +# line. This argument is set to "auto" if the environmnent variable TRAVIS is +# not set, otherwise, it is set to 2. This works around +# https://github.com/pytest-dev/pytest-xdist/issues/9. Currently every Travis +# environnment provides two cores. See +# https://docs.travis-ci.com/user/reference/overview/#Virtualization-environments. + +if ${TRAVIS:-false}; then + NUMPROCESSES="2" +else + NUMPROCESSES="auto" +fi + +pytest --numprocesses "$NUMPROCESSES" "$@" diff --git a/tox.cover.sh b/tox.cover.sh index 2b5a3cf19a9..bc0e5a8bf83 100755 --- a/tox.cover.sh +++ b/tox.cover.sh @@ -51,7 +51,8 @@ cover () { fi pkg_dir=$(echo "$1" | tr _ -) - pytest --cov "$pkg_dir" --cov-append --cov-report= --numprocesses auto --pyargs "$1" + pytest="$(dirname $0)/tools/pytest.sh" + "$pytest" --cov "$pkg_dir" --cov-append --cov-report= --pyargs "$1" coverage report --fail-under="$min" --include="$pkg_dir/*" --show-missing } diff --git a/tox.ini b/tox.ini index 6ebf681ede0..20f5cda32d2 100644 --- a/tox.ini +++ b/tox.ini @@ -61,6 +61,7 @@ commands = deps = setuptools==36.8.0 wheel==0.29.0 +passenv = TRAVIS [testenv] commands = @@ -69,12 +70,16 @@ commands = setenv = PYTHONPATH = {toxinidir} PYTHONHASHSEED = 0 +passenv = + {[testenv:py26]passenv} [testenv:py33] commands = {[testenv]commands} deps = wheel==0.29.0 +passenv = + {[testenv]passenv} [testenv:py27-oldest] commands = @@ -82,6 +87,8 @@ commands = setenv = {[testenv]setenv} CERTBOT_OLDEST=1 +passenv = + {[testenv]passenv} [testenv:py27_install] basepython = python2.7 @@ -93,6 +100,8 @@ basepython = python2.7 commands = {[base]install_packages} ./tox.cover.sh +passenv = + {[testenv]passenv} [testenv:lint] # recent versions of pylint do not support Python 2.6 (#97, #187) From a7d00ee21b454115fc0ce831b13f7902d4b62c37 Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Thu, 4 Jan 2018 13:59:29 -0800 Subject: [PATCH 08/21] print as a string (#5359) --- certbot-nginx/certbot_nginx/configurator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index e9d4e36d454..8af474c5ea3 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -182,7 +182,7 @@ def deploy_cert(self, domain, cert_path, key_path, self.parser.add_server_directives(vhost, cert_directives, replace=True) logger.info("Deployed Certificate to VirtualHost %s for %s", - vhost.filep, vhost.names) + vhost.filep, ", ".join(vhost.names)) self.save_notes += ("Changed vhost at %s with addresses of %s\n" % (vhost.filep, From a3a66cd25d8340e982481e7adf4a521c09f0f35e Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Fri, 5 Jan 2018 00:36:16 +0200 Subject: [PATCH 09/21] Use apache2ctl modules for Gentoo systems. (#5349) * Do not call Apache binary for module reset in cleanup() * Use apache2ctl modules for Gentoo --- .../certbot_apache/override_gentoo.py | 8 +++ .../certbot_apache/tests/gentoo_test.py | 49 +++++++++++++++++-- 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/certbot-apache/certbot_apache/override_gentoo.py b/certbot-apache/certbot_apache/override_gentoo.py index d4d4e96b949..92f1d4a2099 100644 --- a/certbot-apache/certbot_apache/override_gentoo.py +++ b/certbot-apache/certbot_apache/override_gentoo.py @@ -49,6 +49,7 @@ def __init__(self, *args, **kwargs): def update_runtime_variables(self): """ Override for update_runtime_variables for custom parsing """ self.parse_sysconfig_var() + self.update_modules() def parse_sysconfig_var(self): """ Parses Apache CLI options from Gentoo configuration file """ @@ -56,3 +57,10 @@ def parse_sysconfig_var(self): "APACHE2_OPTS") for k in defines.keys(): self.variables[k] = defines[k] + + def update_modules(self): + """Get loaded modules from httpd process, and add them to DOM""" + mod_cmd = [self.configurator.constant("apache_cmd"), "modules"] + matches = self.parse_from_subprocess(mod_cmd, r"(.*)_module") + for mod in matches: + self.add_mod(mod.strip()) diff --git a/certbot-apache/certbot_apache/tests/gentoo_test.py b/certbot-apache/certbot_apache/tests/gentoo_test.py index 0f2b96818df..cfbaffac71d 100644 --- a/certbot-apache/certbot_apache/tests/gentoo_test.py +++ b/certbot-apache/certbot_apache/tests/gentoo_test.py @@ -2,6 +2,8 @@ import os import unittest +import mock + from certbot_apache import override_gentoo from certbot_apache import obj from certbot_apache.tests import util @@ -46,9 +48,10 @@ def setUp(self): # pylint: disable=arguments-differ config_root=config_root, vhost_root=vhost_root) - self.config = util.get_apache_configurator( - self.config_path, self.vhost_path, self.config_dir, self.work_dir, - os_info="gentoo") + with mock.patch("certbot_apache.override_gentoo.GentooParser.update_runtime_variables"): + self.config = util.get_apache_configurator( + self.config_path, self.vhost_path, self.config_dir, self.work_dir, + os_info="gentoo") self.vh_truth = get_vh_truth( self.temp_dir, "gentoo_apache/apache") @@ -78,9 +81,47 @@ def test_get_sysconfig_vars(self): self.config.parser.apacheconfig_filep = os.path.realpath( os.path.join(self.config.parser.root, "../conf.d/apache2")) self.config.parser.variables = {} - self.config.parser.update_runtime_variables() + with mock.patch("certbot_apache.override_gentoo.GentooParser.update_modules"): + self.config.parser.update_runtime_variables() for define in defines: self.assertTrue(define in self.config.parser.variables.keys()) + @mock.patch("certbot_apache.parser.ApacheParser.parse_from_subprocess") + def test_no_binary_configdump(self, mock_subprocess): + """Make sure we don't call binary dumps other than modules from Apache + as this is not supported in Gentoo currently""" + + with mock.patch("certbot_apache.override_gentoo.GentooParser.update_modules"): + self.config.parser.update_runtime_variables() + self.config.parser.reset_modules() + self.assertFalse(mock_subprocess.called) + + self.config.parser.update_runtime_variables() + self.config.parser.reset_modules() + self.assertTrue(mock_subprocess.called) + + @mock.patch("certbot_apache.parser.ApacheParser._get_runtime_cfg") + def test_opportunistic_httpd_runtime_parsing(self, mock_get): + mod_val = ( + 'Loaded Modules:\n' + ' mock_module (static)\n' + ' another_module (static)\n' + ) + def mock_get_cfg(command): + """Mock httpd process stdout""" + if command == ['apache2ctl', 'modules']: + return mod_val + mock_get.side_effect = mock_get_cfg + self.config.parser.modules = set() + + with mock.patch("certbot.util.get_os_info") as mock_osi: + # Make sure we have the have the CentOS httpd constants + mock_osi.return_value = ("gentoo", "123") + self.config.parser.update_runtime_variables() + + self.assertEquals(mock_get.call_count, 1) + self.assertEquals(len(self.config.parser.modules), 4) + self.assertTrue("mod_another.c" in self.config.parser.modules) + if __name__ == "__main__": unittest.main() # pragma: no cover From a1713c0b79b99108ae3a1233cb3e3dc3bef2908a Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Fri, 5 Jan 2018 21:08:38 +0200 Subject: [PATCH 10/21] Broader git ignore for pytest cache files (#5361) Make gitignore take pytest cache directories in to account, even if they reside in subdirectories. If pytest is run for a certain module, ie. `pytest certbot-apache` the cache directory is created under `certbot-apache` directory. --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e018cf938db..a01d2e1c73d 100644 --- a/.gitignore +++ b/.gitignore @@ -37,4 +37,4 @@ tests/letstest/venv/ .venv # pytest cache -/.cache +.cache From 18f6deada8dca329c1a797bcf5b88dbdcbd18cf7 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 5 Jan 2018 19:27:00 -0800 Subject: [PATCH 11/21] Fix letsencrypt-auto name and long forms of -n (#5375) --- letsencrypt-auto-source/letsencrypt-auto | 8 +++++--- letsencrypt-auto-source/letsencrypt-auto.template | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 93e3e7b83bd..46cb5182245 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -68,10 +68,12 @@ for arg in "$@" ; do NO_BOOTSTRAP=1;; --help) HELP=1;; - --noninteractive|--non-interactive|renew) - ASSUME_YES=1;; + --noninteractive|--non-interactive) + NONINTERACTIVE=1;; --quiet) QUIET=1;; + renew) + ASSUME_YES=1;; --verbose) VERBOSE=1;; -[!-]*) @@ -93,7 +95,7 @@ done if [ $BASENAME = "letsencrypt-auto" ]; then # letsencrypt-auto does not respect --help or --yes for backwards compatibility - ASSUME_YES=1 + NONINTERACTIVE=1 HELP=0 fi diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 4eef10c804a..861ef0a6ebc 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -68,10 +68,12 @@ for arg in "$@" ; do NO_BOOTSTRAP=1;; --help) HELP=1;; - --noninteractive|--non-interactive|renew) - ASSUME_YES=1;; + --noninteractive|--non-interactive) + NONINTERACTIVE=1;; --quiet) QUIET=1;; + renew) + ASSUME_YES=1;; --verbose) VERBOSE=1;; -[!-]*) @@ -93,7 +95,7 @@ done if [ $BASENAME = "letsencrypt-auto" ]; then # letsencrypt-auto does not respect --help or --yes for backwards compatibility - ASSUME_YES=1 + NONINTERACTIVE=1 HELP=0 fi From 8585cdd86190c5da753cae1a691e66e586fd2bde Mon Sep 17 00:00:00 2001 From: ohemorange Date: Mon, 8 Jan 2018 13:57:04 -0800 Subject: [PATCH 12/21] Deprecate Python2.6 by using Python3 on CentOS/RHEL 6 (#5329) * If there's no python or there's only python2.6 on red hat systems, install python3 * Always check for python2.6 * address style, documentation, nits * factor out all initialization code * fix up python version return value when no python installed * add no python error and exit * document DeterminePythonVersion parameters * build letsencrypt-auto * close brace * build leauto * fix syntax errors * set USE_PYTHON_3 for all cases * rip out NOCRASH * replace NOCRASH, update LE_PYTHON set logic * use built-in venv for py3 * switch to LE_PYTHON not affecting bootstrap selection and not overwriting LE_PYTHON * python3ify fetch.py * get fetch.py working with python2 and 3 * don't verify server certificates in fetch.py HttpsGetter * Use SSLContext and an environment variable so that our tests continue to never verify server certificates. * typo * build * remove commented out code * address review comments * add documentation for YES_FLAG and QUIET_FLAG * Add tests to centos6 Dockerfile to make sure we install python3 if and only if appropriate to do so. --- letsencrypt-auto-source/Dockerfile.centos6 | 3 +- letsencrypt-auto-source/letsencrypt-auto | 205 +++++++++++++----- .../letsencrypt-auto.template | 67 ++++-- .../pieces/bootstrappers/rpm_common.sh | 75 +------ .../pieces/bootstrappers/rpm_common_base.sh | 78 +++++++ .../pieces/bootstrappers/rpm_python3.sh | 23 ++ letsencrypt-auto-source/pieces/fetch.py | 34 ++- letsencrypt-auto-source/tests/auto_test.py | 2 + .../tests/centos6_tests.sh | 65 ++++++ 9 files changed, 405 insertions(+), 147 deletions(-) create mode 100644 letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh create mode 100644 letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh create mode 100644 letsencrypt-auto-source/tests/centos6_tests.sh diff --git a/letsencrypt-auto-source/Dockerfile.centos6 b/letsencrypt-auto-source/Dockerfile.centos6 index 8c1a4b353c1..47eb48f50c4 100644 --- a/letsencrypt-auto-source/Dockerfile.centos6 +++ b/letsencrypt-auto-source/Dockerfile.centos6 @@ -33,4 +33,5 @@ COPY . /home/lea/certbot/letsencrypt-auto-source USER lea WORKDIR /home/lea -CMD ["pytest", "-v", "-s", "certbot/letsencrypt-auto-source/tests"] +RUN sudo chmod +x certbot/letsencrypt-auto-source/tests/centos6_tests.sh +CMD sudo certbot/letsencrypt-auto-source/tests/centos6_tests.sh diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 46cb5182245..f1361d8eae9 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -246,15 +246,29 @@ DeprecationBootstrap() { fi } - +# Sets LE_PYTHON to Python version string and PYVER to the first two +# digits of the python version DeterminePythonVersion() { - for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do - # Break (while keeping the LE_PYTHON value) if found. - $EXISTS "$LE_PYTHON" > /dev/null && break - done + # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python + if [ -n "$USE_PYTHON_3" ]; then + for LE_PYTHON in "$LE_PYTHON" python3; do + # Break (while keeping the LE_PYTHON value) if found. + $EXISTS "$LE_PYTHON" > /dev/null && break + done + else + for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do + # Break (while keeping the LE_PYTHON value) if found. + $EXISTS "$LE_PYTHON" > /dev/null && break + done + fi if [ "$?" != "0" ]; then - error "Cannot find any Pythons; please install one!" - exit 1 + if [ "$1" != "NOCRASH" ]; then + error "Cannot find any Pythons; please install one!" + exit 1 + else + PYVER=0 + return 0 + fi fi export LE_PYTHON @@ -386,23 +400,19 @@ BootstrapDebCommon() { fi } -# If new packages are installed by BootstrapRpmCommon below, this version -# number must be increased. -BOOTSTRAP_RPM_COMMON_VERSION=1 - -BootstrapRpmCommon() { - # Tested with: - # - Fedora 20, 21, 22, 23 (x64) - # - Centos 7 (x64: on DigitalOcean droplet) - # - CentOS 7 Minimal install in a Hyper-V VM - # - CentOS 6 (EPEL must be installed manually) +# If new packages are installed by BootstrapRpmCommonBase below, version +# numbers in rpm_common.sh and rpm_python3.sh must be increased. +# Sets TOOL to the name of the package manager +# Sets appropriate values for YES_FLAG and QUIET_FLAG based on $ASSUME_YES and $QUIET_FLAG. +# Enables EPEL if applicable and possible. +InitializeRPMCommonBase() { if type dnf 2>/dev/null then - tool=dnf + TOOL=dnf elif type yum 2>/dev/null then - tool=yum + TOOL=yum else error "Neither yum nor dnf found. Aborting bootstrap!" @@ -410,15 +420,15 @@ BootstrapRpmCommon() { fi if [ "$ASSUME_YES" = 1 ]; then - yes_flag="-y" + YES_FLAG="-y" fi if [ "$QUIET" = 1 ]; then QUIET_FLAG='--quiet' fi - if ! $tool list *virtualenv >/dev/null 2>&1; then + if ! $TOOL list *virtualenv >/dev/null 2>&1; then echo "To use Certbot, packages from the EPEL repository need to be installed." - if ! $tool list epel-release >/dev/null 2>&1; then + if ! $TOOL list epel-release >/dev/null 2>&1; then error "Enable the EPEL repository and try running Certbot again." exit 1 fi @@ -430,11 +440,17 @@ BootstrapRpmCommon() { /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 seconds..." sleep 1s fi - if ! $tool install $yes_flag $QUIET_FLAG epel-release; then + if ! $TOOL install $YES_FLAG $QUIET_FLAG epel-release; then error "Could not enable EPEL. Aborting bootstrap!" exit 1 fi fi +} + +BootstrapRpmCommonBase() { + # Arguments: whitespace-delimited python packages to install + + InitializeRPMCommonBase # This call is superfluous in practice pkgs=" gcc @@ -446,10 +462,39 @@ BootstrapRpmCommon() { ca-certificates " - # Most RPM distros use the "python" or "python-" naming convention. Let's try that first. - if $tool list python >/dev/null 2>&1; then + # Add the python packages + pkgs="$pkgs + $1 + " + + if $TOOL list installed "httpd" >/dev/null 2>&1; then pkgs="$pkgs - python + mod_ssl + " + fi + + if ! $TOOL install $YES_FLAG $QUIET_FLAG $pkgs; then + error "Could not install OS dependencies. Aborting bootstrap!" + exit 1 + fi +} + +# If new packages are installed by BootstrapRpmCommon below, this version +# number must be increased. +BOOTSTRAP_RPM_COMMON_VERSION=1 + +BootstrapRpmCommon() { + # Tested with: + # - Fedora 20, 21, 22, 23 (x64) + # - Centos 7 (x64: on DigitalOcean droplet) + # - CentOS 7 Minimal install in a Hyper-V VM + # - CentOS 6 + + InitializeRPMCommonBase + + # Most RPM distros use the "python" or "python-" naming convention. Let's try that first. + if $TOOL list python >/dev/null 2>&1; then + python_pkgs="$python python-devel python-virtualenv python-tools @@ -457,9 +502,8 @@ BootstrapRpmCommon() { " # Fedora 26 starts to use the prefix python2 for python2 based packages. # this elseif is theoretically for any Fedora over version 26: - elif $tool list python2 >/dev/null 2>&1; then - pkgs="$pkgs - python2 + elif $TOOL list python2 >/dev/null 2>&1; then + python_pkgs="$python2 python2-libs python2-setuptools python2-devel @@ -470,8 +514,7 @@ BootstrapRpmCommon() { # Some distros and older versions of current distros use a "python27" # instead of the "python" or "python-" naming convention. else - pkgs="$pkgs - python27 + python_pkgs="$python27 python27-devel python27-virtualenv python27-tools @@ -479,16 +522,31 @@ BootstrapRpmCommon() { " fi - if $tool list installed "httpd" >/dev/null 2>&1; then - pkgs="$pkgs - mod_ssl - " - fi + BootstrapRpmCommonBase "$python_pkgs" +} - if ! $tool install $yes_flag $QUIET_FLAG $pkgs; then - error "Could not install OS dependencies. Aborting bootstrap!" +# If new packages are installed by BootstrapRpmPython3 below, this version +# number must be increased. +BOOTSTRAP_RPM_PYTHON3_VERSION=1 + +BootstrapRpmPython3() { + # Tested with: + # - CentOS 6 + + InitializeRPMCommonBase + + # EPEL uses python34 + if $TOOL list python34 >/dev/null 2>&1; then + python_pkgs="python34 + python34-devel + python34-tools + " + else + error "No supported Python package available to install. Aborting bootstrap!" exit 1 fi + + BootstrapRpmCommonBase "$python_pkgs" } # If new packages are installed by BootstrapSuseCommon below, this version @@ -717,11 +775,24 @@ elif [ -f /etc/mageia-release ]; then } BOOTSTRAP_VERSION="BootstrapMageiaCommon $BOOTSTRAP_MAGEIA_COMMON_VERSION" elif [ -f /etc/redhat-release ]; then - Bootstrap() { - BootstrapMessage "RedHat-based OSes" - BootstrapRpmCommon - } - BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" + prev_le_python="$LE_PYTHON" + unset LE_PYTHON + DeterminePythonVersion "NOCRASH" + if [ "$PYVER" -eq 26 ]; then + Bootstrap() { + BootstrapMessage "RedHat-based OSes that will use Python3" + BootstrapRpmPython3 + } + USE_PYTHON_3=1 + BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION" + else + Bootstrap() { + BootstrapMessage "RedHat-based OSes" + BootstrapRpmCommon + } + BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" + fi + export LE_PYTHON="$prev_le_python" elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then Bootstrap() { BootstrapMessage "openSUSE-based OSes" @@ -860,10 +931,18 @@ if [ "$1" = "--le-auto-phase2" ]; then say "Creating virtual environment..." DeterminePythonVersion rm -rf "$VENV_PATH" - if [ "$VERBOSE" = 1 ]; then - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" + if [ "$PYVER" -le 27 ]; then + if [ "$VERBOSE" = 1 ]; then + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" + else + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null + fi else - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null + if [ "$VERBOSE" = 1 ]; then + "$LE_PYTHON" -m venv "$VENV_PATH" + else + "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null + fi fi if [ -n "$BOOTSTRAP_VERSION" ]; then @@ -1358,17 +1437,22 @@ On failure, return non-zero. """ -from __future__ import print_function +from __future__ import print_function, unicode_literals from distutils.version import LooseVersion from json import loads from os import devnull, environ from os.path import dirname, join import re +import ssl from subprocess import check_call, CalledProcessError from sys import argv, exit -from urllib2 import build_opener, HTTPHandler, HTTPSHandler -from urllib2 import HTTPError, URLError +try: + from urllib2 import build_opener, HTTPHandler, HTTPSHandler + from urllib2 import HTTPError, URLError +except ImportError: + from urllib.request import build_opener, HTTPHandler, HTTPSHandler + from urllib.error import HTTPError, URLError PUBLIC_KEY = environ.get('LE_AUTO_PUBLIC_KEY', """-----BEGIN PUBLIC KEY----- MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6MR8W/galdxnpGqBsYbq @@ -1390,8 +1474,11 @@ class HttpsGetter(object): def __init__(self): """Build an HTTPS opener.""" # Based on pip 1.4.1's URLOpener - # This verifies certs on only Python >=2.7.9. - self._opener = build_opener(HTTPSHandler()) + # This verifies certs on only Python >=2.7.9, and when NO_CERT_VERIFY isn't set. + if environ.get('NO_CERT_VERIFY') == '1' and hasattr(ssl, 'SSLContext'): + self._opener = build_opener(HTTPSHandler(context=create_CERT_NONE_context())) + else: + self._opener = build_opener(HTTPSHandler()) # Strip out HTTPHandler to prevent MITM spoof: for handler in self._opener.handlers: if isinstance(handler, HTTPHandler): @@ -1413,7 +1500,7 @@ class HttpsGetter(object): def write(contents, dir, filename): """Write something to a file in a certain directory.""" - with open(join(dir, filename), 'w') as file: + with open(join(dir, filename), 'wb') as file: file.write(contents) @@ -1421,13 +1508,13 @@ def latest_stable_version(get): """Return the latest stable release of letsencrypt.""" metadata = loads(get( environ.get('LE_AUTO_JSON_URL', - 'https://pypi.python.org/pypi/certbot/json'))) + 'https://pypi.python.org/pypi/certbot/json')).decode('UTF-8')) # metadata['info']['version'] actually returns the latest of any kind of # release release, contrary to https://wiki.python.org/moin/PyPIJSON. # The regex is a sufficient regex for picking out prereleases for most # packages, LE included. return str(max(LooseVersion(r) for r - in metadata['releases'].iterkeys() + in iter(metadata['releases'].keys()) if re.match('^[0-9.]+$', r))) @@ -1444,7 +1531,7 @@ def verified_new_le_auto(get, tag, temp_dir): 'letsencrypt-auto-source/') % tag write(get(le_auto_dir + 'letsencrypt-auto'), temp_dir, 'letsencrypt-auto') write(get(le_auto_dir + 'letsencrypt-auto.sig'), temp_dir, 'letsencrypt-auto.sig') - write(PUBLIC_KEY, temp_dir, 'public_key.pem') + write(PUBLIC_KEY.encode('UTF-8'), temp_dir, 'public_key.pem') try: with open(devnull, 'w') as dev_null: check_call(['openssl', 'dgst', '-sha256', '-verify', @@ -1459,6 +1546,14 @@ def verified_new_le_auto(get, tag, temp_dir): "certbot-auto.", exc) +def create_CERT_NONE_context(): + """Create a SSLContext object to not check hostname.""" + # PROTOCOL_TLS isn't available before 2.7.13 but this code is for 2.7.9+, so use this. + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.verify_mode = ssl.CERT_NONE + return context + + def main(): get = HttpsGetter().get flag = argv[1] diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 861ef0a6ebc..f4c1b202f4d 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -246,15 +246,29 @@ DeprecationBootstrap() { fi } - +# Sets LE_PYTHON to Python version string and PYVER to the first two +# digits of the python version DeterminePythonVersion() { - for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do - # Break (while keeping the LE_PYTHON value) if found. - $EXISTS "$LE_PYTHON" > /dev/null && break - done + # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python + if [ -n "$USE_PYTHON_3" ]; then + for LE_PYTHON in "$LE_PYTHON" python3; do + # Break (while keeping the LE_PYTHON value) if found. + $EXISTS "$LE_PYTHON" > /dev/null && break + done + else + for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do + # Break (while keeping the LE_PYTHON value) if found. + $EXISTS "$LE_PYTHON" > /dev/null && break + done + fi if [ "$?" != "0" ]; then - error "Cannot find any Pythons; please install one!" - exit 1 + if [ "$1" != "NOCRASH" ]; then + error "Cannot find any Pythons; please install one!" + exit 1 + else + PYVER=0 + return 0 + fi fi export LE_PYTHON @@ -267,7 +281,9 @@ DeterminePythonVersion() { } {{ bootstrappers/deb_common.sh }} +{{ bootstrappers/rpm_common_base.sh }} {{ bootstrappers/rpm_common.sh }} +{{ bootstrappers/rpm_python3.sh }} {{ bootstrappers/suse_common.sh }} {{ bootstrappers/arch_common.sh }} {{ bootstrappers/gentoo_common.sh }} @@ -298,11 +314,24 @@ elif [ -f /etc/mageia-release ]; then } BOOTSTRAP_VERSION="BootstrapMageiaCommon $BOOTSTRAP_MAGEIA_COMMON_VERSION" elif [ -f /etc/redhat-release ]; then - Bootstrap() { - BootstrapMessage "RedHat-based OSes" - BootstrapRpmCommon - } - BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" + prev_le_python="$LE_PYTHON" + unset LE_PYTHON + DeterminePythonVersion "NOCRASH" + if [ "$PYVER" -eq 26 ]; then + Bootstrap() { + BootstrapMessage "RedHat-based OSes that will use Python3" + BootstrapRpmPython3 + } + USE_PYTHON_3=1 + BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION" + else + Bootstrap() { + BootstrapMessage "RedHat-based OSes" + BootstrapRpmCommon + } + BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" + fi + export LE_PYTHON="$prev_le_python" elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then Bootstrap() { BootstrapMessage "openSUSE-based OSes" @@ -441,10 +470,18 @@ if [ "$1" = "--le-auto-phase2" ]; then say "Creating virtual environment..." DeterminePythonVersion rm -rf "$VENV_PATH" - if [ "$VERBOSE" = 1 ]; then - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" + if [ "$PYVER" -le 27 ]; then + if [ "$VERBOSE" = 1 ]; then + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" + else + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null + fi else - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null + if [ "$VERBOSE" = 1 ]; then + "$LE_PYTHON" -m venv "$VENV_PATH" + else + "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null + fi fi if [ -n "$BOOTSTRAP_VERSION" ]; then diff --git a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh index 5b120a9e626..80d55a3931e 100755 --- a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh @@ -7,61 +7,13 @@ BootstrapRpmCommon() { # - Fedora 20, 21, 22, 23 (x64) # - Centos 7 (x64: on DigitalOcean droplet) # - CentOS 7 Minimal install in a Hyper-V VM - # - CentOS 6 (EPEL must be installed manually) + # - CentOS 6 - if type dnf 2>/dev/null - then - tool=dnf - elif type yum 2>/dev/null - then - tool=yum - - else - error "Neither yum nor dnf found. Aborting bootstrap!" - exit 1 - fi - - if [ "$ASSUME_YES" = 1 ]; then - yes_flag="-y" - fi - if [ "$QUIET" = 1 ]; then - QUIET_FLAG='--quiet' - fi - - if ! $tool list *virtualenv >/dev/null 2>&1; then - echo "To use Certbot, packages from the EPEL repository need to be installed." - if ! $tool list epel-release >/dev/null 2>&1; then - error "Enable the EPEL repository and try running Certbot again." - exit 1 - fi - if [ "$ASSUME_YES" = 1 ]; then - /bin/echo -n "Enabling the EPEL repository in 3 seconds..." - sleep 1s - /bin/echo -ne "\e[0K\rEnabling the EPEL repository in 2 seconds..." - sleep 1s - /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 seconds..." - sleep 1s - fi - if ! $tool install $yes_flag $QUIET_FLAG epel-release; then - error "Could not enable EPEL. Aborting bootstrap!" - exit 1 - fi - fi - - pkgs=" - gcc - augeas-libs - openssl - openssl-devel - libffi-devel - redhat-rpm-config - ca-certificates - " + InitializeRPMCommonBase # Most RPM distros use the "python" or "python-" naming convention. Let's try that first. - if $tool list python >/dev/null 2>&1; then - pkgs="$pkgs - python + if $TOOL list python >/dev/null 2>&1; then + python_pkgs="$python python-devel python-virtualenv python-tools @@ -69,9 +21,8 @@ BootstrapRpmCommon() { " # Fedora 26 starts to use the prefix python2 for python2 based packages. # this elseif is theoretically for any Fedora over version 26: - elif $tool list python2 >/dev/null 2>&1; then - pkgs="$pkgs - python2 + elif $TOOL list python2 >/dev/null 2>&1; then + python_pkgs="$python2 python2-libs python2-setuptools python2-devel @@ -82,8 +33,7 @@ BootstrapRpmCommon() { # Some distros and older versions of current distros use a "python27" # instead of the "python" or "python-" naming convention. else - pkgs="$pkgs - python27 + python_pkgs="$python27 python27-devel python27-virtualenv python27-tools @@ -91,14 +41,5 @@ BootstrapRpmCommon() { " fi - if $tool list installed "httpd" >/dev/null 2>&1; then - pkgs="$pkgs - mod_ssl - " - fi - - if ! $tool install $yes_flag $QUIET_FLAG $pkgs; then - error "Could not install OS dependencies. Aborting bootstrap!" - exit 1 - fi + BootstrapRpmCommonBase "$python_pkgs" } diff --git a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh new file mode 100644 index 00000000000..d7a9f3133c8 --- /dev/null +++ b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh @@ -0,0 +1,78 @@ +# If new packages are installed by BootstrapRpmCommonBase below, version +# numbers in rpm_common.sh and rpm_python3.sh must be increased. + +# Sets TOOL to the name of the package manager +# Sets appropriate values for YES_FLAG and QUIET_FLAG based on $ASSUME_YES and $QUIET_FLAG. +# Enables EPEL if applicable and possible. +InitializeRPMCommonBase() { + if type dnf 2>/dev/null + then + TOOL=dnf + elif type yum 2>/dev/null + then + TOOL=yum + + else + error "Neither yum nor dnf found. Aborting bootstrap!" + exit 1 + fi + + if [ "$ASSUME_YES" = 1 ]; then + YES_FLAG="-y" + fi + if [ "$QUIET" = 1 ]; then + QUIET_FLAG='--quiet' + fi + + if ! $TOOL list *virtualenv >/dev/null 2>&1; then + echo "To use Certbot, packages from the EPEL repository need to be installed." + if ! $TOOL list epel-release >/dev/null 2>&1; then + error "Enable the EPEL repository and try running Certbot again." + exit 1 + fi + if [ "$ASSUME_YES" = 1 ]; then + /bin/echo -n "Enabling the EPEL repository in 3 seconds..." + sleep 1s + /bin/echo -ne "\e[0K\rEnabling the EPEL repository in 2 seconds..." + sleep 1s + /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 seconds..." + sleep 1s + fi + if ! $TOOL install $YES_FLAG $QUIET_FLAG epel-release; then + error "Could not enable EPEL. Aborting bootstrap!" + exit 1 + fi + fi +} + +BootstrapRpmCommonBase() { + # Arguments: whitespace-delimited python packages to install + + InitializeRPMCommonBase # This call is superfluous in practice + + pkgs=" + gcc + augeas-libs + openssl + openssl-devel + libffi-devel + redhat-rpm-config + ca-certificates + " + + # Add the python packages + pkgs="$pkgs + $1 + " + + if $TOOL list installed "httpd" >/dev/null 2>&1; then + pkgs="$pkgs + mod_ssl + " + fi + + if ! $TOOL install $YES_FLAG $QUIET_FLAG $pkgs; then + error "Could not install OS dependencies. Aborting bootstrap!" + exit 1 + fi +} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh b/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh new file mode 100644 index 00000000000..b011a7235b2 --- /dev/null +++ b/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh @@ -0,0 +1,23 @@ +# If new packages are installed by BootstrapRpmPython3 below, this version +# number must be increased. +BOOTSTRAP_RPM_PYTHON3_VERSION=1 + +BootstrapRpmPython3() { + # Tested with: + # - CentOS 6 + + InitializeRPMCommonBase + + # EPEL uses python34 + if $TOOL list python34 >/dev/null 2>&1; then + python_pkgs="python34 + python34-devel + python34-tools + " + else + error "No supported Python package available to install. Aborting bootstrap!" + exit 1 + fi + + BootstrapRpmCommonBase "$python_pkgs" +} diff --git a/letsencrypt-auto-source/pieces/fetch.py b/letsencrypt-auto-source/pieces/fetch.py index 8f34351c941..ae72a299b48 100644 --- a/letsencrypt-auto-source/pieces/fetch.py +++ b/letsencrypt-auto-source/pieces/fetch.py @@ -11,17 +11,22 @@ """ -from __future__ import print_function +from __future__ import print_function, unicode_literals from distutils.version import LooseVersion from json import loads from os import devnull, environ from os.path import dirname, join import re +import ssl from subprocess import check_call, CalledProcessError from sys import argv, exit -from urllib2 import build_opener, HTTPHandler, HTTPSHandler -from urllib2 import HTTPError, URLError +try: + from urllib2 import build_opener, HTTPHandler, HTTPSHandler + from urllib2 import HTTPError, URLError +except ImportError: + from urllib.request import build_opener, HTTPHandler, HTTPSHandler + from urllib.error import HTTPError, URLError PUBLIC_KEY = environ.get('LE_AUTO_PUBLIC_KEY', """-----BEGIN PUBLIC KEY----- MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6MR8W/galdxnpGqBsYbq @@ -43,8 +48,11 @@ class HttpsGetter(object): def __init__(self): """Build an HTTPS opener.""" # Based on pip 1.4.1's URLOpener - # This verifies certs on only Python >=2.7.9. - self._opener = build_opener(HTTPSHandler()) + # This verifies certs on only Python >=2.7.9, and when NO_CERT_VERIFY isn't set. + if environ.get('NO_CERT_VERIFY') == '1' and hasattr(ssl, 'SSLContext'): + self._opener = build_opener(HTTPSHandler(context=create_CERT_NONE_context())) + else: + self._opener = build_opener(HTTPSHandler()) # Strip out HTTPHandler to prevent MITM spoof: for handler in self._opener.handlers: if isinstance(handler, HTTPHandler): @@ -66,7 +74,7 @@ def get(self, url): def write(contents, dir, filename): """Write something to a file in a certain directory.""" - with open(join(dir, filename), 'w') as file: + with open(join(dir, filename), 'wb') as file: file.write(contents) @@ -74,13 +82,13 @@ def latest_stable_version(get): """Return the latest stable release of letsencrypt.""" metadata = loads(get( environ.get('LE_AUTO_JSON_URL', - 'https://pypi.python.org/pypi/certbot/json'))) + 'https://pypi.python.org/pypi/certbot/json')).decode('UTF-8')) # metadata['info']['version'] actually returns the latest of any kind of # release release, contrary to https://wiki.python.org/moin/PyPIJSON. # The regex is a sufficient regex for picking out prereleases for most # packages, LE included. return str(max(LooseVersion(r) for r - in metadata['releases'].iterkeys() + in iter(metadata['releases'].keys()) if re.match('^[0-9.]+$', r))) @@ -97,7 +105,7 @@ def verified_new_le_auto(get, tag, temp_dir): 'letsencrypt-auto-source/') % tag write(get(le_auto_dir + 'letsencrypt-auto'), temp_dir, 'letsencrypt-auto') write(get(le_auto_dir + 'letsencrypt-auto.sig'), temp_dir, 'letsencrypt-auto.sig') - write(PUBLIC_KEY, temp_dir, 'public_key.pem') + write(PUBLIC_KEY.encode('UTF-8'), temp_dir, 'public_key.pem') try: with open(devnull, 'w') as dev_null: check_call(['openssl', 'dgst', '-sha256', '-verify', @@ -112,6 +120,14 @@ def verified_new_le_auto(get, tag, temp_dir): "certbot-auto.", exc) +def create_CERT_NONE_context(): + """Create a SSLContext object to not check hostname.""" + # PROTOCOL_TLS isn't available before 2.7.13 but this code is for 2.7.9+, so use this. + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.verify_mode = ssl.CERT_NONE + return context + + def main(): get = HttpsGetter().get flag = argv[1] diff --git a/letsencrypt-auto-source/tests/auto_test.py b/letsencrypt-auto-source/tests/auto_test.py index 156466c82df..d187452a125 100644 --- a/letsencrypt-auto-source/tests/auto_test.py +++ b/letsencrypt-auto-source/tests/auto_test.py @@ -202,6 +202,7 @@ def run_le_auto(le_auto_path, venv_dir, base_url, **kwargs): Z5o/NDiQB11m91yNB0MmPYY9QSbnOA9j7IaaC97AwRLuwXY+/R2ablTcxurWou68 iQIDAQAB -----END PUBLIC KEY-----""", + NO_CERT_VERIFY='1', **kwargs) env.update(d) return out_and_err( @@ -349,6 +350,7 @@ def test_openssl_failure(self): self.assertTrue("Couldn't verify signature of downloaded " "certbot-auto." in exc.output) else: + print(out) self.fail('Signature check on certbot-auto erroneously passed.') def test_pip_failure(self): diff --git a/letsencrypt-auto-source/tests/centos6_tests.sh b/letsencrypt-auto-source/tests/centos6_tests.sh new file mode 100644 index 00000000000..e3ebbaec55d --- /dev/null +++ b/letsencrypt-auto-source/tests/centos6_tests.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# Start by making sure your system is up-to-date: +yum update > /dev/null +yum install -y centos-release-scl > /dev/null +yum install -y python27 > /dev/null 2> /dev/null + +# we're going to modify env variables, so do this in a subshell +( +source /opt/rh/python27/enable + +# ensure python 3 isn't installed +python3 --version 2> /dev/null +RESULT=$? +if [ $RESULT -eq 0 ]; then + error "Python3 is already installed." + exit 1 +fi + +# ensure python2.7 is available +python2.7 --version 2> /dev/null +RESULT=$? +if [ $RESULT -ne 0 ]; then + error "Python3 is not available." + exit 1 +fi + +# bootstrap, but don't install python 3. +certbot/letsencrypt-auto-source/letsencrypt-auto --no-self-upgrade -n > /dev/null 2> /dev/null + +# ensure python 3 isn't installed +python3 --version 2> /dev/null +RESULT=$? +if [ $RESULT -eq 0 ]; then + error "letsencrypt-auto installed Python3 even though Python2.7 is present." + exit 1 +fi + +echo "" +echo "PASSED: Did not upgrade to Python3 when Python2.7 is present." +) + +# ensure python2.7 isn't available +python2.7 --version 2> /dev/null +RESULT=$? +if [ $RESULT -eq 0 ]; then + error "Python2.7 is still available." + exit 1 +fi + +# bootstrap, this time installing python3 +certbot/letsencrypt-auto-source/letsencrypt-auto --no-self-upgrade -n > /dev/null 2> /dev/null + +# ensure python 3 is installed +python3 --version > /dev/null +RESULT=$? +if [ $RESULT -ne 0 ]; then + error "letsencrypt-auto failed to install Python3 when only Python2.6 is present." + exit 1 +fi + +echo "PASSED: Successfully upgraded to Python3 when only Python2.6 is present." +echo "" + +# test using python3 +pytest -v -s certbot/letsencrypt-auto-source/tests From 24ddc65cd4765e82067ab5fd963ffe81348e1f02 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 8 Jan 2018 17:02:20 -0800 Subject: [PATCH 13/21] Allow non-interactive revocation without deleting certificates (#5386) * Add --delete-after-revoke flags * Use delete_after_revoke value * Add delete_after_revoke unit tests * Add integration tests for delete-after-revoke. --- certbot/cli.py | 12 ++++++++++ certbot/constants.py | 1 + certbot/main.py | 8 ++++--- certbot/tests/cli_test.py | 14 +++++++++++ certbot/tests/main_test.py | 46 ++++++++++++++++++++++++++++-------- tests/boulder-integration.sh | 9 +++++-- 6 files changed, 75 insertions(+), 15 deletions(-) diff --git a/certbot/cli.py b/certbot/cli.py index 62246227847..f0fa7eb7ebe 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -1220,6 +1220,18 @@ def _create_subparsers(helpful): key=constants.REVOCATION_REASONS.get)), action=_EncodeReasonAction, default=flag_default("reason"), help="Specify reason for revoking certificate. (default: unspecified)") + helpful.add("revoke", + "--delete-after-revoke", action="store_true", + default=flag_default("delete_after_revoke"), + help="Delete certificates after revoking them.") + helpful.add("revoke", + "--no-delete-after-revoke", action="store_false", + dest="delete_after_revoke", + default=flag_default("delete_after_revoke"), + help="Do not delete certificates after revoking them. This " + "option should be used with caution because the 'renew' " + "subcommand will attempt to renew undeleted revoked " + "certificates.") helpful.add("rollback", "--checkpoints", type=int, metavar="N", default=flag_default("rollback_checkpoints"), diff --git a/certbot/constants.py b/certbot/constants.py index 0ac82dafe45..a6878824b58 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -71,6 +71,7 @@ user_agent_comment=None, csr=None, reason=0, + delete_after_revoke=None, rollback_checkpoints=1, init=False, prepare=False, diff --git a/certbot/main.py b/certbot/main.py index 1c6432fd986..e25e030aadb 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -536,9 +536,11 @@ def _delete_if_appropriate(config): # pylint: disable=too-many-locals,too-many-b display = zope.component.getUtility(interfaces.IDisplay) reporter_util = zope.component.getUtility(interfaces.IReporter) - msg = ("Would you like to delete the cert(s) you just revoked?") - attempt_deletion = display.yesno(msg, yes_label="Yes (recommended)", no_label="No", - force_interactive=True, default=True) + attempt_deletion = config.delete_after_revoke + if attempt_deletion is None: + msg = ("Would you like to delete the cert(s) you just revoked?") + attempt_deletion = display.yesno(msg, yes_label="Yes (recommended)", no_label="No", + force_interactive=True, default=True) if not attempt_deletion: reporter_util.add_message("Not deleting revoked certs.", reporter_util.LOW_PRIORITY) diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index 2fce412e230..c5935d7224a 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -164,6 +164,8 @@ def test_help(self): self.assertTrue("--cert-path" in out) self.assertTrue("--key-path" in out) self.assertTrue("--reason" in out) + self.assertTrue("--delete-after-revoke" in out) + self.assertTrue("--no-delete-after-revoke" in out) out = self._help_output(['-h', 'config_changes']) self.assertTrue("--cert-path" not in out) @@ -412,6 +414,18 @@ def test_no_directory_hooks_set(self): def test_no_directory_hooks_unset(self): self.assertTrue(self.parse([]).directory_hooks) + def test_delete_after_revoke(self): + namespace = self.parse(["--delete-after-revoke"]) + self.assertTrue(namespace.delete_after_revoke) + + def test_delete_after_revoke_default(self): + namespace = self.parse([]) + self.assertEqual(namespace.delete_after_revoke, None) + + def test_no_delete_after_revoke(self): + namespace = self.parse(["--no-delete-after-revoke"]) + self.assertFalse(namespace.delete_after_revoke) + class DefaultTest(unittest.TestCase): """Tests for certbot.cli._Default.""" diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 04b71dcc7fe..b1d58542f50 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -298,25 +298,29 @@ def test_revocation_with_prompt(self, mock_get_utility, self._call() self.assertFalse(mock_delete.called) -class DeleteIfAppropriateTest(unittest.TestCase): +class DeleteIfAppropriateTest(test_util.ConfigTestCase): """Tests for certbot.main._delete_if_appropriate """ - def setUp(self): - self.config = mock.Mock() - self.config.namespace = mock.Mock() - self.config.namespace.noninteractive_mode = False - def _call(self, mock_config): from certbot.main import _delete_if_appropriate _delete_if_appropriate(mock_config) - @mock.patch('certbot.cert_manager.delete') + def _test_delete_opt_out_common(self, mock_get_utility): + with mock.patch('certbot.cert_manager.delete') as mock_delete: + self._call(self.config) + mock_delete.assert_not_called() + self.assertTrue(mock_get_utility().add_message.called) + + @test_util.patch_get_utility() + def test_delete_flag_opt_out(self, mock_get_utility): + self.config.delete_after_revoke = False + self._test_delete_opt_out_common(mock_get_utility) + @test_util.patch_get_utility() - def test_delete_opt_out(self, mock_get_utility, mock_delete): + def test_delete_prompt_opt_out(self, mock_get_utility): util_mock = mock_get_utility() util_mock.yesno.return_value = False - self._call(self.config) - mock_delete.assert_not_called() + self._test_delete_opt_out_common(mock_get_utility) # pylint: disable=too-many-arguments @mock.patch('certbot.storage.renewal_file_for_certname') @@ -397,6 +401,28 @@ def test_noninteractive_deletion(self, mock_get_utility, mock_delete, self._call(config) self.assertEqual(mock_delete.call_count, 1) + # pylint: disable=too-many-arguments + @mock.patch('certbot.storage.renewal_file_for_certname') + @mock.patch('certbot.cert_manager.match_and_check_overlaps') + @mock.patch('certbot.storage.full_archive_path') + @mock.patch('certbot.cert_manager.cert_path_to_lineage') + @mock.patch('certbot.cert_manager.delete') + @test_util.patch_get_utility() + def test_opt_in_deletion(self, mock_get_utility, mock_delete, + mock_cert_path_to_lineage, mock_full_archive_dir, + mock_match_and_check_overlaps, mock_renewal_file_for_certname): + # pylint: disable = unused-argument + config = self.config + config.namespace.delete_after_revoke = True + config.cert_path = "/some/reasonable/path" + config.certname = "" + mock_cert_path_to_lineage.return_value = "example.com" + mock_full_archive_dir.return_value = "" + mock_match_and_check_overlaps.return_value = "" + self._call(config) + self.assertEqual(mock_delete.call_count, 1) + self.assertFalse(mock_get_utility().yesno.called) + # pylint: disable=too-many-arguments @mock.patch('certbot.storage.renewal_file_for_certname') @mock.patch('certbot.cert_manager.match_and_check_overlaps') diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 1e0b7754b78..e1aad43365d 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -345,9 +345,14 @@ common auth --must-staple --domains "must-staple.le.wtf" openssl x509 -in "${root}/conf/live/must-staple.le.wtf/cert.pem" -text | grep '1.3.6.1.5.5.7.1.24' # revoke by account key -common revoke --cert-path "$root/conf/live/le.wtf/cert.pem" +common revoke --cert-path "$root/conf/live/le.wtf/cert.pem" --delete-after-revoke # revoke renewed -common revoke --cert-path "$root/conf/live/le1.wtf/cert.pem" +common revoke --cert-path "$root/conf/live/le1.wtf/cert.pem" --no-delete-after-revoke +if [ ! -d "$root/conf/live/le1.wtf" ]; then + echo "cert deleted when --no-delete-after-revoke was used!" + exit 1 +fi +common delete --cert-name le1.wtf # revoke by cert key common revoke --cert-path "$root/conf/live/le2.wtf/cert.pem" \ --key-path "$root/conf/live/le2.wtf/privkey.pem" From e02adec26b9018a3c7fae40fdc13086cd336b05c Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 8 Jan 2018 17:38:03 -0800 Subject: [PATCH 14/21] Have letsencrypt-auto do a real upgrade in leauto-upgrades option 2 (#5390) * Make leauto_upgrades do a real upgrade * Cleanup vars and output * Sleep until the server is ready * add simple_http_server.py * Use a randomly assigned port * s/realpath/readlink * wait for server before getting port * s/localhost/all interfaces --- .../letstest/scripts/test_leauto_upgrades.sh | 47 +++++++++++++++++-- tools/simple_http_server.py | 26 ++++++++++ 2 files changed, 68 insertions(+), 5 deletions(-) create mode 100755 tools/simple_http_server.py diff --git a/tests/letstest/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh index cb659786efd..a83cbd8262b 100755 --- a/tests/letstest/scripts/test_leauto_upgrades.sh +++ b/tests/letstest/scripts/test_leauto_upgrades.sh @@ -15,19 +15,56 @@ if ! command -v git ; then exit 1 fi fi -BRANCH=`git rev-parse --abbrev-ref HEAD` # 0.5.0 is the oldest version of letsencrypt-auto that can be used because it's # the first version that pins package versions, properly supports # --no-self-upgrade, and works with newer versions of pip. -git checkout -f v0.5.0 +git checkout -f v0.5.0 letsencrypt-auto if ! ./letsencrypt-auto -v --debug --version --no-self-upgrade 2>&1 | grep 0.5.0 ; then echo initial installation appeared to fail exit 1 fi -git checkout -f "$BRANCH" -EXPECTED_VERSION=$(grep -m1 LE_AUTO_VERSION letsencrypt-auto | cut -d\" -f2) -if ! ./letsencrypt-auto -v --debug --version --no-self-upgrade 2>&1 | grep $EXPECTED_VERSION ; then +# Now that python and openssl have been installed, we can set up a fake server +# to provide a new version of letsencrypt-auto. First, we start the server and +# directory to be served. +MY_TEMP_DIR=$(mktemp -d) +PORT_FILE="$MY_TEMP_DIR/port" +SERVER_PATH=$(tools/readlink.py tools/simple_http_server.py) +cd "$MY_TEMP_DIR" +"$SERVER_PATH" 0 > $PORT_FILE & +SERVER_PID=$! +trap 'kill "$SERVER_PID" && rm -rf "$MY_TEMP_DIR"' EXIT +cd ~- + +# Then, we set up the files to be served. +FAKE_VERSION_NUM="99.99.99" +echo "{\"releases\": {\"$FAKE_VERSION_NUM\": null}}" > "$MY_TEMP_DIR/json" +LE_AUTO_SOURCE_DIR="$MY_TEMP_DIR/v$FAKE_VERSION_NUM" +NEW_LE_AUTO_PATH="$LE_AUTO_SOURCE_DIR/letsencrypt-auto" +mkdir "$LE_AUTO_SOURCE_DIR" +cp letsencrypt-auto-source/letsencrypt-auto "$LE_AUTO_SOURCE_DIR/letsencrypt-auto" +SIGNING_KEY="letsencrypt-auto-source/tests/signing.key" +openssl dgst -sha256 -sign "$SIGNING_KEY" -out "$NEW_LE_AUTO_PATH.sig" "$NEW_LE_AUTO_PATH" + +# Next, we wait for the server to start and get the port number. +sleep 5s +SERVER_PORT=$(sed -n 's/.*port \([0-9]\+\).*/\1/p' "$PORT_FILE") + +# Finally, we set the necessary certbot-auto environment variables. +export LE_AUTO_DIR_TEMPLATE="http://localhost:$SERVER_PORT/%s/" +export LE_AUTO_JSON_URL="http://localhost:$SERVER_PORT/json" +export LE_AUTO_PUBLIC_KEY="-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsMoSzLYQ7E1sdSOkwelg +tzKIh2qi3bpXuYtcfFC0XrvWig071NwIj+dZiT0OLZ2hPispEH0B7ISuuWg1ll7G +hFW0VdbxL6JdGzS2ShNWkX9hE9z+j8VqwDPOBn3ZHm03qwpYkBDwQib3KqOdYbTT +uUtJmmGcuk3a9Aq/sCT6DdfmTSdP5asdQYwIcaQreDrOosaS84DTWI3IU+UYJVgl +LsIVPBuy9IcgHidUQ96hJnoPsDCWsHwX62495QKEarauyKQrJzFes0EY95orDM47 +Z5o/NDiQB11m91yNB0MmPYY9QSbnOA9j7IaaC97AwRLuwXY+/R2ablTcxurWou68 +iQIDAQAB +-----END PUBLIC KEY----- +" + +if ! ./letsencrypt-auto -v --debug --version || ! diff letsencrypt-auto letsencrypt-auto-source/letsencrypt-auto ; then echo upgrade appeared to fail exit 1 fi diff --git a/tools/simple_http_server.py b/tools/simple_http_server.py new file mode 100755 index 00000000000..26bf231b7bf --- /dev/null +++ b/tools/simple_http_server.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python +"""A version of Python 2.x's SimpleHTTPServer that flushes its output.""" +from BaseHTTPServer import HTTPServer +from SimpleHTTPServer import SimpleHTTPRequestHandler +import sys + +def serve_forever(port=0): + """Spins up an HTTP server on all interfaces and the given port. + + A message is printed to stdout specifying the address and port being used + by the server. + + :param int port: port number to use. + + """ + server = HTTPServer(('', port), SimpleHTTPRequestHandler) + print 'Serving HTTP on {0} port {1} ...'.format(*server.server_address) + sys.stdout.flush() + server.serve_forever() + + +if __name__ == '__main__': + kwargs = {} + if len(sys.argv) > 1: + kwargs['port'] = int(sys.argv[1]) + serve_forever(**kwargs) From d557475bb6f921b7bdb0459d77a162d673044dce Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 9 Jan 2018 07:46:21 -0800 Subject: [PATCH 15/21] update Apache ciphersuites (#5383) --- certbot-apache/certbot_apache/centos-options-ssl-apache.conf | 2 +- certbot-apache/certbot_apache/constants.py | 2 ++ certbot-apache/certbot_apache/options-ssl-apache.conf | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/certbot-apache/certbot_apache/centos-options-ssl-apache.conf b/certbot-apache/certbot_apache/centos-options-ssl-apache.conf index 17ae1be767e..56c946a4e21 100644 --- a/certbot-apache/certbot_apache/centos-options-ssl-apache.conf +++ b/certbot-apache/certbot_apache/centos-options-ssl-apache.conf @@ -8,7 +8,7 @@ SSLEngine on # Intermediate configuration, tweak to your needs SSLProtocol all -SSLv2 -SSLv3 -SSLCipherSuite ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA +SSLCipherSuite ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS SSLHonorCipherOrder on SSLOptions +StrictRequire diff --git a/certbot-apache/certbot_apache/constants.py b/certbot-apache/certbot_apache/constants.py index a13ca04a683..fd6a9eb11fe 100644 --- a/certbot-apache/certbot_apache/constants.py +++ b/certbot-apache/certbot_apache/constants.py @@ -16,6 +16,8 @@ '4066b90268c03c9ba0201068eaa39abbc02acf9558bb45a788b630eb85dadf27', 'f175e2e7c673bd88d0aff8220735f385f916142c44aa83b09f1df88dd4767a88', 'cfdd7c18d2025836ea3307399f509cfb1ebf2612c87dd600a65da2a8e2f2797b', + '80720bd171ccdc2e6b917ded340defae66919e4624962396b992b7218a561791', + 'c0c022ea6b8a51ecc8f1003d0a04af6c3f2bc1c3ce506b3c2dfc1f11ef931082', ] """SHA256 hashes of the contents of previous versions of all versions of MOD_SSL_CONF_SRC""" diff --git a/certbot-apache/certbot_apache/options-ssl-apache.conf b/certbot-apache/certbot_apache/options-ssl-apache.conf index 950a02a8bc2..8113ee81e5a 100644 --- a/certbot-apache/certbot_apache/options-ssl-apache.conf +++ b/certbot-apache/certbot_apache/options-ssl-apache.conf @@ -8,7 +8,7 @@ SSLEngine on # Intermediate configuration, tweak to your needs SSLProtocol all -SSLv2 -SSLv3 -SSLCipherSuite ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA +SSLCipherSuite ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS SSLHonorCipherOrder on SSLCompression off From 62ffcf53738760f901d8ad3b31bff3855d1648c6 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Tue, 9 Jan 2018 17:48:05 +0200 Subject: [PATCH 16/21] Fix macOS builds for Python2.7 in Travis (#5378) * Add OSX Python2 tests * Make sure python2 is originating from homebrew on macOS * Upgrade the already installed python2 instead of trying to reinstall --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3d41bfa4bf9..35666d8e605 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ cache: - $HOME/.cache/pip before_install: - - '([ $TRAVIS_OS_NAME == linux ] && dpkg -s libaugeas0) || (brew update && brew install augeas python3)' + - '([ $TRAVIS_OS_NAME == linux ] && dpkg -s libaugeas0) || (brew update && brew install augeas python3 && brew upgrade python && brew link python)' before_script: - 'if [ $TRAVIS_OS_NAME = osx ] ; then ulimit -n 1024 ; fi' From 288c4d956cf59015590bc24e045138335b46a40d Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Tue, 9 Jan 2018 18:28:52 +0200 Subject: [PATCH 17/21] Automatically install updates in test script (#5394) --- letsencrypt-auto-source/tests/centos6_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt-auto-source/tests/centos6_tests.sh b/letsencrypt-auto-source/tests/centos6_tests.sh index e3ebbaec55d..23b1e16e8a7 100644 --- a/letsencrypt-auto-source/tests/centos6_tests.sh +++ b/letsencrypt-auto-source/tests/centos6_tests.sh @@ -1,6 +1,6 @@ #!/bin/bash # Start by making sure your system is up-to-date: -yum update > /dev/null +yum update -y > /dev/null yum install -y centos-release-scl > /dev/null yum install -y python27 > /dev/null 2> /dev/null From 887a6bcfce73f7b3c556720ccbbc470bf8855606 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 9 Jan 2018 15:40:26 -0800 Subject: [PATCH 18/21] Handle need to rebootstrap before fetch.py (#5389) * Fix #5387 * Add test for #5387 * remove LE_PYTHON * Use environment variable to reduce line length --- letsencrypt-auto-source/letsencrypt-auto | 20 +++++++++++++------ .../letsencrypt-auto.template | 20 +++++++++++++------ .../tests/centos6_tests.sh | 12 +++++++++-- 3 files changed, 38 insertions(+), 14 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index f1361d8eae9..5f46e3a31a1 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -246,10 +246,14 @@ DeprecationBootstrap() { fi } +MIN_PYTHON_VERSION="2.6" +MIN_PYVER=$(echo "$MIN_PYTHON_VERSION" | sed 's/\.//') # Sets LE_PYTHON to Python version string and PYVER to the first two # digits of the python version DeterminePythonVersion() { # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python + # + # If no Python is found, PYVER is set to 0. if [ -n "$USE_PYTHON_3" ]; then for LE_PYTHON in "$LE_PYTHON" python3; do # Break (while keeping the LE_PYTHON value) if found. @@ -273,10 +277,12 @@ DeterminePythonVersion() { export LE_PYTHON PYVER=`"$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` - if [ "$PYVER" -lt 26 ]; then - error "You have an ancient version of Python entombed in your operating system..." - error "This isn't going to work; you'll need at least version 2.6." - exit 1 + if [ "$PYVER" -lt "$MIN_PYVER" ]; then + if [ "$1" != "NOCRASH" ]; then + error "You have an ancient version of Python entombed in your operating system..." + error "This isn't going to work; you'll need at least version $MIN_PYTHON_VERSION." + exit 1 + fi fi } @@ -1575,8 +1581,10 @@ if __name__ == '__main__': UNLIKELY_EOF # --------------------------------------------------------------------------- - DeterminePythonVersion - if ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then + DeterminePythonVersion "NOCRASH" + if [ "$PYVER" -lt "$MIN_PYVER" ]; then + error "WARNING: couldn't find Python $MIN_PYTHON_VERSION+ to check for updates." + elif ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then error "WARNING: unable to check for updates." elif [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then say "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index f4c1b202f4d..7c3cbac088a 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -246,10 +246,14 @@ DeprecationBootstrap() { fi } +MIN_PYTHON_VERSION="2.6" +MIN_PYVER=$(echo "$MIN_PYTHON_VERSION" | sed 's/\.//') # Sets LE_PYTHON to Python version string and PYVER to the first two # digits of the python version DeterminePythonVersion() { # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python + # + # If no Python is found, PYVER is set to 0. if [ -n "$USE_PYTHON_3" ]; then for LE_PYTHON in "$LE_PYTHON" python3; do # Break (while keeping the LE_PYTHON value) if found. @@ -273,10 +277,12 @@ DeterminePythonVersion() { export LE_PYTHON PYVER=`"$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` - if [ "$PYVER" -lt 26 ]; then - error "You have an ancient version of Python entombed in your operating system..." - error "This isn't going to work; you'll need at least version 2.6." - exit 1 + if [ "$PYVER" -lt "$MIN_PYVER" ]; then + if [ "$1" != "NOCRASH" ]; then + error "You have an ancient version of Python entombed in your operating system..." + error "This isn't going to work; you'll need at least version $MIN_PYTHON_VERSION." + exit 1 + fi fi } @@ -586,8 +592,10 @@ else {{ fetch.py }} UNLIKELY_EOF # --------------------------------------------------------------------------- - DeterminePythonVersion - if ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then + DeterminePythonVersion "NOCRASH" + if [ "$PYVER" -lt "$MIN_PYVER" ]; then + error "WARNING: couldn't find Python $MIN_PYTHON_VERSION+ to check for updates." + elif ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then error "WARNING: unable to check for updates." elif [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then say "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." diff --git a/letsencrypt-auto-source/tests/centos6_tests.sh b/letsencrypt-auto-source/tests/centos6_tests.sh index 23b1e16e8a7..a0e96edf8dc 100644 --- a/letsencrypt-auto-source/tests/centos6_tests.sh +++ b/letsencrypt-auto-source/tests/centos6_tests.sh @@ -4,6 +4,8 @@ yum update -y > /dev/null yum install -y centos-release-scl > /dev/null yum install -y python27 > /dev/null 2> /dev/null +LE_AUTO="certbot/letsencrypt-auto-source/letsencrypt-auto" + # we're going to modify env variables, so do this in a subshell ( source /opt/rh/python27/enable @@ -25,7 +27,7 @@ if [ $RESULT -ne 0 ]; then fi # bootstrap, but don't install python 3. -certbot/letsencrypt-auto-source/letsencrypt-auto --no-self-upgrade -n > /dev/null 2> /dev/null +"$LE_AUTO" --no-self-upgrade -n > /dev/null 2> /dev/null # ensure python 3 isn't installed python3 --version 2> /dev/null @@ -47,8 +49,14 @@ if [ $RESULT -eq 0 ]; then exit 1 fi +# Skip self upgrade due to Python 3 not being available. +if ! "$LE_AUTO" 2>&1 | grep -q "WARNING: couldn't find Python"; then + echo "Python upgrade failure warning not printed!" + exit 1 +fi + # bootstrap, this time installing python3 -certbot/letsencrypt-auto-source/letsencrypt-auto --no-self-upgrade -n > /dev/null 2> /dev/null +"$LE_AUTO" --no-self-upgrade -n > /dev/null 2> /dev/null # ensure python 3 is installed python3 --version > /dev/null From f5a02714cd8b610db52ac04e62685bba9c081a47 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Tue, 9 Jan 2018 16:11:04 -0800 Subject: [PATCH 19/21] Add deprecation warning for Python 2.6 (#5391) * Add deprecation warning for Python 2.6 * Allow disabling Python 2.6 warning --- acme/acme/__init__.py | 13 +++++++------ certbot/main.py | 15 ++++++++++++--- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/acme/acme/__init__.py b/acme/acme/__init__.py index 618dda20048..5850fa9551d 100644 --- a/acme/acme/__init__.py +++ b/acme/acme/__init__.py @@ -13,9 +13,10 @@ import sys import warnings -if sys.version_info[:2] == (3, 3): - warnings.warn( - "Python 3.3 support will be dropped in the next release of " - "acme. Please upgrade your Python version.", - PendingDeprecationWarning, - ) #pragma: no cover +for (major, minor) in [(2, 6), (3, 3)]: + if sys.version_info[:2] == (major, minor): + warnings.warn( + "Python {0}.{1} support will be dropped in the next release of " + "acme. Please upgrade your Python version.".format(major, minor), + DeprecationWarning, + ) #pragma: no cover diff --git a/certbot/main.py b/certbot/main.py index e25e030aadb..32dd6925682 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -4,6 +4,7 @@ import logging.handlers import os import sys +import warnings import configobj import josepy as jose @@ -1217,9 +1218,17 @@ def main(cli_args=sys.argv[1:]): # Let plugins_cmd be run as un-privileged user. if config.func != plugins_cmd: raise - if sys.version_info[:2] == (3, 3): - logger.warning("Python 3.3 support will be dropped in the next release " - "of Certbot - please upgrade your Python version.") + deprecation_fmt = ( + "Python %s.%s support will be dropped in the next " + "release of Certbot - please upgrade your Python version.") + # We use the warnings system for Python 2.6 and logging for Python 3 + # because DeprecationWarnings are only reported by default in Python <= 2.6 + # and warnings can be disabled by the user. + if sys.version_info[:2] == (2, 6): + warning = deprecation_fmt % sys.version_info[:2] + warnings.warn(warning, DeprecationWarning) + elif sys.version_info[:2] == (3, 3): + logger.warning(deprecation_fmt, *sys.version_info[:2]) set_displayer(config) From 6eb459354fc28433ac2eabc3be62c809df66a4a1 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Tue, 9 Jan 2018 16:48:16 -0800 Subject: [PATCH 20/21] Address erikrose's comments on #5329 (#5400) --- letsencrypt-auto-source/letsencrypt-auto | 13 ++++++++----- letsencrypt-auto-source/letsencrypt-auto.template | 5 ++++- .../pieces/bootstrappers/rpm_common_base.sh | 2 +- letsencrypt-auto-source/pieces/fetch.py | 6 +++--- 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 5f46e3a31a1..712ef6813a0 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -254,7 +254,7 @@ DeterminePythonVersion() { # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python # # If no Python is found, PYVER is set to 0. - if [ -n "$USE_PYTHON_3" ]; then + if [ "$USE_PYTHON_3" = 1 ]; then for LE_PYTHON in "$LE_PYTHON" python3; do # Break (while keeping the LE_PYTHON value) if found. $EXISTS "$LE_PYTHON" > /dev/null && break @@ -443,7 +443,7 @@ InitializeRPMCommonBase() { sleep 1s /bin/echo -ne "\e[0K\rEnabling the EPEL repository in 2 seconds..." sleep 1s - /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 seconds..." + /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 second..." sleep 1s fi if ! $TOOL install $YES_FLAG $QUIET_FLAG epel-release; then @@ -781,6 +781,9 @@ elif [ -f /etc/mageia-release ]; then } BOOTSTRAP_VERSION="BootstrapMageiaCommon $BOOTSTRAP_MAGEIA_COMMON_VERSION" elif [ -f /etc/redhat-release ]; then + # Run DeterminePythonVersion to decide on the basis of available Python versions + # whether to use 2.x or 3.x on RedHat-like systems. + # Then, revert LE_PYTHON to its previous state. prev_le_python="$LE_PYTHON" unset LE_PYTHON DeterminePythonVersion "NOCRASH" @@ -1482,7 +1485,7 @@ class HttpsGetter(object): # Based on pip 1.4.1's URLOpener # This verifies certs on only Python >=2.7.9, and when NO_CERT_VERIFY isn't set. if environ.get('NO_CERT_VERIFY') == '1' and hasattr(ssl, 'SSLContext'): - self._opener = build_opener(HTTPSHandler(context=create_CERT_NONE_context())) + self._opener = build_opener(HTTPSHandler(context=cert_none_context())) else: self._opener = build_opener(HTTPSHandler()) # Strip out HTTPHandler to prevent MITM spoof: @@ -1520,7 +1523,7 @@ def latest_stable_version(get): # The regex is a sufficient regex for picking out prereleases for most # packages, LE included. return str(max(LooseVersion(r) for r - in iter(metadata['releases'].keys()) + in metadata['releases'].keys() if re.match('^[0-9.]+$', r))) @@ -1552,7 +1555,7 @@ def verified_new_le_auto(get, tag, temp_dir): "certbot-auto.", exc) -def create_CERT_NONE_context(): +def cert_none_context(): """Create a SSLContext object to not check hostname.""" # PROTOCOL_TLS isn't available before 2.7.13 but this code is for 2.7.9+, so use this. context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 7c3cbac088a..b06ac9c8016 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -254,7 +254,7 @@ DeterminePythonVersion() { # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python # # If no Python is found, PYVER is set to 0. - if [ -n "$USE_PYTHON_3" ]; then + if [ "$USE_PYTHON_3" = 1 ]; then for LE_PYTHON in "$LE_PYTHON" python3; do # Break (while keeping the LE_PYTHON value) if found. $EXISTS "$LE_PYTHON" > /dev/null && break @@ -320,6 +320,9 @@ elif [ -f /etc/mageia-release ]; then } BOOTSTRAP_VERSION="BootstrapMageiaCommon $BOOTSTRAP_MAGEIA_COMMON_VERSION" elif [ -f /etc/redhat-release ]; then + # Run DeterminePythonVersion to decide on the basis of available Python versions + # whether to use 2.x or 3.x on RedHat-like systems. + # Then, revert LE_PYTHON to its previous state. prev_le_python="$LE_PYTHON" unset LE_PYTHON DeterminePythonVersion "NOCRASH" diff --git a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh index d7a9f3133c8..326ad8b3f4c 100644 --- a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh @@ -35,7 +35,7 @@ InitializeRPMCommonBase() { sleep 1s /bin/echo -ne "\e[0K\rEnabling the EPEL repository in 2 seconds..." sleep 1s - /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 seconds..." + /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 second..." sleep 1s fi if ! $TOOL install $YES_FLAG $QUIET_FLAG epel-release; then diff --git a/letsencrypt-auto-source/pieces/fetch.py b/letsencrypt-auto-source/pieces/fetch.py index ae72a299b48..1515fe3533b 100644 --- a/letsencrypt-auto-source/pieces/fetch.py +++ b/letsencrypt-auto-source/pieces/fetch.py @@ -50,7 +50,7 @@ def __init__(self): # Based on pip 1.4.1's URLOpener # This verifies certs on only Python >=2.7.9, and when NO_CERT_VERIFY isn't set. if environ.get('NO_CERT_VERIFY') == '1' and hasattr(ssl, 'SSLContext'): - self._opener = build_opener(HTTPSHandler(context=create_CERT_NONE_context())) + self._opener = build_opener(HTTPSHandler(context=cert_none_context())) else: self._opener = build_opener(HTTPSHandler()) # Strip out HTTPHandler to prevent MITM spoof: @@ -88,7 +88,7 @@ def latest_stable_version(get): # The regex is a sufficient regex for picking out prereleases for most # packages, LE included. return str(max(LooseVersion(r) for r - in iter(metadata['releases'].keys()) + in metadata['releases'].keys() if re.match('^[0-9.]+$', r))) @@ -120,7 +120,7 @@ def verified_new_le_auto(get, tag, temp_dir): "certbot-auto.", exc) -def create_CERT_NONE_context(): +def cert_none_context(): """Create a SSLContext object to not check hostname.""" # PROTOCOL_TLS isn't available before 2.7.13 but this code is for 2.7.9+, so use this. context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) From 164121fc1561b8bfee0a1c844c6f7ae634e9970b Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 9 Jan 2018 17:32:07 -0800 Subject: [PATCH 21/21] Revert "Update test-everything (#5397)" This reverts commit 349643c9b8dfb0cb97154ca5310617c7ade4ae21. --- .gitignore | 3 - .travis.yml | 2 +- acme/acme/challenges.py | 2 +- acme/acme/challenges_test.py | 2 +- acme/acme/client.py | 4 +- acme/acme/client_test.py | 2 +- acme/acme/crypto_util_test.py | 2 +- acme/acme/errors.py | 2 +- acme/acme/fields.py | 3 +- acme/acme/fields_test.py | 3 +- acme/acme/jose/__init__.py | 82 +++ acme/acme/jose/b64.py | 61 +++ acme/acme/jose/b64_test.py | 77 +++ acme/acme/jose/errors.py | 35 ++ acme/acme/jose/errors_test.py | 17 + acme/acme/jose/interfaces.py | 216 ++++++++ acme/acme/jose/interfaces_test.py | 114 ++++ acme/acme/jose/json_util.py | 485 ++++++++++++++++++ acme/acme/jose/json_util_test.py | 381 ++++++++++++++ acme/acme/jose/jwa.py | 180 +++++++ acme/acme/jose/jwa_test.py | 104 ++++ acme/acme/jose/jwk.py | 281 ++++++++++ acme/acme/jose/jwk_test.py | 191 +++++++ acme/acme/jose/jws.py | 433 ++++++++++++++++ acme/acme/jose/jws_test.py | 239 +++++++++ acme/acme/jose/util.py | 226 ++++++++ acme/acme/jose/util_test.py | 199 +++++++ acme/acme/jws.py | 6 +- acme/acme/jws_test.py | 3 +- acme/acme/messages.py | 9 +- acme/acme/messages_test.py | 2 +- acme/acme/standalone_test.py | 2 +- acme/acme/test_util.py | 3 +- acme/docs/api/jose.rst | 9 +- acme/docs/api/jose/base64.rst | 5 + acme/docs/api/jose/errors.rst | 5 + acme/docs/api/jose/interfaces.rst | 5 + acme/docs/api/jose/json_util.rst | 5 + acme/docs/api/jose/jwa.rst | 5 + acme/docs/api/jose/jwk.rst | 5 + acme/docs/api/jose/jws.rst | 5 + acme/docs/api/jose/util.rst | 5 + acme/docs/conf.py | 1 - acme/examples/example_client.py | 2 +- acme/setup.py | 7 +- certbot-apache/certbot_apache/apache_util.py | 4 - .../centos-options-ssl-apache.conf | 2 +- certbot-apache/certbot_apache/constants.py | 2 - .../certbot_apache/options-ssl-apache.conf | 2 +- .../certbot_apache/override_gentoo.py | 8 - .../certbot_apache/tests/centos_test.py | 2 - .../certbot_apache/tests/gentoo_test.py | 49 +- .../centos7_apache/apache/sysconfig/httpd | 2 +- certbot-apache/certbot_apache/tests/util.py | 3 +- .../certbot_compatibility_test/util.py | 3 +- certbot-nginx/certbot_nginx/configurator.py | 2 +- certbot-nginx/certbot_nginx/tests/util.py | 3 +- certbot/account.py | 2 +- certbot/achallenges.py | 3 +- certbot/cli.py | 12 - certbot/client.py | 2 +- certbot/constants.py | 1 - certbot/crypto_util.py | 4 +- certbot/main.py | 10 +- certbot/plugins/common.py | 2 +- certbot/plugins/common_test.py | 2 +- certbot/plugins/dns_test_common.py | 2 +- certbot/plugins/dns_test_common_lexicon.py | 2 +- certbot/plugins/standalone_test.py | 2 +- certbot/plugins/webroot_test.py | 2 +- certbot/tests/account_test.py | 2 +- certbot/tests/acme_util.py | 2 +- certbot/tests/cli_test.py | 14 - certbot/tests/client_test.py | 2 +- certbot/tests/display/ops_test.py | 2 +- certbot/tests/main_test.py | 49 +- certbot/tests/util.py | 3 +- letsencrypt-auto-source/Dockerfile.centos6 | 3 +- letsencrypt-auto-source/letsencrypt-auto | 224 +++----- .../letsencrypt-auto.template | 75 +-- .../pieces/bootstrappers/rpm_common.sh | 75 ++- .../pieces/bootstrappers/rpm_common_base.sh | 78 --- .../pieces/bootstrappers/rpm_python3.sh | 23 - .../pieces/dependency-requirements.txt | 11 +- letsencrypt-auto-source/pieces/fetch.py | 34 +- letsencrypt-auto-source/tests/auto_test.py | 32 +- .../tests/centos6_tests.sh | 65 --- pytest.ini | 2 - setup.py | 9 +- tests/boulder-integration.sh | 9 +- .../letstest/scripts/test_leauto_upgrades.sh | 47 +- tools/deactivate.py | 2 +- tools/install_and_test.sh | 7 +- tools/merge_requirements.py | 61 --- tools/oldest_constraints.txt | 51 -- ...ev_constraints.txt => pip_constraints.txt} | 25 +- tools/pip_install.sh | 31 +- tools/pytest.sh | 15 - tools/simple_http_server.py | 26 - tox.cover.sh | 3 +- tox.ini | 45 +- 101 files changed, 3684 insertions(+), 901 deletions(-) create mode 100644 acme/acme/jose/__init__.py create mode 100644 acme/acme/jose/b64.py create mode 100644 acme/acme/jose/b64_test.py create mode 100644 acme/acme/jose/errors.py create mode 100644 acme/acme/jose/errors_test.py create mode 100644 acme/acme/jose/interfaces.py create mode 100644 acme/acme/jose/interfaces_test.py create mode 100644 acme/acme/jose/json_util.py create mode 100644 acme/acme/jose/json_util_test.py create mode 100644 acme/acme/jose/jwa.py create mode 100644 acme/acme/jose/jwa_test.py create mode 100644 acme/acme/jose/jwk.py create mode 100644 acme/acme/jose/jwk_test.py create mode 100644 acme/acme/jose/jws.py create mode 100644 acme/acme/jose/jws_test.py create mode 100644 acme/acme/jose/util.py create mode 100644 acme/acme/jose/util_test.py create mode 100644 acme/docs/api/jose/base64.rst create mode 100644 acme/docs/api/jose/errors.rst create mode 100644 acme/docs/api/jose/interfaces.rst create mode 100644 acme/docs/api/jose/json_util.rst create mode 100644 acme/docs/api/jose/jwa.rst create mode 100644 acme/docs/api/jose/jwk.rst create mode 100644 acme/docs/api/jose/jws.rst create mode 100644 acme/docs/api/jose/util.rst delete mode 100644 letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh delete mode 100644 letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh delete mode 100644 letsencrypt-auto-source/tests/centos6_tests.sh delete mode 100644 pytest.ini delete mode 100755 tools/merge_requirements.py delete mode 100644 tools/oldest_constraints.txt rename tools/{dev_constraints.txt => pip_constraints.txt} (71%) delete mode 100755 tools/pytest.sh delete mode 100755 tools/simple_http_server.py diff --git a/.gitignore b/.gitignore index a01d2e1c73d..b63e40d1cbc 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,3 @@ tests/letstest/*.pem tests/letstest/venv/ .venv - -# pytest cache -.cache diff --git a/.travis.yml b/.travis.yml index 866f7b12abd..b4b48ae719f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ before_install: - cp .travis.yml /tmp/travis.yml - git pull origin master --strategy=recursive --strategy-option=theirs --no-edit - if ! git diff .travis.yml /tmp/travis.yml ; then echo "Please merge master into test-everything"; exit 1; fi - - '([ $TRAVIS_OS_NAME == linux ] && dpkg -s libaugeas0) || (brew update && brew install augeas python3 && brew upgrade python && brew link python)' + - '([ $TRAVIS_OS_NAME == linux ] && dpkg -s libaugeas0) || (brew update && brew install augeas python3)' before_script: - 'if [ $TRAVIS_OS_NAME = osx ] ; then ulimit -n 1024 ; fi' diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 96997297b91..14641af108a 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -6,13 +6,13 @@ import socket from cryptography.hazmat.primitives import hashes # type: ignore -import josepy as jose import OpenSSL import requests from acme import errors from acme import crypto_util from acme import fields +from acme import jose logger = logging.getLogger(__name__) diff --git a/acme/acme/challenges_test.py b/acme/acme/challenges_test.py index 834d569aaf5..49e79010243 100644 --- a/acme/acme/challenges_test.py +++ b/acme/acme/challenges_test.py @@ -1,7 +1,6 @@ """Tests for acme.challenges.""" import unittest -import josepy as jose import mock import OpenSSL import requests @@ -9,6 +8,7 @@ from six.moves.urllib import parse as urllib_parse # pylint: disable=import-error from acme import errors +from acme import jose from acme import test_util CERT = test_util.load_comparable_cert('cert.pem') diff --git a/acme/acme/client.py b/acme/acme/client.py index dc5efbe86f8..2e07d34d70c 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -10,13 +10,13 @@ import six from six.moves import http_client # pylint: disable=import-error -import josepy as jose import OpenSSL import re import requests import sys from acme import errors +from acme import jose from acme import jws from acme import messages @@ -408,7 +408,7 @@ def _get_cert(self, uri): :param str uri: URI of certificate :returns: tuple of the form - (response, :class:`josepy.util.ComparableX509`) + (response, :class:`acme.jose.ComparableX509`) :rtype: tuple """ diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 84620fc994f..4bd762865e3 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -5,12 +5,12 @@ from six.moves import http_client # pylint: disable=import-error -import josepy as jose import mock import requests from acme import challenges from acme import errors +from acme import jose from acme import jws as acme_jws from acme import messages from acme import messages_test diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index 1d7f83ccf22..da433c5a2cf 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -8,10 +8,10 @@ import six from six.moves import socketserver #type: ignore # pylint: disable=import-error -import josepy as jose import OpenSSL from acme import errors +from acme import jose from acme import test_util diff --git a/acme/acme/errors.py b/acme/acme/errors.py index de5f9d1f419..9d991fd75d2 100644 --- a/acme/acme/errors.py +++ b/acme/acme/errors.py @@ -1,5 +1,5 @@ """ACME errors.""" -from josepy import errors as jose_errors +from acme.jose import errors as jose_errors class Error(Exception): diff --git a/acme/acme/fields.py b/acme/acme/fields.py index d7ec78403ed..12d09acf48b 100644 --- a/acme/acme/fields.py +++ b/acme/acme/fields.py @@ -1,9 +1,10 @@ """ACME JSON fields.""" import logging -import josepy as jose import pyrfc3339 +from acme import jose + logger = logging.getLogger(__name__) diff --git a/acme/acme/fields_test.py b/acme/acme/fields_test.py index 69dde8b89bc..de852b6fa23 100644 --- a/acme/acme/fields_test.py +++ b/acme/acme/fields_test.py @@ -2,9 +2,10 @@ import datetime import unittest -import josepy as jose import pytz +from acme import jose + class FixedTest(unittest.TestCase): """Tests for acme.fields.Fixed.""" diff --git a/acme/acme/jose/__init__.py b/acme/acme/jose/__init__.py new file mode 100644 index 00000000000..9116bc43339 --- /dev/null +++ b/acme/acme/jose/__init__.py @@ -0,0 +1,82 @@ +"""Javascript Object Signing and Encryption (jose). + +This package is a Python implementation of the standards developed by +IETF `Javascript Object Signing and Encryption (Active WG)`_, in +particular the following RFCs: + + - `JSON Web Algorithms (JWA)`_ + - `JSON Web Key (JWK)`_ + - `JSON Web Signature (JWS)`_ + + +.. _`Javascript Object Signing and Encryption (Active WG)`: + https://tools.ietf.org/wg/jose/ + +.. _`JSON Web Algorithms (JWA)`: + https://datatracker.ietf.org/doc/draft-ietf-jose-json-web-algorithms/ + +.. _`JSON Web Key (JWK)`: + https://datatracker.ietf.org/doc/draft-ietf-jose-json-web-key/ + +.. _`JSON Web Signature (JWS)`: + https://datatracker.ietf.org/doc/draft-ietf-jose-json-web-signature/ + +""" +from acme.jose.b64 import ( + b64decode, + b64encode, +) + +from acme.jose.errors import ( + DeserializationError, + SerializationError, + Error, + UnrecognizedTypeError, +) + +from acme.jose.interfaces import JSONDeSerializable + +from acme.jose.json_util import ( + Field, + JSONObjectWithFields, + TypedJSONObjectWithFields, + decode_b64jose, + decode_cert, + decode_csr, + decode_hex16, + encode_b64jose, + encode_cert, + encode_csr, + encode_hex16, +) + +from acme.jose.jwa import ( + HS256, + HS384, + HS512, + JWASignature, + PS256, + PS384, + PS512, + RS256, + RS384, + RS512, +) + +from acme.jose.jwk import ( + JWK, + JWKRSA, +) + +from acme.jose.jws import ( + Header, + JWS, + Signature, +) + +from acme.jose.util import ( + ComparableX509, + ComparableKey, + ComparableRSAKey, + ImmutableMap, +) diff --git a/acme/acme/jose/b64.py b/acme/acme/jose/b64.py new file mode 100644 index 00000000000..cf79aa8200e --- /dev/null +++ b/acme/acme/jose/b64.py @@ -0,0 +1,61 @@ +"""JOSE Base64. + +`JOSE Base64`_ is defined as: + + - URL-safe Base64 + - padding stripped + + +.. _`JOSE Base64`: + https://tools.ietf.org/html/draft-ietf-jose-json-web-signature-37#appendix-C + +.. Do NOT try to call this module "base64", as it will "shadow" the + standard library. + +""" +import base64 + +import six + + +def b64encode(data): + """JOSE Base64 encode. + + :param data: Data to be encoded. + :type data: `bytes` + + :returns: JOSE Base64 string. + :rtype: bytes + + :raises TypeError: if `data` is of incorrect type + + """ + if not isinstance(data, six.binary_type): + raise TypeError('argument should be {0}'.format(six.binary_type)) + return base64.urlsafe_b64encode(data).rstrip(b'=') + + +def b64decode(data): + """JOSE Base64 decode. + + :param data: Base64 string to be decoded. If it's unicode, then + only ASCII characters are allowed. + :type data: `bytes` or `unicode` + + :returns: Decoded data. + :rtype: bytes + + :raises TypeError: if input is of incorrect type + :raises ValueError: if input is unicode with non-ASCII characters + + """ + if isinstance(data, six.string_types): + try: + data = data.encode('ascii') + except UnicodeEncodeError: + raise ValueError( + 'unicode argument should contain only ASCII characters') + elif not isinstance(data, six.binary_type): + raise TypeError('argument should be a str or unicode') + + return base64.urlsafe_b64decode(data + b'=' * (4 - (len(data) % 4))) diff --git a/acme/acme/jose/b64_test.py b/acme/acme/jose/b64_test.py new file mode 100644 index 00000000000..cbabe225171 --- /dev/null +++ b/acme/acme/jose/b64_test.py @@ -0,0 +1,77 @@ +"""Tests for acme.jose.b64.""" +import unittest + +import six + + +# https://en.wikipedia.org/wiki/Base64#Examples +B64_PADDING_EXAMPLES = { + b'any carnal pleasure.': (b'YW55IGNhcm5hbCBwbGVhc3VyZS4', b'='), + b'any carnal pleasure': (b'YW55IGNhcm5hbCBwbGVhc3VyZQ', b'=='), + b'any carnal pleasur': (b'YW55IGNhcm5hbCBwbGVhc3Vy', b''), + b'any carnal pleasu': (b'YW55IGNhcm5hbCBwbGVhc3U', b'='), + b'any carnal pleas': (b'YW55IGNhcm5hbCBwbGVhcw', b'=='), +} + + +B64_URL_UNSAFE_EXAMPLES = { + six.int2byte(251) + six.int2byte(239): b'--8', + six.int2byte(255) * 2: b'__8', +} + + +class B64EncodeTest(unittest.TestCase): + """Tests for acme.jose.b64.b64encode.""" + + @classmethod + def _call(cls, data): + from acme.jose.b64 import b64encode + return b64encode(data) + + def test_empty(self): + self.assertEqual(self._call(b''), b'') + + def test_unsafe_url(self): + for text, b64 in six.iteritems(B64_URL_UNSAFE_EXAMPLES): + self.assertEqual(self._call(text), b64) + + def test_different_paddings(self): + for text, (b64, _) in six.iteritems(B64_PADDING_EXAMPLES): + self.assertEqual(self._call(text), b64) + + def test_unicode_fails_with_type_error(self): + self.assertRaises(TypeError, self._call, u'some unicode') + + +class B64DecodeTest(unittest.TestCase): + """Tests for acme.jose.b64.b64decode.""" + + @classmethod + def _call(cls, data): + from acme.jose.b64 import b64decode + return b64decode(data) + + def test_unsafe_url(self): + for text, b64 in six.iteritems(B64_URL_UNSAFE_EXAMPLES): + self.assertEqual(self._call(b64), text) + + def test_input_without_padding(self): + for text, (b64, _) in six.iteritems(B64_PADDING_EXAMPLES): + self.assertEqual(self._call(b64), text) + + def test_input_with_padding(self): + for text, (b64, pad) in six.iteritems(B64_PADDING_EXAMPLES): + self.assertEqual(self._call(b64 + pad), text) + + def test_unicode_with_ascii(self): + self.assertEqual(self._call(u'YQ'), b'a') + + def test_non_ascii_unicode_fails(self): + self.assertRaises(ValueError, self._call, u'\u0105') + + def test_type_error_no_unicode_or_bytes(self): + self.assertRaises(TypeError, self._call, object()) + + +if __name__ == '__main__': + unittest.main() # pragma: no cover diff --git a/acme/acme/jose/errors.py b/acme/acme/jose/errors.py new file mode 100644 index 00000000000..74c9443e1ec --- /dev/null +++ b/acme/acme/jose/errors.py @@ -0,0 +1,35 @@ +"""JOSE errors.""" + + +class Error(Exception): + """Generic JOSE Error.""" + + +class DeserializationError(Error): + """JSON deserialization error.""" + + def __str__(self): + return "Deserialization error: {0}".format( + super(DeserializationError, self).__str__()) + + +class SerializationError(Error): + """JSON serialization error.""" + + +class UnrecognizedTypeError(DeserializationError): + """Unrecognized type error. + + :ivar str typ: The unrecognized type of the JSON object. + :ivar jobj: Full JSON object. + + """ + + def __init__(self, typ, jobj): + self.typ = typ + self.jobj = jobj + super(UnrecognizedTypeError, self).__init__(str(self)) + + def __str__(self): + return '{0} was not recognized, full message: {1}'.format( + self.typ, self.jobj) diff --git a/acme/acme/jose/errors_test.py b/acme/acme/jose/errors_test.py new file mode 100644 index 00000000000..919980920c0 --- /dev/null +++ b/acme/acme/jose/errors_test.py @@ -0,0 +1,17 @@ +"""Tests for acme.jose.errors.""" +import unittest + + +class UnrecognizedTypeErrorTest(unittest.TestCase): + def setUp(self): + from acme.jose.errors import UnrecognizedTypeError + self.error = UnrecognizedTypeError('foo', {'type': 'foo'}) + + def test_str(self): + self.assertEqual( + "foo was not recognized, full message: {'type': 'foo'}", + str(self.error)) + + +if __name__ == '__main__': + unittest.main() # pragma: no cover diff --git a/acme/acme/jose/interfaces.py b/acme/acme/jose/interfaces.py new file mode 100644 index 00000000000..f841848b380 --- /dev/null +++ b/acme/acme/jose/interfaces.py @@ -0,0 +1,216 @@ +"""JOSE interfaces.""" +import abc +import collections +import json + +import six + +from acme.jose import errors +from acme.jose import util + +# pylint: disable=no-self-argument,no-method-argument,no-init,inherit-non-class +# pylint: disable=too-few-public-methods + + +@six.add_metaclass(abc.ABCMeta) +class JSONDeSerializable(object): + # pylint: disable=too-few-public-methods + """Interface for (de)serializable JSON objects. + + Please recall, that standard Python library implements + :class:`json.JSONEncoder` and :class:`json.JSONDecoder` that perform + translations based on respective :ref:`conversion tables + ` that look pretty much like the one below (for + complete tables see relevant Python documentation): + + .. _conversion-table: + + ====== ====== + JSON Python + ====== ====== + object dict + ... ... + ====== ====== + + While the above **conversion table** is about translation of JSON + documents to/from the basic Python types only, + :class:`JSONDeSerializable` introduces the following two concepts: + + serialization + Turning an arbitrary Python object into Python object that can + be encoded into a JSON document. **Full serialization** produces + a Python object composed of only basic types as required by the + :ref:`conversion table `. **Partial + serialization** (accomplished by :meth:`to_partial_json`) + produces a Python object that might also be built from other + :class:`JSONDeSerializable` objects. + + deserialization + Turning a decoded Python object (necessarily one of the basic + types as required by the :ref:`conversion table + `) into an arbitrary Python object. + + Serialization produces **serialized object** ("partially serialized + object" or "fully serialized object" for partial and full + serialization respectively) and deserialization produces + **deserialized object**, both usually denoted in the source code as + ``jobj``. + + Wording in the official Python documentation might be confusing + after reading the above, but in the light of those definitions, one + can view :meth:`json.JSONDecoder.decode` as decoder and + deserializer of basic types, :meth:`json.JSONEncoder.default` as + serializer of basic types, :meth:`json.JSONEncoder.encode` as + serializer and encoder of basic types. + + One could extend :mod:`json` to support arbitrary object + (de)serialization either by: + + - overriding :meth:`json.JSONDecoder.decode` and + :meth:`json.JSONEncoder.default` in subclasses + + - or passing ``object_hook`` argument (or ``object_hook_pairs``) + to :func:`json.load`/:func:`json.loads` or ``default`` argument + for :func:`json.dump`/:func:`json.dumps`. + + Interestingly, ``default`` is required to perform only partial + serialization, as :func:`json.dumps` applies ``default`` + recursively. This is the idea behind making :meth:`to_partial_json` + produce only partial serialization, while providing custom + :meth:`json_dumps` that dumps with ``default`` set to + :meth:`json_dump_default`. + + To make further documentation a bit more concrete, please, consider + the following imaginatory implementation example:: + + class Foo(JSONDeSerializable): + def to_partial_json(self): + return 'foo' + + @classmethod + def from_json(cls, jobj): + return Foo() + + class Bar(JSONDeSerializable): + def to_partial_json(self): + return [Foo(), Foo()] + + @classmethod + def from_json(cls, jobj): + return Bar() + + """ + + @abc.abstractmethod + def to_partial_json(self): # pragma: no cover + """Partially serialize. + + Following the example, **partial serialization** means the following:: + + assert isinstance(Bar().to_partial_json()[0], Foo) + assert isinstance(Bar().to_partial_json()[1], Foo) + + # in particular... + assert Bar().to_partial_json() != ['foo', 'foo'] + + :raises acme.jose.errors.SerializationError: + in case of any serialization error. + :returns: Partially serializable object. + + """ + raise NotImplementedError() + + def to_json(self): + """Fully serialize. + + Again, following the example from before, **full serialization** + means the following:: + + assert Bar().to_json() == ['foo', 'foo'] + + :raises acme.jose.errors.SerializationError: + in case of any serialization error. + :returns: Fully serialized object. + + """ + def _serialize(obj): + if isinstance(obj, JSONDeSerializable): + return _serialize(obj.to_partial_json()) + if isinstance(obj, six.string_types): # strings are Sequence + return obj + elif isinstance(obj, list): + return [_serialize(subobj) for subobj in obj] + elif isinstance(obj, collections.Sequence): + # default to tuple, otherwise Mapping could get + # unhashable list + return tuple(_serialize(subobj) for subobj in obj) + elif isinstance(obj, collections.Mapping): + return dict((_serialize(key), _serialize(value)) + for key, value in six.iteritems(obj)) + else: + return obj + + return _serialize(self) + + @util.abstractclassmethod + def from_json(cls, jobj): # pylint: disable=unused-argument + """Deserialize a decoded JSON document. + + :param jobj: Python object, composed of only other basic data + types, as decoded from JSON document. Not necessarily + :class:`dict` (as decoded from "JSON object" document). + + :raises acme.jose.errors.DeserializationError: + if decoding was unsuccessful, e.g. in case of unparseable + X509 certificate, or wrong padding in JOSE base64 encoded + string, etc. + + """ + # TypeError: Can't instantiate abstract class with + # abstract methods from_json, to_partial_json + return cls() # pylint: disable=abstract-class-instantiated + + @classmethod + def json_loads(cls, json_string): + """Deserialize from JSON document string.""" + try: + loads = json.loads(json_string) + except ValueError as error: + raise errors.DeserializationError(error) + return cls.from_json(loads) + + def json_dumps(self, **kwargs): + """Dump to JSON string using proper serializer. + + :returns: JSON document string. + :rtype: str + + """ + return json.dumps(self, default=self.json_dump_default, **kwargs) + + def json_dumps_pretty(self): + """Dump the object to pretty JSON document string. + + :rtype: str + + """ + return self.json_dumps(sort_keys=True, indent=4, separators=(',', ': ')) + + @classmethod + def json_dump_default(cls, python_object): + """Serialize Python object. + + This function is meant to be passed as ``default`` to + :func:`json.dump` or :func:`json.dumps`. They call + ``default(python_object)`` only for non-basic Python types, so + this function necessarily raises :class:`TypeError` if + ``python_object`` is not an instance of + :class:`IJSONSerializable`. + + Please read the class docstring for more information. + + """ + if isinstance(python_object, JSONDeSerializable): + return python_object.to_partial_json() + else: # this branch is necessary, cannot just "return" + raise TypeError(repr(python_object) + ' is not JSON serializable') diff --git a/acme/acme/jose/interfaces_test.py b/acme/acme/jose/interfaces_test.py new file mode 100644 index 00000000000..cf98ff37110 --- /dev/null +++ b/acme/acme/jose/interfaces_test.py @@ -0,0 +1,114 @@ +"""Tests for acme.jose.interfaces.""" +import unittest + + +class JSONDeSerializableTest(unittest.TestCase): + # pylint: disable=too-many-instance-attributes + + def setUp(self): + from acme.jose.interfaces import JSONDeSerializable + + # pylint: disable=missing-docstring,invalid-name + + class Basic(JSONDeSerializable): + def __init__(self, v): + self.v = v + + def to_partial_json(self): + return self.v + + @classmethod + def from_json(cls, jobj): + return cls(jobj) + + class Sequence(JSONDeSerializable): + def __init__(self, x, y): + self.x = x + self.y = y + + def to_partial_json(self): + return [self.x, self.y] + + @classmethod + def from_json(cls, jobj): + return cls( + Basic.from_json(jobj[0]), Basic.from_json(jobj[1])) + + class Mapping(JSONDeSerializable): + def __init__(self, x, y): + self.x = x + self.y = y + + def to_partial_json(self): + return {self.x: self.y} + + @classmethod + def from_json(cls, jobj): + pass # pragma: no cover + + self.basic1 = Basic('foo1') + self.basic2 = Basic('foo2') + self.seq = Sequence(self.basic1, self.basic2) + self.mapping = Mapping(self.basic1, self.basic2) + self.nested = Basic([[self.basic1]]) + self.tuple = Basic(('foo',)) + + # pylint: disable=invalid-name + self.Basic = Basic + self.Sequence = Sequence + self.Mapping = Mapping + + def test_to_json_sequence(self): + self.assertEqual(self.seq.to_json(), ['foo1', 'foo2']) + + def test_to_json_mapping(self): + self.assertEqual(self.mapping.to_json(), {'foo1': 'foo2'}) + + def test_to_json_other(self): + mock_value = object() + self.assertTrue(self.Basic(mock_value).to_json() is mock_value) + + def test_to_json_nested(self): + self.assertEqual(self.nested.to_json(), [['foo1']]) + + def test_to_json(self): + self.assertEqual(self.tuple.to_json(), (('foo', ))) + + def test_from_json_not_implemented(self): + from acme.jose.interfaces import JSONDeSerializable + self.assertRaises(TypeError, JSONDeSerializable.from_json, 'xxx') + + def test_json_loads(self): + seq = self.Sequence.json_loads('["foo1", "foo2"]') + self.assertTrue(isinstance(seq, self.Sequence)) + self.assertTrue(isinstance(seq.x, self.Basic)) + self.assertTrue(isinstance(seq.y, self.Basic)) + self.assertEqual(seq.x.v, 'foo1') + self.assertEqual(seq.y.v, 'foo2') + + def test_json_dumps(self): + self.assertEqual('["foo1", "foo2"]', self.seq.json_dumps()) + + def test_json_dumps_pretty(self): + self.assertEqual(self.seq.json_dumps_pretty(), + '[\n "foo1",\n "foo2"\n]') + + def test_json_dump_default(self): + from acme.jose.interfaces import JSONDeSerializable + + self.assertEqual( + 'foo1', JSONDeSerializable.json_dump_default(self.basic1)) + + jobj = JSONDeSerializable.json_dump_default(self.seq) + self.assertEqual(len(jobj), 2) + self.assertTrue(jobj[0] is self.basic1) + self.assertTrue(jobj[1] is self.basic2) + + def test_json_dump_default_type_error(self): + from acme.jose.interfaces import JSONDeSerializable + self.assertRaises( + TypeError, JSONDeSerializable.json_dump_default, object()) + + +if __name__ == '__main__': + unittest.main() # pragma: no cover diff --git a/acme/acme/jose/json_util.py b/acme/acme/jose/json_util.py new file mode 100644 index 00000000000..4baadda5eea --- /dev/null +++ b/acme/acme/jose/json_util.py @@ -0,0 +1,485 @@ +"""JSON (de)serialization framework. + +The framework presented here is somewhat based on `Go's "json" package`_ +(especially the ``omitempty`` functionality). + +.. _`Go's "json" package`: http://golang.org/pkg/encoding/json/ + +""" +import abc +import binascii +import logging + +import OpenSSL +import six + +from acme.jose import b64 +from acme.jose import errors +from acme.jose import interfaces +from acme.jose import util + + +logger = logging.getLogger(__name__) + + +class Field(object): + """JSON object field. + + :class:`Field` is meant to be used together with + :class:`JSONObjectWithFields`. + + ``encoder`` (``decoder``) is a callable that accepts a single + parameter, i.e. a value to be encoded (decoded), and returns the + serialized (deserialized) value. In case of errors it should raise + :class:`~acme.jose.errors.SerializationError` + (:class:`~acme.jose.errors.DeserializationError`). + + Note, that ``decoder`` should perform partial serialization only. + + :ivar str json_name: Name of the field when encoded to JSON. + :ivar default: Default value (used when not present in JSON object). + :ivar bool omitempty: If ``True`` and the field value is empty, then + it will not be included in the serialized JSON object, and + ``default`` will be used for deserialization. Otherwise, if ``False``, + field is considered as required, value will always be included in the + serialized JSON objected, and it must also be present when + deserializing. + + """ + __slots__ = ('json_name', 'default', 'omitempty', 'fdec', 'fenc') + + def __init__(self, json_name, default=None, omitempty=False, + decoder=None, encoder=None): + # pylint: disable=too-many-arguments + self.json_name = json_name + self.default = default + self.omitempty = omitempty + + self.fdec = self.default_decoder if decoder is None else decoder + self.fenc = self.default_encoder if encoder is None else encoder + + @classmethod + def _empty(cls, value): + """Is the provided value considered "empty" for this field? + + This is useful for subclasses that might want to override the + definition of being empty, e.g. for some more exotic data types. + + """ + return not isinstance(value, bool) and not value + + def omit(self, value): + """Omit the value in output?""" + return self._empty(value) and self.omitempty + + def _update_params(self, **kwargs): + current = dict(json_name=self.json_name, default=self.default, + omitempty=self.omitempty, + decoder=self.fdec, encoder=self.fenc) + current.update(kwargs) + return type(self)(**current) # pylint: disable=star-args + + def decoder(self, fdec): + """Descriptor to change the decoder on JSON object field.""" + return self._update_params(decoder=fdec) + + def encoder(self, fenc): + """Descriptor to change the encoder on JSON object field.""" + return self._update_params(encoder=fenc) + + def decode(self, value): + """Decode a value, optionally with context JSON object.""" + return self.fdec(value) + + def encode(self, value): + """Encode a value, optionally with context JSON object.""" + return self.fenc(value) + + @classmethod + def default_decoder(cls, value): + """Default decoder. + + Recursively deserialize into immutable types ( + :class:`acme.jose.util.frozendict` instead of + :func:`dict`, :func:`tuple` instead of :func:`list`). + + """ + # bases cases for different types returned by json.loads + if isinstance(value, list): + return tuple(cls.default_decoder(subvalue) for subvalue in value) + elif isinstance(value, dict): + return util.frozendict( + dict((cls.default_decoder(key), cls.default_decoder(value)) + for key, value in six.iteritems(value))) + else: # integer or string + return value + + @classmethod + def default_encoder(cls, value): + """Default (passthrough) encoder.""" + # field.to_partial_json() is no good as encoder has to do partial + # serialization only + return value + + +class JSONObjectWithFieldsMeta(abc.ABCMeta): + """Metaclass for :class:`JSONObjectWithFields` and its subclasses. + + It makes sure that, for any class ``cls`` with ``__metaclass__`` + set to ``JSONObjectWithFieldsMeta``: + + 1. All fields (attributes of type :class:`Field`) in the class + definition are moved to the ``cls._fields`` dictionary, where + keys are field attribute names and values are fields themselves. + + 2. ``cls.__slots__`` is extended by all field attribute names + (i.e. not :attr:`Field.json_name`). Original ``cls.__slots__`` + are stored in ``cls._orig_slots``. + + In a consequence, for a field attribute name ``some_field``, + ``cls.some_field`` will be a slot descriptor and not an instance + of :class:`Field`. For example:: + + some_field = Field('someField', default=()) + + class Foo(object): + __metaclass__ = JSONObjectWithFieldsMeta + __slots__ = ('baz',) + some_field = some_field + + assert Foo.__slots__ == ('some_field', 'baz') + assert Foo._orig_slots == () + assert Foo.some_field is not Field + + assert Foo._fields.keys() == ['some_field'] + assert Foo._fields['some_field'] is some_field + + As an implementation note, this metaclass inherits from + :class:`abc.ABCMeta` (and not the usual :class:`type`) to mitigate + the metaclass conflict (:class:`ImmutableMap` and + :class:`JSONDeSerializable`, parents of :class:`JSONObjectWithFields`, + use :class:`abc.ABCMeta` as its metaclass). + + """ + + def __new__(mcs, name, bases, dikt): + fields = {} + + for base in bases: + fields.update(getattr(base, '_fields', {})) + # Do not reorder, this class might override fields from base classes! + for key, value in tuple(six.iteritems(dikt)): + # not six.iterkeys() (in-place edit!) + if isinstance(value, Field): + fields[key] = dikt.pop(key) + + dikt['_orig_slots'] = dikt.get('__slots__', ()) + dikt['__slots__'] = tuple( + list(dikt['_orig_slots']) + list(six.iterkeys(fields))) + dikt['_fields'] = fields + + return abc.ABCMeta.__new__(mcs, name, bases, dikt) + + +@six.add_metaclass(JSONObjectWithFieldsMeta) +class JSONObjectWithFields(util.ImmutableMap, interfaces.JSONDeSerializable): + # pylint: disable=too-few-public-methods + """JSON object with fields. + + Example:: + + class Foo(JSONObjectWithFields): + bar = Field('Bar') + empty = Field('Empty', omitempty=True) + + @bar.encoder + def bar(value): + return value + 'bar' + + @bar.decoder + def bar(value): + if not value.endswith('bar'): + raise errors.DeserializationError('No bar suffix!') + return value[:-3] + + assert Foo(bar='baz').to_partial_json() == {'Bar': 'bazbar'} + assert Foo.from_json({'Bar': 'bazbar'}) == Foo(bar='baz') + assert (Foo.from_json({'Bar': 'bazbar', 'Empty': '!'}) + == Foo(bar='baz', empty='!')) + assert Foo(bar='baz').bar == 'baz' + + """ + + @classmethod + def _defaults(cls): + """Get default fields values.""" + return dict([(slot, field.default) for slot, field + in six.iteritems(cls._fields)]) + + def __init__(self, **kwargs): + # pylint: disable=star-args + super(JSONObjectWithFields, self).__init__( + **(dict(self._defaults(), **kwargs))) + + def encode(self, name): + """Encode a single field. + + :param str name: Name of the field to be encoded. + + :raises errors.SerializationError: if field cannot be serialized + :raises errors.Error: if field could not be found + + """ + try: + field = self._fields[name] + except KeyError: + raise errors.Error("Field not found: {0}".format(name)) + + return field.encode(getattr(self, name)) + + def fields_to_partial_json(self): + """Serialize fields to JSON.""" + jobj = {} + omitted = set() + for slot, field in six.iteritems(self._fields): + value = getattr(self, slot) + + if field.omit(value): + omitted.add((slot, value)) + else: + try: + jobj[field.json_name] = field.encode(value) + except errors.SerializationError as error: + raise errors.SerializationError( + 'Could not encode {0} ({1}): {2}'.format( + slot, value, error)) + return jobj + + def to_partial_json(self): + return self.fields_to_partial_json() + + @classmethod + def _check_required(cls, jobj): + missing = set() + for _, field in six.iteritems(cls._fields): + if not field.omitempty and field.json_name not in jobj: + missing.add(field.json_name) + + if missing: + raise errors.DeserializationError( + 'The following fields are required: {0}'.format( + ','.join(missing))) + + @classmethod + def fields_from_json(cls, jobj): + """Deserialize fields from JSON.""" + cls._check_required(jobj) + fields = {} + for slot, field in six.iteritems(cls._fields): + if field.json_name not in jobj and field.omitempty: + fields[slot] = field.default + else: + value = jobj[field.json_name] + try: + fields[slot] = field.decode(value) + except errors.DeserializationError as error: + raise errors.DeserializationError( + 'Could not decode {0!r} ({1!r}): {2}'.format( + slot, value, error)) + return fields + + @classmethod + def from_json(cls, jobj): + return cls(**cls.fields_from_json(jobj)) + + +def encode_b64jose(data): + """Encode JOSE Base-64 field. + + :param bytes data: + :rtype: `unicode` + + """ + # b64encode produces ASCII characters only + return b64.b64encode(data).decode('ascii') + + +def decode_b64jose(data, size=None, minimum=False): + """Decode JOSE Base-64 field. + + :param unicode data: + :param int size: Required length (after decoding). + :param bool minimum: If ``True``, then `size` will be treated as + minimum required length, as opposed to exact equality. + + :rtype: bytes + + """ + error_cls = TypeError if six.PY2 else binascii.Error + try: + decoded = b64.b64decode(data.encode()) + except error_cls as error: + raise errors.DeserializationError(error) + + if size is not None and ((not minimum and len(decoded) != size) or + (minimum and len(decoded) < size)): + raise errors.DeserializationError( + "Expected at least or exactly {0} bytes".format(size)) + + return decoded + + +def encode_hex16(value): + """Hexlify. + + :param bytes value: + :rtype: unicode + + """ + return binascii.hexlify(value).decode() + + +def decode_hex16(value, size=None, minimum=False): + """Decode hexlified field. + + :param unicode value: + :param int size: Required length (after decoding). + :param bool minimum: If ``True``, then `size` will be treated as + minimum required length, as opposed to exact equality. + + :rtype: bytes + + """ + value = value.encode() + if size is not None and ((not minimum and len(value) != size * 2) or + (minimum and len(value) < size * 2)): + raise errors.DeserializationError() + error_cls = TypeError if six.PY2 else binascii.Error + try: + return binascii.unhexlify(value) + except error_cls as error: + raise errors.DeserializationError(error) + + +def encode_cert(cert): + """Encode certificate as JOSE Base-64 DER. + + :type cert: `OpenSSL.crypto.X509` wrapped in `.ComparableX509` + :rtype: unicode + + """ + return encode_b64jose(OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_ASN1, cert.wrapped)) + + +def decode_cert(b64der): + """Decode JOSE Base-64 DER-encoded certificate. + + :param unicode b64der: + :rtype: `OpenSSL.crypto.X509` wrapped in `.ComparableX509` + + """ + try: + return util.ComparableX509(OpenSSL.crypto.load_certificate( + OpenSSL.crypto.FILETYPE_ASN1, decode_b64jose(b64der))) + except OpenSSL.crypto.Error as error: + raise errors.DeserializationError(error) + + +def encode_csr(csr): + """Encode CSR as JOSE Base-64 DER. + + :type csr: `OpenSSL.crypto.X509Req` wrapped in `.ComparableX509` + :rtype: unicode + + """ + return encode_b64jose(OpenSSL.crypto.dump_certificate_request( + OpenSSL.crypto.FILETYPE_ASN1, csr.wrapped)) + + +def decode_csr(b64der): + """Decode JOSE Base-64 DER-encoded CSR. + + :param unicode b64der: + :rtype: `OpenSSL.crypto.X509Req` wrapped in `.ComparableX509` + + """ + try: + return util.ComparableX509(OpenSSL.crypto.load_certificate_request( + OpenSSL.crypto.FILETYPE_ASN1, decode_b64jose(b64der))) + except OpenSSL.crypto.Error as error: + raise errors.DeserializationError(error) + + +class TypedJSONObjectWithFields(JSONObjectWithFields): + """JSON object with type.""" + + typ = NotImplemented + """Type of the object. Subclasses must override.""" + + type_field_name = "type" + """Field name used to distinguish different object types. + + Subclasses will probably have to override this. + + """ + + TYPES = NotImplemented + """Types registered for JSON deserialization""" + + @classmethod + def register(cls, type_cls, typ=None): + """Register class for JSON deserialization.""" + typ = type_cls.typ if typ is None else typ + cls.TYPES[typ] = type_cls + return type_cls + + @classmethod + def get_type_cls(cls, jobj): + """Get the registered class for ``jobj``.""" + if cls in six.itervalues(cls.TYPES): + if cls.type_field_name not in jobj: + raise errors.DeserializationError( + "Missing type field ({0})".format(cls.type_field_name)) + # cls is already registered type_cls, force to use it + # so that, e.g Revocation.from_json(jobj) fails if + # jobj["type"] != "revocation". + return cls + + if not isinstance(jobj, dict): + raise errors.DeserializationError( + "{0} is not a dictionary object".format(jobj)) + try: + typ = jobj[cls.type_field_name] + except KeyError: + raise errors.DeserializationError("missing type field") + + try: + return cls.TYPES[typ] + except KeyError: + raise errors.UnrecognizedTypeError(typ, jobj) + + def to_partial_json(self): + """Get JSON serializable object. + + :returns: Serializable JSON object representing ACME typed object. + :meth:`validate` will almost certainly not work, due to reasons + explained in :class:`acme.interfaces.IJSONSerializable`. + :rtype: dict + + """ + jobj = self.fields_to_partial_json() + jobj[self.type_field_name] = self.typ + return jobj + + @classmethod + def from_json(cls, jobj): + """Deserialize ACME object from valid JSON object. + + :raises acme.errors.UnrecognizedTypeError: if type + of the ACME object has not been registered. + + """ + # make sure subclasses don't cause infinite recursive from_json calls + type_cls = cls.get_type_cls(jobj) + return type_cls(**type_cls.fields_from_json(jobj)) diff --git a/acme/acme/jose/json_util_test.py b/acme/acme/jose/json_util_test.py new file mode 100644 index 00000000000..25e36211eec --- /dev/null +++ b/acme/acme/jose/json_util_test.py @@ -0,0 +1,381 @@ +"""Tests for acme.jose.json_util.""" +import itertools +import unittest + +import mock +import six + +from acme import test_util + +from acme.jose import errors +from acme.jose import interfaces +from acme.jose import util + + +CERT = test_util.load_comparable_cert('cert.pem') +CSR = test_util.load_comparable_csr('csr.pem') + + +class FieldTest(unittest.TestCase): + """Tests for acme.jose.json_util.Field.""" + + def test_no_omit_boolean(self): + from acme.jose.json_util import Field + for default, omitempty, value in itertools.product( + [True, False], [True, False], [True, False]): + self.assertFalse( + Field("foo", default=default, omitempty=omitempty).omit(value)) + + def test_descriptors(self): + mock_value = mock.MagicMock() + + # pylint: disable=missing-docstring + + def decoder(unused_value): + return 'd' + + def encoder(unused_value): + return 'e' + + from acme.jose.json_util import Field + field = Field('foo') + + field = field.encoder(encoder) + self.assertEqual('e', field.encode(mock_value)) + + field = field.decoder(decoder) + self.assertEqual('e', field.encode(mock_value)) + self.assertEqual('d', field.decode(mock_value)) + + def test_default_encoder_is_partial(self): + class MockField(interfaces.JSONDeSerializable): + # pylint: disable=missing-docstring + def to_partial_json(self): + return 'foo' # pragma: no cover + + @classmethod + def from_json(cls, jobj): + pass # pragma: no cover + mock_field = MockField() + + from acme.jose.json_util import Field + self.assertTrue(Field.default_encoder(mock_field) is mock_field) + # in particular... + self.assertNotEqual('foo', Field.default_encoder(mock_field)) + + def test_default_encoder_passthrough(self): + mock_value = mock.MagicMock() + from acme.jose.json_util import Field + self.assertTrue(Field.default_encoder(mock_value) is mock_value) + + def test_default_decoder_list_to_tuple(self): + from acme.jose.json_util import Field + self.assertEqual((1, 2, 3), Field.default_decoder([1, 2, 3])) + + def test_default_decoder_dict_to_frozendict(self): + from acme.jose.json_util import Field + obj = Field.default_decoder({'x': 2}) + self.assertTrue(isinstance(obj, util.frozendict)) + self.assertEqual(obj, util.frozendict(x=2)) + + def test_default_decoder_passthrough(self): + mock_value = mock.MagicMock() + from acme.jose.json_util import Field + self.assertTrue(Field.default_decoder(mock_value) is mock_value) + + +class JSONObjectWithFieldsMetaTest(unittest.TestCase): + """Tests for acme.jose.json_util.JSONObjectWithFieldsMeta.""" + + def setUp(self): + from acme.jose.json_util import Field + from acme.jose.json_util import JSONObjectWithFieldsMeta + self.field = Field('Baz') + self.field2 = Field('Baz2') + # pylint: disable=invalid-name,missing-docstring,too-few-public-methods + # pylint: disable=blacklisted-name + + @six.add_metaclass(JSONObjectWithFieldsMeta) + class A(object): + __slots__ = ('bar',) + baz = self.field + + class B(A): + pass + + class C(A): + baz = self.field2 + + self.a_cls = A + self.b_cls = B + self.c_cls = C + + def test_fields(self): + # pylint: disable=protected-access,no-member + self.assertEqual({'baz': self.field}, self.a_cls._fields) + self.assertEqual({'baz': self.field}, self.b_cls._fields) + + def test_fields_inheritance(self): + # pylint: disable=protected-access,no-member + self.assertEqual({'baz': self.field2}, self.c_cls._fields) + + def test_slots(self): + self.assertEqual(('bar', 'baz'), self.a_cls.__slots__) + self.assertEqual(('baz',), self.b_cls.__slots__) + + def test_orig_slots(self): + # pylint: disable=protected-access,no-member + self.assertEqual(('bar',), self.a_cls._orig_slots) + self.assertEqual((), self.b_cls._orig_slots) + + +class JSONObjectWithFieldsTest(unittest.TestCase): + """Tests for acme.jose.json_util.JSONObjectWithFields.""" + # pylint: disable=protected-access + + def setUp(self): + from acme.jose.json_util import JSONObjectWithFields + from acme.jose.json_util import Field + + class MockJSONObjectWithFields(JSONObjectWithFields): + # pylint: disable=invalid-name,missing-docstring,no-self-argument + # pylint: disable=too-few-public-methods + x = Field('x', omitempty=True, + encoder=(lambda x: x * 2), + decoder=(lambda x: x / 2)) + y = Field('y') + z = Field('Z') # on purpose uppercase + + @y.encoder + def y(value): + if value == 500: + raise errors.SerializationError() + return value + + @y.decoder + def y(value): + if value == 500: + raise errors.DeserializationError() + return value + + # pylint: disable=invalid-name + self.MockJSONObjectWithFields = MockJSONObjectWithFields + self.mock = MockJSONObjectWithFields(x=None, y=2, z=3) + + def test_init_defaults(self): + self.assertEqual(self.mock, self.MockJSONObjectWithFields(y=2, z=3)) + + def test_encode(self): + self.assertEqual(10, self.MockJSONObjectWithFields( + x=5, y=0, z=0).encode("x")) + + def test_encode_wrong_field(self): + self.assertRaises(errors.Error, self.mock.encode, 'foo') + + def test_encode_serialization_error_passthrough(self): + self.assertRaises( + errors.SerializationError, + self.MockJSONObjectWithFields(y=500, z=None).encode, "y") + + def test_fields_to_partial_json_omits_empty(self): + self.assertEqual(self.mock.fields_to_partial_json(), {'y': 2, 'Z': 3}) + + def test_fields_from_json_fills_default_for_empty(self): + self.assertEqual( + {'x': None, 'y': 2, 'z': 3}, + self.MockJSONObjectWithFields.fields_from_json({'y': 2, 'Z': 3})) + + def test_fields_from_json_fails_on_missing(self): + self.assertRaises( + errors.DeserializationError, + self.MockJSONObjectWithFields.fields_from_json, {'y': 0}) + self.assertRaises( + errors.DeserializationError, + self.MockJSONObjectWithFields.fields_from_json, {'Z': 0}) + self.assertRaises( + errors.DeserializationError, + self.MockJSONObjectWithFields.fields_from_json, {'x': 0, 'y': 0}) + self.assertRaises( + errors.DeserializationError, + self.MockJSONObjectWithFields.fields_from_json, {'x': 0, 'Z': 0}) + + def test_fields_to_partial_json_encoder(self): + self.assertEqual( + self.MockJSONObjectWithFields(x=1, y=2, z=3).to_partial_json(), + {'x': 2, 'y': 2, 'Z': 3}) + + def test_fields_from_json_decoder(self): + self.assertEqual( + {'x': 2, 'y': 2, 'z': 3}, + self.MockJSONObjectWithFields.fields_from_json( + {'x': 4, 'y': 2, 'Z': 3})) + + def test_fields_to_partial_json_error_passthrough(self): + self.assertRaises( + errors.SerializationError, self.MockJSONObjectWithFields( + x=1, y=500, z=3).to_partial_json) + + def test_fields_from_json_error_passthrough(self): + self.assertRaises( + errors.DeserializationError, + self.MockJSONObjectWithFields.from_json, + {'x': 4, 'y': 500, 'Z': 3}) + + +class DeEncodersTest(unittest.TestCase): + def setUp(self): + self.b64_cert = ( + u'MIIB3jCCAYigAwIBAgICBTkwDQYJKoZIhvcNAQELBQAwdzELMAkGA1UEBhM' + u'CVVMxETAPBgNVBAgMCE1pY2hpZ2FuMRIwEAYDVQQHDAlBbm4gQXJib3IxKz' + u'ApBgNVBAoMIlVuaXZlcnNpdHkgb2YgTWljaGlnYW4gYW5kIHRoZSBFRkYxF' + u'DASBgNVBAMMC2V4YW1wbGUuY29tMB4XDTE0MTIxMTIyMzQ0NVoXDTE0MTIx' + u'ODIyMzQ0NVowdzELMAkGA1UEBhMCVVMxETAPBgNVBAgMCE1pY2hpZ2FuMRI' + u'wEAYDVQQHDAlBbm4gQXJib3IxKzApBgNVBAoMIlVuaXZlcnNpdHkgb2YgTW' + u'ljaGlnYW4gYW5kIHRoZSBFRkYxFDASBgNVBAMMC2V4YW1wbGUuY29tMFwwD' + u'QYJKoZIhvcNAQEBBQADSwAwSAJBAKx1c7RR7R_drnBSQ_zfx1vQLHUbFLh1' + u'AQQQ5R8DZUXd36efNK79vukFhN9HFoHZiUvOjm0c-pVE6K-EdE_twuUCAwE' + u'AATANBgkqhkiG9w0BAQsFAANBAC24z0IdwIVKSlntksllvr6zJepBH5fMnd' + u'fk3XJp10jT6VE-14KNtjh02a56GoraAvJAT5_H67E8GvJ_ocNnB_o' + ) + self.b64_csr = ( + u'MIIBXTCCAQcCAQAweTELMAkGA1UEBhMCVVMxETAPBgNVBAgMCE1pY2hpZ2F' + u'uMRIwEAYDVQQHDAlBbm4gQXJib3IxDDAKBgNVBAoMA0VGRjEfMB0GA1UECw' + u'wWVW5pdmVyc2l0eSBvZiBNaWNoaWdhbjEUMBIGA1UEAwwLZXhhbXBsZS5jb' + u'20wXDANBgkqhkiG9w0BAQEFAANLADBIAkEArHVztFHtH92ucFJD_N_HW9As' + u'dRsUuHUBBBDlHwNlRd3fp580rv2-6QWE30cWgdmJS86ObRz6lUTor4R0T-3' + u'C5QIDAQABoCkwJwYJKoZIhvcNAQkOMRowGDAWBgNVHREEDzANggtleGFtcG' + u'xlLmNvbTANBgkqhkiG9w0BAQsFAANBAHJH_O6BtC9aGzEVCMGOZ7z9iIRHW' + u'Szr9x_bOzn7hLwsbXPAgO1QxEwL-X-4g20Gn9XBE1N9W6HCIEut2d8wACg' + ) + + def test_encode_b64jose(self): + from acme.jose.json_util import encode_b64jose + encoded = encode_b64jose(b'x') + self.assertTrue(isinstance(encoded, six.string_types)) + self.assertEqual(u'eA', encoded) + + def test_decode_b64jose(self): + from acme.jose.json_util import decode_b64jose + decoded = decode_b64jose(u'eA') + self.assertTrue(isinstance(decoded, six.binary_type)) + self.assertEqual(b'x', decoded) + + def test_decode_b64jose_padding_error(self): + from acme.jose.json_util import decode_b64jose + self.assertRaises(errors.DeserializationError, decode_b64jose, u'x') + + def test_decode_b64jose_size(self): + from acme.jose.json_util import decode_b64jose + self.assertEqual(b'foo', decode_b64jose(u'Zm9v', size=3)) + self.assertRaises( + errors.DeserializationError, decode_b64jose, u'Zm9v', size=2) + self.assertRaises( + errors.DeserializationError, decode_b64jose, u'Zm9v', size=4) + + def test_decode_b64jose_minimum_size(self): + from acme.jose.json_util import decode_b64jose + self.assertEqual(b'foo', decode_b64jose(u'Zm9v', size=3, minimum=True)) + self.assertEqual(b'foo', decode_b64jose(u'Zm9v', size=2, minimum=True)) + self.assertRaises(errors.DeserializationError, decode_b64jose, + u'Zm9v', size=4, minimum=True) + + def test_encode_hex16(self): + from acme.jose.json_util import encode_hex16 + encoded = encode_hex16(b'foo') + self.assertEqual(u'666f6f', encoded) + self.assertTrue(isinstance(encoded, six.string_types)) + + def test_decode_hex16(self): + from acme.jose.json_util import decode_hex16 + decoded = decode_hex16(u'666f6f') + self.assertEqual(b'foo', decoded) + self.assertTrue(isinstance(decoded, six.binary_type)) + + def test_decode_hex16_minimum_size(self): + from acme.jose.json_util import decode_hex16 + self.assertEqual(b'foo', decode_hex16(u'666f6f', size=3, minimum=True)) + self.assertEqual(b'foo', decode_hex16(u'666f6f', size=2, minimum=True)) + self.assertRaises(errors.DeserializationError, decode_hex16, + u'666f6f', size=4, minimum=True) + + def test_decode_hex16_odd_length(self): + from acme.jose.json_util import decode_hex16 + self.assertRaises(errors.DeserializationError, decode_hex16, u'x') + + def test_encode_cert(self): + from acme.jose.json_util import encode_cert + self.assertEqual(self.b64_cert, encode_cert(CERT)) + + def test_decode_cert(self): + from acme.jose.json_util import decode_cert + cert = decode_cert(self.b64_cert) + self.assertTrue(isinstance(cert, util.ComparableX509)) + self.assertEqual(cert, CERT) + self.assertRaises(errors.DeserializationError, decode_cert, u'') + + def test_encode_csr(self): + from acme.jose.json_util import encode_csr + self.assertEqual(self.b64_csr, encode_csr(CSR)) + + def test_decode_csr(self): + from acme.jose.json_util import decode_csr + csr = decode_csr(self.b64_csr) + self.assertTrue(isinstance(csr, util.ComparableX509)) + self.assertEqual(csr, CSR) + self.assertRaises(errors.DeserializationError, decode_csr, u'') + + +class TypedJSONObjectWithFieldsTest(unittest.TestCase): + + def setUp(self): + from acme.jose.json_util import TypedJSONObjectWithFields + + # pylint: disable=missing-docstring,abstract-method + # pylint: disable=too-few-public-methods + + class MockParentTypedJSONObjectWithFields(TypedJSONObjectWithFields): + TYPES = {} + type_field_name = 'type' + + @MockParentTypedJSONObjectWithFields.register + class MockTypedJSONObjectWithFields( + MockParentTypedJSONObjectWithFields): + typ = 'test' + __slots__ = ('foo',) + + @classmethod + def fields_from_json(cls, jobj): + return {'foo': jobj['foo']} + + def fields_to_partial_json(self): + return {'foo': self.foo} + + self.parent_cls = MockParentTypedJSONObjectWithFields + self.msg = MockTypedJSONObjectWithFields(foo='bar') + + def test_to_partial_json(self): + self.assertEqual(self.msg.to_partial_json(), { + 'type': 'test', + 'foo': 'bar', + }) + + def test_from_json_non_dict_fails(self): + for value in [[], (), 5, "asd"]: # all possible input types + self.assertRaises( + errors.DeserializationError, self.parent_cls.from_json, value) + + def test_from_json_dict_no_type_fails(self): + self.assertRaises( + errors.DeserializationError, self.parent_cls.from_json, {}) + + def test_from_json_unknown_type_fails(self): + self.assertRaises(errors.UnrecognizedTypeError, + self.parent_cls.from_json, {'type': 'bar'}) + + def test_from_json_returns_obj(self): + self.assertEqual({'foo': 'bar'}, self.parent_cls.from_json( + {'type': 'test', 'foo': 'bar'})) + + +if __name__ == '__main__': + unittest.main() # pragma: no cover diff --git a/acme/acme/jose/jwa.py b/acme/acme/jose/jwa.py new file mode 100644 index 00000000000..9b682ecabbe --- /dev/null +++ b/acme/acme/jose/jwa.py @@ -0,0 +1,180 @@ +"""JSON Web Algorithm. + +https://tools.ietf.org/html/draft-ietf-jose-json-web-algorithms-40 + +""" +import abc +import collections +import logging + +import cryptography.exceptions +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes # type: ignore +from cryptography.hazmat.primitives import hmac # type: ignore +from cryptography.hazmat.primitives.asymmetric import padding # type: ignore + +from acme.jose import errors +from acme.jose import interfaces +from acme.jose import jwk + + +logger = logging.getLogger(__name__) + + +class JWA(interfaces.JSONDeSerializable): # pylint: disable=abstract-method + # pylint: disable=too-few-public-methods + # for some reason disable=abstract-method has to be on the line + # above... + """JSON Web Algorithm.""" + + +class JWASignature(JWA, collections.Hashable): # type: ignore + """JSON Web Signature Algorithm.""" + SIGNATURES = {} # type: dict + + def __init__(self, name): + self.name = name + + def __eq__(self, other): + if not isinstance(other, JWASignature): + return NotImplemented + return self.name == other.name + + def __hash__(self): + return hash((self.__class__, self.name)) + + def __ne__(self, other): + return not self == other + + @classmethod + def register(cls, signature_cls): + """Register class for JSON deserialization.""" + cls.SIGNATURES[signature_cls.name] = signature_cls + return signature_cls + + def to_partial_json(self): + return self.name + + @classmethod + def from_json(cls, jobj): + return cls.SIGNATURES[jobj] + + @abc.abstractmethod + def sign(self, key, msg): # pragma: no cover + """Sign the ``msg`` using ``key``.""" + raise NotImplementedError() + + @abc.abstractmethod + def verify(self, key, msg, sig): # pragma: no cover + """Verify the ``msg` and ``sig`` using ``key``.""" + raise NotImplementedError() + + def __repr__(self): + return self.name + + +class _JWAHS(JWASignature): + + kty = jwk.JWKOct + + def __init__(self, name, hash_): + super(_JWAHS, self).__init__(name) + self.hash = hash_() + + def sign(self, key, msg): + signer = hmac.HMAC(key, self.hash, backend=default_backend()) + signer.update(msg) + return signer.finalize() + + def verify(self, key, msg, sig): + verifier = hmac.HMAC(key, self.hash, backend=default_backend()) + verifier.update(msg) + try: + verifier.verify(sig) + except cryptography.exceptions.InvalidSignature as error: + logger.debug(error, exc_info=True) + return False + else: + return True + + +class _JWARSA(object): + + kty = jwk.JWKRSA + padding = NotImplemented + hash = NotImplemented + + def sign(self, key, msg): + """Sign the ``msg`` using ``key``.""" + try: + signer = key.signer(self.padding, self.hash) + except AttributeError as error: + logger.debug(error, exc_info=True) + raise errors.Error("Public key cannot be used for signing") + except ValueError as error: # digest too large + logger.debug(error, exc_info=True) + raise errors.Error(str(error)) + signer.update(msg) + try: + return signer.finalize() + except ValueError as error: + logger.debug(error, exc_info=True) + raise errors.Error(str(error)) + + def verify(self, key, msg, sig): + """Verify the ``msg` and ``sig`` using ``key``.""" + verifier = key.verifier(sig, self.padding, self.hash) + verifier.update(msg) + try: + verifier.verify() + except cryptography.exceptions.InvalidSignature as error: + logger.debug(error, exc_info=True) + return False + else: + return True + + +class _JWARS(_JWARSA, JWASignature): + + def __init__(self, name, hash_): + super(_JWARS, self).__init__(name) + self.padding = padding.PKCS1v15() + self.hash = hash_() + + +class _JWAPS(_JWARSA, JWASignature): + + def __init__(self, name, hash_): + super(_JWAPS, self).__init__(name) + self.padding = padding.PSS( + mgf=padding.MGF1(hash_()), + salt_length=padding.PSS.MAX_LENGTH) + self.hash = hash_() + + +class _JWAES(JWASignature): # pylint: disable=abstract-class-not-used + + # TODO: implement ES signatures + + def sign(self, key, msg): # pragma: no cover + raise NotImplementedError() + + def verify(self, key, msg, sig): # pragma: no cover + raise NotImplementedError() + + +HS256 = JWASignature.register(_JWAHS('HS256', hashes.SHA256)) +HS384 = JWASignature.register(_JWAHS('HS384', hashes.SHA384)) +HS512 = JWASignature.register(_JWAHS('HS512', hashes.SHA512)) + +RS256 = JWASignature.register(_JWARS('RS256', hashes.SHA256)) +RS384 = JWASignature.register(_JWARS('RS384', hashes.SHA384)) +RS512 = JWASignature.register(_JWARS('RS512', hashes.SHA512)) + +PS256 = JWASignature.register(_JWAPS('PS256', hashes.SHA256)) +PS384 = JWASignature.register(_JWAPS('PS384', hashes.SHA384)) +PS512 = JWASignature.register(_JWAPS('PS512', hashes.SHA512)) + +ES256 = JWASignature.register(_JWAES('ES256')) +ES384 = JWASignature.register(_JWAES('ES384')) +ES512 = JWASignature.register(_JWAES('ES512')) diff --git a/acme/acme/jose/jwa_test.py b/acme/acme/jose/jwa_test.py new file mode 100644 index 00000000000..3328d083ab9 --- /dev/null +++ b/acme/acme/jose/jwa_test.py @@ -0,0 +1,104 @@ +"""Tests for acme.jose.jwa.""" +import unittest + +from acme import test_util + +from acme.jose import errors + + +RSA256_KEY = test_util.load_rsa_private_key('rsa256_key.pem') +RSA512_KEY = test_util.load_rsa_private_key('rsa512_key.pem') +RSA1024_KEY = test_util.load_rsa_private_key('rsa1024_key.pem') + + +class JWASignatureTest(unittest.TestCase): + """Tests for acme.jose.jwa.JWASignature.""" + + def setUp(self): + from acme.jose.jwa import JWASignature + + class MockSig(JWASignature): + # pylint: disable=missing-docstring,too-few-public-methods + # pylint: disable=abstract-class-not-used + def sign(self, key, msg): + raise NotImplementedError() # pragma: no cover + + def verify(self, key, msg, sig): + raise NotImplementedError() # pragma: no cover + + # pylint: disable=invalid-name + self.Sig1 = MockSig('Sig1') + self.Sig2 = MockSig('Sig2') + + def test_eq(self): + self.assertEqual(self.Sig1, self.Sig1) + + def test_ne(self): + self.assertNotEqual(self.Sig1, self.Sig2) + + def test_ne_other_type(self): + self.assertNotEqual(self.Sig1, 5) + + def test_repr(self): + self.assertEqual('Sig1', repr(self.Sig1)) + self.assertEqual('Sig2', repr(self.Sig2)) + + def test_to_partial_json(self): + self.assertEqual(self.Sig1.to_partial_json(), 'Sig1') + self.assertEqual(self.Sig2.to_partial_json(), 'Sig2') + + def test_from_json(self): + from acme.jose.jwa import JWASignature + from acme.jose.jwa import RS256 + self.assertTrue(JWASignature.from_json('RS256') is RS256) + + +class JWAHSTest(unittest.TestCase): # pylint: disable=too-few-public-methods + + def test_it(self): + from acme.jose.jwa import HS256 + sig = ( + b"\xceR\xea\xcd\x94\xab\xcf\xfb\xe0\xacA.:\x1a'\x08i\xe2\xc4" + b"\r\x85+\x0e\x85\xaeUZ\xd4\xb3\x97zO" + ) + self.assertEqual(HS256.sign(b'some key', b'foo'), sig) + self.assertTrue(HS256.verify(b'some key', b'foo', sig) is True) + self.assertTrue(HS256.verify(b'some key', b'foo', sig + b'!') is False) + + +class JWARSTest(unittest.TestCase): + + def test_sign_no_private_part(self): + from acme.jose.jwa import RS256 + self.assertRaises( + errors.Error, RS256.sign, RSA512_KEY.public_key(), b'foo') + + def test_sign_key_too_small(self): + from acme.jose.jwa import RS256 + from acme.jose.jwa import PS256 + self.assertRaises(errors.Error, RS256.sign, RSA256_KEY, b'foo') + self.assertRaises(errors.Error, PS256.sign, RSA256_KEY, b'foo') + + def test_rs(self): + from acme.jose.jwa import RS256 + sig = ( + b'|\xc6\xb2\xa4\xab(\x87\x99\xfa*:\xea\xf8\xa0N&}\x9f\x0f\xc0O' + b'\xc6t\xa3\xe6\xfa\xbb"\x15Y\x80Y\xe0\x81\xb8\x88)\xba\x0c\x9c' + b'\xa4\x99\x1e\x19&\xd8\xc7\x99S\x97\xfc\x85\x0cOV\xe6\x07\x99' + b'\xd2\xb9.>}\xfd' + ) + self.assertEqual(RS256.sign(RSA512_KEY, b'foo'), sig) + self.assertTrue(RS256.verify(RSA512_KEY.public_key(), b'foo', sig)) + self.assertFalse(RS256.verify( + RSA512_KEY.public_key(), b'foo', sig + b'!')) + + def test_ps(self): + from acme.jose.jwa import PS256 + sig = PS256.sign(RSA1024_KEY, b'foo') + self.assertTrue(PS256.verify(RSA1024_KEY.public_key(), b'foo', sig)) + self.assertFalse(PS256.verify( + RSA1024_KEY.public_key(), b'foo', sig + b'!')) + + +if __name__ == '__main__': + unittest.main() # pragma: no cover diff --git a/acme/acme/jose/jwk.py b/acme/acme/jose/jwk.py new file mode 100644 index 00000000000..54423f6709f --- /dev/null +++ b/acme/acme/jose/jwk.py @@ -0,0 +1,281 @@ +"""JSON Web Key.""" +import abc +import binascii +import json +import logging + +import cryptography.exceptions +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes # type: ignore +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import ec # type: ignore +from cryptography.hazmat.primitives.asymmetric import rsa + +import six + +from acme.jose import errors +from acme.jose import json_util +from acme.jose import util + + +logger = logging.getLogger(__name__) + + +class JWK(json_util.TypedJSONObjectWithFields): + # pylint: disable=too-few-public-methods + """JSON Web Key.""" + type_field_name = 'kty' + TYPES = {} # type: dict + cryptography_key_types = () # type: tuple + """Subclasses should override.""" + + required = NotImplemented + """Required members of public key's representation as defined by JWK/JWA.""" + + _thumbprint_json_dumps_params = { + # "no whitespace or line breaks before or after any syntactic + # elements" + 'indent': None, + 'separators': (',', ':'), + # "members ordered lexicographically by the Unicode [UNICODE] + # code points of the member names" + 'sort_keys': True, + } + + def thumbprint(self, hash_function=hashes.SHA256): + """Compute JWK Thumbprint. + + https://tools.ietf.org/html/rfc7638 + + :returns bytes: + + """ + digest = hashes.Hash(hash_function(), backend=default_backend()) + digest.update(json.dumps( + dict((k, v) for k, v in six.iteritems(self.to_json()) + if k in self.required), + **self._thumbprint_json_dumps_params).encode()) + return digest.finalize() + + @abc.abstractmethod + def public_key(self): # pragma: no cover + """Generate JWK with public key. + + For symmetric cryptosystems, this would return ``self``. + + """ + raise NotImplementedError() + + @classmethod + def _load_cryptography_key(cls, data, password=None, backend=None): + backend = default_backend() if backend is None else backend + exceptions = {} + + # private key? + for loader in (serialization.load_pem_private_key, + serialization.load_der_private_key): + try: + return loader(data, password, backend) + except (ValueError, TypeError, + cryptography.exceptions.UnsupportedAlgorithm) as error: + exceptions[loader] = error + + # public key? + for loader in (serialization.load_pem_public_key, + serialization.load_der_public_key): + try: + return loader(data, backend) + except (ValueError, + cryptography.exceptions.UnsupportedAlgorithm) as error: + exceptions[loader] = error + + # no luck + raise errors.Error('Unable to deserialize key: {0}'.format(exceptions)) + + @classmethod + def load(cls, data, password=None, backend=None): + """Load serialized key as JWK. + + :param str data: Public or private key serialized as PEM or DER. + :param str password: Optional password. + :param backend: A `.PEMSerializationBackend` and + `.DERSerializationBackend` provider. + + :raises errors.Error: if unable to deserialize, or unsupported + JWK algorithm + + :returns: JWK of an appropriate type. + :rtype: `JWK` + + """ + try: + key = cls._load_cryptography_key(data, password, backend) + except errors.Error as error: + logger.debug('Loading symmetric key, asymmetric failed: %s', error) + return JWKOct(key=data) + + if cls.typ is not NotImplemented and not isinstance( + key, cls.cryptography_key_types): + raise errors.Error('Unable to deserialize {0} into {1}'.format( + key.__class__, cls.__class__)) + for jwk_cls in six.itervalues(cls.TYPES): + if isinstance(key, jwk_cls.cryptography_key_types): + return jwk_cls(key=key) + raise errors.Error('Unsupported algorithm: {0}'.format(key.__class__)) + + +@JWK.register +class JWKES(JWK): # pragma: no cover + # pylint: disable=abstract-class-not-used + """ES JWK. + + .. warning:: This is not yet implemented! + + """ + typ = 'ES' + cryptography_key_types = ( + ec.EllipticCurvePublicKey, ec.EllipticCurvePrivateKey) + required = ('crv', JWK.type_field_name, 'x', 'y') + + def fields_to_partial_json(self): + raise NotImplementedError() + + @classmethod + def fields_from_json(cls, jobj): + raise NotImplementedError() + + def public_key(self): + raise NotImplementedError() + + +@JWK.register +class JWKOct(JWK): + """Symmetric JWK.""" + typ = 'oct' + __slots__ = ('key',) + required = ('k', JWK.type_field_name) + + def fields_to_partial_json(self): + # TODO: An "alg" member SHOULD also be present to identify the + # algorithm intended to be used with the key, unless the + # application uses another means or convention to determine + # the algorithm used. + return {'k': json_util.encode_b64jose(self.key)} + + @classmethod + def fields_from_json(cls, jobj): + return cls(key=json_util.decode_b64jose(jobj['k'])) + + def public_key(self): + return self + + +@JWK.register +class JWKRSA(JWK): + """RSA JWK. + + :ivar key: `cryptography.hazmat.primitives.rsa.RSAPrivateKey` + or `cryptography.hazmat.primitives.rsa.RSAPublicKey` wrapped + in `.ComparableRSAKey` + + """ + typ = 'RSA' + cryptography_key_types = (rsa.RSAPublicKey, rsa.RSAPrivateKey) + __slots__ = ('key',) + required = ('e', JWK.type_field_name, 'n') + + def __init__(self, *args, **kwargs): + if 'key' in kwargs and not isinstance( + kwargs['key'], util.ComparableRSAKey): + kwargs['key'] = util.ComparableRSAKey(kwargs['key']) + super(JWKRSA, self).__init__(*args, **kwargs) + + @classmethod + def _encode_param(cls, data): + """Encode Base64urlUInt. + + :type data: long + :rtype: unicode + + """ + def _leading_zeros(arg): + if len(arg) % 2: + return '0' + arg + return arg + + return json_util.encode_b64jose(binascii.unhexlify( + _leading_zeros(hex(data)[2:].rstrip('L')))) + + @classmethod + def _decode_param(cls, data): + """Decode Base64urlUInt.""" + try: + return int(binascii.hexlify(json_util.decode_b64jose(data)), 16) + except ValueError: # invalid literal for long() with base 16 + raise errors.DeserializationError() + + def public_key(self): + return type(self)(key=self.key.public_key()) + + @classmethod + def fields_from_json(cls, jobj): + # pylint: disable=invalid-name + n, e = (cls._decode_param(jobj[x]) for x in ('n', 'e')) + public_numbers = rsa.RSAPublicNumbers(e=e, n=n) + if 'd' not in jobj: # public key + key = public_numbers.public_key(default_backend()) + else: # private key + d = cls._decode_param(jobj['d']) + if ('p' in jobj or 'q' in jobj or 'dp' in jobj or + 'dq' in jobj or 'qi' in jobj or 'oth' in jobj): + # "If the producer includes any of the other private + # key parameters, then all of the others MUST be + # present, with the exception of "oth", which MUST + # only be present when more than two prime factors + # were used." + p, q, dp, dq, qi, = all_params = tuple( + jobj.get(x) for x in ('p', 'q', 'dp', 'dq', 'qi')) + if tuple(param for param in all_params if param is None): + raise errors.Error( + 'Some private parameters are missing: {0}'.format( + all_params)) + p, q, dp, dq, qi = tuple( + cls._decode_param(x) for x in all_params) + + # TODO: check for oth + else: + # cryptography>=0.8 + p, q = rsa.rsa_recover_prime_factors(n, e, d) + dp = rsa.rsa_crt_dmp1(d, p) + dq = rsa.rsa_crt_dmq1(d, q) + qi = rsa.rsa_crt_iqmp(p, q) + + key = rsa.RSAPrivateNumbers( + p, q, d, dp, dq, qi, public_numbers).private_key( + default_backend()) + + return cls(key=key) + + def fields_to_partial_json(self): + # pylint: disable=protected-access + if isinstance(self.key._wrapped, rsa.RSAPublicKey): + numbers = self.key.public_numbers() + params = { + 'n': numbers.n, + 'e': numbers.e, + } + else: # rsa.RSAPrivateKey + private = self.key.private_numbers() + public = self.key.public_key().public_numbers() + params = { + 'n': public.n, + 'e': public.e, + 'd': private.d, + 'p': private.p, + 'q': private.q, + 'dp': private.dmp1, + 'dq': private.dmq1, + 'qi': private.iqmp, + } + return dict((key, self._encode_param(value)) + for key, value in six.iteritems(params)) diff --git a/acme/acme/jose/jwk_test.py b/acme/acme/jose/jwk_test.py new file mode 100644 index 00000000000..eea5793bfa6 --- /dev/null +++ b/acme/acme/jose/jwk_test.py @@ -0,0 +1,191 @@ +"""Tests for acme.jose.jwk.""" +import binascii +import unittest + +from acme import test_util + +from acme.jose import errors +from acme.jose import json_util +from acme.jose import util + + +DSA_PEM = test_util.load_vector('dsa512_key.pem') +RSA256_KEY = test_util.load_rsa_private_key('rsa256_key.pem') +RSA512_KEY = test_util.load_rsa_private_key('rsa512_key.pem') + + +class JWKTest(unittest.TestCase): + """Tests for acme.jose.jwk.JWK.""" + + def test_load(self): + from acme.jose.jwk import JWK + self.assertRaises(errors.Error, JWK.load, DSA_PEM) + + def test_load_subclass_wrong_type(self): + from acme.jose.jwk import JWKRSA + self.assertRaises(errors.Error, JWKRSA.load, DSA_PEM) + + +class JWKTestBaseMixin(object): + """Mixin test for JWK subclass tests.""" + + thumbprint = NotImplemented + + def test_thumbprint_private(self): + self.assertEqual(self.thumbprint, self.jwk.thumbprint()) + + def test_thumbprint_public(self): + self.assertEqual(self.thumbprint, self.jwk.public_key().thumbprint()) + + +class JWKOctTest(unittest.TestCase, JWKTestBaseMixin): + """Tests for acme.jose.jwk.JWKOct.""" + + thumbprint = (b"\xf3\xe7\xbe\xa8`\xd2\xdap\xe9}\x9c\xce>" + b"\xd0\xfcI\xbe\xcd\x92'\xd4o\x0e\xf41\xea" + b"\x8e(\x8a\xb2i\x1c") + + def setUp(self): + from acme.jose.jwk import JWKOct + self.jwk = JWKOct(key=b'foo') + self.jobj = {'kty': 'oct', 'k': json_util.encode_b64jose(b'foo')} + + def test_to_partial_json(self): + self.assertEqual(self.jwk.to_partial_json(), self.jobj) + + def test_from_json(self): + from acme.jose.jwk import JWKOct + self.assertEqual(self.jwk, JWKOct.from_json(self.jobj)) + + def test_from_json_hashable(self): + from acme.jose.jwk import JWKOct + hash(JWKOct.from_json(self.jobj)) + + def test_load(self): + from acme.jose.jwk import JWKOct + self.assertEqual(self.jwk, JWKOct.load(b'foo')) + + def test_public_key(self): + self.assertTrue(self.jwk.public_key() is self.jwk) + + +class JWKRSATest(unittest.TestCase, JWKTestBaseMixin): + """Tests for acme.jose.jwk.JWKRSA.""" + # pylint: disable=too-many-instance-attributes + + thumbprint = (b'\x83K\xdc#3\x98\xca\x98\xed\xcb\x80\x80<\x0c' + b'\xf0\x95\xb9H\xb2*l\xbd$\xe5&|O\x91\xd4 \xb0Y') + + def setUp(self): + from acme.jose.jwk import JWKRSA + self.jwk256 = JWKRSA(key=RSA256_KEY.public_key()) + self.jwk256json = { + 'kty': 'RSA', + 'e': 'AQAB', + 'n': 'm2Fylv-Uz7trgTW8EBHP3FQSMeZs2GNQ6VRo1sIVJEk', + } + # pylint: disable=protected-access + self.jwk256_not_comparable = JWKRSA( + key=RSA256_KEY.public_key()._wrapped) + self.jwk512 = JWKRSA(key=RSA512_KEY.public_key()) + self.jwk512json = { + 'kty': 'RSA', + 'e': 'AQAB', + 'n': 'rHVztFHtH92ucFJD_N_HW9AsdRsUuHUBBBDlHwNlRd3fp5' + '80rv2-6QWE30cWgdmJS86ObRz6lUTor4R0T-3C5Q', + } + self.private = JWKRSA(key=RSA256_KEY) + self.private_json_small = self.jwk256json.copy() + self.private_json_small['d'] = ( + 'lPQED_EPTV0UIBfNI3KP2d9Jlrc2mrMllmf946bu-CE') + self.private_json = self.jwk256json.copy() + self.private_json.update({ + 'd': 'lPQED_EPTV0UIBfNI3KP2d9Jlrc2mrMllmf946bu-CE', + 'p': 'zUVNZn4lLLBD1R6NE8TKNQ', + 'q': 'wcfKfc7kl5jfqXArCRSURQ', + 'dp': 'CWJFq43QvT5Bm5iN8n1okQ', + 'dq': 'bHh2u7etM8LKKCF2pY2UdQ', + 'qi': 'oi45cEkbVoJjAbnQpFY87Q', + }) + self.jwk = self.private + + def test_init_auto_comparable(self): + self.assertTrue(isinstance( + self.jwk256_not_comparable.key, util.ComparableRSAKey)) + self.assertEqual(self.jwk256, self.jwk256_not_comparable) + + def test_encode_param_zero(self): + from acme.jose.jwk import JWKRSA + # pylint: disable=protected-access + # TODO: move encode/decode _param to separate class + self.assertEqual('AA', JWKRSA._encode_param(0)) + + def test_equals(self): + self.assertEqual(self.jwk256, self.jwk256) + self.assertEqual(self.jwk512, self.jwk512) + + def test_not_equals(self): + self.assertNotEqual(self.jwk256, self.jwk512) + self.assertNotEqual(self.jwk512, self.jwk256) + + def test_load(self): + from acme.jose.jwk import JWKRSA + self.assertEqual(self.private, JWKRSA.load( + test_util.load_vector('rsa256_key.pem'))) + + def test_public_key(self): + self.assertEqual(self.jwk256, self.private.public_key()) + + def test_to_partial_json(self): + self.assertEqual(self.jwk256.to_partial_json(), self.jwk256json) + self.assertEqual(self.jwk512.to_partial_json(), self.jwk512json) + self.assertEqual(self.private.to_partial_json(), self.private_json) + + def test_from_json(self): + from acme.jose.jwk import JWK + self.assertEqual( + self.jwk256, JWK.from_json(self.jwk256json)) + self.assertEqual( + self.jwk512, JWK.from_json(self.jwk512json)) + self.assertEqual(self.private, JWK.from_json(self.private_json)) + + def test_from_json_private_small(self): + from acme.jose.jwk import JWK + self.assertEqual(self.private, JWK.from_json(self.private_json_small)) + + def test_from_json_missing_one_additional(self): + from acme.jose.jwk import JWK + del self.private_json['q'] + self.assertRaises(errors.Error, JWK.from_json, self.private_json) + + def test_from_json_hashable(self): + from acme.jose.jwk import JWK + hash(JWK.from_json(self.jwk256json)) + + def test_from_json_non_schema_errors(self): + # valid against schema, but still failing + from acme.jose.jwk import JWK + self.assertRaises(errors.DeserializationError, JWK.from_json, + {'kty': 'RSA', 'e': 'AQAB', 'n': ''}) + self.assertRaises(errors.DeserializationError, JWK.from_json, + {'kty': 'RSA', 'e': 'AQAB', 'n': '1'}) + + def test_thumbprint_go_jose(self): + # https://github.com/square/go-jose/blob/4ddd71883fa547d37fbf598071f04512d8bafee3/jwk.go#L155 + # https://github.com/square/go-jose/blob/4ddd71883fa547d37fbf598071f04512d8bafee3/jwk_test.go#L331-L344 + # https://github.com/square/go-jose/blob/4ddd71883fa547d37fbf598071f04512d8bafee3/jwk_test.go#L384 + from acme.jose.jwk import JWKRSA + key = JWKRSA.json_loads("""{ + "kty": "RSA", + "kid": "bilbo.baggins@hobbiton.example", + "use": "sig", + "n": "n4EPtAOCc9AlkeQHPzHStgAbgs7bTZLwUBZdR8_KuKPEHLd4rHVTeT-O-XV2jRojdNhxJWTDvNd7nqQ0VEiZQHz_AJmSCpMaJMRBSFKrKb2wqVwGU_NsYOYL-QtiWN2lbzcEe6XC0dApr5ydQLrHqkHHig3RBordaZ6Aj-oBHqFEHYpPe7Tpe-OfVfHd1E6cS6M1FZcD1NNLYD5lFHpPI9bTwJlsde3uhGqC0ZCuEHg8lhzwOHrtIQbS0FVbb9k3-tVTU4fg_3L_vniUFAKwuCLqKnS2BYwdq_mzSnbLY7h_qixoR7jig3__kRhuaxwUkRz5iaiQkqgc5gHdrNP5zw", + "e": "AQAB" +}""") + self.assertEqual( + binascii.hexlify(key.thumbprint()), + b"f63838e96077ad1fc01c3f8405774dedc0641f558ebb4b40dccf5f9b6d66a932") + + +if __name__ == '__main__': + unittest.main() # pragma: no cover diff --git a/acme/acme/jose/jws.py b/acme/acme/jose/jws.py new file mode 100644 index 00000000000..5f446e4b130 --- /dev/null +++ b/acme/acme/jose/jws.py @@ -0,0 +1,433 @@ +"""JOSE Web Signature.""" +import argparse +import base64 +import sys + +import OpenSSL +import six + +from acme.jose import b64 +from acme.jose import errors +from acme.jose import json_util +from acme.jose import jwa +from acme.jose import jwk +from acme.jose import util + + +class MediaType(object): + """MediaType field encoder/decoder.""" + + PREFIX = 'application/' + """MIME Media Type and Content Type prefix.""" + + @classmethod + def decode(cls, value): + """Decoder.""" + # 4.1.10 + if '/' not in value: + if ';' in value: + raise errors.DeserializationError('Unexpected semi-colon') + return cls.PREFIX + value + return value + + @classmethod + def encode(cls, value): + """Encoder.""" + # 4.1.10 + if ';' not in value: + assert value.startswith(cls.PREFIX) + return value[len(cls.PREFIX):] + return value + + +class Header(json_util.JSONObjectWithFields): + """JOSE Header. + + .. warning:: This class supports **only** Registered Header + Parameter Names (as defined in section 4.1 of the + protocol). If you need Public Header Parameter Names (4.2) + or Private Header Parameter Names (4.3), you must subclass + and override :meth:`from_json` and :meth:`to_partial_json` + appropriately. + + .. warning:: This class does not support any extensions through + the "crit" (Critical) Header Parameter (4.1.11) and as a + conforming implementation, :meth:`from_json` treats its + occurrence as an error. Please subclass if you seek for + a different behaviour. + + :ivar x5tS256: "x5t#S256" + :ivar str typ: MIME Media Type, inc. :const:`MediaType.PREFIX`. + :ivar str cty: Content-Type, inc. :const:`MediaType.PREFIX`. + + """ + alg = json_util.Field( + 'alg', decoder=jwa.JWASignature.from_json, omitempty=True) + jku = json_util.Field('jku', omitempty=True) + jwk = json_util.Field('jwk', decoder=jwk.JWK.from_json, omitempty=True) + kid = json_util.Field('kid', omitempty=True) + x5u = json_util.Field('x5u', omitempty=True) + x5c = json_util.Field('x5c', omitempty=True, default=()) + x5t = json_util.Field( + 'x5t', decoder=json_util.decode_b64jose, omitempty=True) + x5tS256 = json_util.Field( + 'x5t#S256', decoder=json_util.decode_b64jose, omitempty=True) + typ = json_util.Field('typ', encoder=MediaType.encode, + decoder=MediaType.decode, omitempty=True) + cty = json_util.Field('cty', encoder=MediaType.encode, + decoder=MediaType.decode, omitempty=True) + crit = json_util.Field('crit', omitempty=True, default=()) + + def not_omitted(self): + """Fields that would not be omitted in the JSON object.""" + return dict((name, getattr(self, name)) + for name, field in six.iteritems(self._fields) + if not field.omit(getattr(self, name))) + + def __add__(self, other): + if not isinstance(other, type(self)): + raise TypeError('Header cannot be added to: {0}'.format( + type(other))) + + not_omitted_self = self.not_omitted() + not_omitted_other = other.not_omitted() + + if set(not_omitted_self).intersection(not_omitted_other): + raise TypeError('Addition of overlapping headers not defined') + + not_omitted_self.update(not_omitted_other) + return type(self)(**not_omitted_self) # pylint: disable=star-args + + def find_key(self): + """Find key based on header. + + .. todo:: Supports only "jwk" header parameter lookup. + + :returns: (Public) key found in the header. + :rtype: .JWK + + :raises acme.jose.errors.Error: if key could not be found + + """ + if self.jwk is None: + raise errors.Error('No key found') + return self.jwk + + @crit.decoder + def crit(unused_value): + # pylint: disable=missing-docstring,no-self-argument,no-self-use + raise errors.DeserializationError( + '"crit" is not supported, please subclass') + + # x5c does NOT use JOSE Base64 (4.1.6) + + @x5c.encoder # type: ignore + def x5c(value): # pylint: disable=missing-docstring,no-self-argument + return [base64.b64encode(OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_ASN1, cert.wrapped)) for cert in value] + + @x5c.decoder # type: ignore + def x5c(value): # pylint: disable=missing-docstring,no-self-argument + try: + return tuple(util.ComparableX509(OpenSSL.crypto.load_certificate( + OpenSSL.crypto.FILETYPE_ASN1, + base64.b64decode(cert))) for cert in value) + except OpenSSL.crypto.Error as error: + raise errors.DeserializationError(error) + + +class Signature(json_util.JSONObjectWithFields): + """JWS Signature. + + :ivar combined: Combined Header (protected and unprotected, + :class:`Header`). + :ivar unicode protected: JWS protected header (Jose Base-64 decoded). + :ivar header: JWS Unprotected Header (:class:`Header`). + :ivar str signature: The signature. + + """ + header_cls = Header + + __slots__ = ('combined',) + protected = json_util.Field('protected', omitempty=True, default='') + header = json_util.Field( + 'header', omitempty=True, default=header_cls(), + decoder=header_cls.from_json) + signature = json_util.Field( + 'signature', decoder=json_util.decode_b64jose, + encoder=json_util.encode_b64jose) + + @protected.encoder # type: ignore + def protected(value): # pylint: disable=missing-docstring,no-self-argument + # wrong type guess (Signature, not bytes) | pylint: disable=no-member + return json_util.encode_b64jose(value.encode('utf-8')) + + @protected.decoder # type: ignore + def protected(value): # pylint: disable=missing-docstring,no-self-argument + return json_util.decode_b64jose(value).decode('utf-8') + + def __init__(self, **kwargs): + if 'combined' not in kwargs: + kwargs = self._with_combined(kwargs) + super(Signature, self).__init__(**kwargs) + assert self.combined.alg is not None + + @classmethod + def _with_combined(cls, kwargs): + assert 'combined' not in kwargs + header = kwargs.get('header', cls._fields['header'].default) + protected = kwargs.get('protected', cls._fields['protected'].default) + + if protected: + combined = header + cls.header_cls.json_loads(protected) + else: + combined = header + + kwargs['combined'] = combined + return kwargs + + @classmethod + def _msg(cls, protected, payload): + return (b64.b64encode(protected.encode('utf-8')) + b'.' + + b64.b64encode(payload)) + + def verify(self, payload, key=None): + """Verify. + + :param JWK key: Key used for verification. + + """ + key = self.combined.find_key() if key is None else key + return self.combined.alg.verify( + key=key.key, sig=self.signature, + msg=self._msg(self.protected, payload)) + + @classmethod + def sign(cls, payload, key, alg, include_jwk=True, + protect=frozenset(), **kwargs): + """Sign. + + :param JWK key: Key for signature. + + """ + assert isinstance(key, alg.kty) + + header_params = kwargs + header_params['alg'] = alg + if include_jwk: + header_params['jwk'] = key.public_key() + + assert set(header_params).issubset(cls.header_cls._fields) + assert protect.issubset(cls.header_cls._fields) + + protected_params = {} + for header in protect: + if header in header_params: + protected_params[header] = header_params.pop(header) + if protected_params: + # pylint: disable=star-args + protected = cls.header_cls(**protected_params).json_dumps() + else: + protected = '' + + header = cls.header_cls(**header_params) # pylint: disable=star-args + signature = alg.sign(key.key, cls._msg(protected, payload)) + + return cls(protected=protected, header=header, signature=signature) + + def fields_to_partial_json(self): + fields = super(Signature, self).fields_to_partial_json() + if not fields['header'].not_omitted(): + del fields['header'] + return fields + + @classmethod + def fields_from_json(cls, jobj): + fields = super(Signature, cls).fields_from_json(jobj) + fields_with_combined = cls._with_combined(fields) + if 'alg' not in fields_with_combined['combined'].not_omitted(): + raise errors.DeserializationError('alg not present') + return fields_with_combined + + +class JWS(json_util.JSONObjectWithFields): + """JSON Web Signature. + + :ivar str payload: JWS Payload. + :ivar str signature: JWS Signatures. + + """ + __slots__ = ('payload', 'signatures') + + signature_cls = Signature + + def verify(self, key=None): + """Verify.""" + return all(sig.verify(self.payload, key) for sig in self.signatures) + + @classmethod + def sign(cls, payload, **kwargs): + """Sign.""" + return cls(payload=payload, signatures=( + cls.signature_cls.sign(payload=payload, **kwargs),)) + + @property + def signature(self): + """Get a singleton signature. + + :rtype: `signature_cls` + + """ + assert len(self.signatures) == 1 + return self.signatures[0] + + def to_compact(self): + """Compact serialization. + + :rtype: bytes + + """ + assert len(self.signatures) == 1 + + assert 'alg' not in self.signature.header.not_omitted() + # ... it must be in protected + + return ( + b64.b64encode(self.signature.protected.encode('utf-8')) + + b'.' + + b64.b64encode(self.payload) + + b'.' + + b64.b64encode(self.signature.signature)) + + @classmethod + def from_compact(cls, compact): + """Compact deserialization. + + :param bytes compact: + + """ + try: + protected, payload, signature = compact.split(b'.') + except ValueError: + raise errors.DeserializationError( + 'Compact JWS serialization should comprise of exactly' + ' 3 dot-separated components') + + sig = cls.signature_cls( + protected=b64.b64decode(protected).decode('utf-8'), + signature=b64.b64decode(signature)) + return cls(payload=b64.b64decode(payload), signatures=(sig,)) + + def to_partial_json(self, flat=True): # pylint: disable=arguments-differ + assert self.signatures + payload = json_util.encode_b64jose(self.payload) + + if flat and len(self.signatures) == 1: + ret = self.signatures[0].to_partial_json() + ret['payload'] = payload + return ret + else: + return { + 'payload': payload, + 'signatures': self.signatures, + } + + @classmethod + def from_json(cls, jobj): + if 'signature' in jobj and 'signatures' in jobj: + raise errors.DeserializationError('Flat mixed with non-flat') + elif 'signature' in jobj: # flat + return cls(payload=json_util.decode_b64jose(jobj.pop('payload')), + signatures=(cls.signature_cls.from_json(jobj),)) + else: + return cls(payload=json_util.decode_b64jose(jobj['payload']), + signatures=tuple(cls.signature_cls.from_json(sig) + for sig in jobj['signatures'])) + + +class CLI(object): + """JWS CLI.""" + + @classmethod + def sign(cls, args): + """Sign.""" + key = args.alg.kty.load(args.key.read()) + args.key.close() + if args.protect is None: + args.protect = [] + if args.compact: + args.protect.append('alg') + + sig = JWS.sign(payload=sys.stdin.read().encode(), key=key, alg=args.alg, + protect=set(args.protect)) + + if args.compact: + six.print_(sig.to_compact().decode('utf-8')) + else: # JSON + six.print_(sig.json_dumps_pretty()) + + @classmethod + def verify(cls, args): + """Verify.""" + if args.compact: + sig = JWS.from_compact(sys.stdin.read().encode()) + else: # JSON + try: + sig = JWS.json_loads(sys.stdin.read()) + except errors.Error as error: + six.print_(error) + return -1 + + if args.key is not None: + assert args.kty is not None + key = args.kty.load(args.key.read()).public_key() + args.key.close() + else: + key = None + + sys.stdout.write(sig.payload) + return not sig.verify(key=key) + + @classmethod + def _alg_type(cls, arg): + return jwa.JWASignature.from_json(arg) + + @classmethod + def _header_type(cls, arg): + assert arg in Signature.header_cls._fields + return arg + + @classmethod + def _kty_type(cls, arg): + assert arg in jwk.JWK.TYPES + return jwk.JWK.TYPES[arg] + + @classmethod + def run(cls, args=sys.argv[1:]): + """Parse arguments and sign/verify.""" + parser = argparse.ArgumentParser() + parser.add_argument('--compact', action='store_true') + + subparsers = parser.add_subparsers() + parser_sign = subparsers.add_parser('sign') + parser_sign.set_defaults(func=cls.sign) + parser_sign.add_argument( + '-k', '--key', type=argparse.FileType('rb'), required=True) + parser_sign.add_argument( + '-a', '--alg', type=cls._alg_type, default=jwa.RS256) + parser_sign.add_argument( + '-p', '--protect', action='append', type=cls._header_type) + + parser_verify = subparsers.add_parser('verify') + parser_verify.set_defaults(func=cls.verify) + parser_verify.add_argument( + '-k', '--key', type=argparse.FileType('rb'), required=False) + parser_verify.add_argument( + '--kty', type=cls._kty_type, required=False) + + parsed = parser.parse_args(args) + return parsed.func(parsed) + + +if __name__ == '__main__': + exit(CLI.run()) # pragma: no cover diff --git a/acme/acme/jose/jws_test.py b/acme/acme/jose/jws_test.py new file mode 100644 index 00000000000..ec91f6a1bf3 --- /dev/null +++ b/acme/acme/jose/jws_test.py @@ -0,0 +1,239 @@ +"""Tests for acme.jose.jws.""" +import base64 +import unittest + +import mock +import OpenSSL + +from acme import test_util + +from acme.jose import errors +from acme.jose import json_util +from acme.jose import jwa +from acme.jose import jwk + + +CERT = test_util.load_comparable_cert('cert.pem') +KEY = jwk.JWKRSA.load(test_util.load_vector('rsa512_key.pem')) + + +class MediaTypeTest(unittest.TestCase): + """Tests for acme.jose.jws.MediaType.""" + + def test_decode(self): + from acme.jose.jws import MediaType + self.assertEqual('application/app', MediaType.decode('application/app')) + self.assertEqual('application/app', MediaType.decode('app')) + self.assertRaises( + errors.DeserializationError, MediaType.decode, 'app;foo') + + def test_encode(self): + from acme.jose.jws import MediaType + self.assertEqual('app', MediaType.encode('application/app')) + self.assertEqual('application/app;foo', + MediaType.encode('application/app;foo')) + + +class HeaderTest(unittest.TestCase): + """Tests for acme.jose.jws.Header.""" + + def setUp(self): + from acme.jose.jws import Header + self.header1 = Header(jwk='foo') + self.header2 = Header(jwk='bar') + self.crit = Header(crit=('a', 'b')) + self.empty = Header() + + def test_add_non_empty(self): + from acme.jose.jws import Header + self.assertEqual(Header(jwk='foo', crit=('a', 'b')), + self.header1 + self.crit) + + def test_add_empty(self): + self.assertEqual(self.header1, self.header1 + self.empty) + self.assertEqual(self.header1, self.empty + self.header1) + + def test_add_overlapping_error(self): + self.assertRaises(TypeError, self.header1.__add__, self.header2) + + def test_add_wrong_type_error(self): + self.assertRaises(TypeError, self.header1.__add__, 'xxx') + + def test_crit_decode_always_errors(self): + from acme.jose.jws import Header + self.assertRaises(errors.DeserializationError, Header.from_json, + {'crit': ['a', 'b']}) + + def test_x5c_decoding(self): + from acme.jose.jws import Header + header = Header(x5c=(CERT, CERT)) + jobj = header.to_partial_json() + cert_asn1 = OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_ASN1, CERT.wrapped) + cert_b64 = base64.b64encode(cert_asn1) + self.assertEqual(jobj, {'x5c': [cert_b64, cert_b64]}) + self.assertEqual(header, Header.from_json(jobj)) + jobj['x5c'][0] = base64.b64encode(b'xxx' + cert_asn1) + self.assertRaises(errors.DeserializationError, Header.from_json, jobj) + + def test_find_key(self): + self.assertEqual('foo', self.header1.find_key()) + self.assertEqual('bar', self.header2.find_key()) + self.assertRaises(errors.Error, self.crit.find_key) + + +class SignatureTest(unittest.TestCase): + """Tests for acme.jose.jws.Signature.""" + + def test_from_json(self): + from acme.jose.jws import Header + from acme.jose.jws import Signature + self.assertEqual( + Signature(signature=b'foo', header=Header(alg=jwa.RS256)), + Signature.from_json( + {'signature': 'Zm9v', 'header': {'alg': 'RS256'}})) + + def test_from_json_no_alg_error(self): + from acme.jose.jws import Signature + self.assertRaises(errors.DeserializationError, + Signature.from_json, {'signature': 'foo'}) + + +class JWSTest(unittest.TestCase): + """Tests for acme.jose.jws.JWS.""" + + def setUp(self): + self.privkey = KEY + self.pubkey = self.privkey.public_key() + + from acme.jose.jws import JWS + self.unprotected = JWS.sign( + payload=b'foo', key=self.privkey, alg=jwa.RS256) + self.protected = JWS.sign( + payload=b'foo', key=self.privkey, alg=jwa.RS256, + protect=frozenset(['jwk', 'alg'])) + self.mixed = JWS.sign( + payload=b'foo', key=self.privkey, alg=jwa.RS256, + protect=frozenset(['alg'])) + + def test_pubkey_jwk(self): + self.assertEqual(self.unprotected.signature.combined.jwk, self.pubkey) + self.assertEqual(self.protected.signature.combined.jwk, self.pubkey) + self.assertEqual(self.mixed.signature.combined.jwk, self.pubkey) + + def test_sign_unprotected(self): + self.assertTrue(self.unprotected.verify()) + + def test_sign_protected(self): + self.assertTrue(self.protected.verify()) + + def test_sign_mixed(self): + self.assertTrue(self.mixed.verify()) + + def test_compact_lost_unprotected(self): + compact = self.mixed.to_compact() + self.assertEqual( + b'eyJhbGciOiAiUlMyNTYifQ.Zm9v.OHdxFVj73l5LpxbFp1AmYX4yJM0Pyb' + b'_893n1zQjpim_eLS5J1F61lkvrCrCDErTEJnBGOGesJ72M7b6Ve1cAJA', + compact) + + from acme.jose.jws import JWS + mixed = JWS.from_compact(compact) + + self.assertNotEqual(self.mixed, mixed) + self.assertEqual( + set(['alg']), set(mixed.signature.combined.not_omitted())) + + def test_from_compact_missing_components(self): + from acme.jose.jws import JWS + self.assertRaises(errors.DeserializationError, JWS.from_compact, b'.') + + def test_json_omitempty(self): + protected_jobj = self.protected.to_partial_json(flat=True) + unprotected_jobj = self.unprotected.to_partial_json(flat=True) + + self.assertTrue('protected' not in unprotected_jobj) + self.assertTrue('header' not in protected_jobj) + + unprotected_jobj['header'] = unprotected_jobj['header'].to_json() + + from acme.jose.jws import JWS + self.assertEqual(JWS.from_json(protected_jobj), self.protected) + self.assertEqual(JWS.from_json(unprotected_jobj), self.unprotected) + + def test_json_flat(self): + jobj_to = { + 'signature': json_util.encode_b64jose( + self.mixed.signature.signature), + 'payload': json_util.encode_b64jose(b'foo'), + 'header': self.mixed.signature.header, + 'protected': json_util.encode_b64jose( + self.mixed.signature.protected.encode('utf-8')), + } + jobj_from = jobj_to.copy() + jobj_from['header'] = jobj_from['header'].to_json() + + self.assertEqual(self.mixed.to_partial_json(flat=True), jobj_to) + from acme.jose.jws import JWS + self.assertEqual(self.mixed, JWS.from_json(jobj_from)) + + def test_json_not_flat(self): + jobj_to = { + 'signatures': (self.mixed.signature,), + 'payload': json_util.encode_b64jose(b'foo'), + } + jobj_from = jobj_to.copy() + jobj_from['signatures'] = [jobj_to['signatures'][0].to_json()] + + self.assertEqual(self.mixed.to_partial_json(flat=False), jobj_to) + from acme.jose.jws import JWS + self.assertEqual(self.mixed, JWS.from_json(jobj_from)) + + def test_from_json_mixed_flat(self): + from acme.jose.jws import JWS + self.assertRaises(errors.DeserializationError, JWS.from_json, + {'signatures': (), 'signature': 'foo'}) + + def test_from_json_hashable(self): + from acme.jose.jws import JWS + hash(JWS.from_json(self.mixed.to_json())) + + +class CLITest(unittest.TestCase): + + def setUp(self): + self.key_path = test_util.vector_path('rsa512_key.pem') + + def test_unverified(self): + from acme.jose.jws import CLI + with mock.patch('sys.stdin') as sin: + sin.read.return_value = '{"payload": "foo", "signature": "xxx"}' + with mock.patch('sys.stdout'): + self.assertEqual(-1, CLI.run(['verify'])) + + def test_json(self): + from acme.jose.jws import CLI + + with mock.patch('sys.stdin') as sin: + sin.read.return_value = 'foo' + with mock.patch('sys.stdout') as sout: + CLI.run(['sign', '-k', self.key_path, '-a', 'RS256', + '-p', 'jwk']) + sin.read.return_value = sout.write.mock_calls[0][1][0] + self.assertEqual(0, CLI.run(['verify'])) + + def test_compact(self): + from acme.jose.jws import CLI + + with mock.patch('sys.stdin') as sin: + sin.read.return_value = 'foo' + with mock.patch('sys.stdout') as sout: + CLI.run(['--compact', 'sign', '-k', self.key_path]) + sin.read.return_value = sout.write.mock_calls[0][1][0] + self.assertEqual(0, CLI.run([ + '--compact', 'verify', '--kty', 'RSA', + '-k', self.key_path])) + + +if __name__ == '__main__': + unittest.main() # pragma: no cover diff --git a/acme/acme/jose/util.py b/acme/acme/jose/util.py new file mode 100644 index 00000000000..26b7e0c5a69 --- /dev/null +++ b/acme/acme/jose/util.py @@ -0,0 +1,226 @@ +"""JOSE utilities.""" +import collections + +from cryptography.hazmat.primitives.asymmetric import rsa +import OpenSSL +import six + + +class abstractclassmethod(classmethod): + # pylint: disable=invalid-name,too-few-public-methods + """Descriptor for an abstract classmethod. + + It augments the :mod:`abc` framework with an abstract + classmethod. This is implemented as :class:`abc.abstractclassmethod` + in the standard Python library starting with version 3.2. + + This particular implementation, allegedly based on Python 3.3 source + code, is stolen from + http://stackoverflow.com/questions/11217878/python-2-7-combine-abc-abstractmethod-and-classmethod. + + """ + __isabstractmethod__ = True + + def __init__(self, target): + target.__isabstractmethod__ = True + super(abstractclassmethod, self).__init__(target) + + +class ComparableX509(object): # pylint: disable=too-few-public-methods + """Wrapper for OpenSSL.crypto.X509** objects that supports __eq__. + + :ivar wrapped: Wrapped certificate or certificate request. + :type wrapped: `OpenSSL.crypto.X509` or `OpenSSL.crypto.X509Req`. + + """ + def __init__(self, wrapped): + assert isinstance(wrapped, OpenSSL.crypto.X509) or isinstance( + wrapped, OpenSSL.crypto.X509Req) + self.wrapped = wrapped + + def __getattr__(self, name): + return getattr(self.wrapped, name) + + def _dump(self, filetype=OpenSSL.crypto.FILETYPE_ASN1): + """Dumps the object into a buffer with the specified encoding. + + :param int filetype: The desired encoding. Should be one of + `OpenSSL.crypto.FILETYPE_ASN1`, + `OpenSSL.crypto.FILETYPE_PEM`, or + `OpenSSL.crypto.FILETYPE_TEXT`. + + :returns: Encoded X509 object. + :rtype: str + + """ + if isinstance(self.wrapped, OpenSSL.crypto.X509): + func = OpenSSL.crypto.dump_certificate + else: # assert in __init__ makes sure this is X509Req + func = OpenSSL.crypto.dump_certificate_request + return func(filetype, self.wrapped) + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return NotImplemented + # pylint: disable=protected-access + return self._dump() == other._dump() + + def __hash__(self): + return hash((self.__class__, self._dump())) + + def __ne__(self, other): + return not self == other + + def __repr__(self): + return '<{0}({1!r})>'.format(self.__class__.__name__, self.wrapped) + + +class ComparableKey(object): # pylint: disable=too-few-public-methods + """Comparable wrapper for `cryptography` keys. + + See https://github.com/pyca/cryptography/issues/2122. + + """ + __hash__ = NotImplemented + + def __init__(self, wrapped): + self._wrapped = wrapped + + def __getattr__(self, name): + return getattr(self._wrapped, name) + + def __eq__(self, other): + # pylint: disable=protected-access + if (not isinstance(other, self.__class__) or + self._wrapped.__class__ is not other._wrapped.__class__): + return NotImplemented + elif hasattr(self._wrapped, 'private_numbers'): + return self.private_numbers() == other.private_numbers() + elif hasattr(self._wrapped, 'public_numbers'): + return self.public_numbers() == other.public_numbers() + else: + return NotImplemented + + def __ne__(self, other): + return not self == other + + def __repr__(self): + return '<{0}({1!r})>'.format(self.__class__.__name__, self._wrapped) + + def public_key(self): + """Get wrapped public key.""" + return self.__class__(self._wrapped.public_key()) + + +class ComparableRSAKey(ComparableKey): # pylint: disable=too-few-public-methods + """Wrapper for `cryptography` RSA keys. + + Wraps around: + - `cryptography.hazmat.primitives.asymmetric.RSAPrivateKey` + - `cryptography.hazmat.primitives.asymmetric.RSAPublicKey` + + """ + + def __hash__(self): + # public_numbers() hasn't got stable hash! + # https://github.com/pyca/cryptography/issues/2143 + if isinstance(self._wrapped, rsa.RSAPrivateKeyWithSerialization): + priv = self.private_numbers() + pub = priv.public_numbers + return hash((self.__class__, priv.p, priv.q, priv.dmp1, + priv.dmq1, priv.iqmp, pub.n, pub.e)) + elif isinstance(self._wrapped, rsa.RSAPublicKeyWithSerialization): + pub = self.public_numbers() + return hash((self.__class__, pub.n, pub.e)) + + +class ImmutableMap(collections.Mapping, collections.Hashable): # type: ignore + # pylint: disable=too-few-public-methods + """Immutable key to value mapping with attribute access.""" + + __slots__ = () + """Must be overridden in subclasses.""" + + def __init__(self, **kwargs): + if set(kwargs) != set(self.__slots__): + raise TypeError( + '__init__() takes exactly the following arguments: {0} ' + '({1} given)'.format(', '.join(self.__slots__), + ', '.join(kwargs) if kwargs else 'none')) + for slot in self.__slots__: + object.__setattr__(self, slot, kwargs.pop(slot)) + + def update(self, **kwargs): + """Return updated map.""" + items = dict(self) + items.update(kwargs) + return type(self)(**items) # pylint: disable=star-args + + def __getitem__(self, key): + try: + return getattr(self, key) + except AttributeError: + raise KeyError(key) + + def __iter__(self): + return iter(self.__slots__) + + def __len__(self): + return len(self.__slots__) + + def __hash__(self): + return hash(tuple(getattr(self, slot) for slot in self.__slots__)) + + def __setattr__(self, name, value): + raise AttributeError("can't set attribute") + + def __repr__(self): + return '{0}({1})'.format(self.__class__.__name__, ', '.join( + '{0}={1!r}'.format(key, value) + for key, value in six.iteritems(self))) + + +class frozendict(collections.Mapping, collections.Hashable): # type: ignore + # pylint: disable=invalid-name,too-few-public-methods + """Frozen dictionary.""" + __slots__ = ('_items', '_keys') + + def __init__(self, *args, **kwargs): + if kwargs and not args: + items = dict(kwargs) + elif len(args) == 1 and isinstance(args[0], collections.Mapping): + items = args[0] + else: + raise TypeError() + # TODO: support generators/iterators + + object.__setattr__(self, '_items', items) + object.__setattr__(self, '_keys', tuple(sorted(six.iterkeys(items)))) + + def __getitem__(self, key): + return self._items[key] + + def __iter__(self): + return iter(self._keys) + + def __len__(self): + return len(self._items) + + def _sorted_items(self): + return tuple((key, self[key]) for key in self._keys) + + def __hash__(self): + return hash(self._sorted_items()) + + def __getattr__(self, name): + try: + return self._items[name] + except KeyError: + raise AttributeError(name) + + def __setattr__(self, name, value): + raise AttributeError("can't set attribute") + + def __repr__(self): + return 'frozendict({0})'.format(', '.join('{0}={1!r}'.format( + key, value) for key, value in self._sorted_items())) diff --git a/acme/acme/jose/util_test.py b/acme/acme/jose/util_test.py new file mode 100644 index 00000000000..0038a6cc14e --- /dev/null +++ b/acme/acme/jose/util_test.py @@ -0,0 +1,199 @@ +"""Tests for acme.jose.util.""" +import functools +import unittest + +import six + +from acme import test_util + + +class ComparableX509Test(unittest.TestCase): + """Tests for acme.jose.util.ComparableX509.""" + + def setUp(self): + # test_util.load_comparable_{csr,cert} return ComparableX509 + self.req1 = test_util.load_comparable_csr('csr.pem') + self.req2 = test_util.load_comparable_csr('csr.pem') + self.req_other = test_util.load_comparable_csr('csr-san.pem') + + self.cert1 = test_util.load_comparable_cert('cert.pem') + self.cert2 = test_util.load_comparable_cert('cert.pem') + self.cert_other = test_util.load_comparable_cert('cert-san.pem') + + def test_getattr_proxy(self): + self.assertTrue(self.cert1.has_expired()) + + def test_eq(self): + self.assertEqual(self.req1, self.req2) + self.assertEqual(self.cert1, self.cert2) + + def test_ne(self): + self.assertNotEqual(self.req1, self.req_other) + self.assertNotEqual(self.cert1, self.cert_other) + + def test_ne_wrong_types(self): + self.assertNotEqual(self.req1, 5) + self.assertNotEqual(self.cert1, 5) + + def test_hash(self): + self.assertEqual(hash(self.req1), hash(self.req2)) + self.assertNotEqual(hash(self.req1), hash(self.req_other)) + + self.assertEqual(hash(self.cert1), hash(self.cert2)) + self.assertNotEqual(hash(self.cert1), hash(self.cert_other)) + + def test_repr(self): + for x509 in self.req1, self.cert1: + self.assertEqual(repr(x509), + ''.format(x509.wrapped)) + + +class ComparableRSAKeyTest(unittest.TestCase): + """Tests for acme.jose.util.ComparableRSAKey.""" + + def setUp(self): + # test_utl.load_rsa_private_key return ComparableRSAKey + self.key = test_util.load_rsa_private_key('rsa256_key.pem') + self.key_same = test_util.load_rsa_private_key('rsa256_key.pem') + self.key2 = test_util.load_rsa_private_key('rsa512_key.pem') + + def test_getattr_proxy(self): + self.assertEqual(256, self.key.key_size) + + def test_eq(self): + self.assertEqual(self.key, self.key_same) + + def test_ne(self): + self.assertNotEqual(self.key, self.key2) + + def test_ne_different_types(self): + self.assertNotEqual(self.key, 5) + + def test_ne_not_wrapped(self): + # pylint: disable=protected-access + self.assertNotEqual(self.key, self.key_same._wrapped) + + def test_ne_no_serialization(self): + from acme.jose.util import ComparableRSAKey + self.assertNotEqual(ComparableRSAKey(5), ComparableRSAKey(5)) + + def test_hash(self): + self.assertTrue(isinstance(hash(self.key), int)) + self.assertEqual(hash(self.key), hash(self.key_same)) + self.assertNotEqual(hash(self.key), hash(self.key2)) + + def test_repr(self): + self.assertTrue(repr(self.key).startswith( + '=0.6) # rsa_recover_prime_factors (>=0.8) 'cryptography>=0.8', - # formerly known as acme.jose: - 'josepy>=1.0.0', # Connection.set_tlsext_host_name (>=0.13) 'mock', 'PyOpenSSL>=0.13', @@ -76,5 +74,10 @@ 'dev': dev_extras, 'docs': docs_extras, }, + entry_points={ + 'console_scripts': [ + 'jws = acme.jose.jws:CLI.run', + ], + }, test_suite='acme', ) diff --git a/certbot-apache/certbot_apache/apache_util.py b/certbot-apache/certbot_apache/apache_util.py index f03c9da8793..b4a24f137a6 100644 --- a/certbot-apache/certbot_apache/apache_util.py +++ b/certbot-apache/certbot_apache/apache_util.py @@ -93,8 +93,4 @@ def parse_define_file(filepath, varname): if v == "-D" and len(a_opts) >= i+2: var_parts = a_opts[i+1].partition("=") return_vars[var_parts[0]] = var_parts[2] - elif len(v) > 2 and v.startswith("-D"): - # Found var with no whitespace separator - var_parts = v[2:].partition("=") - return_vars[var_parts[0]] = var_parts[2] return return_vars diff --git a/certbot-apache/certbot_apache/centos-options-ssl-apache.conf b/certbot-apache/certbot_apache/centos-options-ssl-apache.conf index 56c946a4e21..17ae1be767e 100644 --- a/certbot-apache/certbot_apache/centos-options-ssl-apache.conf +++ b/certbot-apache/certbot_apache/centos-options-ssl-apache.conf @@ -8,7 +8,7 @@ SSLEngine on # Intermediate configuration, tweak to your needs SSLProtocol all -SSLv2 -SSLv3 -SSLCipherSuite ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS +SSLCipherSuite ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA SSLHonorCipherOrder on SSLOptions +StrictRequire diff --git a/certbot-apache/certbot_apache/constants.py b/certbot-apache/certbot_apache/constants.py index fd6a9eb11fe..a13ca04a683 100644 --- a/certbot-apache/certbot_apache/constants.py +++ b/certbot-apache/certbot_apache/constants.py @@ -16,8 +16,6 @@ '4066b90268c03c9ba0201068eaa39abbc02acf9558bb45a788b630eb85dadf27', 'f175e2e7c673bd88d0aff8220735f385f916142c44aa83b09f1df88dd4767a88', 'cfdd7c18d2025836ea3307399f509cfb1ebf2612c87dd600a65da2a8e2f2797b', - '80720bd171ccdc2e6b917ded340defae66919e4624962396b992b7218a561791', - 'c0c022ea6b8a51ecc8f1003d0a04af6c3f2bc1c3ce506b3c2dfc1f11ef931082', ] """SHA256 hashes of the contents of previous versions of all versions of MOD_SSL_CONF_SRC""" diff --git a/certbot-apache/certbot_apache/options-ssl-apache.conf b/certbot-apache/certbot_apache/options-ssl-apache.conf index 8113ee81e5a..950a02a8bc2 100644 --- a/certbot-apache/certbot_apache/options-ssl-apache.conf +++ b/certbot-apache/certbot_apache/options-ssl-apache.conf @@ -8,7 +8,7 @@ SSLEngine on # Intermediate configuration, tweak to your needs SSLProtocol all -SSLv2 -SSLv3 -SSLCipherSuite ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS +SSLCipherSuite ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA SSLHonorCipherOrder on SSLCompression off diff --git a/certbot-apache/certbot_apache/override_gentoo.py b/certbot-apache/certbot_apache/override_gentoo.py index 92f1d4a2099..d4d4e96b949 100644 --- a/certbot-apache/certbot_apache/override_gentoo.py +++ b/certbot-apache/certbot_apache/override_gentoo.py @@ -49,7 +49,6 @@ def __init__(self, *args, **kwargs): def update_runtime_variables(self): """ Override for update_runtime_variables for custom parsing """ self.parse_sysconfig_var() - self.update_modules() def parse_sysconfig_var(self): """ Parses Apache CLI options from Gentoo configuration file """ @@ -57,10 +56,3 @@ def parse_sysconfig_var(self): "APACHE2_OPTS") for k in defines.keys(): self.variables[k] = defines[k] - - def update_modules(self): - """Get loaded modules from httpd process, and add them to DOM""" - mod_cmd = [self.configurator.constant("apache_cmd"), "modules"] - matches = self.parse_from_subprocess(mod_cmd, r"(.*)_module") - for mod in matches: - self.add_mod(mod.strip()) diff --git a/certbot-apache/certbot_apache/tests/centos_test.py b/certbot-apache/certbot_apache/tests/centos_test.py index d7a2a2fd922..7ca47a4d56f 100644 --- a/certbot-apache/certbot_apache/tests/centos_test.py +++ b/certbot-apache/certbot_apache/tests/centos_test.py @@ -118,8 +118,6 @@ def test_get_sysconfig_vars(self, mock_cfg): self.assertTrue("mock_define_too" in self.config.parser.variables.keys()) self.assertTrue("mock_value" in self.config.parser.variables.keys()) self.assertEqual("TRUE", self.config.parser.variables["mock_value"]) - self.assertTrue("MOCK_NOSEP" in self.config.parser.variables.keys()) - self.assertEqual("NOSEP_VAL", self.config.parser.variables["NOSEP_TWO"]) if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/certbot-apache/certbot_apache/tests/gentoo_test.py b/certbot-apache/certbot_apache/tests/gentoo_test.py index cfbaffac71d..0f2b96818df 100644 --- a/certbot-apache/certbot_apache/tests/gentoo_test.py +++ b/certbot-apache/certbot_apache/tests/gentoo_test.py @@ -2,8 +2,6 @@ import os import unittest -import mock - from certbot_apache import override_gentoo from certbot_apache import obj from certbot_apache.tests import util @@ -48,10 +46,9 @@ def setUp(self): # pylint: disable=arguments-differ config_root=config_root, vhost_root=vhost_root) - with mock.patch("certbot_apache.override_gentoo.GentooParser.update_runtime_variables"): - self.config = util.get_apache_configurator( - self.config_path, self.vhost_path, self.config_dir, self.work_dir, - os_info="gentoo") + self.config = util.get_apache_configurator( + self.config_path, self.vhost_path, self.config_dir, self.work_dir, + os_info="gentoo") self.vh_truth = get_vh_truth( self.temp_dir, "gentoo_apache/apache") @@ -81,47 +78,9 @@ def test_get_sysconfig_vars(self): self.config.parser.apacheconfig_filep = os.path.realpath( os.path.join(self.config.parser.root, "../conf.d/apache2")) self.config.parser.variables = {} - with mock.patch("certbot_apache.override_gentoo.GentooParser.update_modules"): - self.config.parser.update_runtime_variables() + self.config.parser.update_runtime_variables() for define in defines: self.assertTrue(define in self.config.parser.variables.keys()) - @mock.patch("certbot_apache.parser.ApacheParser.parse_from_subprocess") - def test_no_binary_configdump(self, mock_subprocess): - """Make sure we don't call binary dumps other than modules from Apache - as this is not supported in Gentoo currently""" - - with mock.patch("certbot_apache.override_gentoo.GentooParser.update_modules"): - self.config.parser.update_runtime_variables() - self.config.parser.reset_modules() - self.assertFalse(mock_subprocess.called) - - self.config.parser.update_runtime_variables() - self.config.parser.reset_modules() - self.assertTrue(mock_subprocess.called) - - @mock.patch("certbot_apache.parser.ApacheParser._get_runtime_cfg") - def test_opportunistic_httpd_runtime_parsing(self, mock_get): - mod_val = ( - 'Loaded Modules:\n' - ' mock_module (static)\n' - ' another_module (static)\n' - ) - def mock_get_cfg(command): - """Mock httpd process stdout""" - if command == ['apache2ctl', 'modules']: - return mod_val - mock_get.side_effect = mock_get_cfg - self.config.parser.modules = set() - - with mock.patch("certbot.util.get_os_info") as mock_osi: - # Make sure we have the have the CentOS httpd constants - mock_osi.return_value = ("gentoo", "123") - self.config.parser.update_runtime_variables() - - self.assertEquals(mock_get.call_count, 1) - self.assertEquals(len(self.config.parser.modules), 4) - self.assertTrue("mod_another.c" in self.config.parser.modules) - if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/sysconfig/httpd b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/sysconfig/httpd index 4bcb300c29b..0bf6b176c67 100644 --- a/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/sysconfig/httpd +++ b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/sysconfig/httpd @@ -14,7 +14,7 @@ # To pass additional options (for instance, -D definitions) to the # httpd binary at startup, set OPTIONS here. # -OPTIONS="-D mock_define -D mock_define_too -D mock_value=TRUE -DMOCK_NOSEP -DNOSEP_TWO=NOSEP_VAL" +OPTIONS="-D mock_define -D mock_define_too -D mock_value=TRUE" # # This setting ensures the httpd process is started in the "C" locale diff --git a/certbot-apache/certbot_apache/tests/util.py b/certbot-apache/certbot_apache/tests/util.py index ca667465c26..2405110c53f 100644 --- a/certbot-apache/certbot_apache/tests/util.py +++ b/certbot-apache/certbot_apache/tests/util.py @@ -5,10 +5,11 @@ import unittest import augeas -import josepy as jose import mock import zope.component +from acme import jose + from certbot.display import util as display_util from certbot.plugins import common diff --git a/certbot-compatibility-test/certbot_compatibility_test/util.py b/certbot-compatibility-test/certbot_compatibility_test/util.py index 4155944bd5b..af951aa6a99 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/util.py +++ b/certbot-compatibility-test/certbot_compatibility_test/util.py @@ -6,8 +6,7 @@ import shutil import tarfile -import josepy as jose - +from acme import jose from acme import test_util from certbot import constants diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index 8af474c5ea3..e9d4e36d454 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -182,7 +182,7 @@ def deploy_cert(self, domain, cert_path, key_path, self.parser.add_server_directives(vhost, cert_directives, replace=True) logger.info("Deployed Certificate to VirtualHost %s for %s", - vhost.filep, ", ".join(vhost.names)) + vhost.filep, vhost.names) self.save_notes += ("Changed vhost at %s with addresses of %s\n" % (vhost.filep, diff --git a/certbot-nginx/certbot_nginx/tests/util.py b/certbot-nginx/certbot_nginx/tests/util.py index 7b32d8e82fa..6e1b0d8ff58 100644 --- a/certbot-nginx/certbot_nginx/tests/util.py +++ b/certbot-nginx/certbot_nginx/tests/util.py @@ -5,10 +5,11 @@ import tempfile import unittest -import josepy as jose import mock import zope.component +from acme import jose + from certbot import configuration from certbot.tests import util as test_util diff --git a/certbot/account.py b/certbot/account.py index 41e9800974b..389f9679127 100644 --- a/certbot/account.py +++ b/certbot/account.py @@ -7,13 +7,13 @@ import socket from cryptography.hazmat.primitives import serialization -import josepy as jose import pyrfc3339 import pytz import six import zope.component from acme import fields as acme_fields +from acme import jose from acme import messages from certbot import errors diff --git a/certbot/achallenges.py b/certbot/achallenges.py index 6535a6b630d..f39bb4cec00 100644 --- a/certbot/achallenges.py +++ b/certbot/achallenges.py @@ -19,9 +19,8 @@ """ import logging -import josepy as jose - from acme import challenges +from acme import jose logger = logging.getLogger(__name__) diff --git a/certbot/cli.py b/certbot/cli.py index f0fa7eb7ebe..62246227847 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -1220,18 +1220,6 @@ def _create_subparsers(helpful): key=constants.REVOCATION_REASONS.get)), action=_EncodeReasonAction, default=flag_default("reason"), help="Specify reason for revoking certificate. (default: unspecified)") - helpful.add("revoke", - "--delete-after-revoke", action="store_true", - default=flag_default("delete_after_revoke"), - help="Delete certificates after revoking them.") - helpful.add("revoke", - "--no-delete-after-revoke", action="store_false", - dest="delete_after_revoke", - default=flag_default("delete_after_revoke"), - help="Do not delete certificates after revoking them. This " - "option should be used with caution because the 'renew' " - "subcommand will attempt to renew undeleted revoked " - "certificates.") helpful.add("rollback", "--checkpoints", type=int, metavar="N", default=flag_default("rollback_checkpoints"), diff --git a/certbot/client.py b/certbot/client.py index b735421f5a2..ed70fda711a 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -5,13 +5,13 @@ from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.asymmetric import rsa -import josepy as jose import OpenSSL import zope.component from acme import client as acme_client from acme import crypto_util as acme_crypto_util from acme import errors as acme_errors +from acme import jose from acme import messages import certbot diff --git a/certbot/constants.py b/certbot/constants.py index a6878824b58..0ac82dafe45 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -71,7 +71,6 @@ user_agent_comment=None, csr=None, reason=0, - delete_after_revoke=None, rollback_checkpoints=1, init=False, prepare=False, diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index 3ae16529d3f..112ef7c852e 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -14,9 +14,9 @@ import zope.component from cryptography.hazmat.backends import default_backend from cryptography import x509 -import josepy as jose from acme import crypto_util as acme_crypto_util +from acme import jose from certbot import errors from certbot import interfaces @@ -368,7 +368,7 @@ def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM): """Dump certificate chain into a bundle. :param list chain: List of `OpenSSL.crypto.X509` (or wrapped in - :class:`josepy.util.ComparableX509`). + `acme.jose.ComparableX509`). """ # XXX: returns empty string when no chain is available, which diff --git a/certbot/main.py b/certbot/main.py index e25e030aadb..72af7fbba9a 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -6,9 +6,9 @@ import sys import configobj -import josepy as jose import zope.component +from acme import jose from acme import errors as acme_errors import certbot @@ -536,11 +536,9 @@ def _delete_if_appropriate(config): # pylint: disable=too-many-locals,too-many-b display = zope.component.getUtility(interfaces.IDisplay) reporter_util = zope.component.getUtility(interfaces.IReporter) - attempt_deletion = config.delete_after_revoke - if attempt_deletion is None: - msg = ("Would you like to delete the cert(s) you just revoked?") - attempt_deletion = display.yesno(msg, yes_label="Yes (recommended)", no_label="No", - force_interactive=True, default=True) + msg = ("Would you like to delete the cert(s) you just revoked?") + attempt_deletion = display.yesno(msg, yes_label="Yes (recommended)", no_label="No", + force_interactive=True, default=True) if not attempt_deletion: reporter_util.add_message("Not deleting revoked certs.", reporter_util.LOW_PRIORITY) diff --git a/certbot/plugins/common.py b/certbot/plugins/common.py index 002d2f22530..420d15679d3 100644 --- a/certbot/plugins/common.py +++ b/certbot/plugins/common.py @@ -9,7 +9,7 @@ import pkg_resources import zope.interface -from josepy import util as jose_util +from acme.jose import util as jose_util from certbot import constants from certbot import crypto_util diff --git a/certbot/plugins/common_test.py b/certbot/plugins/common_test.py index 1a1ca7dcb80..8ce68bbb5ae 100644 --- a/certbot/plugins/common_test.py +++ b/certbot/plugins/common_test.py @@ -5,11 +5,11 @@ import tempfile import unittest -import josepy as jose import mock import OpenSSL from acme import challenges +from acme import jose from certbot import achallenges from certbot import crypto_util diff --git a/certbot/plugins/dns_test_common.py b/certbot/plugins/dns_test_common.py index 54b656b20ac..d8cd2940445 100644 --- a/certbot/plugins/dns_test_common.py +++ b/certbot/plugins/dns_test_common.py @@ -3,10 +3,10 @@ import os import configobj -import josepy as jose import mock import six from acme import challenges +from acme import jose from certbot import achallenges from certbot.tests import acme_util diff --git a/certbot/plugins/dns_test_common_lexicon.py b/certbot/plugins/dns_test_common_lexicon.py index a221cf1bf0b..f9c5735e80e 100644 --- a/certbot/plugins/dns_test_common_lexicon.py +++ b/certbot/plugins/dns_test_common_lexicon.py @@ -1,7 +1,7 @@ """Base test class for DNS authenticators built on Lexicon.""" -import josepy as jose import mock +from acme import jose from requests.exceptions import HTTPError, RequestException from certbot import errors diff --git a/certbot/plugins/standalone_test.py b/certbot/plugins/standalone_test.py index 5227bc59e39..1ae731e429e 100644 --- a/certbot/plugins/standalone_test.py +++ b/certbot/plugins/standalone_test.py @@ -3,11 +3,11 @@ import socket import unittest -import josepy as jose import mock import six from acme import challenges +from acme import jose from certbot import achallenges from certbot import errors diff --git a/certbot/plugins/webroot_test.py b/certbot/plugins/webroot_test.py index 36e2ffba6ee..92160bdfa39 100644 --- a/certbot/plugins/webroot_test.py +++ b/certbot/plugins/webroot_test.py @@ -10,11 +10,11 @@ import tempfile import unittest -import josepy as jose import mock import six from acme import challenges +from acme import jose from certbot import achallenges from certbot import errors diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index 8ebda56afba..7245ad6a14f 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -6,10 +6,10 @@ import stat import unittest -import josepy as jose import mock import pytz +from acme import jose from acme import messages from certbot import errors diff --git a/certbot/tests/acme_util.py b/certbot/tests/acme_util.py index 53a2f214aed..f0549666a37 100644 --- a/certbot/tests/acme_util.py +++ b/certbot/tests/acme_util.py @@ -1,10 +1,10 @@ """ACME utilities for testing.""" import datetime -import josepy as jose import six from acme import challenges +from acme import jose from acme import messages from certbot import auth_handler diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index c5935d7224a..2fce412e230 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -164,8 +164,6 @@ def test_help(self): self.assertTrue("--cert-path" in out) self.assertTrue("--key-path" in out) self.assertTrue("--reason" in out) - self.assertTrue("--delete-after-revoke" in out) - self.assertTrue("--no-delete-after-revoke" in out) out = self._help_output(['-h', 'config_changes']) self.assertTrue("--cert-path" not in out) @@ -414,18 +412,6 @@ def test_no_directory_hooks_set(self): def test_no_directory_hooks_unset(self): self.assertTrue(self.parse([]).directory_hooks) - def test_delete_after_revoke(self): - namespace = self.parse(["--delete-after-revoke"]) - self.assertTrue(namespace.delete_after_revoke) - - def test_delete_after_revoke_default(self): - namespace = self.parse([]) - self.assertEqual(namespace.delete_after_revoke, None) - - def test_no_delete_after_revoke(self): - namespace = self.parse(["--no-delete-after-revoke"]) - self.assertFalse(namespace.delete_after_revoke) - class DefaultTest(unittest.TestCase): """Tests for certbot.cli._Default.""" diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 204f46323a9..09c4a50ca95 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -4,11 +4,11 @@ import tempfile import unittest -import josepy as jose import OpenSSL import mock from acme import errors as acme_errors +from acme import jose from certbot import account from certbot import errors diff --git a/certbot/tests/display/ops_test.py b/certbot/tests/display/ops_test.py index 57d82f8392f..cb0fb32e35f 100644 --- a/certbot/tests/display/ops_test.py +++ b/certbot/tests/display/ops_test.py @@ -4,10 +4,10 @@ import sys import unittest -import josepy as jose import mock import zope.component +from acme import jose from acme import messages from certbot import account diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index b1d58542f50..1f690df261e 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -11,10 +11,11 @@ import datetime import pytz -import josepy as jose import six from six.moves import reload_module # pylint: disable=import-error +from acme import jose + from certbot import account from certbot import cli from certbot import constants @@ -298,29 +299,25 @@ def test_revocation_with_prompt(self, mock_get_utility, self._call() self.assertFalse(mock_delete.called) -class DeleteIfAppropriateTest(test_util.ConfigTestCase): +class DeleteIfAppropriateTest(unittest.TestCase): """Tests for certbot.main._delete_if_appropriate """ + def setUp(self): + self.config = mock.Mock() + self.config.namespace = mock.Mock() + self.config.namespace.noninteractive_mode = False + def _call(self, mock_config): from certbot.main import _delete_if_appropriate _delete_if_appropriate(mock_config) - def _test_delete_opt_out_common(self, mock_get_utility): - with mock.patch('certbot.cert_manager.delete') as mock_delete: - self._call(self.config) - mock_delete.assert_not_called() - self.assertTrue(mock_get_utility().add_message.called) - - @test_util.patch_get_utility() - def test_delete_flag_opt_out(self, mock_get_utility): - self.config.delete_after_revoke = False - self._test_delete_opt_out_common(mock_get_utility) - + @mock.patch('certbot.cert_manager.delete') @test_util.patch_get_utility() - def test_delete_prompt_opt_out(self, mock_get_utility): + def test_delete_opt_out(self, mock_get_utility, mock_delete): util_mock = mock_get_utility() util_mock.yesno.return_value = False - self._test_delete_opt_out_common(mock_get_utility) + self._call(self.config) + mock_delete.assert_not_called() # pylint: disable=too-many-arguments @mock.patch('certbot.storage.renewal_file_for_certname') @@ -401,28 +398,6 @@ def test_noninteractive_deletion(self, mock_get_utility, mock_delete, self._call(config) self.assertEqual(mock_delete.call_count, 1) - # pylint: disable=too-many-arguments - @mock.patch('certbot.storage.renewal_file_for_certname') - @mock.patch('certbot.cert_manager.match_and_check_overlaps') - @mock.patch('certbot.storage.full_archive_path') - @mock.patch('certbot.cert_manager.cert_path_to_lineage') - @mock.patch('certbot.cert_manager.delete') - @test_util.patch_get_utility() - def test_opt_in_deletion(self, mock_get_utility, mock_delete, - mock_cert_path_to_lineage, mock_full_archive_dir, - mock_match_and_check_overlaps, mock_renewal_file_for_certname): - # pylint: disable = unused-argument - config = self.config - config.namespace.delete_after_revoke = True - config.cert_path = "/some/reasonable/path" - config.certname = "" - mock_cert_path_to_lineage.return_value = "example.com" - mock_full_archive_dir.return_value = "" - mock_match_and_check_overlaps.return_value = "" - self._call(config) - self.assertEqual(mock_delete.call_count, 1) - self.assertFalse(mock_get_utility().yesno.called) - # pylint: disable=too-many-arguments @mock.patch('certbot.storage.renewal_file_for_certname') @mock.patch('certbot.cert_manager.match_and_check_overlaps') diff --git a/certbot/tests/util.py b/certbot/tests/util.py index ddd4a1aecb6..c43b44522f9 100644 --- a/certbot/tests/util.py +++ b/certbot/tests/util.py @@ -14,10 +14,11 @@ from cryptography.hazmat.primitives import serialization import mock import OpenSSL -import josepy as jose import six from six.moves import reload_module # pylint: disable=import-error +from acme import jose + from certbot import constants from certbot import interfaces from certbot import storage diff --git a/letsencrypt-auto-source/Dockerfile.centos6 b/letsencrypt-auto-source/Dockerfile.centos6 index 47eb48f50c4..8c1a4b353c1 100644 --- a/letsencrypt-auto-source/Dockerfile.centos6 +++ b/letsencrypt-auto-source/Dockerfile.centos6 @@ -33,5 +33,4 @@ COPY . /home/lea/certbot/letsencrypt-auto-source USER lea WORKDIR /home/lea -RUN sudo chmod +x certbot/letsencrypt-auto-source/tests/centos6_tests.sh -CMD sudo certbot/letsencrypt-auto-source/tests/centos6_tests.sh +CMD ["pytest", "-v", "-s", "certbot/letsencrypt-auto-source/tests"] diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index f1361d8eae9..8d2e8a6b63a 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -68,12 +68,10 @@ for arg in "$@" ; do NO_BOOTSTRAP=1;; --help) HELP=1;; - --noninteractive|--non-interactive) - NONINTERACTIVE=1;; + --noninteractive|--non-interactive|renew) + ASSUME_YES=1;; --quiet) QUIET=1;; - renew) - ASSUME_YES=1;; --verbose) VERBOSE=1;; -[!-]*) @@ -95,7 +93,7 @@ done if [ $BASENAME = "letsencrypt-auto" ]; then # letsencrypt-auto does not respect --help or --yes for backwards compatibility - NONINTERACTIVE=1 + ASSUME_YES=1 HELP=0 fi @@ -246,29 +244,15 @@ DeprecationBootstrap() { fi } -# Sets LE_PYTHON to Python version string and PYVER to the first two -# digits of the python version + DeterminePythonVersion() { - # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python - if [ -n "$USE_PYTHON_3" ]; then - for LE_PYTHON in "$LE_PYTHON" python3; do - # Break (while keeping the LE_PYTHON value) if found. - $EXISTS "$LE_PYTHON" > /dev/null && break - done - else - for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do - # Break (while keeping the LE_PYTHON value) if found. - $EXISTS "$LE_PYTHON" > /dev/null && break - done - fi + for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do + # Break (while keeping the LE_PYTHON value) if found. + $EXISTS "$LE_PYTHON" > /dev/null && break + done if [ "$?" != "0" ]; then - if [ "$1" != "NOCRASH" ]; then - error "Cannot find any Pythons; please install one!" - exit 1 - else - PYVER=0 - return 0 - fi + error "Cannot find any Pythons; please install one!" + exit 1 fi export LE_PYTHON @@ -400,19 +384,23 @@ BootstrapDebCommon() { fi } -# If new packages are installed by BootstrapRpmCommonBase below, version -# numbers in rpm_common.sh and rpm_python3.sh must be increased. +# If new packages are installed by BootstrapRpmCommon below, this version +# number must be increased. +BOOTSTRAP_RPM_COMMON_VERSION=1 + +BootstrapRpmCommon() { + # Tested with: + # - Fedora 20, 21, 22, 23 (x64) + # - Centos 7 (x64: on DigitalOcean droplet) + # - CentOS 7 Minimal install in a Hyper-V VM + # - CentOS 6 (EPEL must be installed manually) -# Sets TOOL to the name of the package manager -# Sets appropriate values for YES_FLAG and QUIET_FLAG based on $ASSUME_YES and $QUIET_FLAG. -# Enables EPEL if applicable and possible. -InitializeRPMCommonBase() { if type dnf 2>/dev/null then - TOOL=dnf + tool=dnf elif type yum 2>/dev/null then - TOOL=yum + tool=yum else error "Neither yum nor dnf found. Aborting bootstrap!" @@ -420,15 +408,15 @@ InitializeRPMCommonBase() { fi if [ "$ASSUME_YES" = 1 ]; then - YES_FLAG="-y" + yes_flag="-y" fi if [ "$QUIET" = 1 ]; then QUIET_FLAG='--quiet' fi - if ! $TOOL list *virtualenv >/dev/null 2>&1; then + if ! $tool list *virtualenv >/dev/null 2>&1; then echo "To use Certbot, packages from the EPEL repository need to be installed." - if ! $TOOL list epel-release >/dev/null 2>&1; then + if ! $tool list epel-release >/dev/null 2>&1; then error "Enable the EPEL repository and try running Certbot again." exit 1 fi @@ -440,17 +428,11 @@ InitializeRPMCommonBase() { /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 seconds..." sleep 1s fi - if ! $TOOL install $YES_FLAG $QUIET_FLAG epel-release; then + if ! $tool install $yes_flag $QUIET_FLAG epel-release; then error "Could not enable EPEL. Aborting bootstrap!" exit 1 fi fi -} - -BootstrapRpmCommonBase() { - # Arguments: whitespace-delimited python packages to install - - InitializeRPMCommonBase # This call is superfluous in practice pkgs=" gcc @@ -462,39 +444,10 @@ BootstrapRpmCommonBase() { ca-certificates " - # Add the python packages - pkgs="$pkgs - $1 - " - - if $TOOL list installed "httpd" >/dev/null 2>&1; then - pkgs="$pkgs - mod_ssl - " - fi - - if ! $TOOL install $YES_FLAG $QUIET_FLAG $pkgs; then - error "Could not install OS dependencies. Aborting bootstrap!" - exit 1 - fi -} - -# If new packages are installed by BootstrapRpmCommon below, this version -# number must be increased. -BOOTSTRAP_RPM_COMMON_VERSION=1 - -BootstrapRpmCommon() { - # Tested with: - # - Fedora 20, 21, 22, 23 (x64) - # - Centos 7 (x64: on DigitalOcean droplet) - # - CentOS 7 Minimal install in a Hyper-V VM - # - CentOS 6 - - InitializeRPMCommonBase - # Most RPM distros use the "python" or "python-" naming convention. Let's try that first. - if $TOOL list python >/dev/null 2>&1; then - python_pkgs="$python + if $tool list python >/dev/null 2>&1; then + pkgs="$pkgs + python python-devel python-virtualenv python-tools @@ -502,8 +455,9 @@ BootstrapRpmCommon() { " # Fedora 26 starts to use the prefix python2 for python2 based packages. # this elseif is theoretically for any Fedora over version 26: - elif $TOOL list python2 >/dev/null 2>&1; then - python_pkgs="$python2 + elif $tool list python2 >/dev/null 2>&1; then + pkgs="$pkgs + python2 python2-libs python2-setuptools python2-devel @@ -514,7 +468,8 @@ BootstrapRpmCommon() { # Some distros and older versions of current distros use a "python27" # instead of the "python" or "python-" naming convention. else - python_pkgs="$python27 + pkgs="$pkgs + python27 python27-devel python27-virtualenv python27-tools @@ -522,31 +477,16 @@ BootstrapRpmCommon() { " fi - BootstrapRpmCommonBase "$python_pkgs" -} - -# If new packages are installed by BootstrapRpmPython3 below, this version -# number must be increased. -BOOTSTRAP_RPM_PYTHON3_VERSION=1 - -BootstrapRpmPython3() { - # Tested with: - # - CentOS 6 - - InitializeRPMCommonBase - - # EPEL uses python34 - if $TOOL list python34 >/dev/null 2>&1; then - python_pkgs="python34 - python34-devel - python34-tools + if $tool list installed "httpd" >/dev/null 2>&1; then + pkgs="$pkgs + mod_ssl " - else - error "No supported Python package available to install. Aborting bootstrap!" - exit 1 fi - BootstrapRpmCommonBase "$python_pkgs" + if ! $tool install $yes_flag $QUIET_FLAG $pkgs; then + error "Could not install OS dependencies. Aborting bootstrap!" + exit 1 + fi } # If new packages are installed by BootstrapSuseCommon below, this version @@ -775,24 +715,11 @@ elif [ -f /etc/mageia-release ]; then } BOOTSTRAP_VERSION="BootstrapMageiaCommon $BOOTSTRAP_MAGEIA_COMMON_VERSION" elif [ -f /etc/redhat-release ]; then - prev_le_python="$LE_PYTHON" - unset LE_PYTHON - DeterminePythonVersion "NOCRASH" - if [ "$PYVER" -eq 26 ]; then - Bootstrap() { - BootstrapMessage "RedHat-based OSes that will use Python3" - BootstrapRpmPython3 - } - USE_PYTHON_3=1 - BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION" - else - Bootstrap() { - BootstrapMessage "RedHat-based OSes" - BootstrapRpmCommon - } - BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" - fi - export LE_PYTHON="$prev_le_python" + Bootstrap() { + BootstrapMessage "RedHat-based OSes" + BootstrapRpmCommon + } + BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then Bootstrap() { BootstrapMessage "openSUSE-based OSes" @@ -931,18 +858,10 @@ if [ "$1" = "--le-auto-phase2" ]; then say "Creating virtual environment..." DeterminePythonVersion rm -rf "$VENV_PATH" - if [ "$PYVER" -le 27 ]; then - if [ "$VERBOSE" = 1 ]; then - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" - else - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null - fi + if [ "$VERBOSE" = 1 ]; then + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" else - if [ "$VERBOSE" = 1 ]; then - "$LE_PYTHON" -m venv "$VENV_PATH" - else - "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null - fi + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null fi if [ -n "$BOOTSTRAP_VERSION" ]; then @@ -1064,16 +983,9 @@ idna==2.5 \ ipaddress==1.0.16 \ --hash=sha256:935712800ce4760701d89ad677666cd52691fd2f6f0b340c8b4239a3c17988a5 \ --hash=sha256:5a3182b322a706525c46282ca6f064d27a02cffbd449f9f47416f1dc96aa71b0 -josepy==1.0.1 \ - --hash=sha256:354a3513038a38bbcd27c97b7c68a8f3dfaff0a135b20a92c6db4cc4ea72915e \ - --hash=sha256:9f48b88ca37f0244238b1cc77723989f7c54f7b90b2eee6294390bacfe870acc linecache2==1.0.0 \ --hash=sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef \ --hash=sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c -# Using an older version of mock here prevents regressions of #5276. -mock==1.3.0 \ - --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ - --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 ordereddict==1.1 \ --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f packaging==16.8 \ @@ -1150,6 +1062,10 @@ zope.interface==4.1.3 \ --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \ --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \ --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392 +# Using an older version of mock here prevents regressions of #5276. +mock==1.3.0 \ + --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ + --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 # Contains the requirements for the letsencrypt package. # @@ -1437,22 +1353,17 @@ On failure, return non-zero. """ -from __future__ import print_function, unicode_literals +from __future__ import print_function from distutils.version import LooseVersion from json import loads from os import devnull, environ from os.path import dirname, join import re -import ssl from subprocess import check_call, CalledProcessError from sys import argv, exit -try: - from urllib2 import build_opener, HTTPHandler, HTTPSHandler - from urllib2 import HTTPError, URLError -except ImportError: - from urllib.request import build_opener, HTTPHandler, HTTPSHandler - from urllib.error import HTTPError, URLError +from urllib2 import build_opener, HTTPHandler, HTTPSHandler +from urllib2 import HTTPError, URLError PUBLIC_KEY = environ.get('LE_AUTO_PUBLIC_KEY', """-----BEGIN PUBLIC KEY----- MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6MR8W/galdxnpGqBsYbq @@ -1474,11 +1385,8 @@ class HttpsGetter(object): def __init__(self): """Build an HTTPS opener.""" # Based on pip 1.4.1's URLOpener - # This verifies certs on only Python >=2.7.9, and when NO_CERT_VERIFY isn't set. - if environ.get('NO_CERT_VERIFY') == '1' and hasattr(ssl, 'SSLContext'): - self._opener = build_opener(HTTPSHandler(context=create_CERT_NONE_context())) - else: - self._opener = build_opener(HTTPSHandler()) + # This verifies certs on only Python >=2.7.9. + self._opener = build_opener(HTTPSHandler()) # Strip out HTTPHandler to prevent MITM spoof: for handler in self._opener.handlers: if isinstance(handler, HTTPHandler): @@ -1500,7 +1408,7 @@ class HttpsGetter(object): def write(contents, dir, filename): """Write something to a file in a certain directory.""" - with open(join(dir, filename), 'wb') as file: + with open(join(dir, filename), 'w') as file: file.write(contents) @@ -1508,13 +1416,13 @@ def latest_stable_version(get): """Return the latest stable release of letsencrypt.""" metadata = loads(get( environ.get('LE_AUTO_JSON_URL', - 'https://pypi.python.org/pypi/certbot/json')).decode('UTF-8')) + 'https://pypi.python.org/pypi/certbot/json'))) # metadata['info']['version'] actually returns the latest of any kind of # release release, contrary to https://wiki.python.org/moin/PyPIJSON. # The regex is a sufficient regex for picking out prereleases for most # packages, LE included. return str(max(LooseVersion(r) for r - in iter(metadata['releases'].keys()) + in metadata['releases'].iterkeys() if re.match('^[0-9.]+$', r))) @@ -1531,7 +1439,7 @@ def verified_new_le_auto(get, tag, temp_dir): 'letsencrypt-auto-source/') % tag write(get(le_auto_dir + 'letsencrypt-auto'), temp_dir, 'letsencrypt-auto') write(get(le_auto_dir + 'letsencrypt-auto.sig'), temp_dir, 'letsencrypt-auto.sig') - write(PUBLIC_KEY.encode('UTF-8'), temp_dir, 'public_key.pem') + write(PUBLIC_KEY, temp_dir, 'public_key.pem') try: with open(devnull, 'w') as dev_null: check_call(['openssl', 'dgst', '-sha256', '-verify', @@ -1546,14 +1454,6 @@ def verified_new_le_auto(get, tag, temp_dir): "certbot-auto.", exc) -def create_CERT_NONE_context(): - """Create a SSLContext object to not check hostname.""" - # PROTOCOL_TLS isn't available before 2.7.13 but this code is for 2.7.9+, so use this. - context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - context.verify_mode = ssl.CERT_NONE - return context - - def main(): get = HttpsGetter().get flag = argv[1] diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index f4c1b202f4d..4eef10c804a 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -68,12 +68,10 @@ for arg in "$@" ; do NO_BOOTSTRAP=1;; --help) HELP=1;; - --noninteractive|--non-interactive) - NONINTERACTIVE=1;; + --noninteractive|--non-interactive|renew) + ASSUME_YES=1;; --quiet) QUIET=1;; - renew) - ASSUME_YES=1;; --verbose) VERBOSE=1;; -[!-]*) @@ -95,7 +93,7 @@ done if [ $BASENAME = "letsencrypt-auto" ]; then # letsencrypt-auto does not respect --help or --yes for backwards compatibility - NONINTERACTIVE=1 + ASSUME_YES=1 HELP=0 fi @@ -246,29 +244,15 @@ DeprecationBootstrap() { fi } -# Sets LE_PYTHON to Python version string and PYVER to the first two -# digits of the python version + DeterminePythonVersion() { - # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python - if [ -n "$USE_PYTHON_3" ]; then - for LE_PYTHON in "$LE_PYTHON" python3; do - # Break (while keeping the LE_PYTHON value) if found. - $EXISTS "$LE_PYTHON" > /dev/null && break - done - else - for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do - # Break (while keeping the LE_PYTHON value) if found. - $EXISTS "$LE_PYTHON" > /dev/null && break - done - fi + for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do + # Break (while keeping the LE_PYTHON value) if found. + $EXISTS "$LE_PYTHON" > /dev/null && break + done if [ "$?" != "0" ]; then - if [ "$1" != "NOCRASH" ]; then - error "Cannot find any Pythons; please install one!" - exit 1 - else - PYVER=0 - return 0 - fi + error "Cannot find any Pythons; please install one!" + exit 1 fi export LE_PYTHON @@ -281,9 +265,7 @@ DeterminePythonVersion() { } {{ bootstrappers/deb_common.sh }} -{{ bootstrappers/rpm_common_base.sh }} {{ bootstrappers/rpm_common.sh }} -{{ bootstrappers/rpm_python3.sh }} {{ bootstrappers/suse_common.sh }} {{ bootstrappers/arch_common.sh }} {{ bootstrappers/gentoo_common.sh }} @@ -314,24 +296,11 @@ elif [ -f /etc/mageia-release ]; then } BOOTSTRAP_VERSION="BootstrapMageiaCommon $BOOTSTRAP_MAGEIA_COMMON_VERSION" elif [ -f /etc/redhat-release ]; then - prev_le_python="$LE_PYTHON" - unset LE_PYTHON - DeterminePythonVersion "NOCRASH" - if [ "$PYVER" -eq 26 ]; then - Bootstrap() { - BootstrapMessage "RedHat-based OSes that will use Python3" - BootstrapRpmPython3 - } - USE_PYTHON_3=1 - BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION" - else - Bootstrap() { - BootstrapMessage "RedHat-based OSes" - BootstrapRpmCommon - } - BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" - fi - export LE_PYTHON="$prev_le_python" + Bootstrap() { + BootstrapMessage "RedHat-based OSes" + BootstrapRpmCommon + } + BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then Bootstrap() { BootstrapMessage "openSUSE-based OSes" @@ -470,18 +439,10 @@ if [ "$1" = "--le-auto-phase2" ]; then say "Creating virtual environment..." DeterminePythonVersion rm -rf "$VENV_PATH" - if [ "$PYVER" -le 27 ]; then - if [ "$VERBOSE" = 1 ]; then - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" - else - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null - fi + if [ "$VERBOSE" = 1 ]; then + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" else - if [ "$VERBOSE" = 1 ]; then - "$LE_PYTHON" -m venv "$VENV_PATH" - else - "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null - fi + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null fi if [ -n "$BOOTSTRAP_VERSION" ]; then diff --git a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh index 80d55a3931e..5b120a9e626 100755 --- a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh @@ -7,13 +7,61 @@ BootstrapRpmCommon() { # - Fedora 20, 21, 22, 23 (x64) # - Centos 7 (x64: on DigitalOcean droplet) # - CentOS 7 Minimal install in a Hyper-V VM - # - CentOS 6 + # - CentOS 6 (EPEL must be installed manually) - InitializeRPMCommonBase + if type dnf 2>/dev/null + then + tool=dnf + elif type yum 2>/dev/null + then + tool=yum + + else + error "Neither yum nor dnf found. Aborting bootstrap!" + exit 1 + fi + + if [ "$ASSUME_YES" = 1 ]; then + yes_flag="-y" + fi + if [ "$QUIET" = 1 ]; then + QUIET_FLAG='--quiet' + fi + + if ! $tool list *virtualenv >/dev/null 2>&1; then + echo "To use Certbot, packages from the EPEL repository need to be installed." + if ! $tool list epel-release >/dev/null 2>&1; then + error "Enable the EPEL repository and try running Certbot again." + exit 1 + fi + if [ "$ASSUME_YES" = 1 ]; then + /bin/echo -n "Enabling the EPEL repository in 3 seconds..." + sleep 1s + /bin/echo -ne "\e[0K\rEnabling the EPEL repository in 2 seconds..." + sleep 1s + /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 seconds..." + sleep 1s + fi + if ! $tool install $yes_flag $QUIET_FLAG epel-release; then + error "Could not enable EPEL. Aborting bootstrap!" + exit 1 + fi + fi + + pkgs=" + gcc + augeas-libs + openssl + openssl-devel + libffi-devel + redhat-rpm-config + ca-certificates + " # Most RPM distros use the "python" or "python-" naming convention. Let's try that first. - if $TOOL list python >/dev/null 2>&1; then - python_pkgs="$python + if $tool list python >/dev/null 2>&1; then + pkgs="$pkgs + python python-devel python-virtualenv python-tools @@ -21,8 +69,9 @@ BootstrapRpmCommon() { " # Fedora 26 starts to use the prefix python2 for python2 based packages. # this elseif is theoretically for any Fedora over version 26: - elif $TOOL list python2 >/dev/null 2>&1; then - python_pkgs="$python2 + elif $tool list python2 >/dev/null 2>&1; then + pkgs="$pkgs + python2 python2-libs python2-setuptools python2-devel @@ -33,7 +82,8 @@ BootstrapRpmCommon() { # Some distros and older versions of current distros use a "python27" # instead of the "python" or "python-" naming convention. else - python_pkgs="$python27 + pkgs="$pkgs + python27 python27-devel python27-virtualenv python27-tools @@ -41,5 +91,14 @@ BootstrapRpmCommon() { " fi - BootstrapRpmCommonBase "$python_pkgs" + if $tool list installed "httpd" >/dev/null 2>&1; then + pkgs="$pkgs + mod_ssl + " + fi + + if ! $tool install $yes_flag $QUIET_FLAG $pkgs; then + error "Could not install OS dependencies. Aborting bootstrap!" + exit 1 + fi } diff --git a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh deleted file mode 100644 index d7a9f3133c8..00000000000 --- a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh +++ /dev/null @@ -1,78 +0,0 @@ -# If new packages are installed by BootstrapRpmCommonBase below, version -# numbers in rpm_common.sh and rpm_python3.sh must be increased. - -# Sets TOOL to the name of the package manager -# Sets appropriate values for YES_FLAG and QUIET_FLAG based on $ASSUME_YES and $QUIET_FLAG. -# Enables EPEL if applicable and possible. -InitializeRPMCommonBase() { - if type dnf 2>/dev/null - then - TOOL=dnf - elif type yum 2>/dev/null - then - TOOL=yum - - else - error "Neither yum nor dnf found. Aborting bootstrap!" - exit 1 - fi - - if [ "$ASSUME_YES" = 1 ]; then - YES_FLAG="-y" - fi - if [ "$QUIET" = 1 ]; then - QUIET_FLAG='--quiet' - fi - - if ! $TOOL list *virtualenv >/dev/null 2>&1; then - echo "To use Certbot, packages from the EPEL repository need to be installed." - if ! $TOOL list epel-release >/dev/null 2>&1; then - error "Enable the EPEL repository and try running Certbot again." - exit 1 - fi - if [ "$ASSUME_YES" = 1 ]; then - /bin/echo -n "Enabling the EPEL repository in 3 seconds..." - sleep 1s - /bin/echo -ne "\e[0K\rEnabling the EPEL repository in 2 seconds..." - sleep 1s - /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 seconds..." - sleep 1s - fi - if ! $TOOL install $YES_FLAG $QUIET_FLAG epel-release; then - error "Could not enable EPEL. Aborting bootstrap!" - exit 1 - fi - fi -} - -BootstrapRpmCommonBase() { - # Arguments: whitespace-delimited python packages to install - - InitializeRPMCommonBase # This call is superfluous in practice - - pkgs=" - gcc - augeas-libs - openssl - openssl-devel - libffi-devel - redhat-rpm-config - ca-certificates - " - - # Add the python packages - pkgs="$pkgs - $1 - " - - if $TOOL list installed "httpd" >/dev/null 2>&1; then - pkgs="$pkgs - mod_ssl - " - fi - - if ! $TOOL install $YES_FLAG $QUIET_FLAG $pkgs; then - error "Could not install OS dependencies. Aborting bootstrap!" - exit 1 - fi -} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh b/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh deleted file mode 100644 index b011a7235b2..00000000000 --- a/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh +++ /dev/null @@ -1,23 +0,0 @@ -# If new packages are installed by BootstrapRpmPython3 below, this version -# number must be increased. -BOOTSTRAP_RPM_PYTHON3_VERSION=1 - -BootstrapRpmPython3() { - # Tested with: - # - CentOS 6 - - InitializeRPMCommonBase - - # EPEL uses python34 - if $TOOL list python34 >/dev/null 2>&1; then - python_pkgs="python34 - python34-devel - python34-tools - " - else - error "No supported Python package available to install. Aborting bootstrap!" - exit 1 - fi - - BootstrapRpmCommonBase "$python_pkgs" -} diff --git a/letsencrypt-auto-source/pieces/dependency-requirements.txt b/letsencrypt-auto-source/pieces/dependency-requirements.txt index 0e2cec984a8..dec7ae7d065 100644 --- a/letsencrypt-auto-source/pieces/dependency-requirements.txt +++ b/letsencrypt-auto-source/pieces/dependency-requirements.txt @@ -105,16 +105,9 @@ idna==2.5 \ ipaddress==1.0.16 \ --hash=sha256:935712800ce4760701d89ad677666cd52691fd2f6f0b340c8b4239a3c17988a5 \ --hash=sha256:5a3182b322a706525c46282ca6f064d27a02cffbd449f9f47416f1dc96aa71b0 -josepy==1.0.1 \ - --hash=sha256:354a3513038a38bbcd27c97b7c68a8f3dfaff0a135b20a92c6db4cc4ea72915e \ - --hash=sha256:9f48b88ca37f0244238b1cc77723989f7c54f7b90b2eee6294390bacfe870acc linecache2==1.0.0 \ --hash=sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef \ --hash=sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c -# Using an older version of mock here prevents regressions of #5276. -mock==1.3.0 \ - --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ - --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 ordereddict==1.1 \ --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f packaging==16.8 \ @@ -191,3 +184,7 @@ zope.interface==4.1.3 \ --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \ --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \ --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392 +# Using an older version of mock here prevents regressions of #5276. +mock==1.3.0 \ + --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ + --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 diff --git a/letsencrypt-auto-source/pieces/fetch.py b/letsencrypt-auto-source/pieces/fetch.py index ae72a299b48..8f34351c941 100644 --- a/letsencrypt-auto-source/pieces/fetch.py +++ b/letsencrypt-auto-source/pieces/fetch.py @@ -11,22 +11,17 @@ """ -from __future__ import print_function, unicode_literals +from __future__ import print_function from distutils.version import LooseVersion from json import loads from os import devnull, environ from os.path import dirname, join import re -import ssl from subprocess import check_call, CalledProcessError from sys import argv, exit -try: - from urllib2 import build_opener, HTTPHandler, HTTPSHandler - from urllib2 import HTTPError, URLError -except ImportError: - from urllib.request import build_opener, HTTPHandler, HTTPSHandler - from urllib.error import HTTPError, URLError +from urllib2 import build_opener, HTTPHandler, HTTPSHandler +from urllib2 import HTTPError, URLError PUBLIC_KEY = environ.get('LE_AUTO_PUBLIC_KEY', """-----BEGIN PUBLIC KEY----- MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6MR8W/galdxnpGqBsYbq @@ -48,11 +43,8 @@ class HttpsGetter(object): def __init__(self): """Build an HTTPS opener.""" # Based on pip 1.4.1's URLOpener - # This verifies certs on only Python >=2.7.9, and when NO_CERT_VERIFY isn't set. - if environ.get('NO_CERT_VERIFY') == '1' and hasattr(ssl, 'SSLContext'): - self._opener = build_opener(HTTPSHandler(context=create_CERT_NONE_context())) - else: - self._opener = build_opener(HTTPSHandler()) + # This verifies certs on only Python >=2.7.9. + self._opener = build_opener(HTTPSHandler()) # Strip out HTTPHandler to prevent MITM spoof: for handler in self._opener.handlers: if isinstance(handler, HTTPHandler): @@ -74,7 +66,7 @@ def get(self, url): def write(contents, dir, filename): """Write something to a file in a certain directory.""" - with open(join(dir, filename), 'wb') as file: + with open(join(dir, filename), 'w') as file: file.write(contents) @@ -82,13 +74,13 @@ def latest_stable_version(get): """Return the latest stable release of letsencrypt.""" metadata = loads(get( environ.get('LE_AUTO_JSON_URL', - 'https://pypi.python.org/pypi/certbot/json')).decode('UTF-8')) + 'https://pypi.python.org/pypi/certbot/json'))) # metadata['info']['version'] actually returns the latest of any kind of # release release, contrary to https://wiki.python.org/moin/PyPIJSON. # The regex is a sufficient regex for picking out prereleases for most # packages, LE included. return str(max(LooseVersion(r) for r - in iter(metadata['releases'].keys()) + in metadata['releases'].iterkeys() if re.match('^[0-9.]+$', r))) @@ -105,7 +97,7 @@ def verified_new_le_auto(get, tag, temp_dir): 'letsencrypt-auto-source/') % tag write(get(le_auto_dir + 'letsencrypt-auto'), temp_dir, 'letsencrypt-auto') write(get(le_auto_dir + 'letsencrypt-auto.sig'), temp_dir, 'letsencrypt-auto.sig') - write(PUBLIC_KEY.encode('UTF-8'), temp_dir, 'public_key.pem') + write(PUBLIC_KEY, temp_dir, 'public_key.pem') try: with open(devnull, 'w') as dev_null: check_call(['openssl', 'dgst', '-sha256', '-verify', @@ -120,14 +112,6 @@ def verified_new_le_auto(get, tag, temp_dir): "certbot-auto.", exc) -def create_CERT_NONE_context(): - """Create a SSLContext object to not check hostname.""" - # PROTOCOL_TLS isn't available before 2.7.13 but this code is for 2.7.9+, so use this. - context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - context.verify_mode = ssl.CERT_NONE - return context - - def main(): get = HttpsGetter().get flag = argv[1] diff --git a/letsencrypt-auto-source/tests/auto_test.py b/letsencrypt-auto-source/tests/auto_test.py index d187452a125..2fa03105d75 100644 --- a/letsencrypt-auto-source/tests/auto_test.py +++ b/letsencrypt-auto-source/tests/auto_test.py @@ -30,10 +30,6 @@ def tests_dir(): from build import build as build_le_auto -BOOTSTRAP_FILENAME = 'certbot-auto-bootstrap-version.txt' -"""Name of the file where certbot-auto saves its bootstrap version.""" - - class RequestHandler(BaseHTTPRequestHandler): """An HTTPS request handler which is quiet and serves a specific folder.""" @@ -202,7 +198,6 @@ def run_le_auto(le_auto_path, venv_dir, base_url, **kwargs): Z5o/NDiQB11m91yNB0MmPYY9QSbnOA9j7IaaC97AwRLuwXY+/R2ablTcxurWou68 iQIDAQAB -----END PUBLIC KEY-----""", - NO_CERT_VERIFY='1', **kwargs) env.update(d) return out_and_err( @@ -301,31 +296,17 @@ def test_successes(self): def test_phase2_upgrade(self): """Test a phase-2 upgrade without a phase-1 upgrade.""" - resources = {'certbot/json': dumps({'releases': {'99.9.9': None}}), - 'v99.9.9/letsencrypt-auto': self.NEW_LE_AUTO, - 'v99.9.9/letsencrypt-auto.sig': self.NEW_LE_AUTO_SIG} - with serving(resources) as base_url: - pip_find_links=join(tests_dir(), 'fake-letsencrypt', 'dist') - with temp_paths() as (le_auto_path, venv_dir): - install_le_auto(self.NEW_LE_AUTO, le_auto_path) - - # Create venv saving the correct bootstrap script version - out, err = run_le_auto(le_auto_path, venv_dir, base_url, - PIP_FIND_LINKS=pip_find_links) - self.assertFalse('Upgrading certbot-auto ' in out) - self.assertTrue('Creating virtual environment...' in out) - with open(join(venv_dir, BOOTSTRAP_FILENAME)) as f: - bootstrap_version = f.read() - - # Create a new venv with an old letsencrypt version - with temp_paths() as (le_auto_path, venv_dir): + with temp_paths() as (le_auto_path, venv_dir): + resources = {'certbot/json': dumps({'releases': {'99.9.9': None}}), + 'v99.9.9/letsencrypt-auto': self.NEW_LE_AUTO, + 'v99.9.9/letsencrypt-auto.sig': self.NEW_LE_AUTO_SIG} + with serving(resources) as base_url: venv_bin = join(venv_dir, 'bin') makedirs(venv_bin) set_le_script_version(venv_dir, '0.0.1') - with open(join(venv_dir, BOOTSTRAP_FILENAME), 'w') as f: - f.write(bootstrap_version) install_le_auto(self.NEW_LE_AUTO, le_auto_path) + pip_find_links=join(tests_dir(), 'fake-letsencrypt', 'dist') out, err = run_le_auto(le_auto_path, venv_dir, base_url, PIP_FIND_LINKS=pip_find_links) @@ -350,7 +331,6 @@ def test_openssl_failure(self): self.assertTrue("Couldn't verify signature of downloaded " "certbot-auto." in exc.output) else: - print(out) self.fail('Signature check on certbot-auto erroneously passed.') def test_pip_failure(self): diff --git a/letsencrypt-auto-source/tests/centos6_tests.sh b/letsencrypt-auto-source/tests/centos6_tests.sh deleted file mode 100644 index e3ebbaec55d..00000000000 --- a/letsencrypt-auto-source/tests/centos6_tests.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/bash -# Start by making sure your system is up-to-date: -yum update > /dev/null -yum install -y centos-release-scl > /dev/null -yum install -y python27 > /dev/null 2> /dev/null - -# we're going to modify env variables, so do this in a subshell -( -source /opt/rh/python27/enable - -# ensure python 3 isn't installed -python3 --version 2> /dev/null -RESULT=$? -if [ $RESULT -eq 0 ]; then - error "Python3 is already installed." - exit 1 -fi - -# ensure python2.7 is available -python2.7 --version 2> /dev/null -RESULT=$? -if [ $RESULT -ne 0 ]; then - error "Python3 is not available." - exit 1 -fi - -# bootstrap, but don't install python 3. -certbot/letsencrypt-auto-source/letsencrypt-auto --no-self-upgrade -n > /dev/null 2> /dev/null - -# ensure python 3 isn't installed -python3 --version 2> /dev/null -RESULT=$? -if [ $RESULT -eq 0 ]; then - error "letsencrypt-auto installed Python3 even though Python2.7 is present." - exit 1 -fi - -echo "" -echo "PASSED: Did not upgrade to Python3 when Python2.7 is present." -) - -# ensure python2.7 isn't available -python2.7 --version 2> /dev/null -RESULT=$? -if [ $RESULT -eq 0 ]; then - error "Python2.7 is still available." - exit 1 -fi - -# bootstrap, this time installing python3 -certbot/letsencrypt-auto-source/letsencrypt-auto --no-self-upgrade -n > /dev/null 2> /dev/null - -# ensure python 3 is installed -python3 --version > /dev/null -RESULT=$? -if [ $RESULT -ne 0 ]; then - error "letsencrypt-auto failed to install Python3 when only Python2.6 is present." - exit 1 -fi - -echo "PASSED: Successfully upgraded to Python3 when only Python2.6 is present." -echo "" - -# test using python3 -pytest -v -s certbot/letsencrypt-auto-source/tests diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index b64550cb72a..00000000000 --- a/pytest.ini +++ /dev/null @@ -1,2 +0,0 @@ -[pytest] -addopts = --quiet diff --git a/setup.py b/setup.py index ce505a62ef4..ee108c51485 100644 --- a/setup.py +++ b/setup.py @@ -30,9 +30,10 @@ def read_file(filename, encoding='utf8'): changes = read_file(os.path.join(here, 'CHANGES.rst')) version = meta['version'] -# This package relies on PyOpenSSL, requests, and six, however, it isn't -# specified here to avoid masking the more specific request requirements in -# acme. See https://github.com/pypa/pip/issues/988 for more info. +# Please update tox.ini when modifying dependency version requirements +# This package relies on requests, however, it isn't specified here to avoid +# masking the more specific request requirements in acme. See +# https://github.com/pypa/pip/issues/988 for more info. install_requires = [ 'acme=={0}'.format(version), # We technically need ConfigArgParse 0.10.0 for Python 2.6 support, but @@ -43,11 +44,13 @@ def read_file(filename, encoding='utf8'): 'cryptography>=1.2', # load_pem_x509_certificate 'mock', 'parsedatetime>=1.3', # Calendar.parseDT + 'PyOpenSSL', 'pyrfc3339', 'pytz', # For pkg_resources. >=1.0 so pip resolves it to a version cryptography # will tolerate; see #2599: 'setuptools>=1.0', + 'six', 'zope.component', 'zope.interface', ] diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index e1aad43365d..1e0b7754b78 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -345,14 +345,9 @@ common auth --must-staple --domains "must-staple.le.wtf" openssl x509 -in "${root}/conf/live/must-staple.le.wtf/cert.pem" -text | grep '1.3.6.1.5.5.7.1.24' # revoke by account key -common revoke --cert-path "$root/conf/live/le.wtf/cert.pem" --delete-after-revoke +common revoke --cert-path "$root/conf/live/le.wtf/cert.pem" # revoke renewed -common revoke --cert-path "$root/conf/live/le1.wtf/cert.pem" --no-delete-after-revoke -if [ ! -d "$root/conf/live/le1.wtf" ]; then - echo "cert deleted when --no-delete-after-revoke was used!" - exit 1 -fi -common delete --cert-name le1.wtf +common revoke --cert-path "$root/conf/live/le1.wtf/cert.pem" # revoke by cert key common revoke --cert-path "$root/conf/live/le2.wtf/cert.pem" \ --key-path "$root/conf/live/le2.wtf/privkey.pem" diff --git a/tests/letstest/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh index a83cbd8262b..cb659786efd 100755 --- a/tests/letstest/scripts/test_leauto_upgrades.sh +++ b/tests/letstest/scripts/test_leauto_upgrades.sh @@ -15,56 +15,19 @@ if ! command -v git ; then exit 1 fi fi +BRANCH=`git rev-parse --abbrev-ref HEAD` # 0.5.0 is the oldest version of letsencrypt-auto that can be used because it's # the first version that pins package versions, properly supports # --no-self-upgrade, and works with newer versions of pip. -git checkout -f v0.5.0 letsencrypt-auto +git checkout -f v0.5.0 if ! ./letsencrypt-auto -v --debug --version --no-self-upgrade 2>&1 | grep 0.5.0 ; then echo initial installation appeared to fail exit 1 fi -# Now that python and openssl have been installed, we can set up a fake server -# to provide a new version of letsencrypt-auto. First, we start the server and -# directory to be served. -MY_TEMP_DIR=$(mktemp -d) -PORT_FILE="$MY_TEMP_DIR/port" -SERVER_PATH=$(tools/readlink.py tools/simple_http_server.py) -cd "$MY_TEMP_DIR" -"$SERVER_PATH" 0 > $PORT_FILE & -SERVER_PID=$! -trap 'kill "$SERVER_PID" && rm -rf "$MY_TEMP_DIR"' EXIT -cd ~- - -# Then, we set up the files to be served. -FAKE_VERSION_NUM="99.99.99" -echo "{\"releases\": {\"$FAKE_VERSION_NUM\": null}}" > "$MY_TEMP_DIR/json" -LE_AUTO_SOURCE_DIR="$MY_TEMP_DIR/v$FAKE_VERSION_NUM" -NEW_LE_AUTO_PATH="$LE_AUTO_SOURCE_DIR/letsencrypt-auto" -mkdir "$LE_AUTO_SOURCE_DIR" -cp letsencrypt-auto-source/letsencrypt-auto "$LE_AUTO_SOURCE_DIR/letsencrypt-auto" -SIGNING_KEY="letsencrypt-auto-source/tests/signing.key" -openssl dgst -sha256 -sign "$SIGNING_KEY" -out "$NEW_LE_AUTO_PATH.sig" "$NEW_LE_AUTO_PATH" - -# Next, we wait for the server to start and get the port number. -sleep 5s -SERVER_PORT=$(sed -n 's/.*port \([0-9]\+\).*/\1/p' "$PORT_FILE") - -# Finally, we set the necessary certbot-auto environment variables. -export LE_AUTO_DIR_TEMPLATE="http://localhost:$SERVER_PORT/%s/" -export LE_AUTO_JSON_URL="http://localhost:$SERVER_PORT/json" -export LE_AUTO_PUBLIC_KEY="-----BEGIN PUBLIC KEY----- -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsMoSzLYQ7E1sdSOkwelg -tzKIh2qi3bpXuYtcfFC0XrvWig071NwIj+dZiT0OLZ2hPispEH0B7ISuuWg1ll7G -hFW0VdbxL6JdGzS2ShNWkX9hE9z+j8VqwDPOBn3ZHm03qwpYkBDwQib3KqOdYbTT -uUtJmmGcuk3a9Aq/sCT6DdfmTSdP5asdQYwIcaQreDrOosaS84DTWI3IU+UYJVgl -LsIVPBuy9IcgHidUQ96hJnoPsDCWsHwX62495QKEarauyKQrJzFes0EY95orDM47 -Z5o/NDiQB11m91yNB0MmPYY9QSbnOA9j7IaaC97AwRLuwXY+/R2ablTcxurWou68 -iQIDAQAB ------END PUBLIC KEY----- -" - -if ! ./letsencrypt-auto -v --debug --version || ! diff letsencrypt-auto letsencrypt-auto-source/letsencrypt-auto ; then +git checkout -f "$BRANCH" +EXPECTED_VERSION=$(grep -m1 LE_AUTO_VERSION letsencrypt-auto | cut -d\" -f2) +if ! ./letsencrypt-auto -v --debug --version --no-self-upgrade 2>&1 | grep $EXPECTED_VERSION ; then echo upgrade appeared to fail exit 1 fi diff --git a/tools/deactivate.py b/tools/deactivate.py index d43b845523f..5facc8436da 100644 --- a/tools/deactivate.py +++ b/tools/deactivate.py @@ -18,10 +18,10 @@ from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.primitives import serialization -import josepy as jose from acme import client as acme_client from acme import errors as acme_errors +from acme import jose from acme import messages DIRECTORY = os.getenv('DIRECTORY', 'http://localhost:4000/directory') diff --git a/tools/install_and_test.sh b/tools/install_and_test.sh index 0d39e0594bf..d57f0974ea3 100755 --- a/tools/install_and_test.sh +++ b/tools/install_and_test.sh @@ -2,9 +2,8 @@ # pip installs the requested packages in editable mode and runs unit tests on # them. Each package is installed and tested in the order they are provided # before the script moves on to the next package. If CERTBOT_NO_PIN is set not -# set to 1, packages are installed using pinned versions of all of our -# dependencies. See pip_install.sh for more information on the versions pinned -# to. +# set to 1, packages are installed using certbot-auto's requirements file as +# constraints. if [ "$CERTBOT_NO_PIN" = 1 ]; then pip_install="pip install -q -e" @@ -23,5 +22,5 @@ for requirement in "$@" ; do # See https://travis-ci.org/certbot/certbot/jobs/308774157#L1333. pkg=$(echo "$pkg" | tr - _) fi - "$(dirname $0)/pytest.sh" --pyargs $pkg + pytest --numprocesses auto --quiet --pyargs $pkg done diff --git a/tools/merge_requirements.py b/tools/merge_requirements.py deleted file mode 100755 index c8fb9535182..00000000000 --- a/tools/merge_requirements.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python -"""Merges multiple Python requirements files into one file. - -Requirements files specified later take precedence over earlier ones. Only -simple SomeProject==1.2.3 format is currently supported. - -""" - -from __future__ import print_function - -import sys - - -def read_file(file_path): - """Reads in a Python requirements file. - - :param str file_path: path to requirements file - - :returns: mapping from a project to its pinned version - :rtype: dict - - """ - d = {} - with open(file_path) as f: - for line in f: - line = line.strip() - if line and not line.startswith('#'): - project, version = line.split('==') - if not version: - raise ValueError("Unexpected syntax '{0}'".format(line)) - d[project] = version - return d - - -def print_requirements(requirements): - """Prints requirements to stdout. - - :param dict requirements: mapping from a project to its pinned version - - """ - print('\n'.join('{0}=={1}'.format(k, v) - for k, v in sorted(requirements.items()))) - - -def merge_requirements_files(*files): - """Merges multiple requirements files together and prints the result. - - Requirement files specified later in the list take precedence over earlier - files. - - :param tuple files: paths to requirements files - - """ - d = {} - for f in files: - d.update(read_file(f)) - print_requirements(d) - - -if __name__ == '__main__': - merge_requirements_files(*sys.argv[1:]) diff --git a/tools/oldest_constraints.txt b/tools/oldest_constraints.txt deleted file mode 100644 index de2b83ad8b1..00000000000 --- a/tools/oldest_constraints.txt +++ /dev/null @@ -1,51 +0,0 @@ -# This file contains the oldest versions of our dependencies we say we require -# in our packages or versions we need to support to maintain compatibility with -# the versions included in the various Linux distros where we are packaged. - -# CentOS/RHEL 7 EPEL constraints -cffi==1.6.0 -chardet==2.2.1 -configobj==4.7.2 -ipaddress==1.0.16 -mock==1.0.1 -ndg-httpsclient==0.3.2 -ply==3.4 -pyasn1==0.1.9 -pycparser==2.14 -pyOpenSSL==0.13.1 -pyparsing==1.5.6 -pyRFC3339==1.0 -python-augeas==0.5.0 -six==1.9.0 -# setuptools 0.9.8 is the actual version packaged, but some other dependencies -# in this file require setuptools>=1.0 and there are no relevant changes for us -# between these versions. -setuptools==1.0.0 -urllib3==1.10.2 -zope.component==4.1.0 -zope.event==4.0.3 -zope.interface==4.0.5 - -# Debian Jessie Backports constraints -PyICU==1.8 -colorama==0.3.2 -enum34==1.0.3 -html5lib==0.999 -idna==2.0 -pbr==1.8.0 -pytz==2012rc0 - -# Our setup.py constraints -cloudflare==1.5.1 -cryptography==1.2.0 -google-api-python-client==1.5 -oauth2client==2.0 -parsedatetime==1.3 -pyparsing==1.5.5 -python-digitalocean==1.11 -requests[security]==2.4.1 - -# Ubuntu Xenial constraints -ConfigArgParse==0.10.0 -funcsigs==0.4 -zope.hookable==4.0.4 diff --git a/tools/dev_constraints.txt b/tools/pip_constraints.txt similarity index 71% rename from tools/dev_constraints.txt rename to tools/pip_constraints.txt index afc362ff8b5..cacec37d667 100644 --- a/tools/dev_constraints.txt +++ b/tools/pip_constraints.txt @@ -1,15 +1,16 @@ # Specifies Python package versions for packages not specified in -# letsencrypt-auto's requirements file. +# letsencrypt-auto's requirements file. We should avoid listing packages in +# both places because if both files are used as constraints for the same pip +# invocation, some constraints may be ignored due to pip's lack of dependency +# resolution. alabaster==0.7.10 apipkg==1.4 -asn1crypto==0.22.0 astroid==1.3.5 -attrs==17.3.0 Babel==2.5.1 backports.shutil-get-terminal-size==1.0.0 boto3==1.4.7 botocore==1.7.41 -cloudflare==1.5.1 +cloudflare==1.8.1 coverage==4.4.2 decorator==4.1.2 dns-lexicon==2.1.14 @@ -18,7 +19,7 @@ docutils==0.14 execnet==1.5.0 future==0.16.0 futures==3.1.1 -google-api-python-client==1.5 +google-api-python-client==1.6.4 httplib2==0.10.3 imagesize==0.7.1 ipdb==0.10.3 @@ -26,22 +27,20 @@ ipython==5.5.0 ipython-genutils==0.2.0 Jinja2==2.9.6 jmespath==0.9.3 -josepy==1.0.1 -logger==1.4 logilab-common==1.4.1 MarkupSafe==1.0 -ndg-httpsclient==0.3.2 -oauth2client==2.0.0 +oauth2client==4.1.2 pathlib2==2.3.0 pexpect==4.2.1 pickleshare==0.7.4 +pkg-resources==0.0.0 pkginfo==1.4.1 pluggy==0.5.2 prompt-toolkit==1.0.15 ptyprocess==0.5.2 py==1.4.34 -pyasn1==0.1.9 -pyasn1-modules==0.0.10 +pyasn1==0.3.7 +pyasn1-modules==0.1.5 Pygments==2.2.0 pylint==1.4.2 pytest==3.2.5 @@ -49,7 +48,7 @@ pytest-cov==2.5.1 pytest-forked==0.2 pytest-xdist==1.20.1 python-dateutil==2.6.1 -python-digitalocean==1.11 +python-digitalocean==1.12 PyYAML==3.12 repoze.sphinx.autointerface==0.8 requests-file==1.4.2 @@ -66,6 +65,6 @@ tox==2.9.1 tqdm==4.19.4 traitlets==4.3.2 twine==1.9.1 -uritemplate==0.6 +uritemplate==3.0.0 virtualenv==15.1.0 wcwidth==0.1.7 diff --git a/tools/pip_install.sh b/tools/pip_install.sh index d2aae4a43ac..fafd58e5490 100755 --- a/tools/pip_install.sh +++ b/tools/pip_install.sh @@ -1,26 +1,17 @@ -#!/bin/bash -e -# pip installs packages using pinned package versions. If CERTBOT_OLDEST is set -# to 1, a combination of tools/oldest_constraints.txt and -# tools/dev_constraints.txt is used, otherwise, a combination of certbot-auto's -# requirements file and tools/dev_constraints.txt is used. The other file -# always takes precedence over tools/dev_constraints.txt. +#!/bin/sh -e +# pip installs packages using pinned package versions # get the root of the Certbot repo -tools_dir=$(dirname $("$(dirname $0)/readlink.py" $0)) -dev_constraints="$tools_dir/dev_constraints.txt" -merge_reqs="$tools_dir/merge_requirements.py" -test_constraints=$(mktemp) -trap "rm -f $test_constraints" EXIT - -if [ "$CERTBOT_OLDEST" = 1 ]; then - cp "$tools_dir/oldest_constraints.txt" "$test_constraints" -else - repo_root=$(dirname "$tools_dir") - certbot_requirements="$repo_root/letsencrypt-auto-source/pieces/dependency-requirements.txt" - sed -n -e 's/^\([^[:space:]]*==[^[:space:]]*\).*$/\1/p' "$certbot_requirements" > "$test_constraints" -fi +my_path=$("$(dirname $0)/readlink.py" $0) +repo_root=$(dirname $(dirname $my_path)) +requirements="$repo_root/letsencrypt-auto-source/pieces/dependency-requirements.txt" +certbot_auto_constraints=$(mktemp) +trap "rm -f $certbot_auto_constraints" EXIT +# extract pinned requirements without hashes +sed -n -e 's/^\([^[:space:]]*==[^[:space:]]*\).*$/\1/p' $requirements > $certbot_auto_constraints +dev_constraints="$(dirname $my_path)/pip_constraints.txt" set -x # install the requested packages using the pinned requirements as constraints -pip install -q --constraint <("$merge_reqs" "$dev_constraints" "$test_constraints") "$@" +pip install -q --constraint $certbot_auto_constraints --constraint $dev_constraints "$@" diff --git a/tools/pytest.sh b/tools/pytest.sh deleted file mode 100755 index 8e3619d5dbb..00000000000 --- a/tools/pytest.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -# Runs pytest with the provided arguments, adding --numprocesses to the command -# line. This argument is set to "auto" if the environmnent variable TRAVIS is -# not set, otherwise, it is set to 2. This works around -# https://github.com/pytest-dev/pytest-xdist/issues/9. Currently every Travis -# environnment provides two cores. See -# https://docs.travis-ci.com/user/reference/overview/#Virtualization-environments. - -if ${TRAVIS:-false}; then - NUMPROCESSES="2" -else - NUMPROCESSES="auto" -fi - -pytest --numprocesses "$NUMPROCESSES" "$@" diff --git a/tools/simple_http_server.py b/tools/simple_http_server.py deleted file mode 100755 index 26bf231b7bf..00000000000 --- a/tools/simple_http_server.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python -"""A version of Python 2.x's SimpleHTTPServer that flushes its output.""" -from BaseHTTPServer import HTTPServer -from SimpleHTTPServer import SimpleHTTPRequestHandler -import sys - -def serve_forever(port=0): - """Spins up an HTTP server on all interfaces and the given port. - - A message is printed to stdout specifying the address and port being used - by the server. - - :param int port: port number to use. - - """ - server = HTTPServer(('', port), SimpleHTTPRequestHandler) - print 'Serving HTTP on {0} port {1} ...'.format(*server.server_address) - sys.stdout.flush() - server.serve_forever() - - -if __name__ == '__main__': - kwargs = {} - if len(sys.argv) > 1: - kwargs['port'] = int(sys.argv[1]) - serve_forever(**kwargs) diff --git a/tox.cover.sh b/tox.cover.sh index bc0e5a8bf83..2b5a3cf19a9 100755 --- a/tox.cover.sh +++ b/tox.cover.sh @@ -51,8 +51,7 @@ cover () { fi pkg_dir=$(echo "$1" | tr _ -) - pytest="$(dirname $0)/tools/pytest.sh" - "$pytest" --cov "$pkg_dir" --cov-append --cov-report= --pyargs "$1" + pytest --cov "$pkg_dir" --cov-append --cov-report= --numprocesses auto --pyargs "$1" coverage report --fail-under="$min" --include="$pkg_dir/*" --show-missing } diff --git a/tox.ini b/tox.ini index 20f5cda32d2..bb421daa536 100644 --- a/tox.ini +++ b/tox.ini @@ -11,8 +11,9 @@ envlist = modification,py{26,33,34,35,36},cover,lint pip_install = {toxinidir}/tools/pip_install_editable.sh # pip installs the requested packages in editable mode and runs unit tests on # them. Each package is installed and tested in the order they are provided -# before the script moves on to the next package. All dependencies are pinned -# to a specific version for increased stability for developers. +# before the script moves on to the next package. If CERTBOT_NO_PIN is set not +# set to 1, packages are installed using certbot-auto's requirements file as +# constraints. install_and_test = {toxinidir}/tools/install_and_test.sh py26_packages = acme[dev] \ @@ -61,7 +62,6 @@ commands = deps = setuptools==36.8.0 wheel==0.29.0 -passenv = TRAVIS [testenv] commands = @@ -70,25 +70,48 @@ commands = setenv = PYTHONPATH = {toxinidir} PYTHONHASHSEED = 0 -passenv = - {[testenv:py26]passenv} [testenv:py33] commands = {[testenv]commands} deps = wheel==0.29.0 -passenv = - {[testenv]passenv} [testenv:py27-oldest] commands = {[testenv]commands} setenv = {[testenv]setenv} - CERTBOT_OLDEST=1 -passenv = - {[testenv]passenv} + CERTBOT_NO_PIN=1 +deps = + PyOpenSSL==0.13 + cffi==1.5.2 + configargparse==0.10.0 + configargparse==0.10.0 + configobj==4.7.2 + cryptography==1.2.3 + enum34==0.9.23 + google-api-python-client==1.5 + idna==2.0 + ipaddress==1.0.16 + mock==1.0.1 + ndg-httpsclient==0.3.2 + oauth2client==2.0 + parsedatetime==1.4 + pyasn1-modules==0.0.5 + pyasn1==0.1.9 + pyparsing==1.5.6 + pyrfc3339==1.0 + pytest==3.2.5 + python-augeas==0.4.1 + pytz==2012c + requests[security]==2.6.0 + setuptools==0.9.8 + six==1.9.0 + urllib3==1.10 + zope.component==4.0.2 + zope.event==4.0.1 + zope.interface==4.0.5 [testenv:py27_install] basepython = python2.7 @@ -100,8 +123,6 @@ basepython = python2.7 commands = {[base]install_packages} ./tox.cover.sh -passenv = - {[testenv]passenv} [testenv:lint] # recent versions of pylint do not support Python 2.6 (#97, #187)