Merge pull request #22 from Revolution1/fix/tls

add ssl support

.travis.yml

@@ -22,9 +22,9 @@ env:
 
 before_install:
   - docker pull quay.io/coreos/etcd:$ETCD_VER
-  - docker run -d -p 2379:2379 -p 2380:2380 --name etcd3 quay.io/coreos/etcd:$ETCD_VER etcd --name node1 --initial-advertise-peer-urls http://0.0.0.0:2380 --listen-peer-urls http://0.0.0.0:2380 --advertise-client-urls http://0.0.0.0:2379 --listen-client-urls http://0.0.0.0:2379 --initial-cluster node1=http://0.0.0.0:2380
+  - docker run -d -p 2379:2379 -p 2380:2380 --name etcd3_1 quay.io/coreos/etcd:$ETCD_VER etcd --name node1 --initial-advertise-peer-urls http://0.0.0.0:2380 --listen-peer-urls http://0.0.0.0:2380 --advertise-client-urls http://0.0.0.0:2379 --listen-client-urls http://0.0.0.0:2379 --initial-cluster node1=http://0.0.0.0:2380
   - docker ps
-  - sudo docker cp etcd3:/usr/local/bin/etcdctl /usr/bin/etcdctl
+  - sudo docker cp etcd3_1:/usr/local/bin/etcdctl /usr/bin/etcdctl
   - which etcdctl
 
 # command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors

README.md

@@ -24,7 +24,7 @@ Notice: The authentication header through gRPC-JSON-Gateway only supported in [e
 * [x] Support etcd3 gRPC-JSON-Gateway including stream
 * [x] Response modelizing based on etcd3's swagger spec
 * [x] Generate code from swagger spec
-* [ ] TLS Connection
+* [x] TLS Connection
 * [x] support APIs
     * [x] Auth
     * [x] KV
@@ -49,7 +49,7 @@ $ pip install etcd3-py
 **Sync Client**
 ```python
 >>> from etcd3 import Client
->>> client = Client('127.0.0.1', 2379)
+>>> client = Client('127.0.0.1', 2379, cert=(CERT_PATH, KEY_PATH), verify=CA_PATH)
 >>> client.version()
 EtcdVersion(etcdserver='3.3.0-rc.4', etcdcluster='3.3.0')
 >>> client.put('foo', 'bar')

etcd3/aio_client.py

@@ -4,16 +4,18 @@
 
 import json
 import ssl
+import warnings
 
 import aiohttp
+import six
 from aiohttp.client import _RequestContextManager
 
 from .baseclient import BaseClient
 from .baseclient import BaseModelizedStreamResponse
 from .errors import Etcd3Exception
 from .errors import Etcd3StreamError
 from .errors import get_client_error
-from .utils import iter_json_string
+from .utils import iter_json_string, Etcd3Warning
 
 
 class ModelizedResponse(object):
@@ -126,19 +128,39 @@ async def next(self):
 
 class AioClient(BaseClient):
     def __init__(self, host='localhost', port=2379, protocol='http',
-                 ca_cert=None, cert_key=None, cert_cert=None,
+                 cert=(), verify=None,
                  timeout=None, headers=None, user_agent=None, pool_size=30,
                  username=None, password=None, token=None):
         super(AioClient, self).__init__(host=host, port=port, protocol=protocol,
-                                        ca_cert=ca_cert, cert_key=cert_key, cert_cert=cert_cert,
+                                        cert=cert, verify=verify,
                                         timeout=timeout, headers=headers, user_agent=user_agent, pool_size=pool_size,
                                         username=username, password=password, token=token)
+        self.ssl_context = None
         if self.cert:
-            ssl_context = ssl.SSLContext()
+            if verify is False:
+                cert_reqs = ssl.CERT_NONE
+                cafile = None
+            elif verify is True:
+                cert_reqs = ssl.CERT_REQUIRED
+                import certifi
+                cafile = certifi.where()
+            elif isinstance(verify, six.string_types):
+                cert_reqs = ssl.CERT_REQUIRED
+                cafile = verify
+            else:
+                raise TypeError("verify must be bool or string of the ca file path")
+            # the ssl problem is a pain in the ass, seems i can never get it right
+            # https://github.com/requests/requests/issues/1847
+            # https://stackoverflow.com/questions/44316292/ssl-sslerror-tlsv1-alert-protocol-version
+            self.ssl_context = ssl_context = ssl.SSLContext()
+            ssl_context.protocol = ssl.PROTOCOL_TLS
+            if not hasattr(ssl, 'PROTOCOL_TLSv1_1'): # should support TLSv1.2 to pass the test
+                warnings.warn(Etcd3Warning("the openssl version of your python is too old to support TLSv1.1+,"
+                                           "please upgrade you python"))
+            ssl_context.verify_mode = cert_reqs
+            ssl_context.load_verify_locations(cafile=cafile)
             ssl_context.load_cert_chain(*self.cert)
-            connector = aiohttp.TCPConnector(limit=pool_size, ssl=ssl_context)
-        else:
-            connector = aiohttp.TCPConnector(limit=pool_size)
+        connector = aiohttp.TCPConnector(limit=pool_size, ssl=self.ssl_context)
         self.session = aiohttp.ClientSession(connector=connector)
 
     async def close(self):

etcd3/baseclient.py

@@ -26,7 +26,7 @@
 
 rpc_swagger_json = os.path.join(os.path.dirname(__file__), 'rpc.swagger.json')
 
-swaggerSpec = SwaggerSpec(rpc_swagger_json) # TODO: swagger json may changed and need to implement multiple version
+swaggerSpec = SwaggerSpec(rpc_swagger_json)  # TODO: swagger json may changed and need to implement multiple version
 
 
 class BaseModelizedStreamResponse(object):  # pragma: no cover
@@ -48,25 +48,20 @@ def __iter__(self):
 
 class BaseClient(AuthAPI, ClusterAPI, KVAPI, LeaseAPI, MaintenanceAPI, WatchAPI, ExtraAPI):
     def __init__(self, host='localhost', port=2379, protocol='http',
-                 ca_cert=None, cert_key=None, cert_cert=None,
+                 cert=(), verify=None,
                  timeout=None, headers=None, user_agent=None, pool_size=30,
                  username=None, password=None, token=None):
         self.host = host
         self.port = port
         self.cert = None
-        self.ca_cert = ca_cert
-        self.cert_key = cert_key
-        self.cert_cert = cert_cert
-        if ca_cert or cert_key and cert_cert:
+        self.cert = cert
+        self.protocol = protocol
+        if cert:
             self.protocol = 'https'
-        if ca_cert:
-            self.cert = ca_cert
-        if cert_cert and cert_key:
-            self.cert = (cert_cert, cert_key)
+        self.verify = verify or False
         self.user_agent = user_agent
         if not user_agent:
             self.user_agent = 'etcd3-py/' + __version__
-        self.protocol = protocol
         self.timeout = timeout
         self.headers = headers or {}
         self.username = username
@@ -82,7 +77,7 @@ def _retrieve_version(self):  # pragma: no cover
         try:
             import requests
 
-            r = requests.get(self._url('/version'), timeout=0.3)  # 300ms will do
+            r = requests.get(self._url('/version'), cert=self.cert, verify=self.verify, timeout=0.3)  # 300ms will do
             r.raise_for_status()
             v = r.json()
             self.cluster_version = v["etcdcluster"]

etcd3/client.py

@@ -80,14 +80,16 @@ def iter_response(resp):
 
 class Client(BaseClient):
     def __init__(self, host='localhost', port=2379, protocol='http',
-                 ca_cert=None, cert_key=None, cert_cert=None,
+                 cert=(), verify=None,
                  timeout=None, headers=None, user_agent=None, pool_size=30,
                  username=None, password=None, token=None):
         super(Client, self).__init__(host=host, port=port, protocol=protocol,
-                                     ca_cert=ca_cert, cert_key=cert_key, cert_cert=cert_cert,
+                                     cert=cert, verify=verify,
                                      timeout=timeout, headers=headers, user_agent=user_agent, pool_size=pool_size,
                                      username=username, password=password, token=token)
         self._session = requests.session()
+        self._session.cert = self.cert
+        self._session.verify = self.verify
         self.__set_conn_pool(pool_size)
 
     def __set_conn_pool(self, pool_size):
@@ -170,7 +172,6 @@ def call_rpc(self, method, data=None, stream=False, encode=True, raw=False, **kw
         """
         data = data or {}
         kwargs.setdefault('timeout', self.timeout)
-        kwargs.setdefault('cert', self.cert)
         if self.token:
             kwargs.setdefault('headers', {}).setdefault('authorization', self.token)
         kwargs.setdefault('headers', {}).setdefault('user_agent', self.user_agent)

etcd3/utils.py

@@ -2,6 +2,8 @@
 import functools
 import itertools
 import logging
+import os
+import sys
 import time
 import warnings
 from collections import namedtuple, OrderedDict, Hashable
@@ -331,3 +333,45 @@ class Etcd3Warning(UserWarning):
 
 
 warnings.simplefilter('always', Etcd3Warning)
+
+
+# https://gist.github.com/4368898
+# Public domain code by anatoly techtonik <techtonik@gmail.com>
+# AKA Linux `which` and Windows `where`
+def find_executable(executable, path=None):  # pragma: no cover
+    """Find if 'executable' can be run. Looks for it in 'path'
+    (string that lists directories separated by 'os.pathsep';
+    defaults to os.environ['PATH']). Checks for all executable
+    extensions. Returns full path or None if no command is found.
+    """
+    if path is None:
+        path = os.environ['PATH']
+    paths = path.split(os.pathsep)
+    extlist = ['']
+    if os.name == 'os2':
+        (base, ext) = os.path.splitext(executable)
+        # executable files on OS/2 can have an arbitrary extension, but
+        # .exe is automatically appended if no dot is present in the name
+        if not ext:
+            executable = executable + ".exe"
+    elif sys.platform == 'win32':
+        pathext = os.environ['PATHEXT'].lower().split(os.pathsep)
+        (base, ext) = os.path.splitext(executable)
+        if ext.lower() not in pathext:
+            extlist = pathext
+    for ext in extlist:
+        execname = executable + ext
+        if os.path.isfile(execname):
+            return execname
+        else:
+            for p in paths:
+                f = os.path.join(p, execname)
+                if os.path.isfile(f):
+                    return f
+        # violent fix of my wired test environment
+        for p in ['/usr/local/bin']:
+            f = os.path.join(p, execname)
+            if os.path.isfile(f):
+                return f
+    else:
+        return None

requirements.txt

@@ -2,3 +2,7 @@ enum34>=1.1.6
 six>=1.11.0
 requests>=2.10.0
 semantic-version>=2.6.0
+# requests[security]
+pyOpenSSL>=0.14
+cryptography>=1.3.4
+idna>=2.0.0

setup.py

@@ -61,7 +61,7 @@
     include_package_data=True,
     install_requires=requirements,
     license="Apache Software License 2.0",
-    python_requires='>=2.7',
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*',
     zip_safe=False,
     keywords='etcd3',
     classifiers=[

tests/certs/ca-config.json

@@ -0,0 +1,27 @@
+{
+  "signing": {
+    "default": {
+      "expiry": "168h"
+    },
+    "profiles": {
+      "server": {
+        "expiry": "87600h",
+        "usages": [
+          "signing",
+          "key encipherment",
+          "server auth",
+          "client auth"
+        ]
+      },
+      "client": {
+        "expiry": "87600h",
+        "usages": [
+          "signing",
+          "key encipherment",
+          "client auth",
+          "client auth"
+        ]
+      }
+    }
+  }
+}

tests/certs/ca-csr.json

@@ -0,0 +1,14 @@
+{
+  "CN": "etcd3-py-test",
+  "key": {
+    "algo": "ecdsa",
+    "size": 256
+  },
+  "names": [
+    {
+      "C": "US",
+      "ST": "CA",
+      "L": "San Francisco"
+    }
+  ]
+}

tests/certs/ca-key.pem

@@ -0,0 +1,5 @@
+-----BEGIN EC PRIVATE KEY-----
+MHcCAQEEINxvluG24yYuCbjVc3h8y8hObhDf0HsR7OB1OPaO03lOoAoGCCqGSM49
+AwEHoUQDQgAE+R44Lr0jJoA4CXTWAl4M+Id8cSFw9Fl2SKkDInoumJcekqKXHv5Z
+Kl93DzHzNLN4vw54ZMMZ+lbO0/JRkmZNUA==
+-----END EC PRIVATE KEY-----

tests/certs/ca.csr

@@ -0,0 +1,8 @@
+-----BEGIN CERTIFICATE REQUEST-----
+MIIBBjCBrAIBADBKMQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcT
+DVNhbiBGcmFuY2lzY28xFjAUBgNVBAMTDWV0Y2QzLXB5LXRlc3QwWTATBgcqhkjO
+PQIBBggqhkjOPQMBBwNCAAT5HjguvSMmgDgJdNYCXgz4h3xxIXD0WXZIqQMiei6Y
+lx6Sopce/lkqX3cPMfM0s3i/Dnhkwxn6Vs7T8lGSZk1QoAAwCgYIKoZIzj0EAwID
+SQAwRgIhAJnz4iWOi69pSTd/ebKwq5CYOlaUMiD5pbd1Bl+M6hm9AiEAqJOD2nTF
+/+OSZ1J4B7gdXh3rolTyHGxatmdAR7hfwqk=
+-----END CERTIFICATE REQUEST-----

tests/certs/ca.pem

@@ -0,0 +1,12 @@
+-----BEGIN CERTIFICATE-----
+MIIB2DCCAX6gAwIBAgIUf6IKrGmwVXKfoDKLynTSa00PoLMwCgYIKoZIzj0EAwIw
+SjELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMRYwFAYDVQQHEw1TYW4gRnJhbmNp
+c2NvMRYwFAYDVQQDEw1ldGNkMy1weS10ZXN0MB4XDTE4MDMyNjEwMjgwMFoXDTIz
+MDMyNTEwMjgwMFowSjELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMRYwFAYDVQQDEw1ldGNkMy1weS10ZXN0MFkwEwYHKoZI
+zj0CAQYIKoZIzj0DAQcDQgAE+R44Lr0jJoA4CXTWAl4M+Id8cSFw9Fl2SKkDInou
+mJcekqKXHv5ZKl93DzHzNLN4vw54ZMMZ+lbO0/JRkmZNUKNCMEAwDgYDVR0PAQH/
+BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFN3xuRO9hnv8JVBH/vSA
+ySn+hPUQMAoGCCqGSM49BAMCA0gAMEUCIHg2Fk3X64rQ3Y1L9BvFbkOTf/8BfPsU
+MgylrVZmhtmiAiEAteir/Us3nRmlMVP7wdROh5xwqZIjkQWo3thCeUY/kWo=
+-----END CERTIFICATE-----

tests/certs/client-key.pem

@@ -0,0 +1,5 @@
+-----BEGIN EC PRIVATE KEY-----
+MHcCAQEEIOkM48vHZrZ0ENUHt5bbutzYwHz4JpobGegBxwmYyAlyoAoGCCqGSM49
+AwEHoUQDQgAEjUw3P/EYMUJplfVBfVXyfbHeB11acqBI/0Y2/ERJE7Lk/o3wk2Oa
+t3qd/YIx61l/wncFdwt+TAGQh8tpTrXtXA==
+-----END EC PRIVATE KEY-----

tests/certs/client.csr

@@ -0,0 +1,9 @@
+-----BEGIN CERTIFICATE REQUEST-----
+MIIBIzCBygIBADBKMQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcT
+DVNhbiBGcmFuY2lzY28xFjAUBgNVBAMTDWV0Y2QzLXB5LXRlc3QwWTATBgcqhkjO
+PQIBBggqhkjOPQMBBwNCAASNTDc/8RgxQmmV9UF9VfJ9sd4HXVpyoEj/Rjb8REkT
+suT+jfCTY5q3ep39gjHrWX/CdwV3C35MAZCHy2lOte1coB4wHAYJKoZIhvcNAQkO
+MQ8wDTALBgNVHREEBDACggAwCgYIKoZIzj0EAwIDSAAwRQIhAMkc4OYcZGGBXILj
+Ddan5vCYhfcUhHu09cHLGBJbIqS9AiBzt3X1Nh5l0JR3tPP6wc0H0F/q/l6ILFYz
+7GtcHBaWtw==
+-----END CERTIFICATE REQUEST-----

tests/certs/client.json

@@ -0,0 +1,18 @@
+{
+  "CN": "etcd3-py-test",
+  "hosts": [
+    ""
+  ],
+  "key": {
+    "algo": "ecdsa",
+    "size": 256
+  },
+  "names": [
+    {
+      "C": "US",
+      "ST": "CA",
+      "L": "San Francisco"
+    }
+  ]
+}
+

tests/certs/client.pem

@@ -0,0 +1,13 @@
+-----BEGIN CERTIFICATE-----
+MIICATCCAaegAwIBAgIUT9DhId/TTY9mJ8xqc8D2zGyBUxkwCgYIKoZIzj0EAwIw
+SjELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMRYwFAYDVQQHEw1TYW4gRnJhbmNp
+c2NvMRYwFAYDVQQDEw1ldGNkMy1weS10ZXN0MB4XDTE4MDMyNjEwMzAwMFoXDTI4
+MDMyMzEwMzAwMFowSjELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMRYwFAYDVQQDEw1ldGNkMy1weS10ZXN0MFkwEwYHKoZI
+zj0CAQYIKoZIzj0DAQcDQgAEjUw3P/EYMUJplfVBfVXyfbHeB11acqBI/0Y2/ERJ
+E7Lk/o3wk2Oat3qd/YIx61l/wncFdwt+TAGQh8tpTrXtXKNrMGkwDgYDVR0PAQH/
+BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDAjAMBgNVHRMBAf8E
+AjAAMB0GA1UdDgQWBBTv+TSBbN3XHWGFtTesLhs0R2/ZizALBgNVHREEBDACggAw
+CgYIKoZIzj0EAwIDSAAwRQIhAII0p+8yyrnvbIn929Dvo+8Ielei4NWqGKykhBHq
+mOjpAiAswNiZAnpNsxQUnPQsTj8PQsbN5s7Y6DyFl+vmGSJt0w==
+-----END CERTIFICATE-----

tests/certs/server-key.pem

@@ -0,0 +1,5 @@
+-----BEGIN EC PRIVATE KEY-----
+MHcCAQEEINEBhV5Y8peIzddulQ/KBusOkpBRjV/6QUM9h4kC9+MyoAoGCCqGSM49
+AwEHoUQDQgAEQMjDppBcZaM1kajP9vYOKeCXf4vaYUMkG+Zl46QdbH7l2G6A+3hU
+iz66ofCJ/k0Kd8kTiYliSmSXUuV0ZlsMyg==
+-----END EC PRIVATE KEY-----

tests/certs/server.csr

@@ -0,0 +1,9 @@
+-----BEGIN CERTIFICATE REQUEST-----
+MIIBNDCB2wIBADBGMQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcT
+DVNhbiBGcmFuY2lzY28xEjAQBgNVBAMTCWxvY2FsaG9zdDBZMBMGByqGSM49AgEG
+CCqGSM49AwEHA0IABEDIw6aQXGWjNZGoz/b2Dingl3+L2mFDJBvmZeOkHWx+5dhu
+gPt4VIs+uqHwif5NCnfJE4mJYkpkl1LldGZbDMqgMzAxBgkqhkiG9w0BCQ4xJDAi
+MCAGA1UdEQQZMBeCCWxvY2FsaG9zdIcEfwAAAYcEAAAAADAKBggqhkjOPQQDAgNI
+ADBFAiEA7+bfM3BCqGElzs0TNT7t2kY+x1LXidGJMx/rYl3+kFMCICRcs/wr72qZ
+KevM2sYL4NPxpLpI7w4Oms7eMH1Qcm3X
+-----END CERTIFICATE REQUEST-----