Merge pull request #30 from Revolution1/test-etcd-docker-deploy

support etcd v3.2.2+, deploy test etcd docker container automatically…

README.md

@@ -13,7 +13,7 @@ Python client for etcd v3 (Using gRPC-JSON-Gateway)
 * Free software: Apache Software License 2.0
 * Source Code: https://github.com/Revolution1/etcd3-py
 * Documentation: https://etcd3-py.readthedocs.io.
-* etcd version required: v3.3.0+
+* etcd version required: v3.2.2+
 
 Notice: The authentication header through gRPC-JSON-Gateway only supported in [etcd v3.3.0+](https://github.com/coreos/etcd/pull/7999)
 

etcd3/client.py

@@ -70,6 +70,7 @@ def iter_response(resp):
     # https://github.com/coreos/etcd/blob/master/etcdserver/api/v3rpc/maintenance.go#L98
     # 2**15 < ceil(32*1024/3*4) < 2**16 = 65536
     for chunk in resp.iter_content(chunk_size=65536):
+        chunk = chunk.strip()
         chunk = left_chunk + chunk
         for ok, s, _ in iter_json_string(chunk, resp=resp, err_cls=Etcd3StreamError):
             if ok:

etcd3/utils.py

@@ -1,10 +1,12 @@
+import itertools
+import sys
+import time
+
 import enum
 import functools
-import itertools
 import logging
 import os
-import sys
-import time
+import shlex
 import warnings
 from collections import namedtuple, OrderedDict, Hashable
 from subprocess import Popen, PIPE
@@ -320,7 +322,13 @@ def retry(func, max_tries=3, log=logging, err_cls=Exception):  # pragma: no cove
 
 
 def exec_cmd(cmd, envs=None, raise_error=True):  # pragma: no cover
+    if isinstance(cmd, str):
+        cmd = shlex.split(cmd)
     envs = envs or {}
+    cmd = cmd[:]
+    exe = find_executable(cmd[0])
+    if exe:
+        cmd[0] = exe
     p = Popen(cmd, stdout=PIPE, stderr=PIPE, env=envs)
     out, err = p.communicate()
     if p.returncode != 0 and raise_error:

setup.py

@@ -5,7 +5,10 @@
 import os
 import platform
 
-from pip.req import parse_requirements
+try: # for pip >= 10
+    from pip._internal.req import parse_requirements
+except ImportError: # for pip <= 9.0.3
+    from pip.req import parse_requirements
 from setuptools import setup, find_packages
 
 PY2 = platform.python_version_tuple()[0] == '2'

tests/docker_cli.py

@@ -1,7 +1,11 @@
+import time
+
+import json
 import os
 import shlex
+import six
 
-from etcd3.utils import exec_cmd, find_executable
+from etcd3.utils import find_executable, exec_cmd
 from .envs import ETCD_IMG
 
 DOCKER_PATH = find_executable('docker')
@@ -23,20 +27,64 @@ def docker(*args, **kwargs):  # pragma: no cover
     envs = os.environ
     cmd = [DOCKER_PATH]
     cmd.extend(args)
+    # return subprocess.check_call(cmd, env=envs)
     return exec_cmd(cmd, envs, raise_error)
 
 
 # get the etcd server's http port and tcp peer port
 def get_h_t(n):  # pragma: no cover
-    h = 2380 + n  # the http port
+    h = 2379 + 2 * (n - 1)  # the http port
     t = h + 1  # the tcp peer port
     return h, t
 
 
+def docker_run_etcd_main():  # pragma: no cover
+    if NO_DOCKER_SERVICE:
+        return None, 2379, None
+    n = 1
+    h, t = get_h_t(n)  # 2379, 2380
+    name = 'etcd3_1'
+    try:
+        out = docker('inspect %s' % name)
+        if isinstance(out, six.binary_type):
+            out = six.text_type(out, encoding='utf-8')
+        spec = json.loads(out)
+        if isinstance(spec, list):
+            spec = spec[0]
+        image = spec.get('Config', {}).get('Image')
+        if image != ETCD_IMG:
+            if spec.get('Config', {}).get('Labels', {}).get('etcd3.py.test') == 'main':
+                print(docker('rm -f %s' % name))
+            raise RuntimeError
+        cmds = spec.get('Config', {}).get('Cmd', [])
+        for i, c in enumerate(cmds):
+            if c == '--listen-client-urls':
+                h = int(cmds[i + 1].split(':')[-1])
+            if c == '--listen-peer-urls':
+                t = int(cmds[i + 1].split(':')[-1])
+        return '', h, t
+    except:
+        cmd = 'run -d -p {h}:{h} -p {t}:{t} --name {name} ' \
+              '-l etcd3.py.test=main ' \
+              '{img} ' \
+              'etcd --name node{n} ' \
+              '--initial-advertise-peer-urls http://0.0.0.0:{t} ' \
+              '--listen-peer-urls http://0.0.0.0:{t} ' \
+              '--advertise-client-urls http://0.0.0.0:{h} ' \
+              '--listen-client-urls http://0.0.0.0:{h} ' \
+              '--initial-cluster node{n}=http://0.0.0.0:{t}'.format(name=name, n=n, h=h, t=t, img=ETCD_IMG)
+        print(cmd)
+        out = docker(cmd)
+        print(out)
+        time.sleep(5)
+        return out, h, t
+
+
 def docker_run_etcd(n=2):  # pragma: no cover
     n = n or 2  # the node sequence
     h, t = get_h_t(n)
     cmd = 'run -d -p {h}:{h} -p {t}:{t} --name etcd3_{n} ' \
+          '-l etcd3.py.test=node{n} ' \
           '{img} ' \
           'etcd --name node{n} ' \
           '--initial-advertise-peer-urls http://0.0.0.0:{t} ' \
@@ -62,6 +110,7 @@ def docker_run_etcd_ssl():  # pragma: no cover
     n = 10  # the node sequence
     h, t = get_h_t(n)
     cmd = 'run -d -p {h}:{h} -p {t}:{t} --name etcd3_ssl -v {certs_dir}:/certs ' \
+          '-l etcd3.py.test=node{n} ' \
           '{img} ' \
           'etcd --name node_ssl ' \
           '--client-cert-auth ' \
@@ -82,7 +131,7 @@ def docker_run_etcd_ssl():  # pragma: no cover
     return docker(cmd), h, t
 
 
-def docker_rm_etcd_ssl(raise_error=False): # pragma: no cover
+def docker_rm_etcd_ssl(raise_error=False):  # pragma: no cover
     cmd = 'rm -f etcd3_ssl'
     return docker(cmd, raise_error=raise_error)
 

tests/etcd_go_cli.py

@@ -1,6 +1,7 @@
 import shlex
 
 from etcd3.utils import exec_cmd, find_executable
+from tests.docker_cli import NO_DOCKER_SERVICE
 from .envs import ETCD_ENDPOINT
 
 ETCDCTL_PATH = find_executable('etcdctl')
@@ -25,6 +26,8 @@ def etcdctl(*args, **kwargs):  # pragma: no cover
 
 
 NO_ETCD_SERVICE = True
+if not NO_DOCKER_SERVICE:
+    NO_ETCD_SERVICE = False
 try:  # pragma: no cover
     if ETCDCTL_PATH and etcdctl('--dial-timeout=0.2s endpoint health'):
         NO_ETCD_SERVICE = False

tests/test_auth_apis.py

@@ -7,7 +7,8 @@
 from etcd3.errors import ErrUserNotFound
 from etcd3.models import authpbPermissionType
 from etcd3.utils import incr_last_byte
-from .envs import protocol, host, port
+from tests.docker_cli import docker_run_etcd_main
+from .envs import protocol, host
 from .etcd_go_cli import etcdctl, NO_ETCD_SERVICE
 
 
@@ -16,21 +17,22 @@ def client():
     """
     init Etcd3Client, close its connection-pool when teardown
     """
-    c = Client(host, port, protocol)
+    _, p, _ = docker_run_etcd_main()
+    c = Client(host, p, protocol)
     yield c
     c.close()
 
 
-def teardown_auth():  # pragma: no cover
+def teardown_auth(client):  # pragma: no cover
     """
     disable auth, delete all users and roles
     """
-    etcdctl('--user root:root auth disable', raise_error=False)
-    etcdctl('--user root:changed auth disable', raise_error=False)
-    for i in (etcdctl('role list', raise_error=False) or '').splitlines():
-        etcdctl('role', 'delete', i)
-    for i in (etcdctl('user list', raise_error=False) or '').splitlines():
-        etcdctl('user', 'delete', i)
+    etcdctl('--user root:root auth disable', raise_error=False, endpoint=client.baseurl)
+    etcdctl('--user root:changed auth disable', raise_error=False, endpoint=client.baseurl)
+    for i in (etcdctl('role list', raise_error=False, endpoint=client.baseurl) or '').splitlines():
+        etcdctl('role', 'delete', i, endpoint=client.baseurl)
+    for i in (etcdctl('user list', raise_error=False, endpoint=client.baseurl) or '').splitlines():
+        etcdctl('user', 'delete', i, endpoint=client.baseurl)
 
 
 def enable_auth():  # pragma: no cover
@@ -42,8 +44,8 @@ def enable_auth():  # pragma: no cover
 
 @pytest.mark.skipif(NO_ETCD_SERVICE, reason="no etcd service available")
 def test_auth_flow(client, request):
-    teardown_auth()
-    request.addfinalizer(teardown_auth)
+    teardown_auth(client)
+    request.addfinalizer(lambda:teardown_auth(client))
 
     # test error
     with pytest.raises(ErrRootUserNotExist):

tests/test_client.py

@@ -4,9 +4,9 @@
 
 from etcd3.client import Client
 from etcd3.errors import Etcd3Exception
-from .docker_cli import CA_PATH, CERT_PATH, KEY_PATH, NO_DOCKER_SERVICE
+from .docker_cli import CA_PATH, CERT_PATH, KEY_PATH, NO_DOCKER_SERVICE, docker_run_etcd_main
 from .docker_cli import docker_run_etcd_ssl, docker_rm_etcd_ssl
-from .envs import protocol, host, port
+from .envs import protocol, host
 from .etcd_go_cli import etcdctl, NO_ETCD_SERVICE
 from .mocks import fake_request
 
@@ -16,7 +16,8 @@ def client():
     """
     init Etcd3Client, close its connection-pool when teardown
     """
-    c = Client(host, port, protocol)
+    _, p, _ = docker_run_etcd_main()
+    c = Client(host, p, protocol)
     yield c
     c.close()
 

tests/test_extra_apis.py

@@ -1,7 +1,8 @@
 import pytest
 
 from etcd3.client import Client
-from .envs import protocol, host, port
+from tests.docker_cli import docker_run_etcd_main
+from .envs import protocol, host
 from .mocks import fake_request
 
 
@@ -10,7 +11,8 @@ def client():
     """
     init Etcd3Client, close its connection-pool when teardown
     """
-    c = Client(host, port, protocol)
+    _, p, _ = docker_run_etcd_main()
+    c = Client(host, p, protocol)
     yield c
     c.close()
 

tests/test_kv_apis.py

@@ -1,12 +1,12 @@
 import json
-
 import pytest
 import six
 
 from etcd3.client import Client
 from etcd3.models import RangeRequestSortOrder
 from etcd3.models import RangeRequestSortTarget
-from .envs import protocol, host, port
+from tests.docker_cli import docker_run_etcd_main
+from .envs import protocol, host
 from .etcd_go_cli import NO_ETCD_SERVICE
 from .etcd_go_cli import etcdctl
 
@@ -16,7 +16,8 @@ def client():
     """
     init Etcd3Client, close its connection-pool when teardown
     """
-    c = Client(host, port, protocol)
+    _, p, _ = docker_run_etcd_main()
+    c = Client(host, p, protocol)
     yield c
     c.close()
 

tests/test_lease_apis.py

@@ -4,6 +4,7 @@
 import pytest
 
 from etcd3.client import Client
+from tests.docker_cli import docker_run_etcd_main
 from .envs import protocol, host, port
 from .etcd_go_cli import NO_ETCD_SERVICE, etcdctl
 
@@ -13,7 +14,8 @@ def client():
     """
     init Etcd3Client, close its connection-pool when teardown
     """
-    c = Client(host, port, protocol)
+    _, p, _ = docker_run_etcd_main()
+    c = Client(host, p, protocol)
     yield c
     c.close()
 

tests/test_lease_util.py

@@ -1,11 +1,12 @@
-import random
 import time
 
 import mock
 import pytest
+import random
 
 from etcd3.client import Client
-from .envs import protocol, host, port
+from tests.docker_cli import docker_run_etcd_main
+from .envs import protocol, host
 from .etcd_go_cli import NO_ETCD_SERVICE, etcdctl
 
 
@@ -14,7 +15,8 @@ def client():
     """
     init Etcd3Client, close its connection-pool when teardown
     """
-    c = Client(host, port, protocol)
+    _, p, _ = docker_run_etcd_main()
+    c = Client(host, p, protocol)
     yield c
     c.close()
 

tests/test_lock_util.py

@@ -148,6 +148,7 @@ def hold(lock, name, ctx):
     assert l2.holders() == 0
 
 
+@pytest.mark.timeout(60)
 @pytest.mark.skipif(NO_ETCD_SERVICE, reason="no etcd service available")
 def test_reentrant_lock(client):
     clear()

tests/test_maintenance_apis.py

@@ -1,11 +1,11 @@
 import json
-
 import pytest
 import six
 
 from etcd3 import Client
 from etcd3.models import etcdserverpbAlarmType
-from .envs import protocol, host, port
+from tests.docker_cli import docker_run_etcd_main
+from .envs import protocol, host
 from .etcd_go_cli import NO_ETCD_SERVICE, etcdctl
 
 
@@ -14,7 +14,8 @@ def client():
     """
     init Etcd3Client, close its connection-pool when teardown
     """
-    c = Client(host, port, protocol)
+    _, p, _ = docker_run_etcd_main()
+    c = Client(host, p, protocol)
     yield c
     c.close()
 

tests/test_transaction_util.py

@@ -1,7 +1,8 @@
 import pytest
 
 from etcd3 import Client
-from .envs import protocol, host, port
+from tests.docker_cli import docker_run_etcd_main
+from .envs import protocol, host
 from .etcd_go_cli import etcdctl, NO_ETCD_SERVICE
 
 
@@ -10,11 +11,13 @@ def client():
     """
     init Etcd3Client, close its connection-pool when teardown
     """
-    c = Client(host, port, protocol)
+    _, p, _ = docker_run_etcd_main()
+    c = Client(host, p, protocol)
     yield c
     c.close()
 
 
+@pytest.mark.timeout(60)
 @pytest.mark.skipif(NO_ETCD_SERVICE, reason="no etcd service available")
 def test_transaction(client):
     etcdctl('put foo bar')