Fix docker test infrastructure to not conflict (#182)

.travis.yml

@@ -22,8 +22,8 @@ install:
   - pip install coverage==4.0.3 pytest-cov
   - sleep 5
 script:
-  - pytest -s --cov=guillotina -v --cov-report term-missing guillotina
-  - USE_COCKROACH=true pytest -s -v guillotina
+  - pytest -s --cov=guillotina -s --tb=native -v --cov-report term-missing guillotina
+  - USE_COCKROACH=true pytest -s --tb=native -v guillotina
   - flake8 guillotina --config=setup.cfg
 after_success:
   - pip install coveralls

CHANGELOG.rst

@@ -1,14 +1,16 @@
 2.1.9 (unreleased)
 ------------------
 
-- Nothing changed yet.
+- Fix dockers test infrastructure to not conflict with multiple tests running
+  at the same time.
+  [vangheem]
 
 
 2.1.8 (2017-11-21)
 ------------------
 
 - Adding X-Forwarded-Proto in order to allow https rewrite of absolute url
-  [ramon] 
+  [ramon]
 
 - Adding PROPFIND HTTP Verb
   [ramon]

guillotina/db/strategies/resolve.py

@@ -29,7 +29,7 @@ async def tpc_vote(self):
                     modified_keys = [k for k in self._transaction.modified.keys()]
                     logger.warn(
                         f'Could not resolve conflicts in TID: {self._transaction._tid}\n'
-                        f'Conflicted TID: {current_tid}\n',
+                        f'Conflicted TID: {current_tid}\n'
                         f'IDs: {modified_keys}'
                     )
                     return False

guillotina/tests/docker_containers/base.py

@@ -4,78 +4,91 @@
 import os
 
 
-COCKROACH_IMAGE = 'cockroachdb/cockroach:v1.0'
-
-
 class BaseImage:
 
     docker_version = '1.23'
-    label = 'foobar'
+    name = 'foobar'
     image = None
-    to_port = from_port = None
-    image_options = dict(
+    port = None
+    host = ''
+    base_image_options = dict(
         cap_add=['IPC_LOCK'],
         mem_limit='1g',
         environment={},
-        privileged=True)
+        privileged=True,
+        detach=True,
+        publish_all_ports=True)
+
+    def get_image_options(self):
+        image_options = self.base_image_options.copy()
+        return image_options
+
+    def get_port(self):
+        if os.environ.get('TESTING', '') == 'jenkins' or 'TRAVIS' in os.environ:
+            return self.port
+        for port in self.container_obj.attrs['NetworkSettings']['Ports'].keys():
+            if port == '6543/tcp':
+                continue
+            return self.container_obj.attrs['NetworkSettings']['Ports'][port][0]['HostPort']
+
+    def get_host(self):
+        return self.container_obj.attrs['NetworkSettings']['IPAddress']
 
     def check(self, host):
         return True
 
     def run(self):
         docker_client = docker.from_env(version=self.docker_version)
 
-        # Clean up possible other docker containers
-        test_containers = docker_client.containers.list(
-            all=True,
-            filters={'label': self.label})
-        for test_container in test_containers:
-            test_container.stop()
-            test_container.remove(v=True, force=True)
-
         # Create a new one
         container = docker_client.containers.run(
             image=self.image,
-            labels=[self.label],
-            detach=True,
-            ports={
-                f'{self.to_port}/tcp': self.from_port
-            },
-            **self.image_options
+            **self.get_image_options()
         )
         ident = container.id
         count = 1
 
-        container_obj = docker_client.containers.get(ident)
+        self.container_obj = docker_client.containers.get(ident)
 
         opened = False
-        host = ''
 
-        print(f'starting {self.label}')
+        print(f'starting {self.name}')
         while count < 30 and not opened:
+            if count > 0:
+                sleep(1)
             count += 1
             try:
-                container_obj = docker_client.containers.get(ident)
+                self.container_obj = docker_client.containers.get(ident)
             except docker.errors.NotFound:
+                print(f'Container not found for {self.name}')
                 continue
-            sleep(1)
-            if container_obj.attrs['NetworkSettings']['IPAddress'] != '':
+            if self.container_obj.status == 'exited':
+                logs = self.container_obj.logs()
+                self.stop()
+                raise Exception(f'Container failed to start {logs}')
+
+            if self.container_obj.attrs['NetworkSettings']['IPAddress'] != '':
                 if os.environ.get('TESTING', '') == 'jenkins':
-                    host = container_obj.attrs['NetworkSettings']['IPAddress']
+                    self.host = self.container_obj.attrs['NetworkSettings']['IPAddress']
                 else:
-                    host = 'localhost'
+                    self.host = 'localhost'
 
-            if host != '':
-                opened = self.check(host)
-        print(f'{self.label} started')
-        return host
+            if self.host != '':
+                opened = self.check()
+        if not opened:
+            logs = self.container_obj.logs()
+            self.stop()
+            raise Exception(f'Could not start {self.name}: {logs}')
+        print(f'{self.name} started')
+        return self.host, self.get_port()
 
     def stop(self):
-        docker_client = docker.from_env(version=self.docker_version)
-        # Clean up possible other docker containers
-        test_containers = docker_client.containers.list(
-            all=True,
-            filters={'label': self.label})
-        for test_container in test_containers:
-            test_container.kill()
-            test_container.remove(v=True, force=True)
+        if self.container_obj is not None:
+            try:
+                self.container_obj.kill()
+            except docker.errors.APIError:
+                pass
+            try:
+                self.container_obj.remove(v=True, force=True)
+            except docker.errors.APIError:
+                pass

guillotina/tests/docker_containers/cockroach.py

@@ -4,20 +4,28 @@
 
 
 class CockroachDB(BaseImage):
-    label = 'cockroach'
+    name = 'cockroach'
     image = 'cockroachdb/cockroach:v1.0'
-    to_port = from_port = 26257
-    image_options = BaseImage.image_options.copy()
-    image_options.update(dict(
-        command=' '.join([
-            'start --insecure',
-        ])
-    ))
+    port = 26257
 
-    def check(self, host):
+    def get_image_options(self):
+        image_options = super().get_image_options()
+        image_options.update(dict(
+            command=' '.join([
+                'start --insecure',
+            ]),
+            publish_all_ports=False,
+            ports={
+                f'26257/tcp': '26257'
+            }
+        ))
+        return image_options
+
+    def check(self):
         conn = cur = None
         try:
-            conn = psycopg2.connect("dbname=guillotina user=root host=%s port=26257" % host)  # noqa
+            conn = psycopg2.connect(
+                f"dbname=guillotina user=root host={self.host} port={self.get_port()}")
             conn.set_session(autocommit=True)
             cur = conn.cursor()
             cur.execute('SHOW DATABASES;')

guillotina/tests/docker_containers/etcd.py

@@ -7,28 +7,30 @@
 
 
 class ETCD(BaseImage):
-    label = 'etcd'
+    name = 'etcd'
     image = 'quay.io/coreos/etcd:v3.2.0-rc.0'
-    to_port = from_port = 2379
-    image_options = BaseImage.image_options.copy()
-
-    image_options.update(dict(
-        mem_limit='200m',
-        name=image_name,
-        command=' '.join([
-            '/usr/local/bin/etcd',
-            '--name {}'.format(image_name),
-            '--data-dir /etcd-data',
-            '--listen-client-urls http://0.0.0.0:2379',
-            '--advertise-client-urls http://0.0.0.0:2379',
-            '--listen-peer-urls http://0.0.0.0:2380',
-            '--initial-advertise-peer-urls http://0.0.0.0:2380',
-            '--initial-cluster {}=http://0.0.0.0:2380'.format(image_name),
-            '--initial-cluster-token my-etcd-token',
-            '--initial-cluster-state new',
-            '--auto-compaction-retention 1'
-        ])
-    ))
+    port = 2379
+
+    def get_image_options(self):
+        image_options = super().get_image_options()
+        image_options.update(dict(
+            mem_limit='200m',
+            name=image_name,
+            command=' '.join([
+                '/usr/local/bin/etcd',
+                '--name {}'.format(image_name),
+                '--data-dir /etcd-data',
+                '--listen-client-urls http://0.0.0.0:2379',
+                '--advertise-client-urls http://0.0.0.0:2379',
+                '--listen-peer-urls http://0.0.0.0:2380',
+                '--initial-advertise-peer-urls http://0.0.0.0:2380',
+                '--initial-cluster {}=http://0.0.0.0:2380'.format(image_name),
+                '--initial-cluster-token my-etcd-token',
+                '--initial-cluster-state new',
+                '--auto-compaction-retention 1'
+            ])
+        ))
+        return image_options
 
 
 image = ETCD()

guillotina/tests/docker_containers/postgres.py

@@ -4,21 +4,25 @@
 
 
 class Postgresql(BaseImage):
-    label = 'postgresql'
+    name = 'postgresql'
     image = 'postgres:9.6'
-    to_port = from_port = 5432
-    image_options = BaseImage.image_options.copy()
-    image_options.update(dict(
-        environment={
-            'POSTGRES_PASSWORD': '',
-            'POSTGRES_DB': 'guillotina',
-            'POSTGRES_USER': 'postgres'
-        }
-    ))
+    port = 5432
 
-    def check(self, host):
+    def get_image_options(self):
+        image_options = super().get_image_options()
+        image_options.update(dict(
+            environment={
+                'POSTGRES_PASSWORD': '',
+                'POSTGRES_DB': 'guillotina',
+                'POSTGRES_USER': 'postgres'
+            }
+        ))
+        return image_options
+
+    def check(self):
         try:
-            conn = psycopg2.connect("dbname=guillotina user=postgres host=%s port=5432" % host)  # noqa
+            conn = psycopg2.connect(
+                f"dbname=guillotina user=postgres host={self.host} port={self.get_port()}")
             cur = conn.cursor()
             cur.execute("SELECT 1;")
             cur.fetchone()

guillotina/tests/fixtures.py

@@ -51,14 +51,13 @@ def get_pg_settings():
         'dbname': 'guillotina',
         'user': 'postgres',
         'host': getattr(get_pg_settings, 'host', 'localhost'),
+        'port': getattr(get_pg_settings, 'port', 5432),
         'password': '',
-        'port': 5432
     }
     if USE_COCKROACH:
         settings['databases'][0]['db']['storage'] = 'cockroach'
         settings['databases'][0]['db']['dsn'].update({
-            'user': 'root',
-            'port': 26257
+            'user': 'root'
         })
     return settings
 
@@ -70,17 +69,19 @@ def postgres():
     """
 
     if USE_COCKROACH:
-        host = containers.cockroach_image.run()
+        host, port = containers.cockroach_image.run()
     else:
         if not IS_TRAVIS:
-            host = containers.postgres_image.run()
+            host, port = containers.postgres_image.run()
         else:
             host = 'localhost'
+            port = 5432
 
     # mark the function with the actual host
     setattr(get_pg_settings, 'host', host)
+    setattr(get_pg_settings, 'port', port)
 
-    yield host  # provide the fixture value
+    yield host, port  # provide the fixture value
 
     if USE_COCKROACH:
         containers.cockroach_image.stop()
@@ -223,13 +224,17 @@ async def container_requester(guillotina):
 
 
 class CockroachStorageAsyncContextManager(object):
-    def __init__(self, request, loop):
+    def __init__(self, request, loop, postgres):
         self.loop = loop
         self.request = request
         self.storage = None
+        self.postgres = postgres
 
     async def __aenter__(self):
-        dsn = "postgres://root:@localhost:26257/guillotina?sslmode=disable"
+        dsn = "postgres://root:@{}:{}/guillotina?sslmode=disable".format(
+            self.postgres[0],
+            self.postgres[1]
+        )
         self.storage = CockroachStorage(
             dsn=dsn, name='db', pool_size=25,
             conn_acquire_timeout=0.1)
@@ -245,4 +250,4 @@ async def __aexit__(self, exc_type, exc, tb):
 
 @pytest.fixture(scope='function')
 async def cockroach_storage(postgres, dummy_request, loop):
-    return CockroachStorageAsyncContextManager(dummy_request, loop)
+    return CockroachStorageAsyncContextManager(dummy_request, loop, postgres)