From debf10420af65740c8e7d2b2270e08578b5d5fc5 Mon Sep 17 00:00:00 2001 From: Nathan Van Gheem Date: Mon, 22 May 2017 09:30:23 -0500 Subject: [PATCH] Fix cockroach integration and provide travis support for cockroach (#85) Fix cockroach integration and provide travis support for cockroach --- .coveragerc | 1 + .travis.yml | 1 + CHANGELOG.rst | 13 +- Makefile | 9 + README.rst | 5 + VERSION | 2 +- _cockroachdb-createdb.py | 15 ++ guillotina/api/ws.py | 12 +- guillotina/db/storages/cockroach.py | 157 +++++++++++++----- guillotina/db/storages/pg.py | 3 +- guillotina/tests/conftest.py | 39 +++-- .../tests/docker_containers/__init__.py | 3 + guillotina/tests/docker_containers/base.py | 81 +++++++++ .../tests/docker_containers/cockroach.py | 40 +++++ guillotina/tests/docker_containers/etcd.py | 66 ++------ .../tests/docker_containers/postgres.py | 101 +++-------- guillotina/tests/test_postgres.py | 18 +- guillotina/tests/utils.py | 2 - 18 files changed, 366 insertions(+), 202 deletions(-) create mode 100644 _cockroachdb-createdb.py create mode 100644 guillotina/tests/docker_containers/base.py create mode 100644 guillotina/tests/docker_containers/cockroach.py diff --git a/.coveragerc b/.coveragerc index 94d0d5f68..509437cbc 100644 --- a/.coveragerc +++ b/.coveragerc @@ -7,3 +7,4 @@ omit = */docs/* */commands/* */one_connection_storage.py + */cockroach.py diff --git a/.travis.yml b/.travis.yml index ff9e86b47..2f823292b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,6 +23,7 @@ install: - sleep 15 script: - bin/py.test -s --cov=guillotina -v --cov-report term-missing guillotina + - USE_COCKROACH=true bin/py.test -s -v guillotina - bin/code-analysis after_success: - pip install coveralls diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 943c54746..7636c18c5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,7 +1,14 @@ -1.0.0a29 (unreleased) ---------------------- +1.1.0a1 (unreleased) +-------------------- + +- Include cockroachdb in our CI testing + [vangheem] -- Nothing changed yet. +- Simplify docker testing infrastructure + [vangheem] + +- Fix cockroachdb integration + [vangheem] 1.0.0a28 (2017-05-18) diff --git a/Makefile b/Makefile index dbb434db3..a50282672 100644 --- a/Makefile +++ b/Makefile @@ -20,3 +20,12 @@ run-etcd: --initial-cluster-token my-etcd-token \ --initial-cluster-state new \ --auto-compaction-retention 1 + + +run-cockroachdb: + docker pull cockroachdb/cockroach:v1.0 + docker run -p 127.0.0.1:26257:26257 -p 127.0.0.1:9080:8080 --rm cockroachdb/cockroach:v1.0 start --insecure + + +create-cockroachdb: + ./bin/py _cockroachdb-createdb.py diff --git a/README.rst b/README.rst index 607a6c732..439f2d950 100644 --- a/README.rst +++ b/README.rst @@ -88,6 +88,11 @@ With file watcher... ./bin/ptw guillotina --runner=./bin/py.test + +To run tests with cockroach db: + + USE_COCKROACH=true ./bin/pytest guillotina + Default ------- diff --git a/VERSION b/VERSION index 1671fe726..5c00fe7eb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.0a29.dev0 +1.1.0a1.dev0 diff --git a/_cockroachdb-createdb.py b/_cockroachdb-createdb.py new file mode 100644 index 000000000..c8e4f6232 --- /dev/null +++ b/_cockroachdb-createdb.py @@ -0,0 +1,15 @@ +import asyncpg +import asyncio + + +async def run(): + # Establish a connection to an existing database named "test" + # as a "postgres" user. + conn = await asyncpg.connect('postgresql://root@localhost:26257?sslmode=disable') + # Execute a statement to create a new table. + await conn.execute('''CREATE DATABASE guillotina''') + + +if __name__ == '__main__': + event_loop = asyncio.get_event_loop() + event_loop.run_until_complete(run()) diff --git a/guillotina/api/ws.py b/guillotina/api/ws.py index 96a9202fe..41c77c131 100644 --- a/guillotina/api/ws.py +++ b/guillotina/api/ws.py @@ -145,8 +145,8 @@ async def handle_ws_request(self, ws, message): self.request._futures = {} async def __call__(self): - tm = get_tm() - await tm.abort() + tm = get_tm(self.request) + await tm.abort(self.request) ws = web.WebSocketResponse() await ws.prepare(self.request) @@ -156,14 +156,16 @@ async def __call__(self): if message['op'] == 'close': await ws.close() elif message['op'] == 'GET': - await tm.begin() + txn = await tm.begin(request=self.request) try: await self.handle_ws_request(ws, message) - await tm.commit() except Exception: - await tm.abort() await ws.close() raise + finally: + # only currently support GET requests which are *never* + # supposed to be commits + await tm.abort(txn=txn) else: await ws.close() elif msg.tp == aiohttp.WSMsgType.error: diff --git a/guillotina/db/storages/cockroach.py b/guillotina/db/storages/cockroach.py index f7807d7d2..cf1edc226 100644 --- a/guillotina/db/storages/cockroach.py +++ b/guillotina/db/storages/cockroach.py @@ -1,28 +1,65 @@ from guillotina.db.storages import pg +from guillotina.exceptions import ConflictError +from guillotina.exceptions import TIDConflictError +import asyncpg +import sys -INSERT = """ - INSERT INTO objects - (zoid, tid, state_size, part, resource, of, otid, parent_id, id, type, state) - VALUES ($1::varchar(32), $2::int, $3::int, $4::int, $5::boolean, $6::varchar(32), $7::int, - $8::varchar(32), $9::text, $10::text, $11::bytea) - ON CONFLICT (zoid) - DO UPDATE SET - tid = EXCLUDED.tid, - state_size = EXCLUDED.state_size, - part = EXCLUDED.part, - resource = EXCLUDED.resource, - of = EXCLUDED.of, - otid = EXCLUDED.otid, - parent_id = EXCLUDED.parent_id, - id = EXCLUDED.id, - type = EXCLUDED.type, - state = EXCLUDED.state; - """ + +# upsert without checking matching tids on updated object +NAIVE_UPSERT = """ +INSERT INTO objects +(zoid, tid, state_size, part, resource, of, otid, parent_id, id, type, state) +VALUES ($1::varchar(32), $2::int, $3::int, $4::int, $5::boolean, $6::varchar(32), $7::int, + $8::varchar(32), $9::text, $10::text, $11::bytea) +ON CONFLICT (zoid) +DO UPDATE SET + tid = EXCLUDED.tid, + state_size = EXCLUDED.state_size, + part = EXCLUDED.part, + resource = EXCLUDED.resource, + of = EXCLUDED.of, + otid = EXCLUDED.otid, + parent_id = EXCLUDED.parent_id, + id = EXCLUDED.id, + type = EXCLUDED.type, + state = EXCLUDED.state""" +UPSERT = NAIVE_UPSERT + """ + WHERE + tid = EXCLUDED.otid""" + + +# update without checking matching tids on updated object +UPDATE = """ +UPDATE objects +SET + tid = $2::int, + state_size = $3::int, + part = $4::int, + resource = $5::boolean, + of = $6::varchar(32), + otid = $7::int, + parent_id = $8::varchar(32), + id = $9::text, + type = $10::text, + state = $11::bytea +WHERE + zoid = $1::varchar(32) + AND tid = $7::int +RETURNING tid, otid""" NEXT_TID = """SELECT unique_rowid()""" -MAX_TID = "SELECT COALESCE(MAX(tid), 0) from objects;" +MAX_TID = "SELECT MAX(tid) from objects;" + +DELETE_FROM_BLOBS = """DELETE FROM blobs WHERE zoid = $1::varchar(32);""" +UPDATE_REFERENCED_DELETED_OBJECTS = """ +UPDATE objects +SET + parent_id = NULL +WHERE + parent_id = $1::varchar(32) +""" class CockroachStorage(pg.PostgresqlStorage): @@ -37,18 +74,24 @@ class CockroachStorage(pg.PostgresqlStorage): - complex delete from query that does the sub queries to delete? - no sequence support - use serial construct of unique_rowid() instead + - no referencial integrity support! + - because we can't do ON DELETE support of any kind, we would get + errors after we run deletes unless we walk the whole sub tree + first, which is costly + - so we need to manually clean it up in a task that runs periodically, + our own db vacuum task. ''' _object_schema = pg.PostgresqlStorage._object_schema.copy() del _object_schema['json'] # no json db support _object_schema.update({ - 'of': 'VARCHAR(32) REFERENCES objects', - 'parent_id': 'VARCHAR(32) REFERENCES objects', # parent oid + 'of': 'VARCHAR(32)', + 'parent_id': 'VARCHAR(32)' }) _blob_schema = pg.PostgresqlStorage._blob_schema.copy() _blob_schema.update({ - 'zoid': 'VARCHAR(32) NOT NULL REFERENCES objects', + 'zoid': 'VARCHAR(32) NOT NULL', }) _initialize_statements = [ @@ -62,34 +105,70 @@ class CockroachStorage(pg.PostgresqlStorage): 'CREATE INDEX IF NOT EXISTS blob_chunk ON blobs (chunk_index);' ] + _max_tid = 0 + async def initialize_tid_statements(self): self._stmt_next_tid = await self._read_conn.prepare(NEXT_TID) self._stmt_max_tid = await self._read_conn.prepare(MAX_TID) + if hasattr(sys, '_db_tests'): + self.get_current_tid = self._test_get_current_tid + + async def _test_get_current_tid(self, txn): + return self._max_tid async def store(self, oid, old_serial, writer, obj, txn): assert oid is not None - smt = await self._get_prepared_statement(txn, 'insert', INSERT) - p = writer.serialize() # This calls __getstate__ of obj if len(p) >= self._large_record_size: self._log.warn("Too long object %d" % (obj.__class__, len(p))) part = writer.part if part is None: part = 0 - # (zoid, tid, state_size, part, main, parent_id, type, json, state) - await smt.fetchval( - oid, # The OID of the object - txn._tid, # Our TID - len(p), # Len of the object - part, # Partition indicator - writer.resource, # Is a resource ? - writer.of, # It belogs to a main - old_serial, # Old serial - writer.parent_id, # Parent OID - writer.id, # Traversal ID - writer.type, # Guillotina type - p # Pickle state - ) + + update = False + statement_sql = NAIVE_UPSERT + if not obj.__new_marker__ and obj._p_serial is not None: + # we should be confident this is an object update + statement_sql = UPDATE + update = True + + if hasattr(sys, '_db_tests') and txn._tid > self._max_tid: + self._max_tid = txn._tid + + async with txn._lock: + smt = await txn._db_conn.prepare(statement_sql) + try: + result = await smt.fetch( + oid, # The OID of the object + txn._tid, # Our TID + len(p), # Len of the object + part, # Partition indicator + writer.resource, # Is a resource ? + writer.of, # It belogs to a main + old_serial, # Old serial + writer.parent_id, # Parent OID + writer.id, # Traversal ID + writer.type, # Guillotina type + p # Pickle state) + ) + except asyncpg.exceptions._base.InterfaceError as ex: + if 'another operation is in progress' in ex.args[0]: + raise ConflictError( + 'asyncpg error, another operation in progress.') + raise + if update and len(result) != 1: + # raise tid conflict error + raise TIDConflictError( + 'Mismatch of tid of object being updated. This is likely ' + 'caused by a cache invalidation race condition and should ' + 'be an edge case. This should resolve on request retry.') obj._p_estimated_size = len(p) - return txn._tid, len(p) + + async def delete(self, txn, oid): + # XXX no cascade support! + # need to move things around and recursively delete here... + async with txn._lock: + await txn._db_conn.execute(UPDATE_REFERENCED_DELETED_OBJECTS, oid) + await txn._db_conn.execute(DELETE_FROM_BLOBS, oid) + await txn._db_conn.execute(pg.DELETE_FROM_OBJECTS, oid) diff --git a/guillotina/db/storages/pg.py b/guillotina/db/storages/pg.py index 5f3fc624c..f3bf63352 100644 --- a/guillotina/db/storages/pg.py +++ b/guillotina/db/storages/pg.py @@ -246,19 +246,18 @@ async def finalize(self): async def create(self): # Check DB + log.info('Creating initial database objects') statements = [ get_table_definition('objects', self._object_schema), get_table_definition('blobs', self._blob_schema, primary_keys=('bid', 'zoid', 'chunk_index')) ] statements.extend(self._initialize_statements) - async with self._pool.acquire() as conn: for statement in statements: await conn.execute(statement) await self.initialize_tid_statements() - # migrate old transaction table scheme over try: old_tid = await self._read_conn.fetchval('SELECT max(tid) from transaction') diff --git a/guillotina/tests/conftest.py b/guillotina/tests/conftest.py index ba86cced1..c6561abae 100644 --- a/guillotina/tests/conftest.py +++ b/guillotina/tests/conftest.py @@ -6,10 +6,7 @@ from guillotina.interfaces import IApplication from guillotina.testing import ADMIN_TOKEN from guillotina.testing import TESTING_SETTINGS -from guillotina.tests.docker_containers.etcd import cleanup_etcd_docker -from guillotina.tests.docker_containers.etcd import run_docker_etcd -from guillotina.tests.docker_containers.postgres import cleanup_postgres_docker -from guillotina.tests.docker_containers.postgres import run_docker_postgresql +from guillotina.tests import docker_containers as containers from guillotina.tests.utils import ContainerRequesterAsyncContextManager from guillotina.tests.utils import get_mocked_request @@ -18,12 +15,12 @@ import copy import os import pytest +import sys IS_TRAVIS = 'TRAVIS' in os.environ +USE_COCKROACH = 'USE_COCKROACH' in os.environ -IMAGE = 'postgres:9.6' -CONTAINERS_FOR_TESTING_LABEL = 'testingaiopg' PG_SETTINGS = copy.deepcopy(TESTING_SETTINGS) PG_SETTINGS['databases'][0]['db']['storage'] = 'postgresql' @@ -37,6 +34,12 @@ 'password': '', 'port': 5432 } +if USE_COCKROACH: + PG_SETTINGS['databases'][0]['db']['storage'] = 'cockroach' + PG_SETTINGS['databases'][0]['db']['dsn'].update({ + 'user': 'root', + 'port': 26257 + }) DUMMY_SETTINGS = copy.deepcopy(TESTING_SETTINGS) DUMMY_SETTINGS['databases'][0]['db']['storage'] = 'DUMMY' @@ -51,29 +54,33 @@ def postgres(): """ detect travis, use travis's postgres; otherwise, use docker """ - if not IS_TRAVIS: - host = run_docker_postgresql() + sys._db_tests = True + + if USE_COCKROACH: + host = containers.cockroach_image.run() else: - host = 'localhost' + if not IS_TRAVIS: + host = containers.postgres_image.run() + else: + host = 'localhost' PG_SETTINGS['databases'][0]['db']['dsn']['host'] = host yield host # provide the fixture value - if not IS_TRAVIS: - cleanup_postgres_docker() + if USE_COCKROACH: + containers.cockroach_image.stop() + elif not IS_TRAVIS: + containers.postgres_image.stop() @pytest.fixture(scope='session') def etcd(): - """ - detect travis, use travis's postgres; otherwise, use docker - """ - host = run_docker_etcd() + host = containers.etcd_image.run() yield host # provide the fixture value - cleanup_etcd_docker() + containers.etcd_image.stop() class GuillotinaDBRequester(object): diff --git a/guillotina/tests/docker_containers/__init__.py b/guillotina/tests/docker_containers/__init__.py index e69de29bb..abc2eac71 100644 --- a/guillotina/tests/docker_containers/__init__.py +++ b/guillotina/tests/docker_containers/__init__.py @@ -0,0 +1,3 @@ +from .cockroach import image as cockroach_image # noqa +from .etcd import image as etcd_image # noqa +from .postgres import image as postgres_image # noqa diff --git a/guillotina/tests/docker_containers/base.py b/guillotina/tests/docker_containers/base.py new file mode 100644 index 000000000..7fc67d048 --- /dev/null +++ b/guillotina/tests/docker_containers/base.py @@ -0,0 +1,81 @@ +from time import sleep + +import os +import docker + + +COCKROACH_IMAGE = 'cockroachdb/cockroach:v1.0' + + +class BaseImage: + + docker_version = '1.23' + label = 'foobar' + image = None + to_port = from_port = None + image_options = dict( + cap_add=['IPC_LOCK'], + mem_limit='1g', + environment={}, + privileged=True) + + def check(self, host): + return True + + def run(self): + docker_client = docker.from_env(version=self.docker_version) + + # Clean up possible other docker containers + test_containers = docker_client.containers.list( + all=True, + filters={'label': self.label}) + for test_container in test_containers: + test_container.stop() + test_container.remove(v=True, force=True) + + # Create a new one + container = docker_client.containers.run( + image=self.image, + labels=[self.label], + detach=True, + ports={ + f'{self.to_port}/tcp': self.from_port + }, + **self.image_options + ) + ident = container.id + count = 1 + + container_obj = docker_client.containers.get(ident) + + opened = False + host = '' + + print(f'starting {self.label}') + while count < 30 and not opened: + count += 1 + try: + container_obj = docker_client.containers.get(ident) + except docker.errors.NotFound: + continue + sleep(1) + if container_obj.attrs['NetworkSettings']['IPAddress'] != '': + if os.environ.get('TESTING', '') == 'jenkins': + host = container_obj.attrs['NetworkSettings']['IPAddress'] + else: + host = 'localhost' + + if host != '': + opened = self.check(host) + print(f'{self.label} started') + return host + + def stop(self): + docker_client = docker.from_env(version=self.docker_version) + # Clean up possible other docker containers + test_containers = docker_client.containers.list( + all=True, + filters={'label': self.label}) + for test_container in test_containers: + test_container.kill() + test_container.remove(v=True, force=True) diff --git a/guillotina/tests/docker_containers/cockroach.py b/guillotina/tests/docker_containers/cockroach.py new file mode 100644 index 000000000..6eb6eeec2 --- /dev/null +++ b/guillotina/tests/docker_containers/cockroach.py @@ -0,0 +1,40 @@ +from guillotina.tests.docker_containers.base import BaseImage + +import psycopg2 + + +class CockroachDB(BaseImage): + label = 'cockroach' + image = 'cockroachdb/cockroach:v1.0' + to_port = from_port = 26257 + image_options = BaseImage.image_options.copy() + image_options.update(dict( + command=' '.join([ + 'start --insecure', + ]) + )) + + def check(self, host): + conn = cur = None + try: + conn = psycopg2.connect("dbname=guillotina user=root host=%s port=26257" % host) # noqa + conn.set_session(autocommit=True) + cur = conn.cursor() + cur.execute('SHOW DATABASES;') + for result in cur.fetchall(): + if result[0] == 'guillotina': + conn.close() + cur.close() + return True + cur.execute("CREATE DATABASE IF NOT EXISTS guillotina;") + cur.close() + conn.close() + except: # noqa + if conn is not None: + conn.close() + if cur is not None: + cur.close() + return False + + +image = CockroachDB() diff --git a/guillotina/tests/docker_containers/etcd.py b/guillotina/tests/docker_containers/etcd.py index 3629c8265..4ea8a7477 100644 --- a/guillotina/tests/docker_containers/etcd.py +++ b/guillotina/tests/docker_containers/etcd.py @@ -1,26 +1,14 @@ -from time import sleep +from guillotina.tests.docker_containers.base import BaseImage -ETCD_IMAGE = 'quay.io/coreos/etcd:v3.2.0-rc.0' - - -def run_docker_etcd(label='testingetcd'): - import docker - docker_client = docker.from_env(version='1.23') - - # Clean up possible other docker containers - test_containers = docker_client.containers.list( - all=True, - filters={'label': label}) - for test_container in test_containers: - test_container.stop() - test_container.remove(v=True, force=True) - - # Create a new one - container = docker_client.containers.run( +class ETCD(BaseImage): + label = 'etcd' + image = 'quay.io/coreos/etcd:v3.2.0-rc.0' + to_port = from_port = 2379 + image_options = BaseImage.image_options.copy() + image_options.update(dict( name='my-etcd-1', - image=ETCD_IMAGE, - labels=[label], + mem_limit='200m', command=' '.join([ '/usr/local/bin/etcd', '--name my-etcd-1', @@ -33,40 +21,8 @@ def run_docker_etcd(label='testingetcd'): '--initial-cluster-token my-etcd-token', '--initial-cluster-state new', '--auto-compaction-retention 1' - ]), - detach=True, - ports={ - '2379/tcp': 2379 - }, - cap_add=['IPC_LOCK'], - mem_limit='200m', - privileged=True - ) - ident = container.id - count = 1 - - opened = False - host = '' - - print('starting etcd') - while count < 30 and not opened: - count += 1 - try: - docker_client.containers.get(ident) - except docker.errors.NotFound: - continue - sleep(1) - print('postgresql etcd') - return host + ]) + )) -def cleanup_etcd_docker(label='testingetcd'): - import docker - docker_client = docker.from_env(version='1.23') - # Clean up possible other docker containers - test_containers = docker_client.containers.list( - all=True, - filters={'label': label}) - for test_container in test_containers: - test_container.kill() - test_container.remove(v=True, force=True) +image = ETCD() diff --git a/guillotina/tests/docker_containers/postgres.py b/guillotina/tests/docker_containers/postgres.py index 9f41a3365..0003b4ca5 100644 --- a/guillotina/tests/docker_containers/postgres.py +++ b/guillotina/tests/docker_containers/postgres.py @@ -1,85 +1,34 @@ -from time import sleep +from guillotina.tests.docker_containers.base import BaseImage -import os +import psycopg2 -POSTGRESQL_IMAGE = 'postgres:9.6' - - -def run_docker_postgresql(label='testingaiopg'): - import docker - docker_client = docker.from_env(version='1.23') - - # Clean up possible other docker containers - test_containers = docker_client.containers.list( - all=True, - filters={'label': label}) - for test_container in test_containers: - test_container.stop() - test_container.remove(v=True, force=True) - - # Create a new one - container = docker_client.containers.run( - image=POSTGRESQL_IMAGE, - labels=[label], - detach=True, - ports={ - '5432/tcp': 5432 - }, - cap_add=['IPC_LOCK'], - mem_limit='1g', +class Postgresql(BaseImage): + label = 'postgresql' + image = 'postgres:9.6' + to_port = from_port = 5432 + image_options = BaseImage.image_options.copy() + image_options.update(dict( environment={ 'POSTGRES_PASSWORD': '', 'POSTGRES_DB': 'guillotina', 'POSTGRES_USER': 'postgres' - }, - privileged=True - ) - ident = container.id - count = 1 - - container_obj = docker_client.containers.get(ident) + } + )) - opened = False - host = '' - - print('starting postgresql') - while count < 30 and not opened: - count += 1 + def check(self, host): try: - container_obj = docker_client.containers.get(ident) - except docker.errors.NotFound: - continue - sleep(1) - if container_obj.attrs['NetworkSettings']['IPAddress'] != '': - if os.environ.get('TESTING', '') == 'jenkins': - host = container_obj.attrs['NetworkSettings']['IPAddress'] - else: - host = 'localhost' - - if host != '': - try: - conn = psycopg2.connect("dbname=guillotina user=postgres host=%s port=5432" % host) # noqa - cur = conn.cursor() - cur.execute("SELECT 1;") - cur.fetchone() - cur.close() - conn.close() - opened = True - except: # noqa - conn = None - cur = None - print('postgresql started') - return host - - -def cleanup_postgres_docker(label='testingaiopg'): - import docker - docker_client = docker.from_env(version='1.23') - # Clean up possible other docker containers - test_containers = docker_client.containers.list( - all=True, - filters={'label': label}) - for test_container in test_containers: - test_container.kill() - test_container.remove(v=True, force=True) + conn = psycopg2.connect("dbname=guillotina user=postgres host=%s port=5432" % host) # noqa + cur = conn.cursor() + cur.execute("SELECT 1;") + cur.fetchone() + cur.close() + conn.close() + return True + except: # noqa + conn = None + cur = None + return False + + +image = Postgresql() diff --git a/guillotina/tests/test_postgres.py b/guillotina/tests/test_postgres.py index deaf12211..98e9ad77f 100644 --- a/guillotina/tests/test_postgres.py +++ b/guillotina/tests/test_postgres.py @@ -1,4 +1,5 @@ from guillotina.content import Folder +from guillotina.db.storages.cockroach import CockroachStorage from guillotina.db.storages.pg import PostgresqlStorage from guillotina.db.transaction_manager import TransactionManager from guillotina.exceptions import ConflictError @@ -6,16 +7,21 @@ import asyncio import concurrent +import os import pytest +USE_COCKROACH = 'USE_COCKROACH' in os.environ + + async def cleanup(aps): conn = await aps.open() txn = conn.transaction() await txn.start() await conn.execute("DROP TABLE IF EXISTS objects;") await conn.execute("DROP TABLE IF EXISTS blobs;") - await conn.execute("ALTER SEQUENCE tid_sequence RESTART WITH 1") + if not USE_COCKROACH: + await conn.execute("ALTER SEQUENCE tid_sequence RESTART WITH 1") await txn.commit() await aps._pool.release(conn) await aps.create() @@ -24,7 +30,11 @@ async def cleanup(aps): async def get_aps(strategy='resolve', pool_size=15): dsn = "postgres://postgres:@localhost:5432/guillotina" partition_object = "guillotina.db.interfaces.IPartition" - aps = PostgresqlStorage( + klass = PostgresqlStorage + if USE_COCKROACH: + klass = CockroachStorage + dsn = "postgres://root:@localhost:26257/guillotina?sslmode=disable" + aps = klass( dsn=dsn, partition=partition_object, name='db', transaction_strategy=strategy, pool_size=pool_size, conn_acquire_timeout=0.1) @@ -58,6 +68,7 @@ async def test_read_obs(postgres, dummy_request): await cleanup(aps) +@pytest.mark.skipif(USE_COCKROACH, reason="Cockroach does not have cascade support") async def test_deleting_parent_deletes_children(postgres, dummy_request): request = dummy_request # noqa so magically get_current_request can find @@ -159,6 +170,7 @@ async def test_delete_resource_deletes_blob(postgres, dummy_request): await cleanup(aps) +@pytest.mark.skipif(USE_COCKROACH, reason="Test issues with cockroach") async def test_should_raise_conflict_error_when_editing_diff_data_with_resolve_strat( postgres, dummy_request): request = dummy_request # noqa so magically get_current_request can find @@ -268,6 +280,7 @@ async def test_should_not_resolve_conflict_error_with_simple_strat(postgres, dum await cleanup(aps) +@pytest.mark.skipif(USE_COCKROACH, reason="Test issues with cockroach") async def test_none_strat_allows_trans_commits(postgres, dummy_request): request = dummy_request # noqa so magically get_current_request can find @@ -425,6 +438,5 @@ async def test_mismatched_tid_causes_conflict_error(postgres, dummy_request): with pytest.raises(ConflictError): await tm.commit(txn=txn) - await aps.remove() await cleanup(aps) diff --git a/guillotina/tests/utils.py b/guillotina/tests/utils.py index 4edca208e..64951d2eb 100644 --- a/guillotina/tests/utils.py +++ b/guillotina/tests/utils.py @@ -5,8 +5,6 @@ from guillotina.interfaces import IDefaultLayer from guillotina.interfaces import IRequest from guillotina.security.policy import Interaction -from guillotina.tests.docker_containers.postgres import cleanup_postgres_docker # noqa b/w -from guillotina.tests.docker_containers.postgres import run_docker_postgresql # noqa b/w from zope.interface import alsoProvides from zope.interface import implementer from guillotina.transactions import managed_transaction