From 5c2bb3697c29e0acc7fb345098c70698cd70033e Mon Sep 17 00:00:00 2001 From: Piper Merriam Date: Tue, 6 Aug 2019 11:40:05 -0600 Subject: [PATCH] Package up base db and atomic db test suites into re-usable classes --- eth/db/backends/base.py | 2 +- eth/db/backends/level.py | 2 + eth/tools/db/__init__.py | 0 eth/tools/db/atomic.py | 158 +++++++++++++++++++++ eth/tools/db/base.py | 76 ++++++++++ tests/database/test_atomic_database_api.py | 30 ++++ tests/database/test_atomic_db.py | 112 --------------- tests/database/test_base_atomic_db.py | 141 ------------------ tests/database/test_base_db_api.py | 78 ---------- tests/database/test_database_api.py | 31 ++++ 10 files changed, 298 insertions(+), 332 deletions(-) create mode 100644 eth/tools/db/__init__.py create mode 100644 eth/tools/db/atomic.py create mode 100644 eth/tools/db/base.py create mode 100644 tests/database/test_atomic_database_api.py delete mode 100644 tests/database/test_atomic_db.py delete mode 100644 tests/database/test_base_atomic_db.py delete mode 100644 tests/database/test_base_db_api.py create mode 100644 tests/database/test_database_api.py diff --git a/eth/db/backends/base.py b/eth/db/backends/base.py index 52f8f69a09..36b4118d51 100644 --- a/eth/db/backends/base.py +++ b/eth/db/backends/base.py @@ -40,7 +40,7 @@ def delete(self, key: bytes) -> None: try: del self[key] except KeyError: - return None + pass def __iter__(self) -> Iterator[bytes]: raise NotImplementedError("By default, DB classes cannot be iterated.") diff --git a/eth/db/backends/level.py b/eth/db/backends/level.py index 24705013ba..abcedb5ad0 100644 --- a/eth/db/backends/level.py +++ b/eth/db/backends/level.py @@ -62,6 +62,8 @@ def _exists(self, key: bytes) -> bool: return self.db.get(key) is not None def __delitem__(self, key: bytes) -> None: + if self.db.get(key) is None: + raise KeyError(key) self.db.delete(key) @contextmanager diff --git a/eth/tools/db/__init__.py b/eth/tools/db/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/eth/tools/db/atomic.py b/eth/tools/db/atomic.py new file mode 100644 index 0000000000..7b3a2c5fc9 --- /dev/null +++ b/eth/tools/db/atomic.py @@ -0,0 +1,158 @@ +import pytest + +from eth_utils import ValidationError + +from eth.abc import AtomicDatabaseAPI + + +class AtomicDatabaseBatchAPITestSuite: + def test_atomic_batch_set_and_get(self, atomic_db: AtomicDatabaseAPI) -> None: + with atomic_db.atomic_batch() as batch: + batch.set(b'1', b'2') + assert batch.get(b'1') == b'2' + + assert atomic_db.get(b'1') == b'2' + + def test_atomic_db_cannot_recursively_batch(self, atomic_db: AtomicDatabaseAPI) -> None: + with atomic_db.atomic_batch() as batch: + assert not hasattr(batch, 'atomic_batch') + + def test_atomic_db_with_set_and_delete_batch(self, atomic_db: AtomicDatabaseAPI) -> None: + atomic_db[b'key-1'] = b'origin' + + with atomic_db.atomic_batch() as batch: + batch.delete(b'key-1') + + assert b'key-1' not in batch + with pytest.raises(KeyError): + assert batch[b'key-1'] + + with pytest.raises(KeyError): + atomic_db[b'key-1'] + + def test_atomic_db_unbatched_sets_are_immediate(self, atomic_db: AtomicDatabaseAPI) -> None: + atomic_db[b'1'] = b'A' + + with atomic_db.atomic_batch() as batch: + # Unbatched changes are immediate, and show up in batch reads + atomic_db[b'1'] = b'B' + assert batch[b'1'] == b'B' + + batch[b'1'] = b'C1' + + # It doesn't matter what changes happen underlying, all reads now + # show the write applied to the batch db handle + atomic_db[b'1'] = b'C2' + assert batch[b'1'] == b'C1' + + # the batch write should overwrite any intermediate changes + assert atomic_db[b'1'] == b'C1' + + def test_atomic_db_unbatched_deletes_are_immediate(self, atomic_db: AtomicDatabaseAPI) -> None: + atomic_db[b'1'] = b'A' + + with atomic_db.atomic_batch() as batch: + assert b'1' in batch + + # Unbatched changes are immediate, and show up in batch reads + del atomic_db[b'1'] + + assert b'1' not in batch + + batch[b'1'] = b'C1' + + # It doesn't matter what changes happen underlying, all reads now + # show the write applied to the batch db handle + atomic_db[b'1'] = b'C2' + assert batch[b'1'] == b'C1' + + # the batch write should overwrite any intermediate changes + assert atomic_db[b'1'] == b'C1' + + def test_atomic_db_cannot_use_batch_after_context(self, atomic_db: AtomicDatabaseAPI) -> None: + atomic_db[b'1'] = b'A' + + with atomic_db.atomic_batch() as batch: + batch[b'1'] = b'B' + + # set + with pytest.raises(ValidationError): + batch[b'1'] = b'C' + + with pytest.raises(ValidationError): + batch.set(b'1', b'C') + + # get + with pytest.raises(ValidationError): + batch[b'1'] + + with pytest.raises(ValidationError): + batch.get(b'1') + + # exists + with pytest.raises(ValidationError): + b'1' in batch + + with pytest.raises(ValidationError): + batch.exists(b'1') + + # delete + with pytest.raises(ValidationError): + del batch[b'1'] + + with pytest.raises(ValidationError): + batch.delete(b'1') + + # none of the invalid changes above should change the original db + assert atomic_db[b'1'] == b'B' + + def test_atomic_db_with_reverted_delete_batch(self, atomic_db: AtomicDatabaseAPI) -> None: + class CustomException(Exception): + pass + + atomic_db[b'key-1'] = b'origin' + + with pytest.raises(CustomException): + with atomic_db.atomic_batch() as batch: + batch.delete(b'key-1') + + assert b'key-1' not in batch + with pytest.raises(KeyError): + assert batch[b'key-1'] + + raise CustomException('pretend something went wrong') + + assert atomic_db[b'key-1'] == b'origin' + + def test_atomic_db_temporary_state_dropped_across_batches(self, + atomic_db: AtomicDatabaseAPI) -> None: + class CustomException(Exception): + pass + + atomic_db[b'key-1'] = b'origin' + + with pytest.raises(CustomException): + with atomic_db.atomic_batch() as batch: + batch.delete(b'key-1') + batch.set(b'key-2', b'val-2') + raise CustomException('pretend something went wrong') + + with atomic_db.atomic_batch() as batch: + assert batch[b'key-1'] == b'origin' + assert b'key-2' not in batch + + def test_atomic_db_with_exception_batch(self, atomic_db: AtomicDatabaseAPI) -> None: + atomic_db.set(b'key-1', b'value-1') + + try: + with atomic_db.atomic_batch() as batch: + batch.set(b'key-1', b'new-value-1') + batch.set(b'key-2', b'value-2') + raise Exception + except Exception: + pass + + assert atomic_db.get(b'key-1') == b'value-1' + + with pytest.raises(KeyError): + atomic_db[b'key-2'] diff --git a/eth/tools/db/base.py b/eth/tools/db/base.py new file mode 100644 index 0000000000..f0d5011196 --- /dev/null +++ b/eth/tools/db/base.py @@ -0,0 +1,76 @@ +import pytest + +from eth.abc import DatabaseAPI + + +class DatabaseAPITestSuite: + def test_database_api_get(self, db: DatabaseAPI) -> None: + db[b'key-1'] = b'value-1' + assert db.get(b'key-1') == b'value-1' + + def test_database_api_item_getter(self, db: DatabaseAPI) -> None: + db[b'key-1'] = b'value-1' + assert db[b'key-1'] == b'value-1' + + def test_database_api_get_missing_key(self, db: DatabaseAPI) -> None: + assert b'key-1' not in db + assert db.get(b'key-1') is None + + def test_database_api_item_getter_missing_key(self, db: DatabaseAPI) -> None: + assert b'key-1' not in db + with pytest.raises(KeyError): + db[b'key-1'] + + def test_database_api_set(self, db: DatabaseAPI) -> None: + db[b'key-1'] = b'value-1' + assert db[b'key-1'] == b'value-1' + db[b'key-1'] = b'value-2' + assert db[b'key-1'] == b'value-2' + + def test_database_api_item_setter(self, db: DatabaseAPI) -> None: + db.set(b'key-1', b'value-1') + assert db[b'key-1'] == b'value-1' + db.set(b'key-1', b'value-2') + assert db[b'key-1'] == b'value-2' + + def test_database_api_exists(self, db: DatabaseAPI) -> None: + assert db.exists(b'key-1') is False + + db[b'key-1'] = b'value-1' + + assert db.exists(b'key-1') is True + + def test_database_api_contains_checking(self, db: DatabaseAPI) -> None: + assert b'key-1' not in db + + db[b'key-1'] = b'value-1' + + assert b'key-1' in db + + def test_database_api_delete(self, db: DatabaseAPI) -> None: + db[b'key-1'] = b'value-1' + + assert b'key-1' in db + + db.delete(b'key-1') + + assert not db.exists(b'key-1') + assert b'key-1' not in db + + def test_database_api_item_delete(self, db: DatabaseAPI) -> None: + db[b'key-1'] = b'value-1' + + assert b'key-1' in db + + del db[b'key-1'] + + assert b'key-1' not in db + + def test_database_api_delete_missing_key(self, db: DatabaseAPI) -> None: + assert b'key-1' not in db + db.delete(b'key-1') + + def test_database_api_item_delete_missing_key(self, db: DatabaseAPI) -> None: + assert b'key-1' not in db + with pytest.raises(KeyError): + del db[b'key-1'] diff --git a/tests/database/test_atomic_database_api.py b/tests/database/test_atomic_database_api.py new file mode 100644 index 0000000000..20a822074b --- /dev/null +++ b/tests/database/test_atomic_database_api.py @@ -0,0 +1,30 @@ +import pytest + +from eth.db.atomic import AtomicDB +from eth.db.backends.level import LevelDB + +from eth.tools.db.base import DatabaseAPITestSuite +from eth.tools.db.atomic import AtomicDatabaseBatchAPITestSuite + + +@pytest.fixture(params=['atomic', 'level']) +def atomic_db(request, tmpdir): + if request.param == 'atomic': + return AtomicDB() + elif request.param == 'level': + return LevelDB(db_path=tmpdir.mkdir("level_db_path")) + else: + raise ValueError("Unexpected database type: {}".format(request.param)) + + +@pytest.fixture +def db(atomic_db): + return atomic_db + + +class TestAtomicDatabaseBatchAPI(AtomicDatabaseBatchAPITestSuite): + pass + + +class TestAtomicDatabaseAPI(DatabaseAPITestSuite): + pass diff --git a/tests/database/test_atomic_db.py b/tests/database/test_atomic_db.py deleted file mode 100644 index 2a0b2a1a26..0000000000 --- a/tests/database/test_atomic_db.py +++ /dev/null @@ -1,112 +0,0 @@ -import pytest - -from eth.db.atomic import AtomicDB - - -@pytest.fixture -def atomic_db(base_db): - return AtomicDB(base_db) - - -def test_atomic_db_with_set_and_get(base_db, atomic_db): - with atomic_db.atomic_batch() as db: - db.set(b'key-1', b'value-1') - db.set(b'key-2', b'value-2') - assert db.get(b'key-1') == b'value-1' - assert db.get(b'key-2') == b'value-2' - - # keys should not yet be set in base db. - assert b'key-1' not in base_db - assert b'key-2' not in base_db - - assert base_db.get(b'key-1') == b'value-1' - assert base_db.get(b'key-2') == b'value-2' - - -def test_atomic_db_with_set_and_get_unbatched(base_db, atomic_db): - atomic_db.set(b'key-1', b'value-1') - assert atomic_db.get(b'key-1') == b'value-1' - atomic_db.set(b'key-2', b'value-2') - assert atomic_db.get(b'key-2') == b'value-2' - - # keys should be immediately set in base db. - assert base_db.get(b'key-1') == b'value-1' - assert base_db.get(b'key-2') == b'value-2' - - -def test_atomic_db_with_set_and_delete(base_db, atomic_db): - base_db[b'key-1'] = b'origin' - - with atomic_db.atomic_batch() as db: - db.delete(b'key-1') - - assert b'key-1' not in db - with pytest.raises(KeyError): - assert db[b'key-1'] - - # key should still be in base db - assert b'key-1' in base_db - assert b'key-1' not in db - - with pytest.raises(KeyError): - base_db[b'key-1'] - with pytest.raises(KeyError): - atomic_db[b'key-1'] - - -def test_atomic_db_with_set_and_delete_unbatched(base_db, atomic_db): - base_db[b'key-1'] = b'origin' - - atomic_db.delete(b'key-1') - - assert b'key-1' not in atomic_db - with pytest.raises(KeyError): - assert atomic_db[b'key-1'] - - # key should be immediately removed from base atomic_db - with pytest.raises(KeyError): - base_db[b'key-1'] - - -def test_atomic_db_with_exception(base_db, atomic_db): - base_db.set(b'key-1', b'value-1') - - try: - with atomic_db.atomic_batch() as db: - db.set(b'key-1', b'new-value-1') - db.set(b'key-2', b'value-2') - raise Exception - except Exception: - pass - - assert base_db.get(b'key-1') == b'value-1' - - with pytest.raises(KeyError): - base_db[b'key-2'] - with pytest.raises(KeyError): - atomic_db[b'key-2'] - - -def test_atomic_db_with_exception_across_contexts(base_db, atomic_db): - base_db[b'key-1'] = b'origin-1' - base_db[b'key-2'] = b'origin-2' - - try: - with atomic_db.atomic_batch() as db: - db[b'key-1'] = b'value-1' - raise Exception('throw') - except Exception: - pass - - assert base_db[b'key-1'] == b'origin-1' - assert atomic_db[b'key-1'] == b'origin-1' - assert base_db[b'key-2'] == b'origin-2' - assert atomic_db[b'key-2'] == b'origin-2' - - with atomic_db.atomic_batch() as db: - db[b'key-2'] = b'value-2' - - assert base_db[b'key-1'] == b'origin-1' - assert atomic_db[b'key-1'] == b'origin-1' - assert base_db[b'key-2'] == b'value-2' - assert atomic_db[b'key-2'] == b'value-2' diff --git a/tests/database/test_base_atomic_db.py b/tests/database/test_base_atomic_db.py deleted file mode 100644 index 26a5a4bd42..0000000000 --- a/tests/database/test_base_atomic_db.py +++ /dev/null @@ -1,141 +0,0 @@ -import pytest - -from eth_utils import ValidationError - -from eth.db.atomic import AtomicDB -from eth.db.backends.level import LevelDB - - -@pytest.fixture(params=['atomic', 'level']) -def atomic_db(request, tmpdir): - if request.param == 'atomic': - return AtomicDB() - elif request.param == 'level': - return LevelDB(db_path=tmpdir.mkdir("level_db_path")) - else: - raise ValueError("Unexpected database type: {}".format(request.param)) - - -def test_atomic_batch(atomic_db): - with atomic_db.atomic_batch() as db: - db.set(b'1', b'2') - db.set(b'3', b'4') - assert db.get(b'1') == b'2' - - assert atomic_db.get(b'1') == b'2' - assert atomic_db.get(b'3') == b'4' - - -def test_atomic_db_cannot_recursively_batch(atomic_db): - with atomic_db.atomic_batch() as db: - with pytest.raises(AttributeError): - with db.atomic_batch(): - assert False, "LevelDB should not permit recursive batching of changes" - - -def test_atomic_db_with_set_and_delete_batch(atomic_db): - atomic_db[b'key-1'] = b'origin' - - with atomic_db.atomic_batch() as db: - db.delete(b'key-1') - - assert b'key-1' not in db - with pytest.raises(KeyError): - assert db[b'key-1'] - - with pytest.raises(KeyError): - atomic_db[b'key-1'] - - -def test_atomic_db_unbatched_sets_are_immediate(atomic_db): - atomic_db[b'1'] = b'A' - - with atomic_db.atomic_batch() as db: - # Unbatched changes are immediate, and show up in batch reads - atomic_db[b'1'] = b'B' - assert db[b'1'] == b'B' - - db[b'1'] = b'C1' - - # It doesn't matter what changes happen underlying, all reads now - # show the write applied to the batch db handle - atomic_db[b'1'] = b'C2' - assert db[b'1'] == b'C1' - - # the batch write should overwrite any intermediate changes - assert atomic_db[b'1'] == b'C1' - - -def test_atomic_db_cannot_use_write_batch_after_context(atomic_db): - atomic_db[b'1'] = b'A' - - with atomic_db.atomic_batch() as db: - db[b'1'] = b'B' - - with pytest.raises(ValidationError): - db[b'1'] = b'C' - - with pytest.raises(ValidationError): - b'1' in db - - with pytest.raises(ValidationError): - del db[b'1'] - - with pytest.raises(ValidationError): - assert db[b'1'] == 'C' - - # none of the invalid changes above should change the original db - assert atomic_db[b'1'] == b'B' - - -def test_atomic_db_with_reverted_delete_batch(atomic_db): - class CustomException(Exception): - pass - - atomic_db[b'key-1'] = b'origin' - - with pytest.raises(CustomException): - with atomic_db.atomic_batch() as db: - db.delete(b'key-1') - - assert b'key-1' not in db - with pytest.raises(KeyError): - assert db[b'key-1'] - - raise CustomException('pretend something went wrong') - - assert atomic_db[b'key-1'] == b'origin' - - -def test_atomic_db_temporary_state_dropped_across_batches(atomic_db): - class CustomException(Exception): - pass - - atomic_db[b'key-1'] = b'origin' - - with pytest.raises(CustomException): - with atomic_db.atomic_batch() as db: - db.delete(b'key-1') - db.set(b'key-2', b'val-2') - raise CustomException('pretend something went wrong') - - with atomic_db.atomic_batch() as db: - assert db[b'key-1'] == b'origin' - assert b'key-2' not in db - - -def test_atomic_db_with_exception_batch(atomic_db): - atomic_db.set(b'key-1', b'value-1') - - try: - with atomic_db.atomic_batch() as db: - db.set(b'key-1', b'new-value-1') - db.set(b'key-2', b'value-2') - raise Exception - except Exception: - pass - - assert atomic_db.get(b'key-1') == b'value-1' - - with pytest.raises(KeyError): - atomic_db[b'key-2'] diff --git a/tests/database/test_base_db_api.py b/tests/database/test_base_db_api.py deleted file mode 100644 index 4e08fb66ac..0000000000 --- a/tests/database/test_base_db_api.py +++ /dev/null @@ -1,78 +0,0 @@ -import pytest -from eth.db.backends.memory import MemoryDB -from eth.db.journal import JournalDB -from eth.db.batch import BatchDB - - -@pytest.fixture(params=[JournalDB, BatchDB, MemoryDB]) -def db(request): - base_db = MemoryDB() - if request.param is JournalDB: - return JournalDB(base_db) - elif request.param is BatchDB: - return BatchDB(base_db) - elif request.param is MemoryDB: - return base_db - else: - raise Exception("Invariant") - - -def test_database_api_get(db): - db[b'key-1'] = b'value-1' - - assert db.get(b'key-1') == b'value-1' - assert db[b'key-1'] == b'value-1' - - -def test_database_api_set(db): - db[b'key-1'] = b'value-1' - assert db[b'key-1'] == b'value-1' - db[b'key-1'] = b'value-2' - assert db[b'key-1'] == b'value-2' - - db.set(b'key-1', b'value-1') - assert db[b'key-1'] == b'value-1' - db.set(b'key-1', b'value-2') - assert db[b'key-1'] == b'value-2' - - -def test_database_api_existence_checking(db): - assert not db.exists(b'key-1') - assert b'key-1' not in db - - db[b'key-1'] = b'value-1' - - assert db.exists(b'key-1') - assert b'key-1' in db - - -def test_database_api_delete(db): - db[b'key-1'] = b'value-1' - db[b'key-2'] = b'value-2' - - assert db.exists(b'key-1') - assert db.exists(b'key-2') - assert b'key-1' in db - assert b'key-2' in db - - del db[b'key-1'] - db.delete(b'key-2') - - assert not db.exists(b'key-1') - assert not db.exists(b'key-2') - assert b'key-1' not in db - assert b'key-2' not in db - - -def test_database_api_missing_key_retrieval(db): - assert db.get(b'does-not-exist') is None - - with pytest.raises(KeyError): - db[b'does-not-exist'] - - -def test_database_api_missing_key_for_deletion(db): - db.delete(b'does-not-exist') - - with pytest.raises(KeyError): - del db[b'does-not-exist'] diff --git a/tests/database/test_database_api.py b/tests/database/test_database_api.py new file mode 100644 index 0000000000..7a6b459257 --- /dev/null +++ b/tests/database/test_database_api.py @@ -0,0 +1,31 @@ +import pytest +from eth.db.backends.memory import MemoryDB +from eth.db.journal import JournalDB +from eth.db.batch import BatchDB +from eth.db.atomic import AtomicDB +from eth.db.cache import CacheDB + +from eth.tools.db.base import DatabaseAPITestSuite + + +@pytest.fixture(params=[JournalDB, BatchDB, MemoryDB, AtomicDB, CacheDB]) +def db(request): + base_db = MemoryDB() + if request.param is JournalDB: + yield JournalDB(base_db) + elif request.param is BatchDB: + yield BatchDB(base_db) + elif request.param is MemoryDB: + yield base_db + elif request.param is AtomicDB: + atomic_db = AtomicDB(base_db) + with atomic_db.atomic_batch() as batch: + yield batch + elif request.param is CacheDB: + yield CacheDB(base_db) + else: + raise Exception("Invariant") + + +class TestDatabaseAPI(DatabaseAPITestSuite): + pass