From 397220505773152566d113b7fbdc25646229eb40 Mon Sep 17 00:00:00 2001 From: Arne de Laat Date: Thu, 6 Aug 2020 10:27:34 +0200 Subject: [PATCH 01/13] Remove MMFiles only feature isVolatile --- arango/database.py | 7 ------- tests/test_collection.py | 1 - 2 files changed, 8 deletions(-) diff --git a/arango/database.py b/arango/database.py index 9d289620..bcbc3ee7 100644 --- a/arango/database.py +++ b/arango/database.py @@ -952,7 +952,6 @@ def create_collection(self, system=False, journal_size=None, edge=False, - volatile=False, user_keys=True, key_increment=None, key_offset=None, @@ -984,11 +983,6 @@ def create_collection(self, :type journal_size: int :param edge: If set to True, an edge collection is created. :type edge: bool - :param volatile: If set to True, collection data is kept in-memory only - and not made persistent. Unloading the collection will cause the - collection data to be discarded. Stopping or re-starting the server - will also cause full loss of data. - :type volatile: bool :param key_generator: Used for generating document keys. Allowed values are "traditional" or "autoincrement". :type key_generator: str | unicode @@ -1072,7 +1066,6 @@ def create_collection(self, 'waitForSync': sync, 'doCompact': compact, 'isSystem': system, - 'isVolatile': volatile, 'keyOptions': key_options, 'type': 3 if edge else 2 } diff --git a/tests/test_collection.py b/tests/test_collection.py index db2b8240..f8561f66 100644 --- a/tests/test_collection.py +++ b/tests/test_collection.py @@ -154,7 +154,6 @@ def test_collection_management(db, bad_db, cluster): compact=False, journal_size=7774208, system=False, - volatile=False, key_generator='traditional', user_keys=False, key_increment=9, From 9b461d251d69988a8b0f7f4364facb5967aa6686 Mon Sep 17 00:00:00 2001 From: Arne de Laat Date: Thu, 6 Aug 2020 11:50:45 +0200 Subject: [PATCH 02/13] Remove MMFiles only feature indexBuckets --- arango/database.py | 11 ----------- tests/test_collection.py | 1 - 2 files changed, 12 deletions(-) diff --git a/arango/database.py b/arango/database.py index bcbc3ee7..a8b3f176 100644 --- a/arango/database.py +++ b/arango/database.py @@ -958,7 +958,6 @@ def create_collection(self, key_generator='traditional', shard_fields=None, shard_count=None, - index_bucket_count=None, replication_factor=None, shard_like=None, sync_replication=None, @@ -1000,14 +999,6 @@ def create_collection(self, :type shard_fields: [str | unicode] :param shard_count: Number of shards to create. :type shard_count: int - :param index_bucket_count: Number of buckets into which indexes using - hash tables are split. The default is 16, and this number has to be - a power of 2 and less than or equal to 1024. For large collections, - one should increase this to avoid long pauses when the hash table - has to be initially built or re-sized, since buckets are re-sized - individually and can be initially built in parallel. For instance, - 64 may be a sensible value for 100 million documents. - :type index_bucket_count: int :param replication_factor: Number of copies of each shard on different servers in a cluster. Allowed values are 1 (only one copy is kept and no synchronous replication), and n (n-1 replicas are kept and @@ -1075,8 +1066,6 @@ def create_collection(self, data['numberOfShards'] = shard_count if shard_fields is not None: data['shardKeys'] = shard_fields - if index_bucket_count is not None: - data['indexBuckets'] = index_bucket_count if replication_factor is not None: data['replicationFactor'] = replication_factor if shard_like is not None: diff --git a/tests/test_collection.py b/tests/test_collection.py index f8561f66..c3615324 100644 --- a/tests/test_collection.py +++ b/tests/test_collection.py @@ -161,7 +161,6 @@ def test_collection_management(db, bad_db, cluster): edge=True, shard_count=2, shard_fields=['test_attr'], - index_bucket_count=10, replication_factor=1, shard_like='', sync_replication=False, From 1de3f24cde6115bb53533af7221ef28f84c25df3 Mon Sep 17 00:00:00 2001 From: Arne de Laat Date: Thu, 6 Aug 2020 12:33:59 +0200 Subject: [PATCH 03/13] Remove MMFiles only feature doCompact --- arango/database.py | 5 ----- tests/test_collection.py | 1 - 2 files changed, 6 deletions(-) diff --git a/arango/database.py b/arango/database.py index a8b3f176..1f7d568c 100644 --- a/arango/database.py +++ b/arango/database.py @@ -948,7 +948,6 @@ def response_handler(resp): def create_collection(self, name, sync=False, - compact=True, system=False, journal_size=None, edge=False, @@ -972,9 +971,6 @@ def create_collection(self, :param sync: If set to True, document operations via the collection will block until synchronized to disk by default. :type sync: bool - :param compact: If set to True, the collection is compacted. Applies - only to MMFiles storage engine. - :type compact: bool :param system: If set to True, a system collection is created. The collection name must have leading underscore "_" character. :type system: bool @@ -1055,7 +1051,6 @@ def create_collection(self, data = { 'name': name, 'waitForSync': sync, - 'doCompact': compact, 'isSystem': system, 'keyOptions': key_options, 'type': 3 if edge else 2 diff --git a/tests/test_collection.py b/tests/test_collection.py index c3615324..b3022d15 100644 --- a/tests/test_collection.py +++ b/tests/test_collection.py @@ -151,7 +151,6 @@ def test_collection_management(db, bad_db, cluster): col = db.create_collection( name=col_name, sync=True, - compact=False, journal_size=7774208, system=False, key_generator='traditional', From 8750ef7eabeb56ab062bad34e8bc3217ec8c3cab Mon Sep 17 00:00:00 2001 From: Arne de Laat Date: Thu, 6 Aug 2020 12:38:08 +0200 Subject: [PATCH 04/13] Remove MMFiles only feature journalSize Remove related journal rotation API method and error. --- arango/collection.py | 26 +------------------------- arango/database.py | 5 ----- arango/exceptions.py | 4 ---- docs/collection.rst | 2 +- tests/test_collection.py | 4 +--- 5 files changed, 3 insertions(+), 38 deletions(-) diff --git a/arango/collection.py b/arango/collection.py index 46c6837b..6784a643 100644 --- a/arango/collection.py +++ b/arango/collection.py @@ -15,7 +15,6 @@ CollectionResponsibleShardError, CollectionRenameError, CollectionRevisionError, - CollectionRotateJournalError, CollectionStatisticsError, CollectionTruncateError, CollectionUnloadError, @@ -307,13 +306,11 @@ def response_handler(resp): return self._execute(request, response_handler) - def configure(self, sync=None, journal_size=None): + def configure(self, sync=None): """Configure collection properties. :param sync: Block until operations are synchronized to disk. :type sync: bool - :param journal_size: Journal size in bytes. - :type journal_size: int :return: New collection properties. :rtype: dict :raise arango.exceptions.CollectionConfigureError: If operation fails. @@ -321,8 +318,6 @@ def configure(self, sync=None, journal_size=None): data = {} if sync is not None: data['waitForSync'] = sync - if journal_size is not None: - data['journalSize'] = journal_size request = Request( method='put', @@ -470,25 +465,6 @@ def response_handler(resp): return self._execute(request, response_handler) - def rotate(self): - """Rotate the collection journal. - - :return: True if collection journal was rotated successfully. - :rtype: bool - :raise arango.exceptions.CollectionRotateJournalError: If rotate fails. - """ - request = Request( - method='put', - endpoint='/_api/collection/{}/rotate'.format(self.name), - ) - - def response_handler(resp): - if not resp.is_success: - raise CollectionRotateJournalError(resp, request) - return True # pragma: no cover - - return self._execute(request, response_handler) - def truncate(self): """Delete all documents in the collection. diff --git a/arango/database.py b/arango/database.py index 1f7d568c..b90ef3ae 100644 --- a/arango/database.py +++ b/arango/database.py @@ -949,7 +949,6 @@ def create_collection(self, name, sync=False, system=False, - journal_size=None, edge=False, user_keys=True, key_increment=None, @@ -974,8 +973,6 @@ def create_collection(self, :param system: If set to True, a system collection is created. The collection name must have leading underscore "_" character. :type system: bool - :param journal_size: Max size of the journal in bytes. - :type journal_size: int :param edge: If set to True, an edge collection is created. :type edge: bool :param key_generator: Used for generating document keys. Allowed values @@ -1055,8 +1052,6 @@ def create_collection(self, 'keyOptions': key_options, 'type': 3 if edge else 2 } - if journal_size is not None: - data['journalSize'] = journal_size if shard_count is not None: data['numberOfShards'] = shard_count if shard_fields is not None: diff --git a/arango/exceptions.py b/arango/exceptions.py index cea5097f..bee50af3 100644 --- a/arango/exceptions.py +++ b/arango/exceptions.py @@ -245,10 +245,6 @@ class CollectionUnloadError(ArangoServerError): """Failed to unload collection.""" -class CollectionRotateJournalError(ArangoServerError): - """Failed to rotate collection journal.""" - - class CollectionRecalculateCountError(ArangoServerError): """Failed to recalculate document count.""" diff --git a/docs/collection.rst b/docs/collection.rst index fb5225a6..87d7f0f9 100644 --- a/docs/collection.rst +++ b/docs/collection.rst @@ -46,7 +46,7 @@ Here is an example showing how you can manage standard collections: students.load() students.unload() students.truncate() - students.configure(journal_size=3000000) + students.configure() # Delete the collection. db.delete_collection('students') diff --git a/tests/test_collection.py b/tests/test_collection.py index b3022d15..7a809977 100644 --- a/tests/test_collection.py +++ b/tests/test_collection.py @@ -45,7 +45,6 @@ def test_collection_misc_methods(col, bad_col, cluster): prev_sync = properties['sync'] properties = col.configure( sync=not prev_sync, - journal_size=10000000 ) assert properties['name'] == col.name assert properties['system'] is False @@ -53,7 +52,7 @@ def test_collection_misc_methods(col, bad_col, cluster): # Test configure properties with bad collection with assert_raises(CollectionConfigureError) as err: - bad_col.configure(sync=True, journal_size=10000000) + bad_col.configure(sync=True) assert err.value.error_code in {11, 1228} # Test get statistics @@ -151,7 +150,6 @@ def test_collection_management(db, bad_db, cluster): col = db.create_collection( name=col_name, sync=True, - journal_size=7774208, system=False, key_generator='traditional', user_keys=False, From 31837b9a2ce6f445332dcf1eff1767dbe8429096 Mon Sep 17 00:00:00 2001 From: Arne de Laat Date: Thu, 6 Aug 2020 12:38:52 +0200 Subject: [PATCH 05/13] Remove MMFiles only statistics keys --- arango/collection.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/arango/collection.py b/arango/collection.py index 6784a643..19d1f164 100644 --- a/arango/collection.py +++ b/arango/collection.py @@ -350,18 +350,6 @@ def response_handler(resp): raise CollectionStatisticsError(resp, request) stats = resp.body.get('figures', resp.body) - for f in ['compactors', 'datafiles', 'journals']: - if f in stats and 'fileSize' in stats[f]: # pragma: no cover - stats[f]['file_size'] = stats[f].pop('fileSize') - if 'compactionStatus' in stats: # pragma: no cover - status = stats.pop('compactionStatus') - if 'bytesRead' in status: - status['bytes_read'] = status.pop('bytesRead') - if 'bytesWritten' in status: - status['bytes_written'] = status.pop('bytesWritten') - if 'filesCombined' in status: - status['files_combined'] = status.pop('filesCombined') - stats['compaction_status'] = status if 'documentReferences' in stats: # pragma: no cover stats['document_refs'] = stats.pop('documentReferences') if 'lastTick' in stats: # pragma: no cover From e74b501a70bd9bd50dedab4b28c3cac5ca120b68 Mon Sep 17 00:00:00 2001 From: Arne de Laat Date: Thu, 6 Aug 2020 12:55:14 +0200 Subject: [PATCH 06/13] Remove MMFiles only options in replication --- arango/replication.py | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/arango/replication.py b/arango/replication.py index 6c7651a1..4b913891 100644 --- a/arango/replication.py +++ b/arango/replication.py @@ -149,12 +149,7 @@ def response_handler(resp): def dump(self, collection, batch_id=None, - lower=None, - upper=None, chunk_size=None, - include_system=None, - ticks=None, - flush=None, deserialize=False): """Return the events data of one collection. @@ -165,19 +160,6 @@ def dump(self, :type chunk_size: int :param batch_id: Batch ID. For RocksDB engine only. :type batch_id: str | unicode - :param lower: Lower bound tick value for results. For MMFiles only. - :type lower: str | unicode - :param upper: Upper bound tick value for results. For MMFiles only. - :type upper: str | unicode - :param include_system: Include system collections in the result. For - MMFiles only. Default value is True. - :type include_system: bool - :param ticks: Whether to include tick values in the dump. For MMFiles - only. Default value is True. - :type ticks: bool - :param flush: Whether to flush the WAL before dumping. Default value is - True. - :type flush: bool :param deserialize: Deserialize the response content. Default is False. :type deserialize: bool :return: Collection events data. @@ -190,16 +172,6 @@ def dump(self, params['chunkSize'] = chunk_size if batch_id is not None: params['batchId'] = batch_id - if lower is not None: - params['from'] = lower - if upper is not None: - params['to'] = upper - if include_system is not None: - params['includeSystem'] = include_system - if ticks is not None: - params['ticks'] = ticks - if flush is not None: - params['flush '] = flush request = Request( method='get', From 76e08f88663ff42314115d37e654e5e8dde035b7 Mon Sep 17 00:00:00 2001 From: Arne de Laat Date: Thu, 6 Aug 2020 12:55:40 +0200 Subject: [PATCH 07/13] Remove MMFiles as expected engine --- tests/test_database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_database.py b/tests/test_database.py index e101c590..28334ff6 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -210,7 +210,7 @@ def test_database_misc_methods(sys_db, db, bad_db): # Test get storage engine engine = db.engine() - assert engine['name'] in ['mmfiles', 'rocksdb'] + assert engine['name'] in ['rocksdb'] assert 'supports' in engine # Test get storage engine with bad database From c1f341aee94e15140bf52c1a3604102bf979bd2c Mon Sep 17 00:00:00 2001 From: Arne de Laat Date: Thu, 6 Aug 2020 12:55:56 +0200 Subject: [PATCH 08/13] Remove reference to MMFiles in docstring --- arango/wal.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/arango/wal.py b/arango/wal.py index 9bdba7a6..87ee20f5 100644 --- a/arango/wal.py +++ b/arango/wal.py @@ -246,8 +246,7 @@ def tail(self, :param client_info: Short description of the client, used for informative purposes only. :type client_info: str | unicode - :param barrier_id: ID of barrier used to keep WAL entries around. Only - required for the MMFiles storage engine. + :param barrier_id: ID of barrier used to keep WAL entries around. :type barrier_id: int :param deserialize: Deserialize the response content. Default is False. :type deserialize: bool From 78db6280fc85fe7953d85ada36acdc4ab90c5194 Mon Sep 17 00:00:00 2001 From: Arne de Laat Date: Thu, 6 Aug 2020 12:57:14 +0200 Subject: [PATCH 09/13] Remove references to RocksDB in docstrings No reason to specifically mention the only possible engine. --- arango/aql.py | 9 +++------ arango/database.py | 15 +++++---------- arango/executor.py | 3 +-- arango/replication.py | 4 ++-- 4 files changed, 11 insertions(+), 20 deletions(-) diff --git a/arango/aql.py b/arango/aql.py index 9f745423..47e74d75 100644 --- a/arango/aql.py +++ b/arango/aql.py @@ -183,18 +183,15 @@ def execute(self, :param profile: Return additional profiling details in the cursor, unless the query cache is used. :type profile: bool - :param max_transaction_size: Transaction size limit in bytes. Applies - only to RocksDB storage engine. + :param max_transaction_size: Transaction size limit in bytes. :type max_transaction_size: int :param max_warning_count: Max number of warnings returned. :type max_warning_count: int :param intermediate_commit_count: Max number of operations after - which an intermediate commit is performed automatically. Applies - only to RocksDB storage engine. + which an intermediate commit is performed automatically. :type intermediate_commit_count: int :param intermediate_commit_size: Max size of operations in bytes after - which an intermediate commit is performed automatically. Applies - only to RocksDB storage engine. + which an intermediate commit is performed automatically. :type intermediate_commit_size: int :param satellite_sync_wait: Number of seconds in which the server must synchronize the satellite collections involved in the query. When diff --git a/arango/database.py b/arango/database.py index b90ef3ae..5e8ba249 100644 --- a/arango/database.py +++ b/arango/database.py @@ -238,20 +238,17 @@ def execute_transaction(self, ArangoDB server waits indefinitely. If not set, system default value is used. :type timeout: int - :param max_size: Max transaction size limit in bytes. Applies only - to RocksDB storage engine. + :param max_size: Max transaction size limit in bytes. :type max_size: int :param allow_implicit: If set to True, undeclared read collections are loaded lazily. If set to False, transaction fails on any undeclared collections. :type allow_implicit: bool :param intermediate_commit_count: Max number of operations after which - an intermediate commit is performed automatically. Applies only to - RocksDB storage engine. + an intermediate commit is performed automatically. :type intermediate_commit_count: int :param intermediate_commit_size: Max size of operations in bytes after - which an intermediate commit is performed automatically. Applies - only to RocksDB storage engine. + which an intermediate commit is performed automatically. :type intermediate_commit_size: int :return: Return value of **command**. :rtype: str | unicode @@ -2490,8 +2487,7 @@ def begin_transaction(self, given, a default value is used. Setting it to 0 disables the timeout. :type lock_timeout: int - :param max_size: Max transaction size in bytes. Applicable to RocksDB - storage engine only. + :param max_size: Max transaction size in bytes. :type max_size: :return: Database API wrapper object specifically for transactions. :rtype: arango.database.TransactionDatabase @@ -2612,8 +2608,7 @@ class TransactionDatabase(Database): :param lock_timeout: Timeout for waiting on collection locks. If not given, a default value is used. Setting it to 0 disables the timeout. :type lock_timeout: int - :param max_size: Max transaction size in bytes. Applicable to RocksDB - storage engine only. + :param max_size: Max transaction size in bytes. :type max_size: int """ diff --git a/arango/executor.py b/arango/executor.py index 6c9b2033..4733fac3 100644 --- a/arango/executor.py +++ b/arango/executor.py @@ -302,8 +302,7 @@ class TransactionExecutor(Executor): :param lock_timeout: Timeout for waiting on collection locks. If not given, a default value is used. Setting it to 0 disables the timeout. :type lock_timeout: int - :param max_size: Max transaction size in bytes. Applicable to RocksDB - storage engine only. + :param max_size: Max transaction size in bytes. :type max_size: int """ context = 'transaction' diff --git a/arango/replication.py b/arango/replication.py index 4b913891..dc8947ff 100644 --- a/arango/replication.py +++ b/arango/replication.py @@ -40,7 +40,7 @@ def __init__(self, connection, executor): def inventory(self, batch_id, include_system=None, all_databases=None): """Return an overview of collections and indexes. - :param batch_id: Batch ID. For RocksDB engine only. + :param batch_id: Batch ID. :type batch_id: str | unicode :param include_system: Include system collections in the result. Default value is True. @@ -158,7 +158,7 @@ def dump(self, :param chunk_size: Size of the result in bytes. This value is honored approximately only. :type chunk_size: int - :param batch_id: Batch ID. For RocksDB engine only. + :param batch_id: Batch ID. :type batch_id: str | unicode :param deserialize: Deserialize the response content. Default is False. :type deserialize: bool From decba8b4a0f8996214686eb1f5be159840dfe27d Mon Sep 17 00:00:00 2001 From: Arne de Laat Date: Thu, 6 Aug 2020 12:59:31 +0200 Subject: [PATCH 10/13] Remove MMFiles reference in docstring --- arango/aql.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/arango/aql.py b/arango/aql.py index 47e74d75..ad6c67cf 100644 --- a/arango/aql.py +++ b/arango/aql.py @@ -211,11 +211,10 @@ def execute(self, entirety. Results are either returned right away (if the result set is small enough), or stored server-side and accessible via cursors (while respecting the ttl). You should use this parameter only for - short-running queries or without exclusive locks (write-locks on - MMFiles). Note: parameters **cache**, **count** and **full_count** - do not work for streaming queries. Query statistics, warnings and - profiling data are made available only after the query is finished. - Default value is False. + short-running queries or without exclusive locks. Note: parameters + **cache**, **count** and **full_count** do not work for streaming + queries. Query statistics, warnings and profiling data are made + available only after the query is finished. Default value is False. :type stream: bool :param skip_inaccessible_cols: If set to True, collections without user access are skipped, and query executes normally instead of raising From e75c5dcb94dd836c2206cbf8fb040928565d393e Mon Sep 17 00:00:00 2001 From: Arne de Laat Date: Thu, 6 Aug 2020 13:07:38 +0200 Subject: [PATCH 11/13] Remove MMFile only journal rotation test --- tests/test_collection.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/tests/test_collection.py b/tests/test_collection.py index 7a809977..1544ca27 100644 --- a/tests/test_collection.py +++ b/tests/test_collection.py @@ -10,7 +10,6 @@ CollectionPropertiesError, CollectionRenameError, CollectionRevisionError, - CollectionRotateJournalError, CollectionStatisticsError, CollectionTruncateError, CollectionUnloadError, @@ -89,17 +88,6 @@ def test_collection_misc_methods(col, bad_col, cluster): bad_col.unload() assert err.value.error_code in {11, 1228} - # Test rotate journal - try: - assert isinstance(col.rotate(), bool) - except CollectionRotateJournalError as err: - assert err.error_code in {404, 1105} - - # Test rotate journal with bad collection - with assert_raises(CollectionRotateJournalError) as err: - bad_col.rotate() - assert err.value.error_code in {11, 1228} - if cluster: col.insert({}) else: From 31d930da000e2c9d8dc61fb80bc207e9bf19444a Mon Sep 17 00:00:00 2001 From: Arne de Laat Date: Thu, 6 Aug 2020 13:09:22 +0200 Subject: [PATCH 12/13] Remove MMFiles only collection properties --- arango/formatter.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/arango/formatter.py b/arango/formatter.py index 99da8a9b..f2853a8e 100644 --- a/arango/formatter.py +++ b/arango/formatter.py @@ -145,16 +145,6 @@ def format_collection(body): # pragma: no cover if 'writeConcern' in body: result['write_concern'] = body['writeConcern'] - # MMFiles only - if 'doCompact' in body: - result['compact'] = body['doCompact'] - if 'journalSize' in body: - result['journal_size'] = body['journalSize'] - if 'isVolatile' in body: - result['volatile'] = body['isVolatile'] - if 'indexBuckets' in body: - result['index_bucket_count'] = body['indexBuckets'] - # Cluster only if 'shards' in body: result['shards'] = body['shards'] From 2799c9c00b2703aeffb2e97af710ef5d1e2fac7d Mon Sep 17 00:00:00 2001 From: Arne de Laat Date: Thu, 6 Aug 2020 13:18:34 +0200 Subject: [PATCH 13/13] Remove MMFiles only options from replication dump test --- tests/test_replication.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/test_replication.py b/tests/test_replication.py index a845c71e..947ac5b6 100644 --- a/tests/test_replication.py +++ b/tests/test_replication.py @@ -46,12 +46,7 @@ def test_replication_dump_methods(db, bad_db, col, docs, cluster): result = db.replication.dump( collection=col.name, batch_id=batch_id, - lower=0, - upper=1000000, chunk_size=0, - include_system=True, - ticks=0, - flush=True, deserialize=True ) assert 'content' in result