From 1bec75a88b70b90582bacaef3c2f15e12da9e6b1 Mon Sep 17 00:00:00 2001 From: William Barnhart Date: Tue, 26 Jul 2022 14:39:45 -0400 Subject: [PATCH 1/5] allow 'read_only' to be passed through options --- faust/stores/rocksdb.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/faust/stores/rocksdb.py b/faust/stores/rocksdb.py index 3181951b2..ed9ae98aa 100644 --- a/faust/stores/rocksdb.py +++ b/faust/stores/rocksdb.py @@ -161,6 +161,7 @@ def __init__( *, key_index_size: Optional[int] = None, options: Optional[Mapping[str, Any]] = None, + read_only: Optional[bool] = False, **kwargs: Any, ) -> None: if rocksdb is None: @@ -177,6 +178,10 @@ def __init__( if not self.url.path: self.url /= self.table_name self.options = options or {} + self.read_only = read_only + if "read_only" in self.options: + self.read_only = self.options.get("read_only") + del self.options["read_only"] self.rocksdb_options = RocksDBOptions(**self.options) if key_index_size is None: key_index_size = app.conf.table_key_index_size @@ -364,7 +369,9 @@ def _db_for_partition(self, partition: int) -> DB: return db def _open_for_partition(self, partition: int) -> DB: - return self.rocksdb_options.open(self.partition_path(partition)) + return self.rocksdb_options.open( + self.partition_path(partition), read_only=self.read_only + ) def _get(self, key: bytes) -> Optional[bytes]: event = current_event() From c0b1f493b1e33d4531932e856e5c527e21725de0 Mon Sep 17 00:00:00 2001 From: William Barnhart Date: Tue, 26 Jul 2022 14:48:17 -0400 Subject: [PATCH 2/5] only apply read_only if the file doesn't already exist --- faust/stores/rocksdb.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/faust/stores/rocksdb.py b/faust/stores/rocksdb.py index ed9ae98aa..eee13a049 100644 --- a/faust/stores/rocksdb.py +++ b/faust/stores/rocksdb.py @@ -369,8 +369,9 @@ def _db_for_partition(self, partition: int) -> DB: return db def _open_for_partition(self, partition: int) -> DB: + path = self.partition_path(partition) return self.rocksdb_options.open( - self.partition_path(partition), read_only=self.read_only + path, read_only=self.read_only if os.path.isfile(path) else False ) def _get(self, key: bytes) -> Optional[bytes]: From eaa08eded24e58951dbd8706bd60ff9f381f7bcc Mon Sep 17 00:00:00 2001 From: William Barnhart Date: Tue, 26 Jul 2022 15:31:32 -0400 Subject: [PATCH 3/5] pass read_only kwarg to test to fix test bug --- tests/unit/stores/test_rocksdb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/stores/test_rocksdb.py b/tests/unit/stores/test_rocksdb.py index bde79685b..a59eb54f2 100644 --- a/tests/unit/stores/test_rocksdb.py +++ b/tests/unit/stores/test_rocksdb.py @@ -236,7 +236,7 @@ def test_db_for_partition(self, *, store): def test_open_for_partition(self, *, store): open = store.rocksdb_options.open = Mock(name="options.open") assert store._open_for_partition(1) is open.return_value - open.assert_called_once_with(store.partition_path(1)) + open.assert_called_once_with(store.partition_path(1), read_only=False) def test__get__missing(self, *, store): store._get_bucket_for_key = Mock(name="get_bucket_for_key") From 7162716f32cf32ad9abee7ea9d40364bc6c0b297 Mon Sep 17 00:00:00 2001 From: William Barnhart Date: Tue, 26 Jul 2022 15:50:15 -0400 Subject: [PATCH 4/5] call options.pop for brevity --- faust/stores/rocksdb.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/faust/stores/rocksdb.py b/faust/stores/rocksdb.py index eee13a049..28d1fa06e 100644 --- a/faust/stores/rocksdb.py +++ b/faust/stores/rocksdb.py @@ -178,10 +178,7 @@ def __init__( if not self.url.path: self.url /= self.table_name self.options = options or {} - self.read_only = read_only - if "read_only" in self.options: - self.read_only = self.options.get("read_only") - del self.options["read_only"] + self.read_only = self.options.pop("read_only", read_only) self.rocksdb_options = RocksDBOptions(**self.options) if key_index_size is None: key_index_size = app.conf.table_key_index_size From 79081f1aa3c27c9318e2a00ecf3ea42dbc89695a Mon Sep 17 00:00:00 2001 From: William Barnhart Date: Wed, 10 Aug 2022 13:21:08 -0400 Subject: [PATCH 5/5] add note to docs for read_only option --- faust/stores/rocksdb.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/faust/stores/rocksdb.py b/faust/stores/rocksdb.py index 28d1fa06e..9d7b06d7c 100644 --- a/faust/stores/rocksdb.py +++ b/faust/stores/rocksdb.py @@ -138,7 +138,10 @@ def as_options(self) -> Options: class Store(base.SerializedStore): - """RocksDB table storage.""" + """RocksDB table storage. + Pass 'options={'read_only': True}' as an option into a Table class + to allow a RocksDB store be used by multiple apps. + """ offset_key = b"__faust\0offset__"