From 62d385afdf170a13569d1b39590e3c9cbff67678 Mon Sep 17 00:00:00 2001 From: Pavel Kirilin Date: Mon, 16 Mar 2026 15:50:45 +0100 Subject: [PATCH 1/8] Added some stream methods. --- python/natsrpy/_inner/js/__init__.pyi | 2 +- python/natsrpy/_inner/js/stream.pyi | 44 +++ python/natsrpy/js/stream.py | 17 ++ src/exceptions/rust_err.rs | 2 + src/js/consumers/mod.rs | 2 + src/js/consumers/pull.rs | 3 + src/js/consumers/push.rs | 3 + src/js/jetstream.rs | 22 +- src/js/kv.rs | 10 +- src/js/mod.rs | 14 +- src/js/stream.rs | 413 +++++++++++++++++++++++--- src/nats_cls.rs | 6 +- src/subscription.rs | 6 +- 13 files changed, 480 insertions(+), 64 deletions(-) create mode 100644 src/js/consumers/mod.rs create mode 100644 src/js/consumers/pull.rs create mode 100644 src/js/consumers/push.rs diff --git a/python/natsrpy/_inner/js/__init__.pyi b/python/natsrpy/_inner/js/__init__.pyi index 1593b3e..11300ec 100644 --- a/python/natsrpy/_inner/js/__init__.pyi +++ b/python/natsrpy/_inner/js/__init__.pyi @@ -20,4 +20,4 @@ class JetStream: async def create_stream(self, config: StreamConfig) -> Stream: ... async def update_stream(self, config: StreamConfig) -> Stream: ... async def get_stream(self, name: str) -> Stream: ... - async def delete_stream(self, name: str) -> Stream: ... + async def delete_stream(self, name: str) -> bool: ... diff --git a/python/natsrpy/_inner/js/stream.pyi b/python/natsrpy/_inner/js/stream.pyi index 003a26e..78dbc97 100644 --- a/python/natsrpy/_inner/js/stream.pyi +++ b/python/natsrpy/_inner/js/stream.pyi @@ -181,6 +181,50 @@ class StreamMessage: payload: bytes time: datetime +class StreamState: + messages: int + bytes: int + first_sequence: int + first_timestamp: int + last_sequence: int + last_timestamp: int + consumer_count: int + subjects_count: int + deleted_count: int | None + deleted: list[int] | None + +class SourceInfo: + name: str + lag: int + active: timedelta | None + filter_subject: str | None + subject_transform_dest: str | None + subject_transforms: list[SubjectTransform] + +class PeerInfo: + name: str + current: bool + active: timedelta + offline: bool + lag: int | None + +class ClusterInfo: + name: str | None + raft_group: str | None + leader: str | None + leader_since: int | None + system_account: bool + traffic_account: str | None + replicas: list[PeerInfo] + +class StreamInfo: + config: StreamConfig + created: float + state: StreamState + cluster: ClusterInfo | None + mirror: SourceInfo | None + sources: list[SourceInfo] + class Stream: async def direct_get(self, sequence: int) -> StreamMessage: """ diff --git a/python/natsrpy/js/stream.py b/python/natsrpy/js/stream.py index 01e0520..ce25616 100644 --- a/python/natsrpy/js/stream.py +++ b/python/natsrpy/js/stream.py @@ -13,6 +13,11 @@ StreamConfig, StreamMessage, SubjectTransform, + ClusterInfo, + PeerInfo, + SourceInfo, + StreamInfo, + StreamState, ) __all__ = [ @@ -30,4 +35,16 @@ "StreamConfig", "StreamMessage", "SubjectTransform", + "ClusterInfo", + "Compression", + "ConsumerLimits", + "DiscardPolicy", + "Stream", + "PeerInfo", + "PersistenceMode", + "RetentionPolicy", + "SourceInfo", + "StreamConfig", + "StreamInfo", + "StreamState", ] diff --git a/src/exceptions/rust_err.rs b/src/exceptions/rust_err.rs index 78659fb..36a02e2 100644 --- a/src/exceptions/rust_err.rs +++ b/src/exceptions/rust_err.rs @@ -54,6 +54,8 @@ pub enum NatsrpyError { GetStreamError(#[from] async_nats::jetstream::context::GetStreamError), #[error(transparent)] StreamDirectGetError(#[from] async_nats::jetstream::stream::DirectGetError), + #[error(transparent)] + StreamInfoError(#[from] async_nats::jetstream::stream::InfoError), } impl From for pyo3::PyErr { diff --git a/src/js/consumers/mod.rs b/src/js/consumers/mod.rs new file mode 100644 index 0000000..a09e332 --- /dev/null +++ b/src/js/consumers/mod.rs @@ -0,0 +1,2 @@ +pub mod pull; +pub mod push; diff --git a/src/js/consumers/pull.rs b/src/js/consumers/pull.rs new file mode 100644 index 0000000..845f036 --- /dev/null +++ b/src/js/consumers/pull.rs @@ -0,0 +1,3 @@ +#[pyo3::pyclass(from_py_object)] +#[derive(Debug, Clone)] +pub struct PullConsumer; diff --git a/src/js/consumers/push.rs b/src/js/consumers/push.rs new file mode 100644 index 0000000..cbfcba0 --- /dev/null +++ b/src/js/consumers/push.rs @@ -0,0 +1,3 @@ +#[pyo3::pyclass(from_py_object)] +#[derive(Debug, Clone)] +pub struct PushConsumer; diff --git a/src/js/jetstream.rs b/src/js/jetstream.rs index 45d6222..7d56fdd 100644 --- a/src/js/jetstream.rs +++ b/src/js/jetstream.rs @@ -2,7 +2,7 @@ use std::{ops::Deref, sync::Arc}; use async_nats::{Subject, client::traits::Publisher, connection::State}; use pyo3::{ - Bound, PyAny, Python, pyclass, pymethods, + Bound, PyAny, Python, types::{PyBytes, PyBytesMethods, PyDict}, }; use tokio::sync::RwLock; @@ -16,7 +16,7 @@ use crate::{ utils::{headers::NatsrpyHeadermapExt, natsrpy_future}, }; -#[pyclass] +#[pyo3::pyclass] pub struct JetStream { ctx: Arc>, } @@ -30,7 +30,7 @@ impl JetStream { } } -#[pymethods] +#[pyo3::pymethods] impl JetStream { #[pyo3(signature = ( subject, @@ -120,6 +120,18 @@ impl JetStream { }) } + pub fn get_stream<'py>( + &self, + py: Python<'py>, + name: String, + ) -> NatsrpyResult> { + let ctx = self.ctx.clone(); + natsrpy_future(py, async move { + let js = ctx.read().await; + Ok(super::stream::Stream::new(js.get_stream(name).await?)) + }) + } + pub fn create_stream<'py>( &self, py: Python<'py>, @@ -135,7 +147,7 @@ impl JetStream { }) } - pub fn get_stream<'py>( + pub fn delete_stream<'py>( &self, py: Python<'py>, name: String, @@ -143,7 +155,7 @@ impl JetStream { let ctx = self.ctx.clone(); natsrpy_future(py, async move { let js = ctx.read().await; - Ok(super::stream::Stream::new(js.get_stream(name).await?)) + Ok(js.delete_stream(name).await?.success) }) } } diff --git a/src/js/kv.rs b/src/js/kv.rs index 8a15bf5..b39cfd9 100644 --- a/src/js/kv.rs +++ b/src/js/kv.rs @@ -2,7 +2,7 @@ use std::{sync::Arc, time::Duration}; use crate::js; use pyo3::{ - Bound, PyAny, Python, pyclass, pymethods, + Bound, PyAny, Python, types::{PyBytes, PyBytesMethods}, }; use tokio::sync::RwLock; @@ -12,7 +12,7 @@ use crate::{ utils::natsrpy_future, }; -#[pyclass(from_py_object, get_all, set_all)] +#[pyo3::pyclass(from_py_object, get_all, set_all)] #[derive(Clone)] pub struct KVConfig { bucket: String, @@ -32,7 +32,7 @@ pub struct KVConfig { limit_markers: Option, } -#[pymethods] +#[pyo3::pymethods] impl KVConfig { #[new] #[pyo3(signature=( @@ -128,7 +128,7 @@ impl TryFrom for async_nats::jetstream::kv::Config { } } -#[pyclass(from_py_object)] +#[pyo3::pyclass(from_py_object)] #[derive(Clone)] pub struct KeyValue { #[pyo3(get)] @@ -159,7 +159,7 @@ impl KeyValue { } } -#[pymethods] +#[pyo3::pymethods] impl KeyValue { pub fn get<'py>(&self, py: Python<'py>, key: String) -> NatsrpyResult> { let store = self.store.clone(); diff --git a/src/js/mod.rs b/src/js/mod.rs index 53bdb67..9fddb36 100644 --- a/src/js/mod.rs +++ b/src/js/mod.rs @@ -1,17 +1,19 @@ +pub mod consumers; pub mod jetstream; pub mod kv; pub mod stream; #[pyo3::pymodule(submodule, name = "js")] pub mod pymod { + // Classes #[pymodule_export] - pub use super::jetstream::JetStream; - #[pymodule_export] - pub use super::kv::KVConfig; - - #[pymodule_export] - pub use super::kv::KeyValue; + pub use super::{ + consumers::{pull::PullConsumer, push::PushConsumer}, + jetstream::JetStream, + kv::{KVConfig, KeyValue}, + }; + // SubModules #[pymodule_export] pub use super::kv::pymod as kv; #[pymodule_export] diff --git a/src/js/stream.rs b/src/js/stream.rs index 4e43090..46adaff 100644 --- a/src/js/stream.rs +++ b/src/js/stream.rs @@ -1,18 +1,19 @@ use pyo3::{ Py, - types::{PyBytes, PyDateTime, PyTzInfo}, + types::{PyBytes, PyDateTime, PyDict, PyTzInfo}, }; use std::{collections::HashMap, ops::Deref, sync::Arc, time::Duration}; -use tokio::sync::RwLock; use crate::{ exceptions::rust_err::{NatsrpyError, NatsrpyResult}, - utils::{headers::NatsrpyHeadermapExt, natsrpy_future}, + utils::headers::NatsrpyHeadermapExt, + utils::natsrpy_future, }; -use pyo3::{Bound, PyAny, Python, pyclass, pymethods, types::PyDict}; +use pyo3::{Bound, PyAny, Python}; +use tokio::sync::RwLock; -#[pyclass(from_py_object)] -#[derive(Clone, Copy, Default, PartialEq, Eq)] +#[pyo3::pyclass(from_py_object)] +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub enum StorageType { #[default] FILE, @@ -28,8 +29,17 @@ impl From for async_nats::jetstream::stream::StorageType { } } -#[pyclass(from_py_object)] -#[derive(Clone, Copy, Default, PartialEq, Eq)] +impl From for StorageType { + fn from(value: async_nats::jetstream::stream::StorageType) -> Self { + match value { + async_nats::jetstream::stream::StorageType::File => Self::FILE, + async_nats::jetstream::stream::StorageType::Memory => Self::MEMORY, + } + } +} + +#[pyo3::pyclass(from_py_object)] +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub enum DiscardPolicy { #[default] OLD, @@ -45,8 +55,17 @@ impl From for async_nats::jetstream::stream::DiscardPolicy { } } -#[pyclass(from_py_object)] -#[derive(Clone, Copy, Default, PartialEq, Eq)] +impl From for DiscardPolicy { + fn from(value: async_nats::jetstream::stream::DiscardPolicy) -> Self { + match value { + async_nats::jetstream::stream::DiscardPolicy::Old => Self::OLD, + async_nats::jetstream::stream::DiscardPolicy::New => Self::NEW, + } + } +} + +#[pyo3::pyclass(from_py_object)] +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub enum RetentionPolicy { #[default] LIMITS, @@ -64,8 +83,18 @@ impl From for async_nats::jetstream::stream::RetentionPolicy { } } -#[pyclass(from_py_object)] -#[derive(Clone, Copy, PartialEq, Eq)] +impl From for RetentionPolicy { + fn from(value: async_nats::jetstream::stream::RetentionPolicy) -> Self { + match value { + async_nats::jetstream::stream::RetentionPolicy::Limits => Self::LIMITS, + async_nats::jetstream::stream::RetentionPolicy::Interest => Self::INTEREST, + async_nats::jetstream::stream::RetentionPolicy::WorkQueue => Self::WORKQUEUE, + } + } +} + +#[pyo3::pyclass(from_py_object)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Compression { S2, NONE, @@ -80,8 +109,17 @@ impl From for async_nats::jetstream::stream::Compression { } } -#[pyclass(from_py_object)] -#[derive(Default, Debug, Clone, Copy, PartialEq, Eq)] +impl From for Compression { + fn from(value: async_nats::jetstream::stream::Compression) -> Self { + match value { + async_nats::jetstream::stream::Compression::S2 => Self::S2, + async_nats::jetstream::stream::Compression::None => Self::NONE, + } + } +} + +#[pyo3::pyclass(from_py_object)] +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] pub enum PersistenceMode { #[default] Default, @@ -97,14 +135,23 @@ impl From for async_nats::jetstream::stream::PersistenceMode { } } -#[pyclass(from_py_object)] -#[derive(Clone, Debug, PartialEq, Eq, Default)] +impl From for PersistenceMode { + fn from(value: async_nats::jetstream::stream::PersistenceMode) -> Self { + match value { + async_nats::jetstream::stream::PersistenceMode::Default => Self::Default, + async_nats::jetstream::stream::PersistenceMode::Async => Self::Async, + } + } +} + +#[pyo3::pyclass(from_py_object)] +#[derive(Debug, Clone, PartialEq, Eq, Default)] pub struct ConsumerLimits { pub inactive_threshold: Duration, pub max_ack_pending: i64, } -#[pymethods] +#[pyo3::pymethods] impl ConsumerLimits { #[new] #[must_use] @@ -125,15 +172,24 @@ impl From for async_nats::jetstream::stream::ConsumerLimits { } } -#[pyclass(from_py_object, get_all, set_all)] -#[derive(Clone)] +impl From for ConsumerLimits { + fn from(value: async_nats::jetstream::stream::ConsumerLimits) -> Self { + Self { + inactive_threshold: value.inactive_threshold, + max_ack_pending: value.max_ack_pending, + } + } +} + +#[pyo3::pyclass(from_py_object, get_all, set_all)] +#[derive(Debug, Clone)] pub struct Republish { pub source: String, pub destination: String, pub headers_only: bool, } -#[pymethods] +#[pyo3::pymethods] impl Republish { #[new] #[must_use] @@ -156,14 +212,24 @@ impl From for async_nats::jetstream::stream::Republish { } } -#[pyclass(from_py_object, get_all, set_all)] -#[derive(Clone)] +impl From for Republish { + fn from(value: async_nats::jetstream::stream::Republish) -> Self { + Self { + source: value.source, + destination: value.destination, + headers_only: value.headers_only, + } + } +} + +#[pyo3::pyclass(from_py_object, get_all, set_all)] +#[derive(Debug, Clone)] pub struct External { pub api_prefix: String, pub delivery_prefix: Option, } -#[pymethods] +#[pyo3::pymethods] impl External { #[new] #[pyo3(signature = (api_prefix, delivery_prefix=None))] @@ -185,8 +251,17 @@ impl From<&External> for async_nats::jetstream::stream::External { } } -#[pyclass(from_py_object, get_all, set_all)] -#[derive(Clone)] +impl From for External { + fn from(value: async_nats::jetstream::stream::External) -> Self { + Self { + api_prefix: value.api_prefix, + delivery_prefix: value.delivery_prefix, + } + } +} + +#[pyo3::pyclass(from_py_object, get_all, set_all)] +#[derive(Debug, Clone)] pub struct SubjectTransform { pub source: String, pub destination: String, @@ -201,8 +276,17 @@ impl From for async_nats::jetstream::stream::SubjectTransform } } -#[pyclass(from_py_object, get_all, set_all)] -#[derive(Clone)] +impl From for SubjectTransform { + fn from(value: async_nats::jetstream::stream::SubjectTransform) -> Self { + Self { + source: value.source, + destination: value.destination, + } + } +} + +#[pyo3::pyclass(from_py_object, get_all, set_all)] +#[derive(Debug, Clone)] pub struct Source { pub name: String, pub filter_subject: Option, @@ -236,7 +320,25 @@ impl TryFrom for async_nats::jetstream::stream::Source { } } -#[pymethods] +impl From for Source { + fn from(value: async_nats::jetstream::stream::Source) -> Self { + Self { + name: value.name, + filter_subject: value.filter_subject, + external: value.external.map(std::convert::Into::into), + start_sequence: value.start_sequence, + start_time: value.start_time.map(time::OffsetDateTime::unix_timestamp), + domain: value.domain, + subject_transforms: value + .subject_transforms + .into_iter() + .map(Into::into) + .collect(), + } + } +} + +#[pyo3::pymethods] impl Source { #[new] #[pyo3(signature = ( @@ -272,14 +374,14 @@ impl Source { } } -#[pyclass(from_py_object, get_all, set_all)] -#[derive(Clone)] +#[pyo3::pyclass(from_py_object, get_all, set_all)] +#[derive(Debug, Clone)] pub struct Placement { pub cluster: Option, pub tags: Vec, } -#[pymethods] +#[pyo3::pymethods] impl Placement { #[new] #[pyo3(signature=(cluster=None, tags=None))] @@ -301,8 +403,95 @@ impl From for async_nats::jetstream::stream::Placement { } } -#[pyclass(from_py_object, get_all, set_all)] -#[derive(Clone)] +impl From for Placement { + fn from(value: async_nats::jetstream::stream::Placement) -> Self { + Self { + cluster: value.cluster, + tags: value.tags, + } + } +} + +#[pyo3::pyclass(from_py_object, get_all)] +#[derive(Debug, Clone)] +pub struct PeerInfo { + pub name: String, + pub current: bool, + pub active: Duration, + pub offline: bool, + pub lag: Option, +} + +impl From for async_nats::jetstream::stream::PeerInfo { + fn from(value: PeerInfo) -> Self { + Self { + name: value.name, + current: value.current, + active: value.active, + offline: value.offline, + lag: value.lag, + } + } +} + +impl From for PeerInfo { + fn from(value: async_nats::jetstream::stream::PeerInfo) -> Self { + Self { + name: value.name, + current: value.current, + active: value.active, + offline: value.offline, + lag: value.lag, + } + } +} + +#[pyo3::pyclass(from_py_object, get_all)] +#[derive(Debug, Clone)] +pub struct ClusterInfo { + pub name: Option, + pub raft_group: Option, + pub leader: Option, + pub leader_since: Option, + pub system_account: bool, + pub traffic_account: Option, + pub replicas: Vec, +} + +impl TryFrom for async_nats::jetstream::stream::ClusterInfo { + type Error = NatsrpyError; + fn try_from(value: ClusterInfo) -> Result { + Ok(Self { + name: value.name, + raft_group: value.raft_group, + leader: value.leader, + leader_since: value + .leader_since + .map(time::OffsetDateTime::from_unix_timestamp) + .transpose()?, + system_account: value.system_account, + traffic_account: value.traffic_account, + replicas: value.replicas.into_iter().map(Into::into).collect(), + }) + } +} + +impl From for ClusterInfo { + fn from(value: async_nats::jetstream::stream::ClusterInfo) -> Self { + Self { + name: value.name, + raft_group: value.raft_group, + leader: value.leader, + leader_since: value.leader_since.map(time::OffsetDateTime::unix_timestamp), + system_account: value.system_account, + traffic_account: value.traffic_account, + replicas: value.replicas.into_iter().map(Into::into).collect(), + } + } +} + +#[pyo3::pyclass(from_py_object, get_all, set_all)] +#[derive(Debug, Clone)] pub struct StreamConfig { pub name: String, pub subjects: Vec, @@ -345,7 +534,7 @@ pub struct StreamConfig { pub allow_message_counter: Option, } -#[pymethods] +#[pyo3::pymethods] impl StreamConfig { #[new] #[pyo3(signature=( @@ -474,6 +663,56 @@ impl StreamConfig { } } +impl TryFrom for StreamConfig { + type Error = NatsrpyError; + + fn try_from(value: async_nats::jetstream::stream::Config) -> Result { + Ok(Self { + name: value.name, + subjects: value.subjects, + max_bytes: Some(value.max_bytes), + max_messages: Some(value.max_messages), + max_messages_per_subject: Some(value.max_messages_per_subject), + discard: Some(value.discard.into()), + discard_new_per_subject: Some(value.discard_new_per_subject), + retention: Some(value.retention.into()), + max_consumers: Some(value.max_consumers), + max_age: Some(value.max_age), + max_message_size: Some(value.max_message_size), + storage: Some(value.storage.into()), + num_replicas: Some(value.num_replicas), + no_ack: Some(value.no_ack), + duplicate_window: Some(value.duplicate_window), + template_owner: Some(value.template_owner), + sealed: Some(value.sealed), + description: value.description, + allow_rollup: Some(value.allow_rollup), + deny_delete: Some(value.deny_delete), + deny_purge: Some(value.deny_purge), + republish: value.republish.map(Into::into), + allow_direct: Some(value.allow_direct), + mirror_direct: Some(value.mirror_direct), + mirror: value.mirror.map(Into::into), + sources: value + .sources + .map(|val| val.into_iter().map(Into::into).collect()), + metadata: Some(value.metadata), + subject_transform: value.subject_transform.map(Into::into), + compression: value.compression.map(Into::into), + consumer_limits: value.consumer_limits.map(Into::into), + first_sequence: value.first_sequence, + placement: value.placement.map(Into::into), + persist_mode: value.persist_mode.map(Into::into), + pause_until: value.pause_until.map(time::OffsetDateTime::unix_timestamp), + allow_message_ttl: Some(value.allow_message_ttl), + subject_delete_marker_ttl: value.subject_delete_marker_ttl, + allow_atomic_publish: Some(value.allow_atomic_publish), + allow_message_schedules: Some(value.allow_message_schedules), + allow_message_counter: Some(value.allow_message_counter), + }) + } +} + impl TryFrom for async_nats::jetstream::stream::Config { type Error = NatsrpyError; @@ -552,7 +791,7 @@ impl TryFrom for async_nats::jetstream::stream::Config { } } -#[pyclass(get_all)] +#[pyo3::pyclass(get_all)] #[derive(Debug)] pub struct StreamMessage { pub subject: String, @@ -590,7 +829,7 @@ impl StreamMessage { } } -#[pymethods] +#[pyo3::pymethods] impl StreamMessage { #[must_use] pub fn __repr__(&self) -> String { @@ -604,7 +843,92 @@ impl StreamMessage { } } -#[pyclass(from_py_object)] +#[pyo3::pyclass(from_py_object, get_all)] +#[derive(Debug, Clone)] +pub struct StreamState { + pub messages: u64, + pub bytes: u64, + pub first_sequence: u64, + pub first_timestamp: i64, + pub last_sequence: u64, + pub last_timestamp: i64, + pub consumer_count: usize, + pub subjects_count: u64, + pub deleted_count: Option, + pub deleted: Option>, +} + +impl From for StreamState { + fn from(value: async_nats::jetstream::stream::State) -> Self { + Self { + messages: value.messages, + bytes: value.bytes, + first_sequence: value.first_sequence, + first_timestamp: value.first_timestamp.unix_timestamp(), + last_sequence: value.last_sequence, + last_timestamp: value.last_timestamp.unix_timestamp(), + consumer_count: value.consumer_count, + subjects_count: value.subjects_count, + deleted_count: value.deleted_count, + deleted: value.deleted, + } + } +} + +#[pyo3::pyclass(from_py_object, get_all)] +#[derive(Debug, Clone)] +pub struct SourceInfo { + pub name: String, + pub lag: u64, + pub active: Option, + pub filter_subject: Option, + pub subject_transform_dest: Option, + pub subject_transforms: Vec, +} + +impl From for SourceInfo { + fn from(value: async_nats::jetstream::stream::SourceInfo) -> Self { + Self { + name: value.name, + lag: value.lag, + active: value.active, + filter_subject: value.filter_subject, + subject_transform_dest: value.subject_transform_dest, + subject_transforms: value + .subject_transforms + .into_iter() + .map(Into::into) + .collect(), + } + } +} + +#[pyo3::pyclass(from_py_object)] +#[derive(Debug, Clone)] +pub struct StreamInfo { + pub config: StreamConfig, + pub created: time::OffsetDateTime, + pub state: StreamState, + pub cluster: Option, + pub mirror: Option, + pub sources: Vec, +} + +impl TryFrom for StreamInfo { + type Error = NatsrpyError; + fn try_from(value: async_nats::jetstream::stream::Info) -> Result { + Ok(Self { + config: value.config.try_into()?, + created: value.created, + state: value.state.into(), + cluster: value.cluster.map(Into::into), + mirror: value.mirror.map(Into::into), + sources: value.sources.into_iter().map(Into::into).collect(), + }) + } +} + +#[pyo3::pyclass(from_py_object)] #[derive(Debug, Clone)] pub struct Stream { stream: Arc>>, @@ -620,7 +944,7 @@ impl Stream { } } -#[pymethods] +#[pyo3::pymethods] impl Stream { pub fn direct_get<'py>( &self, @@ -635,14 +959,21 @@ impl Stream { Ok(result) }) } + + pub fn get_info<'py>(&self, py: Python<'py>) -> NatsrpyResult> { + let ctx = self.stream.clone(); + natsrpy_future(py, async move { + StreamInfo::try_from(ctx.read().await.get_info().await?) + }) + } } #[pyo3::pymodule(submodule, name = "stream")] pub mod pymod { #[pymodule_export] pub use super::{ - Compression, ConsumerLimits, DiscardPolicy, External, PersistenceMode, Placement, - Republish, RetentionPolicy, Source, StorageType, Stream, StreamConfig, StreamMessage, - SubjectTransform, + ClusterInfo, Compression, ConsumerLimits, DiscardPolicy, External, PeerInfo, + PersistenceMode, Placement, Republish, RetentionPolicy, Source, SourceInfo, StorageType, + Stream, StreamConfig, StreamInfo, StreamMessage, StreamState, SubjectTransform, }; } diff --git a/src/nats_cls.rs b/src/nats_cls.rs index 254a493..3f113a1 100644 --- a/src/nats_cls.rs +++ b/src/nats_cls.rs @@ -1,6 +1,6 @@ use async_nats::{Subject, client::traits::Publisher, message::OutboundMessage}; use pyo3::{ - Bound, PyAny, PyResult, Python, pyclass, pymethods, + Bound, PyAny, PyResult, Python, types::{PyBytes, PyBytesMethods, PyDict}, }; use std::{sync::Arc, time::Duration}; @@ -12,7 +12,7 @@ use crate::{ utils::{headers::NatsrpyHeadermapExt, natsrpy_future}, }; -#[pyclass(name = "Nats")] +#[pyo3::pyclass(name = "Nats")] pub struct NatsCls { nats_session: Arc>>, addr: Vec, @@ -27,7 +27,7 @@ pub struct NatsCls { request_timeout: Option, } -#[pymethods] +#[pyo3::pymethods] impl NatsCls { #[new] #[pyo3(signature = ( diff --git a/src/subscription.rs b/src/subscription.rs index 473c821..3535f44 100644 --- a/src/subscription.rs +++ b/src/subscription.rs @@ -2,7 +2,7 @@ use futures_util::StreamExt; use pyo3::exceptions::PyStopAsyncIteration; use std::{sync::Arc, time::Duration}; -use pyo3::{Bound, PyAny, PyRef, Python, pyclass, pymethods}; +use pyo3::{Bound, PyAny, PyRef, Python}; use tokio::sync::Mutex; use crate::{ @@ -10,7 +10,7 @@ use crate::{ utils::natsrpy_future, }; -#[pyclass] +#[pyo3::pyclass] pub struct Subscription { inner: Option>>, } @@ -24,7 +24,7 @@ impl Subscription { } } -#[pymethods] +#[pyo3::pymethods] impl Subscription { #[must_use] pub const fn __aiter__(slf: PyRef) -> PyRef { From 4ffe0e7edb23099f317fb86a177d2e7be0188fdc Mon Sep 17 00:00:00 2001 From: Pavel Kirilin Date: Thu, 19 Mar 2026 17:40:31 +0100 Subject: [PATCH 2/8] TMP Big updates. --- Cargo.toml | 2 +- pyproject.toml | 2 +- python/natsrpy/__init__.py | 2 +- python/natsrpy/_inner/message.pyi | 10 - .../{_inner => _natsrpy_rs}/__init__.pyi | 4 +- .../{_inner => _natsrpy_rs}/js/__init__.pyi | 4 +- .../natsrpy/{_inner => _natsrpy_rs}/js/kv.pyi | 2 +- .../{_inner => _natsrpy_rs}/js/stream.pyi | 29 +- python/natsrpy/_natsrpy_rs/message.pyi | 26 ++ python/natsrpy/js/__init__.py | 2 +- python/natsrpy/js/kv.py | 2 +- python/natsrpy/js/stream.py | 34 +- src/exceptions/rust_err.rs | 8 + src/js/consumers/_pull.rs | 55 +++ src/js/consumers/{push.rs => _push.rs} | 0 src/js/consumers/common.rs | 104 ++++++ src/js/consumers/mod.rs | 11 + src/js/consumers/pull.rs | 3 - src/js/consumers/pull/config.rs | 82 +++++ src/js/consumers/pull/consumer.rs | 50 +++ src/js/consumers/pull/mod.rs | 5 + src/js/consumers/push/config.rs | 78 ++++ src/js/consumers/push/mod.rs | 3 + src/js/message.rs | 1 + src/js/mod.rs | 3 +- src/js/stream.rs | 332 +++++++++++------- src/lib.rs | 2 +- src/message.rs | 34 +- src/subscription.rs | 8 +- 29 files changed, 712 insertions(+), 186 deletions(-) delete mode 100644 python/natsrpy/_inner/message.pyi rename python/natsrpy/{_inner => _natsrpy_rs}/__init__.pyi (93%) rename python/natsrpy/{_inner => _natsrpy_rs}/js/__init__.pyi (86%) rename python/natsrpy/{_inner => _natsrpy_rs}/js/kv.pyi (95%) rename python/natsrpy/{_inner => _natsrpy_rs}/js/stream.pyi (88%) create mode 100644 python/natsrpy/_natsrpy_rs/message.pyi create mode 100644 src/js/consumers/_pull.rs rename src/js/consumers/{push.rs => _push.rs} (100%) create mode 100644 src/js/consumers/common.rs delete mode 100644 src/js/consumers/pull.rs create mode 100644 src/js/consumers/pull/config.rs create mode 100644 src/js/consumers/pull/consumer.rs create mode 100644 src/js/consumers/pull/mod.rs create mode 100644 src/js/consumers/push/config.rs create mode 100644 src/js/consumers/push/mod.rs create mode 100644 src/js/message.rs diff --git a/Cargo.toml b/Cargo.toml index 089506f..2ef53aa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ publish = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [lib] crate-type = ["cdylib"] -name = "_inner" +name = "_natsrpy_rs" [dependencies] async-nats = "0.46" diff --git a/pyproject.toml b/pyproject.toml index 63f07da..f7f4336 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ build-backend = "maturin" [tool.maturin] bindings = "pyo3" features = ["pyo3/extension-module"] -module-name = "natsrpy._inner" +module-name = "natsrpy._natsrpy_rs" python-source = "python" [tool.mypy] diff --git a/python/natsrpy/__init__.py b/python/natsrpy/__init__.py index 84b6e8e..91e6406 100644 --- a/python/natsrpy/__init__.py +++ b/python/natsrpy/__init__.py @@ -1,4 +1,4 @@ -from natsrpy._inner import Message, Nats, Subscription +from natsrpy._natsrpy_rs import Message, Nats, Subscription __all__ = [ "Message", diff --git a/python/natsrpy/_inner/message.pyi b/python/natsrpy/_inner/message.pyi deleted file mode 100644 index 31911b5..0000000 --- a/python/natsrpy/_inner/message.pyi +++ /dev/null @@ -1,10 +0,0 @@ -from typing import Any - -class Message: - subject: str - reply: str | None - payload: bytes - headers: dict[str, Any] - status: int | None - description: str | None - length: int diff --git a/python/natsrpy/_inner/__init__.pyi b/python/natsrpy/_natsrpy_rs/__init__.pyi similarity index 93% rename from python/natsrpy/_inner/__init__.pyi rename to python/natsrpy/_natsrpy_rs/__init__.pyi index 532e71c..caa19ba 100644 --- a/python/natsrpy/_inner/__init__.pyi +++ b/python/natsrpy/_natsrpy_rs/__init__.pyi @@ -1,8 +1,8 @@ from datetime import timedelta from typing import Any -from natsrpy._inner.js import JetStream -from natsrpy._inner.message import Message +from natsrpy._natsrpy_rs.js import JetStream +from natsrpy._natsrpy_rs.message import Message class Subscription: def __aiter__(self) -> Subscription: ... diff --git a/python/natsrpy/_inner/js/__init__.pyi b/python/natsrpy/_natsrpy_rs/js/__init__.pyi similarity index 86% rename from python/natsrpy/_inner/js/__init__.pyi rename to python/natsrpy/_natsrpy_rs/js/__init__.pyi index 11300ec..2eb4d5a 100644 --- a/python/natsrpy/_inner/js/__init__.pyi +++ b/python/natsrpy/_natsrpy_rs/js/__init__.pyi @@ -1,5 +1,5 @@ -from natsrpy._inner.js.kv import KeyValue, KVConfig -from natsrpy._inner.js.stream import Stream, StreamConfig +from natsrpy._natsrpy_rs.js.kv import KeyValue, KVConfig +from natsrpy._natsrpy_rs.js.stream import Stream, StreamConfig class JetStream: async def publish( diff --git a/python/natsrpy/_inner/js/kv.pyi b/python/natsrpy/_natsrpy_rs/js/kv.pyi similarity index 95% rename from python/natsrpy/_inner/js/kv.pyi rename to python/natsrpy/_natsrpy_rs/js/kv.pyi index f164eae..94baf4e 100644 --- a/python/natsrpy/_inner/js/kv.pyi +++ b/python/natsrpy/_natsrpy_rs/js/kv.pyi @@ -1,4 +1,4 @@ -from natsrpy._inner.js.stream import Placement, Republish, Source, StorageType +from natsrpy._natsrpy_rs.js.stream import Placement, Republish, Source, StorageType class KVConfig: """ diff --git a/python/natsrpy/_inner/js/stream.pyi b/python/natsrpy/_natsrpy_rs/js/stream.pyi similarity index 88% rename from python/natsrpy/_inner/js/stream.pyi rename to python/natsrpy/_natsrpy_rs/js/stream.pyi index 78dbc97..41b0ffe 100644 --- a/python/natsrpy/_inner/js/stream.pyi +++ b/python/natsrpy/_natsrpy_rs/js/stream.pyi @@ -228,8 +228,31 @@ class StreamInfo: class Stream: async def direct_get(self, sequence: int) -> StreamMessage: """ - Get direct message from a stream. + Get direct message from the stream. - Please note, that this method will throw an error - in case of stream being configured without `allow_direct=True`. + :param sequence: sequence number of the message to get. + :return: Message. + """ + + async def get_info(self) -> StreamInfo: + """ + Get information about the stream. + + :return: Stream info. + """ + + async def purge( + self, + filter: str | None = None, + sequence: int | None = None, + keep: int | None = None, + ) -> int: + """ + Purge current stream. + + :param filter: filter of subjects to purge, defaults to None + :param sequence: Message sequence to purge up to (inclusive), defaults to None + :param keep: Message count to keep starting from the end of the stream, + defaults to None + :return: number of messages purged """ diff --git a/python/natsrpy/_natsrpy_rs/message.pyi b/python/natsrpy/_natsrpy_rs/message.pyi new file mode 100644 index 0000000..a2d2a9f --- /dev/null +++ b/python/natsrpy/_natsrpy_rs/message.pyi @@ -0,0 +1,26 @@ +from typing import Any + +class Message: + """ + Simple NATS message. + + Attributes: + subject: subject where message was published + reply: subject where reply should be sent, if any + payload: message payload + headers: dictionary of message headers, + every value can be a simple value or a list. + status: status is used for reply messages to indicate the status of the reply. + It is None for regular messages. + description: message description is used for reply messages to + provide additional information about the status. + length: a length of the message payload in bytes. + """ + + subject: str + reply: str | None + payload: bytes + headers: dict[str, Any] + status: int | None + description: str | None + length: int diff --git a/python/natsrpy/js/__init__.py b/python/natsrpy/js/__init__.py index 79c8773..bd23a18 100644 --- a/python/natsrpy/js/__init__.py +++ b/python/natsrpy/js/__init__.py @@ -1,4 +1,4 @@ -from natsrpy._inner.js import JetStream +from natsrpy._natsrpy_rs.js import JetStream from natsrpy.js.kv import KeyValue, KVConfig from natsrpy.js.stream import ( Compression, diff --git a/python/natsrpy/js/kv.py b/python/natsrpy/js/kv.py index faabf69..6eee56e 100644 --- a/python/natsrpy/js/kv.py +++ b/python/natsrpy/js/kv.py @@ -1,4 +1,4 @@ -from natsrpy._inner.js.kv import KeyValue, KVConfig +from natsrpy._natsrpy_rs.js.kv import KeyValue, KVConfig __all__ = [ "KVConfig", diff --git a/python/natsrpy/js/stream.py b/python/natsrpy/js/stream.py index ce25616..edf828c 100644 --- a/python/natsrpy/js/stream.py +++ b/python/natsrpy/js/stream.py @@ -1,50 +1,50 @@ -from natsrpy._inner.js.stream import ( +from natsrpy._natsrpy_rs.js.stream import ( + ClusterInfo, Compression, ConsumerLimits, DiscardPolicy, External, + PeerInfo, PersistenceMode, Placement, Republish, RetentionPolicy, Source, + SourceInfo, StorageType, Stream, StreamConfig, - StreamMessage, - SubjectTransform, - ClusterInfo, - PeerInfo, - SourceInfo, StreamInfo, + StreamMessage, StreamState, + SubjectTransform, ) __all__ = [ + "ClusterInfo", + "Compression", "Compression", "ConsumerLimits", + "ConsumerLimits", + "DiscardPolicy", "DiscardPolicy", "External", + "PeerInfo", + "PersistenceMode", "PersistenceMode", "Placement", "Republish", "RetentionPolicy", + "RetentionPolicy", "Source", + "SourceInfo", "StorageType", "Stream", - "StreamConfig", - "StreamMessage", - "SubjectTransform", - "ClusterInfo", - "Compression", - "ConsumerLimits", - "DiscardPolicy", "Stream", - "PeerInfo", - "PersistenceMode", - "RetentionPolicy", - "SourceInfo", + "StreamConfig", "StreamConfig", "StreamInfo", + "StreamMessage", "StreamState", + "SubjectTransform", ] diff --git a/src/exceptions/rust_err.rs b/src/exceptions/rust_err.rs index 36a02e2..93658c1 100644 --- a/src/exceptions/rust_err.rs +++ b/src/exceptions/rust_err.rs @@ -56,6 +56,14 @@ pub enum NatsrpyError { StreamDirectGetError(#[from] async_nats::jetstream::stream::DirectGetError), #[error(transparent)] StreamInfoError(#[from] async_nats::jetstream::stream::InfoError), + #[error(transparent)] + StreamPurgeError(#[from] async_nats::jetstream::stream::PurgeError), + #[error(transparent)] + UnknownError(#[from] Box), + #[error(transparent)] + PullMessageError(#[from] async_nats::jetstream::consumer::pull::MessagesError), + #[error(transparent)] + PullConsumerError(#[from] async_nats::jetstream::stream::ConsumerError), } impl From for pyo3::PyErr { diff --git a/src/js/consumers/_pull.rs b/src/js/consumers/_pull.rs new file mode 100644 index 0000000..3a8d3e6 --- /dev/null +++ b/src/js/consumers/_pull.rs @@ -0,0 +1,55 @@ +use std::sync::Arc; + +use futures_util::StreamExt; +use pyo3::{Bound, PyAny, Python}; +use tokio::sync::RwLock; + +use crate::{exceptions::rust_err::NatsrpyResult, utils::natsrpy_future}; + +type NatsPullConsumer = + async_nats::jetstream::consumer::Consumer; + +#[pyo3::pyclass(from_py_object)] +#[derive(Debug, Clone)] +pub struct PullConsumer { + consumer: Arc>, +} + +#[pyo3::pyclass(from_py_object)] +#[derive(Debug, Clone)] +pub struct OrderedPullConsumer { + consumer: Arc>, +} + +impl PullConsumer { + pub fn new(consumer: NatsPullConsumer) -> Self { + Self { + consumer: Arc::new(RwLock::new(consumer)), + } + } +} + +#[pyo3::pyclass] +pub struct PullMessageIterator { + inner: Arc>, +} + +#[pyo3::pymethods] +impl PullConsumer { + pub fn messages<'py>(&self, py: Python<'py>) -> NatsrpyResult> { + let consumer_lock = self.consumer.clone(); + natsrpy_future(py, async move { + let mut messages = consumer_lock.read().await.messages().await.unwrap(); + while let Some(message) = messages.next().await { + let msg = message?; + log::info!("{:#?}", msg.message.payload); + msg.ack().await?; + } + + Ok(()) + }) + } +} + +#[pyo3::pymethods] +impl PullMessageIterator {} diff --git a/src/js/consumers/push.rs b/src/js/consumers/_push.rs similarity index 100% rename from src/js/consumers/push.rs rename to src/js/consumers/_push.rs diff --git a/src/js/consumers/common.rs b/src/js/consumers/common.rs new file mode 100644 index 0000000..84c4022 --- /dev/null +++ b/src/js/consumers/common.rs @@ -0,0 +1,104 @@ +use crate::exceptions::rust_err::{NatsrpyError, NatsrpyResult}; + +#[pyo3::pyclass(from_py_object)] +#[derive(Debug, Clone, Default, Copy, PartialEq, Eq, PartialOrd)] +pub enum DeliverPolicy { + #[default] + All, + Last, + New, + ByStartSequence, + ByStartTime, + LastPerSubject, +} + +impl DeliverPolicy { + pub fn to_nats_delivery_policy( + &self, + start_sequence: Option, + start_time: Option, + ) -> NatsrpyResult { + let result = match self { + Self::All => async_nats::jetstream::consumer::DeliverPolicy::All, + Self::Last => async_nats::jetstream::consumer::DeliverPolicy::Last, + Self::New => async_nats::jetstream::consumer::DeliverPolicy::New, + Self::LastPerSubject => async_nats::jetstream::consumer::DeliverPolicy::Last, + Self::ByStartSequence => { + let Some(start_sequence) = start_sequence else { + return Err(NatsrpyError::SessionError(String::from( + "Start sequence is not present", + ))); + }; + async_nats::jetstream::consumer::DeliverPolicy::ByStartSequence { start_sequence } + } + Self::ByStartTime => { + let Some(start_time) = start_time else { + return Err(NatsrpyError::SessionError(String::from( + "Start sequence is not present", + ))); + }; + async_nats::jetstream::consumer::DeliverPolicy::ByStartTime { + start_time: time::OffsetDateTime::from_unix_timestamp(start_time)?, + } + } + }; + Ok(result) + } +} + +#[pyo3::pyclass(from_py_object)] +#[derive(Debug, Clone, Default, Copy, PartialEq, Eq, PartialOrd)] +pub enum AckPolicy { + #[default] + Explicit, + None, + All, +} + +impl From for async_nats::jetstream::consumer::AckPolicy { + fn from(value: AckPolicy) -> Self { + match value { + AckPolicy::Explicit => Self::Explicit, + AckPolicy::None => Self::None, + AckPolicy::All => Self::All, + } + } +} + +#[pyo3::pyclass(from_py_object)] +#[derive(Debug, Clone, Default, Copy, PartialEq, Eq, PartialOrd)] +pub enum ReplayPolicy { + #[default] + Instant, + Original, +} + +impl From for async_nats::jetstream::consumer::ReplayPolicy { + fn from(value: ReplayPolicy) -> Self { + match value { + ReplayPolicy::Instant => Self::Instant, + ReplayPolicy::Original => Self::Original, + } + } +} + +#[pyo3::pyclass(from_py_object)] +#[derive(Debug, Clone, Default, Copy, PartialEq, Eq, PartialOrd)] +pub enum PriorityPolicy { + #[default] + None, + Overflow, + PinnedClient, + Prioritized, +} + +impl From for async_nats::jetstream::consumer::PriorityPolicy { + fn from(value: PriorityPolicy) -> Self { + match value { + PriorityPolicy::None => Self::None, + PriorityPolicy::Overflow => Self::Overflow, + PriorityPolicy::PinnedClient => Self::PinnedClient, + PriorityPolicy::Prioritized => Self::Prioritized, + } + } +} diff --git a/src/js/consumers/mod.rs b/src/js/consumers/mod.rs index a09e332..601ffc5 100644 --- a/src/js/consumers/mod.rs +++ b/src/js/consumers/mod.rs @@ -1,2 +1,13 @@ +pub mod common; pub mod pull; pub mod push; + +#[pyo3::pymodule(submodule, name = "consumers")] +pub mod pymod { + #[pymodule_export] + use super::common::{AckPolicy, DeliverPolicy, PriorityPolicy, ReplayPolicy}; + #[pymodule_export] + pub use super::pull::{config::PullConsumerConfig, consumer::PullConsumer}; + #[pymodule_export] + pub use super::push::config::PushConsumerConfig; +} diff --git a/src/js/consumers/pull.rs b/src/js/consumers/pull.rs deleted file mode 100644 index 845f036..0000000 --- a/src/js/consumers/pull.rs +++ /dev/null @@ -1,3 +0,0 @@ -#[pyo3::pyclass(from_py_object)] -#[derive(Debug, Clone)] -pub struct PullConsumer; diff --git a/src/js/consumers/pull/config.rs b/src/js/consumers/pull/config.rs new file mode 100644 index 0000000..d0ddef0 --- /dev/null +++ b/src/js/consumers/pull/config.rs @@ -0,0 +1,82 @@ +use std::{collections::HashMap, time::Duration}; + +use crate::{ + exceptions::rust_err::NatsrpyError, + js::consumers::common::{AckPolicy, DeliverPolicy, PriorityPolicy, ReplayPolicy}, +}; + +#[pyo3::pyclass(from_py_object, get_all, set_all)] +#[derive(Clone, Debug)] +pub struct PullConsumerConfig { + pub durable_name: Option, + pub name: Option, + pub description: Option, + pub deliver_policy: DeliverPolicy, + pub delivery_start_sequence: Option, + pub delivery_start_time: Option, + pub ack_policy: AckPolicy, + pub ack_wait: Duration, + pub max_deliver: i64, + pub filter_subject: String, + pub filter_subjects: Vec, + pub replay_policy: ReplayPolicy, + pub rate_limit: u64, + pub sample_frequency: u8, + pub max_waiting: i64, + pub max_ack_pending: i64, + pub headers_only: bool, + pub max_batch: i64, + pub max_bytes: i64, + pub max_expires: Duration, + pub inactive_threshold: Duration, + pub num_replicas: usize, + pub memory_storage: bool, + pub metadata: HashMap, + pub backoff: Vec, + pub priority_policy: PriorityPolicy, + pub priority_groups: Vec, + pub pause_until: Option, +} + +impl PullConsumerConfig {} + +impl TryFrom for async_nats::jetstream::consumer::pull::Config { + type Error = NatsrpyError; + + fn try_from(value: PullConsumerConfig) -> Result { + Ok(Self { + durable_name: value.durable_name, + name: value.name, + description: value.description, + deliver_policy: value.deliver_policy.to_nats_delivery_policy( + value.delivery_start_sequence, + value.delivery_start_time, + )?, + ack_policy: value.ack_policy.into(), + ack_wait: value.ack_wait, + max_deliver: value.max_deliver, + filter_subject: value.filter_subject, + filter_subjects: value.filter_subjects, + replay_policy: value.replay_policy.into(), + rate_limit: value.rate_limit, + sample_frequency: value.sample_frequency, + max_waiting: value.max_waiting, + max_ack_pending: value.max_ack_pending, + headers_only: value.headers_only, + max_batch: value.max_batch, + max_bytes: value.max_bytes, + max_expires: value.max_expires, + inactive_threshold: value.inactive_threshold, + num_replicas: value.num_replicas, + memory_storage: value.memory_storage, + metadata: value.metadata, + backoff: value.backoff, + priority_policy: value.priority_policy.into(), + priority_groups: value.priority_groups, + pause_until: value + .pause_until + .map(time::OffsetDateTime::from_unix_timestamp) + .transpose()?, + }) + } +} diff --git a/src/js/consumers/pull/consumer.rs b/src/js/consumers/pull/consumer.rs new file mode 100644 index 0000000..f24aaaa --- /dev/null +++ b/src/js/consumers/pull/consumer.rs @@ -0,0 +1,50 @@ +use std::sync::Arc; + +use futures_util::StreamExt; +use pyo3::{Bound, PyAny, Python}; +use tokio::sync::RwLock; + +use crate::{exceptions::rust_err::NatsrpyResult, utils::natsrpy_future}; + +type NatsPullConsumer = + async_nats::jetstream::consumer::Consumer; + +#[pyo3::pyclass(from_py_object)] +#[derive(Debug, Clone)] +pub struct PullConsumer { + consumer: Arc>, +} + +impl PullConsumer { + #[must_use] + pub fn new(consumer: NatsPullConsumer) -> Self { + Self { + consumer: Arc::new(RwLock::new(consumer)), + } + } +} + +#[pyo3::pyclass] +pub struct PullMessageIterator { + inner: Arc>, +} + +#[pyo3::pymethods] +impl PullConsumer { + pub fn messages<'py>(&self, py: Python<'py>) -> NatsrpyResult> { + let consumer_lock = self.consumer.clone(); + natsrpy_future(py, async move { + let mut messages = consumer_lock.read().await.messages().await.unwrap(); + while let Some(message) = messages.next().await { + let msg = message?; + log::info!("{:#?}", msg.message.payload); + msg.ack().await?; + } + + Ok(()) + }) + } +} + +#[pyo3::pymethods] +impl PullMessageIterator {} diff --git a/src/js/consumers/pull/mod.rs b/src/js/consumers/pull/mod.rs new file mode 100644 index 0000000..98b9001 --- /dev/null +++ b/src/js/consumers/pull/mod.rs @@ -0,0 +1,5 @@ +pub mod config; +pub mod consumer; + +pub use config::PullConsumerConfig; +pub use consumer::PullConsumer; diff --git a/src/js/consumers/push/config.rs b/src/js/consumers/push/config.rs new file mode 100644 index 0000000..fb888b9 --- /dev/null +++ b/src/js/consumers/push/config.rs @@ -0,0 +1,78 @@ +use std::{collections::HashMap, time::Duration}; + +use crate::{ + exceptions::rust_err::NatsrpyError, + js::consumers::common::{AckPolicy, DeliverPolicy, ReplayPolicy}, +}; + +#[pyo3::pyclass(from_py_object, get_all, set_all)] +#[derive(Clone, Debug)] +pub struct PushConsumerConfig { + pub deliver_subject: String, + pub durable_name: Option, + pub name: Option, + pub description: Option, + pub deliver_group: Option, + pub deliver_policy: DeliverPolicy, + pub delivery_start_sequence: Option, + pub delivery_start_time: Option, + pub ack_policy: AckPolicy, + pub ack_wait: Duration, + pub max_deliver: i64, + pub filter_subject: String, + pub filter_subjects: Vec, + pub replay_policy: ReplayPolicy, + pub rate_limit: u64, + pub sample_frequency: u8, + pub max_waiting: i64, + pub max_ack_pending: i64, + pub headers_only: bool, + pub flow_control: bool, + pub idle_heartbeat: Duration, + pub num_replicas: usize, + pub memory_storage: bool, + pub metadata: HashMap, + pub backoff: Vec, + pub inactive_threshold: Duration, + pub pause_until: Option, +} + +impl TryFrom for async_nats::jetstream::consumer::push::Config { + type Error = NatsrpyError; + + fn try_from(value: PushConsumerConfig) -> Result { + Ok(Self { + deliver_subject: value.deliver_subject, + durable_name: value.durable_name, + name: value.name, + description: value.description, + deliver_group: value.deliver_group, + deliver_policy: value.deliver_policy.to_nats_delivery_policy( + value.delivery_start_sequence, + value.delivery_start_time, + )?, + ack_policy: value.ack_policy.into(), + ack_wait: value.ack_wait, + max_deliver: value.max_deliver, + filter_subject: value.filter_subject, + filter_subjects: value.filter_subjects, + replay_policy: value.replay_policy.into(), + rate_limit: value.rate_limit, + sample_frequency: value.sample_frequency, + max_waiting: value.max_waiting, + max_ack_pending: value.max_ack_pending, + headers_only: value.headers_only, + flow_control: value.flow_control, + idle_heartbeat: value.idle_heartbeat, + num_replicas: value.num_replicas, + memory_storage: value.memory_storage, + metadata: value.metadata, + backoff: value.backoff, + inactive_threshold: value.inactive_threshold, + pause_until: value + .pause_until + .map(time::OffsetDateTime::from_unix_timestamp) + .transpose()?, + }) + } +} diff --git a/src/js/consumers/push/mod.rs b/src/js/consumers/push/mod.rs new file mode 100644 index 0000000..3aa043f --- /dev/null +++ b/src/js/consumers/push/mod.rs @@ -0,0 +1,3 @@ +pub mod config; + +pub use config::PushConsumerConfig; diff --git a/src/js/message.rs b/src/js/message.rs new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/js/message.rs @@ -0,0 +1 @@ + diff --git a/src/js/mod.rs b/src/js/mod.rs index 9fddb36..5672495 100644 --- a/src/js/mod.rs +++ b/src/js/mod.rs @@ -8,13 +8,14 @@ pub mod pymod { // Classes #[pymodule_export] pub use super::{ - consumers::{pull::PullConsumer, push::PushConsumer}, jetstream::JetStream, kv::{KVConfig, KeyValue}, }; // SubModules #[pymodule_export] + pub use super::consumers::pymod as consumers; + #[pymodule_export] pub use super::kv::pymod as kv; #[pymodule_export] pub use super::stream::pymod as stream; diff --git a/src/js/stream.rs b/src/js/stream.rs index 46adaff..4995102 100644 --- a/src/js/stream.rs +++ b/src/js/stream.rs @@ -6,8 +6,8 @@ use std::{collections::HashMap, ops::Deref, sync::Arc, time::Duration}; use crate::{ exceptions::rust_err::{NatsrpyError, NatsrpyResult}, - utils::headers::NatsrpyHeadermapExt, - utils::natsrpy_future, + js::consumers::{self}, + utils::{headers::NatsrpyHeadermapExt, natsrpy_future}, }; use pyo3::{Bound, PyAny, Python}; use tokio::sync::RwLock; @@ -94,8 +94,9 @@ impl From for RetentionPolicy { } #[pyo3::pyclass(from_py_object)] -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum Compression { + #[default] S2, NONE, } @@ -491,35 +492,35 @@ impl From for ClusterInfo { } #[pyo3::pyclass(from_py_object, get_all, set_all)] -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default)] pub struct StreamConfig { pub name: String, pub subjects: Vec, - pub max_bytes: Option, - pub max_messages: Option, - pub max_messages_per_subject: Option, - pub discard: Option, - pub discard_new_per_subject: Option, - pub retention: Option, - pub max_consumers: Option, - pub max_age: Option, - pub max_message_size: Option, - pub storage: Option, - pub num_replicas: Option, - pub no_ack: Option, - pub duplicate_window: Option, - pub template_owner: Option, - pub sealed: Option, + pub max_bytes: i64, + pub max_messages: i64, + pub max_messages_per_subject: i64, + pub discard: DiscardPolicy, + pub discard_new_per_subject: bool, + pub retention: RetentionPolicy, + pub max_consumers: i32, + pub max_age: Duration, + pub max_message_size: i32, + pub storage: StorageType, + pub num_replicas: usize, + pub no_ack: bool, + pub duplicate_window: Duration, + pub template_owner: String, + pub sealed: bool, pub description: Option, - pub allow_rollup: Option, - pub deny_delete: Option, - pub deny_purge: Option, + pub allow_rollup: bool, + pub deny_delete: bool, + pub deny_purge: bool, pub republish: Option, - pub allow_direct: Option, - pub mirror_direct: Option, + pub allow_direct: bool, + pub mirror_direct: bool, pub mirror: Option, pub sources: Option>, - pub metadata: Option>, + pub metadata: HashMap, pub subject_transform: Option, pub compression: Option, pub consumer_limits: Option, @@ -527,11 +528,11 @@ pub struct StreamConfig { pub placement: Option, pub persist_mode: Option, pub pause_until: Option, - pub allow_message_ttl: Option, + pub allow_message_ttl: bool, pub subject_delete_marker_ttl: Option, - pub allow_atomic_publish: Option, - pub allow_message_schedules: Option, - pub allow_message_counter: Option, + pub allow_atomic_publish: bool, + pub allow_message_schedules: bool, + pub allow_message_counter: bool, } #[pyo3::pymethods] @@ -578,7 +579,7 @@ impl StreamConfig { allow_message_schedules=None, allow_message_counter=None, ))] - pub const fn __new__( + pub fn __new__( name: String, subjects: Vec, max_bytes: Option, @@ -619,34 +620,13 @@ impl StreamConfig { allow_message_schedules: Option, allow_message_counter: Option, ) -> NatsrpyResult { - Ok(Self { + let mut config = Self { name, subjects, - max_bytes, - max_messages, - max_messages_per_subject, - discard, - discard_new_per_subject, - retention, - max_consumers, - max_age, - max_message_size, - storage, - num_replicas, - no_ack, - duplicate_window, - template_owner, - sealed, description, - allow_rollup, - deny_delete, - deny_purge, republish, - allow_direct, - mirror_direct, mirror, sources, - metadata, subject_transform, compression, consumer_limits, @@ -654,12 +634,41 @@ impl StreamConfig { placement, persist_mode, pause_until, - allow_message_ttl, subject_delete_marker_ttl, - allow_atomic_publish, - allow_message_schedules, - allow_message_counter, - }) + ..Default::default() + }; + + config.max_bytes = max_bytes.unwrap_or(config.max_bytes); + config.max_messages = max_messages.unwrap_or(config.max_messages); + config.max_messages_per_subject = + max_messages_per_subject.unwrap_or(config.max_messages_per_subject); + config.discard = discard.unwrap_or(config.discard); + config.discard_new_per_subject = + discard_new_per_subject.unwrap_or(config.discard_new_per_subject); + config.retention = retention.unwrap_or(config.retention); + config.max_consumers = max_consumers.unwrap_or(config.max_consumers); + config.max_age = max_age.unwrap_or(config.max_age); + config.max_message_size = max_message_size.unwrap_or(config.max_message_size); + config.storage = storage.unwrap_or(config.storage); + config.num_replicas = num_replicas.unwrap_or(config.num_replicas); + config.no_ack = no_ack.unwrap_or(config.no_ack); + config.duplicate_window = duplicate_window.unwrap_or(config.duplicate_window); + config.template_owner = template_owner.unwrap_or(config.template_owner); + config.sealed = sealed.unwrap_or(config.sealed); + config.allow_rollup = allow_rollup.unwrap_or(config.allow_rollup); + config.deny_delete = deny_delete.unwrap_or(config.deny_delete); + config.deny_purge = deny_purge.unwrap_or(config.deny_purge); + config.allow_direct = allow_direct.unwrap_or(config.allow_direct); + config.mirror_direct = mirror_direct.unwrap_or(config.mirror_direct); + config.metadata = metadata.unwrap_or(config.metadata); + config.allow_message_ttl = allow_message_ttl.unwrap_or(config.allow_message_ttl); + config.allow_atomic_publish = allow_atomic_publish.unwrap_or(config.allow_atomic_publish); + config.allow_message_schedules = + allow_message_schedules.unwrap_or(config.allow_message_schedules); + config.allow_message_counter = + allow_message_counter.unwrap_or(config.allow_message_counter); + + Ok(config) } } @@ -670,33 +679,33 @@ impl TryFrom for StreamConfig { Ok(Self { name: value.name, subjects: value.subjects, - max_bytes: Some(value.max_bytes), - max_messages: Some(value.max_messages), - max_messages_per_subject: Some(value.max_messages_per_subject), - discard: Some(value.discard.into()), - discard_new_per_subject: Some(value.discard_new_per_subject), - retention: Some(value.retention.into()), - max_consumers: Some(value.max_consumers), - max_age: Some(value.max_age), - max_message_size: Some(value.max_message_size), - storage: Some(value.storage.into()), - num_replicas: Some(value.num_replicas), - no_ack: Some(value.no_ack), - duplicate_window: Some(value.duplicate_window), - template_owner: Some(value.template_owner), - sealed: Some(value.sealed), + max_bytes: value.max_bytes, + max_messages: value.max_messages, + max_messages_per_subject: value.max_messages_per_subject, + discard: value.discard.into(), + discard_new_per_subject: value.discard_new_per_subject, + retention: value.retention.into(), + max_consumers: value.max_consumers, + max_age: value.max_age, + max_message_size: value.max_message_size, + storage: value.storage.into(), + num_replicas: value.num_replicas, + no_ack: value.no_ack, + duplicate_window: value.duplicate_window, + template_owner: value.template_owner, + sealed: value.sealed, description: value.description, - allow_rollup: Some(value.allow_rollup), - deny_delete: Some(value.deny_delete), - deny_purge: Some(value.deny_purge), + allow_rollup: value.allow_rollup, + deny_delete: value.deny_delete, + deny_purge: value.deny_purge, republish: value.republish.map(Into::into), - allow_direct: Some(value.allow_direct), - mirror_direct: Some(value.mirror_direct), + allow_direct: value.allow_direct, + mirror_direct: value.mirror_direct, mirror: value.mirror.map(Into::into), sources: value .sources .map(|val| val.into_iter().map(Into::into).collect()), - metadata: Some(value.metadata), + metadata: value.metadata, subject_transform: value.subject_transform.map(Into::into), compression: value.compression.map(Into::into), consumer_limits: value.consumer_limits.map(Into::into), @@ -704,11 +713,11 @@ impl TryFrom for StreamConfig { placement: value.placement.map(Into::into), persist_mode: value.persist_mode.map(Into::into), pause_until: value.pause_until.map(time::OffsetDateTime::unix_timestamp), - allow_message_ttl: Some(value.allow_message_ttl), + allow_message_ttl: value.allow_message_ttl, subject_delete_marker_ttl: value.subject_delete_marker_ttl, - allow_atomic_publish: Some(value.allow_atomic_publish), - allow_message_schedules: Some(value.allow_message_schedules), - allow_message_counter: Some(value.allow_message_counter), + allow_atomic_publish: value.allow_atomic_publish, + allow_message_schedules: value.allow_message_schedules, + allow_message_counter: value.allow_message_counter, }) } } @@ -729,44 +738,34 @@ impl TryFrom for async_nats::jetstream::stream::Config { // Optional values that have defaults. // If the value is not present, we just use the one // that nats' config defaults to. - conf.max_bytes = value.max_bytes.unwrap_or(conf.max_bytes); - conf.max_messages = value.max_messages.unwrap_or(conf.max_messages); - conf.max_messages_per_subject = value - .max_messages_per_subject - .unwrap_or(conf.max_messages_per_subject); - conf.discard_new_per_subject = value - .discard_new_per_subject - .unwrap_or(conf.discard_new_per_subject); - conf.max_consumers = value.max_consumers.unwrap_or(conf.max_consumers); - conf.max_age = value.max_age.unwrap_or(conf.max_age); - conf.max_message_size = value.max_message_size.unwrap_or(conf.max_message_size); - conf.num_replicas = value.num_replicas.unwrap_or(conf.num_replicas); - conf.no_ack = value.no_ack.unwrap_or(conf.no_ack); - conf.duplicate_window = value.duplicate_window.unwrap_or(conf.duplicate_window); - conf.template_owner = value.template_owner.unwrap_or(conf.template_owner); - conf.sealed = value.sealed.unwrap_or(conf.sealed); - conf.allow_rollup = value.allow_rollup.unwrap_or(conf.allow_rollup); - conf.deny_delete = value.deny_delete.unwrap_or(conf.deny_delete); - conf.deny_purge = value.deny_purge.unwrap_or(conf.deny_purge); - conf.allow_direct = value.allow_direct.unwrap_or(conf.allow_direct); - conf.mirror_direct = value.mirror_direct.unwrap_or(conf.mirror_direct); - conf.metadata = value.metadata.unwrap_or(conf.metadata); - conf.allow_message_ttl = value.allow_message_ttl.unwrap_or(conf.allow_message_ttl); - conf.allow_atomic_publish = value - .allow_atomic_publish - .unwrap_or(conf.allow_atomic_publish); - conf.allow_message_schedules = value - .allow_message_schedules - .unwrap_or(conf.allow_message_schedules); - conf.allow_message_counter = value - .allow_message_counter - .unwrap_or(conf.allow_message_counter); + conf.max_bytes = value.max_bytes; + conf.max_messages = value.max_messages; + conf.max_messages_per_subject = value.max_messages_per_subject; + conf.discard_new_per_subject = value.discard_new_per_subject; + conf.max_consumers = value.max_consumers; + conf.max_age = value.max_age; + conf.max_message_size = value.max_message_size; + conf.num_replicas = value.num_replicas; + conf.no_ack = value.no_ack; + conf.duplicate_window = value.duplicate_window; + conf.template_owner = value.template_owner; + conf.sealed = value.sealed; + conf.allow_rollup = value.allow_rollup; + conf.deny_delete = value.deny_delete; + conf.deny_purge = value.deny_purge; + conf.allow_direct = value.allow_direct; + conf.mirror_direct = value.mirror_direct; + conf.metadata = value.metadata; + conf.allow_message_ttl = value.allow_message_ttl; + conf.allow_atomic_publish = value.allow_atomic_publish; + conf.allow_message_schedules = value.allow_message_schedules; + conf.allow_message_counter = value.allow_message_counter; // Values that require conversion between python -> rust types. conf.republish = value.republish.map(Into::into); - conf.storage = value.storage.map_or(conf.storage, Into::into); - conf.retention = value.retention.map_or(conf.retention, Into::into); - conf.discard = value.discard.map_or(conf.discard, Into::into); + conf.storage = value.storage.into(); + conf.retention = value.retention.into(); + conf.discard = value.discard.into(); conf.mirror = value.mirror.map(TryInto::try_into).transpose()?; conf.sources = value .sources @@ -903,23 +902,31 @@ impl From for SourceInfo { } } -#[pyo3::pyclass(from_py_object)] +#[pyo3::pyclass(from_py_object, get_all)] #[derive(Debug, Clone)] pub struct StreamInfo { pub config: StreamConfig, - pub created: time::OffsetDateTime, + pub created: i64, pub state: StreamState, pub cluster: Option, pub mirror: Option, pub sources: Vec, } +#[pyo3::pymethods] +impl StreamInfo { + #[must_use] + pub fn __str__(&self) -> String { + format!("{self:#?}") + } +} + impl TryFrom for StreamInfo { type Error = NatsrpyError; fn try_from(value: async_nats::jetstream::stream::Info) -> Result { Ok(Self { config: value.config.try_into()?, - created: value.created, + created: value.created.unix_timestamp(), state: value.state.into(), cluster: value.cluster.map(Into::into), mirror: value.mirror.map(Into::into), @@ -928,6 +935,22 @@ impl TryFrom for StreamInfo { } } +#[pyo3::pyclass(from_py_object, get_all)] +#[derive(Clone, Debug)] +pub struct PurgeResponse { + success: bool, + purged: u64, +} + +impl From for PurgeResponse { + fn from(value: async_nats::jetstream::stream::PurgeResponse) -> Self { + Self { + success: value.success, + purged: value.purged, + } + } +} + #[pyo3::pyclass(from_py_object)] #[derive(Debug, Clone)] pub struct Stream { @@ -966,6 +989,75 @@ impl Stream { StreamInfo::try_from(ctx.read().await.get_info().await?) }) } + + #[pyo3(signature=( + filter=None, + sequence=None, + keep=None, + ))] + pub fn purge<'py>( + &self, + py: Python<'py>, + filter: Option, + sequence: Option, + keep: Option, + ) -> NatsrpyResult> { + let ctx = self.stream.clone(); + natsrpy_future(py, async move { + let mut purge_request = ctx.read().await.purge(); + if let Some(filter) = filter { + purge_request = purge_request.filter(filter); + } + let purge_response = match (sequence, keep) { + (None, None) => purge_request.await, + (Some(seq), None) => purge_request.sequence(seq).await, + (None, Some(keep)) => purge_request.keep(keep).await, + _ => { + return Err(NatsrpyError::InvalidArgument(String::from( + "Either keep or sequence can be set, but not both.", + ))); + } + }; + let resp = purge_response?; + if !resp.success { + return Err(NatsrpyError::SessionError(String::from( + "Purge failed. Check server logs for more info.", + ))); + } + Ok(resp.purged) + }) + } + + pub fn create_pull_consumer<'py>( + &self, + py: Python<'py>, + config: consumers::pull::PullConsumerConfig, + ) -> NatsrpyResult> { + let ctx = self.stream.clone(); + natsrpy_future(py, async move { + Ok(super::consumers::pull::consumer::PullConsumer::new( + ctx.read() + .await + .create_consumer(async_nats::jetstream::consumer::pull::Config::try_from( + config, + )?) + .await?, + )) + }) + } + + pub fn get_pull_consumer<'py>( + &self, + py: Python<'py>, + name: String, + ) -> NatsrpyResult> { + let ctx = self.stream.clone(); + natsrpy_future(py, async move { + Ok(super::consumers::pull::consumer::PullConsumer::new( + ctx.read().await.get_consumer(&name).await?, + )) + }) + } } #[pyo3::pymodule(submodule, name = "stream")] diff --git a/src/lib.rs b/src/lib.rs index a22d823..60286ba 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -25,7 +25,7 @@ pub mod subscription; pub mod utils; #[pyo3::pymodule] -pub mod _inner { +pub mod _natsrpy_rs { use pyo3::{Bound, PyResult, types::PyModule}; use crate::utils::py::PyModuleSubmoduleExt; diff --git a/src/message.rs b/src/message.rs index 79704a9..956bbf0 100644 --- a/src/message.rs +++ b/src/message.rs @@ -3,7 +3,7 @@ use pyo3::{ types::{PyBytes, PyDict}, }; -use crate::{exceptions::rust_err::NatsrpyResult, utils::headers::NatsrpyHeadermapExt}; +use crate::{exceptions::rust_err::NatsrpyError, utils::headers::NatsrpyHeadermapExt}; #[pyo3::pyclass(get_all, set_all)] #[derive(Debug)] @@ -17,20 +17,24 @@ pub struct Message { pub length: usize, } -impl Message { - pub fn from_nats_message(py: Python<'_>, message: async_nats::Message) -> NatsrpyResult { - let headers = match message.headers { - Some(headermap) => headermap.to_pydict(py)?, - None => PyDict::new(py).unbind(), - }; - Ok(Self { - subject: message.subject.to_string(), - reply: message.reply.as_deref().map(ToString::to_string), - payload: PyBytes::new(py, &message.payload).unbind(), - headers, - status: message.status.map(Into::::into), - description: message.description, - length: message.length, +impl TryFrom for Message { + type Error = NatsrpyError; + + fn try_from(value: async_nats::Message) -> Result { + Python::attach(move |gil| { + let headers = match value.headers { + Some(headermap) => headermap.to_pydict(gil)?, + None => PyDict::new(gil).unbind(), + }; + Ok(Self { + subject: value.subject.to_string(), + reply: value.reply.as_deref().map(ToString::to_string), + payload: PyBytes::new(gil, &value.payload).unbind(), + headers, + status: value.status.map(Into::::into), + description: value.description, + length: value.length, + }) }) } } diff --git a/src/subscription.rs b/src/subscription.rs index 3535f44..bce844a 100644 --- a/src/subscription.rs +++ b/src/subscription.rs @@ -42,14 +42,10 @@ impl Subscription { let future = async move { let Some(message) = inner.lock().await.next().await else { - return Err(NatsrpyError::from(PyStopAsyncIteration::new_err( - "End of the stream.", - ))); + return Err(PyStopAsyncIteration::new_err("End of the stream.").into()); }; - Python::attach(move |gil| -> NatsrpyResult<_> { - crate::message::Message::from_nats_message(gil, message) - }) + crate::message::Message::try_from(message) }; natsrpy_future(py, async move { From 2ec5f2f6ea4b7044b6f23e9521aeb40db7cf7c0e Mon Sep 17 00:00:00 2001 From: Pavel Kirilin Date: Fri, 20 Mar 2026 16:42:48 +0100 Subject: [PATCH 3/8] Made few small changes. --- Cargo.lock | 122 ++++-------------- python/natsrpy/_natsrpy_rs/__init__.pyi | 2 +- python/natsrpy/_natsrpy_rs/js/__init__.pyi | 18 +-- .../_natsrpy_rs/js/consumers/__init__.pyi | 19 +++ .../_natsrpy_rs/js/consumers/common.pyi | 22 ++++ .../natsrpy/_natsrpy_rs/js/consumers/pull.pyi | 72 +++++++++++ .../natsrpy/_natsrpy_rs/js/consumers/push.pyi | 69 ++++++++++ .../_natsrpy_rs/js/managers/__init__.pyi | 0 .../_natsrpy_rs/js/managers/consumers.pyi | 14 ++ python/natsrpy/_natsrpy_rs/js/managers/kv.pyi | 8 ++ .../_natsrpy_rs/js/managers/streams.pyi | 8 ++ python/natsrpy/_natsrpy_rs/js/stream.pyi | 5 + python/natsrpy/js/consumers/__init__.py | 17 +++ src/js/consumers/_pull.rs | 55 -------- src/js/consumers/_push.rs | 3 - src/js/consumers/common.rs | 66 +++++----- src/js/consumers/mod.rs | 2 +- src/js/consumers/pull/config.rs | 103 ++++++++++++++- src/js/consumers/pull/consumer.rs | 19 +-- src/js/consumers/push/config.rs | 96 ++++++++++++++ src/js/consumers/push/consumer.rs | 24 ++++ src/js/consumers/push/mod.rs | 2 + src/js/jetstream.rs | 97 ++------------ src/js/managers/consumers.rs | 89 +++++++++++++ src/js/managers/kv.rs | 86 ++++++++++++ src/js/managers/mod.rs | 13 ++ src/js/managers/streams.rs | 87 +++++++++++++ src/js/mod.rs | 3 + src/js/stream.rs | 99 +++++--------- src/lib.rs | 3 + src/nats_cls.rs | 7 +- src/utils/mod.rs | 1 + src/utils/py_types.rs | 42 ++++++ 33 files changed, 897 insertions(+), 376 deletions(-) create mode 100644 python/natsrpy/_natsrpy_rs/js/consumers/__init__.pyi create mode 100644 python/natsrpy/_natsrpy_rs/js/consumers/common.pyi create mode 100644 python/natsrpy/_natsrpy_rs/js/consumers/pull.pyi create mode 100644 python/natsrpy/_natsrpy_rs/js/consumers/push.pyi create mode 100644 python/natsrpy/_natsrpy_rs/js/managers/__init__.pyi create mode 100644 python/natsrpy/_natsrpy_rs/js/managers/consumers.pyi create mode 100644 python/natsrpy/_natsrpy_rs/js/managers/kv.pyi create mode 100644 python/natsrpy/_natsrpy_rs/js/managers/streams.pyi create mode 100644 python/natsrpy/js/consumers/__init__.py delete mode 100644 src/js/consumers/_pull.rs delete mode 100644 src/js/consumers/_push.rs create mode 100644 src/js/consumers/push/consumer.rs create mode 100644 src/js/managers/consumers.rs create mode 100644 src/js/managers/kv.rs create mode 100644 src/js/managers/mod.rs create mode 100644 src/js/managers/streams.rs create mode 100644 src/utils/py_types.rs diff --git a/Cargo.lock b/Cargo.lock index 6797575..7e565e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -95,9 +95,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.56" +version = "1.2.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2" +checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423" dependencies = [ "find-msvc-tools", "shlex", @@ -482,9 +482,9 @@ checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" [[package]] name = "libc" -version = "0.2.182" +version = "0.2.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112" +checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" [[package]] name = "litemap" @@ -572,9 +572,9 @@ checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" [[package]] name = "once_cell" -version = "1.21.3" +version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "openssl-probe" @@ -952,9 +952,9 @@ checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] name = "schannel" -version = "0.1.28" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" dependencies = [ "windows-sys 0.61.2", ] @@ -1120,12 +1120,12 @@ checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" [[package]] name = "socket2" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f4aa3ad99f2088c990dfa82d367e19cb29268ed67c574d10d0a4bfe71f07e0" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -1461,16 +1461,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", + "windows-targets", ] [[package]] @@ -1488,31 +1479,14 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", -] - -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] [[package]] @@ -1521,96 +1495,48 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - [[package]] name = "writeable" version = "0.6.2" @@ -1642,18 +1568,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.40" +version = "0.8.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a789c6e490b576db9f7e6b6d661bcc9799f7c0ac8352f56ea20193b2681532e5" +checksum = "5c5030500cb2d66bdfbb4ebc9563be6ce7005a4b5d0f26be0c523870fe372ca6" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.40" +version = "0.8.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f65c489a7071a749c849713807783f70672b28094011623e200cb86dcb835953" +checksum = "a5f86989a046a79640b9d8867c823349a139367bda96549794fcc3313ce91f4e" dependencies = [ "proc-macro2", "quote", diff --git a/python/natsrpy/_natsrpy_rs/__init__.pyi b/python/natsrpy/_natsrpy_rs/__init__.pyi index caa19ba..54b4ae0 100644 --- a/python/natsrpy/_natsrpy_rs/__init__.pyi +++ b/python/natsrpy/_natsrpy_rs/__init__.pyi @@ -28,7 +28,7 @@ class Nats: async def publish( self, subject: str, - payload: bytes, + payload: bytes | str | bytearray | memoryview, *, headers: dict[str, Any] | None = None, reply: str | None = None, diff --git a/python/natsrpy/_natsrpy_rs/js/__init__.pyi b/python/natsrpy/_natsrpy_rs/js/__init__.pyi index 2eb4d5a..43fdf44 100644 --- a/python/natsrpy/_natsrpy_rs/js/__init__.pyi +++ b/python/natsrpy/_natsrpy_rs/js/__init__.pyi @@ -1,5 +1,5 @@ -from natsrpy._natsrpy_rs.js.kv import KeyValue, KVConfig -from natsrpy._natsrpy_rs.js.stream import Stream, StreamConfig +from .managers.kv import KVManager +from .managers.streams import StreamsManager class JetStream: async def publish( @@ -11,13 +11,7 @@ class JetStream: reply: str | None = None, err_on_disconnect: bool = False, ) -> None: ... - # KV - async def create_kv(self, config: KVConfig) -> KeyValue: ... - async def update_kv(self, config: KVConfig) -> KeyValue: ... - async def get_kv(self, bucket: str) -> KeyValue: ... - async def delete_kv(self, bucket: str) -> bool: ... - # Streams - async def create_stream(self, config: StreamConfig) -> Stream: ... - async def update_stream(self, config: StreamConfig) -> Stream: ... - async def get_stream(self, name: str) -> Stream: ... - async def delete_stream(self, name: str) -> bool: ... + @property + def kv(self) -> KVManager: ... + @property + def streams(self) -> StreamsManager: ... diff --git a/python/natsrpy/_natsrpy_rs/js/consumers/__init__.pyi b/python/natsrpy/_natsrpy_rs/js/consumers/__init__.pyi new file mode 100644 index 0000000..dd9a648 --- /dev/null +++ b/python/natsrpy/_natsrpy_rs/js/consumers/__init__.pyi @@ -0,0 +1,19 @@ +from .common import ( + AckPolicy, + DeliverPolicy, + PriorityPolicy, + ReplayPolicy, +) +from .pull import PullConsumer, PullConsumerConfig +from .push import PushConsumer, PushConsumerConfig + +__all__ = [ + "AckPolicy", + "DeliverPolicy", + "PriorityPolicy", + "PullConsumer", + "PullConsumerConfig", + "PushConsumer", + "PushConsumerConfig", + "ReplayPolicy", +] diff --git a/python/natsrpy/_natsrpy_rs/js/consumers/common.pyi b/python/natsrpy/_natsrpy_rs/js/consumers/common.pyi new file mode 100644 index 0000000..1816020 --- /dev/null +++ b/python/natsrpy/_natsrpy_rs/js/consumers/common.pyi @@ -0,0 +1,22 @@ +class DeliverPolicy: + ALL: DeliverPolicy + LAST: DeliverPolicy + NEW: DeliverPolicy + BY_START_SEQUENCE: DeliverPolicy + BY_START_TIME: DeliverPolicy + LAST_PER_SUBJECT: DeliverPolicy + +class AckPolicy: + EXPLICIT: AckPolicy + NONE: AckPolicy + ALL: AckPolicy + +class ReplayPolicy: + INSTANT: ReplayPolicy + ORIGINAL: ReplayPolicy + +class PriorityPolicy: + NONE: PriorityPolicy + OVERFLOW: PriorityPolicy + PINNED_CLIENT: PriorityPolicy + PRIORITIZED: PriorityPolicy diff --git a/python/natsrpy/_natsrpy_rs/js/consumers/pull.pyi b/python/natsrpy/_natsrpy_rs/js/consumers/pull.pyi new file mode 100644 index 0000000..27d5fc5 --- /dev/null +++ b/python/natsrpy/_natsrpy_rs/js/consumers/pull.pyi @@ -0,0 +1,72 @@ +from datetime import timedelta + +from natsrpy._natsrpy_rs.js.consumers.common import ( + AckPolicy, + DeliverPolicy, + PriorityPolicy, + ReplayPolicy, +) + +class PullConsumerConfig: + durable_name: str | None + name: str | None + description: str | None + deliver_policy: DeliverPolicy + delivery_start_sequence: int | None + delivery_start_time: int | None + ack_policy: AckPolicy + ack_wait: timedelta + max_deliver: int + filter_subject: str + filter_subjects: list[str] + replay_policy: ReplayPolicy + rate_limit: int + sample_frequency: int + max_waiting: int + max_ack_pending: int + headers_only: bool + max_batch: int + max_bytes: int + max_expires: timedelta + inactive_threshold: timedelta + num_replicas: int + memory_storage: bool + metadata: dict[str, str] + backoff: list[timedelta] + priority_policy: PriorityPolicy + priority_groups: list[str] + pause_until: int | None + + def __init__( + self, + durable_name: str | None = None, + name: str | None = None, + description: str | None = None, + deliver_policy: DeliverPolicy | None = None, + delivery_start_sequence: int | None = None, + delivery_start_time: int | None = None, + ack_policy: AckPolicy | None = None, + ack_wait: timedelta | None = None, + max_deliver: int | None = None, + filter_subject: str | None = None, + filter_subjects: list[str] | None = None, + replay_policy: ReplayPolicy | None = None, + rate_limit: int | None = None, + sample_frequency: int | None = None, + max_waiting: int | None = None, + max_ack_pending: int | None = None, + headers_only: bool | None = None, + max_batch: int | None = None, + max_bytes: int | None = None, + max_expires: timedelta | None = None, + inactive_threshold: timedelta | None = None, + num_replicas: int | None = None, + memory_storage: bool | None = None, + metadata: dict[str, str] | None = None, + backoff: list[timedelta] | None = None, + priority_policy: PriorityPolicy | None = None, + priority_groups: list[str] | None = None, + pause_until: int | None = None, + ) -> None: ... + +class PullConsumer: ... diff --git a/python/natsrpy/_natsrpy_rs/js/consumers/push.pyi b/python/natsrpy/_natsrpy_rs/js/consumers/push.pyi new file mode 100644 index 0000000..e816408 --- /dev/null +++ b/python/natsrpy/_natsrpy_rs/js/consumers/push.pyi @@ -0,0 +1,69 @@ +from datetime import timedelta + +from natsrpy._natsrpy_rs.js.consumers.common import ( + AckPolicy, + DeliverPolicy, + ReplayPolicy, +) + +class PushConsumerConfig: + deliver_subject: str + durable_name: str | None + name: str | None + description: str | None + deliver_group: str | None + deliver_policy: DeliverPolicy + delivery_start_sequence: int | None + delivery_start_time: int | None + ack_policy: AckPolicy + ack_wait: timedelta + max_deliver: int + filter_subject: str + filter_subjects: list[str] + replay_policy: ReplayPolicy + rate_limit: int + sample_frequency: int + max_waiting: int + max_ack_pending: int + headers_only: bool + flow_control: bool + idle_heartbeat: timedelta + num_replicas: int + memory_storage: bool + metadata: dict[str, str] + backoff: list[timedelta] + inactive_threshold: timedelta + pause_until: int | None + + def __init__( + self, + deliver_subject: str, + durable_name: str | None = None, + name: str | None = None, + description: str | None = None, + deliver_group: str | None = None, + deliver_policy: DeliverPolicy | None = None, + delivery_start_sequence: int | None = None, + delivery_start_time: int | None = None, + ack_policy: AckPolicy | None = None, + ack_wait: timedelta | None = None, + max_deliver: int | None = None, + filter_subject: str | None = None, + filter_subjects: list[str] | None = None, + replay_policy: ReplayPolicy | None = None, + rate_limit: int | None = None, + sample_frequency: int | None = None, + max_waiting: int | None = None, + max_ack_pending: int | None = None, + headers_only: bool | None = None, + flow_control: bool | None = None, + idle_heartbeat: timedelta | None = None, + num_replicas: int | None = None, + memory_storage: bool | None = None, + metadata: dict[str, str] | None = None, + backoff: list[timedelta] | None = None, + inactive_threshold: timedelta | None = None, + pause_until: int | None = None, + ) -> None: ... + +class PushConsumer: ... diff --git a/python/natsrpy/_natsrpy_rs/js/managers/__init__.pyi b/python/natsrpy/_natsrpy_rs/js/managers/__init__.pyi new file mode 100644 index 0000000..e69de29 diff --git a/python/natsrpy/_natsrpy_rs/js/managers/consumers.pyi b/python/natsrpy/_natsrpy_rs/js/managers/consumers.pyi new file mode 100644 index 0000000..fb04773 --- /dev/null +++ b/python/natsrpy/_natsrpy_rs/js/managers/consumers.pyi @@ -0,0 +1,14 @@ +from typing import overload + +from natsrpy._natsrpy_rs.js.consumers import ( + PullConsumer, + PullConsumerConfig, + PushConsumer, + PushConsumerConfig, +) + +class ConsumersManager: + @overload + async def create(self, config: PullConsumerConfig) -> PullConsumer: ... + @overload + async def create(self, config: PushConsumerConfig) -> PushConsumer: ... diff --git a/python/natsrpy/_natsrpy_rs/js/managers/kv.pyi b/python/natsrpy/_natsrpy_rs/js/managers/kv.pyi new file mode 100644 index 0000000..b998951 --- /dev/null +++ b/python/natsrpy/_natsrpy_rs/js/managers/kv.pyi @@ -0,0 +1,8 @@ +from natsrpy._natsrpy_rs.js.kv import KeyValue, KVConfig + +class KVManager: + async def create(self, config: KVConfig) -> KeyValue: ... + async def create_or_update(self, config: KVConfig) -> KeyValue: ... + async def get(self, bucket: str) -> KeyValue: ... + async def delete(self, bucket: str) -> None: ... + async def update(self, config: KVConfig) -> KeyValue: ... diff --git a/python/natsrpy/_natsrpy_rs/js/managers/streams.pyi b/python/natsrpy/_natsrpy_rs/js/managers/streams.pyi new file mode 100644 index 0000000..2a47b81 --- /dev/null +++ b/python/natsrpy/_natsrpy_rs/js/managers/streams.pyi @@ -0,0 +1,8 @@ +from natsrpy._natsrpy_rs.js.stream import Stream, StreamConfig + +class StreamsManager: + async def create(self, config: StreamConfig) -> Stream: ... + async def create_or_update(self, config: StreamConfig) -> Stream: ... + async def get(self, bucket: str) -> Stream: ... + async def delete(self, bucket: str) -> None: ... + async def update(self, config: StreamConfig) -> Stream: ... diff --git a/python/natsrpy/_natsrpy_rs/js/stream.pyi b/python/natsrpy/_natsrpy_rs/js/stream.pyi index 41b0ffe..506e08a 100644 --- a/python/natsrpy/_natsrpy_rs/js/stream.pyi +++ b/python/natsrpy/_natsrpy_rs/js/stream.pyi @@ -1,6 +1,8 @@ from datetime import datetime, timedelta from typing import Any +from natsrpy._natsrpy_rs.js.managers.consumers import ConsumersManager + class StorageType: FILE: StorageType MEMORY: StorageType @@ -256,3 +258,6 @@ class Stream: defaults to None :return: number of messages purged """ + + @property + def consumers(self) -> ConsumersManager: ... diff --git a/python/natsrpy/js/consumers/__init__.py b/python/natsrpy/js/consumers/__init__.py new file mode 100644 index 0000000..03d442f --- /dev/null +++ b/python/natsrpy/js/consumers/__init__.py @@ -0,0 +1,17 @@ +from natsrpy._natsrpy_rs.js.consumers import ( + AckPolicy, + DeliverPolicy, + PriorityPolicy, + PullConsumerConfig, + PushConsumerConfig, + ReplayPolicy, +) + +__all__ = [ + "AckPolicy", + "DeliverPolicy", + "PriorityPolicy", + "PullConsumerConfig", + "PushConsumerConfig", + "ReplayPolicy", +] diff --git a/src/js/consumers/_pull.rs b/src/js/consumers/_pull.rs deleted file mode 100644 index 3a8d3e6..0000000 --- a/src/js/consumers/_pull.rs +++ /dev/null @@ -1,55 +0,0 @@ -use std::sync::Arc; - -use futures_util::StreamExt; -use pyo3::{Bound, PyAny, Python}; -use tokio::sync::RwLock; - -use crate::{exceptions::rust_err::NatsrpyResult, utils::natsrpy_future}; - -type NatsPullConsumer = - async_nats::jetstream::consumer::Consumer; - -#[pyo3::pyclass(from_py_object)] -#[derive(Debug, Clone)] -pub struct PullConsumer { - consumer: Arc>, -} - -#[pyo3::pyclass(from_py_object)] -#[derive(Debug, Clone)] -pub struct OrderedPullConsumer { - consumer: Arc>, -} - -impl PullConsumer { - pub fn new(consumer: NatsPullConsumer) -> Self { - Self { - consumer: Arc::new(RwLock::new(consumer)), - } - } -} - -#[pyo3::pyclass] -pub struct PullMessageIterator { - inner: Arc>, -} - -#[pyo3::pymethods] -impl PullConsumer { - pub fn messages<'py>(&self, py: Python<'py>) -> NatsrpyResult> { - let consumer_lock = self.consumer.clone(); - natsrpy_future(py, async move { - let mut messages = consumer_lock.read().await.messages().await.unwrap(); - while let Some(message) = messages.next().await { - let msg = message?; - log::info!("{:#?}", msg.message.payload); - msg.ack().await?; - } - - Ok(()) - }) - } -} - -#[pyo3::pymethods] -impl PullMessageIterator {} diff --git a/src/js/consumers/_push.rs b/src/js/consumers/_push.rs deleted file mode 100644 index cbfcba0..0000000 --- a/src/js/consumers/_push.rs +++ /dev/null @@ -1,3 +0,0 @@ -#[pyo3::pyclass(from_py_object)] -#[derive(Debug, Clone)] -pub struct PushConsumer; diff --git a/src/js/consumers/common.rs b/src/js/consumers/common.rs index 84c4022..e8f2e08 100644 --- a/src/js/consumers/common.rs +++ b/src/js/consumers/common.rs @@ -4,12 +4,12 @@ use crate::exceptions::rust_err::{NatsrpyError, NatsrpyResult}; #[derive(Debug, Clone, Default, Copy, PartialEq, Eq, PartialOrd)] pub enum DeliverPolicy { #[default] - All, - Last, - New, - ByStartSequence, - ByStartTime, - LastPerSubject, + ALL, + LAST, + NEW, + BY_START_SEQUENCE, + BY_START_TIME, + LAST_PER_SUBJECT, } impl DeliverPolicy { @@ -19,22 +19,24 @@ impl DeliverPolicy { start_time: Option, ) -> NatsrpyResult { let result = match self { - Self::All => async_nats::jetstream::consumer::DeliverPolicy::All, - Self::Last => async_nats::jetstream::consumer::DeliverPolicy::Last, - Self::New => async_nats::jetstream::consumer::DeliverPolicy::New, - Self::LastPerSubject => async_nats::jetstream::consumer::DeliverPolicy::Last, - Self::ByStartSequence => { + Self::ALL => async_nats::jetstream::consumer::DeliverPolicy::All, + Self::LAST => async_nats::jetstream::consumer::DeliverPolicy::Last, + Self::NEW => async_nats::jetstream::consumer::DeliverPolicy::New, + Self::LAST_PER_SUBJECT => { + async_nats::jetstream::consumer::DeliverPolicy::LastPerSubject + } + Self::BY_START_SEQUENCE => { let Some(start_sequence) = start_sequence else { return Err(NatsrpyError::SessionError(String::from( - "Start sequence is not present", + "Start sequence is not present, but deliver_policy is set to BY_START_SEQUENCE", ))); }; async_nats::jetstream::consumer::DeliverPolicy::ByStartSequence { start_sequence } } - Self::ByStartTime => { + Self::BY_START_TIME => { let Some(start_time) = start_time else { return Err(NatsrpyError::SessionError(String::from( - "Start sequence is not present", + "Start time is not present, but deliver_policy is set to BY_START_TIME", ))); }; async_nats::jetstream::consumer::DeliverPolicy::ByStartTime { @@ -50,17 +52,17 @@ impl DeliverPolicy { #[derive(Debug, Clone, Default, Copy, PartialEq, Eq, PartialOrd)] pub enum AckPolicy { #[default] - Explicit, - None, - All, + EXPLICIT, + NONE, + ALL, } impl From for async_nats::jetstream::consumer::AckPolicy { fn from(value: AckPolicy) -> Self { match value { - AckPolicy::Explicit => Self::Explicit, - AckPolicy::None => Self::None, - AckPolicy::All => Self::All, + AckPolicy::EXPLICIT => Self::Explicit, + AckPolicy::NONE => Self::None, + AckPolicy::ALL => Self::All, } } } @@ -69,15 +71,15 @@ impl From for async_nats::jetstream::consumer::AckPolicy { #[derive(Debug, Clone, Default, Copy, PartialEq, Eq, PartialOrd)] pub enum ReplayPolicy { #[default] - Instant, - Original, + INSTANT, + ORIGINAL, } impl From for async_nats::jetstream::consumer::ReplayPolicy { fn from(value: ReplayPolicy) -> Self { match value { - ReplayPolicy::Instant => Self::Instant, - ReplayPolicy::Original => Self::Original, + ReplayPolicy::INSTANT => Self::Instant, + ReplayPolicy::ORIGINAL => Self::Original, } } } @@ -86,19 +88,19 @@ impl From for async_nats::jetstream::consumer::ReplayPolicy { #[derive(Debug, Clone, Default, Copy, PartialEq, Eq, PartialOrd)] pub enum PriorityPolicy { #[default] - None, - Overflow, - PinnedClient, - Prioritized, + NONE, + OVERFLOW, + PINNED_CLIENT, + PRIORITIZED, } impl From for async_nats::jetstream::consumer::PriorityPolicy { fn from(value: PriorityPolicy) -> Self { match value { - PriorityPolicy::None => Self::None, - PriorityPolicy::Overflow => Self::Overflow, - PriorityPolicy::PinnedClient => Self::PinnedClient, - PriorityPolicy::Prioritized => Self::Prioritized, + PriorityPolicy::NONE => Self::None, + PriorityPolicy::OVERFLOW => Self::Overflow, + PriorityPolicy::PINNED_CLIENT => Self::PinnedClient, + PriorityPolicy::PRIORITIZED => Self::Prioritized, } } } diff --git a/src/js/consumers/mod.rs b/src/js/consumers/mod.rs index 601ffc5..41680d5 100644 --- a/src/js/consumers/mod.rs +++ b/src/js/consumers/mod.rs @@ -9,5 +9,5 @@ pub mod pymod { #[pymodule_export] pub use super::pull::{config::PullConsumerConfig, consumer::PullConsumer}; #[pymodule_export] - pub use super::push::config::PushConsumerConfig; + pub use super::push::{config::PushConsumerConfig, consumer::PushConsumer}; } diff --git a/src/js/consumers/pull/config.rs b/src/js/consumers/pull/config.rs index d0ddef0..a963433 100644 --- a/src/js/consumers/pull/config.rs +++ b/src/js/consumers/pull/config.rs @@ -6,7 +6,7 @@ use crate::{ }; #[pyo3::pyclass(from_py_object, get_all, set_all)] -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Default)] pub struct PullConsumerConfig { pub durable_name: Option, pub name: Option, @@ -38,7 +38,106 @@ pub struct PullConsumerConfig { pub pause_until: Option, } -impl PullConsumerConfig {} +#[pyo3::pymethods] +impl PullConsumerConfig { + #[new] + #[pyo3(signature=( + durable_name=None, + name=None, + description=None, + deliver_policy=None, + delivery_start_sequence=None, + delivery_start_time=None, + ack_policy=None, + ack_wait=None, + max_deliver=None, + filter_subject=None, + filter_subjects=None, + replay_policy=None, + rate_limit=None, + sample_frequency=None, + max_waiting=None, + max_ack_pending=None, + headers_only=None, + max_batch=None, + max_bytes=None, + max_expires=None, + inactive_threshold=None, + num_replicas=None, + memory_storage=None, + metadata=None, + backoff=None, + priority_policy=None, + priority_groups=None, + pause_until=None, + ))] + #[must_use] + pub fn __new__( + durable_name: Option, + name: Option, + description: Option, + deliver_policy: Option, + delivery_start_sequence: Option, + delivery_start_time: Option, + ack_policy: Option, + ack_wait: Option, + max_deliver: Option, + filter_subject: Option, + filter_subjects: Option>, + replay_policy: Option, + rate_limit: Option, + sample_frequency: Option, + max_waiting: Option, + max_ack_pending: Option, + headers_only: Option, + max_batch: Option, + max_bytes: Option, + max_expires: Option, + inactive_threshold: Option, + num_replicas: Option, + memory_storage: Option, + metadata: Option>, + backoff: Option>, + priority_policy: Option, + priority_groups: Option>, + pause_until: Option, + ) -> Self { + let mut conf = Self { + durable_name, + name, + description, + delivery_start_sequence, + delivery_start_time, + pause_until, + ..Default::default() + }; + + conf.deliver_policy = deliver_policy.unwrap_or_default(); + conf.ack_policy = ack_policy.unwrap_or_default(); + conf.ack_wait = ack_wait.unwrap_or_default(); + conf.max_deliver = max_deliver.unwrap_or_default(); + conf.filter_subject = filter_subject.unwrap_or_default(); + conf.filter_subjects = filter_subjects.unwrap_or_default(); + conf.replay_policy = replay_policy.unwrap_or_default(); + conf.rate_limit = rate_limit.unwrap_or_default(); + conf.sample_frequency = sample_frequency.unwrap_or_default(); + conf.max_waiting = max_waiting.unwrap_or_default(); + conf.max_ack_pending = max_ack_pending.unwrap_or_default(); + conf.headers_only = headers_only.unwrap_or_default(); + conf.max_batch = max_batch.unwrap_or_default(); + conf.max_bytes = max_bytes.unwrap_or_default(); + conf.max_expires = max_expires.unwrap_or_default(); + conf.inactive_threshold = inactive_threshold.unwrap_or_default(); + conf.num_replicas = num_replicas.unwrap_or_default(); + conf.memory_storage = memory_storage.unwrap_or_default(); + conf.metadata = metadata.unwrap_or_default(); + conf.backoff = backoff.unwrap_or_default(); + conf.priority_policy = priority_policy.unwrap_or_default(); + conf.priority_groups = priority_groups.unwrap_or_default(); + + conf + } +} impl TryFrom for async_nats::jetstream::consumer::pull::Config { type Error = NatsrpyError; diff --git a/src/js/consumers/pull/consumer.rs b/src/js/consumers/pull/consumer.rs index f24aaaa..a8fcb71 100644 --- a/src/js/consumers/pull/consumer.rs +++ b/src/js/consumers/pull/consumer.rs @@ -1,10 +1,7 @@ use std::sync::Arc; -use futures_util::StreamExt; -use pyo3::{Bound, PyAny, Python}; use tokio::sync::RwLock; -use crate::{exceptions::rust_err::NatsrpyResult, utils::natsrpy_future}; type NatsPullConsumer = async_nats::jetstream::consumer::Consumer; @@ -30,21 +27,7 @@ pub struct PullMessageIterator { } #[pyo3::pymethods] -impl PullConsumer { - pub fn messages<'py>(&self, py: Python<'py>) -> NatsrpyResult> { - let consumer_lock = self.consumer.clone(); - natsrpy_future(py, async move { - let mut messages = consumer_lock.read().await.messages().await.unwrap(); - while let Some(message) = messages.next().await { - let msg = message?; - log::info!("{:#?}", msg.message.payload); - msg.ack().await?; - } - - Ok(()) - }) - } -} +impl PullConsumer {} #[pyo3::pymethods] impl PullMessageIterator {} diff --git a/src/js/consumers/push/config.rs b/src/js/consumers/push/config.rs index fb888b9..71cfc46 100644 --- a/src/js/consumers/push/config.rs +++ b/src/js/consumers/push/config.rs @@ -37,6 +37,102 @@ pub struct PushConsumerConfig { pub pause_until: Option, } +#[pyo3::pymethods] +impl PushConsumerConfig { + #[new] + #[pyo3(signature=( + deliver_subject, + durable_name=None, + name=None, + description=None, + deliver_group=None, + deliver_policy=None, + delivery_start_sequence=None, + delivery_start_time=None, + ack_policy=None, + ack_wait=None, + max_deliver=None, + filter_subject=None, + filter_subjects=None, + replay_policy=None, + rate_limit=None, + sample_frequency=None, + max_waiting=None, + max_ack_pending=None, + headers_only=None, + flow_control=None, + idle_heartbeat=None, + num_replicas=None, + memory_storage=None, + metadata=None, + backoff=None, + inactive_threshold=None, + pause_until=None, + + ))] + #[must_use] + pub fn __new__( + deliver_subject: String, + durable_name: Option, + name: Option, + description: Option, + deliver_group: Option, + deliver_policy: Option, + delivery_start_sequence: Option, + delivery_start_time: Option, + ack_policy: Option, + ack_wait: Option, + max_deliver: Option, + filter_subject: Option, + filter_subjects: Option>, + replay_policy: Option, + rate_limit: Option, + sample_frequency: Option, + max_waiting: Option, + max_ack_pending: Option, + headers_only: Option, + flow_control: Option, + idle_heartbeat: Option, + num_replicas: Option, + memory_storage: Option, + metadata: Option>, + backoff: Option>, + inactive_threshold: Option, + pause_until: Option, + ) -> Self { + Self { + deliver_subject, + durable_name, + name, + description, + deliver_group, + delivery_start_sequence, + delivery_start_time, + pause_until, + + deliver_policy: deliver_policy.unwrap_or_default(), + ack_policy: ack_policy.unwrap_or_default(), + ack_wait: ack_wait.unwrap_or_default(), + max_deliver: max_deliver.unwrap_or_default(), + filter_subject: filter_subject.unwrap_or_default(), + filter_subjects: filter_subjects.unwrap_or_default(), + replay_policy: replay_policy.unwrap_or_default(), + rate_limit: rate_limit.unwrap_or_default(), + sample_frequency: sample_frequency.unwrap_or_default(), + max_waiting: max_waiting.unwrap_or_default(), + max_ack_pending: max_ack_pending.unwrap_or_default(), + headers_only: headers_only.unwrap_or_default(), + flow_control: flow_control.unwrap_or_default(), + idle_heartbeat: idle_heartbeat.unwrap_or_default(), + num_replicas: num_replicas.unwrap_or_default(), + memory_storage: memory_storage.unwrap_or_default(), + metadata: metadata.unwrap_or_default(), + backoff: backoff.unwrap_or_default(), + inactive_threshold: inactive_threshold.unwrap_or_default(), + } + } +} + impl TryFrom for async_nats::jetstream::consumer::push::Config { type Error = NatsrpyError; diff --git a/src/js/consumers/push/consumer.rs b/src/js/consumers/push/consumer.rs new file mode 100644 index 0000000..05c88b3 --- /dev/null +++ b/src/js/consumers/push/consumer.rs @@ -0,0 +1,24 @@ +use std::sync::Arc; + +use tokio::sync::RwLock; + +type NatsPushConsumer = + async_nats::jetstream::consumer::Consumer; + +#[pyo3::pyclass(from_py_object)] +#[derive(Debug, Clone)] +pub struct PushConsumer { + consumer: Arc>, +} + +impl PushConsumer { + #[must_use] + pub fn new(consumer: NatsPushConsumer) -> Self { + Self { + consumer: Arc::new(RwLock::new(consumer)), + } + } +} + +#[pyo3::pymethods] +impl PushConsumer {} diff --git a/src/js/consumers/push/mod.rs b/src/js/consumers/push/mod.rs index 3aa043f..f7a77e0 100644 --- a/src/js/consumers/push/mod.rs +++ b/src/js/consumers/push/mod.rs @@ -1,3 +1,5 @@ pub mod config; +pub mod consumer; pub use config::PushConsumerConfig; +pub use consumer::PushConsumer; diff --git a/src/js/jetstream.rs b/src/js/jetstream.rs index 7d56fdd..49e32d6 100644 --- a/src/js/jetstream.rs +++ b/src/js/jetstream.rs @@ -1,4 +1,4 @@ -use std::{ops::Deref, sync::Arc}; +use std::sync::Arc; use async_nats::{Subject, client::traits::Publisher, connection::State}; use pyo3::{ @@ -9,10 +9,7 @@ use tokio::sync::RwLock; use crate::{ exceptions::rust_err::{NatsrpyError, NatsrpyResult}, - js::{ - kv::{KVConfig, KeyValue}, - stream::StreamConfig, - }, + js::managers::{kv::KVManager, streams::StreamsManager}, utils::{headers::NatsrpyHeadermapExt, natsrpy_future}, }; @@ -73,89 +70,15 @@ impl JetStream { }) } - pub fn create_kv<'py>( - &self, - py: Python<'py>, - config: &Bound<'py, KVConfig>, - ) -> NatsrpyResult> { - let ctx = self.ctx.clone(); - let config = config.borrow().deref().clone().try_into()?; - - natsrpy_future(py, async move { - let js = ctx.read().await; - Ok(KeyValue::new(js.create_key_value(config).await?)) - }) - } - - pub fn get_kv<'py>(&self, py: Python<'py>, bucket: String) -> NatsrpyResult> { - let ctx = self.ctx.clone(); - natsrpy_future(py, async move { - let js = ctx.read().await; - Ok(KeyValue::new(js.get_key_value(bucket).await?)) - }) - } - - pub fn update_kv<'py>( - &self, - py: Python<'py>, - config: &Bound<'py, KVConfig>, - ) -> NatsrpyResult> { - let ctx = self.ctx.clone(); - let config = config.borrow().deref().clone().try_into()?; - natsrpy_future(py, async move { - let js = ctx.read().await; - Ok(KeyValue::new(js.update_key_value(config).await?)) - }) - } - - pub fn delete_kv<'py>( - &self, - py: Python<'py>, - bucket: String, - ) -> NatsrpyResult> { - let ctx = self.ctx.clone(); - natsrpy_future(py, async move { - let js = ctx.read().await; - Ok(js.delete_key_value(bucket).await?.success) - }) - } - - pub fn get_stream<'py>( - &self, - py: Python<'py>, - name: String, - ) -> NatsrpyResult> { - let ctx = self.ctx.clone(); - natsrpy_future(py, async move { - let js = ctx.read().await; - Ok(super::stream::Stream::new(js.get_stream(name).await?)) - }) - } - - pub fn create_stream<'py>( - &self, - py: Python<'py>, - config: StreamConfig, - ) -> NatsrpyResult> { - let ctx = self.ctx.clone(); - natsrpy_future(py, async move { - let js = ctx.read().await; - Ok(super::stream::Stream::new( - js.create_stream(async_nats::jetstream::stream::Config::try_from(config)?) - .await?, - )) - }) + #[getter] + #[must_use] + pub fn kv(&self) -> KVManager { + KVManager::new(self.ctx.clone()) } - pub fn delete_stream<'py>( - &self, - py: Python<'py>, - name: String, - ) -> NatsrpyResult> { - let ctx = self.ctx.clone(); - natsrpy_future(py, async move { - let js = ctx.read().await; - Ok(js.delete_stream(name).await?.success) - }) + #[getter] + #[must_use] + pub fn streams(&self) -> StreamsManager { + StreamsManager::new(self.ctx.clone()) } } diff --git a/src/js/managers/consumers.rs b/src/js/managers/consumers.rs new file mode 100644 index 0000000..12e346a --- /dev/null +++ b/src/js/managers/consumers.rs @@ -0,0 +1,89 @@ +use std::sync::Arc; + +use pyo3::{Bound, FromPyObject, IntoPyObjectExt, PyAny, Python}; +use tokio::sync::RwLock; + +use crate::{ + exceptions::rust_err::{NatsrpyError, NatsrpyResult}, + js::consumers::{self, pull::PullConsumer, push::PushConsumer}, + utils::natsrpy_future, +}; + +#[pyo3::pyclass] +pub struct ConsumersManager { + stream: Arc>>, +} + +impl ConsumersManager { + pub const fn new( + stream: Arc< + RwLock>, + >, + ) -> Self { + Self { stream } + } +} + +pub enum ConsumerConfigs { + Pull(consumers::pull::PullConsumerConfig), + Push(consumers::push::PushConsumerConfig), +} + +impl<'py> FromPyObject<'_, 'py> for ConsumerConfigs { + type Error = NatsrpyError; + + fn extract(obj: pyo3::Borrowed<'_, 'py, PyAny>) -> Result { + #[allow(clippy::option_if_let_else)] + if let Ok(conf) = obj.extract::() { + Ok(Self::Pull(conf)) + } else if let Ok(conf) = obj.extract::() { + Ok(Self::Push(conf)) + } else { + Err(NatsrpyError::InvalidArgument(String::from( + "Unknown value passed as consumer config. Only consumer config classes are accepted.", + ))) + } + } +} + +#[pyo3::pyclass] +pub enum Consumers { + Pull(consumers::pull::PullConsumer), + Push(consumers::push::PushConsumer), +} + +#[pyo3::pymethods] +impl ConsumersManager { + pub fn create<'py>( + &self, + py: Python<'py>, + config: ConsumerConfigs, + ) -> NatsrpyResult> { + let ctx = self.stream.clone(); + natsrpy_future(py, async move { + match config { + ConsumerConfigs::Pull(config) => { + let consumer = PullConsumer::new( + ctx.read().await.create_consumer(config.try_into()?).await?, + ); + Ok(Python::attach(|gil| consumer.into_py_any(gil))?) + } + ConsumerConfigs::Push(config) => { + let consumer = PushConsumer::new( + ctx.read().await.create_consumer(config.try_into()?).await?, + ); + Ok(Python::attach(|gil| consumer.into_py_any(gil))?) + } + } + }) + } + + pub fn get<'py>(&self, py: Python<'py>, name: String) -> NatsrpyResult> { + let ctx = self.stream.clone(); + natsrpy_future(py, async move { + Ok(consumers::pull::consumer::PullConsumer::new( + ctx.read().await.get_consumer(&name).await?, + )) + }) + } +} diff --git a/src/js/managers/kv.rs b/src/js/managers/kv.rs new file mode 100644 index 0000000..f805a2f --- /dev/null +++ b/src/js/managers/kv.rs @@ -0,0 +1,86 @@ +use std::sync::Arc; + +use pyo3::{Bound, PyAny, Python}; +use tokio::sync::RwLock; + +use crate::{ + exceptions::rust_err::NatsrpyResult, + js::kv::{KVConfig, KeyValue}, + utils::natsrpy_future, +}; + +#[pyo3::pyclass] +pub struct KVManager { + ctx: Arc>, +} + +impl KVManager { + pub const fn new(ctx: Arc>) -> Self { + Self { ctx } + } +} + +#[pyo3::pymethods] +impl KVManager { + pub fn create<'py>( + &self, + py: Python<'py>, + config: KVConfig, + ) -> NatsrpyResult> { + let ctx = self.ctx.clone(); + natsrpy_future(py, async move { + Ok(KeyValue::new( + ctx.read() + .await + .create_key_value(config.try_into()?) + .await?, + )) + }) + } + + pub fn create_or_update<'py>( + &self, + py: Python<'py>, + config: KVConfig, + ) -> NatsrpyResult> { + let ctx = self.ctx.clone(); + natsrpy_future(py, async move { + Ok(KeyValue::new( + ctx.read() + .await + .create_or_update_key_value(config.try_into()?) + .await?, + )) + }) + } + + pub fn get<'py>(&self, py: Python<'py>, bucket: String) -> NatsrpyResult> { + let ctx = self.ctx.clone(); + natsrpy_future(py, async move { + Ok(KeyValue::new(ctx.read().await.get_key_value(bucket).await?)) + }) + } + + pub fn delete<'py>(&self, py: Python<'py>, bucket: String) -> NatsrpyResult> { + let ctx = self.ctx.clone(); + natsrpy_future(py, async move { + Ok(ctx.read().await.delete_key_value(bucket).await?.success) + }) + } + + pub fn update<'py>( + &self, + py: Python<'py>, + config: KVConfig, + ) -> NatsrpyResult> { + let ctx = self.ctx.clone(); + natsrpy_future(py, async move { + Ok(KeyValue::new( + ctx.read() + .await + .update_key_value(config.try_into()?) + .await?, + )) + }) + } +} diff --git a/src/js/managers/mod.rs b/src/js/managers/mod.rs new file mode 100644 index 0000000..99d5e38 --- /dev/null +++ b/src/js/managers/mod.rs @@ -0,0 +1,13 @@ +pub mod consumers; +pub mod kv; +pub mod streams; + +#[pyo3::pymodule(submodule, name = "managers")] +pub mod pymod { + #[pymodule_export] + use super::consumers::ConsumersManager; + #[pymodule_export] + use super::kv::KVManager; + #[pymodule_export] + use super::streams::StreamsManager; +} diff --git a/src/js/managers/streams.rs b/src/js/managers/streams.rs new file mode 100644 index 0000000..66700f2 --- /dev/null +++ b/src/js/managers/streams.rs @@ -0,0 +1,87 @@ +use std::sync::Arc; + +use crate::js::stream::Stream; +use pyo3::{Bound, PyAny, Python}; +use tokio::sync::RwLock; + +use crate::{exceptions::rust_err::NatsrpyResult, js::stream::StreamConfig, utils::natsrpy_future}; + +#[pyo3::pyclass] +pub struct StreamsManager { + ctx: Arc>, +} + +impl StreamsManager { + pub const fn new(ctx: Arc>) -> Self { + Self { ctx } + } +} + +#[pyo3::pymethods] +impl StreamsManager { + pub fn create<'py>( + &self, + py: Python<'py>, + config: StreamConfig, + ) -> NatsrpyResult> { + let ctx = self.ctx.clone(); + natsrpy_future(py, async move { + let js = ctx.read().await; + Ok(Stream::new( + js.create_stream(async_nats::jetstream::stream::Config::try_from(config)?) + .await?, + )) + }) + } + + pub fn create_or_update<'py>( + &self, + py: Python<'py>, + config: StreamConfig, + ) -> NatsrpyResult> { + let ctx = self.ctx.clone(); + natsrpy_future(py, async move { + let info = ctx + .read() + .await + .create_or_update_stream(async_nats::jetstream::stream::Config::try_from(config)?) + .await?; + Ok(Stream::new( + ctx.read().await.get_stream(info.config.name).await?, + )) + }) + } + + pub fn get<'py>(&self, py: Python<'py>, name: String) -> NatsrpyResult> { + let ctx = self.ctx.clone(); + natsrpy_future(py, async move { + Ok(Stream::new(ctx.read().await.get_stream(name).await?)) + }) + } + + pub fn delete<'py>(&self, py: Python<'py>, name: String) -> NatsrpyResult> { + let ctx = self.ctx.clone(); + natsrpy_future(py, async move { + let js = ctx.read().await; + Ok(js.delete_stream(name).await?.success) + }) + } + + pub fn update<'py>( + &self, + py: Python<'py>, + config: StreamConfig, + ) -> NatsrpyResult> { + let ctx = self.ctx.clone(); + natsrpy_future(py, async move { + let info = ctx + .read() + .await + .update_stream(async_nats::jetstream::stream::Config::try_from(config)?) + .await?; + Ok(Stream::new( + ctx.read().await.get_stream(info.config.name).await?, + )) + }) + } +} diff --git a/src/js/mod.rs b/src/js/mod.rs index 5672495..7d1548f 100644 --- a/src/js/mod.rs +++ b/src/js/mod.rs @@ -1,6 +1,7 @@ pub mod consumers; pub mod jetstream; pub mod kv; +pub mod managers; pub mod stream; #[pyo3::pymodule(submodule, name = "js")] @@ -18,5 +19,7 @@ pub mod pymod { #[pymodule_export] pub use super::kv::pymod as kv; #[pymodule_export] + pub use super::managers::pymod as managers; + #[pymodule_export] pub use super::stream::pymod as stream; } diff --git a/src/js/stream.rs b/src/js/stream.rs index 4995102..de300f0 100644 --- a/src/js/stream.rs +++ b/src/js/stream.rs @@ -6,7 +6,7 @@ use std::{collections::HashMap, ops::Deref, sync::Arc, time::Duration}; use crate::{ exceptions::rust_err::{NatsrpyError, NatsrpyResult}, - js::consumers::{self}, + js::managers::consumers::ConsumersManager, utils::{headers::NatsrpyHeadermapExt, natsrpy_future}, }; use pyo3::{Bound, PyAny, Python}; @@ -493,6 +493,7 @@ impl From for ClusterInfo { #[pyo3::pyclass(from_py_object, get_all, set_all)] #[derive(Debug, Clone, Default)] +#[allow(clippy::struct_excessive_bools)] pub struct StreamConfig { pub name: String, pub subjects: Vec, @@ -620,7 +621,7 @@ impl StreamConfig { allow_message_schedules: Option, allow_message_counter: Option, ) -> NatsrpyResult { - let mut config = Self { + let config = Self { name, subjects, description, @@ -635,38 +636,33 @@ impl StreamConfig { persist_mode, pause_until, subject_delete_marker_ttl, - ..Default::default() - }; - config.max_bytes = max_bytes.unwrap_or(config.max_bytes); - config.max_messages = max_messages.unwrap_or(config.max_messages); - config.max_messages_per_subject = - max_messages_per_subject.unwrap_or(config.max_messages_per_subject); - config.discard = discard.unwrap_or(config.discard); - config.discard_new_per_subject = - discard_new_per_subject.unwrap_or(config.discard_new_per_subject); - config.retention = retention.unwrap_or(config.retention); - config.max_consumers = max_consumers.unwrap_or(config.max_consumers); - config.max_age = max_age.unwrap_or(config.max_age); - config.max_message_size = max_message_size.unwrap_or(config.max_message_size); - config.storage = storage.unwrap_or(config.storage); - config.num_replicas = num_replicas.unwrap_or(config.num_replicas); - config.no_ack = no_ack.unwrap_or(config.no_ack); - config.duplicate_window = duplicate_window.unwrap_or(config.duplicate_window); - config.template_owner = template_owner.unwrap_or(config.template_owner); - config.sealed = sealed.unwrap_or(config.sealed); - config.allow_rollup = allow_rollup.unwrap_or(config.allow_rollup); - config.deny_delete = deny_delete.unwrap_or(config.deny_delete); - config.deny_purge = deny_purge.unwrap_or(config.deny_purge); - config.allow_direct = allow_direct.unwrap_or(config.allow_direct); - config.mirror_direct = mirror_direct.unwrap_or(config.mirror_direct); - config.metadata = metadata.unwrap_or(config.metadata); - config.allow_message_ttl = allow_message_ttl.unwrap_or(config.allow_message_ttl); - config.allow_atomic_publish = allow_atomic_publish.unwrap_or(config.allow_atomic_publish); - config.allow_message_schedules = - allow_message_schedules.unwrap_or(config.allow_message_schedules); - config.allow_message_counter = - allow_message_counter.unwrap_or(config.allow_message_counter); + max_bytes: max_bytes.unwrap_or_default(), + max_messages: max_messages.unwrap_or_default(), + max_messages_per_subject: max_messages_per_subject.unwrap_or_default(), + discard: discard.unwrap_or_default(), + discard_new_per_subject: discard_new_per_subject.unwrap_or_default(), + retention: retention.unwrap_or_default(), + max_consumers: max_consumers.unwrap_or_default(), + max_age: max_age.unwrap_or_default(), + max_message_size: max_message_size.unwrap_or_default(), + storage: storage.unwrap_or_default(), + num_replicas: num_replicas.unwrap_or_default(), + no_ack: no_ack.unwrap_or_default(), + duplicate_window: duplicate_window.unwrap_or_default(), + template_owner: template_owner.unwrap_or_default(), + sealed: sealed.unwrap_or_default(), + allow_rollup: allow_rollup.unwrap_or_default(), + deny_delete: deny_delete.unwrap_or_default(), + deny_purge: deny_purge.unwrap_or_default(), + allow_direct: allow_direct.unwrap_or_default(), + mirror_direct: mirror_direct.unwrap_or_default(), + metadata: metadata.unwrap_or_default(), + allow_message_ttl: allow_message_ttl.unwrap_or_default(), + allow_atomic_publish: allow_atomic_publish.unwrap_or_default(), + allow_message_schedules: allow_message_schedules.unwrap_or_default(), + allow_message_counter: allow_message_counter.unwrap_or_default(), + }; Ok(config) } @@ -969,6 +965,12 @@ impl Stream { #[pyo3::pymethods] impl Stream { + #[getter] + #[must_use] + pub fn consumers(&self) -> ConsumersManager { + ConsumersManager::new(self.stream.clone()) + } + pub fn direct_get<'py>( &self, py: Python<'py>, @@ -1027,37 +1029,6 @@ impl Stream { Ok(resp.purged) }) } - - pub fn create_pull_consumer<'py>( - &self, - py: Python<'py>, - config: consumers::pull::PullConsumerConfig, - ) -> NatsrpyResult> { - let ctx = self.stream.clone(); - natsrpy_future(py, async move { - Ok(super::consumers::pull::consumer::PullConsumer::new( - ctx.read() - .await - .create_consumer(async_nats::jetstream::consumer::pull::Config::try_from( - config, - )?) - .await?, - )) - }) - } - - pub fn get_pull_consumer<'py>( - &self, - py: Python<'py>, - name: String, - ) -> NatsrpyResult> { - let ctx = self.stream.clone(); - natsrpy_future(py, async move { - Ok(super::consumers::pull::consumer::PullConsumer::new( - ctx.read().await.get_consumer(&name).await?, - )) - }) - } } #[pyo3::pymodule(submodule, name = "stream")] diff --git a/src/lib.rs b/src/lib.rs index 60286ba..e6abd67 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8,6 +8,9 @@ )] #![ allow( + // Beacsue some types naming conventions are + // just different to the ones that rust have. + non_camel_case_types, // I don't care about this. clippy::module_name_repetitions, // Yo, the hell you should put diff --git a/src/nats_cls.rs b/src/nats_cls.rs index 3f113a1..83b3e72 100644 --- a/src/nats_cls.rs +++ b/src/nats_cls.rs @@ -9,7 +9,7 @@ use tokio::sync::RwLock; use crate::{ exceptions::rust_err::NatsrpyError, subscription::Subscription, - utils::{headers::NatsrpyHeadermapExt, natsrpy_future}, + utils::{headers::NatsrpyHeadermapExt, natsrpy_future, py_types::SendableValue}, }; #[pyo3::pyclass(name = "Nats")] @@ -118,13 +118,14 @@ impl NatsCls { &self, py: Python<'py>, subject: String, - payload: &Bound, + payload: SendableValue, headers: Option>, reply: Option, err_on_disconnect: bool, ) -> PyResult> { let session = self.nats_session.clone(); - let data = bytes::Bytes::copy_from_slice(payload.as_bytes()); + log::info!("Payload: {payload:?}"); + let data = payload.into(); let headermap = headers .map(async_nats::HeaderMap::from_pydict) .transpose()?; diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 1b9e060..d574516 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -1,5 +1,6 @@ pub mod futures; pub mod headers; pub mod py; +pub mod py_types; pub use futures::natsrpy_future; diff --git a/src/utils/py_types.rs b/src/utils/py_types.rs new file mode 100644 index 0000000..f1130c6 --- /dev/null +++ b/src/utils/py_types.rs @@ -0,0 +1,42 @@ +use pyo3::{ + FromPyObject, + types::{PyBytes, PyBytesMethods}, +}; + +use crate::exceptions::rust_err::NatsrpyError; + +#[derive(Clone, Debug)] +pub enum SendableValue { + Bytes(bytes::Bytes), + String(String), +} + +impl<'py> FromPyObject<'_, 'py> for SendableValue { + type Error = NatsrpyError; + + fn extract(obj: pyo3::Borrowed<'_, 'py, pyo3::PyAny>) -> Result { + #[allow(clippy::option_if_let_else)] + if let Ok(pybytes) = obj.cast::() { + Ok(Self::Bytes(bytes::Bytes::copy_from_slice( + pybytes.as_bytes(), + ))) + } else if let Ok(pybytes) = obj.extract::>() { + Ok(Self::Bytes(bytes::Bytes::from(pybytes))) + } else if let Ok(str_data) = obj.extract::() { + Ok(Self::String(str_data)) + } else { + Err(NatsrpyError::InvalidArgument(String::from( + "String or bytes are the only accepted values", + ))) + } + } +} + +impl From for bytes::Bytes { + fn from(value: SendableValue) -> Self { + match value { + SendableValue::Bytes(bytes) => bytes, + SendableValue::String(str) => Self::from(str.into_bytes()), + } + } +} From 88f90632a3a8407b53920a832c353f4889545bd1 Mon Sep 17 00:00:00 2001 From: Pavel Kirilin Date: Fri, 20 Mar 2026 16:47:16 +0100 Subject: [PATCH 4/8] Fixed enum variants. --- src/js/consumers/pull/consumer.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/js/consumers/pull/consumer.rs b/src/js/consumers/pull/consumer.rs index a8fcb71..f765ec3 100644 --- a/src/js/consumers/pull/consumer.rs +++ b/src/js/consumers/pull/consumer.rs @@ -2,7 +2,6 @@ use std::sync::Arc; use tokio::sync::RwLock; - type NatsPullConsumer = async_nats::jetstream::consumer::Consumer; From e6b9e4c47804c9432f1a0952fe25a36a08b4a785 Mon Sep 17 00:00:00 2001 From: Pavel Kirilin Date: Fri, 20 Mar 2026 16:51:12 +0100 Subject: [PATCH 5/8] Adde deny warnings to clippy. --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cfc4ba7..8be8894 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -48,6 +48,7 @@ jobs: - uses: auguwu/clippy-action@1.4.0 with: token: ${{secrets.GITHUB_TOKEN}} + deny: warnings pytest: runs-on: ubuntu-latest steps: From d1bafcae5be670b694d74a4e495a59d386a7845f Mon Sep 17 00:00:00 2001 From: Pavel Kirilin Date: Fri, 20 Mar 2026 17:24:06 +0100 Subject: [PATCH 6/8] Fixed python layout. --- python/natsrpy/_natsrpy_rs/js/__init__.pyi | 5 +- python/natsrpy/_natsrpy_rs/js/consumers.pyi | 149 ++++++++++++++++++ .../_natsrpy_rs/js/consumers/__init__.pyi | 19 --- .../_natsrpy_rs/js/consumers/common.pyi | 22 --- .../natsrpy/_natsrpy_rs/js/consumers/pull.pyi | 72 --------- .../natsrpy/_natsrpy_rs/js/consumers/push.pyi | 69 -------- python/natsrpy/_natsrpy_rs/js/managers.pyi | 30 ++++ .../_natsrpy_rs/js/managers/__init__.pyi | 0 .../_natsrpy_rs/js/managers/consumers.pyi | 14 -- python/natsrpy/_natsrpy_rs/js/managers/kv.pyi | 8 - .../_natsrpy_rs/js/managers/streams.pyi | 8 - python/natsrpy/_natsrpy_rs/js/stream.pyi | 2 +- python/natsrpy/js/__init__.py | 18 +++ .../{consumers/__init__.py => consumers.py} | 4 + src/js/jetstream.rs | 11 +- src/js/mod.rs | 5 +- 16 files changed, 209 insertions(+), 227 deletions(-) create mode 100644 python/natsrpy/_natsrpy_rs/js/consumers.pyi delete mode 100644 python/natsrpy/_natsrpy_rs/js/consumers/__init__.pyi delete mode 100644 python/natsrpy/_natsrpy_rs/js/consumers/common.pyi delete mode 100644 python/natsrpy/_natsrpy_rs/js/consumers/pull.pyi delete mode 100644 python/natsrpy/_natsrpy_rs/js/consumers/push.pyi create mode 100644 python/natsrpy/_natsrpy_rs/js/managers.pyi delete mode 100644 python/natsrpy/_natsrpy_rs/js/managers/__init__.pyi delete mode 100644 python/natsrpy/_natsrpy_rs/js/managers/consumers.pyi delete mode 100644 python/natsrpy/_natsrpy_rs/js/managers/kv.pyi delete mode 100644 python/natsrpy/_natsrpy_rs/js/managers/streams.pyi rename python/natsrpy/js/{consumers/__init__.py => consumers.py} (80%) diff --git a/python/natsrpy/_natsrpy_rs/js/__init__.pyi b/python/natsrpy/_natsrpy_rs/js/__init__.pyi index 43fdf44..db925b1 100644 --- a/python/natsrpy/_natsrpy_rs/js/__init__.pyi +++ b/python/natsrpy/_natsrpy_rs/js/__init__.pyi @@ -1,11 +1,10 @@ -from .managers.kv import KVManager -from .managers.streams import StreamsManager +from .managers import KVManager, StreamsManager class JetStream: async def publish( self, subject: str, - payload: bytes, + payload: str | bytes | bytearray | memoryview, *, headers: dict[str, str] | None = None, reply: str | None = None, diff --git a/python/natsrpy/_natsrpy_rs/js/consumers.pyi b/python/natsrpy/_natsrpy_rs/js/consumers.pyi new file mode 100644 index 0000000..e3d5c60 --- /dev/null +++ b/python/natsrpy/_natsrpy_rs/js/consumers.pyi @@ -0,0 +1,149 @@ +from datetime import timedelta + +class DeliverPolicy: + ALL: DeliverPolicy + LAST: DeliverPolicy + NEW: DeliverPolicy + BY_START_SEQUENCE: DeliverPolicy + BY_START_TIME: DeliverPolicy + LAST_PER_SUBJECT: DeliverPolicy + +class AckPolicy: + EXPLICIT: AckPolicy + NONE: AckPolicy + ALL: AckPolicy + +class ReplayPolicy: + INSTANT: ReplayPolicy + ORIGINAL: ReplayPolicy + +class PriorityPolicy: + NONE: PriorityPolicy + OVERFLOW: PriorityPolicy + PINNED_CLIENT: PriorityPolicy + PRIORITIZED: PriorityPolicy + +class PullConsumerConfig: + durable_name: str | None + name: str | None + description: str | None + deliver_policy: DeliverPolicy + delivery_start_sequence: int | None + delivery_start_time: int | None + ack_policy: AckPolicy + ack_wait: timedelta + max_deliver: int + filter_subject: str + filter_subjects: list[str] + replay_policy: ReplayPolicy + rate_limit: int + sample_frequency: int + max_waiting: int + max_ack_pending: int + headers_only: bool + max_batch: int + max_bytes: int + max_expires: timedelta + inactive_threshold: timedelta + num_replicas: int + memory_storage: bool + metadata: dict[str, str] + backoff: list[timedelta] + priority_policy: PriorityPolicy + priority_groups: list[str] + pause_until: int | None + + def __init__( + self, + durable_name: str | None = None, + name: str | None = None, + description: str | None = None, + deliver_policy: DeliverPolicy | None = None, + delivery_start_sequence: int | None = None, + delivery_start_time: int | None = None, + ack_policy: AckPolicy | None = None, + ack_wait: timedelta | None = None, + max_deliver: int | None = None, + filter_subject: str | None = None, + filter_subjects: list[str] | None = None, + replay_policy: ReplayPolicy | None = None, + rate_limit: int | None = None, + sample_frequency: int | None = None, + max_waiting: int | None = None, + max_ack_pending: int | None = None, + headers_only: bool | None = None, + max_batch: int | None = None, + max_bytes: int | None = None, + max_expires: timedelta | None = None, + inactive_threshold: timedelta | None = None, + num_replicas: int | None = None, + memory_storage: bool | None = None, + metadata: dict[str, str] | None = None, + backoff: list[timedelta] | None = None, + priority_policy: PriorityPolicy | None = None, + priority_groups: list[str] | None = None, + pause_until: int | None = None, + ) -> None: ... + +class PushConsumerConfig: + deliver_subject: str + durable_name: str | None + name: str | None + description: str | None + deliver_group: str | None + deliver_policy: DeliverPolicy + delivery_start_sequence: int | None + delivery_start_time: int | None + ack_policy: AckPolicy + ack_wait: timedelta + max_deliver: int + filter_subject: str + filter_subjects: list[str] + replay_policy: ReplayPolicy + rate_limit: int + sample_frequency: int + max_waiting: int + max_ack_pending: int + headers_only: bool + flow_control: bool + idle_heartbeat: timedelta + num_replicas: int + memory_storage: bool + metadata: dict[str, str] + backoff: list[timedelta] + inactive_threshold: timedelta + pause_until: int | None + + def __init__( + self, + deliver_subject: str, + durable_name: str | None = None, + name: str | None = None, + description: str | None = None, + deliver_group: str | None = None, + deliver_policy: DeliverPolicy | None = None, + delivery_start_sequence: int | None = None, + delivery_start_time: int | None = None, + ack_policy: AckPolicy | None = None, + ack_wait: timedelta | None = None, + max_deliver: int | None = None, + filter_subject: str | None = None, + filter_subjects: list[str] | None = None, + replay_policy: ReplayPolicy | None = None, + rate_limit: int | None = None, + sample_frequency: int | None = None, + max_waiting: int | None = None, + max_ack_pending: int | None = None, + headers_only: bool | None = None, + flow_control: bool | None = None, + idle_heartbeat: timedelta | None = None, + num_replicas: int | None = None, + memory_storage: bool | None = None, + metadata: dict[str, str] | None = None, + backoff: list[timedelta] | None = None, + inactive_threshold: timedelta | None = None, + pause_until: int | None = None, + ) -> None: ... + +class PushConsumer: ... +class PullConsumer: ... diff --git a/python/natsrpy/_natsrpy_rs/js/consumers/__init__.pyi b/python/natsrpy/_natsrpy_rs/js/consumers/__init__.pyi deleted file mode 100644 index dd9a648..0000000 --- a/python/natsrpy/_natsrpy_rs/js/consumers/__init__.pyi +++ /dev/null @@ -1,19 +0,0 @@ -from .common import ( - AckPolicy, - DeliverPolicy, - PriorityPolicy, - ReplayPolicy, -) -from .pull import PullConsumer, PullConsumerConfig -from .push import PushConsumer, PushConsumerConfig - -__all__ = [ - "AckPolicy", - "DeliverPolicy", - "PriorityPolicy", - "PullConsumer", - "PullConsumerConfig", - "PushConsumer", - "PushConsumerConfig", - "ReplayPolicy", -] diff --git a/python/natsrpy/_natsrpy_rs/js/consumers/common.pyi b/python/natsrpy/_natsrpy_rs/js/consumers/common.pyi deleted file mode 100644 index 1816020..0000000 --- a/python/natsrpy/_natsrpy_rs/js/consumers/common.pyi +++ /dev/null @@ -1,22 +0,0 @@ -class DeliverPolicy: - ALL: DeliverPolicy - LAST: DeliverPolicy - NEW: DeliverPolicy - BY_START_SEQUENCE: DeliverPolicy - BY_START_TIME: DeliverPolicy - LAST_PER_SUBJECT: DeliverPolicy - -class AckPolicy: - EXPLICIT: AckPolicy - NONE: AckPolicy - ALL: AckPolicy - -class ReplayPolicy: - INSTANT: ReplayPolicy - ORIGINAL: ReplayPolicy - -class PriorityPolicy: - NONE: PriorityPolicy - OVERFLOW: PriorityPolicy - PINNED_CLIENT: PriorityPolicy - PRIORITIZED: PriorityPolicy diff --git a/python/natsrpy/_natsrpy_rs/js/consumers/pull.pyi b/python/natsrpy/_natsrpy_rs/js/consumers/pull.pyi deleted file mode 100644 index 27d5fc5..0000000 --- a/python/natsrpy/_natsrpy_rs/js/consumers/pull.pyi +++ /dev/null @@ -1,72 +0,0 @@ -from datetime import timedelta - -from natsrpy._natsrpy_rs.js.consumers.common import ( - AckPolicy, - DeliverPolicy, - PriorityPolicy, - ReplayPolicy, -) - -class PullConsumerConfig: - durable_name: str | None - name: str | None - description: str | None - deliver_policy: DeliverPolicy - delivery_start_sequence: int | None - delivery_start_time: int | None - ack_policy: AckPolicy - ack_wait: timedelta - max_deliver: int - filter_subject: str - filter_subjects: list[str] - replay_policy: ReplayPolicy - rate_limit: int - sample_frequency: int - max_waiting: int - max_ack_pending: int - headers_only: bool - max_batch: int - max_bytes: int - max_expires: timedelta - inactive_threshold: timedelta - num_replicas: int - memory_storage: bool - metadata: dict[str, str] - backoff: list[timedelta] - priority_policy: PriorityPolicy - priority_groups: list[str] - pause_until: int | None - - def __init__( - self, - durable_name: str | None = None, - name: str | None = None, - description: str | None = None, - deliver_policy: DeliverPolicy | None = None, - delivery_start_sequence: int | None = None, - delivery_start_time: int | None = None, - ack_policy: AckPolicy | None = None, - ack_wait: timedelta | None = None, - max_deliver: int | None = None, - filter_subject: str | None = None, - filter_subjects: list[str] | None = None, - replay_policy: ReplayPolicy | None = None, - rate_limit: int | None = None, - sample_frequency: int | None = None, - max_waiting: int | None = None, - max_ack_pending: int | None = None, - headers_only: bool | None = None, - max_batch: int | None = None, - max_bytes: int | None = None, - max_expires: timedelta | None = None, - inactive_threshold: timedelta | None = None, - num_replicas: int | None = None, - memory_storage: bool | None = None, - metadata: dict[str, str] | None = None, - backoff: list[timedelta] | None = None, - priority_policy: PriorityPolicy | None = None, - priority_groups: list[str] | None = None, - pause_until: int | None = None, - ) -> None: ... - -class PullConsumer: ... diff --git a/python/natsrpy/_natsrpy_rs/js/consumers/push.pyi b/python/natsrpy/_natsrpy_rs/js/consumers/push.pyi deleted file mode 100644 index e816408..0000000 --- a/python/natsrpy/_natsrpy_rs/js/consumers/push.pyi +++ /dev/null @@ -1,69 +0,0 @@ -from datetime import timedelta - -from natsrpy._natsrpy_rs.js.consumers.common import ( - AckPolicy, - DeliverPolicy, - ReplayPolicy, -) - -class PushConsumerConfig: - deliver_subject: str - durable_name: str | None - name: str | None - description: str | None - deliver_group: str | None - deliver_policy: DeliverPolicy - delivery_start_sequence: int | None - delivery_start_time: int | None - ack_policy: AckPolicy - ack_wait: timedelta - max_deliver: int - filter_subject: str - filter_subjects: list[str] - replay_policy: ReplayPolicy - rate_limit: int - sample_frequency: int - max_waiting: int - max_ack_pending: int - headers_only: bool - flow_control: bool - idle_heartbeat: timedelta - num_replicas: int - memory_storage: bool - metadata: dict[str, str] - backoff: list[timedelta] - inactive_threshold: timedelta - pause_until: int | None - - def __init__( - self, - deliver_subject: str, - durable_name: str | None = None, - name: str | None = None, - description: str | None = None, - deliver_group: str | None = None, - deliver_policy: DeliverPolicy | None = None, - delivery_start_sequence: int | None = None, - delivery_start_time: int | None = None, - ack_policy: AckPolicy | None = None, - ack_wait: timedelta | None = None, - max_deliver: int | None = None, - filter_subject: str | None = None, - filter_subjects: list[str] | None = None, - replay_policy: ReplayPolicy | None = None, - rate_limit: int | None = None, - sample_frequency: int | None = None, - max_waiting: int | None = None, - max_ack_pending: int | None = None, - headers_only: bool | None = None, - flow_control: bool | None = None, - idle_heartbeat: timedelta | None = None, - num_replicas: int | None = None, - memory_storage: bool | None = None, - metadata: dict[str, str] | None = None, - backoff: list[timedelta] | None = None, - inactive_threshold: timedelta | None = None, - pause_until: int | None = None, - ) -> None: ... - -class PushConsumer: ... diff --git a/python/natsrpy/_natsrpy_rs/js/managers.pyi b/python/natsrpy/_natsrpy_rs/js/managers.pyi new file mode 100644 index 0000000..64a4a72 --- /dev/null +++ b/python/natsrpy/_natsrpy_rs/js/managers.pyi @@ -0,0 +1,30 @@ +from typing import overload + +from .consumers import ( + PullConsumer, + PullConsumerConfig, + PushConsumer, + PushConsumerConfig, +) +from .kv import KeyValue, KVConfig +from .stream import Stream, StreamConfig + +class StreamsManager: + async def create(self, config: StreamConfig) -> Stream: ... + async def create_or_update(self, config: StreamConfig) -> Stream: ... + async def get(self, bucket: str) -> Stream: ... + async def delete(self, bucket: str) -> None: ... + async def update(self, config: StreamConfig) -> Stream: ... + +class KVManager: + async def create(self, config: KVConfig) -> KeyValue: ... + async def create_or_update(self, config: KVConfig) -> KeyValue: ... + async def get(self, bucket: str) -> KeyValue: ... + async def delete(self, bucket: str) -> None: ... + async def update(self, config: KVConfig) -> KeyValue: ... + +class ConsumersManager: + @overload + async def create(self, config: PullConsumerConfig) -> PullConsumer: ... + @overload + async def create(self, config: PushConsumerConfig) -> PushConsumer: ... diff --git a/python/natsrpy/_natsrpy_rs/js/managers/__init__.pyi b/python/natsrpy/_natsrpy_rs/js/managers/__init__.pyi deleted file mode 100644 index e69de29..0000000 diff --git a/python/natsrpy/_natsrpy_rs/js/managers/consumers.pyi b/python/natsrpy/_natsrpy_rs/js/managers/consumers.pyi deleted file mode 100644 index fb04773..0000000 --- a/python/natsrpy/_natsrpy_rs/js/managers/consumers.pyi +++ /dev/null @@ -1,14 +0,0 @@ -from typing import overload - -from natsrpy._natsrpy_rs.js.consumers import ( - PullConsumer, - PullConsumerConfig, - PushConsumer, - PushConsumerConfig, -) - -class ConsumersManager: - @overload - async def create(self, config: PullConsumerConfig) -> PullConsumer: ... - @overload - async def create(self, config: PushConsumerConfig) -> PushConsumer: ... diff --git a/python/natsrpy/_natsrpy_rs/js/managers/kv.pyi b/python/natsrpy/_natsrpy_rs/js/managers/kv.pyi deleted file mode 100644 index b998951..0000000 --- a/python/natsrpy/_natsrpy_rs/js/managers/kv.pyi +++ /dev/null @@ -1,8 +0,0 @@ -from natsrpy._natsrpy_rs.js.kv import KeyValue, KVConfig - -class KVManager: - async def create(self, config: KVConfig) -> KeyValue: ... - async def create_or_update(self, config: KVConfig) -> KeyValue: ... - async def get(self, bucket: str) -> KeyValue: ... - async def delete(self, bucket: str) -> None: ... - async def update(self, config: KVConfig) -> KeyValue: ... diff --git a/python/natsrpy/_natsrpy_rs/js/managers/streams.pyi b/python/natsrpy/_natsrpy_rs/js/managers/streams.pyi deleted file mode 100644 index 2a47b81..0000000 --- a/python/natsrpy/_natsrpy_rs/js/managers/streams.pyi +++ /dev/null @@ -1,8 +0,0 @@ -from natsrpy._natsrpy_rs.js.stream import Stream, StreamConfig - -class StreamsManager: - async def create(self, config: StreamConfig) -> Stream: ... - async def create_or_update(self, config: StreamConfig) -> Stream: ... - async def get(self, bucket: str) -> Stream: ... - async def delete(self, bucket: str) -> None: ... - async def update(self, config: StreamConfig) -> Stream: ... diff --git a/python/natsrpy/_natsrpy_rs/js/stream.pyi b/python/natsrpy/_natsrpy_rs/js/stream.pyi index 506e08a..bef603d 100644 --- a/python/natsrpy/_natsrpy_rs/js/stream.pyi +++ b/python/natsrpy/_natsrpy_rs/js/stream.pyi @@ -1,7 +1,7 @@ from datetime import datetime, timedelta from typing import Any -from natsrpy._natsrpy_rs.js.managers.consumers import ConsumersManager +from .managers import ConsumersManager class StorageType: FILE: StorageType diff --git a/python/natsrpy/js/__init__.py b/python/natsrpy/js/__init__.py index bd23a18..24b1788 100644 --- a/python/natsrpy/js/__init__.py +++ b/python/natsrpy/js/__init__.py @@ -1,4 +1,14 @@ from natsrpy._natsrpy_rs.js import JetStream +from natsrpy.js.consumers import ( + AckPolicy, + DeliverPolicy, + PriorityPolicy, + PullConsumer, + PullConsumerConfig, + PushConsumer, + PushConsumerConfig, + ReplayPolicy, +) from natsrpy.js.kv import KeyValue, KVConfig from natsrpy.js.stream import ( Compression, @@ -18,8 +28,10 @@ ) __all__ = [ + "AckPolicy", "Compression", "ConsumerLimits", + "DeliverPolicy", "DiscardPolicy", "External", "JetStream", @@ -27,6 +39,12 @@ "KeyValue", "PersistenceMode", "Placement", + "PriorityPolicy", + "PullConsumer", + "PullConsumerConfig", + "PushConsumer", + "PushConsumerConfig", + "ReplayPolicy", "Republish", "RetentionPolicy", "Source", diff --git a/python/natsrpy/js/consumers/__init__.py b/python/natsrpy/js/consumers.py similarity index 80% rename from python/natsrpy/js/consumers/__init__.py rename to python/natsrpy/js/consumers.py index 03d442f..e90e8f5 100644 --- a/python/natsrpy/js/consumers/__init__.py +++ b/python/natsrpy/js/consumers.py @@ -2,7 +2,9 @@ AckPolicy, DeliverPolicy, PriorityPolicy, + PullConsumer, PullConsumerConfig, + PushConsumer, PushConsumerConfig, ReplayPolicy, ) @@ -11,7 +13,9 @@ "AckPolicy", "DeliverPolicy", "PriorityPolicy", + "PullConsumer", "PullConsumerConfig", + "PushConsumer", "PushConsumerConfig", "ReplayPolicy", ] diff --git a/src/js/jetstream.rs b/src/js/jetstream.rs index 49e32d6..b2f0dac 100644 --- a/src/js/jetstream.rs +++ b/src/js/jetstream.rs @@ -1,16 +1,13 @@ use std::sync::Arc; use async_nats::{Subject, client::traits::Publisher, connection::State}; -use pyo3::{ - Bound, PyAny, Python, - types::{PyBytes, PyBytesMethods, PyDict}, -}; +use pyo3::{Bound, PyAny, Python, types::PyDict}; use tokio::sync::RwLock; use crate::{ exceptions::rust_err::{NatsrpyError, NatsrpyResult}, js::managers::{kv::KVManager, streams::StreamsManager}, - utils::{headers::NatsrpyHeadermapExt, natsrpy_future}, + utils::{headers::NatsrpyHeadermapExt, natsrpy_future, py_types::SendableValue}, }; #[pyo3::pyclass] @@ -41,13 +38,13 @@ impl JetStream { &self, py: Python<'py>, subject: String, - payload: &Bound, + payload: SendableValue, headers: Option>, reply: Option, err_on_disconnect: bool, ) -> NatsrpyResult> { let ctx = self.ctx.clone(); - let data = bytes::Bytes::from(payload.as_bytes().to_vec()); + let data = payload.into(); let headermap = headers .map(async_nats::HeaderMap::from_pydict) .transpose()?; diff --git a/src/js/mod.rs b/src/js/mod.rs index 7d1548f..84c2ca2 100644 --- a/src/js/mod.rs +++ b/src/js/mod.rs @@ -8,10 +8,7 @@ pub mod stream; pub mod pymod { // Classes #[pymodule_export] - pub use super::{ - jetstream::JetStream, - kv::{KVConfig, KeyValue}, - }; + pub use super::jetstream::JetStream; // SubModules #[pymodule_export] From 3e77b14dae173816ff8a4d78cc72e665a4586db3 Mon Sep 17 00:00:00 2001 From: Pavel Kirilin Date: Sat, 21 Mar 2026 14:25:13 +0100 Subject: [PATCH 7/8] Added pull consumer fetch function. --- python/natsrpy/_natsrpy_rs/js/__init__.pyi | 13 ++++ python/natsrpy/_natsrpy_rs/js/consumers.pyi | 16 ++++- src/exceptions/rust_err.rs | 2 + src/js/consumers/pull/consumer.rs | 79 ++++++++++++++++++--- src/js/message.rs | 60 ++++++++++++++++ src/js/mod.rs | 3 + src/js/stream.rs | 2 +- src/message.rs | 2 +- src/utils/headers.rs | 8 +-- 9 files changed, 168 insertions(+), 17 deletions(-) diff --git a/python/natsrpy/_natsrpy_rs/js/__init__.pyi b/python/natsrpy/_natsrpy_rs/js/__init__.pyi index db925b1..cc53a76 100644 --- a/python/natsrpy/_natsrpy_rs/js/__init__.pyi +++ b/python/natsrpy/_natsrpy_rs/js/__init__.pyi @@ -1,3 +1,5 @@ +from typing import Any + from .managers import KVManager, StreamsManager class JetStream: @@ -14,3 +16,14 @@ class JetStream: def kv(self) -> KVManager: ... @property def streams(self) -> StreamsManager: ... + +class JetStreamMessage: + @property + def subject(self) -> str: ... + @property + def reply(self) -> str | None: ... + @property + def payload(self) -> bytes: ... + @property + def headers(self) -> dict[str, Any]: ... + async def ack(self) -> None: ... diff --git a/python/natsrpy/_natsrpy_rs/js/consumers.pyi b/python/natsrpy/_natsrpy_rs/js/consumers.pyi index e3d5c60..14bb850 100644 --- a/python/natsrpy/_natsrpy_rs/js/consumers.pyi +++ b/python/natsrpy/_natsrpy_rs/js/consumers.pyi @@ -1,5 +1,7 @@ from datetime import timedelta +from natsrpy._natsrpy_rs.js import JetStreamMessage + class DeliverPolicy: ALL: DeliverPolicy LAST: DeliverPolicy @@ -146,4 +148,16 @@ class PushConsumerConfig: ) -> None: ... class PushConsumer: ... -class PullConsumer: ... + +class PullConsumer: + async def fetch( + self, + max_messages: int | None = None, + group: str | None = None, + priority: int | None = None, + max_bytes: int | None = None, + heartbeat: timedelta | None = None, + expires: timedelta | None = None, + min_pending: int | None = None, + min_ack_pending: int | None = None, + ) -> list[JetStreamMessage]: ... diff --git a/src/exceptions/rust_err.rs b/src/exceptions/rust_err.rs index 93658c1..cfac744 100644 --- a/src/exceptions/rust_err.rs +++ b/src/exceptions/rust_err.rs @@ -64,6 +64,8 @@ pub enum NatsrpyError { PullMessageError(#[from] async_nats::jetstream::consumer::pull::MessagesError), #[error(transparent)] PullConsumerError(#[from] async_nats::jetstream::stream::ConsumerError), + #[error(transparent)] + PullConsumerBatchError(#[from] async_nats::jetstream::consumer::pull::BatchError), } impl From for pyo3::PyErr { diff --git a/src/js/consumers/pull/consumer.rs b/src/js/consumers/pull/consumer.rs index f765ec3..e2e4960 100644 --- a/src/js/consumers/pull/consumer.rs +++ b/src/js/consumers/pull/consumer.rs @@ -1,7 +1,11 @@ -use std::sync::Arc; +use std::{sync::Arc, time::Duration}; +use futures_util::StreamExt; +use pyo3::{Bound, PyAny, Python}; use tokio::sync::RwLock; +use crate::{exceptions::rust_err::NatsrpyResult, utils::natsrpy_future}; + type NatsPullConsumer = async_nats::jetstream::consumer::Consumer; @@ -20,13 +24,68 @@ impl PullConsumer { } } -#[pyo3::pyclass] -pub struct PullMessageIterator { - inner: Arc>, -} - -#[pyo3::pymethods] -impl PullConsumer {} - #[pyo3::pymethods] -impl PullMessageIterator {} +impl PullConsumer { + #[pyo3(signature=( + max_messages=None, + group=None, + priority=None, + max_bytes=None, + heartbeat=None, + expires=None, + min_pending=None, + min_ack_pending=None, + ))] + pub fn fetch<'py>( + &self, + py: Python<'py>, + max_messages: Option, + group: Option, + priority: Option, + max_bytes: Option, + heartbeat: Option, + expires: Option, + min_pending: Option, + min_ack_pending: Option, + ) -> NatsrpyResult> { + let ctx = self.consumer.clone(); + #[allow(clippy::significant_drop_tightening)] + natsrpy_future(py, async move { + // Because we borrow created value + // later for modifications. + let consumer = ctx.read().await; + let mut fetch_builder = consumer.fetch(); + if let Some(max_messages) = max_messages { + fetch_builder = fetch_builder.max_messages(max_messages); + } + if let Some(group) = group { + fetch_builder = fetch_builder.group(group); + } + if let Some(priority) = priority { + fetch_builder = fetch_builder.priority(priority); + } + if let Some(max_bytes) = max_bytes { + fetch_builder = fetch_builder.max_bytes(max_bytes); + } + if let Some(heartbeat) = heartbeat { + fetch_builder = fetch_builder.heartbeat(heartbeat); + } + if let Some(expires) = expires { + fetch_builder = fetch_builder.expires(expires); + } + if let Some(min_pending) = min_pending { + fetch_builder = fetch_builder.min_pending(min_pending); + } + if let Some(min_ack_pending) = min_ack_pending { + fetch_builder = fetch_builder.min_ack_pending(min_ack_pending); + } + let mut messages = fetch_builder.messages().await?; + let mut ret_messages = Vec::new(); + while let Some(msg) = messages.next().await { + let raw_msg = msg?; + ret_messages.push(crate::js::message::JetStreamMessage::from(raw_msg)); + } + Ok(ret_messages) + }) + } +} diff --git a/src/js/message.rs b/src/js/message.rs index 8b13789..64f59a0 100644 --- a/src/js/message.rs +++ b/src/js/message.rs @@ -1 +1,61 @@ +use pyo3::{Bound, Py, PyAny, Python, types::PyDict}; +use std::sync::Arc; +use tokio::sync::RwLock; +use crate::{ + exceptions::rust_err::NatsrpyResult, + utils::{headers::NatsrpyHeadermapExt, natsrpy_future}, +}; + +#[pyo3::pyclass] +pub struct JetStreamMessage { + message: async_nats::Message, + headers: Option>, + acker: Arc>, +} + +impl From for JetStreamMessage { + fn from(value: async_nats::jetstream::Message) -> Self { + let (message, acker) = value.split(); + Self { + message, + headers: None, + acker: Arc::new(RwLock::new(acker)), + } + } +} + +#[pyo3::pymethods] +impl JetStreamMessage { + #[getter] + pub fn subject(&self) -> &str { + self.message.subject.as_str() + } + #[getter] + pub fn reply(&self) -> Option<&str> { + self.message.reply.as_ref().map(async_nats::Subject::as_str) + } + #[getter] + pub fn payload(&self) -> &[u8] { + &self.message.payload + } + #[getter] + pub fn headers(&mut self, py: Python<'_>) -> NatsrpyResult> { + if let Some(headers) = &self.headers { + Ok(headers.clone_ref(py)) + } else { + let headermap = self.message.headers.clone().unwrap_or_default(); + let headers = headermap.to_pydict(py)?.unbind(); + self.headers = Some(headers.clone_ref(py)); + Ok(headers) + } + } + + pub fn ack<'py>(&self, py: Python<'py>) -> NatsrpyResult> { + let acker_guard = self.acker.clone(); + natsrpy_future(py, async move { + acker_guard.read().await.ack().await?; + Ok(()) + }) + } +} diff --git a/src/js/mod.rs b/src/js/mod.rs index 84c2ca2..1ebefa8 100644 --- a/src/js/mod.rs +++ b/src/js/mod.rs @@ -2,6 +2,7 @@ pub mod consumers; pub mod jetstream; pub mod kv; pub mod managers; +pub mod message; pub mod stream; #[pyo3::pymodule(submodule, name = "js")] @@ -9,6 +10,8 @@ pub mod pymod { // Classes #[pymodule_export] pub use super::jetstream::JetStream; + #[pymodule_export] + pub use super::message::JetStreamMessage; // SubModules #[pymodule_export] diff --git a/src/js/stream.rs b/src/js/stream.rs index de300f0..5196331 100644 --- a/src/js/stream.rs +++ b/src/js/stream.rs @@ -817,7 +817,7 @@ impl StreamMessage { Ok(Self { subject: msg.subject.to_string(), payload: PyBytes::new(py, &msg.payload).unbind(), - headers: msg.headers.to_pydict(py)?, + headers: msg.headers.to_pydict(py)?.unbind(), sequence: msg.sequence, time: time.unbind(), }) diff --git a/src/message.rs b/src/message.rs index 956bbf0..bedbcf9 100644 --- a/src/message.rs +++ b/src/message.rs @@ -23,7 +23,7 @@ impl TryFrom for Message { fn try_from(value: async_nats::Message) -> Result { Python::attach(move |gil| { let headers = match value.headers { - Some(headermap) => headermap.to_pydict(gil)?, + Some(headermap) => headermap.to_pydict(gil)?.unbind(), None => PyDict::new(gil).unbind(), }; Ok(Self { diff --git a/src/utils/headers.rs b/src/utils/headers.rs index f99b0c8..30b2d00 100644 --- a/src/utils/headers.rs +++ b/src/utils/headers.rs @@ -1,5 +1,5 @@ use pyo3::{ - Bound, Py, Python, + Bound, Python, types::{PyAnyMethods, PyDict}, }; @@ -7,7 +7,7 @@ use crate::exceptions::rust_err::NatsrpyResult; pub trait NatsrpyHeadermapExt: Sized { fn from_pydict(pydict: Bound) -> NatsrpyResult; - fn to_pydict(&self, py: Python) -> NatsrpyResult>; + fn to_pydict<'py>(&self, py: Python<'py>) -> NatsrpyResult>; } impl NatsrpyHeadermapExt for async_nats::HeaderMap { @@ -30,7 +30,7 @@ impl NatsrpyHeadermapExt for async_nats::HeaderMap { Ok(headermap) } - fn to_pydict(&self, py: Python) -> NatsrpyResult> { + fn to_pydict<'py>(&self, py: Python<'py>) -> NatsrpyResult> { let dict = PyDict::new(py); for (header_name, header_val) in self.iter() { let py_val = header_val @@ -46,6 +46,6 @@ impl NatsrpyHeadermapExt for async_nats::HeaderMap { } dict.set_item(header_name.to_string(), py_val)?; } - Ok(dict.unbind()) + Ok(dict) } } From 6641f3f891a8a4508ff6b6d324f4c150424da2c5 Mon Sep 17 00:00:00 2001 From: Pavel Kirilin Date: Sat, 21 Mar 2026 14:28:43 +0100 Subject: [PATCH 8/8] Added tmp clippy ignores. --- src/js/consumers/push/consumer.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/js/consumers/push/consumer.rs b/src/js/consumers/push/consumer.rs index 05c88b3..9825712 100644 --- a/src/js/consumers/push/consumer.rs +++ b/src/js/consumers/push/consumer.rs @@ -7,6 +7,7 @@ type NatsPushConsumer = #[pyo3::pyclass(from_py_object)] #[derive(Debug, Clone)] +#[allow(dead_code)] // TODO! remove later. pub struct PushConsumer { consumer: Arc>, }