-
Notifications
You must be signed in to change notification settings - Fork 15
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: add destination/writer trait (#838)
And implements them for tokio channel(s).
- Loading branch information
1 parent
944b648
commit 7508a6d
Showing
26 changed files
with
412 additions
and
131 deletions.
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,5 +1,6 @@ | ||
[workspace] | ||
members = ["crates/*"] | ||
resolver = "2" | ||
|
||
[workspace.package] | ||
authors = ["Kaskada Developers"] | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,47 @@ | ||
use sparrow_batch::Batch; | ||
use std::fmt::Debug; | ||
|
||
use crate::types::Partition; | ||
|
||
/// Interface defining how output is written to a particular destination. | ||
pub trait Destination: Send + Sync + Debug { | ||
/// Creates a new writer to this destination. | ||
/// | ||
/// Arguments: | ||
/// * partition: The partition this writer will write to. | ||
fn new_writer(&self, partition: Partition) -> error_stack::Result<Box<dyn Writer>, WriteError>; | ||
} | ||
|
||
/// Writer for a specific destination. | ||
pub trait Writer: Send + Sync + Debug { | ||
/// Write a batch to the given writer. | ||
/// | ||
/// NOTE: Some destinations (such as Parquet) may actually rotate files | ||
/// during / after calls to `write_batch`. | ||
/// | ||
/// Arguments: | ||
/// * batch - The batch to write. | ||
fn write_batch(&mut self, batch: Batch) -> error_stack::Result<(), WriteError>; | ||
|
||
/// Close this writer. | ||
fn close(&self) -> error_stack::Result<(), WriteError>; | ||
} | ||
|
||
#[non_exhaustive] | ||
#[derive(derive_more::Display, Debug)] | ||
pub enum WriteError { | ||
#[display(fmt = "internal error on write: {}", _0)] | ||
Internal(&'static str), | ||
} | ||
|
||
impl error_stack::Context for WriteError {} | ||
|
||
impl WriteError { | ||
pub fn internal() -> Self { | ||
WriteError::Internal("no additional context") | ||
} | ||
|
||
pub fn internal_msg(msg: &'static str) -> Self { | ||
WriteError::Internal(msg) | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
mod partition; | ||
pub use partition::*; |
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,4 +1,3 @@ | ||
mod source; | ||
mod destination; | ||
|
||
pub use source::InMemoryBatches; | ||
pub use source::InMemorySource; | ||
pub use destination::*; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,68 @@ | ||
use arrow_array::RecordBatch; | ||
use arrow_schema::SchemaRef; | ||
use error_stack::{IntoReport, ResultExt}; | ||
use sparrow_batch::Batch; | ||
use sparrow_interfaces::{ | ||
destination::{Destination, WriteError, Writer}, | ||
types::Partition, | ||
}; | ||
|
||
#[derive(Debug)] | ||
pub struct ChannelDestination { | ||
/// The expected output schema. | ||
schema: SchemaRef, | ||
// TODO: The output current expects [RecordBatch] instead of [Batch] | ||
// but we should standardize on [Batch]. | ||
txs: Vec<tokio::sync::mpsc::Sender<RecordBatch>>, | ||
} | ||
|
||
impl ChannelDestination { | ||
/// Construct a new channel destination with the given senders. | ||
/// | ||
/// The number of `txs` should equal the number of partitions the output | ||
/// produces from. | ||
pub fn new(schema: SchemaRef, txs: Vec<tokio::sync::mpsc::Sender<RecordBatch>>) -> Self { | ||
Self { schema, txs } | ||
} | ||
} | ||
|
||
impl Destination for ChannelDestination { | ||
fn new_writer(&self, partition: Partition) -> error_stack::Result<Box<dyn Writer>, WriteError> { | ||
let partition: usize = partition.into(); | ||
let tx = self | ||
.txs | ||
.get(partition) | ||
.ok_or_else(|| WriteError::internal_msg("expected channel for partition {partition}"))? | ||
.clone(); // Senders can be cloned cheaply | ||
Ok(Box::new(ChannelWriter { | ||
schema: self.schema.clone(), | ||
tx, | ||
})) | ||
} | ||
} | ||
|
||
#[derive(Debug)] | ||
struct ChannelWriter { | ||
schema: SchemaRef, | ||
tx: tokio::sync::mpsc::Sender<RecordBatch>, | ||
} | ||
|
||
impl Writer for ChannelWriter { | ||
fn write_batch(&mut self, batch: Batch) -> error_stack::Result<(), WriteError> { | ||
// HACK: This converts `Batch` to `RecordBatch` because the current execution logic | ||
// expects `RecordBatch` outputs. This should be changed to standardize on `Batch` | ||
// which makes it easier to carry a primitive value out. | ||
if let Some(batch) = batch.into_record_batch(self.schema.clone()) { | ||
self.tx | ||
.blocking_send(batch) | ||
.into_report() | ||
.change_context(WriteError::internal())?; | ||
} | ||
Ok(()) | ||
} | ||
|
||
fn close(&self) -> error_stack::Result<(), WriteError> { | ||
self.tx.downgrade(); | ||
Ok(()) | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -7,4 +7,5 @@ | |
clippy::print_stderr | ||
)] | ||
|
||
pub mod channel; | ||
pub mod in_memory; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.