From c693f05717a2d5c53d778ae09d45e83dcfca22af Mon Sep 17 00:00:00 2001 From: alecmocatta Date: Tue, 21 Jul 2020 19:12:30 +0100 Subject: [PATCH] combine Sink::{poll, output}, make naming consistent --- .../src/into_par_stream/collections.rs | 1 + amadeus-core/src/into_par_stream/iterator.rs | 5 +- amadeus-core/src/into_par_stream/slice.rs | 2 +- amadeus-core/src/par_pipe.rs | 91 +++-- amadeus-core/src/par_sink.rs | 83 ++-- amadeus-core/src/par_sink/all.rs | 101 +++-- amadeus-core/src/par_sink/any.rs | 101 +++-- amadeus-core/src/par_sink/collect.rs | 238 +++++------- amadeus-core/src/par_sink/combine.rs | 23 +- amadeus-core/src/par_sink/combiner.rs | 42 +- amadeus-core/src/par_sink/count.rs | 22 +- amadeus-core/src/par_sink/fold.rs | 37 +- amadeus-core/src/par_sink/folder.rs | 104 +++-- amadeus-core/src/par_sink/for_each.rs | 74 ++-- amadeus-core/src/par_sink/fork.rs | 80 ++-- amadeus-core/src/par_sink/group_by.rs | 363 ++++++++++-------- amadeus-core/src/par_sink/histogram.rs | 33 +- amadeus-core/src/par_sink/max.rs | 90 ++--- amadeus-core/src/par_sink/pipe.rs | 41 +- amadeus-core/src/par_sink/sample.rs | 77 ++-- amadeus-core/src/par_sink/sum.rs | 41 +- amadeus-core/src/par_sink/tuple.rs | 212 ++++------ amadeus-core/src/par_stream.rs | 67 ++-- amadeus-core/src/par_stream/chain.rs | 16 +- amadeus-core/src/par_stream/cloned.rs | 62 +-- amadeus-core/src/par_stream/filter.rs | 48 ++- amadeus-core/src/par_stream/flat_map.rs | 44 ++- amadeus-core/src/par_stream/identity.rs | 8 +- amadeus-core/src/par_stream/inspect.rs | 56 +-- amadeus-core/src/par_stream/map.rs | 49 ++- amadeus-core/src/par_stream/sum_type.rs | 28 +- amadeus-core/src/par_stream/update.rs | 56 +-- amadeus-core/src/pipe.rs | 152 ++++---- amadeus-core/src/pipe/filter.rs | 28 +- amadeus-core/src/pipe/flat_map.rs | 18 +- amadeus-core/src/pipe/map.rs | 31 +- amadeus-core/src/util.rs | 14 +- src/source.rs | 27 +- 38 files changed, 1244 insertions(+), 1321 deletions(-) diff --git a/amadeus-core/src/into_par_stream/collections.rs b/amadeus-core/src/into_par_stream/collections.rs index 921c97cb..2c8c62bf 100644 --- a/amadeus-core/src/into_par_stream/collections.rs +++ b/amadeus-core/src/into_par_stream/collections.rs @@ -13,6 +13,7 @@ impl<'a, 'b, I: Iterator, A: Clone + 'a, B: Clone + 'b> I for TupleCloned { type Item = (A, B); + fn next(&mut self) -> Option { self.0.next().map(|(a, b)| (a.clone(), b.clone())) } diff --git a/amadeus-core/src/into_par_stream/iterator.rs b/amadeus-core/src/into_par_stream/iterator.rs index 8aa264bd..1945aa31 100644 --- a/amadeus-core/src/into_par_stream/iterator.rs +++ b/amadeus-core/src/into_par_stream/iterator.rs @@ -21,6 +21,7 @@ pub trait IteratorExt: Iterator + Sized { impl IteratorExt for I {} impl_par_dist_rename! { + #[pin_project] pub struct IterParStream(pub(crate) I); impl ParallelStream for IterParStream @@ -33,8 +34,8 @@ impl_par_dist_rename! { fn size_hint(&self) -> (usize, Option) { self.0.size_hint() } - fn next_task(&mut self) -> Option { - self.0.next().map(IterStreamTask::new) + fn next_task(mut self: Pin<&mut Self>, _cx: &mut Context) -> Poll> { + Poll::Ready(self.0.next().map(IterStreamTask::new)) } } } diff --git a/amadeus-core/src/into_par_stream/slice.rs b/amadeus-core/src/into_par_stream/slice.rs index a47b95ab..fe70ed1b 100644 --- a/amadeus-core/src/into_par_stream/slice.rs +++ b/amadeus-core/src/into_par_stream/slice.rs @@ -47,7 +47,7 @@ impl_par_dist_rename! { fn size_hint(&self) -> (usize, Option) { unreachable!() } - fn next_task(&mut self) -> Option { + fn next_task(self: Pin<&mut Self>, _cx: &mut Context) -> Poll> { unreachable!() } } diff --git a/amadeus-core/src/par_pipe.rs b/amadeus-core/src/par_pipe.rs index 3638e23b..949f10ec 100644 --- a/amadeus-core/src/par_pipe.rs +++ b/amadeus-core/src/par_pipe.rs @@ -5,14 +5,13 @@ use futures::Stream; use serde_closure::traits; use std::{cmp::Ordering, hash::Hash, iter, ops}; -use crate::{pipe::Pipe, pool::ProcessSend}; - use super::{par_sink::*, par_stream::*}; +use crate::{pipe::Pipe, pool::ProcessSend}; #[must_use] -pub trait PipeTask { - type Item; - type Async: Pipe; +pub trait PipeTask { + type Output; + type Async: Pipe; fn into_async(self) -> Self::Async; } @@ -21,15 +20,15 @@ macro_rules! pipe { ($pipe:ident $sink:ident $from_sink:ident $send:ident $fns:ident $assert_pipe:ident $assert_sink:ident $($meta:meta)*) => { $(#[$meta])* #[must_use] - pub trait $pipe { - type Item; - type Task: PipeTask + $send; + pub trait $pipe { + type Output; + type Task: PipeTask + $send; fn task(&self) -> Self::Task; fn inspect(self, f: F) -> Inspect where - F: $fns::FnMut(&Self::Item) + Clone + $send + 'static, + F: $fns::FnMut(&Self::Output) + Clone + $send + 'static, Self: Sized, { $assert_pipe(Inspect::new(self, f)) @@ -37,7 +36,7 @@ macro_rules! pipe { fn update(self, f: F) -> Update where - F: $fns::FnMut(&mut Self::Item) + Clone + $send + 'static, + F: $fns::FnMut(&mut Self::Output) + Clone + $send + 'static, Self: Sized, { $assert_pipe(Update::new(self, f)) @@ -45,7 +44,7 @@ macro_rules! pipe { fn map(self, f: F) -> Map where - F: $fns::FnMut(Self::Item) -> B + Clone + $send + 'static, + F: $fns::FnMut(Self::Output) -> B + Clone + $send + 'static, Self: Sized, { $assert_pipe(Map::new(self, f)) @@ -53,7 +52,7 @@ macro_rules! pipe { fn flat_map(self, f: F) -> FlatMap where - F: $fns::FnMut(Self::Item) -> B + Clone + $send + 'static, + F: $fns::FnMut(Self::Output) -> B + Clone + $send + 'static, B: Stream, Self: Sized, { @@ -62,17 +61,17 @@ macro_rules! pipe { fn filter(self, f: F) -> Filter where - F: $fns::FnMut(&Self::Item) -> bool + Clone + $send + 'static, + F: $fns::FnMut(&Self::Output) -> bool + Clone + $send + 'static, Self: Sized, { $assert_pipe(Filter::new(self, f)) } - fn cloned<'a, T>(self) -> Cloned + fn cloned<'a, T>(self) -> Cloned where T: Clone + 'a, - Source: 'a, - Self: $pipe<&'a Source, Item = &'a T> + Sized, + Input: 'a, + Self: $pipe<&'a Input, Output = &'a T> + Sized, { $assert_pipe(Cloned::new(self)) } @@ -80,7 +79,7 @@ macro_rules! pipe { // #[must_use] // fn chain(self, chain: C) -> Chain // where - // C: IntoParallelStream, + // C: IntoParallelStream, // Self: Sized, // { // $assert_pipe(Chain::new(self, chain.into_par_stream())) @@ -88,7 +87,7 @@ macro_rules! pipe { fn pipe(self, sink: S) -> super::par_sink::Pipe where - S: $sink, + S: $sink, Self: Sized, { $assert_sink(super::par_sink::Pipe::new(self, sink)) @@ -96,10 +95,10 @@ macro_rules! pipe { fn fork( self, sink: A, sink_ref: B, - ) -> Fork + ) -> Fork where - A: $sink, - B: for<'a> $sink<&'a Self::Item>, + A: $sink, + B: for<'a> $sink<&'a Self::Output>, Self: Sized, { $assert_sink(Fork::new(self, sink, sink_ref)) @@ -107,7 +106,7 @@ macro_rules! pipe { fn for_each(self, f: F) -> ForEach where - F: $fns::FnMut(Self::Item) + Clone + $send + 'static, + F: $fns::FnMut(Self::Output) + Clone + $send + 'static, Self: Sized, { $assert_sink(ForEach::new(self, f)) @@ -116,7 +115,7 @@ macro_rules! pipe { fn fold(self, identity: ID, op: F) -> Fold where ID: $fns::FnMut() -> B + Clone + $send + 'static, - F: $fns::FnMut(B, Either) -> B + Clone + $send + 'static, + F: $fns::FnMut(B, Either) -> B + Clone + $send + 'static, B: $send + 'static, Self: Sized, { @@ -130,15 +129,15 @@ macro_rules! pipe { >::Task: Clone + $send + 'static, S::ReduceA: 'static, S::ReduceC: Clone, - S::Output: $send + 'static, - Self: $pipe + Sized, + S::Done: $send + 'static, + Self: $pipe + Sized, { $assert_sink(GroupBy::new(self, sink)) } fn histogram(self) -> Histogram where - Self::Item: Hash + Ord + $send + 'static, + Self::Output: Hash + Ord + $send + 'static, Self: Sized, { $assert_sink(Histogram::new(self)) @@ -153,7 +152,7 @@ macro_rules! pipe { fn sum(self) -> Sum where - B: iter::Sum + iter::Sum + $send + 'static, + B: iter::Sum + iter::Sum + $send + 'static, Self: Sized, { $assert_sink(Sum::new(self)) @@ -161,8 +160,8 @@ macro_rules! pipe { fn combine(self, f: F) -> Combine where - F: $fns::FnMut(Self::Item, Self::Item) -> Self::Item + Clone + $send + 'static, - Self::Item: $send + 'static, + F: $fns::FnMut(Self::Output, Self::Output) -> Self::Output + Clone + $send + 'static, + Self::Output: $send + 'static, Self: Sized, { $assert_sink(Combine::new(self, f)) @@ -170,7 +169,7 @@ macro_rules! pipe { fn max(self) -> Max where - Self::Item: Ord + $send + 'static, + Self::Output: Ord + $send + 'static, Self: Sized, { $assert_sink(Max::new(self)) @@ -178,8 +177,8 @@ macro_rules! pipe { fn max_by(self, f: F) -> MaxBy where - F: $fns::FnMut(&Self::Item, &Self::Item) -> Ordering + Clone + $send + 'static, - Self::Item: $send + 'static, + F: $fns::FnMut(&Self::Output, &Self::Output) -> Ordering + Clone + $send + 'static, + Self::Output: $send + 'static, Self: Sized, { $assert_sink(MaxBy::new(self, f)) @@ -187,9 +186,9 @@ macro_rules! pipe { fn max_by_key(self, f: F) -> MaxByKey where - F: $fns::FnMut(&Self::Item) -> B + Clone + $send + 'static, + F: $fns::FnMut(&Self::Output) -> B + Clone + $send + 'static, B: Ord + 'static, - Self::Item: $send + 'static, + Self::Output: $send + 'static, Self: Sized, { $assert_sink(MaxByKey::new(self, f)) @@ -197,7 +196,7 @@ macro_rules! pipe { fn min(self) -> Min where - Self::Item: Ord + $send + 'static, + Self::Output: Ord + $send + 'static, Self: Sized, { $assert_sink(Min::new(self)) @@ -205,8 +204,8 @@ macro_rules! pipe { fn min_by(self, f: F) -> MinBy where - F: $fns::FnMut(&Self::Item, &Self::Item) -> Ordering + Clone + $send + 'static, - Self::Item: $send + 'static, + F: $fns::FnMut(&Self::Output, &Self::Output) -> Ordering + Clone + $send + 'static, + Self::Output: $send + 'static, Self: Sized, { $assert_sink(MinBy::new(self, f)) @@ -214,9 +213,9 @@ macro_rules! pipe { fn min_by_key(self, f: F) -> MinByKey where - F: $fns::FnMut(&Self::Item) -> B + Clone + $send + 'static, + F: $fns::FnMut(&Self::Output) -> B + Clone + $send + 'static, B: Ord + 'static, - Self::Item: $send + 'static, + Self::Output: $send + 'static, Self: Sized, { $assert_sink(MinByKey::new(self, f)) @@ -224,7 +223,7 @@ macro_rules! pipe { fn most_frequent(self, n: usize, probability: f64, tolerance: f64) -> MostFrequent where - Self::Item: Hash + Eq + Clone + $send + 'static, + Self::Output: Hash + Eq + Clone + $send + 'static, Self: Sized, { $assert_sink(MostFrequent::new(self, n, probability, tolerance)) @@ -234,7 +233,7 @@ macro_rules! pipe { self, n: usize, probability: f64, tolerance: f64, error_rate: f64, ) -> MostDistinct where - Self: $pipe + Sized, + Self: $pipe + Sized, A: Hash + Eq + Clone + $send + 'static, B: Hash + 'static, { @@ -249,7 +248,7 @@ macro_rules! pipe { fn sample_unstable(self, samples: usize) -> SampleUnstable where - Self::Item: $send + 'static, + Self::Output: $send + 'static, Self: Sized, { $assert_sink(SampleUnstable::new(self, samples)) @@ -257,7 +256,7 @@ macro_rules! pipe { fn all(self, f: F) -> All where - F: $fns::FnMut(Self::Item) -> bool + Clone + $send + 'static, + F: $fns::FnMut(Self::Output) -> bool + Clone + $send + 'static, Self: Sized, { $assert_sink(All::new(self, f)) @@ -265,7 +264,7 @@ macro_rules! pipe { fn any(self, f: F) -> Any where - F: $fns::FnMut(Self::Item) -> bool + Clone + $send + 'static, + F: $fns::FnMut(Self::Output) -> bool + Clone + $send + 'static, Self: Sized, { $assert_sink(Any::new(self, f)) @@ -273,7 +272,7 @@ macro_rules! pipe { fn collect(self) -> Collect where - B: $from_sink, + B: $from_sink, Self: Sized, { $assert_sink(Collect::new(self)) @@ -281,7 +280,7 @@ macro_rules! pipe { } #[inline(always)] - pub(crate) fn $assert_pipe, Source>(i: I) -> I { + pub(crate) fn $assert_pipe, Input>(i: I) -> I { i } }; diff --git a/amadeus-core/src/par_sink.rs b/amadeus-core/src/par_sink.rs index 21d97a15..96eaf1f8 100644 --- a/amadeus-core/src/par_sink.rs +++ b/amadeus-core/src/par_sink.rs @@ -16,103 +16,70 @@ mod sample; mod sum; mod tuple; -use std::{future::Future, ops::DerefMut, pin::Pin}; - -use crate::{pipe::Sink, pool::ProcessSend}; - use super::par_pipe::*; +use crate::{pipe::Sink, pool::ProcessSend}; pub use self::{ all::*, any::*, collect::*, combine::*, combiner::*, count::*, fold::*, folder::*, for_each::*, fork::*, group_by::*, histogram::*, max::*, pipe::*, sample::*, sum::*, tuple::* }; #[must_use] -pub trait Reducer { - type Output; - type Async: ReducerAsync; +pub trait Reducer { + type Done; + type Async: Sink; fn into_async(self) -> Self::Async; } -pub trait ReducerSend: - Reducer>::Output> -{ - type Output: Send + 'static; +pub trait ReducerSend: Reducer>::Done> { + type Done: Send + 'static; } -pub trait ReducerProcessSend: - ReducerSend>::Output> +pub trait ReducerProcessSend: + ReducerSend>::Done> { - type Output: ProcessSend + 'static; -} -#[must_use] -pub trait ReducerAsync: Sink { - type Output; - - fn output<'a>(self: Pin<&'a mut Self>) -> Pin + 'a>>; -} - -impl ReducerAsync for Pin

-where - P: DerefMut + Unpin, - P::Target: ReducerAsync, -{ - type Output = >::Output; - - fn output<'a>(self: Pin<&'a mut Self>) -> Pin + 'a>> { - self.get_mut().as_mut().output() - } -} -impl ReducerAsync for &mut T -where - T: ReducerAsync + Unpin, -{ - type Output = T::Output; - - fn output<'a>(mut self: Pin<&'a mut Self>) -> Pin + 'a>> { - Box::pin(async move { Pin::new(&mut **self).output().await }) - } + type Done: ProcessSend + 'static; } #[must_use] -pub trait ParallelSink { - type Output; - type Pipe: ParallelPipe; - type ReduceA: ReducerSend<>::Item> + Clone + Send; +pub trait ParallelSink { + type Done; + type Pipe: ParallelPipe; + type ReduceA: ReducerSend<>::Output> + Clone + Send; type ReduceC: Reducer< - >::Item>>::Output, - Output = Self::Output, + >::Output>>::Done, + Done = Self::Done, >; fn reducers(self) -> (Self::Pipe, Self::ReduceA, Self::ReduceC); } #[inline(always)] -pub(crate) fn assert_parallel_sink, Source>(r: R) -> R { +pub(crate) fn assert_parallel_sink, Input>(r: R) -> R { r } #[must_use] -pub trait DistributedSink { - type Output; - type Pipe: DistributedPipe; - type ReduceA: ReducerSend<>::Item> +pub trait DistributedSink { + type Done; + type Pipe: DistributedPipe; + type ReduceA: ReducerSend<>::Output> + Clone + ProcessSend + Send; type ReduceB: ReducerProcessSend< - >::Item>>::Output, + >::Output>>::Done, > + Clone + ProcessSend; type ReduceC: Reducer< >::Item>>::Output, - >>::Output, - Output = Self::Output, + >::Output>>::Done, + >>::Done, + Done = Self::Done, >; fn reducers(self) -> (Self::Pipe, Self::ReduceA, Self::ReduceB, Self::ReduceC); } #[inline(always)] -pub(crate) fn assert_distributed_sink, Source>(r: R) -> R { +pub(crate) fn assert_distributed_sink, Input>(r: R) -> R { r } diff --git a/amadeus-core/src/par_sink/all.rs b/amadeus-core/src/par_sink/all.rs index b963d196..10c3a9a6 100644 --- a/amadeus-core/src/par_sink/all.rs +++ b/amadeus-core/src/par_sink/all.rs @@ -5,47 +5,47 @@ use pin_project::pin_project; use serde::{Deserialize, Serialize}; use serde_closure::traits::FnMut; use std::{ - future::Future, marker::PhantomData, pin::Pin, task::{Context, Poll} + marker::PhantomData, pin::Pin, task::{Context, Poll} }; use super::{ - DistributedPipe, DistributedSink, ParallelPipe, ParallelSink, Reducer, ReducerAsync, ReducerProcessSend, ReducerSend + DistributedPipe, DistributedSink, ParallelPipe, ParallelSink, Reducer, ReducerProcessSend, ReducerSend }; use crate::{pipe::Sink, pool::ProcessSend}; #[derive(new)] #[must_use] -pub struct All { - i: I, +pub struct All { + pipe: P, f: F, } -impl, Source, F> ParallelSink for All +impl, Input, F> ParallelSink for All where - F: FnMut<(I::Item,), Output = bool> + Clone + Send + 'static, + F: FnMut<(P::Output,), Output = bool> + Clone + Send + 'static, { - type Output = bool; - type Pipe = I; - type ReduceA = AllReducer; + type Done = bool; + type Pipe = P; + type ReduceA = AllReducer; type ReduceC = BoolAndReducer; fn reducers(self) -> (Self::Pipe, Self::ReduceA, Self::ReduceC) { - (self.i, AllReducer(self.f, PhantomData), BoolAndReducer) + (self.pipe, AllReducer(self.f, PhantomData), BoolAndReducer) } } -impl, Source, F> DistributedSink for All +impl, Input, F> DistributedSink for All where - F: FnMut<(I::Item,), Output = bool> + Clone + ProcessSend + 'static, + F: FnMut<(P::Output,), Output = bool> + Clone + ProcessSend + 'static, { - type Output = bool; - type Pipe = I; - type ReduceA = AllReducer; + type Done = bool; + type Pipe = P; + type ReduceA = AllReducer; type ReduceB = BoolAndReducer; type ReduceC = BoolAndReducer; fn reducers(self) -> (Self::Pipe, Self::ReduceA, Self::ReduceB, Self::ReduceC) { ( - self.i, + self.pipe, AllReducer(self.f, PhantomData), BoolAndReducer, BoolAndReducer, @@ -59,30 +59,30 @@ where bound(serialize = "F: Serialize"), bound(deserialize = "F: Deserialize<'de>") )] -pub struct AllReducer(F, PhantomData A>); +pub struct AllReducer(F, PhantomData Item>); -impl Reducer for AllReducer +impl Reducer for AllReducer where - F: FnMut<(A,), Output = bool>, + F: FnMut<(Item,), Output = bool>, { - type Output = bool; - type Async = AllReducerAsync; + type Done = bool; + type Async = AllReducerAsync; fn into_async(self) -> Self::Async { AllReducerAsync(self.0, true, PhantomData) } } -impl ReducerProcessSend for AllReducer +impl ReducerProcessSend for AllReducer where - F: FnMut<(A,), Output = bool>, + F: FnMut<(Item,), Output = bool>, { - type Output = bool; + type Done = bool; } -impl ReducerSend for AllReducer +impl ReducerSend for AllReducer where - F: FnMut<(A,), Output = bool>, + F: FnMut<(Item,), Output = bool>, { - type Output = bool; + type Done = bool; } #[pin_project] @@ -91,16 +91,18 @@ where bound(serialize = "F: Serialize"), bound(deserialize = "F: Deserialize<'de>") )] -pub struct AllReducerAsync(F, bool, PhantomData A>); +pub struct AllReducerAsync(F, bool, PhantomData Item>); -impl Sink for AllReducerAsync +impl Sink for AllReducerAsync where - F: FnMut<(A,), Output = bool>, + F: FnMut<(Item,), Output = bool>, { + type Done = bool; + #[inline(always)] - fn poll_pipe( - self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>, - ) -> Poll<()> { + fn poll_forward( + self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>, + ) -> Poll { let self_ = self.project(); while *self_.1 { if let Some(item) = ready!(stream.as_mut().poll_next(cx)) { @@ -109,17 +111,7 @@ where break; } } - Poll::Ready(()) - } -} -impl ReducerAsync for AllReducerAsync -where - F: FnMut<(A,), Output = bool>, -{ - type Output = bool; - - fn output<'a>(self: Pin<&'a mut Self>) -> Pin + 'a>> { - Box::pin(async move { self.1 }) + Poll::Ready(*self_.1) } } @@ -127,7 +119,7 @@ where pub struct BoolAndReducer; impl Reducer for BoolAndReducer { - type Output = bool; + type Done = bool; type Async = BoolAndReducerAsync; fn into_async(self) -> Self::Async { @@ -135,19 +127,21 @@ impl Reducer for BoolAndReducer { } } impl ReducerProcessSend for BoolAndReducer { - type Output = bool; + type Done = bool; } impl ReducerSend for BoolAndReducer { - type Output = bool; + type Done = bool; } #[pin_project] pub struct BoolAndReducerAsync(bool); impl Sink for BoolAndReducerAsync { + type Done = bool; + #[inline(always)] - fn poll_pipe( + fn poll_forward( mut self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>, - ) -> Poll<()> { + ) -> Poll { while self.0 { if let Some(item) = ready!(stream.as_mut().poll_next(cx)) { self.0 = self.0 && item; @@ -155,13 +149,6 @@ impl Sink for BoolAndReducerAsync { break; } } - Poll::Ready(()) - } -} -impl ReducerAsync for BoolAndReducerAsync { - type Output = bool; - - fn output<'a>(self: Pin<&'a mut Self>) -> Pin + 'a>> { - Box::pin(async move { self.0 }) + Poll::Ready(self.0) } } diff --git a/amadeus-core/src/par_sink/any.rs b/amadeus-core/src/par_sink/any.rs index ca77c21a..ccdfdb66 100644 --- a/amadeus-core/src/par_sink/any.rs +++ b/amadeus-core/src/par_sink/any.rs @@ -5,47 +5,47 @@ use pin_project::pin_project; use serde::{Deserialize, Serialize}; use serde_closure::traits::FnMut; use std::{ - future::Future, marker::PhantomData, pin::Pin, task::{Context, Poll} + marker::PhantomData, pin::Pin, task::{Context, Poll} }; use super::{ - DistributedPipe, DistributedSink, ParallelPipe, ParallelSink, Reducer, ReducerAsync, ReducerProcessSend, ReducerSend + DistributedPipe, DistributedSink, ParallelPipe, ParallelSink, Reducer, ReducerProcessSend, ReducerSend }; use crate::{pipe::Sink, pool::ProcessSend}; #[derive(new)] #[must_use] -pub struct Any { - i: I, +pub struct Any { + pipe: P, f: F, } -impl, Source, F> ParallelSink for Any +impl, Input, F> ParallelSink for Any where - F: FnMut<(I::Item,), Output = bool> + Clone + Send + 'static, + F: FnMut<(P::Output,), Output = bool> + Clone + Send + 'static, { - type Output = bool; - type Pipe = I; - type ReduceA = AnyReducer; + type Done = bool; + type Pipe = P; + type ReduceA = AnyReducer; type ReduceC = BoolOrReducer; fn reducers(self) -> (Self::Pipe, Self::ReduceA, Self::ReduceC) { - (self.i, AnyReducer(self.f, PhantomData), BoolOrReducer) + (self.pipe, AnyReducer(self.f, PhantomData), BoolOrReducer) } } -impl, Source, F> DistributedSink for Any +impl, Input, F> DistributedSink for Any where - F: FnMut<(I::Item,), Output = bool> + Clone + ProcessSend + 'static, + F: FnMut<(P::Output,), Output = bool> + Clone + ProcessSend + 'static, { - type Output = bool; - type Pipe = I; - type ReduceA = AnyReducer; + type Done = bool; + type Pipe = P; + type ReduceA = AnyReducer; type ReduceB = BoolOrReducer; type ReduceC = BoolOrReducer; fn reducers(self) -> (Self::Pipe, Self::ReduceA, Self::ReduceB, Self::ReduceC) { ( - self.i, + self.pipe, AnyReducer(self.f, PhantomData), BoolOrReducer, BoolOrReducer, @@ -59,30 +59,30 @@ where bound(serialize = "F: Serialize"), bound(deserialize = "F: Deserialize<'de>") )] -pub struct AnyReducer(F, PhantomData A>); +pub struct AnyReducer(F, PhantomData Item>); -impl Reducer for AnyReducer +impl Reducer for AnyReducer where - F: FnMut<(A,), Output = bool>, + F: FnMut<(Item,), Output = bool>, { - type Output = bool; - type Async = AnyReducerAsync; + type Done = bool; + type Async = AnyReducerAsync; fn into_async(self) -> Self::Async { AnyReducerAsync(self.0, true, PhantomData) } } -impl ReducerProcessSend for AnyReducer +impl ReducerProcessSend for AnyReducer where - F: FnMut<(A,), Output = bool>, + F: FnMut<(Item,), Output = bool>, { - type Output = bool; + type Done = bool; } -impl ReducerSend for AnyReducer +impl ReducerSend for AnyReducer where - F: FnMut<(A,), Output = bool>, + F: FnMut<(Item,), Output = bool>, { - type Output = bool; + type Done = bool; } #[pin_project] @@ -91,16 +91,18 @@ where bound(serialize = "F: Serialize"), bound(deserialize = "F: Deserialize<'de>") )] -pub struct AnyReducerAsync(F, bool, PhantomData A>); +pub struct AnyReducerAsync(F, bool, PhantomData Item>); -impl Sink for AnyReducerAsync +impl Sink for AnyReducerAsync where - F: FnMut<(A,), Output = bool>, + F: FnMut<(Item,), Output = bool>, { + type Done = bool; + #[inline(always)] - fn poll_pipe( - self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>, - ) -> Poll<()> { + fn poll_forward( + self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>, + ) -> Poll { let self_ = self.project(); while *self_.1 { if let Some(item) = ready!(stream.as_mut().poll_next(cx)) { @@ -109,17 +111,7 @@ where break; } } - Poll::Ready(()) - } -} -impl ReducerAsync for AnyReducerAsync -where - F: FnMut<(A,), Output = bool>, -{ - type Output = bool; - - fn output<'a>(self: Pin<&'a mut Self>) -> Pin + 'a>> { - Box::pin(async move { !self.1 }) + Poll::Ready(!*self_.1) } } @@ -127,7 +119,7 @@ where pub struct BoolOrReducer; impl Reducer for BoolOrReducer { - type Output = bool; + type Done = bool; type Async = BoolOrReducerAsync; fn into_async(self) -> Self::Async { @@ -135,10 +127,10 @@ impl Reducer for BoolOrReducer { } } impl ReducerProcessSend for BoolOrReducer { - type Output = bool; + type Done = bool; } impl ReducerSend for BoolOrReducer { - type Output = bool; + type Done = bool; } #[pin_project] @@ -146,10 +138,12 @@ impl ReducerSend for BoolOrReducer { pub struct BoolOrReducerAsync(bool); impl Sink for BoolOrReducerAsync { + type Done = bool; + #[inline(always)] - fn poll_pipe( + fn poll_forward( mut self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>, - ) -> Poll<()> { + ) -> Poll { while self.0 { if let Some(item) = ready!(stream.as_mut().poll_next(cx)) { self.0 = self.0 && !item; @@ -157,13 +151,6 @@ impl Sink for BoolOrReducerAsync { break; } } - Poll::Ready(()) - } -} -impl ReducerAsync for BoolOrReducerAsync { - type Output = bool; - - fn output<'a>(self: Pin<&'a mut Self>) -> Pin + 'a>> { - Box::pin(async move { !self.0 }) + Poll::Ready(!self.0) } } diff --git a/amadeus-core/src/par_sink/collect.rs b/amadeus-core/src/par_sink/collect.rs index 815cc6b9..704cec65 100644 --- a/amadeus-core/src/par_sink/collect.rs +++ b/amadeus-core/src/par_sink/collect.rs @@ -4,163 +4,154 @@ use futures::{pin_mut, ready, Stream, StreamExt}; use pin_project::pin_project; use serde::{Deserialize, Serialize}; use std::{ - collections::{BTreeMap, BTreeSet, BinaryHeap, HashMap, HashSet, LinkedList, VecDeque}, future::Future, hash::{BuildHasher, Hash}, iter, marker::PhantomData, pin::Pin, task::{Context, Poll} + collections::{BTreeMap, BTreeSet, BinaryHeap, HashMap, HashSet, LinkedList, VecDeque}, hash::{BuildHasher, Hash}, iter, marker::PhantomData, pin::Pin, task::{Context, Poll} }; use super::{ - DistributedPipe, DistributedSink, ParallelPipe, ParallelSink, Reducer, ReducerAsync, ReducerProcessSend, ReducerSend + DistributedPipe, DistributedSink, ParallelPipe, ParallelSink, Reducer, ReducerProcessSend, ReducerSend }; use crate::{pipe::Sink, pool::ProcessSend}; #[derive(new)] #[must_use] -pub struct Collect { - i: I, +pub struct Collect { + pipe: P, marker: PhantomData A>, } -impl, Source, T: FromParallelStream> ParallelSink - for Collect +impl, Input, T: FromParallelStream> ParallelSink + for Collect { - type Output = T; - type Pipe = I; + type Done = T; + type Pipe = P; type ReduceA = T::ReduceA; type ReduceC = T::ReduceC; fn reducers(self) -> (Self::Pipe, Self::ReduceA, Self::ReduceC) { let (a, b) = T::reducers(); - (self.i, a, b) + (self.pipe, a, b) } } -impl, Source, T: FromDistributedStream> DistributedSink - for Collect +impl, Input, T: FromDistributedStream> DistributedSink + for Collect { - type Output = T; - type Pipe = I; + type Done = T; + type Pipe = P; type ReduceA = T::ReduceA; type ReduceB = T::ReduceB; type ReduceC = T::ReduceC; fn reducers(self) -> (Self::Pipe, Self::ReduceA, Self::ReduceB, Self::ReduceC) { let (a, b, c) = T::reducers(); - (self.i, a, b, c) + (self.pipe, a, b, c) } } pub trait FromParallelStream: Sized { type ReduceA: ReducerSend + Clone + Send; - type ReduceC: Reducer<>::Output, Output = Self>; + type ReduceC: Reducer<>::Done, Done = Self>; fn reducers() -> (Self::ReduceA, Self::ReduceC); } pub trait FromDistributedStream: Sized { type ReduceA: ReducerSend + Clone + ProcessSend; - type ReduceB: ReducerProcessSend<>::Output> - + Clone - + ProcessSend; + type ReduceB: ReducerProcessSend<>::Done> + Clone + ProcessSend; type ReduceC: Reducer< - >::Output>>::Output, - Output = Self, + >::Done>>::Done, + Done = Self, >; fn reducers() -> (Self::ReduceA, Self::ReduceB, Self::ReduceC); - // fn from_dist_stream(dist_stream: I, pool: &Pool) -> Self where T: Serialize + DeserializeOwned + Send + 'static, I: IntoDistributedStream, <::Iter as DistributedStream>::Task: Serialize + DeserializeOwned + Send + 'static; + // fn from_dist_stream

(dist_stream: P, pool: &Pool) -> Self where T: Serialize + DeserializeOwned + Send + 'static, P: IntoDistributedStream, <

::Iter as DistributedStream>::Task: Serialize + DeserializeOwned + Send + 'static; } #[derive(Educe, Serialize, Deserialize, new)] #[educe(Clone, Default)] #[serde(bound = "")] -pub struct PushReducer(PhantomData (T, A)>); +pub struct PushReducer(PhantomData (T, Item)>); -impl> Reducer for PushReducer { - type Output = T; - type Async = PushReducerAsync; +impl> Reducer for PushReducer { + type Done = T; + type Async = PushReducerAsync; fn into_async(self) -> Self::Async { PushReducerAsync(Some(Default::default()), PhantomData) } } -impl> ReducerProcessSend for PushReducer +impl> ReducerProcessSend for PushReducer where T: ProcessSend + 'static, { - type Output = T; + type Done = T; } -impl> ReducerSend for PushReducer +impl> ReducerSend for PushReducer where T: Send + 'static, { - type Output = T; + type Done = T; } #[pin_project] -pub struct PushReducerAsync(Option, PhantomData A>); -impl> Sink for PushReducerAsync { +pub struct PushReducerAsync(Option, PhantomData Item>); +impl> Sink for PushReducerAsync { + type Done = T; + #[inline] - fn poll_pipe( - self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>, - ) -> Poll<()> { + fn poll_forward( + self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>, + ) -> Poll { let self_ = self.project(); while let Some(item) = ready!(stream.as_mut().poll_next(cx)) { self_.0.as_mut().unwrap().extend(iter::once(item)); } - Poll::Ready(()) - } -} -impl> ReducerAsync for PushReducerAsync { - type Output = T; - - fn output<'a>(mut self: Pin<&'a mut Self>) -> Pin + 'a>> { - Box::pin(async move { self.0.take().unwrap() }) + Poll::Ready(self_.0.take().unwrap()) } } #[derive(Educe, Serialize, Deserialize)] #[educe(Clone, Default)] #[serde(bound = "")] -pub struct ExtendReducer(PhantomData (T, A)>); -impl, T: Default + Extend, B> Reducer for ExtendReducer { - type Output = T; - type Async = ExtendReducerAsync; +pub struct ExtendReducer(PhantomData (T, Item)>); +impl, T: Default + Extend, B> Reducer + for ExtendReducer +{ + type Done = T; + type Async = ExtendReducerAsync; fn into_async(self) -> Self::Async { ExtendReducerAsync(Some(T::default()), PhantomData) } } -impl, T: Default + Extend, B> ReducerProcessSend - for ExtendReducer +impl, T: Default + Extend, B> ReducerProcessSend + for ExtendReducer where T: ProcessSend + 'static, { - type Output = T; + type Done = T; } -impl, T: Default + Extend, B> ReducerSend for ExtendReducer +impl, T: Default + Extend, B> ReducerSend + for ExtendReducer where T: Send + 'static, { - type Output = T; + type Done = T; } #[pin_project] -pub struct ExtendReducerAsync(Option, PhantomData A>); -impl, T: Extend, B> Sink for ExtendReducerAsync { +pub struct ExtendReducerAsync(Option, PhantomData Item>); +impl, T: Extend, B> Sink for ExtendReducerAsync { + type Done = T; + #[inline] - fn poll_pipe( - self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>, - ) -> Poll<()> { + fn poll_forward( + self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>, + ) -> Poll { let self_ = self.project(); while let Some(item) = ready!(stream.as_mut().poll_next(cx)) { self_.0.as_mut().unwrap().extend(item); } - Poll::Ready(()) - } -} -impl, T: Extend, B> ReducerAsync for ExtendReducerAsync { - type Output = T; - - fn output<'a>(mut self: Pin<&'a mut Self>) -> Pin + 'a>> { - Box::pin(async move { self.0.take().unwrap() }) + Poll::Ready(self_.0.take().unwrap()) } } @@ -174,9 +165,9 @@ pub struct IntoReducer(R, PhantomData T>); impl, T, Item> Reducer for IntoReducer where - R::Output: Into, + R::Done: Into, { - type Output = T; + type Done = T; type Async = IntoReducerAsync; fn into_async(self) -> Self::Async { @@ -187,34 +178,26 @@ where #[pin_project] pub struct IntoReducerAsync(#[pin] R, PhantomData T>); -impl, T, Item> Sink for IntoReducerAsync +impl, T, Item> Sink for IntoReducerAsync where - R::Output: Into, + R::Done: Into, { + type Done = T; + #[inline] - fn poll_pipe( + fn poll_forward( self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, - ) -> Poll<()> { + ) -> Poll { let stream = stream.map(Into::into); pin_mut!(stream); - self.project().0.poll_pipe(cx, stream) - } -} -impl, T, Item> ReducerAsync for IntoReducerAsync -where - R::Output: Into, -{ - type Output = T; - - fn output<'a>(self: Pin<&'a mut Self>) -> Pin + 'a>> { - Box::pin(async move { self.project().0.output().await.into() }) + self.project().0.poll_forward(cx, stream).map(Into::into) } } #[derive(Clone, Default, Serialize, Deserialize)] pub struct OptionReducer(R); impl, Item> Reducer> for OptionReducer { - type Output = Option; + type Done = Option; type Async = OptionReducerAsync; fn into_async(self) -> Self::Async { @@ -223,42 +206,33 @@ impl, Item> Reducer> for OptionReducer { } impl, Item> ReducerProcessSend> for OptionReducer where - R::Output: ProcessSend + 'static, + R::Done: ProcessSend + 'static, { - type Output = Option; + type Done = Option; } impl, Item> ReducerSend> for OptionReducer where - R::Output: Send + 'static, + R::Done: Send + 'static, { - type Output = Option; + type Done = Option; } #[pin_project] pub struct OptionReducerAsync(#[pin] Option); -impl, Item> Sink> for OptionReducerAsync { - #[inline] - fn poll_pipe( - self: Pin<&mut Self>, _cx: &mut Context, - _stream: Pin<&mut impl Stream>>, - ) -> Poll<()> { - todo!("Tracking at https://github.com/constellation-rs/amadeus/projects/3#card-40276549"); - // let self_ = self.project(); - // match (self_.0, item.is_some()) { - // (&mut Some(ref mut a), true) => { - // return a.push(item.unwrap()); - // } - // (self_, _) => *self_ = None, - // } - // self_.0.is_some() - } -} -impl, Item> ReducerAsync> for OptionReducerAsync { - type Output = Option; +impl, Item> Sink> for OptionReducerAsync { + type Done = Option; - fn output<'a>(self: Pin<&'a mut Self>) -> Pin + 'a>> { - Box::pin(async move { Some(self.project().0.as_pin_mut()?.output().await) }) + #[inline] + fn poll_forward( + self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>>, + ) -> Poll { + let stream = stream.map(|x|x.expect("Not yet implemented: Tracking at https://github.com/constellation-rs/amadeus/projects/3#card-40276549")); + pin_mut!(stream); + match self.project().0.as_pin_mut() { + Some(a) => a.poll_forward(cx, stream).map(Some), + None => Poll::Ready(None), + } } } @@ -271,7 +245,7 @@ impl, Item> ReducerAsync> for OptionReducerAs pub struct ResultReducer(R, PhantomData E>); impl, E, Item> Reducer> for ResultReducer { - type Output = Result; + type Done = Result; type Async = ResultReducerAsync; fn into_async(self) -> Self::Async { @@ -280,17 +254,17 @@ impl, E, Item> Reducer> for ResultReducer } impl, E, Item> ReducerProcessSend> for ResultReducer where - R::Output: ProcessSend + 'static, + R::Done: ProcessSend + 'static, E: ProcessSend + 'static, { - type Output = Result; + type Done = Result; } impl, E, Item> ReducerSend> for ResultReducer where - R::Output: Send + 'static, + R::Done: Send + 'static, E: Send + 'static, { - type Output = Result; + type Done = Result; } #[pin_project(project = ResultReducerAsyncProj)] @@ -298,34 +272,20 @@ pub enum ResultReducerAsync { Ok(#[pin] R), Err(Option), } -impl, E, Item> Sink> for ResultReducerAsync { +impl, E, Item> Sink> for ResultReducerAsync { + type Done = Result; + #[inline] - fn poll_pipe( - self: Pin<&mut Self>, _cx: &mut Context, - _stream: Pin<&mut impl Stream>>, - ) -> Poll<()> { - todo!("Tracking at https://github.com/constellation-rs/amadeus/projects/3#card-40276549"); - // let self_ = self.project(); - // match (self_.0, item.is_ok()) { - // (&mut Ok(ref mut a), true) => { - // return a.push(item.ok().unwrap()); - // } - // (self_, false) => *self_ = Err(item.err().unwrap()), - // _ => (), - // } - // self_.0.is_ok() - } -} -impl, E, Item> ReducerAsync> for ResultReducerAsync { - type Output = Result; - - fn output<'a>(self: Pin<&'a mut Self>) -> Pin + 'a>> { - Box::pin(async move { - match self.project() { - ResultReducerAsyncProj::Ok(a) => Ok(a.output().await), - ResultReducerAsyncProj::Err(b) => Err(b.take().unwrap()), - } - }) + fn poll_forward( + self: Pin<&mut Self>, cx: &mut Context, + stream: Pin<&mut impl Stream>>, + ) -> Poll { + let stream = stream.map(|x|x.ok().expect("Not yet implemented: Tracking at https://github.com/constellation-rs/amadeus/projects/3#card-40276549")); + pin_mut!(stream); + match self.project() { + ResultReducerAsyncProj::Ok(a) => a.poll_forward(cx, stream).map(Ok), + ResultReducerAsyncProj::Err(b) => Poll::Ready(Err(b.take().unwrap())), + } } } diff --git a/amadeus-core/src/par_sink/combine.rs b/amadeus-core/src/par_sink/combine.rs index e4b82f9c..a11c713f 100644 --- a/amadeus-core/src/par_sink/combine.rs +++ b/amadeus-core/src/par_sink/combine.rs @@ -13,17 +13,17 @@ use super::{combiner_par_sink, FolderSync, FolderSyncReducer, ParallelPipe, Para bound(deserialize = "F: Deserialize<'de>") )] pub struct ReduceFn(F, PhantomData A>); -impl FolderSync for ReduceFn +impl FolderSync for ReduceFn where F: FnMut<(A, A), Output = A>, - T: Into>, + Item: Into>, { - type Output = Option; + type Done = Option; - fn zero(&mut self) -> Self::Output { + fn zero(&mut self) -> Self::Done { None } - fn push(&mut self, state: &mut Self::Output, item: T) { + fn push(&mut self, state: &mut Self::Done, item: Item) { if let Some(item) = item.into() { *state = Some(if let Some(state) = state.take() { self.0.call_mut((state, item)) @@ -36,18 +36,17 @@ where #[derive(new)] #[must_use] -pub struct Combine { - i: I, +pub struct Combine { + pipe: P, f: F, } impl_par_dist! { - impl, Source, F> ParallelSink - for Combine + impl, Input, F> ParallelSink for Combine where - F: FnMut<(I::Item, I::Item,), Output = I::Item> + Clone + Send + 'static, - I::Item: Send + 'static, + F: FnMut<(P::Output, P::Output), Output = P::Output> + Clone + Send + 'static, + P::Output: Send + 'static, { - combiner_par_sink!(ReduceFn, self, ReduceFn::new(self.f)); + combiner_par_sink!(ReduceFn, self, ReduceFn::new(self.f)); } } diff --git a/amadeus-core/src/par_sink/combiner.rs b/amadeus-core/src/par_sink/combiner.rs index 92b2f6d8..187e0813 100644 --- a/amadeus-core/src/par_sink/combiner.rs +++ b/amadeus-core/src/par_sink/combiner.rs @@ -6,15 +6,15 @@ mod macros { #[macro_export] macro_rules! combiner_par_sink { ($combiner:ty, $self:ident, $init:expr) => { - type Output = >::Output>>::Output; - type Pipe = I; - type ReduceA = FolderSyncReducer; - type ReduceC = FolderSyncReducer<>::Output, $combiner>; + type Done = >::Done>>::Done; + type Pipe = P; + type ReduceA = FolderSyncReducer; + type ReduceC = FolderSyncReducer<>::Done, $combiner>; - fn reducers($self) -> (I, Self::ReduceA, Self::ReduceC) { + fn reducers($self) -> (P, Self::ReduceA, Self::ReduceC) { let init = $init; ( - $self.i, + $self.pipe, FolderSyncReducer::new(init.clone()), FolderSyncReducer::new(init), ) @@ -24,16 +24,16 @@ mod macros { #[macro_export] macro_rules! combiner_dist_sink { ($combiner:ty, $self:ident, $init:expr) => { - type Output = >::Output>>::Output>>::Output; - type Pipe = I; - type ReduceA = FolderSyncReducer; - type ReduceB = FolderSyncReducer<>::Output, $combiner>; - type ReduceC = FolderSyncReducer<>::Output>>::Output, $combiner>; + type Done = >::Done>>::Done>>::Done; + type Pipe = P; + type ReduceA = FolderSyncReducer; + type ReduceB = FolderSyncReducer<>::Done, $combiner>; + type ReduceC = FolderSyncReducer<>::Done>>::Done, $combiner>; - fn reducers($self) -> (I, Self::ReduceA, Self::ReduceB, Self::ReduceC) { + fn reducers($self) -> (P, Self::ReduceA, Self::ReduceB, Self::ReduceC) { let init = $init; ( - $self.i, + $self.pipe, FolderSyncReducer::new(init.clone()), FolderSyncReducer::new(init.clone()), FolderSyncReducer::new(init), @@ -48,21 +48,21 @@ mod macros { pub(crate) use macros::{combiner_dist_sink, combiner_par_sink}; pub trait CombinerSync { - type Output; + type Done; - fn combine(&mut self, a: Self::Output, b: Self::Output) -> Self::Output; + fn combine(&mut self, a: Self::Done, b: Self::Done) -> Self::Done; } -impl FolderSync for C +impl FolderSync for C where - C: CombinerSync, - A: Into>, + C: CombinerSync, + Item: Into>, { - type Output = Option; + type Done = Option; - fn zero(&mut self) -> Self::Output { + fn zero(&mut self) -> Self::Done { None } - fn push(&mut self, state: &mut Self::Output, item: A) { + fn push(&mut self, state: &mut Self::Done, item: Item) { if let Some(item) = item.into() { *state = Some(if let Some(state) = state.take() { self.combine(state, item) diff --git a/amadeus-core/src/par_sink/count.rs b/amadeus-core/src/par_sink/count.rs index 128496b0..8da8e73d 100644 --- a/amadeus-core/src/par_sink/count.rs +++ b/amadeus-core/src/par_sink/count.rs @@ -7,26 +7,32 @@ use super::{ #[derive(new)] #[must_use] -pub struct Count { - i: I, +pub struct Count

{ + pipe: P, } impl_par_dist! { - impl, Source> ParallelSink for Count { - folder_par_sink!(CountFolder, SumFolder, self, CountFolder::new(), SumFolder::new()); + impl, Input> ParallelSink for Count

{ + folder_par_sink!( + CountFolder, + SumFolder, + self, + CountFolder::new(), + SumFolder::new() + ); } } #[derive(Clone, Serialize, Deserialize, new)] pub struct CountFolder; -impl FolderSync for CountFolder { - type Output = usize; +impl FolderSync for CountFolder { + type Done = usize; - fn zero(&mut self) -> Self::Output { + fn zero(&mut self) -> Self::Done { 0 } - fn push(&mut self, state: &mut Self::Output, _item: A) { + fn push(&mut self, state: &mut Self::Done, _item: Item) { *state += 1; } } diff --git a/amadeus-core/src/par_sink/fold.rs b/amadeus-core/src/par_sink/fold.rs index 07c128eb..8a04f718 100644 --- a/amadeus-core/src/par_sink/fold.rs +++ b/amadeus-core/src/par_sink/fold.rs @@ -12,22 +12,21 @@ use super::{folder_par_sink, FolderSync, FolderSyncReducer, ParallelPipe, Parall #[derive(new)] #[must_use] -pub struct Fold { - i: I, +pub struct Fold { + pipe: P, identity: ID, op: F, marker: PhantomData B>, } impl_par_dist! { - impl, Source, ID, F, B> ParallelSink - for Fold + impl, Input, ID, F, B> ParallelSink for Fold where ID: FnMut<(), Output = B> + Clone + Send + 'static, - F: FnMut<(B, Either,), Output = B> + Clone + Send + 'static, + F: FnMut<(B, Either), Output = B> + Clone + Send + 'static, B: Send + 'static, { - folder_par_sink!(FoldFolder, FoldFolder, self, FoldFolder::new(self.identity.clone(), self.op.clone()), FoldFolder::new(self.identity, self.op)); + folder_par_sink!(FoldFolder, FoldFolder, self, FoldFolder::new(self.identity.clone(), self.op.clone()), FoldFolder::new(self.identity, self.op)); } } @@ -37,40 +36,40 @@ impl_par_dist! { bound(serialize = "ID: Serialize, F: Serialize"), bound(deserialize = "ID: Deserialize<'de>, F: Deserialize<'de>") )] -pub struct FoldFolder { +pub struct FoldFolder { identity: ID, op: F, - marker: PhantomData (A, B, Step)>, + marker: PhantomData (Item, B, Step)>, } pub struct StepA; pub struct StepB; -impl FolderSync for FoldFolder +impl FolderSync for FoldFolder where ID: FnMut<(), Output = B>, - F: FnMut<(B, Either), Output = B>, + F: FnMut<(B, Either), Output = B>, { - type Output = B; + type Done = B; - fn zero(&mut self) -> Self::Output { + fn zero(&mut self) -> Self::Done { self.identity.call_mut(()) } - fn push(&mut self, state: &mut Self::Output, item: A) { + fn push(&mut self, state: &mut Self::Done, item: Item) { replace_with_or_abort(state, |state| self.op.call_mut((state, Either::Left(item)))) } } -impl FolderSync for FoldFolder +impl FolderSync for FoldFolder where - ID: FnMut<(), Output = B>, - F: FnMut<(B, Either), Output = B>, + ID: FnMut<(), Output = Item>, + F: FnMut<(Item, Either), Output = Item>, { - type Output = B; + type Done = Item; - fn zero(&mut self) -> Self::Output { + fn zero(&mut self) -> Self::Done { self.identity.call_mut(()) } - fn push(&mut self, state: &mut Self::Output, item: B) { + fn push(&mut self, state: &mut Self::Done, item: Item) { replace_with_or_abort(state, |state| { self.op.call_mut((state, Either::Right(item))) }) diff --git a/amadeus-core/src/par_sink/folder.rs b/amadeus-core/src/par_sink/folder.rs index 0fef0d84..d0c2b2fa 100644 --- a/amadeus-core/src/par_sink/folder.rs +++ b/amadeus-core/src/par_sink/folder.rs @@ -9,23 +9,23 @@ use std::{ future::Future, marker::PhantomData, pin::Pin, task::{Context, Poll} }; -use super::{Reducer, ReducerAsync, ReducerProcessSend, ReducerSend}; +use super::{Reducer, ReducerProcessSend, ReducerSend}; use crate::{pipe::Sink, pool::ProcessSend}; mod macros { #[macro_export] macro_rules! folder_par_sink { ($folder_a:ty, $folder_b:ty, $self:ident, $init_a:expr, $init_b:expr) => { - type Output = >::Output>>::Output; - type Pipe = I; - type ReduceA = FolderSyncReducer; - type ReduceC = FolderSyncReducer<>::Output, $folder_b>; + type Done = >::Done>>::Done; + type Pipe = P; + type ReduceA = FolderSyncReducer; + type ReduceC = FolderSyncReducer<>::Done, $folder_b>; - fn reducers($self) -> (I, Self::ReduceA, Self::ReduceC) { + fn reducers($self) -> (P, Self::ReduceA, Self::ReduceC) { let init_a = $init_a; let init_b = $init_b; ( - $self.i, + $self.pipe, FolderSyncReducer::new(init_a), FolderSyncReducer::new(init_b), ) @@ -35,17 +35,17 @@ mod macros { #[macro_export] macro_rules! folder_dist_sink { ($folder_a:ty, $folder_b:ty, $self:ident, $init_a:expr, $init_b:expr) => { - type Output = >::Output>>::Output>>::Output; - type Pipe = I; - type ReduceA = FolderSyncReducer; - type ReduceB = FolderSyncReducer<>::Output, $folder_b>; - type ReduceC = FolderSyncReducer<>::Output>>::Output, $folder_b>; + type Done = >::Done>>::Done>>::Done; + type Pipe = P; + type ReduceA = FolderSyncReducer; + type ReduceB = FolderSyncReducer<>::Done, $folder_b>; + type ReduceC = FolderSyncReducer<>::Done>>::Done, $folder_b>; - fn reducers($self) -> (I, Self::ReduceA, Self::ReduceB, Self::ReduceC) { + fn reducers($self) -> (P, Self::ReduceA, Self::ReduceB, Self::ReduceC) { let init_a = $init_a; let init_b = $init_b; ( - $self.i, + $self.pipe, FolderSyncReducer::new(init_a), FolderSyncReducer::new(init_b.clone()), FolderSyncReducer::new(init_b), @@ -59,30 +59,30 @@ mod macros { pub(crate) use macros::{folder_dist_sink, folder_par_sink}; -pub trait FolderSync { - type Output; +pub trait FolderSync { + type Done; - fn zero(&mut self) -> Self::Output; - fn push(&mut self, state: &mut Self::Output, item: A); + fn zero(&mut self) -> Self::Done; + fn push(&mut self, state: &mut Self::Done, item: Item); } #[derive(Educe, Serialize, Deserialize, new)] -#[educe(Clone(bound = "C: Clone"))] +#[educe(Clone(bound = "F: Clone"))] #[serde( - bound(serialize = "C: Serialize"), - bound(deserialize = "C: Deserialize<'de>") + bound(serialize = "F: Serialize"), + bound(deserialize = "F: Deserialize<'de>") )] -pub struct FolderSyncReducer { - folder: C, - marker: PhantomData A>, +pub struct FolderSyncReducer { + folder: F, + marker: PhantomData Item>, } -impl Reducer for FolderSyncReducer +impl Reducer for FolderSyncReducer where - C: FolderSync, + F: FolderSync, { - type Output = C::Output; - type Async = FolderSyncReducerAsync; + type Done = F::Done; + type Async = FolderSyncReducerAsync; fn into_async(mut self) -> Self::Async { FolderSyncReducerAsync { @@ -92,50 +92,42 @@ where } } } -impl ReducerProcessSend for FolderSyncReducer +impl ReducerProcessSend for FolderSyncReducer where - C: FolderSync, - C::Output: ProcessSend + 'static, + F: FolderSync, + F::Done: ProcessSend + 'static, { - type Output = C::Output; + type Done = F::Done; } -impl ReducerSend for FolderSyncReducer +impl ReducerSend for FolderSyncReducer where - C: FolderSync, - C::Output: Send + 'static, + F: FolderSync, + F::Done: Send + 'static, { - type Output = C::Output; + type Done = F::Done; } #[pin_project] -pub struct FolderSyncReducerAsync { - state: Option, - folder: C, - marker: PhantomData A>, +pub struct FolderSyncReducerAsync { + state: Option, + folder: F, + marker: PhantomData Item>, } -impl Sink for FolderSyncReducerAsync +impl Sink for FolderSyncReducerAsync where - C: FolderSync, + F: FolderSync, { + type Done = F::Done; + #[inline(always)] - fn poll_pipe( - self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>, - ) -> Poll<()> { + fn poll_forward( + self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>, + ) -> Poll { let self_ = self.project(); let folder = self_.folder; while let Some(item) = ready!(stream.as_mut().poll_next(cx)) { folder.push(self_.state.as_mut().unwrap(), item); } - Poll::Ready(()) - } -} -impl ReducerAsync for FolderSyncReducerAsync -where - C: FolderSync, -{ - type Output = C::Output; - - fn output<'a>(self: Pin<&'a mut Self>) -> Pin + 'a>> { - Box::pin(async move { self.project().state.take().unwrap() }) + Poll::Ready(self_.state.take().unwrap()) } } diff --git a/amadeus-core/src/par_sink/for_each.rs b/amadeus-core/src/par_sink/for_each.rs index be021d99..24603d45 100644 --- a/amadeus-core/src/par_sink/for_each.rs +++ b/amadeus-core/src/par_sink/for_each.rs @@ -5,51 +5,51 @@ use pin_project::pin_project; use serde::{Deserialize, Serialize}; use serde_closure::traits::FnMut; use std::{ - future::Future, marker::PhantomData, pin::Pin, task::{Context, Poll} + marker::PhantomData, pin::Pin, task::{Context, Poll} }; use super::{ - DistributedPipe, DistributedSink, ParallelPipe, ParallelSink, PushReducer, Reducer, ReducerAsync, ReducerProcessSend, ReducerSend + DistributedPipe, DistributedSink, ParallelPipe, ParallelSink, PushReducer, Reducer, ReducerProcessSend, ReducerSend }; use crate::{pipe::Sink, pool::ProcessSend}; #[derive(new)] #[must_use] -pub struct ForEach { - i: I, +pub struct ForEach { + pipe: P, f: F, } -impl, Source, F> ParallelSink for ForEach +impl, Input, F> ParallelSink for ForEach where - F: FnMut<(I::Item,), Output = ()> + Clone + Send + 'static, + F: FnMut<(P::Output,), Output = ()> + Clone + Send + 'static, { - type Output = (); - type Pipe = I; - type ReduceA = ForEachReducer; + type Done = (); + type Pipe = P; + type ReduceA = ForEachReducer; type ReduceC = PushReducer<()>; fn reducers(self) -> (Self::Pipe, Self::ReduceA, Self::ReduceC) { ( - self.i, + self.pipe, ForEachReducer(self.f, PhantomData), PushReducer::new(), ) } } -impl, Source, F> DistributedSink for ForEach +impl, Input, F> DistributedSink for ForEach where - F: FnMut<(I::Item,), Output = ()> + Clone + ProcessSend + 'static, + F: FnMut<(P::Output,), Output = ()> + Clone + ProcessSend + 'static, { - type Output = (); - type Pipe = I; - type ReduceA = ForEachReducer; + type Done = (); + type Pipe = P; + type ReduceA = ForEachReducer; type ReduceB = PushReducer<()>; type ReduceC = PushReducer<()>; fn reducers(self) -> (Self::Pipe, Self::ReduceA, Self::ReduceB, Self::ReduceC) { ( - self.i, + self.pipe, ForEachReducer(self.f, PhantomData), PushReducer::new(), PushReducer::new(), @@ -64,40 +64,42 @@ where bound(serialize = "F: Serialize"), bound(deserialize = "F: Deserialize<'de>") )] -pub struct ForEachReducer(F, PhantomData A>); +pub struct ForEachReducer(F, PhantomData Item>); -impl Reducer for ForEachReducer +impl Reducer for ForEachReducer where - F: FnMut<(A,), Output = ()> + Clone, + F: FnMut<(Item,), Output = ()>, { - type Output = (); + type Done = (); type Async = Self; fn into_async(self) -> Self::Async { self } } -impl ReducerProcessSend for ForEachReducer +impl ReducerProcessSend for ForEachReducer where - F: FnMut<(A,), Output = ()> + Clone, + F: FnMut<(Item,), Output = ()>, { - type Output = (); + type Done = (); } -impl ReducerSend for ForEachReducer +impl ReducerSend for ForEachReducer where - F: FnMut<(A,), Output = ()> + Clone, + F: FnMut<(Item,), Output = ()>, { - type Output = (); + type Done = (); } -impl Sink for ForEachReducer +impl Sink for ForEachReducer where - F: FnMut<(A,), Output = ()> + Clone, + F: FnMut<(Item,), Output = ()>, { + type Done = (); + #[inline(always)] - fn poll_pipe( - self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>, - ) -> Poll<()> { + fn poll_forward( + self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>, + ) -> Poll { let self_ = self.project(); while let Some(item) = ready!(stream.as_mut().poll_next(cx)) { self_.0.call_mut((item,)); @@ -105,13 +107,3 @@ where Poll::Ready(()) } } -impl ReducerAsync for ForEachReducer -where - F: FnMut<(A,), Output = ()> + Clone, -{ - type Output = (); - - fn output<'a>(self: Pin<&'a mut Self>) -> Pin + 'a>> { - Box::pin(async move {}) - } -} diff --git a/amadeus-core/src/par_sink/fork.rs b/amadeus-core/src/par_sink/fork.rs index 9b702619..e4bc1b13 100644 --- a/amadeus-core/src/par_sink/fork.rs +++ b/amadeus-core/src/par_sink/fork.rs @@ -18,9 +18,11 @@ use crate::{ par_stream::{ParallelStream, StreamTask}, pipe::Pipe, util::transmute }; +#[pin_project] #[derive(new)] #[must_use] pub struct Fork { + #[pin] a: A, b: B, c: C, @@ -35,48 +37,58 @@ impl_par_dist! { C: ParallelPipe, RefAItem: 'static, { - type Item = Sum2; + type Item = Sum2; type Task = JoinTask; fn size_hint(&self) -> (usize, Option) { self.a.size_hint() } - fn next_task(&mut self) -> Option { - self.a.next_task().map(|task| JoinTask { - stream: task, - pipe: self.b.task(), - pipe_ref: self.c.task(), - marker: PhantomData, + fn next_task(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + let self_ = self.project(); + let b = self_.b; + let c = self_.c; + self_.a.next_task(cx).map(|task| { + task.map(|task| JoinTask { + stream: task, + pipe: b.task(), + pipe_ref: c.task(), + marker: PhantomData, + }) }) } } - impl ParallelPipe for Fork + impl ParallelPipe for Fork where - A: ParallelPipe, - B: ParallelPipe, + A: ParallelPipe, + B: ParallelPipe, C: ParallelPipe, RefAItem: 'static, { - type Item = Sum2; + type Output = Sum2; type Task = JoinTask; fn task(&self) -> Self::Task { let stream = self.a.task(); let pipe = self.b.task(); let pipe_ref = self.c.task(); - JoinTask { stream, pipe, pipe_ref, marker: PhantomData } + JoinTask { + stream, + pipe, + pipe_ref, + marker: PhantomData, + } } } } -impl ParallelSink for Fork +impl ParallelSink for Fork where - A: ParallelPipe, - B: ParallelSink, + A: ParallelPipe, + B: ParallelSink, C: ParallelSink, RefAItem: 'static, { - type Output = (B::Output, C::Output); + type Done = (B::Done, C::Done); type Pipe = Fork; type ReduceA = ReduceA2; type ReduceC = ReduceC2; @@ -91,14 +103,14 @@ where ) } } -impl DistributedSink for Fork +impl DistributedSink for Fork where - A: DistributedPipe, - B: DistributedSink, + A: DistributedPipe, + B: DistributedSink, C: DistributedSink, RefAItem: 'static, { - type Output = (B::Output, C::Output); + type Done = (B::Done, C::Done); type Pipe = Fork; type ReduceA = ReduceA2; type ReduceB = ReduceC2; @@ -133,7 +145,7 @@ where B: PipeTask, C: PipeTask, { - type Item = Sum2; + type Item = Sum2; type Async = JoinStreamTaskAsync; fn into_async(self) -> Self::Async { @@ -147,14 +159,14 @@ where } } } -impl PipeTask for JoinTask +impl PipeTask for JoinTask where - A: PipeTask, - B: PipeTask, + A: PipeTask, + B: PipeTask, C: PipeTask, { - type Item = Sum2; - type Async = JoinStreamTaskAsync; + type Output = Sum2; + type Async = JoinStreamTaskAsync; fn into_async(self) -> Self::Async { JoinStreamTaskAsync { @@ -187,7 +199,7 @@ where C: Pipe, { // TODO: fairness - fn poll(&mut self, cx: &mut Context) -> Option>>> { + fn poll(&mut self, cx: &mut Context) -> Option>>> { if let pending @ Some(_) = self.pending.as_mut().unwrap() { let ref_given = &mut *self.ref_given; { @@ -300,7 +312,7 @@ where B: Pipe, C: Pipe, { - type Item = Sum2; + type Item = Sum2; fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { let mut self_ = self.project(); @@ -315,17 +327,17 @@ where } } -impl Pipe for JoinStreamTaskAsync +impl Pipe for JoinStreamTaskAsync where - A: Pipe, - B: Pipe, + A: Pipe, + B: Pipe, C: Pipe, { - type Item = Sum2; + type Output = Sum2; fn poll_next( - self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>, - ) -> Poll> { + self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>, + ) -> Poll> { let mut self_ = self.project(); loop { if self_.pending.is_none() { diff --git a/amadeus-core/src/par_sink/group_by.rs b/amadeus-core/src/par_sink/group_by.rs index 84570f08..370fc2ee 100644 --- a/amadeus-core/src/par_sink/group_by.rs +++ b/amadeus-core/src/par_sink/group_by.rs @@ -2,15 +2,16 @@ use derive_new::new; use educe::Educe; -use futures::{future::join_all, pin_mut, ready, stream, Stream, StreamExt}; +use futures::{pin_mut, ready, stream, Stream, StreamExt}; use pin_project::pin_project; use serde::{Deserialize, Serialize}; use std::{ - collections::HashMap, future::Future, hash::Hash, marker::PhantomData, mem, pin::Pin, task::{Context, Poll} + collections::HashMap, hash::Hash, marker::PhantomData, mem, pin::Pin, task::{Context, Poll} }; +use sum::Sum2; use super::{ - DistributedPipe, DistributedSink, ParallelPipe, ParallelSink, PipeTask, Reducer, ReducerAsync, ReducerProcessSend, ReducerSend + DistributedPipe, DistributedSink, ParallelPipe, ParallelSink, PipeTask, Reducer, ReducerProcessSend, ReducerSend }; use crate::{ pipe::{Pipe, Sink, StreamExt as _}, pool::ProcessSend @@ -23,22 +24,22 @@ pub struct GroupBy { b: B, } -impl, B: ParallelSink, Source, T, U> ParallelSink +impl, B: ParallelSink, Input, T, U> ParallelSink for GroupBy where T: Eq + Hash + Send + 'static, >::Task: Clone + Send + 'static, B::ReduceA: Clone + Send + 'static, B::ReduceC: Clone, - B::Output: Send + 'static, + B::Done: Send + 'static, { - type Output = HashMap; + type Done = HashMap; type Pipe = A; type ReduceA = GroupByReducerA<>::Task, B::ReduceA, T, U>; type ReduceC = GroupByReducerB< B::ReduceC, T, - >::Item>>::Output, + >::Output>>::Done, >; fn reducers(self) -> (Self::Pipe, Self::ReduceA, Self::ReduceC) { @@ -51,30 +52,30 @@ where } } -impl, B: DistributedSink, Source, T, U> - DistributedSink for GroupBy +impl, B: DistributedSink, Input, T, U> + DistributedSink for GroupBy where T: Eq + Hash + ProcessSend + 'static, >::Task: Clone + ProcessSend + 'static, B::ReduceA: Clone + ProcessSend + 'static, B::ReduceB: Clone, B::ReduceC: Clone, - B::Output: ProcessSend + 'static, + B::Done: ProcessSend + 'static, { - type Output = HashMap; + type Done = HashMap; type Pipe = A; type ReduceA = GroupByReducerA<>::Task, B::ReduceA, T, U>; type ReduceB = GroupByReducerB< B::ReduceB, T, - >::Item>>::Output, + >::Output>>::Done, >; type ReduceC = GroupByReducerB< B::ReduceC, T, >::Item>>::Output, - >>::Output, + >::Output>>::Done, + >>::Done, >; fn reducers(self) -> (Self::Pipe, Self::ReduceA, Self::ReduceB, Self::ReduceC) { @@ -99,11 +100,11 @@ pub struct GroupByReducerA(P, R, PhantomData (R, T, U)>); impl Reducer<(T, U)> for GroupByReducerA where P: PipeTask, - R: Reducer + Clone, + R: Reducer + Clone, T: Eq + Hash, { - type Output = HashMap; - type Async = GroupByReducerAAsync; + type Done = HashMap; + type Async = GroupByReducerAAsync; fn into_async(self) -> Self::Async { GroupByReducerAAsync::new(self.0.into_async(), self.1) @@ -112,118 +113,132 @@ where impl ReducerProcessSend<(T, U)> for GroupByReducerA where P: PipeTask, - R: Reducer + Clone, + R: Reducer + Clone, T: Eq + Hash + ProcessSend + 'static, - R::Output: ProcessSend + 'static, + R::Done: ProcessSend + 'static, { - type Output = HashMap; + type Done = HashMap; } impl ReducerSend<(T, U)> for GroupByReducerA where P: PipeTask, - R: Reducer + Clone, + R: Reducer + Clone, T: Eq + Hash + Send + 'static, - R::Output: Send + 'static, + R::Done: Send + 'static, { - type Output = HashMap; + type Done = HashMap; } #[pin_project] #[derive(new)] -pub struct GroupByReducerAAsync { +pub struct GroupByReducerAAsync +where + P: Pipe, + R: Reducer, +{ #[pin] pipe: P, factory: R, #[new(default)] - pending: Option, Option>>)>>, + pending: Option, Option>>), Vec>>>, #[new(default)] - map: HashMap>>, + map: HashMap>>, marker: PhantomData (R, U)>, } -impl Sink<(T, U)> for GroupByReducerAAsync +impl Sink<(T, U)> for GroupByReducerAAsync where P: Pipe, - R: Reducer + Clone, + R: Reducer + Clone, T: Eq + Hash, { + type Done = HashMap; + #[inline(always)] - fn poll_pipe( + fn poll_forward( self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>, - ) -> Poll<()> { + ) -> Poll { let mut self_ = self.project(); loop { if !self_.pending.is_some() { - *self_.pending = Some(ready!(stream.as_mut().poll_next(cx)).map(|(k, u)| { - let r = if !self_.map.contains_key(&k) { - Some(Box::pin(self_.factory.clone().into_async())) - } else { - None - }; - (k, Some(u), r) - })); + *self_.pending = Some( + ready!(stream.as_mut().poll_next(cx)) + .map(|(k, u)| { + let r = if !self_.map.contains_key(&k) { + Some(Box::pin(self_.factory.clone().into_async())) + } else { + None + }; + (k, Some(u), r) + }) + .map_or_else( + || Sum2::B((0..self_.map.len()).map(|_| None).collect()), + Sum2::A, + ), + ); } - if let Some((k, u, r)) = self_.pending.as_mut().unwrap() { - let waker = cx.waker(); - let stream = stream::poll_fn(|cx| { - if let Some(u) = u.take() { - Poll::Ready(Some(u)) - } else { - let waker_ = cx.waker(); - if !waker.will_wake(waker_) { - waker_.wake_by_ref(); + match self_.pending.as_mut().unwrap() { + Sum2::A((k, u, r)) => { + let waker = cx.waker(); + let stream = stream::poll_fn(|cx| { + if let Some(u) = u.take() { + Poll::Ready(Some(u)) + } else { + let waker_ = cx.waker(); + if !waker.will_wake(waker_) { + waker_.wake_by_ref(); + } + Poll::Pending } - Poll::Pending + }) + .fuse() + .pipe(self_.pipe.as_mut()); + pin_mut!(stream); + let map = &mut *self_.map; + let r_ = r.as_mut().unwrap_or_else(|| map.get_mut(&k).unwrap()); + if r_.as_mut().poll_forward(cx, stream).is_ready() { + let _ = u.take(); + } + if u.is_some() { + return Poll::Pending; + } + let (k, _u, r) = self_.pending.take().unwrap().a().unwrap(); + if let Some(r) = r { + let _ = self_.map.insert(k, r); } - }) - .fuse() - .pipe(self_.pipe.as_mut()); - pin_mut!(stream); - let map = &mut *self_.map; - let r_ = r.as_mut().unwrap_or_else(|| map.get_mut(&k).unwrap()); - if r_.as_mut().poll_pipe(cx, stream).is_ready() { - let _ = u.take(); - } - if u.is_some() { - return Poll::Pending; - } - let (k, _u, r) = self_.pending.take().unwrap().unwrap(); - if let Some(r) = r { - let _ = self_.map.insert(k, r); } - } else { - for r in self_.map.values_mut() { - let stream = stream::empty(); - pin_mut!(stream); - ready!(r.as_mut().poll_pipe(cx, stream)); + Sum2::B(done) => { + let mut done_ = true; + self_ + .map + .values_mut() + .zip(done.iter_mut()) + .for_each(|(r, done)| { + if done.is_none() { + let stream = stream::empty(); + pin_mut!(stream); + if let Poll::Ready(done_) = r.as_mut().poll_forward(cx, stream) { + *done = Some(done_); + } else { + done_ = false; + } + } + }); + if !done_ { + return Poll::Pending; + } + let ret = self_ + .map + .drain() + .zip(done.iter_mut()) + .map(|((k, _), v)| (k, v.take().unwrap())) + .collect(); + return Poll::Ready(ret); } - return Poll::Ready(()); } } } } -impl ReducerAsync<(T, U)> for GroupByReducerAAsync -where - P: Pipe, - R: Reducer + Clone, - T: Eq + Hash, -{ - type Output = HashMap; - - fn output<'a>(self: Pin<&'a mut Self>) -> Pin + 'a>> { - Box::pin(async move { - let self_ = self.project(); - join_all( - mem::take(self_.map) - .into_iter() - .map(|(k, mut v)| async move { (k, v.as_mut().output().await) }), - ) - .await - .into_iter() - .collect() - }) - } -} #[derive(Educe, Serialize, Deserialize, new)] #[educe(Clone(bound = "R: Clone"))] @@ -238,8 +253,8 @@ where R: Reducer + Clone, T: Eq + Hash, { - type Output = HashMap; - type Async = GroupByReducerBAsync; + type Done = HashMap; + type Async = GroupByReducerBAsync; fn into_async(self) -> Self::Async { GroupByReducerBAsync::new(self.0) @@ -249,119 +264,133 @@ impl ReducerProcessSend> for GroupByReducerB where R: Reducer + Clone, T: Eq + Hash + ProcessSend + 'static, - R::Output: ProcessSend + 'static, + R::Done: ProcessSend + 'static, { - type Output = HashMap; + type Done = HashMap; } impl ReducerSend> for GroupByReducerB where R: Reducer + Clone, T: Eq + Hash + Send + 'static, - R::Output: Send + 'static, + R::Done: Send + 'static, { - type Output = HashMap; + type Done = HashMap; } #[pin_project] #[derive(new)] -pub struct GroupByReducerBAsync { +pub struct GroupByReducerBAsync +where + R: Reducer, +{ f: R, #[new(default)] - pending: Option>>)>>>, + pending: Option>>)>, Vec>>>, #[new(default)] - map: HashMap>>, + map: HashMap>>, marker: PhantomData (T, U)>, } -impl Sink> for GroupByReducerBAsync +impl Sink> for GroupByReducerBAsync where R: Reducer + Clone, T: Eq + Hash, { + type Done = HashMap; + #[inline(always)] - fn poll_pipe( + fn poll_forward( self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>>, - ) -> Poll<()> { + ) -> Poll { let self_ = self.project(); loop { if self_.pending.is_none() { - *self_.pending = Some(ready!(stream.as_mut().poll_next(cx)).map(|item| { - item.into_iter() - .map(|(k, v)| { - let r = if !self_.map.contains_key(&k) { - Some(Box::pin(self_.f.clone().into_async())) - } else { - None - }; - (k, (v, r)) + *self_.pending = Some( + ready!(stream.as_mut().poll_next(cx)) + .map(|item| { + item.into_iter() + .map(|(k, v)| { + let r = if !self_.map.contains_key(&k) { + Some(Box::pin(self_.f.clone().into_async())) + } else { + None + }; + (k, (v, r)) + }) + .collect() }) - .collect() - })); + .map_or_else( + || Sum2::B((0..self_.map.len()).map(|_| None).collect()), + Sum2::A, + ), + ); } - if let Some(pending) = self_.pending.as_mut().unwrap() { - while let Some((k, (v, mut r))) = pop(pending) { - let mut v = Some(v); - let waker = cx.waker(); - let stream = stream::poll_fn(|cx| { - if let Some(v) = v.take() { - Poll::Ready(Some(v)) - } else { - let waker_ = cx.waker(); - if !waker.will_wake(waker_) { - waker_.wake_by_ref(); + match self_.pending.as_mut().unwrap() { + Sum2::A(pending) => { + while let Some((k, (v, mut r))) = pop(pending) { + let mut v = Some(v); + let waker = cx.waker(); + let stream = stream::poll_fn(|cx| { + if let Some(v) = v.take() { + Poll::Ready(Some(v)) + } else { + let waker_ = cx.waker(); + if !waker.will_wake(waker_) { + waker_.wake_by_ref(); + } + Poll::Pending } - Poll::Pending + }) + .fuse(); + pin_mut!(stream); + let map = &mut *self_.map; + let r_ = r.as_mut().unwrap_or_else(|| map.get_mut(&k).unwrap()); + if r_.as_mut().poll_forward(cx, stream).is_ready() { + let _ = v.take(); + } + if let Some(v) = v { + let _ = pending.insert(k, (v, r)); + return Poll::Pending; + } + if let Some(r) = r { + let _ = self_.map.insert(k, r); } - }) - .fuse(); - pin_mut!(stream); - let map = &mut *self_.map; - let r_ = r.as_mut().unwrap_or_else(|| map.get_mut(&k).unwrap()); - if r_.as_mut().poll_pipe(cx, stream).is_ready() { - let _ = v.take(); } - if let Some(v) = v { - let _ = pending.insert(k, (v, r)); + *self_.pending = None; + } + Sum2::B(done) => { + let mut done_ = true; + self_ + .map + .values_mut() + .zip(done.iter_mut()) + .for_each(|(r, done)| { + if done.is_none() { + let stream = stream::empty(); + pin_mut!(stream); + if let Poll::Ready(done_) = r.as_mut().poll_forward(cx, stream) { + *done = Some(done_); + } else { + done_ = false; + } + } + }); + if !done_ { return Poll::Pending; } - if let Some(r) = r { - let _ = self_.map.insert(k, r); - } + let ret = self_ + .map + .drain() + .zip(done.iter_mut()) + .map(|((k, _), v)| (k, v.take().unwrap())) + .collect(); + return Poll::Ready(ret); } - *self_.pending = None; - } else { - for r in self_.map.values_mut() { - let stream = stream::empty(); - pin_mut!(stream); - ready!(r.as_mut().poll_pipe(cx, stream)); - } - return Poll::Ready(()); } } } } -impl ReducerAsync> for GroupByReducerBAsync -where - R: Reducer + Clone, - T: Eq + Hash, -{ - type Output = HashMap; - - fn output<'a>(self: Pin<&'a mut Self>) -> Pin + 'a>> { - Box::pin(async move { - let self_ = self.project(); - join_all( - mem::take(self_.map) - .into_iter() - .map(|(k, mut v)| async move { (k, v.as_mut().output().await) }), - ) - .await - .into_iter() - .collect() - }) - } -} // https://github.com/rust-lang/rfcs/issues/1800#issuecomment-653757340 #[allow(clippy::unnecessary_filter_map)] diff --git a/amadeus-core/src/par_sink/histogram.rs b/amadeus-core/src/par_sink/histogram.rs index bbd3c416..459aff52 100644 --- a/amadeus-core/src/par_sink/histogram.rs +++ b/amadeus-core/src/par_sink/histogram.rs @@ -11,17 +11,16 @@ use super::{folder_par_sink, FolderSync, FolderSyncReducer, ParallelPipe, Parall #[derive(new)] #[must_use] -pub struct Histogram { - i: I, +pub struct Histogram

{ + pipe: P, } impl_par_dist! { - impl, Source> ParallelSink - for Histogram + impl, Input> ParallelSink for Histogram

where - I::Item: Hash + Ord + Send + 'static, + P::Output: Hash + Ord + Send + 'static, { - folder_par_sink!(HistogramFolder, HistogramFolder, self, HistogramFolder::new(), HistogramFolder::new()); + folder_par_sink!(HistogramFolder, HistogramFolder, self, HistogramFolder::new(), HistogramFolder::new()); } } @@ -35,16 +34,16 @@ pub struct HistogramFolder { pub struct StepA; pub struct StepB; -impl FolderSync for HistogramFolder +impl FolderSync for HistogramFolder where - B: Hash + Ord, + Item: Hash + Ord, { - type Output = HashMap; + type Done = HashMap; - fn zero(&mut self) -> Self::Output { + fn zero(&mut self) -> Self::Done { HashMap::new() } - fn push(&mut self, state: &mut Self::Output, item: B) { + fn push(&mut self, state: &mut Self::Done, item: Item) { *state.entry(item).or_insert(0) += 1; } } @@ -52,12 +51,12 @@ impl FolderSync> for HistogramFolder where B: Hash + Ord, { - type Output = Vec<(B, usize)>; + type Done = Vec<(B, usize)>; - fn zero(&mut self) -> Self::Output { + fn zero(&mut self) -> Self::Done { Vec::new() } - fn push(&mut self, state: &mut Self::Output, b: HashMap) { + fn push(&mut self, state: &mut Self::Done, b: HashMap) { let mut b = b.into_iter().collect::>(); b.sort_by(|a, b| a.0.cmp(&b.0)); replace_with_or_default(state, |state| { @@ -79,12 +78,12 @@ impl FolderSync> for HistogramFolder where B: Hash + Ord, { - type Output = Vec<(B, usize)>; + type Done = Vec<(B, usize)>; - fn zero(&mut self) -> Self::Output { + fn zero(&mut self) -> Self::Done { Vec::new() } - fn push(&mut self, state: &mut Self::Output, b: Vec<(B, usize)>) { + fn push(&mut self, state: &mut Self::Done, b: Vec<(B, usize)>) { replace_with_or_default(state, |state| { state .into_iter() diff --git a/amadeus-core/src/par_sink/max.rs b/amadeus-core/src/par_sink/max.rs index ed02f1fd..cc40088e 100644 --- a/amadeus-core/src/par_sink/max.rs +++ b/amadeus-core/src/par_sink/max.rs @@ -8,99 +8,101 @@ use super::{combiner_par_sink, CombinerSync, FolderSyncReducer, ParallelPipe, Pa #[derive(new)] #[must_use] -pub struct Max { - i: I, +pub struct Max

{ + pipe: P, } impl_par_dist! { - impl, Source> ParallelSink for Max + impl, Input> ParallelSink for Max

where - I::Item: Ord + Send + 'static, + P::Output: Ord + Send + 'static, { - combiner_par_sink!(combine::Max, self, combine::Max::new()); + combiner_par_sink!(combine::Max, self, combine::Max::new()); } } #[derive(new)] #[must_use] -pub struct MaxBy { - i: I, +pub struct MaxBy { + pipe: P, f: F, } impl_par_dist! { - impl, Source, F> ParallelSink - for MaxBy + impl, Input, F> ParallelSink for MaxBy where - F: for<'a,'b> FnMut<(&'a I::Item, &'b I::Item,), Output = Ordering> + Clone + Send + 'static, - I::Item: Send + 'static, + F: for<'a, 'b> FnMut<(&'a P::Output, &'b P::Output), Output = Ordering> + + Clone + + Send + + 'static, + P::Output: Send + 'static, { - combiner_par_sink!(combine::MaxBy, self, combine::MaxBy::new(self.f)); + combiner_par_sink!(combine::MaxBy, self, combine::MaxBy::new(self.f)); } } #[derive(new)] #[must_use] -pub struct MaxByKey { - i: I, +pub struct MaxByKey { + pipe: P, f: F, } impl_par_dist! { - impl, Source, F, B> ParallelSink - for MaxByKey + impl, Input, F, B> ParallelSink for MaxByKey where - F: for<'a> FnMut<(&'a I::Item,), Output = B> + Clone + Send + 'static, + F: for<'a> FnMut<(&'a P::Output,), Output = B> + Clone + Send + 'static, B: Ord + 'static, - I::Item: Send + 'static, + P::Output: Send + 'static, { - combiner_par_sink!(combine::MaxByKey, self, combine::MaxByKey::new(self.f)); + combiner_par_sink!(combine::MaxByKey, self, combine::MaxByKey::new(self.f)); } } #[derive(new)] #[must_use] -pub struct Min { - i: I, +pub struct Min

{ + pipe: P, } impl_par_dist! { - impl, Source> ParallelSink for Min + impl, Input> ParallelSink for Min

where - I::Item: Ord + Send + 'static, + P::Output: Ord + Send + 'static, { - combiner_par_sink!(combine::Min, self, combine::Min::new()); + combiner_par_sink!(combine::Min, self, combine::Min::new()); } } #[derive(new)] #[must_use] -pub struct MinBy { - i: I, +pub struct MinBy { + pipe: P, f: F, } impl_par_dist! { - impl, Source, F> ParallelSink - for MinBy + impl, Input, F> ParallelSink for MinBy where - F: for<'a,'b> FnMut<(&'a I::Item, &'b I::Item,), Output = Ordering> + Clone + Send + 'static, - I::Item: Send + 'static, + F: for<'a, 'b> FnMut<(&'a P::Output, &'b P::Output), Output = Ordering> + + Clone + + Send + + 'static, + P::Output: Send + 'static, { - combiner_par_sink!(combine::MinBy, self, combine::MinBy::new(self.f)); + combiner_par_sink!(combine::MinBy, self, combine::MinBy::new(self.f)); } } #[derive(new)] #[must_use] -pub struct MinByKey { - i: I, +pub struct MinByKey { + pipe: P, f: F, } impl_par_dist! { - impl, Source, F, B> ParallelSink - for MinByKey + impl, Input, F, B> ParallelSink for MinByKey where - F: for<'a> FnMut<(&'a I::Item,), Output = B> + Clone + Send + 'static, + F: for<'a> FnMut<(&'a P::Output,), Output = B> + Clone + Send + 'static, B: Ord + 'static, - I::Item: Send + 'static, + P::Output: Send + 'static, { - combiner_par_sink!(combine::MinByKey, self, combine::MinByKey::new(self.f)); + combiner_par_sink!(combine::MinByKey, self, combine::MinByKey::new(self.f)); } } @@ -112,7 +114,7 @@ mod combine { #[serde(bound = "")] pub struct Max(PhantomData A>); impl CombinerSync for Max { - type Output = A; + type Done = A; fn combine(&mut self, a: A, b: A) -> A { // switch to b even if it is only equal, to preserve stability. @@ -132,7 +134,7 @@ mod combine { )] pub struct MaxBy(pub F, PhantomData A>); impl FnMut<(&'a A, &'b A), Output = Ordering>> CombinerSync for MaxBy { - type Output = A; + type Done = A; fn combine(&mut self, a: A, b: A) -> A { if self.0.call_mut((&a, &b)) != Ordering::Greater { @@ -151,7 +153,7 @@ mod combine { )] pub struct MaxByKey(pub F, pub PhantomData); impl FnMut<(&'a A,), Output = B>, B: Ord> CombinerSync for MaxByKey { - type Output = A; + type Done = A; fn combine(&mut self, a: A, b: A) -> A { if self.0.call_mut((&a,)).cmp(&self.0.call_mut((&b,))) != Ordering::Greater { @@ -167,7 +169,7 @@ mod combine { #[serde(bound = "")] pub struct Min(PhantomData A>); impl CombinerSync for Min { - type Output = A; + type Done = A; fn combine(&mut self, a: A, b: A) -> A { // switch to b even if it is strictly smaller, to preserve stability. @@ -187,7 +189,7 @@ mod combine { )] pub struct MinBy(pub F, PhantomData A>); impl FnMut<(&'a A, &'b A), Output = Ordering>> CombinerSync for MinBy { - type Output = A; + type Done = A; fn combine(&mut self, a: A, b: A) -> A { if self.0.call_mut((&a, &b)) == Ordering::Greater { @@ -206,7 +208,7 @@ mod combine { )] pub struct MinByKey(pub F, pub PhantomData); impl FnMut<(&'a A,), Output = B>, B: Ord> CombinerSync for MinByKey { - type Output = A; + type Done = A; fn combine(&mut self, a: A, b: A) -> A { if self.0.call_mut((&a,)).cmp(&self.0.call_mut((&b,))) == Ordering::Greater { diff --git a/amadeus-core/src/par_sink/pipe.rs b/amadeus-core/src/par_sink/pipe.rs index d0bf1a41..d5381e15 100644 --- a/amadeus-core/src/par_sink/pipe.rs +++ b/amadeus-core/src/par_sink/pipe.rs @@ -1,36 +1,47 @@ use derive_new::new; use pin_project::pin_project; use serde::{Deserialize, Serialize}; +use std::{ + pin::Pin, task::{Context, Poll} +}; use super::{DistributedPipe, DistributedSink, ParallelPipe, ParallelSink, PipeTask}; use crate::{ par_stream::{ParallelStream, StreamTask}, pipe::{Pipe as _, PipePipe, StreamExt, StreamPipe} }; +#[pin_project] #[derive(new)] #[must_use] pub struct Pipe { + #[pin] a: A, b: B, } impl_par_dist! { impl> ParallelStream for Pipe { - type Item = B::Item; + type Item = B::Output; type Task = JoinTask; - fn next_task(&mut self) -> Option { - self.a.next_task().map(|a| { - let b = self.b.task(); - JoinTask { a, b } + fn next_task(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + let self_ = self.project(); + let b = self_.b; + self_.a.next_task(cx).map(|task| { + task.map(|a| { + let b = b.task(); + JoinTask { a, b } + }) }) } fn size_hint(&self) -> (usize, Option) { self.a.size_hint() } } - impl, B: ParallelPipe, Source> ParallelPipe for Pipe { - type Item = B::Item; + impl, B: ParallelPipe, Input> ParallelPipe + for Pipe + { + type Output = B::Output; type Task = JoinTask; fn task(&self) -> Self::Task { @@ -41,10 +52,8 @@ impl_par_dist! { } } -impl, B: ParallelSink, Source> ParallelSink - for Pipe -{ - type Output = B::Output; +impl, B: ParallelSink, Input> ParallelSink for Pipe { + type Done = B::Done; type Pipe = Pipe; type ReduceA = B::ReduceA; type ReduceC = B::ReduceC; @@ -54,10 +63,10 @@ impl, B: ParallelSink, Source> ParallelSink, B: DistributedSink, Source> DistributedSink +impl, B: DistributedSink, Input> DistributedSink for Pipe { - type Output = B::Output; + type Done = B::Done; type Pipe = Pipe; type ReduceA = B::ReduceA; type ReduceB = B::ReduceB; @@ -79,7 +88,7 @@ pub struct JoinTask { } impl> StreamTask for JoinTask { - type Item = B::Item; + type Item = B::Output; type Async = StreamPipe; fn into_async(self) -> Self::Async { @@ -87,8 +96,8 @@ impl> StreamTask for JoinTask { } } -impl, B: PipeTask, Source> PipeTask for JoinTask { - type Item = B::Item; +impl, B: PipeTask, Input> PipeTask for JoinTask { + type Output = B::Output; type Async = PipePipe; fn into_async(self) -> Self::Async { diff --git a/amadeus-core/src/par_sink/sample.rs b/amadeus-core/src/par_sink/sample.rs index 1798f552..0eec0dff 100644 --- a/amadeus-core/src/par_sink/sample.rs +++ b/amadeus-core/src/par_sink/sample.rs @@ -12,18 +12,23 @@ use super::{ #[derive(new)] #[must_use] -pub struct SampleUnstable { - i: I, +pub struct SampleUnstable

{ + pipe: P, samples: usize, } impl_par_dist! { - impl, Source> ParallelSink - for SampleUnstable + impl, Input> ParallelSink for SampleUnstable

where - I::Item: Send + 'static, + P::Output: Send + 'static, { - folder_par_sink!(SampleUnstableFolder, SumFolder>, self, SampleUnstableFolder::new(self.samples), SumFolder::new()); + folder_par_sink!( + SampleUnstableFolder, + SumFolder>, + self, + SampleUnstableFolder::new(self.samples), + SumFolder::new() + ); } } @@ -32,33 +37,38 @@ pub struct SampleUnstableFolder { samples: usize, } -impl FolderSync for SampleUnstableFolder { - type Output = SASampleUnstable; +impl FolderSync for SampleUnstableFolder { + type Done = SASampleUnstable; - fn zero(&mut self) -> Self::Output { + fn zero(&mut self) -> Self::Done { SASampleUnstable::new(self.samples) } - fn push(&mut self, state: &mut Self::Output, item: A) { + fn push(&mut self, state: &mut Self::Done, item: Item) { state.push(item, &mut thread_rng()) } } #[derive(new)] #[must_use] -pub struct MostFrequent { - i: I, +pub struct MostFrequent

{ + pipe: P, n: usize, probability: f64, tolerance: f64, } impl_par_dist! { - impl, Source> ParallelSink - for MostFrequent + impl, Input> ParallelSink for MostFrequent

where - I::Item: Clone + Hash + Eq + Send + 'static, + P::Output: Clone + Hash + Eq + Send + 'static, { - folder_par_sink!(MostFrequentFolder, SumZeroFolder>, self, MostFrequentFolder::new(self.n, self.probability, self.tolerance), SumZeroFolder::new(Top::new(self.n, self.probability, self.tolerance, ()))); + folder_par_sink!( + MostFrequentFolder, + SumZeroFolder>, + self, + MostFrequentFolder::new(self.n, self.probability, self.tolerance), + SumZeroFolder::new(Top::new(self.n, self.probability, self.tolerance, ())) + ); } } @@ -69,24 +79,24 @@ pub struct MostFrequentFolder { tolerance: f64, } -impl FolderSync for MostFrequentFolder +impl FolderSync for MostFrequentFolder where - A: Clone + Hash + Eq + Send + 'static, + Item: Clone + Hash + Eq + Send + 'static, { - type Output = Top; + type Done = Top; - fn zero(&mut self) -> Self::Output { + fn zero(&mut self) -> Self::Done { Top::new(self.n, self.probability, self.tolerance, ()) } - fn push(&mut self, state: &mut Self::Output, item: A) { + fn push(&mut self, state: &mut Self::Done, item: Item) { state.push(item, &1) } } #[derive(new)] #[must_use] -pub struct MostDistinct { - i: I, +pub struct MostDistinct

{ + pipe: P, n: usize, probability: f64, tolerance: f64, @@ -94,12 +104,23 @@ pub struct MostDistinct { } impl_par_dist! { - impl, Source, A, B> ParallelSink for MostDistinct + impl, Input, A, B> ParallelSink for MostDistinct

where A: Clone + Hash + Eq + Send + 'static, B: Hash + 'static, { - folder_par_sink!(MostDistinctFolder, SumZeroFolder>>, self, MostDistinctFolder::new(self.n, self.probability, self.tolerance, self.error_rate), SumZeroFolder::new(Top::new(self.n, self.probability, self.tolerance, self.error_rate))); + folder_par_sink!( + MostDistinctFolder, + SumZeroFolder>>, + self, + MostDistinctFolder::new(self.n, self.probability, self.tolerance, self.error_rate), + SumZeroFolder::new(Top::new( + self.n, + self.probability, + self.tolerance, + self.error_rate + )) + ); } } @@ -116,12 +137,12 @@ where A: Clone + Hash + Eq + Send + 'static, B: Hash + 'static, { - type Output = Top>; + type Done = Top>; - fn zero(&mut self) -> Self::Output { + fn zero(&mut self) -> Self::Done { Top::new(self.n, self.probability, self.tolerance, self.error_rate) } - fn push(&mut self, state: &mut Self::Output, item: (A, B)) { + fn push(&mut self, state: &mut Self::Done, item: (A, B)) { state.push(item.0, &item.1) } } diff --git a/amadeus-core/src/par_sink/sum.rs b/amadeus-core/src/par_sink/sum.rs index aaaa7aab..b68eb5d3 100644 --- a/amadeus-core/src/par_sink/sum.rs +++ b/amadeus-core/src/par_sink/sum.rs @@ -8,18 +8,23 @@ use super::{folder_par_sink, FolderSync, FolderSyncReducer, ParallelPipe, Parall #[derive(new)] #[must_use] -pub struct Sum { - i: I, +pub struct Sum { + pipe: P, marker: PhantomData B>, } impl_par_dist! { - impl, Source, B> ParallelSink - for Sum + impl, Input, B> ParallelSink for Sum where - B: iter::Sum + iter::Sum + Send + 'static, + B: iter::Sum + iter::Sum + Send + 'static, { - folder_par_sink!(SumFolder, SumFolder, self, SumFolder::new(), SumFolder::new()); + folder_par_sink!( + SumFolder, + SumFolder, + self, + SumFolder::new(), + SumFolder::new() + ); } } @@ -30,16 +35,16 @@ pub struct SumFolder { marker: PhantomData B>, } -impl FolderSync for SumFolder +impl FolderSync for SumFolder where - B: iter::Sum + iter::Sum, + B: iter::Sum + iter::Sum, { - type Output = B; + type Done = B; - fn zero(&mut self) -> Self::Output { + fn zero(&mut self) -> Self::Done { iter::empty::().sum() } - fn push(&mut self, state: &mut Self::Output, item: A) { + fn push(&mut self, state: &mut Self::Done, item: Item) { *state = iter::once(mem::replace(state, iter::empty::().sum())) .chain(iter::once(iter::once(item).sum::())) .sum(); @@ -56,20 +61,20 @@ impl SumZeroFolder { } } -impl FolderSync for SumZeroFolder +impl FolderSync for SumZeroFolder where - Option: iter::Sum, + Option: iter::Sum, { - type Output = B; + type Done = Item; - fn zero(&mut self) -> Self::Output { + fn zero(&mut self) -> Self::Done { self.zero.take().unwrap() } - fn push(&mut self, state: &mut Self::Output, item: B) { + fn push(&mut self, state: &mut Self::Done, item: Item) { replace_with_or_abort(state, |state| { iter::once(state) - .chain(iter::once(iter::once(item).sum::>().unwrap())) - .sum::>() + .chain(iter::once(iter::once(item).sum::>().unwrap())) + .sum::>() .unwrap() }) } diff --git a/amadeus-core/src/par_sink/tuple.rs b/amadeus-core/src/par_sink/tuple.rs index da89a0b2..eaf52232 100644 --- a/amadeus-core/src/par_sink/tuple.rs +++ b/amadeus-core/src/par_sink/tuple.rs @@ -1,20 +1,16 @@ #![allow(non_snake_case, clippy::type_complexity, irrefutable_let_patterns, clippy::new_without_default, unused_mut, unreachable_code, clippy::too_many_arguments)] use derive_new::new; -use futures::{ - future::{ - join as Join2, join3 as Join3, join4 as Join4, join5 as Join5, maybe_done, FusedFuture, MaybeDone - }, pin_mut, ready, stream, Stream, StreamExt -}; +use futures::{pin_mut, ready, stream, Stream, StreamExt}; use pin_project::pin_project; use serde::{Deserialize, Serialize}; use std::{ - future::Future, pin::Pin, task::{Context, Poll} + pin::Pin, task::{Context, Poll} }; use sum::{Sum0, Sum1, Sum2, Sum3, Sum4, Sum5, Sum6, Sum7, Sum8}; use super::{ - DistributedPipe, DistributedSink, ParallelPipe, ParallelSink, PipeTask, Reducer, ReducerAsync, ReducerProcessSend, ReducerSend + DistributedPipe, DistributedSink, ParallelPipe, ParallelSink, PipeTask, Reducer, ReducerProcessSend, ReducerSend }; use crate::{ pipe::{Pipe, Sink}, pool::ProcessSend @@ -53,13 +49,13 @@ where macro_rules! impl_tuple { ($reducea:ident $reduceaasync:ident $reduceb:ident $reducebasync:ident $async:ident $enum:ident $join:ident $($copy:ident)? : $($num:tt $t:ident $s:ident $i:ident $r:ident $o:ident $c:ident $iterator:ident $reducera:ident $reducerb:ident $($copyb:ident)? , $comma:tt)*) => ( impl< - Source, - $($r: ParallelSink,)* + Input, + $($r: ParallelSink,)* $($o,)* - > ParallelSink for ($($r,)*) - where Source: $($copy)*, + > ParallelSink for ($($r,)*) + where Input: $($copy)*, { - type Output = ($($o,)*); + type Done = ($($o,)*); type Pipe = ($($r::Pipe,)*); type ReduceA = $reducea<$($r::ReduceA,)*>; type ReduceC = $reduceb<$($r::ReduceC,)*>; @@ -74,13 +70,13 @@ macro_rules! impl_tuple { } } impl< - Source, - $($r: DistributedSink,)* + Input, + $($r: DistributedSink,)* $($o,)* - > DistributedSink for ($($r,)*) - where Source: $($copy)*, + > DistributedSink for ($($r,)*) + where Input: $($copy)*, { - type Output = ($($o,)*); + type Done = ($($o,)*); type Pipe = ($($r::Pipe,)*); type ReduceA = $reducea<$($r::ReduceA,)*>; type ReduceB = $reduceb<$($r::ReduceB,)*>; @@ -97,22 +93,22 @@ macro_rules! impl_tuple { } } - impl,)*> - ParallelPipe for ($($i,)*) - where Source: $($copy)*, + impl,)*> + ParallelPipe for ($($i,)*) + where Input: $($copy)*, { - type Item = $enum<$($i::Item,)*>; + type Output = $enum<$($i::Output,)*>; type Task = ($($i::Task,)*); fn task(&self) -> Self::Task { ($(self.$num.task(),)*) } } - impl,)*> - DistributedPipe for ($($i,)*) - where Source: $($copy)*, + impl,)*> + DistributedPipe for ($($i,)*) + where Input: $($copy)*, { - type Item = $enum<$($i::Item,)*>; + type Output = $enum<$($i::Output,)*>; type Task = ($($i::Task,)*); fn task(&self) -> Self::Task { @@ -120,12 +116,13 @@ macro_rules! impl_tuple { } } - impl,)*> PipeTask for ($($c,)*) + impl,)*> PipeTask for ($($c,)*) where - Source: $($copy)*, + Input: $($copy)*, { - type Item = $enum<$($c::Item,)*>; - type Async = $async; + type Output = $enum<$($c::Output,)*>; + type Async = $async; + fn into_async(self) -> Self::Async { $async{ $($t: Some(self.$num.into_async()),)* @@ -136,23 +133,23 @@ macro_rules! impl_tuple { } #[pin_project] - pub struct $async { + pub struct $async { $(#[pin] $t: Option<$c>,)* - pending: Option>, + pending: Option>, given: ($(bool $comma)*), } #[allow(unused_variables)] - impl,)*> Pipe for $async + impl,)*> Pipe for $async where - Source: $($copy)*, + Input: $($copy)*, { - type Item = $enum<$($c::Item,)*>; + type Output = $enum<$($c::Output,)*>; #[allow(non_snake_case)] fn poll_next( - self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>, - ) -> Poll> { + self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>, + ) -> Poll> { let mut self_ = self.project(); // buffer, copy to each loop { @@ -160,7 +157,7 @@ macro_rules! impl_tuple { *self_.pending = Some(ready!(stream.as_mut().poll_next(cx))); } $({ - let pending: &mut Option = self_.pending.as_mut().unwrap(); + let pending: &mut Option = self_.pending.as_mut().unwrap(); let given = &mut self_.given.$num; let waker = cx.waker(); let stream_ = stream::poll_fn(|cx| { @@ -205,32 +202,34 @@ macro_rules! impl_tuple { $($t: $t,)* } impl<$($t: Reducer<$s>,)* $($s,)*> Reducer<$enum<$($s,)*>> for $reducea<$($t,)*> { - type Output = ($($t::Output,)*); + type Done = ($($t::Done,)*); type Async = $reduceaasync<$($t::Async,)* $($s,)*>; fn into_async(self) -> Self::Async { $reduceaasync{ $($t: self.$t.into_async(),)* peeked: None, - ready: ($(false $comma)*), + ready: ($(None::<$t::Done>,)*), } } } - impl<$($t: Reducer<$s>,)* $($s,)*> ReducerProcessSend<$enum<$($s,)*>> for $reducea<$($t,)*> where $($t::Output: ProcessSend + 'static,)* { - type Output = ($($t::Output,)*); + impl<$($t: Reducer<$s>,)* $($s,)*> ReducerProcessSend<$enum<$($s,)*>> for $reducea<$($t,)*> where $($t::Done: ProcessSend + 'static,)* { + type Done = ($($t::Done,)*); } - impl<$($t: Reducer<$s>,)* $($s,)*> ReducerSend<$enum<$($s,)*>> for $reducea<$($t,)*> where $($t::Output: Send + 'static,)* { - type Output = ($($t::Output,)*); + impl<$($t: Reducer<$s>,)* $($s,)*> ReducerSend<$enum<$($s,)*>> for $reducea<$($t,)*> where $($t::Done: Send + 'static,)* { + type Done = ($($t::Done,)*); } #[pin_project] - pub struct $reduceaasync<$($t,)* $($s,)*> where $($t: ReducerAsync<$s>,)* { + pub struct $reduceaasync<$($t,)* $($s,)*> where $($t: Sink<$s>,)* { $(#[pin] $t: $t,)* peeked: Option<$enum<$($s,)*>>, - ready: ($(bool $comma)*), + ready: ($(Option<$t::Done>,)*), } #[allow(unused_variables)] - impl<$($t: ReducerAsync<$s>,)* $($s,)*> Sink<$enum<$($s,)*>> for $reduceaasync<$($t,)* $($s,)*> { - fn poll_pipe(self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>>) -> Poll<()> { + impl<$($t: Sink<$s>,)* $($s,)*> Sink<$enum<$($s,)*>> for $reduceaasync<$($t,)* $($s,)*> { + type Done = ($($t::Done,)*); + + fn poll_forward(self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>>) -> Poll { let mut self_ = self.project(); loop { let mut progress = false; @@ -239,13 +238,14 @@ macro_rules! impl_tuple { pin_mut!(stream); let stream_ = substream(cx, stream, |item| if let $enum::$t(_) = item { true } else { false }, |item| { progress = true; if let $enum::$t(item) = item { item } else { unreachable!() } }); pin_mut!(stream_); - match if !self_.ready.$num { Some(self_.$t.as_mut().poll_pipe(cx, stream_)) } else { None } { - Some(Poll::Ready(())) | None => self_.ready.$num = true, - Some(Poll::Pending) => () + if self_.ready.$num.is_none() { + if let Poll::Ready(done) = self_.$t.as_mut().poll_forward(cx, stream_) { + self_.ready.$num = Some(done); + } } })* - if $(self_.ready.$num &&)* true { - break Poll::Ready(()) + if $(self_.ready.$num.is_some() &&)* true { + break Poll::Ready(($(self_.ready.$num.take().unwrap(),)*)); } if !progress { break Poll::Pending; @@ -253,49 +253,40 @@ macro_rules! impl_tuple { } } } - #[allow(unused_variables)] - impl<$($t: ReducerAsync<$s>,)* $($s,)*> ReducerAsync<$enum<$($s,)*>> for $reduceaasync<$($t,)* $($s,)*> { - type Output = ($($t::Output,)*); - - fn output<'a>(self: Pin<&'a mut Self>) -> Pin + 'a>> { - Box::pin(async move { - let self_ = self.project(); - $join($(self_.$t.output(),)*).await - }) - } - } #[derive(Clone, Serialize, Deserialize, new)] pub struct $reduceb<$($t,)*> { $($t: $t,)* } impl<$($t: Reducer<$s>,)* $($s,)*> Reducer<($($s,)*)> for $reduceb<$($t,)*> { - type Output = ($($t::Output,)*); + type Done = ($($t::Done,)*); type Async = $reducebasync<$($t::Async,)* $($s,)*>; fn into_async(self) -> Self::Async { $reducebasync{ $($t: self.$t.into_async(),)* peeked: None, - ready: ($(false $comma)*), + ready: ($(None::<$t::Done>,)*), } } } impl<$($t: ReducerProcessSend<$s>,)* $($s,)*> ReducerProcessSend<($($s,)*)> for $reduceb<$($t,)*> { - type Output = ($(<$t as ReducerProcessSend<$s>>::Output,)*); + type Done = ($(<$t as ReducerProcessSend<$s>>::Done,)*); } impl<$($t: ReducerSend<$s>,)* $($s,)*> ReducerSend<($($s,)*)> for $reduceb<$($t,)*> { - type Output = ($(<$t as ReducerSend<$s>>::Output,)*); + type Done = ($(<$t as ReducerSend<$s>>::Done,)*); } #[pin_project] - pub struct $reducebasync<$($t,)* $($s,)*> where $($t: ReducerAsync<$s>,)* { + pub struct $reducebasync<$($t,)* $($s,)*> where $($t: Sink<$s>,)* { $(#[pin] $t: $t,)* peeked: Option<($(Option<$s>,)*)>, - ready: ($(bool $comma)*), + ready: ($(Option<$t::Done>,)*), } #[allow(unused_variables)] - impl<$($t: ReducerAsync<$s>,)* $($s,)*> Sink<($($s,)*)> for $reducebasync<$($t,)* $($s,)*> { - fn poll_pipe(self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>) -> Poll<()> { + impl<$($t: Sink<$s>,)* $($s,)*> Sink<($($s,)*)> for $reducebasync<$($t,)* $($s,)*> { + type Done = ($($t::Done,)*); + + fn poll_forward(self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>) -> Poll { let mut self_ = self.project(); let stream = stream.map(|item| ($(Some(item.$num),)*)); pin_mut!(stream); @@ -321,13 +312,14 @@ macro_rules! impl_tuple { None => Poll::Ready(None), }).fuse(); pin_mut!(stream); - match if !self_.ready.$num { Some(self_.$t.as_mut().poll_pipe(cx, stream)) } else{ None } { - Some(Poll::Ready(())) | None => self_.ready.$num = true, - Some(Poll::Pending) => () + if self_.ready.$num.is_none() { + if let Poll::Ready(done) = self_.$t.as_mut().poll_forward(cx, stream) { + self_.ready.$num = Some(done); + } } })* - if $(self_.ready.$num &&)* true { - break Poll::Ready(()); + if $(self_.ready.$num.is_some() &&)* true { + break Poll::Ready(($(self_.ready.$num.take().unwrap(),)*)); } if let Some(peeked) = self_.peeked { if $(peeked.$num.is_none() &&)* true { @@ -341,17 +333,6 @@ macro_rules! impl_tuple { } } } - #[allow(unused_variables)] - impl<$($t: ReducerAsync<$s>,)* $($s,)*> ReducerAsync<($($s,)*)> for $reducebasync<$($t,)* $($s,)*> { - type Output = ($($t::Output,)*); - - fn output<'a>(self: Pin<&'a mut Self>) -> Pin + 'a>> { - Box::pin(async move { - let self_ = self.project(); - $join($(self_.$t.output(),)*).await - }) - } - } ); } impl_tuple!(ReduceA0 ReduceA0Async ReduceC0 ReduceC0Async AsyncTuple0 Sum0 Join0:); @@ -411,60 +392,3 @@ impl<'a, S: Stream> Stream for Peekable<'a, S> { (lower, upper) } } - -// from https://github.com/rust-lang/futures-rs/blob/19831dbecd13961fbd4243c1114663e24299c33d/futures-util/src/future/join.rs -macro_rules! generate { - ($( - ($Join:ident, <$($Fut:ident),*>), - )*) => ($( - #[pin_project] - struct $Join<$($Fut: Future),*> { - $(#[pin] $Fut: MaybeDone<$Fut>,)* - dummy: (), - } - - fn $Join<$($Fut: Future),*>($($Fut: $Fut),*) -> $Join<$($Fut),*> { - $Join { - $($Fut: maybe_done($Fut),)* - dummy: (), - } - } - - #[allow(unused_variables)] - impl<$($Fut: Future),*> Future for $Join<$($Fut),*> { - type Output = ($($Fut::Output,)*); - - fn poll( - self: Pin<&mut Self>, cx: &mut Context<'_> - ) -> Poll { - let mut all_done = true; - let mut futures = self.project(); - $( - all_done &= futures.$Fut.as_mut().poll(cx).is_ready(); - )* - - if all_done { - Poll::Ready(($(futures.$Fut.take_output().unwrap(),)*)) - } else { - Poll::Pending - } - } - } - - impl<$($Fut: FusedFuture),*> FusedFuture for $Join<$($Fut),*> { - fn is_terminated(&self) -> bool { - $( - self.$Fut.is_terminated() && - )* true - } - } - )*) -} - -generate! { - (Join0, <>), - (Join1, ), - (Join6, ), - (Join7, ), - (Join8, ), -} diff --git a/amadeus-core/src/par_stream.rs b/amadeus-core/src/par_stream.rs index 54b4fc1c..acdc0b71 100644 --- a/amadeus-core/src/par_stream.rs +++ b/amadeus-core/src/par_stream.rs @@ -15,10 +15,13 @@ mod update; use async_trait::async_trait; use either::Either; -use futures::{pin_mut, stream, stream::StreamExt as _, Stream}; +use futures::{future, pin_mut, stream, stream::StreamExt as _, Stream}; use serde_closure::{traits, FnOnce}; -use std::{cmp::Ordering, collections::HashMap, hash::Hash, iter, ops, vec}; +use std::{ + cmp::Ordering, collections::HashMap, hash::Hash, iter, ops, pin::Pin, task::{Context, Poll}, vec +}; +use super::{par_pipe::*, par_sink::*}; use crate::{ into_par_stream::{IntoDistributedStream, IntoParallelStream}, pipe::StreamExt, pool::{ProcessPool, ProcessSend, ThreadPool} }; @@ -27,8 +30,6 @@ pub use self::{ chain::*, cloned::*, filter::*, flat_map::*, identity::*, inspect::*, map::*, update::* }; -use super::{par_pipe::*, par_sink::*}; - #[must_use] pub trait StreamTask { type Item; @@ -45,8 +46,9 @@ macro_rules! stream { pub trait $stream { type Item; type Task: StreamTask + $send; + + fn next_task(self: Pin<&mut Self>, cx: &mut Context) -> Poll>; fn size_hint(&self) -> (usize, Option); - fn next_task(&mut self) -> Option; $($items)* @@ -338,17 +340,19 @@ stream!(ParallelStream ParallelPipe ParallelSink FromParallelStream IntoParallel where P: ThreadPool, R1: ReducerSend + Clone + Send + 'static, - R3: Reducer<>::Output, Output = B>, + R3: Reducer<>::Done, Done = B>, Self::Task: 'static, Self: Sized, { + let self_ = self; + pin_mut!(self_); // TODO: don't buffer tasks before sending. requires changes to ThreadPool let mut tasks = (0..pool.threads()).map(|_| vec![]).collect::>(); let mut allocated = 0; 'a: loop { for i in 0..tasks.len() { loop { - let (mut lower, _upper) = self.size_hint(); + let (mut lower, _upper) = self_.size_hint(); if lower == 0 { lower = 1; } @@ -361,7 +365,7 @@ stream!(ParallelStream ParallelPipe ParallelSink FromParallelStream IntoParallel break; } for _ in 0..batch { - if let Some(task) = self.next_task() { + if let Some(task) = future::poll_fn(|cx| self_.as_mut().next_task(cx)).await { tasks[i].push(task); allocated += 1; } else { @@ -394,8 +398,7 @@ stream!(ParallelStream ParallelPipe ParallelSink FromParallelStream IntoParallel pin_mut!(sink); let tasks = stream::iter(tasks.into_iter().map(StreamTask::into_async)).flatten(); - tasks.sink(sink.as_mut()).await; - sink.output().await + tasks.sink(sink.as_mut()).await }) }) .collect::>(); @@ -404,14 +407,13 @@ stream!(ParallelStream ParallelPipe ParallelSink FromParallelStream IntoParallel }); let reduce_c = reduce_c.into_async(); pin_mut!(reduce_c); - stream.sink(reduce_c.as_mut()).await; - reduce_c.output().await + stream.sink(reduce_c.as_mut()).await } async fn pipe(self, pool: &P, sink: ParSink) -> A where P: ThreadPool, - ParSink: ParallelSink, + ParSink: ParallelSink, >::Task: 'static, ParSink::ReduceA: 'static, Self::Task: 'static, @@ -429,8 +431,8 @@ stream!(ParallelStream ParallelPipe ParallelSink FromParallelStream IntoParallel ) -> (A, B) where P: ThreadPool, - ParSinkA: ParallelSink, - ParSinkB: for<'a> ParallelSink<&'a Self::Item, Output = B> + 'static, + ParSinkA: ParallelSink, + ParSinkB: for<'a> ParallelSink<&'a Self::Item, Done = B> + 'static, >::Task: 'static, ParSinkA::ReduceA: 'static, >::ReduceA: 'static, @@ -452,7 +454,7 @@ stream!(ParallelStream ParallelPipe ParallelSink FromParallelStream IntoParallel .await } - async fn group_by(self, pool: &P, sink: S) -> HashMap + async fn group_by(self, pool: &P, sink: S) -> HashMap where P: ThreadPool, A: Eq + Hash + Send + 'static, @@ -461,7 +463,7 @@ stream!(ParallelStream ParallelPipe ParallelSink FromParallelStream IntoParallel >::Task: Clone + Send + 'static, S::ReduceA: 'static, S::ReduceC: Clone, - S::Output: Send + 'static, + S::Done: Send + 'static, Self::Task: 'static, Self: ParallelStream + Sized, { @@ -489,24 +491,26 @@ stream!(DistributedStream DistributedPipe DistributedSink FromDistributedStream where P: ProcessPool, R1: ReducerSend + Clone + ProcessSend + 'static, - R2: ReducerProcessSend<>::Output> + R2: ReducerProcessSend<>::Done> + Clone + ProcessSend + 'static, R3: Reducer< - >::Output>>::Output, - Output = B, + >::Done>>::Done, + Done = B, >, Self::Task: 'static, Self: Sized, { + let self_ = self; + pin_mut!(self_); // TODO: don't buffer tasks before sending. requires changes to ProcessPool let mut tasks = (0..pool.processes()).map(|_| vec![]).collect::>(); let mut allocated = 0; 'a: loop { for i in 0..tasks.len() { loop { - let (mut lower, _upper) = self.size_hint(); + let (mut lower, _upper) = self_.size_hint(); if lower == 0 { lower = 1; } @@ -519,7 +523,7 @@ stream!(DistributedStream DistributedPipe DistributedSink FromDistributedStream break; } for _ in 0..batch { - if let Some(task) = self.next_task() { + if let Some(task) = future::poll_fn(|cx| self_.as_mut().next_task(cx)).await { tasks[i].push(task); allocated += 1; } else { @@ -602,8 +606,7 @@ stream!(DistributedStream DistributedPipe DistributedSink FromDistributedStream let tasks = stream::iter(tasks.into_iter().map(StreamTask::into_async)) .flatten(); - tasks.sink(sink.as_mut()).await; - sink.output().await + tasks.sink(sink.as_mut()).await }) }) .collect::>(); @@ -616,8 +619,7 @@ stream!(DistributedStream DistributedPipe DistributedSink FromDistributedStream let reduce_b = reduce_b.into_async(); async move { pin_mut!(reduce_b); - stream.sink(reduce_b.as_mut()).await; - reduce_b.output().await + stream.sink(reduce_b.as_mut()).await } })) }) @@ -627,14 +629,13 @@ stream!(DistributedStream DistributedPipe DistributedSink FromDistributedStream }); let reduce_c = reduce_c.into_async(); pin_mut!(reduce_c); - stream.sink(reduce_c.as_mut()).await; - reduce_c.output().await + stream.sink(reduce_c.as_mut()).await } async fn pipe(self, pool: &P, sink: DistSink) -> A where P: ProcessPool, - DistSink: DistributedSink, + DistSink: DistributedSink, >::Task: 'static, DistSink::ReduceA: 'static, DistSink::ReduceB: 'static, @@ -653,8 +654,8 @@ stream!(DistributedStream DistributedPipe DistributedSink FromDistributedStream ) -> (A, B) where P: ProcessPool, - DistSinkA: DistributedSink, - DistSinkB: for<'a> DistributedSink<&'a Self::Item, Output = B> + 'static, + DistSinkA: DistributedSink, + DistSinkB: for<'a> DistributedSink<&'a Self::Item, Done = B> + 'static, >::Task: 'static, DistSinkA::ReduceA: 'static, DistSinkA::ReduceB: 'static, @@ -679,7 +680,7 @@ stream!(DistributedStream DistributedPipe DistributedSink FromDistributedStream .await } - async fn group_by(self, pool: &P, sink: S) -> HashMap + async fn group_by(self, pool: &P, sink: S) -> HashMap where P: ProcessPool, A: Eq + Hash + ProcessSend + 'static, @@ -689,7 +690,7 @@ stream!(DistributedStream DistributedPipe DistributedSink FromDistributedStream S::ReduceA: 'static, S::ReduceB: 'static, S::ReduceC: Clone, - S::Output: ProcessSend + 'static, + S::Done: ProcessSend + 'static, Self::Task: 'static, Self: DistributedStream + Sized, { diff --git a/amadeus-core/src/par_stream/chain.rs b/amadeus-core/src/par_stream/chain.rs index 13a52cf4..1afcc46a 100644 --- a/amadeus-core/src/par_stream/chain.rs +++ b/amadeus-core/src/par_stream/chain.rs @@ -8,10 +8,13 @@ use std::{ use super::{ParallelStream, StreamTask}; +#[pin_project] #[derive(new)] #[must_use] pub struct Chain { + #[pin] a: A, + #[pin] b: B, } @@ -32,11 +35,13 @@ impl_par_dist! { }, ) } - fn next_task(&mut self) -> Option { - self.a - .next_task() - .map(ChainTask::A) - .or_else(|| self.b.next_task().map(ChainTask::B)) + fn next_task(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + let self_ = self.project(); + match self_.a.next_task(cx) { + Poll::Ready(Some(a)) => Poll::Ready(Some(ChainTask::A(a))), + Poll::Ready(None) => self_.b.next_task(cx).map(|task| task.map(ChainTask::B)), + Poll::Pending => Poll::Pending, + } } } } @@ -50,6 +55,7 @@ pub enum ChainTask { impl> StreamTask for ChainTask { type Item = A::Item; type Async = ChainTask; + fn into_async(self) -> Self::Async { match self { ChainTask::A(a) => ChainTask::A(a.into_async()), diff --git a/amadeus-core/src/par_stream/cloned.rs b/amadeus-core/src/par_stream/cloned.rs index 281ee8d7..7985e7f5 100644 --- a/amadeus-core/src/par_stream/cloned.rs +++ b/amadeus-core/src/par_stream/cloned.rs @@ -11,41 +11,22 @@ use crate::pipe::Pipe; #[derive(new)] #[must_use] -pub struct Cloned { - i: I, - marker: PhantomData, +pub struct Cloned { + pipe: P, + marker: PhantomData, } -// impl<'a,I,T:'a> DistributedStream for Cloned -// where -// I: DistributedStream, -// T: Clone, -// { -// type Item = T; -// type Task = ClonedTask; -// fn size_hint(&self) -> (usize, Option) { -// self.i.size_hint() -// } -// fn next_task(&mut self) -> Option { -// self.i.next_task().map(|task| ClonedTask { task }) -// } -// } - -// https://github.com/rust-lang/rust/issues/55731 -// https://play.rust-lang.org/?version=nightly&mode=debug&edition=2015&gist=238651c4992913bcd62b68b4832fcd9a -// https://play.rust-lang.org/?version=nightly&mode=debug&edition=2015&gist=2f1da304878b050cc313c0279047b0fa - impl_par_dist! { - impl<'a, I, Source, T: 'a> ParallelPipe<&'a Source> for Cloned + impl<'a, P, Input, T: 'a> ParallelPipe<&'a Input> for Cloned where - I: ParallelPipe<&'a Source, Item = &'a T>, + P: ParallelPipe<&'a Input, Output = &'a T>, T: Clone, { - type Item = T; - type Task = ClonedTask; + type Output = T; + type Task = ClonedTask; fn task(&self) -> Self::Task { - let task = self.i.task(); + let task = self.pipe.task(); ClonedTask { task } } } @@ -57,12 +38,12 @@ pub struct ClonedTask { #[pin] task: T, } -impl<'a, C, Source, T: 'a> PipeTask<&'a Source> for ClonedTask +impl<'a, C, Input, T: 'a> PipeTask<&'a Input> for ClonedTask where - C: PipeTask<&'a Source, Item = &'a T>, + C: PipeTask<&'a Input, Output = &'a T>, T: Clone, { - type Item = T; + type Output = T; type Async = ClonedTask; fn into_async(self) -> Self::Async { @@ -71,26 +52,17 @@ where } } } -// impl<'a,C,T:'a> StreamTask for ClonedTask -// where -// C: StreamTask, -// T: Clone, -// { -// type Item = T; -// fn run(self, i: &mut impl FnMut(Self::Item) -> bool) -> bool { -// self.task.run(&mut |item| i(item.clone())) -// } -// } -impl<'a, C, Source: 'a, T: 'a> Pipe<&'a Source> for ClonedTask + +impl<'a, C, Input: 'a, T: 'a> Pipe<&'a Input> for ClonedTask where - C: Pipe<&'a Source, Item = &'a T>, + C: Pipe<&'a Input, Output = &'a T>, T: Clone, { - type Item = T; + type Output = T; fn poll_next( - self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, - ) -> Poll> { + self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, + ) -> Poll> { self.project() .task .poll_next(cx, stream) diff --git a/amadeus-core/src/par_stream/filter.rs b/amadeus-core/src/par_stream/filter.rs index 2c370e74..f2067a86 100644 --- a/amadeus-core/src/par_stream/filter.rs +++ b/amadeus-core/src/par_stream/filter.rs @@ -1,44 +1,54 @@ use derive_new::new; +use pin_project::pin_project; use serde::{Deserialize, Serialize}; use serde_closure::traits::FnMut; +use std::{ + pin::Pin, task::{Context, Poll} +}; use super::{ParallelPipe, ParallelStream, PipeTask, StreamTask}; +#[pin_project] #[derive(new)] #[must_use] -pub struct Filter { - i: I, +pub struct Filter { + #[pin] + pipe: P, f: F, } impl_par_dist! { - impl ParallelStream for Filter + impl ParallelStream for Filter where - F: for<'a> FnMut<(&'a I::Item,), Output = bool> + Clone + Send + 'static, + F: for<'a> FnMut<(&'a P::Item,), Output = bool> + Clone + Send + 'static, { - type Item = I::Item; - type Task = FilterTask; + type Item = P::Item; + type Task = FilterTask; fn size_hint(&self) -> (usize, Option) { - (0, self.i.size_hint().1) + (0, self.pipe.size_hint().1) } - fn next_task(&mut self) -> Option { - self.i.next_task().map(|task| { - let f = self.f.clone(); - FilterTask { task, f } + fn next_task(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + let self_ = self.project(); + let f = self_.f; + self_.pipe.next_task(cx).map(|task| { + task.map(|task| { + let f = f.clone(); + FilterTask { task, f } + }) }) } } - impl, F, Source> ParallelPipe for Filter + impl, F, Input> ParallelPipe for Filter where - F: for<'a> FnMut<(&'a I::Item,), Output = bool> + Clone + Send + 'static, + F: for<'a> FnMut<(&'a P::Output,), Output = bool> + Clone + Send + 'static, { - type Item = I::Item; - type Task = FilterTask; + type Output = P::Output; + type Task = FilterTask; fn task(&self) -> Self::Task { - let task = self.i.task(); + let task = self.pipe.task(); let f = self.f.clone(); FilterTask { task, f } } @@ -62,11 +72,11 @@ where crate::pipe::Filter::new(self.task.into_async(), self.f) } } -impl, F, Source> PipeTask for FilterTask +impl, F, Input> PipeTask for FilterTask where - F: for<'a> FnMut<(&'a C::Item,), Output = bool>, + F: for<'a> FnMut<(&'a C::Output,), Output = bool>, { - type Item = C::Item; + type Output = C::Output; type Async = crate::pipe::Filter; fn into_async(self) -> Self::Async { diff --git a/amadeus-core/src/par_stream/flat_map.rs b/amadeus-core/src/par_stream/flat_map.rs index 3ac329db..6a2cc4c2 100644 --- a/amadeus-core/src/par_stream/flat_map.rs +++ b/amadeus-core/src/par_stream/flat_map.rs @@ -1,45 +1,55 @@ use derive_new::new; use futures::Stream; +use pin_project::pin_project; use serde::{Deserialize, Serialize}; use serde_closure::traits::FnMut; +use std::{ + pin::Pin, task::{Context, Poll} +}; use super::{ParallelPipe, ParallelStream, PipeTask, StreamTask}; +#[pin_project] #[derive(new)] #[must_use] -pub struct FlatMap { - i: I, +pub struct FlatMap { + #[pin] + pipe: P, f: F, } impl_par_dist! { - impl ParallelStream for FlatMap + impl ParallelStream for FlatMap where - F: FnMut<(I::Item,), Output = R> + Clone + Send + 'static, + F: FnMut<(P::Item,), Output = R> + Clone + Send + 'static, { type Item = R::Item; - type Task = FlatMapTask; + type Task = FlatMapTask; fn size_hint(&self) -> (usize, Option) { (0, None) } - fn next_task(&mut self) -> Option { - self.i.next_task().map(|task| { - let f = self.f.clone(); - FlatMapTask { task, f } + fn next_task(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + let self_ = self.project(); + let f = self_.f; + self_.pipe.next_task(cx).map(|task| { + task.map(|task| { + let f = f.clone(); + FlatMapTask { task, f } + }) }) } } - impl, F, R: Stream, Source> ParallelPipe for FlatMap + impl, F, R: Stream, Input> ParallelPipe for FlatMap where - F: FnMut<(I::Item,), Output = R> + Clone + Send + 'static, + F: FnMut<(P::Output,), Output = R> + Clone + Send + 'static, { - type Item = R::Item; - type Task = FlatMapTask; + type Output = R::Item; + type Task = FlatMapTask; fn task(&self) -> Self::Task { - let task = self.i.task(); + let task = self.pipe.task(); let f = self.f.clone(); FlatMapTask { task, f } } @@ -61,10 +71,10 @@ impl + Clone, R: Stream> StreamT crate::pipe::FlatMap::new(self.task.into_async(), self.f) } } -impl, F: FnMut<(C::Item,), Output = R> + Clone, R: Stream, Source> - PipeTask for FlatMapTask +impl, F: FnMut<(C::Output,), Output = R> + Clone, R: Stream, Input> + PipeTask for FlatMapTask { - type Item = R::Item; + type Output = R::Item; type Async = crate::pipe::FlatMap; fn into_async(self) -> Self::Async { diff --git a/amadeus-core/src/par_stream/identity.rs b/amadeus-core/src/par_stream/identity.rs index 86344ab7..be1bf48d 100644 --- a/amadeus-core/src/par_stream/identity.rs +++ b/amadeus-core/src/par_stream/identity.rs @@ -16,7 +16,7 @@ pub struct Identity; impl_par_dist! { impl ParallelPipe for Identity { - type Item = Item; + type Output = Item; type Task = IdentityTask; fn task(&self) -> Self::Task { @@ -200,7 +200,7 @@ mod workaround { #[derive(Clone, Serialize, Deserialize)] pub struct IdentityTask; impl PipeTask for IdentityTask { - type Item = Item; + type Output = Item; type Async = IdentityTask; fn into_async(self) -> Self::Async { @@ -208,11 +208,11 @@ impl PipeTask for IdentityTask { } } impl crate::pipe::Pipe for IdentityTask { - type Item = Item; + type Output = Item; fn poll_next( self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, - ) -> Poll> { + ) -> Poll> { stream.poll_next(cx) } } diff --git a/amadeus-core/src/par_stream/inspect.rs b/amadeus-core/src/par_stream/inspect.rs index bb878ffb..ddd0b854 100644 --- a/amadeus-core/src/par_stream/inspect.rs +++ b/amadeus-core/src/par_stream/inspect.rs @@ -10,41 +10,47 @@ use std::{ use super::{ParallelPipe, ParallelStream, PipeTask, StreamTask}; use crate::pipe::Pipe; +#[pin_project] #[derive(new)] #[must_use] -pub struct Inspect { - i: I, +pub struct Inspect { + #[pin] + pipe: P, f: F, } impl_par_dist! { - impl ParallelStream for Inspect + impl ParallelStream for Inspect where - F: for<'a> FnMut<(&'a I::Item,), Output = ()> + Clone + Send + 'static, + F: for<'a> FnMut<(&'a P::Item,), Output = ()> + Clone + Send + 'static, { - type Item = I::Item; - type Task = InspectTask; + type Item = P::Item; + type Task = InspectTask; fn size_hint(&self) -> (usize, Option) { - self.i.size_hint() + self.pipe.size_hint() } - fn next_task(&mut self) -> Option { - self.i.next_task().map(|task| { - let f = self.f.clone(); - InspectTask { task, f } + fn next_task(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + let self_ = self.project(); + let f = self_.f; + self_.pipe.next_task(cx).map(|task| { + task.map(|task| { + let f = f.clone(); + InspectTask { task, f } + }) }) } } - impl, F, Source> ParallelPipe for Inspect + impl, F, Input> ParallelPipe for Inspect where - F: for<'a> FnMut<(&'a I::Item,), Output = ()> + Clone + Send + 'static, + F: for<'a> FnMut<(&'a P::Output,), Output = ()> + Clone + Send + 'static, { - type Item = I::Item; - type Task = InspectTask; + type Output = P::Output; + type Task = InspectTask; fn task(&self) -> Self::Task { - let task = self.i.task(); + let task = self.pipe.task(); let f = self.f.clone(); InspectTask { task, f } } @@ -65,6 +71,7 @@ where { type Item = C::Item; type Async = InspectTask; + fn into_async(self) -> Self::Async { InspectTask { task: self.task.into_async(), @@ -72,12 +79,13 @@ where } } } -impl, F, Source> PipeTask for InspectTask +impl, F, Input> PipeTask for InspectTask where - F: for<'a> FnMut<(&'a C::Item,), Output = ()> + Clone, + F: for<'a> FnMut<(&'a C::Output,), Output = ()> + Clone, { - type Item = C::Item; + type Output = C::Output; type Async = InspectTask; + fn into_async(self) -> Self::Async { InspectTask { task: self.task.into_async(), @@ -104,15 +112,15 @@ where } } -impl, F, Source> Pipe for InspectTask +impl, F, Input> Pipe for InspectTask where - F: for<'a> FnMut<(&'a C::Item,), Output = ()> + Clone, + F: for<'a> FnMut<(&'a C::Output,), Output = ()> + Clone, { - type Item = C::Item; + type Output = C::Output; fn poll_next( - self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, - ) -> Poll> { + self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, + ) -> Poll> { let mut self_ = self.project(); let (task, f) = (self_.task, &mut self_.f); task.poll_next(cx, stream).map(|item| { diff --git a/amadeus-core/src/par_stream/map.rs b/amadeus-core/src/par_stream/map.rs index 006e467b..bf87cb71 100644 --- a/amadeus-core/src/par_stream/map.rs +++ b/amadeus-core/src/par_stream/map.rs @@ -2,44 +2,53 @@ use derive_new::new; use pin_project::pin_project; use serde::{Deserialize, Serialize}; use serde_closure::traits::FnMut; +use std::{ + pin::Pin, task::{Context, Poll} +}; use super::{ParallelPipe, ParallelStream, PipeTask, StreamTask}; +#[pin_project] #[derive(new)] #[must_use] -pub struct Map { - i: I, +pub struct Map { + #[pin] + pipe: P, f: F, } impl_par_dist! { - impl ParallelStream for Map + impl ParallelStream for Map where - F: FnMut<(I::Item,), Output = R> + Clone + Send + 'static, + F: FnMut<(P::Item,), Output = R> + Clone + Send + 'static, { type Item = R; - type Task = MapTask; + type Task = MapTask; fn size_hint(&self) -> (usize, Option) { - self.i.size_hint() + self.pipe.size_hint() } - fn next_task(&mut self) -> Option { - self.i.next_task().map(|task| { - let f = self.f.clone(); - MapTask { task, f } + fn next_task(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + let self_ = self.project(); + let f = self_.f; + self_.pipe.next_task(cx).map(|task| { + task.map(|task| { + let f = f.clone(); + MapTask { task, f } + }) }) } } - impl, F, R, Source> ParallelPipe for Map + impl, F, R, Input> ParallelPipe for Map where - F: FnMut<(I::Item,), Output = R> + Clone + Send + 'static, + F: FnMut<(P::Output,), Output = R> + Clone + Send + 'static, { - type Item = R; - type Task = MapTask; + type Output = R; + type Task = MapTask; fn task(&self) -> Self::Task { - let task = self.i.task(); + let task = self.pipe.task(); let f = self.f.clone(); MapTask { task, f } } @@ -59,18 +68,18 @@ where F: FnMut<(C::Item,), Output = R> + Clone, { type Item = R; - type Async = crate::pipe::Map; + type Async = crate::pipe::Map; fn into_async(self) -> Self::Async { crate::pipe::Map::new(self.task.into_async(), self.f) } } -impl, F, R, Source> PipeTask for MapTask +impl, F, R, Input> PipeTask for MapTask where - F: FnMut<(C::Item,), Output = R> + Clone, + F: FnMut<(C::Output,), Output = R> + Clone, { - type Item = R; - type Async = crate::pipe::Map; + type Output = R; + type Async = crate::pipe::Map; fn into_async(self) -> Self::Async { crate::pipe::Map::new(self.task.into_async(), self.f) diff --git a/amadeus-core/src/par_stream/sum_type.rs b/amadeus-core/src/par_stream/sum_type.rs index 8037d82d..eee0f57b 100644 --- a/amadeus-core/src/par_stream/sum_type.rs +++ b/amadeus-core/src/par_stream/sum_type.rs @@ -18,18 +18,18 @@ impl_par_dist! { Self::B(i) => i.size_hint(), } } - fn next_task(&mut self) -> Option { - match self { - Self::A(i) => i.next_task().map(Sum2::A), - Self::B(i) => i.next_task().map(Sum2::B), + fn next_task(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + match self.as_pin_mut() { + Sum2::A(i) => i.next_task(cx).map(|task| task.map(Sum2::A)), + Sum2::B(i) => i.next_task(cx).map(|task| task.map(Sum2::B)), } } } - impl, B: ParallelPipe, Source> - ParallelPipe for Sum2 + impl, B: ParallelPipe, Input> + ParallelPipe for Sum2 { - type Item = A::Item; + type Output = A::Output; type Task = Sum2; fn task(&self) -> Self::Task { @@ -44,6 +44,7 @@ impl_par_dist! { impl> StreamTask for Sum2 { type Item = A::Item; type Async = Sum2; + fn into_async(self) -> Self::Async { match self { Sum2::A(a) => Sum2::A(a.into_async()), @@ -51,11 +52,12 @@ impl> StreamTask for Sum2 { } } } -impl, B: PipeTask, Source> PipeTask +impl, B: PipeTask, Input> PipeTask for Sum2 { - type Item = A::Item; + type Output = A::Output; type Async = Sum2; + fn into_async(self) -> Self::Async { match self { Sum2::A(a) => Sum2::A(a.into_async()), @@ -64,12 +66,12 @@ impl, B: PipeTask, Source> PipeTask< } } -impl, B: Pipe, Source> Pipe for Sum2 { - type Item = A::Item; +impl, B: Pipe, Input> Pipe for Sum2 { + type Output = A::Output; fn poll_next( - self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, - ) -> Poll> { + self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, + ) -> Poll> { match self.as_pin_mut() { Sum2::A(task) => task.poll_next(cx, stream), Sum2::B(task) => task.poll_next(cx, stream), diff --git a/amadeus-core/src/par_stream/update.rs b/amadeus-core/src/par_stream/update.rs index 75b9b681..da5b28e4 100644 --- a/amadeus-core/src/par_stream/update.rs +++ b/amadeus-core/src/par_stream/update.rs @@ -10,41 +10,47 @@ use std::{ use super::{ParallelPipe, ParallelStream, PipeTask, StreamTask}; use crate::pipe::Pipe; +#[pin_project] #[derive(new)] #[must_use] -pub struct Update { - i: I, +pub struct Update { + #[pin] + pipe: P, f: F, } impl_par_dist! { - impl ParallelStream for Update + impl ParallelStream for Update where - F: for<'a>FnMut<(&'a mut I::Item,), Output = ()> + Clone + Send + 'static, + F: for<'a> FnMut<(&'a mut P::Item,), Output = ()> + Clone + Send + 'static, { - type Item = I::Item; - type Task = UpdateTask; + type Item = P::Item; + type Task = UpdateTask; fn size_hint(&self) -> (usize, Option) { - self.i.size_hint() + self.pipe.size_hint() } - fn next_task(&mut self) -> Option { - self.i.next_task().map(|task| { - let f = self.f.clone(); - UpdateTask { task, f } + fn next_task(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + let self_ = self.project(); + let f = self_.f; + self_.pipe.next_task(cx).map(|task| { + task.map(|task| { + let f = f.clone(); + UpdateTask { task, f } + }) }) } } - impl, F, Source> ParallelPipe for Update + impl, F, Input> ParallelPipe for Update where - F: for<'a>FnMut<(&'a mut I::Item,), Output = ()> + Clone + Send + 'static, + F: for<'a> FnMut<(&'a mut P::Output,), Output = ()> + Clone + Send + 'static, { - type Item = I::Item; - type Task = UpdateTask; + type Output = P::Output; + type Task = UpdateTask; fn task(&self) -> Self::Task { - let task = self.i.task(); + let task = self.pipe.task(); let f = self.f.clone(); UpdateTask { task, f } } @@ -65,6 +71,7 @@ where { type Item = C::Item; type Async = UpdateTask; + fn into_async(self) -> Self::Async { UpdateTask { task: self.task.into_async(), @@ -72,12 +79,13 @@ where } } } -impl, F, Source> PipeTask for UpdateTask +impl, F, Input> PipeTask for UpdateTask where - F: for<'a> FnMut<(&'a mut >::Item,), Output = ()> + Clone, + F: for<'a> FnMut<(&'a mut >::Output,), Output = ()> + Clone, { - type Item = C::Item; + type Output = C::Output; type Async = UpdateTask; + fn into_async(self) -> Self::Async { UpdateTask { task: self.task.into_async(), @@ -104,15 +112,15 @@ where } } -impl, F, Source> Pipe for UpdateTask +impl, F, Input> Pipe for UpdateTask where - F: for<'a> FnMut<(&'a mut C::Item,), Output = ()> + Clone, + F: for<'a> FnMut<(&'a mut C::Output,), Output = ()> + Clone, { - type Item = C::Item; + type Output = C::Output; fn poll_next( - self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, - ) -> Poll> { + self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, + ) -> Poll> { let mut self_ = self.project(); let (task, f) = (self_.task, &mut self_.f); task.poll_next(cx, stream).map(|item| { diff --git a/amadeus-core/src/pipe.rs b/amadeus-core/src/pipe.rs index 8157ebb7..7e1ac22e 100644 --- a/amadeus-core/src/pipe.rs +++ b/amadeus-core/src/pipe.rs @@ -10,7 +10,7 @@ use std::{ pub use self::{filter::*, flat_map::*, map::*}; -// Sink takes Source as an input parameter rather than associated type to accept +// Sink takes Input as an input parameter rather than associated type to accept // for<'a> &'a T, but this might not be necessary in future? // https://github.com/rust-lang/rust/issues/49601 @@ -32,39 +32,20 @@ pub trait StreamExt: Stream { { assert_future(StreamSink { stream: self, sink }) } - - #[inline(always)] - fn flat_map(self, f: F) -> FlatMap - where - F: FnMut(Self::Item) -> R, - R: Stream, - Self: Sized, - { - assert_stream(FlatMap::new(self, f)) - } - - #[inline(always)] - fn filter(self, f: F) -> Filter - where - F: FnMut(&Self::Item) -> bool, - Self: Sized, - { - assert_stream(Filter::new(self, f)) - } } impl StreamExt for S where S: Stream {} -pub trait Pipe { - type Item; +pub trait Pipe { + type Output; fn poll_next( - self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, - ) -> Poll>; + self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, + ) -> Poll>; #[inline(always)] fn pipe

(self, pipe: P) -> PipePipe where - P: Pipe, + P: Pipe, Self: Sized, { assert_pipe(PipePipe { a: self, b: pipe }) @@ -73,16 +54,25 @@ pub trait Pipe { #[inline(always)] fn sink(self, sink: S) -> PipeSink where - S: Sink, + S: Sink, Self: Sized, { assert_sink(PipeSink { pipe: self, sink }) } + #[inline(always)] + fn filter(self, f: F) -> Filter + where + F: FnMut(&Self::Output) -> bool, + Self: Sized, + { + assert_pipe(Filter::new(self, f)) + } + #[inline(always)] fn flat_map(self, f: F) -> FlatMap where - F: FnMut(Self::Item) -> R, + F: FnMut(Self::Output) -> R, R: Stream, Self: Sized, { @@ -90,22 +80,24 @@ pub trait Pipe { } #[inline(always)] - fn filter(self, f: F) -> Filter + fn map(self, f: F) -> Map where - F: FnMut(&Self::Item) -> bool, + F: FnMut(Self::Output) -> R, Self: Sized, { - assert_pipe(Filter::new(self, f)) + assert_pipe(Map::new(self, f)) } } -pub trait Sink { +pub trait Sink { + type Done; + /// Returns `Poll::Ready` when a) it can't accept any more elements from `stream` and b) all /// accepted elements have been fully processed. By convention, `stream` yielding `None` /// typically triggers (a). - fn poll_pipe( - self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, - ) -> Poll<()>; + fn poll_forward( + self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, + ) -> Poll; } #[inline(always)] @@ -116,16 +108,16 @@ where s } #[inline(always)] -fn assert_pipe(p: P) -> P +fn assert_pipe(p: P) -> P where - P: Pipe, + P: Pipe, { p } #[inline(always)] -fn assert_sink(s: S) -> S +fn assert_sink(s: S) -> S where - S: Sink, + S: Sink, { s } @@ -150,7 +142,7 @@ where S: Stream, P: Pipe, { - type Item = P::Item; + type Item = P::Output; #[inline(always)] fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { @@ -167,17 +159,17 @@ pub struct PipePipe { b: B, } -impl Pipe for PipePipe +impl Pipe for PipePipe where - A: Pipe, - B: Pipe, + A: Pipe, + B: Pipe, { - type Item = B::Item; + type Output = B::Output; #[inline(always)] fn poll_next( - self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, - ) -> Poll> { + self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, + ) -> Poll> { let self_ = self.project(); let stream = stream.pipe(self_.a); pin_mut!(stream); @@ -193,19 +185,21 @@ pub struct PipeSink { sink: S, } -impl Sink for PipeSink +impl Sink for PipeSink where - P: Pipe, - S: Sink, + P: Pipe, + S: Sink, { + type Done = S::Done; + #[inline(always)] - fn poll_pipe( - self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, - ) -> Poll<()> { + fn poll_forward( + self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, + ) -> Poll { let self_ = self.project(); let stream = stream.pipe(self_.pipe); pin_mut!(stream); - self_.sink.poll_pipe(cx, stream) + self_.sink.poll_forward(cx, stream) } } @@ -222,65 +216,69 @@ where A: Stream, B: Sink, { - type Output = (); + type Output = B::Done; #[inline(always)] - fn poll(self: Pin<&mut Self>, cx: &mut Context) -> Poll<()> { + fn poll(self: Pin<&mut Self>, cx: &mut Context) -> Poll { let self_ = self.project(); - self_.sink.poll_pipe(cx, self_.stream) + self_.sink.poll_forward(cx, self_.stream) } } -impl Pipe for Pin

+impl Pipe for Pin

where P: DerefMut + Unpin, - P::Target: Pipe, + P::Target: Pipe, { - type Item = >::Item; + type Output = >::Output; #[inline(always)] fn poll_next( - self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, - ) -> Poll> { + self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, + ) -> Poll> { self.get_mut().as_mut().poll_next(cx, stream) } } -impl Pipe for &mut T +impl Pipe for &mut T where - T: Pipe + Unpin, + T: Pipe + Unpin, { - type Item = T::Item; + type Output = T::Output; #[inline(always)] fn poll_next( - mut self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, - ) -> Poll> { + mut self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, + ) -> Poll> { Pin::new(&mut **self).poll_next(cx, stream) } } -impl Sink for Pin

+impl Sink for Pin

where P: DerefMut + Unpin, - P::Target: Sink, + P::Target: Sink, { + type Done = >::Done; + #[inline(always)] - fn poll_pipe( - self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, - ) -> Poll<()> { - self.get_mut().as_mut().poll_pipe(cx, stream) + fn poll_forward( + self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, + ) -> Poll { + self.get_mut().as_mut().poll_forward(cx, stream) } } -impl Sink for &mut T +impl Sink for &mut T where - T: Sink + Unpin, + T: Sink + Unpin, { + type Done = T::Done; + #[inline(always)] - fn poll_pipe( - mut self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, - ) -> Poll<()> { - Pin::new(&mut **self).poll_pipe(cx, stream) + fn poll_forward( + mut self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, + ) -> Poll { + Pin::new(&mut **self).poll_forward(cx, stream) } } diff --git a/amadeus-core/src/pipe/filter.rs b/amadeus-core/src/pipe/filter.rs index f8a0e8e0..d456450b 100644 --- a/amadeus-core/src/pipe/filter.rs +++ b/amadeus-core/src/pipe/filter.rs @@ -10,23 +10,23 @@ use super::Pipe; #[pin_project] #[derive(new)] -pub struct Filter { +pub struct Filter { #[pin] - task: C, + pipe: P, f: F, } -impl Stream for Filter +impl Stream for Filter where - F: for<'a> FnMut<(&'a C::Item,), Output = bool>, + F: for<'a> FnMut<(&'a P::Item,), Output = bool>, { - type Item = C::Item; + type Item = P::Item; fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { let mut self_ = self.project(); - let (mut task, f) = (self_.task, &mut self_.f); + let (mut pipe, f) = (self_.pipe, &mut self_.f); Poll::Ready(loop { - match ready!(task.as_mut().poll_next(cx)) { + match ready!(pipe.as_mut().poll_next(cx)) { Some(t) if f.call_mut((&t,)) => break Some(t), Some(_) => (), None => break None, @@ -35,19 +35,19 @@ where } } -impl, F, Source> Pipe for Filter +impl, F, Input> Pipe for Filter where - F: for<'a> FnMut<(&'a C::Item,), Output = bool>, + F: for<'a> FnMut<(&'a P::Output,), Output = bool>, { - type Item = C::Item; + type Output = P::Output; fn poll_next( - self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>, - ) -> Poll> { + self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>, + ) -> Poll> { let mut self_ = self.project(); - let (mut task, f) = (self_.task, &mut self_.f); + let (mut pipe, f) = (self_.pipe, &mut self_.f); Poll::Ready(loop { - match ready!(task.as_mut().poll_next(cx, stream.as_mut())) { + match ready!(pipe.as_mut().poll_next(cx, stream.as_mut())) { Some(t) if f.call_mut((&t,)) => break Some(t), Some(_) => (), None => break None, diff --git a/amadeus-core/src/pipe/flat_map.rs b/amadeus-core/src/pipe/flat_map.rs index 3ee16bae..ede0d720 100644 --- a/amadeus-core/src/pipe/flat_map.rs +++ b/amadeus-core/src/pipe/flat_map.rs @@ -10,18 +10,18 @@ use super::Pipe; #[pin_project] #[derive(new)] -pub struct FlatMap { +pub struct FlatMap { #[pin] - pipe: C, + pipe: P, f: F, #[pin] #[new(default)] next: Option, } -impl Stream for FlatMap +impl Stream for FlatMap where - F: FnMut<(C::Item,), Output = R>, + F: FnMut<(P::Item,), Output = R>, R: Stream, { type Item = R::Item; @@ -44,16 +44,16 @@ where } } -impl, F, R, Source> Pipe for FlatMap +impl, F, R, Input> Pipe for FlatMap where - F: FnMut<(C::Item,), Output = R>, + F: FnMut<(P::Output,), Output = R>, R: Stream, { - type Item = R::Item; + type Output = R::Item; fn poll_next( - self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>, - ) -> Poll> { + self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>, + ) -> Poll> { let mut self_ = self.project(); Poll::Ready(loop { if let Some(s) = self_.next.as_mut().as_pin_mut() { diff --git a/amadeus-core/src/pipe/map.rs b/amadeus-core/src/pipe/map.rs index da7f33ad..827afb1f 100644 --- a/amadeus-core/src/pipe/map.rs +++ b/amadeus-core/src/pipe/map.rs @@ -3,44 +3,43 @@ use futures::{ready, Stream}; use pin_project::pin_project; use serde_closure::traits::FnMut; use std::{ - marker::PhantomData, pin::Pin, task::{Context, Poll} + pin::Pin, task::{Context, Poll} }; use super::Pipe; #[pin_project] #[derive(new)] -pub struct Map { +pub struct Map { #[pin] - task: C, + pipe: P, f: F, - marker: PhantomData R>, } -impl Stream for Map +impl Stream for Map where - F: FnMut<(C::Item,), Output = R>, + F: FnMut<(P::Item,)>, { - type Item = R; + type Item = F::Output; fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { let mut self_ = self.project(); - let (mut task, f) = (self_.task, &mut self_.f); - Poll::Ready(ready!(task.as_mut().poll_next(cx)).map(|t| f.call_mut((t,)))) + let (mut pipe, f) = (self_.pipe, &mut self_.f); + Poll::Ready(ready!(pipe.as_mut().poll_next(cx)).map(|t| f.call_mut((t,)))) } } -impl, F, R, Source> Pipe for Map +impl, F, Input> Pipe for Map where - F: FnMut<(C::Item,), Output = R>, + F: FnMut<(P::Output,)>, { - type Item = R; + type Output = F::Output; fn poll_next( - self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>, - ) -> Poll> { + self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>, + ) -> Poll> { let mut self_ = self.project(); - let (mut task, f) = (self_.task, &mut self_.f); - Poll::Ready(ready!(task.as_mut().poll_next(cx, stream.as_mut())).map(|t| f.call_mut((t,)))) + let (mut pipe, f) = (self_.pipe, &mut self_.f); + Poll::Ready(ready!(pipe.as_mut().poll_next(cx, stream.as_mut())).map(|t| f.call_mut((t,)))) } } diff --git a/amadeus-core/src/util.rs b/amadeus-core/src/util.rs index 366cb93d..e34bc714 100644 --- a/amadeus-core/src/util.rs +++ b/amadeus-core/src/util.rs @@ -15,6 +15,7 @@ where { type Item = Result; type IntoIter = ResultExpandIter; + fn into_iter(self) -> Self::IntoIter { ResultExpandIter::new(self.0.map(IntoIterator::into_iter)) } @@ -37,6 +38,7 @@ where T: Iterator, { type Item = Result; + fn next(&mut self) -> Option { match self { Self::Ok(t) => t.next().map(Ok), @@ -49,6 +51,7 @@ where T: Stream, { type Item = Result; + fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { let ret = match self.project() { ResultExpandIterProj::Ok(t) => ready!(t.poll_next(cx)).map(Ok), @@ -88,9 +91,10 @@ impl From for io::Error { } } +#[pin_project] #[derive(new)] #[repr(transparent)] -pub struct DistParStream(S); +pub struct DistParStream(#[pin] S); impl ParallelStream for DistParStream where S: DistributedStream, @@ -101,8 +105,8 @@ where fn size_hint(&self) -> (usize, Option) { self.0.size_hint() } - fn next_task(&mut self) -> Option { - self.0.next_task() + fn next_task(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + self.project().0.next_task(cx) } } @@ -123,7 +127,7 @@ impl DistributedStream for ImplDistributedStream { fn size_hint(&self) -> (usize, Option) { unreachable!() } - fn next_task(&mut self) -> Option { + fn next_task(self: Pin<&mut Self>, _cx: &mut Context) -> Poll> { unreachable!() } } @@ -134,7 +138,7 @@ impl ParallelStream for ImplDistributedStream { fn size_hint(&self) -> (usize, Option) { unreachable!() } - fn next_task(&mut self) -> Option { + fn next_task(self: Pin<&mut Self>, _cx: &mut Context) -> Poll> { unreachable!() } } diff --git a/src/source.rs b/src/source.rs index c42e50ec..0fc597c3 100644 --- a/src/source.rs +++ b/src/source.rs @@ -49,8 +49,8 @@ pub trait Destination: Clone + Debug { type Item: crate::data::Data; type Error: Error; - type ParSink: ParallelSink>; - type DistSink: DistributedSink>; + type ParSink: ParallelSink>; + type DistSink: DistributedSink>; fn par_sink(self) -> Self::ParSink; fn dist_sink(self) -> Self::DistSink; @@ -162,8 +162,9 @@ impl Source for CommonCrawl { } } +#[pin_project] #[derive(new)] -pub struct IntoStream(I, PhantomData U>); +pub struct IntoStream(#[pin] I, PhantomData U>); impl ParallelStream for IntoStream where I: ParallelStream>, @@ -176,10 +177,12 @@ where fn size_hint(&self) -> (usize, Option) { self.0.size_hint() } - fn next_task(&mut self) -> Option { - self.0.next_task().map(|task| IntoTask { - task, - marker: PhantomData, + fn next_task(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + self.project().0.next_task(cx).map(|task| { + task.map(|task| IntoTask { + task, + marker: PhantomData, + }) }) } } @@ -195,10 +198,12 @@ where fn size_hint(&self) -> (usize, Option) { self.0.size_hint() } - fn next_task(&mut self) -> Option { - self.0.next_task().map(|task| IntoTask { - task, - marker: PhantomData, + fn next_task(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + self.project().0.next_task(cx).map(|task| { + task.map(|task| IntoTask { + task, + marker: PhantomData, + }) }) } }