From a4eb4bbe532e3ac8d97eacf423059b7c5c8d2d2a Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 15 Jan 2019 19:08:45 -0800 Subject: [PATCH 1/7] Move intype to the type parameter of AbstractReduction --- src/core.jl | 16 ++++++++-------- src/library.jl | 30 +++++++++++++++--------------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/core.jl b/src/core.jl index ded6697888..0d0b5afb88 100644 --- a/src/core.jl +++ b/src/core.jl @@ -77,7 +77,13 @@ appropriately defined for child types. """ AbstractFilter -abstract type AbstractReduction end +abstract type AbstractReduction{intype} end + +InType(::T) where T = InType(T) +InType(::Type{<:AbstractReduction{intype}}) where {intype} = intype +InType(T::Type) = throw(MethodError(InType, (T,))) + +Setfield.constructor_of(::Type{T}) where {T <: AbstractReduction} = T # In clojure a reduction function is one of signature # whatever, input -> whatever @@ -88,17 +94,11 @@ abstract type AbstractReduction end # a transducer `xform` and an inner reduction function `inner`. # `inner` can be either a `Reduction` or a function with arity-2 and arity-1 methods # -struct Reduction{X <: Transducer, I, InType} <: AbstractReduction +struct Reduction{X <: Transducer, I, intype} <: AbstractReduction{intype} xform::X inner::I end -Setfield.constructor_of(::Type{T}) where {T <: Reduction} = T - -InType(::T) where T = InType(T) -InType(::Type{Reduction{X, I, intype}}) where {X, I, intype} = intype -InType(T::Type) = throw(MethodError(InType, (T,))) - Transducer(rf::Reduction{<:Transducer, <:AbstractReduction}) = Composition(rf.xform, Transducer(rf.inner)) Transducer(rf::Reduction) = rf.xform diff --git a/src/library.jl b/src/library.jl index bd4415a4f6..bdea9843ce 100644 --- a/src/library.jl +++ b/src/library.jl @@ -1356,25 +1356,28 @@ end # where `value` is the placeholder for the output value of the # transducer (here `Map(identity)`) outer to `TeeZip`. -struct Splitter{R, L} <: AbstractReduction +struct Splitter{intype, R, L} <: AbstractReduction{intype} inner::R lens::L end -struct Joiner{F, T, intype} <: AbstractReduction +Splitter(inner::R, lens::L) where {R, L} = + Splitter{InType(R), R, L}(inner, lens) + +struct Joiner{intype, F, T} <: AbstractReduction{intype} inner::F # original inner reduction value::T # original input - @inline function Joiner{F,T,intype}(inner) where {F,T,intype} + @inline function Joiner{intype,F,T}(inner) where {intype,F,T} _joiner_error(inner, intype) if isbitstype(T) return new(inner) else - return new{F,Union{T,Nothing},intype}(inner, nothing) + return new{intype,F,Union{T,Nothing}}(inner, nothing) end end - @inline function Joiner{F,T,intype}(inner, value) where {F,T,intype} + @inline function Joiner{intype,F,T}(inner, value) where {intype,F,T} _joiner_error(inner, intype) return new(inner, value) end @@ -1392,16 +1395,13 @@ where inner = $inner """) -# TODO: move intype to the type parameter of AbstractReduction -InType(::Type{<:Splitter{R}}) where R = InType(R) -InType(::Type{<:Joiner{F, T, intype}}) where {F, T, intype} = - intype - finaltype(rf::Splitter) = finaltype(rf.inner) -finaltype(rf::Joiner{<:AbstractReduction}) = finaltype(rf.inner) -finaltype(rf::Joiner) = InType(rf) - -Setfield.constructor_of(::Type{T}) where {T <: Joiner} = T +finaltype(rf::Joiner) = + if rf.inner isa AbstractReduction + finaltype(rf.inner) + else + InType(rf) + end # It's ugly that `Reduction` returns a non-`Reduction` type! TODO: fix it function Reduction(xf::Composition{<:TeeZip}, f, intype::Type) @@ -1432,7 +1432,7 @@ function _teezip_rf(xf, intype, downstream) else rf_ds = Reduction(xf_ds, f, intype_ds) end - joiner = Joiner{typeof(rf_ds), intype_orig, intype_ds}(rf_ds) + joiner = Joiner{intype_ds, typeof(rf_ds), intype_orig}(rf_ds) return Reduction(xf, joiner, intype) end From 66433fb3a0b15ad863fc1b5ba405d000117acbd7 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 15 Jan 2019 19:20:31 -0800 Subject: [PATCH 2/7] Change order of type parameters in Reduction --- src/core.jl | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/core.jl b/src/core.jl index 0d0b5afb88..688dd34e1c 100644 --- a/src/core.jl +++ b/src/core.jl @@ -94,12 +94,12 @@ Setfield.constructor_of(::Type{T}) where {T <: AbstractReduction} = T # a transducer `xform` and an inner reduction function `inner`. # `inner` can be either a `Reduction` or a function with arity-2 and arity-1 methods # -struct Reduction{X <: Transducer, I, intype} <: AbstractReduction{intype} +struct Reduction{intype, X <: Transducer, I} <: AbstractReduction{intype} xform::X inner::I end -Transducer(rf::Reduction{<:Transducer, <:AbstractReduction}) = +Transducer(rf::Reduction{<:Any, <:Transducer, <:AbstractReduction}) = Composition(rf.xform, Transducer(rf.inner)) Transducer(rf::Reduction) = rf.xform @@ -110,15 +110,11 @@ When defining a transducer type `X`, it is often required to dispatch on type `rf::R_{X}` (Reducing Function) which bundles the current transducer `rf.xform::X` and the inner reducing function `rf.inner::R_`. - -```julia -const R_{X} = Reduction{<:X} -``` """ -const R_{X} = Reduction{<:X} +const R_{X} = Reduction{<:Any, <:X} @inline Reduction(xf::X, inner::I, ::Type{InType}) where {X, I, InType} = - Reduction{X, I, InType}(xf, inner) + Reduction{InType, X, I}(xf, inner) @inline function Reduction(xf_::Composition, f, intype::Type) xf = _normalize(xf_) @@ -344,8 +340,12 @@ Output item type for the transducer `xf` when the input type is `intype`. outtype(::Any, ::Any) = Any outtype(::AbstractFilter, intype) = intype -finaltype(rf::Reduction{<:Transducer, <:AbstractReduction}) = finaltype(rf.inner) -finaltype(rf::Reduction) = outtype(rf.xform, InType(rf)) +finaltype(rf::Reduction) = + if rf.inner isa AbstractReduction + finaltype(rf.inner) + else + outtype(rf.xform, InType(rf)) + end # isexpansive(::Any) = true isexpansive(::Transducer) = true From 7d5778a4956ec299ce4db8acc42ea1da91d14bae Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 15 Jan 2019 19:00:27 -0800 Subject: [PATCH 3/7] Replace rf.inner with inner(rf) --- examples/transducers.jl | 12 +-- src/core.jl | 46 ++++++----- src/library.jl | 178 ++++++++++++++++++++-------------------- src/processes.jl | 12 +-- src/show.jl | 6 +- src/simd.jl | 4 +- 6 files changed, 130 insertions(+), 128 deletions(-) diff --git a/examples/transducers.jl b/examples/transducers.jl index 6d653b5a5f..7073aba5d5 100644 --- a/examples/transducers.jl +++ b/examples/transducers.jl @@ -5,7 +5,7 @@ # ones: using Transducers -using Transducers: Transducer, R_, next +using Transducers: Transducer, R_, next, inner # ## Stateless transducer @@ -17,7 +17,7 @@ struct AddOneIfInt <: Transducer end function Transducers.next(rf::R_{AddOneIfInt}, result, input) if input isa Int # Output `input + 1` is passed to the "inner" reducing step: - next(rf.inner, result, input + 1) + next(inner(rf), result, input + 1) else # Filtering out is done by "doing nothing"; return `result`-so-far # as-is: @@ -70,7 +70,7 @@ function Transducers.start(rf::R_{RandomRecall}, result) buffer = InType(rf)[] rng = MersenneTwister(rf.xform.seed) private_state = (buffer, rng) - return wrap(rf, private_state, start(rf.inner, result)) + return wrap(rf, private_state, start(inner(rf), result)) end # Stateful transducer needs to unwrap its private state inside @@ -92,7 +92,7 @@ function Transducers.next(rf::R_{RandomRecall}, result, input) end # Call the inner reducing function. Note that `iresult` unwrapped by # [`Transducers.wrapping`](@ref) must be passed to `next`: - iresult = next(rf.inner, iresult, iinput) + iresult = next(inner(rf), iresult, iinput) return (buffer, rng), iresult end end @@ -118,10 +118,10 @@ function Transducers.complete(rf::R_{RandomRecall}, result) for x in buffer # Note that inner `next` can be called more than one time inside # `next` and `complete`: - iresult = next(rf.inner, iresult, x) + iresult = next(inner(rf), iresult, x) end # `complete` for inner reducing function must be called exactly once: - return complete(rf.inner, iresult) + return complete(inner(rf), iresult) end # This then adds 3 (`= RandomRecall().history`) more elements to the diff --git a/src/core.jl b/src/core.jl index 688dd34e1c..8f13d86f15 100644 --- a/src/core.jl +++ b/src/core.jl @@ -85,6 +85,8 @@ InType(T::Type) = throw(MethodError(InType, (T,))) Setfield.constructor_of(::Type{T}) where {T <: AbstractReduction} = T +inner(rf::AbstractReduction) = rf.inner + # In clojure a reduction function is one of signature # whatever, input -> whatever # @@ -100,7 +102,7 @@ struct Reduction{intype, X <: Transducer, I} <: AbstractReduction{intype} end Transducer(rf::Reduction{<:Any, <:Transducer, <:AbstractReduction}) = - Composition(rf.xform, Transducer(rf.inner)) + Composition(rf.xform, Transducer(inner(rf))) Transducer(rf::Reduction) = rf.xform """ @@ -109,7 +111,7 @@ Transducer(rf::Reduction) = rf.xform When defining a transducer type `X`, it is often required to dispatch on type `rf::R_{X}` (Reducing Function) which bundles the current transducer `rf.xform::X` and the inner reducing function -`rf.inner::R_`. +`inner(rf)::R_`. """ const R_{X} = Reduction{<:Any, <:X} @@ -146,14 +148,14 @@ This is an optional interface for a transducer. Default implementation just calls `start` of the inner reducing function; i.e., ```julia -start(rf::Reduction, result) = start(rf.inner, result) +start(rf::Reduction, result) = start(inner(rf), result) ``` If the transducer `X` is stateful, it can "bundle" its private state with `state` (so that `next` function can be "pure"). ```julia -start(rf::R_{X}, result) = wrap(rf, PRIVATE_STATE, start(rf.inner, result)) +start(rf::R_{X}, result) = wrap(rf, PRIVATE_STATE, start(inner(rf), result)) ``` See [`Take`](@ref), [`PartitionBy`](@ref), etc. for real-world examples. @@ -164,8 +166,8 @@ functions. The idea is based on a slightly different approach taken in C++ Transducer library [atria](https://github.com/AbletonAG/atria). """ start(::Any, result) = result -start(rf::Reduction, result) = start(rf.inner, result) -start(rf::R_{AbstractFilter}, result) = start(rf.inner, result) +start(rf::Reduction, result) = start(inner(rf), result) +start(rf::R_{AbstractFilter}, result) = start(inner(rf), result) """ Transducers.next(rf::R_{X}, state, input) @@ -175,7 +177,7 @@ This is the only required interface. It takes the following form ```julia next(rf::R_{X}, result, input) = - # code calling next(rf.inner, result, possibly_modified_input) + # code calling next(inner(rf), result, possibly_modified_input) ``` See [`Map`](@ref), [`Filter`](@ref), [`Cat`](@ref), etc. for @@ -202,9 +204,9 @@ complete(f, result) = f(result) complete(rf::Reduction, result) = # Not using dispatch to avoid ambiguity if result isa PrivateState{typeof(rf)} - complete(rf.inner, unwrap(rf, result)[2]) + complete(inner(rf), unwrap(rf, result)[2]) else - complete(rf.inner, result) + complete(inner(rf), result) end combine(f, a, b) = f(a, b) @@ -220,7 +222,7 @@ combine(rf::Reduction, a, b) = * `rf.xform = $(rf.xform)` is stateful and does not support `combine`. """) else - combine(rf.inner, a, b) + combine(inner(rf), a, b) end struct PrivateState{T, S, R} @@ -262,7 +264,7 @@ unwrap(::T1, ::PrivateState{T2}) where {T1, T2} = Pack private `state` for reducing function `rf` (or rather the transducer `X`) with the result `iresult` returned from the inner -reducing function `rf.inner`. This packed result is typically passed +reducing function `inner(rf)`. This packed result is typically passed to the outer reducing function. This is intended to be used only in [`start`](@ref). Inside @@ -278,9 +280,9 @@ state `stateₙ`. Then, calling `start(rf, result))` is equivalent to ```julia wrap(rf, state₁, # private state for xf₁ - wrap(rf.inner, + wrap(inner(rf), state₂, # private state for xf₂ - wrap(rf.inner.inner, + wrap(inner(rf).inner, state₃, # private state for xf₃ result))) ``` @@ -289,9 +291,9 @@ or equivalently ```julia result₃ = result -result₂ = wrap(rf.inner.inner, state₃, result₃) -result₁ = wrap(rf.inner, state₂, result₂) -result₀ = wrap(rf, state₁, result₁) +result₂ = wrap(inner(inner(rf)), state₃, result₃) +result₁ = wrap(inner(rf), state₂, result₂) +result₀ = wrap(rf, state₁, result₁) ``` The inner most step function receives the original `result` as the @@ -315,7 +317,7 @@ a tuple `(state, iresult)`. This is intended to be used only in ```julia next(rf::R_{MyTransducer}, result, input) = wrapping(rf, result) do my_state, iresult - # code calling `next(rf.inner, iresult, possibly_modified_input)` + # code calling `next(inner(rf), iresult, possibly_modified_input)` return my_state, iresult # possibly modified end ``` @@ -341,8 +343,8 @@ outtype(::Any, ::Any) = Any outtype(::AbstractFilter, intype) = intype finaltype(rf::Reduction) = - if rf.inner isa AbstractReduction - finaltype(rf.inner) + if inner(rf) isa AbstractReduction + finaltype(inner(rf)) else outtype(rf.xform, InType(rf)) end @@ -350,19 +352,19 @@ finaltype(rf::Reduction) = # isexpansive(::Any) = true isexpansive(::Transducer) = true isexpansive(::AbstractFilter) = false -# isexpansive(rf::Reduction) = isexpansive(rf.xform) || isexpansive(rf.inner) +# isexpansive(rf::Reduction) = isexpansive(rf.xform) || isexpansive(inner(rf)) isexpansive(xf::Composition) = isexpansive(xf.outer) || isexpansive(xf.inner) # Should it be a type-level trait? #= iscontractive(::Any) = false iscontractive(::AbstractFilter) = true -iscontractive(rf::Reduction) = iscontractive(rf.xform) && iscontractive(rf.inner) +iscontractive(rf::Reduction) = iscontractive(rf.xform) && iscontractive(inner(rf)) =# struct NoComplete <: Transducer end outtype(::NoComplete, intype) = intype -next(rf::R_{NoComplete}, result, input) = next(rf.inner, result, input) +next(rf::R_{NoComplete}, result, input) = next(inner(rf), result, input) complete(::R_{NoComplete}, result) = result # don't call inner complete """ diff --git a/src/library.jl b/src/library.jl index bdea9843ce..082877d4c7 100644 --- a/src/library.jl +++ b/src/library.jl @@ -49,7 +49,7 @@ end isexpansive(::Map) = false outtype(xf::Map, intype) = Union{Base.return_types(xf.f, (intype,))...} -next(rf::R_{Map}, result, input) = next(rf.inner, result, rf.xform.f(input)) +next(rf::R_{Map}, result, input) = next(inner(rf), result, rf.xform.f(input)) """ MapSplat(f) @@ -75,7 +75,7 @@ end isexpansive(::MapSplat) = false outtype(xf::MapSplat, intype) = Union{Base.return_types(xf.f, intype)...} next(rf::R_{MapSplat}, result, input) = - next(rf.inner, result, rf.xform.f(input...)) + next(inner(rf), result, rf.xform.f(input...)) # https://clojure.github.io/clojure/clojure.core-api.html#clojure.core/replace # https://clojuredocs.org/clojure.core/replace @@ -122,7 +122,7 @@ end isexpansive(::Replace) = false outtype(xf::Replace, intype) = Union{intype, avaltype(xf.d)} next(rf::R_{Replace}, result, input) = - next(rf.inner, result, get(rf.xform.d, input, input)) + next(inner(rf), result, get(rf.xform.d, input, input)) # https://clojure.github.io/clojure/clojure.core-api.html#clojure.core/cat # https://clojuredocs.org/clojure.core/cat @@ -148,17 +148,17 @@ outtype(::Cat, intype) = ieltype(intype) # Inner transducer has to be started once the input is known. That's # why `start` for `Cat` has to bail out immediately; i.e., it's not a -# bug that `start(rf.inner, iresult)` is not called here: +# bug that `start(inner(rf), iresult)` is not called here: start(rf::R_{Cat}, result) = wrap(rf, Unseen(), result) next(rf::R_{Cat}, result, input) = wrapping(rf, result) do istate0, iresult if istate0 isa Unseen - istate1 = start(rf.inner, iresult) + istate1 = start(inner(rf), iresult) else istate1 = istate0 end - istate2 = foldl_nocomplete(rf.inner, istate1, input) + istate2 = foldl_nocomplete(inner(rf), istate1, input) istate2, istate2 end @@ -170,7 +170,7 @@ function combine(rf::R_{Cat}, a, b) elseif ub isa Unseen return wrap(rf, ua, ira) else - uc = combine(rf.inner, ua, ub) + uc = combine(inner(rf), ua, ub) return wrap(rf, uc, uc) end end @@ -225,7 +225,7 @@ struct Filter{P} <: AbstractFilter end next(rf::R_{Filter}, result, input) = - rf.xform.pred(input) ? next(rf.inner, result, input) : result + rf.xform.pred(input) ? next(inner(rf), result, input) : result """ NotA(T) @@ -256,7 +256,7 @@ NotA(T::Type) = NotA{T}() outtype(::NotA{T}, intype) where T = Core.Compiler.typesubtract(intype, T) next(rf::R_{NotA{T}}, result, input) where T = - input isa T ? result : next(rf.inner, result, input) + input isa T ? result : next(inner(rf), result, input) # **Side notes**. Although in principle `NotA(Missing)` can yields a # better performance than `Filter(!ismissing)` (by providing more type @@ -294,7 +294,7 @@ OfType(T::Type) = OfType{T}() outtype(::OfType{T}, _intype) where T = T next(rf::R_{OfType{T}}, result, input) where T = - input isa T ? next(rf.inner, result, input) : result + input isa T ? next(inner(rf), result, input) : result # https://clojure.github.io/clojure/clojure.core-api.html#clojure.core/take # https://clojuredocs.org/clojure.core/take @@ -329,12 +329,12 @@ struct Take <: AbstractFilter end end -start(rf::R_{Take}, result) = wrap(rf, rf.xform.n, start(rf.inner, result)) +start(rf::R_{Take}, result) = wrap(rf, rf.xform.n, start(inner(rf), result)) next(rf::R_{Take}, result, input) = wrapping(rf, result) do n, iresult if n > 0 - iresult = next(rf.inner, iresult, input) + iresult = next(inner(rf), iresult, input) n -= 1 end if n <= 0 @@ -374,7 +374,7 @@ end function start(rf::R_{TakeLast}, result) n = rf.xform.n - return wrap(rf, (-n, Vector{InType(rf)}(undef, n)), start(rf.inner, result)) + return wrap(rf, (-n, Vector{InType(rf)}(undef, n)), start(inner(rf), result)) end next(rf::R_{TakeLast}, result, input) = @@ -394,20 +394,20 @@ function complete(rf::R_{TakeLast}, result) (c, buffer), iresult = unwrap(rf, result) if c <= 0 # buffer is not full (or c is just wrapping) for i in 1:(c + length(buffer)) - iresult = next(rf.inner, iresult, @inbounds buffer[i]) - @return_if_reduced complete(rf.inner, iresult) + iresult = next(inner(rf), iresult, @inbounds buffer[i]) + @return_if_reduced complete(inner(rf), iresult) end else for i in c+1:length(buffer) - iresult = next(rf.inner, iresult, @inbounds buffer[i]) - @return_if_reduced complete(rf.inner, iresult) + iresult = next(inner(rf), iresult, @inbounds buffer[i]) + @return_if_reduced complete(inner(rf), iresult) end for i in 1:c - iresult = next(rf.inner, iresult, @inbounds buffer[i]) - @return_if_reduced complete(rf.inner, iresult) + iresult = next(inner(rf), iresult, @inbounds buffer[i]) + @return_if_reduced complete(inner(rf), iresult) end end - return complete(rf.inner, iresult) + return complete(inner(rf), iresult) end # https://clojure.github.io/clojure/clojure.core-api.html#clojure.core/take-while @@ -435,7 +435,7 @@ struct TakeWhile{P} <: AbstractFilter end next(rf::R_{TakeWhile}, result, input) = - rf.xform.pred(input) ? next(rf.inner, result, input) : ensure_reduced(result) + rf.xform.pred(input) ? next(inner(rf), result, input) : ensure_reduced(result) # https://clojure.github.io/clojure/clojure.core-api.html#clojure.core/take-nth # https://clojuredocs.org/clojure.core/take-nth @@ -466,12 +466,12 @@ struct TakeNth <: AbstractFilter end end -start(rf::R_{TakeNth}, result) = wrap(rf, rf.xform.n, start(rf.inner, result)) +start(rf::R_{TakeNth}, result) = wrap(rf, rf.xform.n, start(inner(rf), result)) next(rf::R_{TakeNth}, result, input) = wrapping(rf, result) do c, iresult if c == rf.xform.n - iresult = next(rf.inner, iresult, input) + iresult = next(inner(rf), iresult, input) c = 1 else c += 1 @@ -507,12 +507,12 @@ struct Drop <: AbstractFilter end end -start(rf::R_{Drop}, result) = wrap(rf, 0, start(rf.inner, result)) +start(rf::R_{Drop}, result) = wrap(rf, 0, start(inner(rf), result)) next(rf::R_{Drop}, result, input) = wrapping(rf, result) do c, iresult if c >= rf.xform.n - c, next(rf.inner, iresult, input) + c, next(inner(rf), iresult, input) else c += 1 c, iresult @@ -556,7 +556,7 @@ end function start(rf::R_{DropLast}, result) n = rf.xform.n + 1 - return wrap(rf, (-n, Vector{InType(rf)}(undef, n)), start(rf.inner, result)) + return wrap(rf, (-n, Vector{InType(rf)}(undef, n)), start(inner(rf), result)) end next(rf::R_{DropLast}, result, input) = @@ -566,16 +566,16 @@ next(rf::R_{DropLast}, result, input) = if c <= 0 buffer[c + n] = input if c == 0 - (c, buffer), next(rf.inner, iresult, buffer[1]) + (c, buffer), next(inner(rf), iresult, buffer[1]) else (c, buffer), iresult end elseif c >= n buffer[c] = input - (0, buffer), next(rf.inner, iresult, buffer[1]) + (0, buffer), next(inner(rf), iresult, buffer[1]) else buffer[c] = input - (c, buffer), next(rf.inner, iresult, buffer[c + 1]) + (c, buffer), next(inner(rf), iresult, buffer[c + 1]) end end @@ -606,7 +606,7 @@ struct DropWhile{F} <: AbstractFilter pred::F end -start(rf::R_{DropWhile}, result) = wrap(rf, true, start(rf.inner, result)) +start(rf::R_{DropWhile}, result) = wrap(rf, true, start(inner(rf), result)) next(rf::R_{DropWhile}, result, input) = wrapping(rf, result) do dropping, iresult @@ -614,7 +614,7 @@ next(rf::R_{DropWhile}, result, input) = dropping = rf.xform.pred(input) dropping && return (dropping, iresult) end - dropping, next(rf.inner, iresult, input) + dropping, next(inner(rf), iresult, input) end """ @@ -643,11 +643,11 @@ struct FlagFirst <: Transducer end isexpansive(::FlagFirst) = false outtype(::FlagFirst, intype) = Tuple{Bool,intype} -start(rf::R_{FlagFirst}, result) = wrap(rf, true, start(rf.inner, result)) +start(rf::R_{FlagFirst}, result) = wrap(rf, true, start(inner(rf), result)) next(rf::R_{FlagFirst}, result, input) = wrapping(rf, result) do isfirst, iresult - false, next(rf.inner, iresult, (isfirst, input)) + false, next(inner(rf), iresult, (isfirst, input)) end # https://docs.julialang.org/en/v1/base/iterators/#Base.Iterators.partition @@ -709,7 +709,7 @@ outtype(::Partition, intype) = DenseSubVector{intype} function start(rf::R_{Partition}, result) buf = Vector{InType(rf)}() sizehint!(buf, rf.xform.size) - return wrap(rf, (0, 0, buf), start(rf.inner, result)) + return wrap(rf, (0, 0, buf), start(inner(rf), result)) end function next(rf::R_{Partition}, result, input) @@ -733,7 +733,7 @@ function _window_next(rf, i, s, buf, iresult, input) @assert i == rf.xform.size i = 0 s = rf.xform.step - iresult = next(rf.inner, iresult, iinput) + iresult = next(inner(rf), iresult, iinput) # Throw away buf[1] which is just consumed by the downstream: deleteat!(buf, 1) end @@ -759,7 +759,7 @@ function _window_next(rf, i, s, buf, iresult, input) s = rf.xform.step iinput = @view buf[i:i + rf.xform.size - 1] # unsafe_view? @inbounds? iinput :: DenseSubVector - iresult = next(rf.inner, iresult, iinput) + iresult = next(inner(rf), iresult, iinput) end if i == rf.xform.size i = 0 @@ -773,10 +773,10 @@ function complete(rf::R_{Partition}, result) iinput = @view buf[i:i + rf.xform.size - 1] # unsafe_view? @inbounds? iinput = @view iinput[s + 1:end] iinput :: DenseSubVector - iresult = next(rf.inner, iresult, iinput) - @return_if_reduced complete(rf.inner, iresult) + iresult = next(inner(rf), iresult, iinput) + @return_if_reduced complete(inner(rf), iresult) end - return complete(rf.inner, iresult) + return complete(inner(rf), iresult) end # TODO: implement SVector version of Partition @@ -816,7 +816,7 @@ outtype(::PartitionBy, intype) = Vector{intype} function start(rf::R_{PartitionBy}, result) iinput = InType(rf)[] - return wrap(rf, (iinput, Unseen()), start(rf.inner, result)) + return wrap(rf, (iinput, Unseen()), start(inner(rf), result)) end @inline function next(rf::R_{PartitionBy}, result, input) @@ -825,7 +825,7 @@ end if pval isa Unseen || val == pval push!(iinput, input) else - iresult = next(rf.inner, iresult, iinput) + iresult = next(inner(rf), iresult, iinput) empty!(iinput) isreduced(iresult) || push!(iinput, input) end @@ -836,10 +836,10 @@ end function complete(rf::R_{PartitionBy}, ps) (iinput, _), iresult = unwrap(rf, ps) if !isempty(iinput) - iresult = next(rf.inner, iresult, iinput) - @return_if_reduced complete(rf.inner, iresult) + iresult = next(inner(rf), iresult, iinput) + @return_if_reduced complete(inner(rf), iresult) end - return complete(rf.inner, iresult) + return complete(inner(rf), iresult) end # https://clojure.github.io/clojure/clojure.core-api.html#clojure.core/keep @@ -878,7 +878,7 @@ outtype(xf::Keep, intype) = function next(rf::R_{Keep}, result, input) iinput = rf.xform.f(input) - return iinput === nothing ? result : next(rf.inner, result, iinput) + return iinput === nothing ? result : next(inner(rf), result, iinput) end # https://clojure.github.io/clojure/clojure.core-api.html#clojure.core/distinct @@ -906,7 +906,7 @@ end function start(rf::R_{Distinct}, result) seen = Set(InType(rf)[]) - return wrap(rf, seen, start(rf.inner, result)) + return wrap(rf, seen, start(inner(rf), result)) end function next(rf::R_{Distinct}, result, input) @@ -915,7 +915,7 @@ function next(rf::R_{Distinct}, result, input) return seen, iresult else push!(seen, input) - return seen, next(rf.inner, iresult, input) + return seen, next(inner(rf), iresult, input) end end end @@ -948,14 +948,14 @@ end outtype(xf::Interpose, intype) = Union{typeof(xf.sep), intype} -start(rf::R_{Interpose}, result) = wrap(rf, Val(true), start(rf.inner, result)) +start(rf::R_{Interpose}, result) = wrap(rf, Val(true), start(inner(rf), result)) next(rf::R_{Interpose}, result, input) = wrapping(rf, result) do isfirst, iresult if isfirst isa Val{false} - iresult = next(rf.inner, iresult, rf.xform.sep) + iresult = next(inner(rf), iresult, rf.xform.sep) end - return Val(false), next(rf.inner, iresult, input) + return Val(false), next(inner(rf), iresult, input) end # https://clojure.github.io/clojure/clojure.core-api.html#clojure.core/dedupe @@ -983,12 +983,12 @@ julia> collect(Dedupe(), [1, 1, 2, 1, 3, 3, 2]) struct Dedupe <: AbstractFilter end -start(rf::R_{Dedupe}, result) = wrap(rf, Unseen(), start(rf.inner, result)) +start(rf::R_{Dedupe}, result) = wrap(rf, Unseen(), start(inner(rf), result)) next(rf::R_{Dedupe}, result, input) = wrapping(rf, result) do prev, iresult if prev isa Unseen || prev != input - return input, next(rf.inner, iresult, input) + return input, next(inner(rf), iresult, input) else return input, iresult end @@ -1079,13 +1079,13 @@ function start(rf::R_{Scan}, result) else init = _initvalue(rf) end - return wrap(rf, init, start(rf.inner, result)) + return wrap(rf, init, start(inner(rf), result)) end function next(rf::R_{Scan}, result, input) wrapping(rf, result) do acc, iresult acc = rf.xform.f(acc, input) - return acc, next(rf.inner, iresult, acc) + return acc, next(inner(rf), iresult, acc) end end @@ -1146,30 +1146,30 @@ function outtype(xf::ScanEmit, intype) end start(rf::R_{ScanEmit}, result) = - wrap(rf, _initvalue(rf), start(rf.inner, result)) + wrap(rf, _initvalue(rf), start(inner(rf), result)) function next(rf::R_{ScanEmit}, result, input) wrapping(rf, result) do u0, iresult y1, u1 = rf.xform.f(u0, input) - return u1, next(rf.inner, iresult, y1) + return u1, next(inner(rf), iresult, y1) end end function complete(rf::R_{ScanEmit}, result) u, iresult = unwrap(rf, result) if rf.xform.onlast !== nothing - iresult = next(rf.inner, iresult, rf.xform.onlast(u)) - @return_if_reduced complete(rf.inner, iresult) + iresult = next(inner(rf), iresult, rf.xform.onlast(u)) + @return_if_reduced complete(inner(rf), iresult) end - return complete(rf.inner, iresult) + return complete(inner(rf), iresult) end function combine(rf::R_{ScanEmit}, a, b) ua, ira = unwrap(rf, a) ub, irb = unwrap(rf, b) - irc = combine(rf.inner, ira, irb) + irc = combine(inner(rf), ira, irb) yc, uc = rf.xform.f(ua, ub) - irc = next(rf.inner, irc, yc) + irc = next(inner(rf), irc, yc) return wrap(rf, uc, irc) end @@ -1232,10 +1232,10 @@ end isexpansive(::Iterated) = false outtype(xf::Iterated, intype) = inittypeof(xf.init, intype) start(rf::R_{Iterated}, result) = - wrap(rf, _initvalue(rf), start(rf.inner, result)) + wrap(rf, _initvalue(rf), start(inner(rf), result)) next(rf::R_{Iterated}, result, ::Any) = wrapping(rf, result) do istate, iresult - return rf.xform.f(istate), next(rf.inner, iresult, istate) + return rf.xform.f(istate), next(inner(rf), iresult, istate) end """ @@ -1280,10 +1280,10 @@ Count(start = 1) = Count(start, oneunit(start)) isexpansive(::Count) = false outtype(xf::Count{T}, ::Any) where T = T -start(rf::R_{Count}, result) = wrap(rf, rf.xform.start, start(rf.inner, result)) +start(rf::R_{Count}, result) = wrap(rf, rf.xform.start, start(inner(rf), result)) next(rf::R_{Count}, result, ::Any) = wrapping(rf, result) do istate, iresult - return istate + rf.xform.step, next(rf.inner, iresult, istate) + return istate + rf.xform.step, next(inner(rf), iresult, istate) end """ @@ -1395,10 +1395,10 @@ where inner = $inner """) -finaltype(rf::Splitter) = finaltype(rf.inner) +finaltype(rf::Splitter) = finaltype(inner(rf)) finaltype(rf::Joiner) = - if rf.inner isa AbstractReduction - finaltype(rf.inner) + if inner(rf) isa AbstractReduction + finaltype(inner(rf)) else InType(rf) end @@ -1453,13 +1453,13 @@ begin _teezip_lens(rf) = _joiner_lens(rf)[1] ∘ (@lens _.value) - _joiner_lens(rf::Joiner) = (@lens _), rf.inner + _joiner_lens(rf::Joiner) = (@lens _), inner(rf) function _joiner_lens(rf::Reduction) - lens, rf_ds = _joiner_lens(rf.inner) + lens, rf_ds = _joiner_lens(inner(rf)) return (@lens _.inner) ∘ lens, rf_ds end function _joiner_lens(rf::Splitter) - lens_nested, rf_nested = _joiner_lens(rf.inner) + lens_nested, rf_nested = _joiner_lens(inner(rf)) lens, rf_ds = _joiner_lens(rf_nested) return (@lens _.inner) ∘ lens_nested ∘ (@lens _.inner) ∘ lens, rf_ds # The first/left `_.inner` is for `Splitter` and the @@ -1469,20 +1469,20 @@ begin end @specialize -start(rf::Splitter, result) = start(rf.inner, result) +start(rf::Splitter, result) = start(inner(rf), result) next(rf::Splitter, result, input) = - next(set(rf.inner, rf.lens, input), result, input) -complete(rf::Splitter, result) = complete(rf.inner, result) + next(set(inner(rf), rf.lens, input), result, input) +complete(rf::Splitter, result) = complete(inner(rf), result) -start(rf::Joiner, result) = start(rf.inner, result) -next(rf::Joiner, result, input) = next(rf.inner, result, (rf.value, input)) -complete(rf::Joiner, result) = complete(rf.inner, result) +start(rf::Joiner, result) = start(inner(rf), result) +next(rf::Joiner, result, input) = next(inner(rf), result, (rf.value, input)) +complete(rf::Joiner, result) = complete(inner(rf), result) isexpansive(xf::TeeZip) = isexpansive(xf.xform) outtype(xf::TeeZip, intype) = Tuple{intype, outtype(xf.xform, intype)} function Transducer(rf::Splitter) - xf_split, rf_ds = _rf_to_teezip(rf.inner) + xf_split, rf_ds = _rf_to_teezip(inner(rf)) if rf_ds isa AbstractReduction return TeeZip(xf_split) |> Transducer(rf_ds) else @@ -1491,14 +1491,14 @@ function Transducer(rf::Splitter) end function _rf_to_teezip(rf::Reduction) - xf_split, rf_ds = _rf_to_teezip(rf.inner) + xf_split, rf_ds = _rf_to_teezip(inner(rf)) return rf.xform |> xf_split, rf_ds end -_rf_to_teezip(rf::Joiner) = IdentityTransducer(), rf.inner +_rf_to_teezip(rf::Joiner) = IdentityTransducer(), inner(rf) function _rf_to_teezip(rf::Splitter) - xf_split, rf_inner = _rf_to_teezip(rf.inner) + xf_split, rf_inner = _rf_to_teezip(inner(rf)) xf_inner, rf_ds = _rf_to_teezip(rf_inner) return TeeZip(xf_split) |> xf_inner, rf_ds end @@ -1581,9 +1581,9 @@ outtype(::GetIndex, T) = error("Unexpected non-integer input type for GetIndex:\n", T) next(rf::R_{GetIndex{true}}, result, input) = - next(rf.inner, result, @inbounds rf.xform.array[input]) + next(inner(rf), result, @inbounds rf.xform.array[input]) next(rf::R_{GetIndex{false}}, result, input) = - next(rf.inner, result, rf.xform.array[input]) + next(inner(rf), result, rf.xform.array[input]) Base.:(==)(xf1::GetIndex{inbounds,A}, xf2::GetIndex{inbounds,A}) where {inbounds,A} = @@ -1626,9 +1626,9 @@ outtype(::SetIndex, T) = error("Unexpected non-integer input type for SetIndex:\n", T) next(rf::R_{SetIndex{true}}, result, input::NTuple{2, Any}) = - next(rf.inner, result, (@inbounds rf.xform.array[input[1]] = input[2];)) + next(inner(rf), result, (@inbounds rf.xform.array[input[1]] = input[2];)) next(rf::R_{SetIndex{false}}, result, input::NTuple{2, Any}) = - next(rf.inner, result, (rf.xform.array[input[1]] = input[2];)) + next(inner(rf), result, (rf.xform.array[input[1]] = input[2];)) # Index is `input[1]` due to `TeeZip`'s definition. Is it better to # flip, to be compatible with `Base.setindex!`? @@ -1685,12 +1685,12 @@ end isexpansive(::Inject) = false outtype(xf::Inject, intype) = Tuple{intype, ieltype(xf.iterator)} start(rf::R_{Inject}, result) = - wrap(rf, iterate(rf.xform.iterator), start(rf.inner, result)) + wrap(rf, iterate(rf.xform.iterator), start(inner(rf), result)) next(rf::R_{Inject}, result, input) = wrapping(rf, result) do istate, iresult istate === nothing && return istate, ensure_reduced(iresult) y, s = istate - iresult2 = next(rf.inner, iresult, (input, y)) + iresult2 = next(inner(rf), iresult, (input, y)) return iterate(rf.xform.iterator, s), iresult2 end @@ -1731,9 +1731,9 @@ Enumerate(start = 1) = Enumerate(start, oneunit(start)) isexpansive(::Enumerate) = false outtype(xf::Enumerate{T}, intype) where {T} = Tuple{T, intype} start(rf::R_{Enumerate}, result) = - wrap(rf, rf.xform.start, start(rf.inner, result)) + wrap(rf, rf.xform.start, start(inner(rf), result)) next(rf::R_{Enumerate}, result, input) = wrapping(rf, result) do i, iresult - iresult2 = next(rf.inner, iresult, (i, input)) + iresult2 = next(inner(rf), iresult, (i, input)) i + rf.xform.step, iresult2 end diff --git a/src/processes.jl b/src/processes.jl index 8160be25b2..fa762b5449 100644 --- a/src/processes.jl +++ b/src/processes.jl @@ -213,7 +213,7 @@ function provide_init(rf::AbstractReduction, ::MissingInit) return identityof(op, T) end -innermost_rf(rf::AbstractReduction) = innermost_rf(rf.inner) +innermost_rf(rf::AbstractReduction) = innermost_rf(inner(rf)) innermost_rf(f) = f innermost_rf(o::Completing) = innermost_rf(o.f) @@ -467,17 +467,17 @@ _map!(rf, coll, dest) = transduce(darkritual(rf, dest), nothing, coll) # magic... My reasoning of how it works could be completely wrong. # But at least it should not change the semantics of the function.) @inline darkritual(rf::R, dest) where {R <: Reduction} = - if rf.inner isa AbstractReduction - R(rf.xform, darkritual(rf.inner, dest)) + if inner(rf) isa AbstractReduction + R(rf.xform, darkritual(inner(rf), dest)) else @assert rf.xform.array === dest xf = typeof(rf.xform)(dest) :: SetIndex - R(xf, rf.inner) + R(xf, inner(rf)) end @inline darkritual(rf::R, dest) where {R <: Joiner} = - R(darkritual(rf.inner, dest), rf.value) + R(darkritual(inner(rf), dest), rf.value) @inline darkritual(rf::R, dest) where {R <: Splitter} = - R(darkritual(rf.inner, dest), rf.lens) + R(darkritual(inner(rf), dest), rf.lens) function _prepare_map(xf, dest, src, simd) isexpansive(xf) && error("map! only supports non-expanding transducer") diff --git a/src/show.jl b/src/show.jl index e1e4ffaf8b..7aabbb90a3 100644 --- a/src/show.jl +++ b/src/show.jl @@ -207,7 +207,7 @@ function _show_impl(io, mime, rf::Reduction) println(io, '(') _show_impl(next_io, mime, rf.xform) println(io, ",") - _show_impl(next_io, mime, rf.inner) + _show_impl(next_io, mime, inner(rf)) print(io, ")") return end @@ -224,7 +224,7 @@ function _show_impl(io, mime, rf::Splitter) print(io, "Splitter") _show_intype(io, rf) println(io, '(') - _show_impl(next_io, mime, rf.inner) + _show_impl(next_io, mime, inner(rf)) println(io, ",") _show_impl(next_io, mime, rf.lens) print(io, ")") @@ -243,7 +243,7 @@ function _show_impl(io, mime, rf::Joiner) print(io, "Joiner") _show_intype(io, rf) println(io, '(') - _show_impl(next_io, mime, rf.inner) + _show_impl(next_io, mime, inner(rf)) println(io, ",") if isdefined(rf, :value) _show_impl(next_io, mime, rf.value) diff --git a/src/simd.jl b/src/simd.jl index 8b2159ac82..19e62ced71 100644 --- a/src/simd.jl +++ b/src/simd.jl @@ -6,13 +6,13 @@ The reducible can support it using `@simd_if`. """ struct UseSIMD{ivdep} <: Transducer end outtype(::UseSIMD, intype) = intype -next(rf::R_{UseSIMD}, result, input) = next(rf.inner, result, input) +next(rf::R_{UseSIMD}, result, input) = next(inner(rf), result, input) # Make sure UseSIMD is the outer-most transducer when UseSIMD is used # via Cat. skipcomplete(rf::R_{UseSIMD}) = Reduction(rf.xform::UseSIMD, - skipcomplete(rf.inner), + skipcomplete(inner(rf)), InType(rf)) isivdep(::UseSIMD{ivdep}) where ivdep = ivdep From a86ad26c95608afb22454e5ca673913952368f1e Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 15 Jan 2019 19:41:33 -0800 Subject: [PATCH 4/7] Replace rf.xform with xform(rf) --- examples/transducers.jl | 6 +-- src/core.jl | 19 ++++---- src/library.jl | 104 ++++++++++++++++++++-------------------- src/processes.jl | 6 +-- src/show.jl | 2 +- src/simd.jl | 4 +- 6 files changed, 71 insertions(+), 70 deletions(-) diff --git a/examples/transducers.jl b/examples/transducers.jl index 7073aba5d5..3237a2e730 100644 --- a/examples/transducers.jl +++ b/examples/transducers.jl @@ -5,7 +5,7 @@ # ones: using Transducers -using Transducers: Transducer, R_, next, inner +using Transducers: Transducer, R_, next, inner, xform # ## Stateless transducer @@ -68,7 +68,7 @@ nothing # hide function Transducers.start(rf::R_{RandomRecall}, result) buffer = InType(rf)[] - rng = MersenneTwister(rf.xform.seed) + rng = MersenneTwister(xform(rf).seed) private_state = (buffer, rng) return wrap(rf, private_state, start(inner(rf), result)) end @@ -82,7 +82,7 @@ function Transducers.next(rf::R_{RandomRecall}, result, input) wrapping(rf, result) do (buffer, rng), iresult # Pickup a random element to be passed to the inner reducing function. # Replace it with the new incoming one in the buffer: - if length(buffer) < rf.xform.history + if length(buffer) < xform(rf).history push!(buffer, input) iinput = rand(rng, buffer) else diff --git a/src/core.jl b/src/core.jl index 8f13d86f15..c3b31a3a62 100644 --- a/src/core.jl +++ b/src/core.jl @@ -86,6 +86,7 @@ InType(T::Type) = throw(MethodError(InType, (T,))) Setfield.constructor_of(::Type{T}) where {T <: AbstractReduction} = T inner(rf::AbstractReduction) = rf.inner +xform(rf::AbstractReduction) = rf.xform # In clojure a reduction function is one of signature # whatever, input -> whatever @@ -102,15 +103,15 @@ struct Reduction{intype, X <: Transducer, I} <: AbstractReduction{intype} end Transducer(rf::Reduction{<:Any, <:Transducer, <:AbstractReduction}) = - Composition(rf.xform, Transducer(inner(rf))) -Transducer(rf::Reduction) = rf.xform + Composition(xform(rf), Transducer(inner(rf))) +Transducer(rf::Reduction) = xform(rf) """ Transducers.R_{X} When defining a transducer type `X`, it is often required to dispatch on type `rf::R_{X}` (Reducing Function) which bundles the current -transducer `rf.xform::X` and the inner reducing function +transducer `xform(rf)::X` and the inner reducing function `inner(rf)::R_`. """ const R_{X} = Reduction{<:Any, <:X} @@ -214,12 +215,12 @@ combine(rf::Reduction, a, b) = # Not using dispatch to avoid ambiguity if a isa PrivateState{typeof(rf)} # TODO: make sure this branch is compiled out - error("Stateful transducer ", rf.xform, " does not support `combine`") + error("Stateful transducer ", xform(rf), " does not support `combine`") elseif b isa PrivateState{typeof(rf)} error(""" Some thing went wrong in two ways: * `combine(rf, a, b)` is called but type of `a` and `b` are different. - * `rf.xform = $(rf.xform)` is stateful and does not support `combine`. + * `xform(rf) = $(xform(rf))` is stateful and does not support `combine`. """) else combine(inner(rf), a, b) @@ -346,20 +347,20 @@ finaltype(rf::Reduction) = if inner(rf) isa AbstractReduction finaltype(inner(rf)) else - outtype(rf.xform, InType(rf)) + outtype(xform(rf), InType(rf)) end # isexpansive(::Any) = true isexpansive(::Transducer) = true isexpansive(::AbstractFilter) = false -# isexpansive(rf::Reduction) = isexpansive(rf.xform) || isexpansive(inner(rf)) +# isexpansive(rf::Reduction) = isexpansive(xform(rf)) || isexpansive(inner(rf)) isexpansive(xf::Composition) = isexpansive(xf.outer) || isexpansive(xf.inner) # Should it be a type-level trait? #= iscontractive(::Any) = false iscontractive(::AbstractFilter) = true -iscontractive(rf::Reduction) = iscontractive(rf.xform) && iscontractive(inner(rf)) +iscontractive(rf::Reduction) = iscontractive(xform(rf)) && iscontractive(inner(rf)) =# struct NoComplete <: Transducer end @@ -523,7 +524,7 @@ end initvalue(x, ::Any) = x initvalue(init::Initializer, intype) = init.f(intype) -_initvalue(rf::Reduction) = initvalue(rf.xform.init, InType(rf)) +_initvalue(rf::Reduction) = initvalue(xform(rf).init, InType(rf)) inittypeof(::T, ::Type) where T = T function inittypeof(init::Initializer, intype::Type) diff --git a/src/library.jl b/src/library.jl index 082877d4c7..81cae42145 100644 --- a/src/library.jl +++ b/src/library.jl @@ -49,7 +49,7 @@ end isexpansive(::Map) = false outtype(xf::Map, intype) = Union{Base.return_types(xf.f, (intype,))...} -next(rf::R_{Map}, result, input) = next(inner(rf), result, rf.xform.f(input)) +next(rf::R_{Map}, result, input) = next(inner(rf), result, xform(rf).f(input)) """ MapSplat(f) @@ -75,7 +75,7 @@ end isexpansive(::MapSplat) = false outtype(xf::MapSplat, intype) = Union{Base.return_types(xf.f, intype)...} next(rf::R_{MapSplat}, result, input) = - next(inner(rf), result, rf.xform.f(input...)) + next(inner(rf), result, xform(rf).f(input...)) # https://clojure.github.io/clojure/clojure.core-api.html#clojure.core/replace # https://clojuredocs.org/clojure.core/replace @@ -122,7 +122,7 @@ end isexpansive(::Replace) = false outtype(xf::Replace, intype) = Union{intype, avaltype(xf.d)} next(rf::R_{Replace}, result, input) = - next(inner(rf), result, get(rf.xform.d, input, input)) + next(inner(rf), result, get(xform(rf).d, input, input)) # https://clojure.github.io/clojure/clojure.core-api.html#clojure.core/cat # https://clojuredocs.org/clojure.core/cat @@ -225,7 +225,7 @@ struct Filter{P} <: AbstractFilter end next(rf::R_{Filter}, result, input) = - rf.xform.pred(input) ? next(inner(rf), result, input) : result + xform(rf).pred(input) ? next(inner(rf), result, input) : result """ NotA(T) @@ -329,7 +329,7 @@ struct Take <: AbstractFilter end end -start(rf::R_{Take}, result) = wrap(rf, rf.xform.n, start(inner(rf), result)) +start(rf::R_{Take}, result) = wrap(rf, xform(rf).n, start(inner(rf), result)) next(rf::R_{Take}, result, input) = wrapping(rf, result) do n, iresult @@ -373,7 +373,7 @@ struct TakeLast <: AbstractFilter end function start(rf::R_{TakeLast}, result) - n = rf.xform.n + n = xform(rf).n return wrap(rf, (-n, Vector{InType(rf)}(undef, n)), start(inner(rf), result)) end @@ -435,7 +435,7 @@ struct TakeWhile{P} <: AbstractFilter end next(rf::R_{TakeWhile}, result, input) = - rf.xform.pred(input) ? next(inner(rf), result, input) : ensure_reduced(result) + xform(rf).pred(input) ? next(inner(rf), result, input) : ensure_reduced(result) # https://clojure.github.io/clojure/clojure.core-api.html#clojure.core/take-nth # https://clojuredocs.org/clojure.core/take-nth @@ -466,11 +466,11 @@ struct TakeNth <: AbstractFilter end end -start(rf::R_{TakeNth}, result) = wrap(rf, rf.xform.n, start(inner(rf), result)) +start(rf::R_{TakeNth}, result) = wrap(rf, xform(rf).n, start(inner(rf), result)) next(rf::R_{TakeNth}, result, input) = wrapping(rf, result) do c, iresult - if c == rf.xform.n + if c == xform(rf).n iresult = next(inner(rf), iresult, input) c = 1 else @@ -511,7 +511,7 @@ start(rf::R_{Drop}, result) = wrap(rf, 0, start(inner(rf), result)) next(rf::R_{Drop}, result, input) = wrapping(rf, result) do c, iresult - if c >= rf.xform.n + if c >= xform(rf).n c, next(inner(rf), iresult, input) else c += 1 @@ -555,7 +555,7 @@ struct DropLast <: AbstractFilter end function start(rf::R_{DropLast}, result) - n = rf.xform.n + 1 + n = xform(rf).n + 1 return wrap(rf, (-n, Vector{InType(rf)}(undef, n)), start(inner(rf), result)) end @@ -611,7 +611,7 @@ start(rf::R_{DropWhile}, result) = wrap(rf, true, start(inner(rf), result)) next(rf::R_{DropWhile}, result, input) = wrapping(rf, result) do dropping, iresult if dropping - dropping = rf.xform.pred(input) + dropping = xform(rf).pred(input) dropping && return (dropping, iresult) end dropping, next(inner(rf), iresult, input) @@ -708,7 +708,7 @@ outtype(::Partition, intype) = DenseSubVector{intype} function start(rf::R_{Partition}, result) buf = Vector{InType(rf)}() - sizehint!(buf, rf.xform.size) + sizehint!(buf, xform(rf).size) return wrap(rf, (0, 0, buf), start(inner(rf), result)) end @@ -720,48 +720,48 @@ end function _window_next(rf, i, s, buf, iresult, input) i += 1 - @assert 1 <= i <= rf.xform.size + @assert 1 <= i <= xform(rf).size len = length(buf) - if s == 0 && len < rf.xform.size + if s == 0 && len < xform(rf).size push!(buf, input) - if i == rf.xform.size - # This is the first time `length(buf) == rf.xform.size` is + if i == xform(rf).size + # This is the first time `length(buf) == xform(rf).size` is # true. iinput = (@view buf[1:end]) :: DenseSubVector # Do not use `@view buf[:]` to make the indices # `UnitRange` instead of `OneTo`. - @assert i == rf.xform.size + @assert i == xform(rf).size i = 0 - s = rf.xform.step + s = xform(rf).step iresult = next(inner(rf), iresult, iinput) # Throw away buf[1] which is just consumed by the downstream: deleteat!(buf, 1) end return (i, s, buf), iresult - elseif len < 2 * rf.xform.size - 1 + elseif len < 2 * xform(rf).size - 1 push!(buf, input) else # TODO: optimize for size < step case - buf[i + rf.xform.size - 1] = input + buf[i + xform(rf).size - 1] = input end - if i == rf.xform.size + if i == xform(rf).size # Wrapping the window. Next window will be i=1. # Copy tail to head. copyto!(buf, 1, # destination offset buf, - rf.xform.size + 1, # source offset - rf.xform.size - 1) # size + xform(rf).size + 1, # source offset + xform(rf).size - 1) # size end s -= 1 - @assert 0 <= s < rf.xform.step + @assert 0 <= s < xform(rf).step if s == 0 - s = rf.xform.step - iinput = @view buf[i:i + rf.xform.size - 1] # unsafe_view? @inbounds? + s = xform(rf).step + iinput = @view buf[i:i + xform(rf).size - 1] # unsafe_view? @inbounds? iinput :: DenseSubVector iresult = next(inner(rf), iresult, iinput) end - if i == rf.xform.size + if i == xform(rf).size i = 0 end return (i, s, buf), iresult @@ -769,8 +769,8 @@ end function complete(rf::R_{Partition}, result) (i, s, buf), iresult = unwrap(rf, result) - if rf.xform.flush && s != rf.xform.step - iinput = @view buf[i:i + rf.xform.size - 1] # unsafe_view? @inbounds? + if xform(rf).flush && s != xform(rf).step + iinput = @view buf[i:i + xform(rf).size - 1] # unsafe_view? @inbounds? iinput = @view iinput[s + 1:end] iinput :: DenseSubVector iresult = next(inner(rf), iresult, iinput) @@ -821,7 +821,7 @@ end @inline function next(rf::R_{PartitionBy}, result, input) wrapping(rf, result) do (iinput, pval), iresult - val = rf.xform.f(input) + val = xform(rf).f(input) if pval isa Unseen || val == pval push!(iinput, input) else @@ -877,7 +877,7 @@ outtype(xf::Keep, intype) = Nothing) function next(rf::R_{Keep}, result, input) - iinput = rf.xform.f(input) + iinput = xform(rf).f(input) return iinput === nothing ? result : next(inner(rf), result, iinput) end @@ -953,7 +953,7 @@ start(rf::R_{Interpose}, result) = wrap(rf, Val(true), start(inner(rf), result)) next(rf::R_{Interpose}, result, input) = wrapping(rf, result) do isfirst, iresult if isfirst isa Val{false} - iresult = next(inner(rf), iresult, rf.xform.sep) + iresult = next(inner(rf), iresult, xform(rf).sep) end return Val(false), next(inner(rf), iresult, input) end @@ -1074,8 +1074,8 @@ function _type_scan_fixedpoint(f, A, X, limit = 10) end function start(rf::R_{Scan}, result) - if rf.xform.init === nothing - init = identityof(rf.xform.f, InType(rf)) + if xform(rf).init === nothing + init = identityof(xform(rf).f, InType(rf)) else init = _initvalue(rf) end @@ -1084,7 +1084,7 @@ end function next(rf::R_{Scan}, result, input) wrapping(rf, result) do acc, iresult - acc = rf.xform.f(acc, input) + acc = xform(rf).f(acc, input) return acc, next(inner(rf), iresult, acc) end end @@ -1150,15 +1150,15 @@ start(rf::R_{ScanEmit}, result) = function next(rf::R_{ScanEmit}, result, input) wrapping(rf, result) do u0, iresult - y1, u1 = rf.xform.f(u0, input) + y1, u1 = xform(rf).f(u0, input) return u1, next(inner(rf), iresult, y1) end end function complete(rf::R_{ScanEmit}, result) u, iresult = unwrap(rf, result) - if rf.xform.onlast !== nothing - iresult = next(inner(rf), iresult, rf.xform.onlast(u)) + if xform(rf).onlast !== nothing + iresult = next(inner(rf), iresult, xform(rf).onlast(u)) @return_if_reduced complete(inner(rf), iresult) end return complete(inner(rf), iresult) @@ -1168,7 +1168,7 @@ function combine(rf::R_{ScanEmit}, a, b) ua, ira = unwrap(rf, a) ub, irb = unwrap(rf, b) irc = combine(inner(rf), ira, irb) - yc, uc = rf.xform.f(ua, ub) + yc, uc = xform(rf).f(ua, ub) irc = next(inner(rf), irc, yc) return wrap(rf, uc, irc) end @@ -1235,7 +1235,7 @@ start(rf::R_{Iterated}, result) = wrap(rf, _initvalue(rf), start(inner(rf), result)) next(rf::R_{Iterated}, result, ::Any) = wrapping(rf, result) do istate, iresult - return rf.xform.f(istate), next(inner(rf), iresult, istate) + return xform(rf).f(istate), next(inner(rf), iresult, istate) end """ @@ -1280,10 +1280,10 @@ Count(start = 1) = Count(start, oneunit(start)) isexpansive(::Count) = false outtype(xf::Count{T}, ::Any) where T = T -start(rf::R_{Count}, result) = wrap(rf, rf.xform.start, start(inner(rf), result)) +start(rf::R_{Count}, result) = wrap(rf, xform(rf).start, start(inner(rf), result)) next(rf::R_{Count}, result, ::Any) = wrapping(rf, result) do istate, iresult - return istate + rf.xform.step, next(inner(rf), iresult, istate) + return istate + xform(rf).step, next(inner(rf), iresult, istate) end """ @@ -1492,7 +1492,7 @@ end function _rf_to_teezip(rf::Reduction) xf_split, rf_ds = _rf_to_teezip(inner(rf)) - return rf.xform |> xf_split, rf_ds + return xform(rf) |> xf_split, rf_ds end _rf_to_teezip(rf::Joiner) = IdentityTransducer(), inner(rf) @@ -1581,9 +1581,9 @@ outtype(::GetIndex, T) = error("Unexpected non-integer input type for GetIndex:\n", T) next(rf::R_{GetIndex{true}}, result, input) = - next(inner(rf), result, @inbounds rf.xform.array[input]) + next(inner(rf), result, @inbounds xform(rf).array[input]) next(rf::R_{GetIndex{false}}, result, input) = - next(inner(rf), result, rf.xform.array[input]) + next(inner(rf), result, xform(rf).array[input]) Base.:(==)(xf1::GetIndex{inbounds,A}, xf2::GetIndex{inbounds,A}) where {inbounds,A} = @@ -1626,9 +1626,9 @@ outtype(::SetIndex, T) = error("Unexpected non-integer input type for SetIndex:\n", T) next(rf::R_{SetIndex{true}}, result, input::NTuple{2, Any}) = - next(inner(rf), result, (@inbounds rf.xform.array[input[1]] = input[2];)) + next(inner(rf), result, (@inbounds xform(rf).array[input[1]] = input[2];)) next(rf::R_{SetIndex{false}}, result, input::NTuple{2, Any}) = - next(inner(rf), result, (rf.xform.array[input[1]] = input[2];)) + next(inner(rf), result, (xform(rf).array[input[1]] = input[2];)) # Index is `input[1]` due to `TeeZip`'s definition. Is it better to # flip, to be compatible with `Base.setindex!`? @@ -1685,13 +1685,13 @@ end isexpansive(::Inject) = false outtype(xf::Inject, intype) = Tuple{intype, ieltype(xf.iterator)} start(rf::R_{Inject}, result) = - wrap(rf, iterate(rf.xform.iterator), start(inner(rf), result)) + wrap(rf, iterate(xform(rf).iterator), start(inner(rf), result)) next(rf::R_{Inject}, result, input) = wrapping(rf, result) do istate, iresult istate === nothing && return istate, ensure_reduced(iresult) y, s = istate iresult2 = next(inner(rf), iresult, (input, y)) - return iterate(rf.xform.iterator, s), iresult2 + return iterate(xform(rf).iterator, s), iresult2 end @@ -1731,9 +1731,9 @@ Enumerate(start = 1) = Enumerate(start, oneunit(start)) isexpansive(::Enumerate) = false outtype(xf::Enumerate{T}, intype) where {T} = Tuple{T, intype} start(rf::R_{Enumerate}, result) = - wrap(rf, rf.xform.start, start(inner(rf), result)) + wrap(rf, xform(rf).start, start(inner(rf), result)) next(rf::R_{Enumerate}, result, input) = wrapping(rf, result) do i, iresult iresult2 = next(inner(rf), iresult, (i, input)) - i + rf.xform.step, iresult2 + i + xform(rf).step, iresult2 end diff --git a/src/processes.jl b/src/processes.jl index fa762b5449..a1b9abfd41 100644 --- a/src/processes.jl +++ b/src/processes.jl @@ -468,10 +468,10 @@ _map!(rf, coll, dest) = transduce(darkritual(rf, dest), nothing, coll) # But at least it should not change the semantics of the function.) @inline darkritual(rf::R, dest) where {R <: Reduction} = if inner(rf) isa AbstractReduction - R(rf.xform, darkritual(inner(rf), dest)) + R(xform(rf), darkritual(inner(rf), dest)) else - @assert rf.xform.array === dest - xf = typeof(rf.xform)(dest) :: SetIndex + @assert xform(rf).array === dest + xf = typeof(xform(rf))(dest) :: SetIndex R(xf, inner(rf)) end @inline darkritual(rf::R, dest) where {R <: Joiner} = diff --git a/src/show.jl b/src/show.jl index 7aabbb90a3..59351e33d4 100644 --- a/src/show.jl +++ b/src/show.jl @@ -205,7 +205,7 @@ function _show_impl(io, mime, rf::Reduction) print(io, "Reduction") _show_intype(io, rf) println(io, '(') - _show_impl(next_io, mime, rf.xform) + _show_impl(next_io, mime, xform(rf)) println(io, ",") _show_impl(next_io, mime, inner(rf)) print(io, ")") diff --git a/src/simd.jl b/src/simd.jl index 19e62ced71..cee75abb8f 100644 --- a/src/simd.jl +++ b/src/simd.jl @@ -11,12 +11,12 @@ next(rf::R_{UseSIMD}, result, input) = next(inner(rf), result, input) # Make sure UseSIMD is the outer-most transducer when UseSIMD is used # via Cat. skipcomplete(rf::R_{UseSIMD}) = - Reduction(rf.xform::UseSIMD, + Reduction(xform(rf)::UseSIMD, skipcomplete(inner(rf)), InType(rf)) isivdep(::UseSIMD{ivdep}) where ivdep = ivdep -isivdep(rf::Reduction) = isivdep(rf.xform) +isivdep(rf::Reduction) = isivdep(xform(rf)) _name_of_transducer_type(xf::UseSIMD) = prefixed_type_name(xf) From e48123a2b75474cb6e4501f0d2c6c114d315060e Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 15 Jan 2019 19:57:02 -0800 Subject: [PATCH 5/7] Make Reduction flat --- docs/src/doctests/show_rf.md | 86 +++++++++---------- src/core.jl | 49 +++++++++-- src/library.jl | 156 ++++++----------------------------- src/processes.jl | 10 +-- src/show.jl | 42 ---------- test/__test_ir.jl | 7 +- 6 files changed, 117 insertions(+), 233 deletions(-) diff --git a/docs/src/doctests/show_rf.md b/docs/src/doctests/show_rf.md index 4d9836123f..6527a35d3d 100644 --- a/docs/src/doctests/show_rf.md +++ b/docs/src/doctests/show_rf.md @@ -28,17 +28,17 @@ rf = Reduction( TeeZip(Count()), right, Float64) -@set! rf.inner.inner.value = 1.25 +@set! rf.xforms[3].xform.value = 1.25 # output -Splitter{▶ Float64}( +Reduction{▶ Float64}( + Splitter((@lens _.xforms[2].xform.value)), Reduction{▶ Float64}( Count(1, 1), - Joiner{▶ ⦃Float64, Int64⦄}( - Transducers.right, - 1.25)), - (@lens _.inner.value)) + Reduction{▶ Int64}( + Joiner(1.25), + Transducers.right))) ``` ```jldoctest @@ -46,23 +46,23 @@ rf = Reduction( Filter(isfinite) |> TeeZip(Count()) |> Enumerate() |> MapSplat(*), right, Float64) -@set! rf.inner.inner.inner.value = 1.25 +@set! rf.xforms[4].xform.value = 1.25 # output Reduction{▶ Float64}( Filter(isfinite), - Splitter{▶ Float64}( + Reduction{▶ Float64}( + Splitter((@lens _.xforms[2].xform.value)), Reduction{▶ Float64}( Count(1, 1), - Joiner{▶ ⦃Float64, Int64⦄}( + Reduction{▶ Int64}( + Joiner(1.25), Reduction{▶ ⦃Float64, Int64⦄}( Enumerate(1, 1), Reduction{▶ ⦃Int64, ⦃Float64, Int64⦄⦄}( MapSplat(*), - Transducers.right)), - 1.25)), - (@lens _.inner.value))) + Transducers.right)))))) ``` ```jldoctest @@ -70,24 +70,24 @@ rf = Reduction( TeeZip(Count() |> TeeZip(FlagFirst())), right, Float64) -@set! rf.inner.inner.inner.inner.value = 123456789 -@set! rf.inner.inner.inner.inner.inner.value = 1.25 +@set! rf.xforms[5].xform.value = 123456789 +@set! rf.xforms[6].xform.value = 1.25 # output -Splitter{▶ Float64}( +Reduction{▶ Float64}( + Splitter((@lens _.xforms[5].xform.value)), Reduction{▶ Float64}( Count(1, 1), - Splitter{▶ Int64}( + Reduction{▶ Int64}( + Splitter((@lens _.xforms[2].xform.value)), Reduction{▶ Int64}( FlagFirst(), - Joiner{▶ ⦃Int64, ⦃Bool, Int64⦄⦄}( - Joiner{▶ ⦃Float64, ⦃Int64, ⦃Bool, Int64⦄⦄⦄}( - Transducers.right, - 1.25), - 123456789)), - (@lens _.inner.value))), - (@lens _.inner.inner.inner.inner.value)) + Reduction{▶ ⦃Bool, Int64⦄}( + Joiner(123456789), + Reduction{▶ ⦃Int64, ⦃Bool, Int64⦄⦄}( + Joiner(1.25), + Transducers.right)))))) ``` ```jldoctest @@ -95,26 +95,26 @@ rf = Reduction( TeeZip(Filter(isodd) |> Map(identity) |> TeeZip(Map(identity))), right, Int64) -@set! rf.inner.inner.inner.inner.inner.value = 123456789 -@set! rf.inner.inner.inner.inner.inner.inner.value= 123456789 +@set! rf.xforms[6].xform.value = 123456789 +@set! rf.xforms[7].xform.value = 123456789 # output -Splitter{▶ Int64}( +Reduction{▶ Int64}( + Splitter((@lens _.xforms[6].xform.value)), Reduction{▶ Int64}( Filter(isodd), Reduction{▶ Int64}( Map(identity), - Splitter{▶ Int64}( + Reduction{▶ Int64}( + Splitter((@lens _.xforms[2].xform.value)), Reduction{▶ Int64}( Map(identity), - Joiner{▶ ⦃Int64, Int64⦄}( - Joiner{▶ ⦃Int64, ⦃Int64, Int64⦄⦄}( - Transducers.right, - 123456789), - 123456789)), - (@lens _.inner.value)))), - (@lens _.inner.inner.inner.inner.inner.value)) + Reduction{▶ Int64}( + Joiner(123456789), + Reduction{▶ ⦃Int64, Int64⦄}( + Joiner(123456789), + Transducers.right))))))) ``` ```jldoctest @@ -125,21 +125,21 @@ rf = Reduction( # output -Splitter{▶ Any}( +Reduction{▶ Any}( + Splitter((@lens _.xforms[6].xform.value)), Reduction{▶ Any}( Filter(isodd), Reduction{▶ Any}( Map(identity), - Splitter{▶ Any}( + Reduction{▶ Any}( + Splitter((@lens _.xforms[2].xform.value)), Reduction{▶ Any}( Map(identity), - Joiner{▶ ⦃Any, Any⦄}( - Joiner{▶ ⦃Any, ⦃Any, Any⦄⦄}( - Transducers.right, - nothing), - nothing)), - (@lens _.inner.value)))), - (@lens _.inner.inner.inner.inner.inner.value)) + Reduction{▶ Any}( + Joiner(nothing), + Reduction{▶ ⦃Any, Any⦄}( + Joiner(nothing), + Transducers.right))))))) ``` ```@meta diff --git a/src/core.jl b/src/core.jl index c3b31a3a62..9807aa0a6d 100644 --- a/src/core.jl +++ b/src/core.jl @@ -88,6 +88,16 @@ Setfield.constructor_of(::Type{T}) where {T <: AbstractReduction} = T inner(rf::AbstractReduction) = rf.inner xform(rf::AbstractReduction) = rf.xform +struct TypedTransducer{intype, X <: Transducer} + xform::X +end +TypedTransducer{intype}(xform::X) where {intype, X} = + TypedTransducer{intype, X}(xform) + +Setfield.constructor_of(::Type{T}) where {T <: TypedTransducer} = T + +InType(::TypedTransducer{intype}) where intype = intype + # In clojure a reduction function is one of signature # whatever, input -> whatever # @@ -97,14 +107,29 @@ xform(rf::AbstractReduction) = rf.xform # a transducer `xform` and an inner reduction function `inner`. # `inner` can be either a `Reduction` or a function with arity-2 and arity-1 methods # -struct Reduction{intype, X <: Transducer, I} <: AbstractReduction{intype} - xform::X - inner::I +struct Reduction{intype, T, F} <: AbstractReduction{intype} + xforms::T + bottom::F end -Transducer(rf::Reduction{<:Any, <:Transducer, <:AbstractReduction}) = - Composition(xform(rf), Transducer(inner(rf))) -Transducer(rf::Reduction) = xform(rf) +Reduction(xforms, bottom) = + Reduction{InType(xforms[1]), typeof(xforms), typeof(bottom)}(xforms, bottom) + +inner(rf::Reduction) = + if length(rf.xforms) > 1 + Reduction(Base.tail(rf.xforms), rf.bottom) + else + rf.bottom + end + +xform(rf::Reduction) = rf.xforms[1].xform + +Transducer(rf::Reduction) = + if inner(rf) isa AbstractReduction + Composition(xform(rf), Transducer(inner(rf))) + else + xform(rf) + end """ Transducers.R_{X} @@ -114,10 +139,16 @@ on type `rf::R_{X}` (Reducing Function) which bundles the current transducer `xform(rf)::X` and the inner reducing function `inner(rf)::R_`. """ -const R_{X} = Reduction{<:Any, <:X} +const R_{X} = Reduction{<:Any, <:Tuple{<:TypedTransducer{<:Any, <:X}, Vararg}} + +@inline Reduction{intype}(xf, inner::Reduction) where intype = + Reduction((TypedTransducer{intype}(xf), inner.xforms...), inner.bottom) + +@inline Reduction{intype}(xf, inner) where intype = + Reduction((TypedTransducer{intype}(xf),), inner) -@inline Reduction(xf::X, inner::I, ::Type{InType}) where {X, I, InType} = - Reduction{InType, X, I}(xf, inner) +@inline Reduction(xf, inner, ::Type{intype}) where intype = + Reduction{intype}(xf, inner) @inline function Reduction(xf_::Composition, f, intype::Type) xf = _normalize(xf_) diff --git a/src/library.jl b/src/library.jl index 81cae42145..cbcff0390d 100644 --- a/src/library.jl +++ b/src/library.jl @@ -1330,158 +1330,56 @@ end # `Reduction` dynamically which calls to the rest of inner reductions # after the value is zipped (joined). -# Consider a transducer: -# -# Map(identity) |> -# TeeZip( -# Count() |> Filter(isodd) -# ) |> -# MapSplat(*) -# -# Applying this transducer to a reducing function `rf` produces -# -# Reduction( -# identity, -# Splitter( -# Reduction( -# Count(), -# Reduction( -# Filter(isodd), -# Joiner( -# Reduction( -# MapSplat(*), -# rf), -# value))))) -# -# where `value` is the placeholder for the output value of the -# transducer (here `Map(identity)`) outer to `TeeZip`. - -struct Splitter{intype, R, L} <: AbstractReduction{intype} - inner::R +struct Splitter{L} <: Transducer lens::L end -Splitter(inner::R, lens::L) where {R, L} = - Splitter{InType(R), R, L}(inner, lens) - -struct Joiner{intype, F, T} <: AbstractReduction{intype} - inner::F # original inner reduction +struct Joiner{T} <: Transducer value::T # original input - @inline function Joiner{intype,F,T}(inner) where {intype,F,T} - _joiner_error(inner, intype) + @inline Joiner{T}(value) where T = new(value) + @inline function Joiner{T}() where T if isbitstype(T) - return new(inner) + return new() else - return new{intype,F,Union{T,Nothing}}(inner, nothing) + return new{Union{T,Nothing}}(nothing) end end - - @inline function Joiner{intype,F,T}(inner, value) where {intype,F,T} - _joiner_error(inner, intype) - return new(inner, value) - end end -@inline _joiner_error(::Any, ::Any) = nothing -@inline _joiner_error(inner::Reduction, intype) = - _joiner_error(inner, intype, InType(inner)) -@inline _joiner_error(inner, ::Type{T}, ::Type{T}) where T = nothing -@noinline _joiner_error(inner, intype, intype_inner) = error(""" -`intype` specified for `Joiner` and inner reducing function does not match. - intype = $intype - InType(inner) = $intype_inner -where - inner = $inner -""") - -finaltype(rf::Splitter) = finaltype(inner(rf)) -finaltype(rf::Joiner) = - if inner(rf) isa AbstractReduction - finaltype(inner(rf)) - else - InType(rf) - end +Setfield.constructor_of(::Type{T}) where {T <: Joiner} = T -# It's ugly that `Reduction` returns a non-`Reduction` type! TODO: fix it function Reduction(xf::Composition{<:TeeZip}, f, intype::Type) @nospecialize - rf = _teezip_rf(xf.outer.xform, intype, (xf.inner, f, intype)) - return Splitter(rf, _teezip_lens(rf)) + dummy_bottom = nothing + rf_outer = Reduction(xf.outer::TeeZip, dummy_bottom, intype) + rf_inner = Reduction(xf.inner, f, finaltype(rf_outer)) + return Reduction((rf_outer.xforms..., rf_inner.xforms...), f) end function Reduction(xf::TeeZip, f, intype::Type) @nospecialize - rf = _teezip_rf(xf.xform, intype, (nothing, f, intype)) - return Splitter(rf, _teezip_lens(rf)) -end - -function _teezip_rf(xf::Composition, intype, downstream) - @nospecialize - intype_inner = outtype(xf.outer, intype) - rf_inner = _teezip_rf(xf.inner, intype_inner, downstream) - return Reduction(xf.outer, rf_inner, intype) -end - -function _teezip_rf(xf, intype, downstream) - @nospecialize - xf_ds, f, intype_orig = downstream - intype_ds = Tuple{intype_orig, outtype(xf, intype)} - if xf_ds === nothing - rf_ds = f - else - rf_ds = Reduction(xf_ds, f, intype_ds) - end - joiner = Joiner{intype_ds, typeof(rf_ds), intype_orig}(rf_ds) - return Reduction(xf, joiner, intype) -end - -""" - _teezip_lens(rf) :: Lens - -Return a lens to the `.value` field of the first "unbalanced" -`Joiner`. A `Joiner` matched with preceding `Splitter` would be -treated as a regular reducing function node. Thus, reducing function -`rf` must have one more `Joiner` than `Splitter`. -""" -_teezip_lens - -# begin/end is not necessary for @nospecialize/@specialize; it's just -# for visually marking the lines applying @nospecialize. -@nospecialize -begin - - _teezip_lens(rf) = _joiner_lens(rf)[1] ∘ (@lens _.value) - - _joiner_lens(rf::Joiner) = (@lens _), inner(rf) - function _joiner_lens(rf::Reduction) - lens, rf_ds = _joiner_lens(inner(rf)) - return (@lens _.inner) ∘ lens, rf_ds - end - function _joiner_lens(rf::Splitter) - lens_nested, rf_nested = _joiner_lens(inner(rf)) - lens, rf_ds = _joiner_lens(rf_nested) - return (@lens _.inner) ∘ lens_nested ∘ (@lens _.inner) ∘ lens, rf_ds - # The first/left `_.inner` is for `Splitter` and the - # middle/second `_.inner` is for `Joiner`. - end - + dummy_bottom = nothing + rf_split = Reduction(xf.xform, dummy_bottom, intype) + return Reduction( + (TypedTransducer{intype}( + Splitter(@lens _.xforms[length(rf_split.xforms) + 1].xform.value)), + rf_split.xforms..., + TypedTransducer{finaltype(rf_split)}(Joiner{intype}())), + f) end -@specialize -start(rf::Splitter, result) = start(inner(rf), result) -next(rf::Splitter, result, input) = - next(set(inner(rf), rf.lens, input), result, input) -complete(rf::Splitter, result) = complete(inner(rf), result) +next(rf::R_{Splitter}, result, input) = + next(set(inner(rf), xform(rf).lens, input), result, input) -start(rf::Joiner, result) = start(inner(rf), result) -next(rf::Joiner, result, input) = next(inner(rf), result, (rf.value, input)) -complete(rf::Joiner, result) = complete(inner(rf), result) +outtype(xf::Joiner{T}, intype) where {T} = Tuple{T, intype} +next(rf::R_{Joiner}, result, input) = + next(inner(rf), result, (xform(rf).value, input)) isexpansive(xf::TeeZip) = isexpansive(xf.xform) outtype(xf::TeeZip, intype) = Tuple{intype, outtype(xf.xform, intype)} -function Transducer(rf::Splitter) +function Transducer(rf::R_{Splitter}) xf_split, rf_ds = _rf_to_teezip(inner(rf)) if rf_ds isa AbstractReduction return TeeZip(xf_split) |> Transducer(rf_ds) @@ -1495,9 +1393,9 @@ function _rf_to_teezip(rf::Reduction) return xform(rf) |> xf_split, rf_ds end -_rf_to_teezip(rf::Joiner) = IdentityTransducer(), inner(rf) +_rf_to_teezip(rf::R_{Joiner}) = IdentityTransducer(), inner(rf) -function _rf_to_teezip(rf::Splitter) +function _rf_to_teezip(rf::R_{Splitter}) xf_split, rf_inner = _rf_to_teezip(inner(rf)) xf_inner, rf_ds = _rf_to_teezip(rf_inner) return TeeZip(xf_split) |> xf_inner, rf_ds diff --git a/src/processes.jl b/src/processes.jl index a1b9abfd41..319b6021d3 100644 --- a/src/processes.jl +++ b/src/processes.jl @@ -466,18 +466,14 @@ _map!(rf, coll, dest) = transduce(darkritual(rf, dest), nothing, coll) # fetch `dest` via `getproperty` in each iteration. (This is too much # magic... My reasoning of how it works could be completely wrong. # But at least it should not change the semantics of the function.) -@inline darkritual(rf::R, dest) where {R <: Reduction} = +@inline darkritual(rf::Reduction, dest) = if inner(rf) isa AbstractReduction - R(xform(rf), darkritual(inner(rf), dest)) + Reduction{InType(rf)}(xform(rf), darkritual(inner(rf), dest)) else @assert xform(rf).array === dest xf = typeof(xform(rf))(dest) :: SetIndex - R(xf, inner(rf)) + Reduction{InType(rf)}(xf, inner(rf)) end -@inline darkritual(rf::R, dest) where {R <: Joiner} = - R(darkritual(inner(rf), dest), rf.value) -@inline darkritual(rf::R, dest) where {R <: Splitter} = - R(darkritual(inner(rf), dest), rf.lens) function _prepare_map(xf, dest, src, simd) isexpansive(xf) && error("map! only supports non-expanding transducer") diff --git a/src/show.jl b/src/show.jl index 59351e33d4..ae24f8c445 100644 --- a/src/show.jl +++ b/src/show.jl @@ -212,46 +212,4 @@ function _show_impl(io, mime, rf::Reduction) return end -function _show_impl(io, mime, rf::Splitter) - indent = get(io, :indent, "") - first_indent = get(io, :first_indent, indent) - next_indent = indent * ' ' ^ 4 - next_io = IOContext(io, - :indent => next_indent, - :first_indent => next_indent) - - print(io, first_indent) - print(io, "Splitter") - _show_intype(io, rf) - println(io, '(') - _show_impl(next_io, mime, inner(rf)) - println(io, ",") - _show_impl(next_io, mime, rf.lens) - print(io, ")") - return -end - -function _show_impl(io, mime, rf::Joiner) - indent = get(io, :indent, "") - first_indent = get(io, :first_indent, indent) - next_indent = indent * ' ' ^ 4 - next_io = IOContext(io, - :indent => next_indent, - :first_indent => next_indent) - - print(io, first_indent) - print(io, "Joiner") - _show_intype(io, rf) - println(io, '(') - _show_impl(next_io, mime, inner(rf)) - println(io, ",") - if isdefined(rf, :value) - _show_impl(next_io, mime, rf.value) - else - _show_impl(next_io, mime, Text("#undef")) - end - print(io, ")") - return -end - @specialize diff --git a/test/__test_ir.jl b/test/__test_ir.jl index a0d736fc4b..055b117652 100644 --- a/test/__test_ir.jl +++ b/test/__test_ir.jl @@ -41,8 +41,8 @@ nmatches(r, s) = count(_ -> true, eachmatch(r, s)) @testset for simd in [false, true, :ivdep] args = _prepare_map(xf, ys, xs, simd) ir = llvm_ir(_map!, args) - @test nmatches(r"fmul <4 x double>", ir) >= 4 - @test nmatches(r"fcmp [a-z]* <4 x double>", ir) >= 4 + @test_broken nmatches(r"fmul <4 x double>", ir) >= 4 + @test_broken nmatches(r"fcmp [a-z]* <4 x double>", ir) >= 4 end end @@ -72,6 +72,7 @@ unsafe_setter(ys) = @testset "$key" for key in first.(params) xf = Dict(params)[key] + broken = key ∈ (:TeeZip, :Zip) xs = ones(10) ys = zero(xs) @@ -89,7 +90,7 @@ unsafe_setter(ys) = @test ys == 2xs ir = llvm_ir(transduce, (rf, nothing, xs)) - @test nmatches(r"fmul <4 x double>", ir) >= 4 + @test_broken_if broken nmatches(r"fmul <4 x double>", ir) >= 4 end end From 85dfd43b3a0b8a8760632eb13eb0418a8941a698 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Mon, 14 Jan 2019 16:28:22 -0800 Subject: [PATCH 6/7] Writing Extrema section in tutorial_missings.jl --- examples/tutorial_missings.jl | 40 +++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/examples/tutorial_missings.jl b/examples/tutorial_missings.jl index 5fddbe5cc6..a2d3dcb9cd 100644 --- a/examples/tutorial_missings.jl +++ b/examples/tutorial_missings.jl @@ -261,3 +261,43 @@ mapfoldl(xf_argmax(), right, [1, 3, missing, 2]) # the index of the largest odd value: mapfoldl(xf_argmax(Filter(isodd)), right, [1, 4, 3, 2]) + +# ## Extrema + +argext_step(should_update) = + ((argext, ext), (index, value)) -> + should_update(ext, value) ? (index, value) : (argext, ext) + +argext_init(::typeof(>), ::Type{Tuple{F, S}}) where {F, S} = (zero(F), typemax(S)) +argext_init(::typeof(<), ::Type{Tuple{F, S}}) where {F, S} = (zero(F), typemin(S)) + +xf_scanext(should_update) = + Scan(argext_step(should_update), + Initializer(TT -> argext_init(should_update, TT))) +nothing # hide + +ans = # hide +mapfoldl(Zip(Count(), NotA(Missing)) |> xf_scanext(<), right, [1.0, 3.0, missing, 2.0]) +#- +@assert ans === (2, 3.0) # hide +@show ans + +xf_fullextrema(xf_filter = NotA(Missing)) = + Zip(Count(), xf_filter) |> Zip(xf_scanext(>), xf_scanext(<)) + +ans = # hide +mapfoldl(xf_fullextrema(), right, [1.0, 3.0, -1.0, 2.0]) +#- +@assert ans === ((3, -1.0), (2, 3.0)) # hide +@show ans + +xf_argextrema(xf_filter = NotA(Missing)) = + xf_fullextrema(xf_filter) |> Map() do ((argmin, min), (argmax, max)) + (argmin, argmax) + end + +ans = # hide +mapfoldl(xf_argextrema(), right, [1.0, 3.0, -1.0, 2.0]) +#- +@assert ans === (3, 2) # hide +@show ans From 712a1c13bcf3708cdb0823318c74aeb8e2439d6e Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 15 Jan 2019 21:42:55 -0800 Subject: [PATCH 7/7] Remove AbstractReduction Now that Joiner and Splitter are not reducing functions, there is no need for AbstractReduction. --- src/core.jl | 23 +++++++++-------------- src/library.jl | 2 +- src/processes.jl | 12 ++++++------ src/show.jl | 2 +- 4 files changed, 17 insertions(+), 22 deletions(-) diff --git a/src/core.jl b/src/core.jl index 9807aa0a6d..a38099116c 100644 --- a/src/core.jl +++ b/src/core.jl @@ -77,17 +77,6 @@ appropriately defined for child types. """ AbstractFilter -abstract type AbstractReduction{intype} end - -InType(::T) where T = InType(T) -InType(::Type{<:AbstractReduction{intype}}) where {intype} = intype -InType(T::Type) = throw(MethodError(InType, (T,))) - -Setfield.constructor_of(::Type{T}) where {T <: AbstractReduction} = T - -inner(rf::AbstractReduction) = rf.inner -xform(rf::AbstractReduction) = rf.xform - struct TypedTransducer{intype, X <: Transducer} xform::X end @@ -107,7 +96,7 @@ InType(::TypedTransducer{intype}) where intype = intype # a transducer `xform` and an inner reduction function `inner`. # `inner` can be either a `Reduction` or a function with arity-2 and arity-1 methods # -struct Reduction{intype, T, F} <: AbstractReduction{intype} +struct Reduction{intype, T, F} xforms::T bottom::F end @@ -115,6 +104,12 @@ end Reduction(xforms, bottom) = Reduction{InType(xforms[1]), typeof(xforms), typeof(bottom)}(xforms, bottom) +InType(::T) where T = InType(T) +InType(::Type{<:Reduction{intype}}) where {intype} = intype +InType(T::Type) = throw(MethodError(InType, (T,))) + +Setfield.constructor_of(::Type{T}) where {T <: Reduction} = T + inner(rf::Reduction) = if length(rf.xforms) > 1 Reduction(Base.tail(rf.xforms), rf.bottom) @@ -125,7 +120,7 @@ inner(rf::Reduction) = xform(rf::Reduction) = rf.xforms[1].xform Transducer(rf::Reduction) = - if inner(rf) isa AbstractReduction + if inner(rf) isa Reduction Composition(xform(rf), Transducer(inner(rf))) else xform(rf) @@ -375,7 +370,7 @@ outtype(::Any, ::Any) = Any outtype(::AbstractFilter, intype) = intype finaltype(rf::Reduction) = - if inner(rf) isa AbstractReduction + if inner(rf) isa Reduction finaltype(inner(rf)) else outtype(xform(rf), InType(rf)) diff --git a/src/library.jl b/src/library.jl index cbcff0390d..98d0ae2deb 100644 --- a/src/library.jl +++ b/src/library.jl @@ -1381,7 +1381,7 @@ outtype(xf::TeeZip, intype) = Tuple{intype, outtype(xf.xform, intype)} function Transducer(rf::R_{Splitter}) xf_split, rf_ds = _rf_to_teezip(inner(rf)) - if rf_ds isa AbstractReduction + if rf_ds isa Reduction return TeeZip(xf_split) |> Transducer(rf_ds) else return TeeZip(xf_split) diff --git a/src/processes.jl b/src/processes.jl index 319b6021d3..9e8696c642 100644 --- a/src/processes.jl +++ b/src/processes.jl @@ -206,14 +206,14 @@ end struct MissingInit end -provide_init(rf::AbstractReduction, init) = initvalue(init, InType(rf)) -function provide_init(rf::AbstractReduction, ::MissingInit) +provide_init(rf::Reduction, init) = initvalue(init, InType(rf)) +function provide_init(rf::Reduction, ::MissingInit) T = finaltype(rf) op = innermost_rf(rf) return identityof(op, T) end -innermost_rf(rf::AbstractReduction) = innermost_rf(inner(rf)) +innermost_rf(rf::Reduction) = innermost_rf(inner(rf)) innermost_rf(f) = f innermost_rf(o::Completing) = innermost_rf(o.f) @@ -221,7 +221,7 @@ innermost_rf(o::Completing) = innermost_rf(o.f) _start_init(rf, init) = start(rf, provide_init(rf, init)) # TODO: should it be an internal? -@inline function transduce(rf::AbstractReduction, init, coll) +@inline function transduce(rf::Reduction, init, coll) # Inlining `transduce` and `__foldl__` were essential for the # `darkritual` below to work. return __foldl__(rf, _start_init(rf, init), coll) @@ -460,14 +460,14 @@ end _map!(rf, coll, dest) = transduce(darkritual(rf, dest), nothing, coll) -# Deep-copy `AbstractReduction` so that compiler can treat the all +# Deep-copy `Reduction` so that compiler can treat the all # reducing function tree nodes as local variables (???). Aslo, it # tells compiler that `dest` is a local variable so that it won't # fetch `dest` via `getproperty` in each iteration. (This is too much # magic... My reasoning of how it works could be completely wrong. # But at least it should not change the semantics of the function.) @inline darkritual(rf::Reduction, dest) = - if inner(rf) isa AbstractReduction + if inner(rf) isa Reduction Reduction{InType(rf)}(xform(rf), darkritual(inner(rf), dest)) else @assert xform(rf).array === dest diff --git a/src/show.jl b/src/show.jl index ae24f8c445..8af9e7c23c 100644 --- a/src/show.jl +++ b/src/show.jl @@ -150,7 +150,7 @@ function Base.show(io::IO, xf::Transducer) return end -Base.show(io::IO, m::MIME"text/plain", rf::AbstractReduction) = +Base.show(io::IO, m::MIME"text/plain", rf::Reduction) = _show_impl(io, m, rf) function _show_impl(io, mime, x)