Merge branch 'master'

.github/workflows/benchmark.yml

@@ -10,7 +10,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: julia-actions/setup-julia@latest
         with:
-          version: 1.4
+          version: 1.5
       - name: Install dependencies
         run: julia -e 'using Pkg; pkg"add PkgBenchmark BenchmarkCI@0.1"'
       - name: Run benchmarks

.github/workflows/check-xfail.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        julia-version: ['1.4']
+        julia-version: ['^1']
       fail-fast: false
     name: Test xfail Julia ${{ matrix.julia-version }}
     steps:

.github/workflows/multi-thread-benchmark.yml

@@ -10,7 +10,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: julia-actions/setup-julia@latest
         with:
-          version: 1.4
+          version: 1.5
       - name: Install dependencies
         run: julia -e 'using Pkg; pkg"add PkgBenchmark BenchmarkCI@0.1"'
       - name: Run benchmarks

.github/workflows/vanilla-test-push.yml

@@ -1,4 +1,4 @@
-name: Run test via Pkg.test()
+name: Run test via Pkg.test() on push
 
 on:
   push:
@@ -7,15 +7,16 @@ on:
     paths:
       - Project.toml
       - test/environments/main/Project.toml
+  workflow_dispatch:
 
 jobs:
-  vanilla-test:
+  vanilla-test-push:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: julia-actions/setup-julia@v1
         with:
-          version: 1.4
+          version: ^1
       - uses: julia-actions/julia-buildpkg@latest
       - uses: julia-actions/julia-runtest@latest
         env:

.github/workflows/vanilla-test.yml

@@ -36,7 +36,7 @@ jobs:
       - uses: julia-actions/setup-julia@v1
         if: ${{ steps.check-project-toml.outputs.need_test == 'yes' }}
         with:
-          version: 1.4
+          version: ^1
       - uses: julia-actions/julia-buildpkg@latest
         if: ${{ steps.check-project-toml.outputs.need_test == 'yes' }}
       - uses: julia-actions/julia-runtest@latest

.travis.yml

@@ -3,8 +3,8 @@ language: julia
 os:
   - linux
 julia:
+  - 1.5  # to be used in benchmarks as well
   - 1.4
-  - 1.5
   - nightly
 env:
    global:

benchmark/Project.toml

@@ -9,6 +9,7 @@ Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a"
 MicroCollections = "128add7d-3638-4c79-886c-908ea0c25c34"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 PkgBenchmark = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 Referenceables = "42d2dcc6-99eb-4e98-b66c-637b7d73030e"
 SplitApplyCombine = "03a91e81-4c3e-53e1-a0a4-9c0c8f19dd66"
 SplittablesBase = "171d559e-b47b-412a-8079-5efa626c420e"

benchmark/bench_cartesian.jl

@@ -0,0 +1,41 @@
+module BenchCartesian
+
+using BenchmarkTools
+using Transducers
+
+function copyto_manual!(ys::AbstractMatrix, xs::AbstractMatrix)
+    @assert axes(ys) == axes(xs)
+    for j in 1:size(xs, 2), i in 1:size(xs, 1)
+        @inbounds ys[i, j] = xs[i, j]
+    end
+    return ys
+end
+
+function copyto_iter!(ys, xs)
+    @assert axes(ys) == axes(xs)
+    for I in CartesianIndices(xs)
+        @inbounds ys[I] = xs[I]
+    end
+    return ys
+end
+
+function copyto_xf!(ys, xs)
+    foreach(Map(identity), CartesianIndices(xs)) do I
+        @inbounds ys[I] = xs[I]
+        nothing
+    end
+    return ys
+end
+
+const SUITE = BenchmarkGroup()
+
+let xs = randn(3, 10^3)
+    ys = zero(xs)
+    s1 = SUITE["copyto!"] = BenchmarkGroup()
+    s1["man"] = @benchmarkable(copyto_manual!($ys, $xs))
+    s1["iter"] = @benchmarkable(copyto_iter!($ys, $xs))
+    s1["xf"] = @benchmarkable(copyto_xf!($ys, $xs))
+end
+
+end  # module
+BenchCartesian.SUITE

benchmark/bench_filter_sum.jl

@@ -0,0 +1,39 @@
+module BenchFilterSum
+
+import Random
+using BenchmarkTools
+using Transducers
+
+const SUITE = BenchmarkGroup()
+
+function naive_sum(xs, acc = false)
+    for x in xs
+        acc += x
+    end
+    return acc
+end
+
+Random.seed!(12345)
+for n in [1000, 10000]
+    s0 = SUITE[string(n)] = BenchmarkGroup()
+
+    for (xslabel, xs, init) in [
+        ("UnitRange", (x for x in 1:n if isodd(x)), 0),
+        ("RandomFloats", (x for x in randn(n) if x > 0), 0.0),
+    ]
+        s1 = s0[xslabel] = BenchmarkGroup()
+
+        s2 = s1["noinit"] = BenchmarkGroup()
+        s2["naive"] = @benchmarkable naive_sum($xs)
+        s2["base"] = @benchmarkable sum($xs)
+        s2["xf"] = @benchmarkable sum($(eduction(xs)))
+
+        s2 = s1["withinit"] = BenchmarkGroup()
+        s2["naive"] = @benchmarkable naive_sum($xs, $init)
+        s2["base"] = @benchmarkable foldl(+, $xs; init = $init)
+        s2["xf"] = @benchmarkable foldl(+, $(eduction(xs)); init = $init)
+    end
+end
+
+end  # module
+BenchFilterSum.SUITE

benchmark/bench_sum_transpose.jl

@@ -0,0 +1,42 @@
+module BenchSumTranspose
+
+import Random
+using BenchmarkTools
+using Transducers
+
+const SUITE = BenchmarkGroup()
+
+function iter_sum(xs, acc = false)
+    for x in xs
+        acc += x
+    end
+    return acc
+end
+
+function man_sum(xs::AbstractMatrix, acc = false)
+    for i in axes(xs, 2), j in axes(xs, 1)
+        acc += @inbounds xs[j, i]
+    end
+    return acc
+end
+
+Random.seed!(12345)
+# for n in [30, 100]
+let n = 30
+    s1 = SUITE[string(n)] = BenchmarkGroup()
+
+    xs = randn(n, n)'
+
+    s2 = s1["noinit"] = BenchmarkGroup()
+    s2["iter"] = @benchmarkable iter_sum($xs)
+    s2["man"] = @benchmarkable man_sum($xs)
+    s2["xf"] = @benchmarkable foldxl(+, $xs)
+
+    s2 = s1["withinit"] = BenchmarkGroup()
+    s2["iter"] = @benchmarkable iter_sum($xs, 0.0)
+    s2["man"] = @benchmarkable man_sum($xs, 0.0)
+    s2["xf"] = @benchmarkable foldxl(+, $xs; init = 0.0)
+end
+
+end  # module
+BenchSumTranspose.SUITE

benchmark/bench_teerf_filter.jl

@@ -0,0 +1,16 @@
+module BenchTeeRFFilter
+
+using BenchmarkTools
+using Transducers
+
+const SUITE = BenchmarkGroup()
+
+let xs = 1:1000
+    rf = TeeRF(Filter(isodd)'(+), Filter(iseven)'(+))
+
+    SUITE["noinit"] = @benchmarkable foldxl($rf, $xs)
+    SUITE["withinit"] = @benchmarkable foldxl($rf, $xs; init = (0, 0))
+end
+
+end  # module
+BenchTeeRFFilter.SUITE

src/basics.jl

@@ -95,3 +95,36 @@ abstract type _Function <: Function end
 Base.show(io::IO, ::MIME"text/plain", f::_Function) = show(io, f)
 Base.print(io::IO, f::_Function) = show(io, f)
 @specialize
+
+
+# A macro for "manual Union splitting".  It is sometimes useful to let
+# the compiler know that it is beneficial to type-specialize `body`.
+# * https://github.com/JuliaFolds/Transducers.jl/pull/188
+# * https://github.com/JuliaLang/julia/pull/34293#discussion_r363550608
+macro manual_union_split(cond, body)
+    quote
+        if $cond
+            $body
+        else
+            $body
+        end
+    end |> esc
+end
+
+@inline _firstindex(arr) = firstindex(arr)
+@inline _lastindex(arr) = lastindex(arr)
+
+# Define `firstindex` and `lastindex` for `Broadcasted` with linear
+# index style:
+@inline _firstindex(bc::Broadcasted) = first((axes(bc)::Tuple{Any})[1])
+@inline _lastindex(bc::Broadcasted) = last((axes(bc)::Tuple{Any})[1])
+
+# Define `CartesianIndices` for `Broadcasted`
+@inline _CartesianIndices(arr) = CartesianIndices(arr)
+@inline _CartesianIndices(bc::Broadcasted) = CartesianIndices(axes(bc)::Tuple)
+
+# Define `IndexStyle` for `Broadcasted`
+_IndexStyle(arr) = IndexStyle(arr)
+_IndexStyle(bc::Broadcasted) = _IndexStyle(typeof(bc))
+_IndexStyle(::Type{<:Broadcasted{<:Any,<:Tuple{Any}}}) = IndexLinear()
+_IndexStyle(::Type{<:Broadcasted{<:Any}}) = IndexCartesian()

src/core.jl

@@ -324,6 +324,7 @@ end
 
 prependxf(rf::AbstractReduction, xf) = Reduction(xf, rf)
 setinner(rf::Reduction, inner) = Reduction(xform(rf), inner)
+setxform(rf::Reduction, xform) = Reduction(xform, inner(rf))
 
 Transducer(rf::Reduction) =
     if inner(rf) isa BottomRF
@@ -564,6 +565,11 @@ combine(rf::Reduction, a, b) =
         combine(inner(rf), a, b)
     end
 
+is_prelude(_) = false
+is_prelude(::InitialValues.InitialValue) = true
+is_prelude(xs::Tuple) = any(map(is_prelude, xs))
+is_prelude(xs::NamedTuple) = is_prelude(Tuple(xs))
+
 privatestate(::T, state, result) where {T <: AbstractReduction} =
     privatestate(T, state, result)
 
@@ -597,6 +603,8 @@ ownsstate(::R, ::PrivateState{T}) where {R, T} = R === T
 # took more than 10 min).  See also:
 # https://github.com/JuliaLang/julia/issues/30125
 
+@inline is_prelude(ps::PrivateState) = is_prelude(psstate(ps)) || is_prelude(psresult(ps))
+
 """
     unwrap(rf, result)
 

src/library.jl

@@ -237,15 +237,16 @@ end
 
 next(rf::R_{TCat}, result, input) =
     wrapping(rf, result) do init, acc
+        rfi, itr = retransform(inner(rf), input)
         subresult = _transduce_assoc_nocomplete(
-            inner(rf),
+            rfi,
             init,
-            input,
+            itr,
             xform(rf).basesize,
         )
         subresult isa Reduced && return init, subresult
         acc isa Unseen && return init, subresult
-        return init, combine(inner(rf), acc, subresult)
+        return init, combine(rfi, acc, subresult)
     end
 
 function combine(rf::R_{TCat}, a, b)
@@ -926,6 +927,7 @@ struct PartitionBy{F} <: Transducer
 end
 
 struct Unseen end
+is_prelude(::Unseen) = true
 
 isexpansive(::PartitionBy) = false