Merge pull request #24187 from stevengj/reverseiter

Iterators.Reverse type for reverse-order iteration

NEWS.md

@@ -280,6 +280,9 @@ Library improvements
     For example, `x^-1` is now essentially a synonym for `inv(x)`, and works
     in a type-stable way even if `typeof(x) != typeof(inv(x))` ([#24240]).
 
+  * New `Iterators.reverse(itr)` for reverse-order iteration ([#24187]).  Iterator
+    types `T` can implement `start` etc. for `Iterators.Reverse{T}` to support this.
+
   * The functions `nextind` and `prevind` now accept `nchar` argument that indicates
     the number of characters to move ([#23805]).
 

base/array.jl

@@ -620,7 +620,7 @@ function _collect(cont, itr, ::HasEltype, isz::SizeUnknown)
     return a
 end
 
-_collect_indices(::Tuple{}, A) = copy!(Vector{eltype(A)}(), A)
+_collect_indices(::Tuple{}, A) = copy!(Array{eltype(A)}(), A)
 _collect_indices(indsA::Tuple{Vararg{OneTo}}, A) =
     copy!(Array{eltype(A)}(length.(indsA)), A)
 function _collect_indices(indsA, A)
@@ -1450,7 +1450,8 @@ end
 """
     reverse(v [, start=1 [, stop=length(v) ]] )
 
-Return a copy of `v` reversed from start to stop.
+Return a copy of `v` reversed from start to stop.  See also [`Iterators.reverse`](@ref)
+for reverse-order iteration without making a copy.
 
 # Examples
 ```jldoctest

base/iterators.jl

@@ -5,10 +5,10 @@ Methods for working with Iterators.
 """
 module Iterators
 
-import Base: start, done, next, isempty, length, size, eltype, iteratorsize, iteratoreltype, indices, ndims, pairs
+import Base: start, done, next, isempty, length, size, eltype, iteratorsize, iteratoreltype, indices, ndims, pairs, last, first
 
 using Base: tail, tuple_type_head, tuple_type_tail, tuple_type_cons, SizeUnknown, HasLength, HasShape,
-            IsInfinite, EltypeUnknown, HasEltype, OneTo, @propagate_inbounds
+            IsInfinite, EltypeUnknown, HasEltype, OneTo, @propagate_inbounds, Generator, AbstractRange
 
 export enumerate, zip, rest, countfrom, take, drop, cycle, repeated, product, flatten, partition
 
@@ -30,6 +30,52 @@ and_iteratorsize(a, b) = SizeUnknown()
 and_iteratoreltype(iel::T, ::T) where {T} = iel
 and_iteratoreltype(a, b) = EltypeUnknown()
 
+## Reverse-order iteration for arrays and other collections.  Collections
+## should implement start/next/done etcetera if possible/practical.
+"""
+    Iterators.reverse(itr)
+
+Given an iterator `itr`, then `reverse(itr)` is an iterator over the
+same collection but in the reverse order.
+
+This iterator is "lazy" in that it does not make a copy of the collection in
+order to reverse it; see [`Base.reverse`](@ref) for an eager implementation.
+
+Not all iterator types `T` support reverse-order iteration.  If `T`
+doesn't, then iterating over `Iterators.reverse(itr::T)` will throw a [`MethodError`](@ref)
+because of the missing [`start`](@ref), [`next`](@ref), and [`done`](@ref)
+methods for `Iterators.Reverse{T}`.  (To implement these methods, the original iterator
+`itr::T` can be obtained from `r = Iterators.reverse(itr)` by `r.itr`.)
+"""
+reverse(itr) = Reverse(itr)
+
+struct Reverse{T}
+    itr::T
+end
+eltype(r::Reverse) = eltype(r.itr)
+length(r::Reverse) = length(r.itr)
+size(r::Reverse) = size(r.itr)
+iteratorsize(r::Reverse) = iteratorsize(r.itr)
+iteratoreltype(r::Reverse) = iteratoreltype(r.itr)
+last(r::Reverse) = first(r.itr) # the first shall be last
+first(r::Reverse) = last(r.itr) # and the last shall be first
+
+# reverse-order array iterators: assumes more-specialized Reverse for eachindex
+@inline start(A::Reverse{<:AbstractArray}) = (itr = reverse(eachindex(A.itr)); (itr, start(itr)))
+@propagate_inbounds next(A::Reverse{<:AbstractArray}, i) = ((idx, s) = next(i[1], i[2]); (A.itr[idx], (i[1], s)))
+@propagate_inbounds done(A::Reverse{<:AbstractArray}, i) = done(i[1], i[2])
+
+reverse(R::AbstractRange) = Base.reverse(R) # copying ranges is cheap
+reverse(G::Generator) = Generator(G.f, reverse(G.iter))
+reverse(r::Reverse) = r.itr
+reverse(x::Union{Number,Char}) = x
+reverse(p::Pair) = Base.reverse(p) # copying pairs is cheap
+
+start(r::Reverse{<:Tuple}) = length(r.itr)
+done(r::Reverse{<:Tuple}, i::Int) = i < 1
+next(r::Reverse{<:Tuple}, i::Int) = (r.itr[i], i-1)
+
+
 # enumerate
 
 struct Enumerate{I}
@@ -75,6 +121,16 @@ eltype(::Type{Enumerate{I}}) where {I} = Tuple{Int, eltype(I)}
 iteratorsize(::Type{Enumerate{I}}) where {I} = iteratorsize(I)
 iteratoreltype(::Type{Enumerate{I}}) where {I} = iteratoreltype(I)
 
+@inline function start(r::Reverse{<:Enumerate})
+    ri = reverse(r.itr.itr)
+    return (length(ri), ri, start(ri))
+end
+@inline function next(r::Reverse{<:Enumerate}, state)
+    n = next(state[2],state[3])
+    (state[1],n[1]), (state[1]-1,state[2],n[2])
+end
+@inline done(r::Reverse{<:Enumerate}, state) = state[1] < 1
+
 struct IndexValue{I,A<:AbstractArray}
     data::A
     itr::I
@@ -147,6 +203,8 @@ eltype(::Type{IndexValue{I,A}}) where {I,A} = Pair{eltype(I), eltype(A)}
 iteratorsize(::Type{IndexValue{I}}) where {I} = iteratorsize(I)
 iteratoreltype(::Type{IndexValue{I}}) where {I} = iteratoreltype(I)
 
+reverse(v::IndexValue) = IndexValue(v.data, reverse(v.itr))
+
 # zip
 
 abstract type AbstractZipIterator end
@@ -246,6 +304,10 @@ end
 iteratorsize(::Type{Zip{I1,I2}}) where {I1,I2} = zip_iteratorsize(iteratorsize(I1),iteratorsize(I2))
 iteratoreltype(::Type{Zip{I1,I2}}) where {I1,I2} = and_iteratoreltype(iteratoreltype(I1),iteratoreltype(I2))
 
+reverse(z::Zip1) = Zip1(reverse(z.a))
+reverse(z::Zip2) = Zip2(reverse(z.a), reverse(z.b))
+reverse(z::Zip) = Zip(reverse(z.a), reverse(z.z))
+
 # filter
 
 struct Filter{F,I}
@@ -313,6 +375,8 @@ eltype(::Type{Filter{F,I}}) where {F,I} = eltype(I)
 iteratoreltype(::Type{Filter{F,I}}) where {F,I} = iteratoreltype(I)
 iteratorsize(::Type{<:Filter}) = SizeUnknown()
 
+reverse(f::Filter) = Filter(f.flt, reverse(f.itr))
+
 # Rest -- iterate starting at the given state
 
 struct Rest{I,S}
@@ -346,7 +410,6 @@ rest_iteratorsize(a) = SizeUnknown()
 rest_iteratorsize(::IsInfinite) = IsInfinite()
 iteratorsize(::Type{Rest{I,S}}) where {I,S} = rest_iteratorsize(iteratorsize(I))
 
-
 # Count -- infinite counting
 
 struct Count{S<:Number}
@@ -539,6 +602,7 @@ end
 
 done(it::Cycle, state) = state[2]
 
+reverse(it::Cycle) = Cycle(reverse(it.xs))
 
 # Repeated - repeat an object infinitely many times
 
@@ -576,6 +640,7 @@ done(it::Repeated, state) = false
 iteratorsize(::Type{<:Repeated}) = IsInfinite()
 iteratoreltype(::Type{<:Repeated}) = HasEltype()
 
+reverse(it::Union{Repeated,Take{<:Repeated}}) = it
 
 # Product -- cartesian product of iterators
 struct ProductIterator{T<:Tuple}
@@ -706,6 +771,8 @@ function _prod_next(iterators, states, nvalues)
     end
 end
 
+reverse(p::ProductIterator) = ProductIterator(map(reverse, p.iterators))
+
 # flatten an iterator of iterators
 
 struct Flatten{I}
@@ -781,6 +848,7 @@ end
     return done(f.it, s) && done(inner, s2)
 end
 
+reverse(f::Flatten) = Flatten(reverse(itr) for itr in reverse(f.it))
 
 """
     partition(collection, n)

base/multidimensional.jl

@@ -9,6 +9,7 @@ module IteratorsMD
     import Base: +, -, *
     import Base: simd_outer_range, simd_inner_length, simd_index
     using Base: IndexLinear, IndexCartesian, AbstractCartesianIndex, fill_to_length, tail
+    using Base.Iterators: Reverse
 
     export CartesianIndex, CartesianRange
 
@@ -314,6 +315,33 @@ module IteratorsMD
         i, j = split(R.indices, V)
         CartesianRange(i), CartesianRange(j)
     end
+
+    # reversed CartesianRange iteration
+    @inline function start(r::Reverse{<:CartesianRange})
+        iterfirst, iterlast = last(r.itr), first(r.itr)
+        if any(map(<, iterfirst.I, iterlast.I))
+            return iterlast-1
+        end
+        iterfirst
+    end
+    @inline function next(r::Reverse{<:CartesianRange}, state)
+        state, CartesianIndex(dec(state.I, last(r.itr).I, first(r.itr).I))
+    end
+    # decrement & carry
+    @inline dec(::Tuple{}, ::Tuple{}, ::Tuple{}) = ()
+    @inline dec(state::Tuple{Int}, start::Tuple{Int}, stop::Tuple{Int}) = (state[1]-1,)
+    @inline function dec(state, start, stop)
+        if state[1] > stop[1]
+            return (state[1]-1,tail(state)...)
+        end
+        newtail = dec(tail(state), tail(start), tail(stop))
+        (start[1], newtail...)
+    end
+    @inline done(r::Reverse{<:CartesianRange}, state) = state.I[end] < first(r.itr.indices[end])
+    # 0-d cartesian ranges are special-cased to iterate once and only once
+    start(iter::Reverse{<:CartesianRange{0}}) = false
+    next(iter::Reverse{<:CartesianRange{0}}, state) = CartesianIndex(), true
+    done(iter::Reverse{<:CartesianRange{0}}, state) = state
 end  # IteratorsMD
 
 

base/strings/basic.jl

@@ -638,3 +638,11 @@ function last(str::AbstractString, nchar::Integer)
     end
     str[prevind(str, e, nchar-1):e]
 end
+
+# reverse-order iteration for strings and indices thereof
+start(r::Iterators.Reverse{<:AbstractString}) = endof(r.itr)
+done(r::Iterators.Reverse{<:AbstractString}, i) = i < start(r.itr)
+next(r::Iterators.Reverse{<:AbstractString}, i) = (r.itr[i], prevind(r.itr, i))
+start(r::Iterators.Reverse{<:EachStringIndex}) = endof(r.itr.s)
+done(r::Iterators.Reverse{<:EachStringIndex}, i) = i < start(r.itr.s)
+next(r::Iterators.Reverse{<:EachStringIndex}, i) = (i, prevind(r.itr.s, i))

base/strings/types.jl

@@ -134,6 +134,7 @@ main utility is for reversed-order string processing, especially for reversed
 regular-expression searches.  See also [`reverseind`](@ref) to convert indices
 in `s` to indices in `reverse(s)` and vice-versa, and [`graphemes`](@ref)
 to operate on user-visible "characters" (graphemes) rather than codepoints.
+See also [`Iterators.reverse`](@ref) for reverse-order iteration without making a copy.
 
 # Examples
 ```jldoctest

doc/src/manual/interfaces.md

@@ -136,6 +136,25 @@ define an informal interface that enable many fancier behaviors. In some cases,
 to additionally specialize those extra behaviors when they know a more efficient algorithm can
 be used in their specific case.
 
+It is also often useful to allow iteration over a collection in *reverse order*
+by iterating over [`Iterators.reverse(iterator)`](@ref).  To actually support
+reverse-order iteration, however, an iterator
+type `T` needs to implement `start`, `next`, and `done` methods for `Iterators.Reverse{T}`.
+(Given `r::Iterators.Reverse{T}`, the underling iterator of type `T` is `r.itr`.)
+In our `Squares` example, we would implement `Iterators.Reverse{Squares}` methods:
+
+```jldoctest squaretype
+julia> Base.start(rS::Iterators.Reverse{Squares}) = rS.itr.count
+
+julia> Base.next(::Iterators.Reverse{Squares}, state) = (state*state, state-1)
+
+julia> Base.done(::Iterators.Reverse{Squares}, state) = state < 1
+
+julia> collect(Iterators.reverse(Squares(10)))' # transposed to save space
+1×10 RowVector{Int64,Array{Int64,1}}:
+ 100  81  64  49  36  25  16  9  4  1
+```
+
 ## Indexing
 
 | Methods to implement | Brief description                |

doc/src/stdlib/iterators.md

@@ -12,4 +12,6 @@ Base.Iterators.repeated
 Base.Iterators.product
 Base.Iterators.flatten
 Base.Iterators.partition
+Base.Iterators.filter
+Base.Iterators.reverse
 ```

test/abstractarray.jl

@@ -872,3 +872,8 @@ end
         copy!(Array{T,n}(size(a)), a)
     @test isa(similar(Dict(:a=>1, :b=>2.0), Pair{Union{},Union{}}), Dict{Union{}, Union{}})
 end
+
+@testset "zero-dimensional copy" begin
+    Z = Array{Int}(); Z[] = 17
+    @test Z == collect(Z) == copy(Z)
+end

test/iterators.jl

@@ -429,3 +429,21 @@ end
     @test length(arr) == 0
     @test eltype(arr) == Int
 end
+
+@testset "reverse iterators" begin
+    squash(A) = reshape(A, length(A))
+    Z = Array{Int}(); Z[] = 17 # zero-dimensional test case
+    for itr in (2:10, "∀ϵ>0", 1:0, "", (2,3,5,7,11), [2,3,5,7,11], rand(5,6), Z, 3, true, 'x', 4=>5,
+                eachindex("∀ϵ>0"), view(Z), view(rand(5,6),2:4,2:6), (x^2 for x in 1:10),
+                Iterators.Filter(isodd, 1:10), flatten((1:10, 50:60)), enumerate("foo"),
+                pairs(50:60), zip(1:10,21:30,51:60), product(1:3, 10:12), repeated(3.14159, 5))
+        @test squash(collect(Iterators.reverse(itr))) == reverse(squash(collect(itr)))
+    end
+    @test collect(take(Iterators.reverse(cycle(1:3)), 7)) == collect(take(cycle(3:-1:1), 7))
+    let r = repeated(3.14159)
+        @test Iterators.reverse(r) === r
+    end
+    let t = (2,3,5,7,11)
+        @test Iterators.reverse(Iterators.reverse(t)) === t
+    end
+end