# JuliaLang/julia

say hello to tuple comprehensions (x for i in y)

```but, you still shouldn't make very long tuples

remove some unnecessary uses of ntuple()
if we like the new way, it will be fully deprecated```
1 parent 93b8c23 commit f988e0eeb89f1d45ba1b92e8e867f8e58b02e391 JeffBezanson committed Oct 7, 2012
Showing with 75 additions and 63 deletions.
1. +7 −12 base/abstractarray.jl
2. +1 −1 base/array.jl
3. +19 −31 base/darray.jl
4. +4 −5 base/inference.jl
5. +1 −1 base/operators.jl
6. +1 −1 base/pcre.jl
7. +1 −1 base/regex.jl
8. +3 −2 base/subarray.jl
9. +4 −4 base/tuple.jl
10. +1 −1 extras/sparse.jl
11. +19 −3 src/builtins.c
12. +6 −1 src/julia-parser.scm
13. +8 −0 src/julia-syntax.scm
19 base/abstractarray.jl
 @@ -359,7 +359,7 @@ ref(t::AbstractArray, r::Real...) = ref(t,map(to_index,r)...) # index A[:,:,...,i,:,:,...] where "i" is in dimension "d" # TODO: more optimized special cases slicedim(A::AbstractArray, d::Integer, i) = - A[ntuple(ndims(A), n->(n==d ? i : (1:size(A,n))))...] + A[[ n==d ? i : (1:size(A,n)) for n in 1:ndims(A) ]...] function flipdim(A::AbstractArray, d::Integer) nd = ndims(A) @@ -379,12 +379,9 @@ function flipdim(A::AbstractArray, d::Integer) end return B end - alli = ntuple(nd, n->(n==d ? 0 : (1:size(B,n)))) - local ri - b_ind = n->(n==d ? ri : alli[n]) + alli = [ 1:size(B,n) for n in 1:nd ] for i = 1:sd - ri = sd+1-i - B[ntuple(nd, b_ind)...] = slicedim(A, d, i) + B[[ n==d ? sd+1-i : alli[n] for n in 1:nd ]...] = slicedim(A, d, i) end return B end @@ -531,7 +528,7 @@ function cat(catdim::Integer, X...) end end - cat_ranges = ntuple(nargs, i->(catdim <= ndimsX[i] ? dimsX[i][catdim] : 1)) + cat_ranges = [ catdim <= ndimsX[i] ? dimsX[i][catdim] : 1 for i=1:nargs ] function compute_dims(d) if d == catdim @@ -557,8 +554,7 @@ function cat(catdim::Integer, X...) range = 1 for k=1:nargs nextrange = range+cat_ranges[k] - cat_one = ntuple(ndimsC, i->(i != catdim ? - (1:dimsC[i]) : (range:nextrange-1) )) + cat_one = [ i != catdim ? (1:dimsC[i]) : (range:nextrange-1) for i=1:ndimsC ] C[cat_one...] = X[k] range = nextrange end @@ -594,7 +590,7 @@ function cat(catdim::Integer, A::AbstractArray...) end end - cat_ranges = ntuple(nargs, i->(catdim <= ndimsA[i] ? dimsA[i][catdim] : 1)) + cat_ranges = [ catdim <= ndimsA[i] ? dimsA[i][catdim] : 1 for i=1:nargs ] function compute_dims(d) if d == catdim @@ -620,8 +616,7 @@ function cat(catdim::Integer, A::AbstractArray...) range = 1 for k=1:nargs nextrange = range+cat_ranges[k] - cat_one = ntuple(ndimsC, i->(i != catdim ? - (1:dimsC[i]) : (range:nextrange-1) )) + cat_one = [ i != catdim ? (1:dimsC[i]) : (range:nextrange-1) for i=1:ndimsC ] C[cat_one...] = A[k] range = nextrange end
2 base/array.jl
 @@ -1761,7 +1761,7 @@ function permute(A::StridedArray, perm) ndimsA = length(dimsA) dimsP = ntuple(ndimsA, i->dimsA[perm[i]]) P = similar(A, dimsP) - ranges = ntuple(ndimsA, i->(colon(1,dimsP[i]))) + ranges = ntuple(ndimsA, i->(1:dimsP[i])) while length(stridenames) < ndimsA push(stridenames, gensym()) end
50 base/darray.jl
 @@ -477,7 +477,7 @@ function _jl_da_sub(d::DArray, I::Range1{Int}...) offs = d.dist[d.localpiece]-1 J = ntuple(ndims(d), i -> (i == d.distdim ? I[i]-offs : I[i])) - return sub(localize(d), J...) + return sub(localize(d), J) end # Nd ref with Range1 indexes @@ -486,15 +486,13 @@ function ref{T}(d::DArray{T}, I::Range1{Int}...) np = length(pmap) if np == 1 && pmap[1] == d.localpiece offs = d.dist[pmap[1]]-1 - J = ntuple(ndims(d), i -> (i == d.distdim ? I[i]-offs : - I[i])) + J = [ i == d.distdim ? I[i]-offs : I[i] for i=1:ndims(d) ] return localize(d)[J...] end A = Array(T, map(length, I)) deps = cell(np) for p = 1:np - K = ntuple(ndims(d),i->(i==d.distdim ? (dist[p]:(dist[p+1]-1)) : - I[i])) + K = [ i==d.distdim ? (dist[p]:(dist[p+1]-1)) : I[i] for i=1:ndims(d) ] if np == 1 # use remote_call_fetch if we only need to communicate with 1 proc deps[p] = remote_call_fetch(d.pmap[pmap[p]], _jl_da_sub, d, K...) @@ -504,8 +502,7 @@ function ref{T}(d::DArray{T}, I::Range1{Int}...) end for p = 1:np offs = I[d.distdim][1] - 1 - J = ntuple(ndims(d),i->(i==d.distdim ? (dist[p]:(dist[p+1]-1))-offs : - (1:length(I[i])))) + J = [ i==d.distdim ? (dist[p]:(dist[p+1]-1))-offs : (1:length(I[i])) for i=1:ndims(d) ] A[J...] = fetch(deps[p]) end return A @@ -516,7 +513,7 @@ ref(d::DArray, I::Range1{Int}, j::Int) = d[I, j:j] ref(d::DArray, i::Int, J::Range1{Int}) = d[i:i, J] ref(d::DArray, I::Union(Int,Range1{Int})...) = - d[ntuple(length(I),i->(isa(I[i],Int) ? (I[i]:I[i]) : I[i] ))...] + d[[isa(i,Int) ? (i:i) : i for i in I ]...] # Nd ref with vector indexes @@ -525,8 +522,7 @@ function ref{T}(d::DArray{T}, I::AbstractVector{Int}...) np = length(pmap) if np == 1 && pmap[1] == d.localpiece offs = d.dist[pmap[1]]-1 - J = ntuple(ndims(d), i -> (i == d.distdim ? I[i]-offs : - I[i])) + J = [ i == d.distdim ? I[i]-offs : I[i] for i=1:ndims(d) ] return localize(d)[J...] end A = Array(T, map(length, I)) @@ -540,8 +536,7 @@ function ref{T}(d::DArray{T}, I::AbstractVector{Int}...) while j <= n && II[j] < dist[p+1] j += 1 end - K = ntuple(ndims(d),i->(i==d.distdim ? II[lower:(j-1)] : - I[i])) + K = [ i==d.distdim ? II[lower:(j-1)] : I[i] for i=1:ndims(d) ] if np == 1 deps[p] = remote_call_fetch(d.pmap[pmap[p]], ref, d, K...) else @@ -555,8 +550,7 @@ function ref{T}(d::DArray{T}, I::AbstractVector{Int}...) while j <= n && II[j] < dist[p+1] j += 1 end - J = ntuple(ndims(d),i->(i==d.distdim ? perm[lower:(j-1)] : - (1:length(I[i])))) + J = [ i==d.distdim ? perm[lower:(j-1)] : (1:length(I[i])) for i=1:ndims(d) ] A[J...] = fetch(deps[p]) end return A @@ -567,7 +561,7 @@ ref(d::DArray, I::AbstractVector{Int}, j::Int) = d[I, [j]] ref(d::DArray, i::Int, J::AbstractVector{Int}) = d[[i], J] ref(d::DArray, I::Union(Int,AbstractVector{Int})...) = - d[ntuple(length(I),i->(isa(I[i],Int) ? [I[i]] : I[i] ))...] + d[[isa(i,Int) ? [i] : i for i in I]...] # Nd scalar assign function assign_elt(d::DArray, v, sub::(Int...)) @@ -596,14 +590,12 @@ function assign(d::DArray, v, I::Range1{Int}...) (pmap, dist) = locate(d, I[d.distdim]) if length(pmap) == 1 && pmap[1] == d.localpiece offs = d.dist[pmap[1]]-1 - J = ntuple(ndims(d), i -> (i == d.distdim ? I[i]-offs : - I[i])) + J = [ i == d.distdim ? I[i]-offs : I[i] for i=1:ndims(d) ] localize(d)[J...] = v return d end for p = 1:length(pmap) - K = ntuple(ndims(d),i->(i==d.distdim ? (dist[p]:(dist[p+1]-1)) : - I[i])) + K = [ i==d.distdim ? (dist[p]:(dist[p+1]-1)) : I[i] for i=1:ndims(d) ] sync_add(remote_call(d.pmap[pmap[p]], assign, d, v, K...)) end return d @@ -615,8 +607,7 @@ function assign(d::DArray, v::AbstractArray, I::Range1{Int}...) (pmap, dist) = locate(d, I[d.distdim]) if length(pmap) == 1 && pmap[1] == d.localpiece offs = d.dist[pmap[1]]-1 - J = ntuple(ndims(d), i -> (i == d.distdim ? I[i]-offs : - I[i])) + J = [ i == d.distdim ? I[i]-offs : I[i] for i=1:ndims(d) ] localize(d)[J...] = v return d end @@ -629,7 +620,7 @@ function assign(d::DArray, v::AbstractArray, I::Range1{Int}...) (1:length(I[i])))) K = ntuple(ndims(d),i->(i==d.distdim ? (dist[p]:(dist[p+1]-1)) : I[i])) - sync_add(remote_call(d.pmap[pmap[p]], assign, d, sub(v,J...), K...)) + sync_add(remote_call(d.pmap[pmap[p]], assign, d, sub(v,J), K...)) end return d end @@ -639,8 +630,7 @@ function assign(d::DArray, v, I::AbstractVector{Int}...) (pmap, dist, perm) = locate(d, I[d.distdim]) if length(pmap) == 1 && pmap[1] == d.localpiece offs = d.dist[pmap[1]]-1 - J = ntuple(ndims(d), i -> (i == d.distdim ? I[i]-offs : - I[i])) + J = [ i == d.distdim ? I[i]-offs : I[i] for i=1:ndims(d) ] localize(d)[J...] = v return d end @@ -653,8 +643,7 @@ function assign(d::DArray, v, I::AbstractVector{Int}...) while j <= n && II[j] < dist[p+1] j += 1 end - K = ntuple(ndims(d),i->(i==d.distdim ? II[lower:(j-1)] : - I[i])) + K = [ i==d.distdim ? II[lower:(j-1)] : I[i] for i=1:ndims(d) ] sync_add(remote_call(d.pmap[pmap[p]], assign, d, v, K...)) end return d @@ -666,8 +655,7 @@ function assign(d::DArray, v::AbstractArray, I::AbstractVector{Int}...) (pmap, dist, perm) = locate(d, I[d.distdim]) if length(pmap) == 1 && pmap[1] == d.localpiece offs = d.dist[pmap[1]]-1 - J = ntuple(ndims(d), i -> (i == d.distdim ? I[i]-offs : - I[i])) + J = [ i == d.distdim ? I[i]-offs : I[i] for i=1:ndims(d) ] localize(d)[J...] = v return d end @@ -687,18 +675,18 @@ function assign(d::DArray, v::AbstractArray, I::AbstractVector{Int}...) (1:length(I[i])))) K = ntuple(ndims(d),i->(i==d.distdim ? II[lower:(j-1)] : I[i])) - sync_add(remote_call(d.pmap[pmap[p]], assign, d, sub(v,J...), K...)) + sync_add(remote_call(d.pmap[pmap[p]], assign, d, sub(v,J), K...)) end return d end # assign with combinations of Range1 and scalar indexes assign(d::DArray, v, I::Union(Int,Range1{Int})...) = - assign(d,v,ntuple(length(I),i->(isa(I[i],Int) ? (I[i]:I[i]) : I[i] ))...) + assign(d, v, [isa(i,Int) ? (i:i) : i for i in I]...) # assign with combinations of vector and scalar indexes assign(d::DArray, v, I::Union(Int,AbstractVector{Int})...) = - assign(d,v,ntuple(length(I),i->(isa(I[i],Int) ? [I[i]] : I[i] ))...) + assign(d, v, [isa(i,Int) ? [i] : i for i in I]...) ## matrix multiply ##
9 base/inference.jl
 @@ -572,7 +572,7 @@ end function abstract_eval_call(e, vtypes, sv::StaticVarInfo) fargs = e.args[2:] - argtypes = ntuple(length(fargs), i->abstract_eval(fargs[i],vtypes,sv)) + argtypes = (abstract_eval(fargs[i],vtypes,sv) for i=1:length(fargs)) if anyp(x->is(x,None), argtypes) return None end @@ -1266,7 +1266,7 @@ function type_annotate(ast::Expr, states::Array{Any,1}, sv::ANY, rettype::ANY, end end na = length(a.args[1]) - typeinf(li, ntuple(na+1, i->(i>na ? Tuple[1] : Any)), + typeinf(li, (i>na ? (Tuple)[1] : Any for i=1:na+1), li.sparams, li, false) end end @@ -1427,7 +1427,7 @@ function inlineable(f, e::Expr, sv, enclosing_ast) return NF end argexprs = e.args[2:] - atypes = limit_tuple_type(ntuple(length(argexprs), i->exprtype(argexprs[i]))) + atypes = limit_tuple_type((exprtype(ae) for ae in argexprs)) if is(f, convert_default) && length(atypes)==3 # builtin case of convert. convert(T,x::S) => x, when S<:T @@ -1552,8 +1552,7 @@ function mk_tupleref(texpr, i) end function mk_tuplecall(args) - Expr(:call1, {_jl_top_tuple, args...}, - ntuple(length(args), i->exprtype(args[i]))) + Expr(:call1, {_jl_top_tuple, args...}, (exprtype(a) for a in args)) end function inlining_pass(e::Expr, sv, ast)
2 base/operators.jl
 @@ -180,7 +180,7 @@ end function ref_shape(I...) n = length(I) while n > 0 && isa(I[n],Real); n-=1; end - ntuple(n, i->length(I[i])) + (length(I[i]) for i=1:n) end ref_shape(i::Real) = ()
2 base/pcre.jl
 @@ -97,7 +97,7 @@ end study(re::Array{Uint8}) = study(re, int32(0)) function exec(regex::Array{Uint8}, extra::Ptr{Void}, - str::ByteString, offset::Integer, options::Integer, cap::Bool) + str::ByteString, offset::Integer, options::Integer, cap::Bool) if offset < 0 || length(str) < offset error("index out of range") end
2 base/regex.jl
 @@ -96,7 +96,7 @@ function match(re::Regex, str::ByteString, idx::Integer, opts::Integer) if isempty(m); return nothing; end mat = str[m[1]+1:m[2]] cap = ntuple(n, i->(m[2i+1] < 0 ? nothing : str[m[2i+1]+1:m[2i+2]])) - off = map(i->m[2i+1]+1, [1:n]) + off = [ m[2i+1]::Int32+1 for i=1:n ] RegexMatch(mat, cap, m[1]+1, off) end match(r::Regex, s::String, i::Integer, o::Integer) = match(r, bytestring(s), i, o)
5 base/subarray.jl
 @@ -55,8 +55,9 @@ function sub{T,N}(A::AbstractArray{T,N}, i::NTuple{N,RangeIndex}) i = ntuple(length(i), k->(k<=L ? i0[k] : i[k])) SubArray{T,L,typeof(A),typeof(i)}(A, i) end -sub(A::AbstractArray, i::RangeIndex...) = - sub(A, i) + +sub(A::AbstractArray, i::RangeIndex...) = sub(A, i) + function sub(A::SubArray, i::RangeIndex...) L = length(i) while L > 0 && isa(i[L], Int); L-=1; end
8 base/tuple.jl
 @@ -4,7 +4,7 @@ length(t::Tuple) = tuplelen(t) size(t::Tuple, d) = d==1 ? tuplelen(t) : error("invalid tuple dimension") ref(t::Tuple, i::Int) = tupleref(t, i) ref(t::Tuple, i::Integer) = tupleref(t, int(i)) -ref(t::Tuple, r::Ranges) = ntuple(length(r), i->t[r[i]]) +ref(t::Tuple, r::Ranges) = (t[ri] for ri in r) ## iterating ## @@ -30,7 +30,7 @@ map(f, t::(Any,)) = (f(t[1]),) map(f, t::(Any, Any)) = (f(t[1]), f(t[2])) map(f, t::(Any, Any, Any)) = (f(t[1]), f(t[2]), f(t[3])) map(f, t::(Any, Any, Any, Any)) = (f(t[1]), f(t[2]), f(t[3]), f(t[4])) -map(f, t::Tuple) = ntuple(length(t), i->f(t[i])) +map(f, t::Tuple) = (f(ti) for ti in t) # 2 argument function map(f, t::(), s::()) = () map(f, t::(Any,), s::(Any,)) = (f(t[1],s[1]),) @@ -40,7 +40,7 @@ map(f, t::(Any,Any,Any), s::(Any,Any,Any)) = map(f, t::(Any,Any,Any,Any), s::(Any,Any,Any,Any)) = (f(t[1],s[1]), f(t[2],s[2]), f(t[3],s[3]), f(t[4],s[4])) # n argument function -map(f, ts::Tuple...) = ntuple(length_checked_equal(ts...), n->f(map(t->t[n],ts)...)) +map(f, ts::Tuple...) = (f(map(t->t[n],ts)...) for n=1:length_checked_equal(ts...)) function length_checked_equal(args...) n = length(args[1]) @@ -82,4 +82,4 @@ end isempty(x::()) = true isempty(x::Tuple) = false -reverse(x::Tuple) = (n=length(x); ntuple(n, k->x[n-k+1])) +reverse(x::Tuple) = (n=length(x); (x[n-k+1] for k=1:n))
2 extras/sparse.jl
 @@ -910,7 +910,7 @@ function hvcat(rows::(Int...), X::SparseMatrixCSC...) tmp_rows[i] = hcat(X[(1 : rows[i]) + k]...) k += rows[i] end - vcat(ntuple(nbr, x->tmp_rows[x])...) + vcat(tmp_rows...) end
22 src/builtins.c
 @@ -199,9 +199,25 @@ JL_CALLABLE(jl_f_apply) { JL_NARGSV(apply, 1); JL_TYPECHK(apply, function, args[0]); - if (nargs == 2 && jl_is_tuple(args[1])) { - return jl_apply((jl_function_t*)args[0], &jl_tupleref(args[1],0), - jl_tuple_len(args[1])); + if (nargs == 2) { + if (((jl_function_t*)args[0])->fptr == &jl_f_tuple) { + if (jl_is_tuple(args[1])) + return args[1]; + if (jl_is_array(args[1])) { + size_t n = jl_array_len(args[1]); + jl_tuple_t *t = jl_alloc_tuple(n); + JL_GC_PUSH(&t); + for(size_t i=0; i < n; i++) { + jl_tupleset(t, i, jl_arrayref((jl_array_t*)args[1], i)); + } + JL_GC_POP(); + return (jl_value_t*)t; + } + } + if (jl_is_tuple(args[1])) { + return jl_apply((jl_function_t*)args[0], &jl_tupleref(args[1],0), + jl_tuple_len(args[1])); + } } size_t n=0, i, j; for(i=1; i < nargs; i++) {
7 src/julia-parser.scm
 @@ -1301,9 +1301,10 @@ (t (require-token s))) (cond ((eqv? t #\) ) (take-token s) - ;; value in parentheses (x) (if (and (pair? ex) (eq? (car ex) '...)) + ;; (ex...) `(tuple ,ex) + ;; value in parentheses (x) ex)) ((eqv? t #\, ) ;; tuple (x,) (x,y) (x...) etc. @@ -1323,6 +1324,10 @@ (error "unexpected line break in tuple")) ((memv t '(#\] #\})) (error (string "unexpected " t " in tuple"))) + ((eq? t 'for) + (take-token s) + (cons 'tuple-comprehension + (cdr (parse-comprehension s ex #\) )))) (else (error "missing separator in tuple")))))))))
8 src/julia-syntax.scm
 @@ -1193,6 +1193,14 @@ ,(construct-loops (reverse ranges) (reverse rs)) ,result))))) + ;; tuple comprehension + (pattern-lambda + (tuple-comprehension expr . ranges) + (begin (if (not (length= ranges 1)) + (error "tuple comprehension must have exactly one range")) + `(call (top tuple) + (... (comprehension ,expr ,@ranges))))) + )) ;; lower-comprehensions

#### 10 comments on commit `f988e0e`

The Julia Language member
commented on `f988e0e` Oct 7, 2012

Interesting. I think I like it but I'm not entirely sure... Will take a little sinking in.

The Julia Language member
commented on `f988e0e` Oct 7, 2012

These look somewhat uncomfortably like Python generator expressions, which are desugared to something like:

```def genexp:
for i in y:
yield x(i)```

So usually, substituting a genexp for a listcomp saves memory and is worth it for large `y`, but here the opposite is the case. Not that we should always necessarily avoid collisions with Python habits, but this one isn't particularly obvious.

The Julia Language member
commented on `f988e0e` Oct 7, 2012

@pao: I'm pretty unclear on this similarity to generator expressions. There's no coroutine-like behavior going on here – it's essentially just a syntax for what `ntuple` did, except that it can be lowered more trivially because `ntuple` was higher order, using a closure that would have to be lowered for maximal efficiency, whereas the syntax can be lowered much more trivially.

The Julia Language member
commented on `f988e0e` Oct 7, 2012

@pao has a point; this syntax may be too good for something you shouldn't really do. We could also lower it as `@task (for i in y;produce(x);end)` or as some other iterator.

The Julia Language member
commented on `f988e0e` Oct 7, 2012

@StefanKarpinski I meant syntactic similarity. Obviously it does something very different, which is what I was getting at.

The Julia Language member
commented on `f988e0e` Oct 7, 2012

True, you shouldn't make things people shouldn't do too easy. There are other syntaxes that would be nice are are far more useful than tuple comprehensions:

```# dict comprehensions:
d = { k(x) => v(x) for x in a }

# non-comprehensions:
f(x) for x in a```

The latter is just a single-line for loop modifier syntax. But it kind of makes sense given the trend here.

The Julia Language member
commented on `f988e0e` Oct 7, 2012

Yes, I am going to roll this back. I was too impulsive here. Generator expressions would be much more useful, for things like `sum(x^2 for x in y)`.

The Julia Language member
commented on `f988e0e` Oct 7, 2012

Yes, generator expressions would be great for that kind of thing. As a side effect, in a context where you don't consume them we could maybe just execute f(x) for every x in a.

The Julia Language member
commented on `f988e0e` Oct 8, 2012

I have already found myself doing these kinds of things, e.g. on the mailing list where I posted `[@time (randn(1,10000000); gc()) for x = 1:10];`, when really all I want to do is run that `randn()` call followed by a `gc()` 10 times. So I guess that's a `+1` from me on generator expressions, both with and without return values. (I personally do not, however, think this should include the other half of generators in Python, e.g. the `yield` statement)

The Julia Language member
commented on `f988e0e` Oct 8, 2012

`for x=1:10 @time (randn(1,10000000); gc()) end` does also work.