Skip to content

Commit

Permalink
Merge pull request #16622 from JuliaLang/jb/compr2
Browse files Browse the repository at this point in the history 
0.5-style comprehensions
  • Loading branch information
@JeffBezanson
JeffBezanson committed Jul 11, 2016
2 parents 6f9a592 + b363cc7 commit 0572533
Show file tree
Hide file tree
Showing 23 changed files with 265 additions and 403 deletions.
10 changes: 10 additions & 0 deletions NEWS.md
View file
Expand Up @@ -7,6 +7,9 @@ New language features
* Generator expressions, e.g. `f(i) for i in 1:n` ([#4470]). This returns an iterator
that computes the specified values on demand.

* Generators and comprehensions support filtering using `if` ([#550]) and nested
iteration using multiple `for` keywords ([#4867]).

* Broadcasting syntax: ``f.(args...)`` is equivalent to ``broadcast(f, args...)`` ([#15032]).

* Macro expander functions are now generic, so macros can have multiple definitions
Expand Down Expand Up @@ -69,6 +72,13 @@ Language changes
* The built-in `NTuple` type has been removed; `NTuple{N,T}` is now
implemented internally as `Tuple{Vararg{T,N}}` ([#11242]).

* Array comprehensions preserve the dimensions of the input ranges. For example,
`[ 2x for x in A]` will have the same dimensions as `A`.

* The result type of an array comprehension depends only on the types of elements
computed, instead of using type inference ([#7258]). If the result is empty, then
type inference is still used to determine the element type.

Command-line option changes
---------------------------

Expand Down
4 changes: 2 additions & 2 deletions base/abstractarray.jl
View file
Expand Up @@ -813,8 +813,8 @@ typed_hcat{T}(::Type{T}) = Array{T}(0)
## cat: special cases
vcat{T}(X::T...) = T[ X[i] for i=1:length(X) ]
vcat{T<:Number}(X::T...) = T[ X[i] for i=1:length(X) ]
hcat{T}(X::T...) = T[ X[j] for i=1, j=1:length(X) ]
hcat{T<:Number}(X::T...) = T[ X[j] for i=1, j=1:length(X) ]
hcat{T}(X::T...) = T[ X[j] for i=1:1, j=1:length(X) ]
hcat{T<:Number}(X::T...) = T[ X[j] for i=1:1, j=1:length(X) ]

vcat(X::Number...) = hvcat_fill(Array{promote_typeof(X...)}(length(X)), X)
hcat(X::Number...) = hvcat_fill(Array{promote_typeof(X...)}(1,length(X)), X)
Expand Down
33 changes: 29 additions & 4 deletions base/array.jl
View file
Expand Up @@ -244,16 +244,41 @@ function _collect(cont, itr, ::HasEltype, isz::SizeUnknown)
return a
end

_default_eltype(itr::ANY) = Union{}
_default_eltype{I,T}(::Generator{I,Type{T}}) = T
if isdefined(Core, :Inference)
function _default_eltype(itrt::ANY)
rt = Core.Inference.return_type(first, Tuple{itrt})
return isleaftype(rt) ? rt : Union{}
end
else
_default_eltype(itr::ANY) = Union{}
end
_default_eltype{I,T}(::Type{Generator{I,Type{T}}}) = T

_array_for(T, itr, ::HasLength) = Array(T, Int(length(itr)::Integer))
_array_for(T, itr, ::HasShape) = Array(T, convert(Dims,size(itr)))

function collect(itr::Generator)
isz = iteratorsize(itr.iter)
et = _default_eltype(typeof(itr))
if isa(isz, SizeUnknown)
return grow_to!(Array(et, 0), itr)
else
st = start(itr)
if done(itr,st)
return _array_for(et, itr.iter, isz)
end
v1, st = next(itr, st)
collect_to_with_first!(_array_for(typeof(v1), itr.iter, isz), v1, itr, st)
end
end

_collect(c, itr, ::EltypeUnknown, isz::SizeUnknown) =
grow_to!(_similar_for(c, _default_eltype(itr), itr, isz), itr)
grow_to!(_similar_for(c, _default_eltype(typeof(itr)), itr, isz), itr)

function _collect(c, itr, ::EltypeUnknown, isz::Union{HasLength,HasShape})
st = start(itr)
if done(itr,st)
return _similar_for(c, _default_eltype(itr), itr, isz)
return _similar_for(c, _default_eltype(typeof(itr)), itr, isz)
end
v1, st = next(itr, st)
collect_to_with_first!(_similar_for(c, typeof(v1), itr, isz), v1, itr, st)
Expand Down
4 changes: 2 additions & 2 deletions base/arraymath.jl
View file
Expand Up @@ -322,8 +322,8 @@ function ctranspose(A::AbstractMatrix)
end
ctranspose{T<:Real}(A::AbstractVecOrMat{T}) = transpose(A)

transpose(x::AbstractVector) = [ transpose(v) for i=1, v in x ]
ctranspose{T}(x::AbstractVector{T}) = T[ ctranspose(v) for i=1, v in x ] #Fixme comprehension
transpose(x::AbstractVector) = [ transpose(v) for i=1:1, v in x ]
ctranspose{T}(x::AbstractVector{T}) = T[ ctranspose(v) for i=1:1, v in x ]

_cumsum_type{T<:Number}(v::AbstractArray{T}) = typeof(+zero(T))
_cumsum_type(v) = typeof(v[1]+v[1])
Expand Down
10 changes: 6 additions & 4 deletions base/generator.jl
View file
Expand Up @@ -5,8 +5,9 @@
Given a function `f` and an iterator `iter`, construct an iterator that yields
the values of `f` applied to the elements of `iter`.
The syntax `f(x) for x in iter` is syntax for constructing an instance of this
type.
The syntax `f(x) [if cond(x)::Bool] for x in iter` is syntax for constructing an instance of this
type. The `[if cond(x)::Bool]` expression is optional and acts as a "guard", effectively
filtering out values where the condition is false.
"""
immutable Generator{I,F}
f::F
Expand All @@ -17,9 +18,10 @@ Generator(f, I1, I2, Is...) = Generator(a->f(a...), zip(I1, I2, Is...))

Generator{T,I}(::Type{T}, iter::I) = Generator{I,Type{T}}(T, iter)

start(g::Generator) = start(g.iter)
done(g::Generator, s) = done(g.iter, s)
start(g::Generator) = (@_inline_meta; start(g.iter))
done(g::Generator, s) = (@_inline_meta; done(g.iter, s))
function next(g::Generator, s)
@_inline_meta
v, s2 = next(g.iter, s)
g.f(v), s2
end
Expand Down
147 changes: 38 additions & 109 deletions base/inference.jl
View file
Expand Up @@ -43,10 +43,8 @@ end
type InferenceState
sp::SimpleVector # static parameters
label_counter::Int # index of the current highest label for this function
fedbackvars::Dict{SSAValue, Bool}
mod::Module
currpc::LineNum
static_typeof::Bool

# info on the state of inference and the linfo
linfo::LambdaInfo
Expand All @@ -71,7 +69,6 @@ type InferenceState
backedges::Vector{Tuple{InferenceState, Vector{LineNum}}}
# iteration fixed-point detection
fixedpoint::Bool
typegotoredo::Bool
inworkq::Bool
# optimization
optimize::Bool
Expand Down Expand Up @@ -158,13 +155,13 @@ type InferenceState

inmodule = isdefined(linfo, :def) ? linfo.def.module : current_module() # toplevel thunks are inferred in the current module
frame = new(
sp, nl, Dict{SSAValue, Bool}(), inmodule, 0, false,
sp, nl, inmodule, 0,
linfo, la, s, Union{}, W, n,
cur_hand, handler_at, n_handlers,
ssavalue_uses, ssavalue_init,
ObjectIdDict(), #Dict{InferenceState, Vector{LineNum}}(),
Vector{Tuple{InferenceState, Vector{LineNum}}}(),
false, false, false, optimize, inlining, needtree, false)
false, false, optimize, inlining, needtree, false)
push!(active, frame)
nactive[] += 1
return frame
Expand Down Expand Up @@ -926,13 +923,32 @@ function abstract_apply(af::ANY, fargs, aargtypes::Vector{Any}, vtypes::VarTable
return abstract_call(af, (), Any[type_typeof(af), Vararg{Any}], vtypes, sv)
end

function pure_eval_call(f::ANY, argtypes::ANY, atype, sv)
function pure_eval_call(f::ANY, argtypes::ANY, atype, vtypes, sv)
for a in drop(argtypes,1)
if !(isa(a,Const) || (isType(a) && !has_typevars(a.parameters[1])))
return false
end
end

if f === return_type && length(argtypes) == 3
tt = argtypes[3]
if isType(tt)
af_argtype = tt.parameters[1]
if af_argtype <: Tuple && isa(af_argtype, DataType)
af = argtypes[2]
rt = abstract_call(isa(af,Const) ? af.val : af.parameters[1],
(), Any[argtypes[2], af_argtype.parameters...], vtypes, sv)
if isa(rt,Const)
return Type{widenconst(rt)}
elseif isleaftype(rt) || isleaftype(af_argtype) || rt === Bottom
return Type{rt}
else
return Type{TypeVar(:R, rt)}
end
end
end
end

meth = _methods_by_ftype(atype, 1)
if meth === false || length(meth) != 1
return false
Expand Down Expand Up @@ -1011,7 +1027,7 @@ function abstract_call(f::ANY, fargs, argtypes::Vector{Any}, vtypes::VarTable, s
end

atype = argtypes_to_type(argtypes)
t = pure_eval_call(f, argtypes, atype, sv)
t = pure_eval_call(f, argtypes, atype, vtypes, sv)
t !== false && return t

if istopfunction(tm, f, :promote_type) || istopfunction(tm, f, :typejoin)
Expand Down Expand Up @@ -1068,8 +1084,6 @@ function abstract_eval(e::ANY, vtypes::VarTable, sv::InferenceState)
return abstract_eval_constant(e)
end
e = e::Expr
# handle:
# call null new & static_typeof
if is(e.head,:call)
t = abstract_eval_call(e, vtypes, sv)
elseif is(e.head,:null)
Expand Down Expand Up @@ -1103,42 +1117,6 @@ function abstract_eval(e::ANY, vtypes::VarTable, sv::InferenceState)
t = abstract_eval_constant(val)
end
end
elseif is(e.head,:static_typeof)
var = e.args[1]
t = widenconst(abstract_eval(var, vtypes, sv))
if isa(t,DataType) && typeseq(t,t.name.primary)
# remove unnecessary typevars
t = t.name.primary
end
if is(t,Bottom)
# if we haven't gotten fed-back type info yet, return Bottom. otherwise
# Bottom is the actual type of the variable, so return Type{Bottom}.
if get!(sv.fedbackvars, var, false)
t = Type{Bottom}
else
sv.static_typeof = true
end
elseif isleaftype(t)
t = Type{t}
elseif isleaftype(sv.linfo.specTypes)
if isa(t,TypeVar)
t = Type{t.ub}
else
t = Type{t}
end
else
# if there is any type uncertainty in the arguments, we are
# effectively predicting what static_typeof will say when
# the function is compiled with actual arguments. in that case
# abstract types yield Type{<:T} instead of Type{T}.
# this doesn't really model the situation perfectly, but
# "isleaftype(inference_stack.types)" should be good enough.
if isa(t,TypeVar) || isvarargtype(t)
t = Type{t}
else
t = Type{TypeVar(:_,t)}
end
end
elseif is(e.head,:method)
t = (length(e.args) == 1) ? Any : Void
elseif is(e.head,:copyast)
Expand Down Expand Up @@ -1666,23 +1644,19 @@ function typeinf_frame(frame)
W = frame.ip
s = frame.stmt_types
n = frame.nstmts
@label restart_typeinf
while !isempty(W)
# make progress on the active ip set
local pc::Int = first(W), pc´::Int
while true # inner loop optimizes the common case where it can run straight from pc to pc + 1
#print(pc,": ",s[pc],"\n")
delete!(W, pc)
frame.currpc = pc
frame.static_typeof = false
frame.cur_hand = frame.handler_at[pc]
stmt = frame.linfo.code[pc]
changes = abstract_interpret(stmt, s[pc]::Array{Any,1}, frame)
if changes === ()
# if there was a Expr(:static_typeof) on this line,
# need to continue to the next pc even though the return type was Bottom
# otherwise, this line threw an error and there is no need to continue
frame.static_typeof || break
# this line threw an error and there is no need to continue
break
changes = s[pc]
end
if frame.cur_hand !== ()
Expand Down Expand Up @@ -1732,26 +1706,6 @@ function typeinf_frame(frame)
s[l] = newstate
end
end
elseif is(hd, :type_goto)
for i = 2:length(stmt.args)
var = stmt.args[i]::SSAValue
# Store types that need to be fed back via type_goto
# in ssavalue_init. After finishing inference, if any
# of these types changed, start over with the fed-back
# types known from the beginning.
# See issue #3821 (using !typeseq instead of !subtype),
# and issue #7810.
id = var.id+1
vt = frame.linfo.ssavaluetypes[id]
ot = frame.ssavalue_init[id]
if ot===NF || !(vt⊑ot && ot⊑vt)
frame.ssavalue_init[id] = vt
if get(frame.fedbackvars, var, false)
frame.typegotoredo = true
end
end
frame.fedbackvars[var] = true
end
elseif is(hd, :return)
pc´ = n + 1
rt = abstract_eval(stmt.args[1], s[pc], frame)
Expand Down Expand Up @@ -1821,39 +1775,6 @@ function typeinf_frame(frame)
end

if finished || frame.fixedpoint
if frame.typegotoredo
# if any type_gotos changed, clear state and restart.
frame.typegotoredo = false
for ll = 2:length(s)
s[ll] = ()
end
empty!(W)
push!(W, 1)
frame.cur_hand = ()
frame.handler_at = Any[ () for i=1:n ]
frame.n_handlers = 0
frame.linfo.ssavaluetypes[:] = frame.ssavalue_init
@goto restart_typeinf
else
# if a static_typeof was never reached,
# use Union{} as its real type and continue
# running type inference from its uses
# (one of which is the static_typeof)
# TODO: this restart should happen just before calling finish()
for (fbvar, seen) in frame.fedbackvars
if !seen
frame.fedbackvars[fbvar] = true
id = (fbvar::SSAValue).id + 1
for r in frame.ssavalue_uses[id]
if !is(s[r], ()) # s[r] === () => unreached statement
push!(W, r)
end
end
@goto restart_typeinf
end
end
end

if finished
finish(frame)
else # fixedpoint propagation
Expand Down Expand Up @@ -2056,7 +1977,7 @@ function eval_annotate(e::ANY, vtypes::ANY, sv::InferenceState, undefs, pass)

e = e::Expr
head = e.head
if is(head,:static_typeof) || is(head,:line) || is(head,:const)
if is(head,:line) || is(head,:const)
return e
elseif is(head,:(=))
e.args[2] = eval_annotate(e.args[2], vtypes, sv, undefs, pass)
Expand Down Expand Up @@ -2282,9 +2203,6 @@ function effect_free(e::ANY, sv, allow_volatile::Bool)
elseif isa(e,Expr)
e = e::Expr
head = e.head
if head === :static_typeof
return true
end
if head === :static_parameter || head === :meta || head === :line ||
head === :inbounds || head === :boundscheck
return true
Expand Down Expand Up @@ -2550,7 +2468,8 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference
methsp = meth[2]
method = meth[3]::Method
# check whether call can be inlined to just a quoted constant value
if isa(f, widenconst(ft)) && !method.isstaged && method.lambda_template.pure && (isType(e.typ) || isa(e.typ,Const))
if isa(f, widenconst(ft)) && !method.isstaged && (method.lambda_template.pure || f === return_type) &&
(isType(e.typ) || isa(e.typ,Const))
if isType(e.typ)
if !has_typevars(e.typ.parameters[1])
return inline_as_constant(e.typ.parameters[1], argexprs, sv)
Expand Down Expand Up @@ -3668,6 +3587,16 @@ function reindex_labels!(linfo::LambdaInfo, sv::InferenceState)
end
end

function return_type(f::ANY, t::ANY)
rt = Union{}
for m in _methods(f, t, -1)
_, ty, inferred = typeinf(m[3], m[1], m[2], false)
!inferred && return Any
rt = tmerge(rt, ty)
rt === Any && break
end
return rt
end

#### bootstrapping ####

Expand Down

0 comments on commit 0572533

Please sign in to comment.