Skip to content

Commit

Permalink
add simple keyword arguments
Browse files Browse the repository at this point in the history 
Function calls: keyword arguments are collected and passed via a new
`Keywords` object:

    julia> f(kw::Keywords, args...) = (args, kw)
    # method added to generic function f

    julia> f(1, a=3, 2, b=4)
    ((1,2),Keywords(a=3, b=4))

Method definitions: here, a keyword argument creates a new local variable
whose value is overridden if called with a keyword of the same name:

    julia> g(kw::Keywords, args..., x=0) = (args, kw, x)
    # method added to generic function g

    julia> g(1, a=3, 2, b=4)
    ((1,2),Keywords(a=3, b=4),0)

    julia> g(1, a=3, 2, b=4, x=7)
    ((1,2),Keywords(a=3, b=4, x=7),7)

Internally,

    function g(kw::Keywords, a, b, x=0)
        ...
    end

get converted to

    function g(kw::Keywords, a, b)
        x = get(kw, :x, 0)
        ...
    end

Methods lacking the `::Keywords` catch-all only accept their specific
keywords:

    julia> h(args..., x=0) = (args, x)
    # method added to generic function h

    julia> h(1, 2, x=7)
    ((1,2),7)

    julia> h(1, a=3, 2, x=7)
    ERROR: unrecognized keyword a
     in h at no file
  • Loading branch information
@nolta
nolta committed Feb 25, 2013
1 parent 3778656 commit f4a2427
Show file tree
Hide file tree
Showing 5 changed files with 75 additions and 2 deletions.
4 changes: 4 additions & 0 deletions base/exports.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ export
IOBuffer,
ImaginaryUnit,
IntSet,
Keywords,
LocalProcess,
Matrix,
ObjectIdDict,
Expand Down Expand Up @@ -666,6 +667,9 @@ export
filter,
filter!,

# keywords
keywords,

# strings and text output
ascii,
begins_with,
Expand Down
34 changes: 34 additions & 0 deletions base/keywords.jl
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@

type Keywords <: Associative{Symbol,Any}
w::Dict{Symbol,Any}
end
Keywords(ks::(Symbol...), vs::(Any...)) = Keywords(Dict{Symbol,Any}(ks,vs))
Keywords() = Keywords(Dict{Symbol,Any}())

keywords(x) = convert(Keywords, x)
convert(::Type{Keywords}, kw::Keywords) = kw
convert(::Type{Keywords}, d::Dict{Symbol,Any}) = Keywords(d)
convert(::Type{Keywords}, d::Dict) =
Keywords((Symbol=>Any)[symbol(k) => v for (k,v) in d])

length(k::Keywords) = length(k.w)
start(k::Keywords) = start(k.w)
done(k::Keywords, i) = done(k.w, i)
next(k::Keywords, i) = next(k.w, i)

get(k::Keywords, s::Symbol, default::ANY) =
has(k.w,s) ? ref(k.w,s) : default
has(k::Keywords, s::Symbol) = has(k.w, s)
ref(k::Keywords, s::Symbol) = ref(k.w, s)
assign(k::Keywords, v::ANY, s::Symbol) = assign(k.w, v, s)

function show(io::IO, k::Keywords)
print(io, "Keywords(")
first = true
for (a,b) in k.w
first || print(io, ", ")
first = false
print(io, a, '=', b)
end
print(io, ')')
end
1 change: 1 addition & 0 deletions base/sysimg.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ include("bitarray.jl")
include("intset.jl")
include("dict.jl")
include("set.jl")
include("keywords.jl")

# compiler
import Core.Undef # used internally by compiler
Expand Down
6 changes: 4 additions & 2 deletions src/julia-parser.scm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1125,11 +1125,13 @@
(define (separate-keywords argl)
(receive
(kws args) (separate (lambda (x)
(and (pair? x) (eq? (car x) '=)))
(and (pair? x)
(eq? (car x) '=)
(symbol? (cadr x))))
argl)
(if (null? kws)
args
`(,@args (keywords ,@kws)))))
`((keywords ,@kws) ,@args))))

; handle function call argument list, or any comma-delimited list.
; . an extra comma at the end is allowed
Expand Down
32 changes: 32 additions & 0 deletions src/julia-syntax.scm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,32 @@
(error "malformed type parameter list"))))

(define (method-def-expr name sparams argl body)
(if (and (pair? argl)
(pair? (car argl))
(eqv? (caar argl) 'keywords))
(let* ((keys (map cadr (cdar argl)))
(vals (map cddr (cdar argl)))
(accepts-all (and (pair? (cdr argl))
(eqv? (arg-type (cadr argl)) 'Keywords)))
(pargl (if accepts-all
(cdr argl)
`((:: ,(gensy) Keywords) ,@(cdr argl))))
(kw (arg-name (car pargl)))
(check (if accepts-all '()
(let ((g (gensy)) (i (gensy)) (j (gensy)))
`(block (= ,g (call (top Set) ,@(map (lambda (k) `(quote ,k)) keys)))
(for (= (tuple ,i ,j) ,kw)
(if (call (top !) (call (top has) ,g ,i))
(call (top error) "unrecognized keyword " ,i)))))))
(newbody `(block ,@check
,@(map (lambda (k v)
`(= ,k (call (top get) ,kw (quote ,k) ,@v)))
keys vals)
,@(cdr body))))
(method-def-expr- name sparams pargl newbody))
(method-def-expr- name sparams argl body)))

(define (method-def-expr- name sparams argl body)
(if (has-dups (llist-vars argl))
(error "function argument names not unique"))
(if (not (symbol? name))
Expand Down Expand Up @@ -914,6 +940,12 @@
x))
args)))

;; keywords syntax
(pattern-lambda (keywords . args)
`(call (top Keywords)
(tuple ,@(map (lambda (x) `(quote ,(cadr x))) args))
(tuple ,@(map caddr args))))

;; dict syntax
(pattern-lambda (dict . args)
`(call (top Dict)
Expand Down

18 comments on commit f4a2427

@tshort
Copy link
Contributor

@tshort tshort commented on f4a2427 Feb 25, 2013

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I played with your branch some. It's great to see your work here. This is my #1 feature request. Here's one thing I was hoping would work but didn't:

julia> f1(x, y = 1.0) = x + y
# method added to generic function f1

julia> f1(3)
ERROR: no method f1(Int64,)

julia> f1
# methods for generic function f1
f1(::Keywords,x)

@nolta
Copy link
Member Author

@nolta nolta commented on f4a2427 Feb 26, 2013

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, should work now.

@tshort
Copy link
Contributor

@tshort tshort commented on f4a2427 Feb 26, 2013

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks, Nick. Overall, I think this meets most of my needs.

I did do a little performance testing as shown below. Keyword args do slow things down. (This won't affect most of my uses for keyword args.)

f1(x, y = 1.0) = x + y

f2(x, y) = x + y
f2(x) = x + 1


function t1()
    x = rand(100000)
    y = 0.0
    for i in 1:100000
        y += f1(x[i], y = 1.0)
    end
    y
end

function t2()
    x = rand(100000)
    y = 0.0
    for i in 1:100000
        y += f1(x[i])
    end
    y
end

function t3()
    x = rand(100000)
    y = 0.0
    for i in 1:100000
        y += f2(x[i], 4.4)
    end
    y
end

@time t1()  # elapsed time: 0.5172491073608398 seconds
@time t2()  # elapsed time: 0.02160811424255371 seconds
@time t3()  # elapsed time: 0.017750978469848633 seconds

@JeffBezanson
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is cool. I was planning to do a version that doesn't use heap allocation, but this might be worth exploring, in the interest of not complicating lots of internals with keywords. On the assumption that the number of keywords will be fairly small, I might try using a simple array like {:a,3,:b,4} instead of a Dict, which should save a bunch of time and space.
We can also use double dispatch to intercept the types of the keyword arguments:

    function g(kw::Keywords, a, b)
        _g_internal(a, b, get(kw, :x, 0), ...)
    end

@JeffBezanson
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Here's another idea. We could define

immutable KWCount
    n::Int
end

and then lower calls as g(KWCount(2), x, y, :a, 3, :b, 4). That way keywords are passed at the end of the normal varargs list, and at least some calls could happen with no allocation. Makes it trickier to deal with varargs however.

@nolta
Copy link
Member Author

@nolta nolta commented on f4a2427 Feb 26, 2013

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, i'll give this a shot.

@nolta
Copy link
Member Author

@nolta nolta commented on f4a2427 Feb 27, 2013

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure how immutable is going to work, but what about something like:

abstract Keywords

immutable Keywords1{T1} <: Keywords
    key1::Symbol
    val1::T1
end

immutable Keywords2{T1,T2} <: Keywords
    key1::Symbol
    val1::T1
    key2::Symbol
    val2::T2
end

Would they get passed on the stack?

@tshort
Copy link
Contributor

@tshort tshort commented on f4a2427 Feb 27, 2013

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would it help to put the keyword symbol in the type? I was thinking of something like:

immutable KW{S,T}
    val::T
end

# the function definition
f2(x, y, a::Int = 1, b::String = "s") = (x, y, a, b)
# could become
function f2(x, y)
    a::Int = 1
    b::String = "s"
    (x, y, a, b)
end
function f2{T1<:Int,T2<:String}(x, y, kw1::Union(KW{:a,T1},KW{:b,T2}))
    a::Int = 1
    b::String = "s"
    S1 = kw1.val
    (x, y, a, b)
end
function f2{T1<:Int,T2<:String}(x, y, kw1::Union(KW{:a,T1},KW{:b,T2}), kw2::Union(KW{:a,T1},KW{:b,T2}))
    a::Int = 1
    b::String = "s"
    S1 = kw1.val
    S2 = kw2.val
    (x, y, a, b)
end
# the function call
f2(xx, yy, b = "asdf")
# becomes
f2(xx, yy, KW{:b,ASCIIString}("asdf"))

@tshort
Copy link
Contributor

@tshort tshort commented on f4a2427 Feb 27, 2013

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My function definitions are messed up, so that might be just a dead end.

@nolta
Copy link
Member Author

@nolta nolta commented on f4a2427 Feb 27, 2013

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, and my typedefs don't really work either, so this probably is a dead end. Seemed tantalizingly close, though.

@tshort
Copy link
Contributor

@tshort tshort commented on f4a2427 Feb 27, 2013

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The best code I could find that actually works is here, but it's not really that flexible. It seems like we need some sort of look-up, whether it's done something like that below (possibly runs faster) or with a Dict (a lot easier to implement and probably easier for the end user).

type KW
    sym::Symbol
    val
end
# the function definition
## f2(x, y, a::Int = 1, b::String = "s") = (x, y, a, b)
# could become
function f2(x, y)
    a::Int = 1
    b::String = "s"
    (x, y, a, b)
end
function f2(x, y, kw1::KW)
    a::Int = 1
    b::String = "s"
    kw1.sym == :a ? a = kw1.val :
    kw1.sym == :b ? b = kw1.val : nothing
    (x, y, a, b)
end
function f2(x, y, kw1::KW, kw2::KW)
    a::Int = 1
    b::String = "s"
    kw1.sym == :a ? a = kw1.val :
    kw1.sym == :b ? b = kw1.val : nothing
    kw2.sym == :a ? a = kw2.val :
    kw2.sym == :b ? b = kw2.val : nothing
    (x, y, a, b)
end
@show f2(1,2) 
@show f2(1,2,KW(:a,44))
@show f2(1,2,KW(:b,"asdf"))
@show f2(1,2,KW(:c,"asdf"))
@show f2(1,2,KW(:a,11),KW(:b,"a"))

@JeffBezanson
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm trying my approach of passing a keyword count and keywords as varargs on branch jb/kwargs.

@tshort
Copy link
Contributor

@tshort tshort commented on f4a2427 Feb 28, 2013

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Great, Jeff!

Here are some issues based on the examples I tried below:

  • The method definitions shown at the REPL lose their filename link.
  • It'd be great if the method definitions shown in the REPL included the defined keyword arguments.
  • The method definitions enforce a type for each argument. It'd be nice if typing was left looser unless the user specified it. g(x, i::Int = 4) looks natural for this, but I don't know if it is feasible.
  • I couldn't figure out how to collect keywords with varargs following standard args (example below).
g(x, y = 1.0, z = 9) = (x, y, z)
#
@show g(2, y = 9.)      
@show g(2, z = 0)
@show g(z = 0, 2)
@show g(2, z = 0, y = 9.)
@show g(2, z = 0, y = 9.)
@show g(2, z = 0, y = 9., zz = "top")
@show g(2, z = 0.)   # doesn't work: z has to be Int
#
g1(args...) = args
@show g1(z = 0)
@show g1(4, zz = "top")
@show g1(4, z = 0, 9)
@show g(g1(4, z = 0, y = 99.)...)
#
# I can't figure out how to get these to work:
g2(x, args...) = (x, args...)
@show g2(2, z = 0)
g3(n::Base.NKeywords, x, args...) = (x, n, args...)
@show g3(2, z = 0)

@JeffBezanson
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I haven't gotten to all the features yet. I will let your obvious enthusiasm for keyword args inspire me :)

@tshort
Copy link
Contributor

@tshort tshort commented on f4a2427 Feb 28, 2013 via email

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@JeffBezanson
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, I have addressed your points 1 and 3, and added support for using keywords and varargs together. The next feature is "rest keywords", i.e. collecting all unmatched keywords, and passing a block of keywords with f(a, va...; keywords...).

It is great that this approach uses only source-level rewriting. And everything is passed on the stack, so the overhead is similar to normal varargs, and we can leverage existing and future vararg-related optimizations. The big difference, of course, is the keyword matching code. But, that might as well be generated by the front end, since it is much easier that way, and code equivalent to that would have to run somewhere in any keyword arg implementation anyway.

By adding metadata describing keywords to a function's AST, we can gain useful printing of methods, and also the ability to optimize away keyword matching completely (I've isolated matching to its own method, so it is easier to skip). For known function calls, the overhead ought to be zero.

So far I only see one big downside, which is that screwy things happen to functions that don't support keywords, and have an Any-typed first argument:

julia> f(x, rest...) = x
# method added to generic function f

julia> f(1, y=1)
NKeywords(0x0000000000000001)

We could potentially do something drastic like making the types of the keyword-related tags disjoint from the rest of the type hierarchy (not subtypes of Any!). Or we could add a check to every loosely typed function that throws an error if isa(firstarg,NKeywords).

@nolta
Copy link
Member Author

@nolta nolta commented on f4a2427 Mar 1, 2013

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I took another crack at keywords, with an eye towards making them as fast as possible. The solution i came up with is a little goofy, splitting keywords into 2 types: fast/static & slow/dynamic. See ee32a01's commit message on mn/kwargs2 for details.

On mn/kwargs2 @tshort's t1() test runs about 10x faster than jb/kwargs, which in turn is about 25x faster than mn/kwargs.

@JeffBezanson
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think that approach is going to work, since functions are not always called by their "official" names. It should be possible to pass keywords to function arguments:

foo(x, y=1) = x+y

function other(f, a)
    f(a, y=2)
end

other(foo, 1)

It is also a problem that it generates 2^n method definitions. We will have to recover the performance of mn/kwargs2 by adding a compiler pass to statically process keyword args when possible. That way each call site is transformed in one of 2^n possible ways, instead of keeping all 2^n around explicitly.

Please sign in to comment.