Merge pull request #19 from timholy/dalum-dalum/0.1

More 0.7/1.0 fixes

.travis.yml

@@ -4,8 +4,9 @@ os:
   - linux
   - osx
 julia:
-  - 0.6
-  #- nightly
+  - 0.7
+  - 1.0
+  - nightly
 notifications:
   email: false
 # uncomment the following lines to override the default test script

README.md

@@ -15,28 +15,31 @@ scripts that work.
 
 ## Usage
 
-The easiest way to describe SnoopCompile is to show a snoop script, in this case for the `Images` package:
+The easiest way to describe SnoopCompile is to show a snoop script, in this case for the `ColorTypes` package:
 
-```jl
+```julia
 using SnoopCompile
 
 ### Log the compiles
-# This only needs to be run once (to generate "/tmp/images_compiles.csv")
+# This only needs to be run once (to generate "/tmp/colortypes_compiles.csv")
 
-SnoopCompile.@snoop "/tmp/images_compiles.csv" begin
-    include(Pkg.dir("Images", "test","runtests.jl"))
+SnoopCompile.@snoop "/tmp/colortypes_compiles.csv" begin
+    using ColorTypes, Pkg
+    include(joinpath(dirname(dirname(pathof(ColorTypes))), "test", "runtests.jl"))
 end
 
 ### Parse the compiles and generate precompilation scripts
 # This can be run repeatedly to tweak the scripts
 
-data = SnoopCompile.read("/tmp/images_compiles.csv")
+data = SnoopCompile.read("/tmp/colortypes_compiles.csv")
 
 pc = SnoopCompile.parcel(reverse!(data[2]))
 SnoopCompile.write("/tmp/precompile", pc)
 ```
 
-After the conclusion of this script, the `"/tmp/precompile"` folder will contain a number of `*.jl` files, organized by package.  These files could be added to a package like this:
+After the conclusion of this script, the `"/tmp/precompile"` folder will contain a number of `*.jl` files, organized by package.
+For each package, you could copy its corresponding `*.jl` file into the package's `src/` directory
+and `include` it into the package:
 
 ```jl
 module SomeModule
@@ -53,6 +56,22 @@ end # module SomeModule
 
 There's a more complete example illustrating potential options in the `examples/` directory.
 
+### Additional flags
+
+When calling the `@snoop` macro, a new julia process is spawned using the function `Base.julia_cmd()`.
+Advanced users may want to tweak the flags passed to this process to suit specific needs.
+This can be done by passing an array of flags of the form `["--flag1", "--flag2"]` as the first argument to the `@snoop` macro.
+For instance, if you want to pass the `--project=/path/to/dir` flag to the process, to cause the julia process to load the project specified by the path, a snoop script may look like:
+```julia
+using SnoopCompile
+
+SnoopCompile.@snoop ["--project=/path/to/dir"] "/tmp/compiles.csv" begin
+    # ... statement to snoop on
+end
+
+# ... processing the precompile statements
+```
+
 ## `userimg.jl`
 
 Currently, precompilation does not cache functions from other modules; as a consequence, your speedup in execution time might be smaller than you'd like. In such cases, one strategy is to generate a script for your `base/userimg.jl` file and build the packages (with precompiles) into julia itself.  Simply append/replace the last two lines of the above script with

REQUIRE

@@ -1,2 +1 @@
-julia 0.6
-Compat
+julia 0.7

appveyor.yml

@@ -1,9 +1,18 @@
 environment:
   matrix:
-  - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x86/0.6/julia-0.6-latest-win32.exe"
-  - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x64/0.6/julia-0.6-latest-win64.exe"
-  #- JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x86/julia-latest-win32.exe"
-  #- JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x64/julia-latest-win64.exe"
+  - julia_version: 0.7
+  - julia_version: 1
+  - julia_version: nightly
+
+platform:
+  - x86 # 32-bit
+  - x64 # 64-bit
+
+# # Uncomment the following lines to allow failures on nightly julia
+# # (tests will run but not make your overall status red)
+# matrix:
+#   allow_failures:
+#   - julia_version: nightly
 
 branches:
   only:
@@ -17,21 +26,18 @@ notifications:
     on_build_status_changed: false
 
 install:
-  - ps: "[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::Tls12"
-# Download most recent Julia Windows binary
-  - ps: (new-object net.webclient).DownloadFile(
-        $env:JULIA_URL,
-        "C:\projects\julia-binary.exe")
-# Run installer silently, output to C:\projects\julia
-  - C:\projects\julia-binary.exe /S /D=C:\projects\julia
+  - ps: iex ((new-object net.webclient).DownloadString("https://raw.githubusercontent.com/JuliaCI/Appveyor.jl/version-1/bin/install.ps1"))
 
 build_script:
-# Need to convert from shallow to complete for Pkg.clone to work
-  - IF EXIST .git\shallow (git fetch --unshallow)
-  - C:\projects\julia\bin\julia -e "
-      versioninfo()
-      Pkg.clone(pwd(), \"SnoopCompile\")
-      Pkg.build(\"SnoopCompile\")"
+  - echo "%JL_BUILD_SCRIPT%"
+  - C:\julia\bin\julia -e "%JL_BUILD_SCRIPT%"
 
 test_script:
-  - C:\projects\julia\bin\julia --check-bounds=yes -e "Pkg.test(\"SnoopCompile\")"
+  - echo "%JL_TEST_SCRIPT%"
+  - C:\julia\bin\julia -e "%JL_TEST_SCRIPT%"
+
+# # Uncomment to support code coverage upload. Should only be enabled for packages
+# # which would have coverage gaps without running on Windows
+# on_success:
+#   - echo "%JL_CODECOV_SCRIPT%"
+#   - C:\julia\bin\julia -e "%JL_CODECOV_SCRIPT%"

examples/gadfly.jl

@@ -1,9 +1,15 @@
+# For this to work, you need to be able to run
+#     using Gadfly
+#     include(joinpath(dirname(dirname(pathof(Gadfly))), "test", "runtests.jl"))
+# successfully. Even if you have the Gadfly package, you may need to add packages.
+
 # Open a new julia session and run this:
 
 using SnoopCompile
 
-SnoopCompile.@snoop1 "/tmp/gadfly_compiles.csv" begin
-    include(Pkg.dir("Gadfly", "test","runtests.jl"))
+SnoopCompile.@snoop "/tmp/gadfly_compiles.csv" begin
+    using Gadfly, Pkg
+    include(joinpath(dirname(dirname(pathof(Gadfly))), "test", "runtests.jl"))
 end
 
 data = SnoopCompile.read("/tmp/gadfly_compiles.csv")

examples/images.jl

@@ -1,10 +1,16 @@
+# For this to work, you need to be able to run
+#     using Images
+#     include(joinpath(dirname(dirname(pathof(Images))), "test", "runtests.jl"))
+# successfully. Even if you have the Images package, you may need to add packages.
+
 using SnoopCompile
 
 ### Log the compiles
 # This only needs to be run once (to generate "/tmp/images_compiles.csv")
 
 SnoopCompile.@snoop "/tmp/images_compiles.csv" begin
-    include(Pkg.dir("Images", "test", "runtests.jl"))
+    using Images, Pkg
+    include(joinpath(dirname(dirname(pathof(Images))), "test", "runtests.jl"))
 end
 
 ### Parse the compiles and generate precompilation scripts

src/SnoopCompile.jl

@@ -1,10 +1,9 @@
-__precompile__()
-
 module SnoopCompile
 
+using Serialization
+
 export
-    @snoop,
-    @snoop1
+    @snoop
 
 """
 ```
@@ -16,67 +15,48 @@ causes the julia compiler to log all functions compiled in the course
 of executing the commands to the file "compiledata.csv". This file
 can be used for the input to `SnoopCompile.read`.
 """
+macro snoop(flags, filename, commands)
+    return :(snoop($(esc(flags)), $(esc(filename)), $(QuoteNode(commands))))
+end
 macro snoop(filename, commands)
-    return :(snoop($(esc(filename)), $(QuoteNode(commands))))
+    return :(snoop(String[], $(esc(filename)), $(QuoteNode(commands))))
 end
 
-function snoop(filename, commands)
+function snoop(flags, filename, commands)
     println("Launching new julia process to run commands...")
     # addprocs will run the unmodified version of julia, so we
     # launch it as a command.
     code_object = """
-        while !eof(STDIN)
-            eval(Main, deserialize(STDIN))
+            using Serialization
+            while !eof(stdin)
+                Core.eval(Main, deserialize(stdin))
+            end
+            """
+    process = open(`$(Base.julia_cmd()) $flags --eval $code_object`, stdout, write=true)
+    serialize(process, quote
+        let io = open($filename, "w")
+            ccall(:jl_dump_compiles, Nothing, (Ptr{Nothing},), io.handle)
+            try
+                $commands
+            finally
+                ccall(:jl_dump_compiles, Nothing, (Ptr{Nothing},), C_NULL)
+                close(io)
+            end
         end
-        """
-    in, io = open(`$(Base.julia_cmd()) --eval $code_object`, "w", STDOUT)
-    serialize(in, quote
-        import SnoopCompile
-    end)
-    # Now that the new process knows about SnoopCompile, it can
-    # expand the macro in this next expression
-    serialize(in, quote
-          SnoopCompile.@snoop1 $filename $commands
+        exit()
     end)
-    close(in)
-    wait(io)
+    wait(process)
     println("done.")
     nothing
 end
 
 function split2(str, on)
-    i = search(str, on)
-    first(i) == 0 && return str, ""
-    return (SubString(str, start(str), prevind(str, first(i))),
+    i = findfirst(isequal(on), str)
+    i === nothing && return str, ""
+    return (SubString(str, firstindex(str), prevind(str, first(i))),
             SubString(str, nextind(str, last(i))))
 end
 
-"""
-```
-@snoop1 "compiledata.csv" begin
-    # Commands to execute
-end
-```
-causes the julia compiler to log all functions compiled in the course
-of executing the commands to the file "compiledata.csv". This file
-can be used for the input to `SnoopCompile.read`.
-"""
-macro snoop1(filename, commands)
-    filename = esc(filename)
-    commands = esc(commands)
-    return quote
-        let io = open($filename, "w")
-            ccall(:jl_dump_compiles, Void, (Ptr{Void},), io.handle)
-            try
-                $commands
-            finally
-                ccall(:jl_dump_compiles, Void, (Ptr{Void},), C_NULL)
-                close(io)
-            end
-        end
-    end
-end
-
 """
 `SnoopCompile.read("compiledata.csv")` reads the log file produced by the compiler and returns the functions as a pair of arrays. The first array is the amount of time required to compile each function, the second is the corresponding function + types. The functions are sorted in order of increasing compilation time. (The time does not include the cost of nested compiles.)
 """
@@ -94,22 +74,23 @@ function read(filename)
         time, str = split2(line, '\t')
         length(str) < 2 && continue
         (str[1] == '"' && str[end] == '"') || continue
-        if startswith(str, "\"<toplevel thunk> -> ")
+        if startswith(str, """"<toplevel thunk> -> """)
             # consume lines until we find the terminating " character
             toplevel = true
             continue
         end
         tm = tryparse(UInt64, time)
-        isnull(tm) && continue
-        push!(times, get(tm))
-        push!(data, str[2:prevind(str, endof(str))])
+        tm === nothing && continue
+        push!(times, tm)
+        push!(data, str[2:prevind(str, lastindex(str))])
     end
     # Save the most costly for last
     p = sortperm(times)
     return (permute!(times, p), permute!(data, p))
 end
 
 # pattern match on the known output of jl_static_show
+extract_topmod(e::QuoteNode) = extract_topmod(e.value)
 function extract_topmod(e)
     Meta.isexpr(e, :.) &&
         return extract_topmod(e.args[1])
@@ -119,29 +100,26 @@ function extract_topmod(e)
         return extract_topmod(e.args[2])
     #Meta.isexpr(e, :call) && length(e.args) == 2 && e.args[1] == :Symbol &&
     #    return Symbol(e.args[2])
+    # parametrized anonymous functions
+    Meta.isexpr(e, :call) && e.args[1].args[1] == :getfield &&
+        return extract_topmod(e.args[1].args[2].args[2])
     isa(e, Symbol) &&
         return e
     return :unknown
 end
 
 function parse_call(line; subst=Vector{Pair{String, String}}(), blacklist=String[])
     for (k, v) in subst
-        line = replace(line, k, v)
+        line = replace(line, k=>v)
     end
-    if any(b -> contains(line, b), blacklist)
+    if any(b -> occursin(b, line), blacklist)
         println(line, " contains a blacklisted substring")
         return false, line, :unknown
     end
 
-    argsidx = search(line, '(') + 1
-    if argsidx == 1 || !endswith(line, ")")
-        warn("unexpected characters at end of line: ", line)
-        return false, line, :unknown
-    end
-    line = "Tuple{$(line[argsidx:prevind(line, endof(line))])}"
-    curly = parse(line, raise=false)
+    curly = Meta.parse(line, raise=false)
     if !Meta.isexpr(curly, :curly)
-        warn("failed parse of line: ", line)
+        @warn("failed parse of line: ", line)
         return false, line, :unknown
     end
     func = curly.args[2]
@@ -231,15 +209,22 @@ function write(filename::AbstractString, pc::Vector)
     nothing
 end
 
-# Write each modules' precompiles to a separate file
-function write(prefix::AbstractString, pc::Dict)
+"""
+    write(prefix::AbstractString, pc::Dict; always::Bool = false)
+
+Write each modules' precompiles to a separate file.  If `always` is
+true, the generated function will always run the precompile statements
+when called, otherwise the statements will only be called during
+package precompilation.
+"""
+function write(prefix::AbstractString, pc::Dict; always::Bool = false)
     if !isdir(prefix)
         mkpath(prefix)
     end
     for (k, v) in pc
         open(joinpath(prefix, "precompile_$k.jl"), "w") do io
             println(io, "function _precompile_()")
-            println(io, "    ccall(:jl_generating_output, Cint, ()) == 1 || return nothing")
+            !always && println(io, "    ccall(:jl_generating_output, Cint, ()) == 1 || return nothing")
             for ln in v
                 println(io, "    ", ln)
             end

test/REQUIRE

@@ -1 +1,2 @@
 ColorTypes
+FixedPointNumbers

test/colortypes.jl

@@ -1,4 +1,4 @@
-using SnoopCompile
+using SnoopCompile, Test, Pkg
 
 mktempdir() do tmpdir
     tmpfile = joinpath(tmpdir, "colortypes_compiles.csv")
@@ -7,7 +7,8 @@ mktempdir() do tmpdir
 
     ### Log the compiles (in a separate process)
     SnoopCompile.@snoop tmpfile begin
-        include(Pkg.dir("ColorTypes", "test", "runtests.jl"))
+        using ColorTypes, Pkg
+        include(joinpath(dirname(dirname(pathof(ColorTypes))), "test", "runtests.jl"))
     end
 
     ### Parse the compiles and generate precompilation scripts