Split SnoopCompile into subpackages (#98)

Fixes #95

.github/workflows/Documenter.yml

@@ -0,0 +1,18 @@
+name: Documenter
+on:
+  push:
+    branches: [master]
+    tags: [v*]
+  pull_request:
+
+jobs:
+  Documenter:
+    name: Documentation
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: julia-actions/julia-buildpkg@latest
+      - uses: julia-actions/julia-docdeploy@releases/v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }}

.github/workflows/ci.yml

@@ -8,12 +8,12 @@ jobs:
     name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }}
     runs-on: ${{ matrix.os }}
     strategy:
-      fail-fast: false
+      fail-fast: true
       matrix:
         version:
-          - '1.0.5'
+          - '1.0'
           - '1.2'
-          - '1.4.2'
+          - '1.4'
           - 'nightly'
         os:
           - ubuntu-latest

.gitignore

@@ -2,5 +2,5 @@
 *.jl.*.cov
 *.jl.mem
 Manifest.toml
-test/bot/*/deps/SnoopCompile/precompile/*
-test/bot/*/src/precompile_includer.jl
+SnoopCompileBot/test/*/deps/SnoopCompile/precompile/*
+SnoopCompileBot/test/*/src/precompile_includer.jl

Project.toml

@@ -1,19 +1,19 @@
 name = "SnoopCompile"
 uuid = "aa65fe97-06da-5843-b5b1-d5d13cad87d2"
 author = ["Tim Holy <tim.holy@gmail.com>"]
-version = "1.5.0"
+version = "1.6.0"
 
 [deps]
-FilePathsBase = "48062228-2e41-5def-b9a4-89aafe57970f"
-OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
-YAML = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6"
+SnoopCompileAnalysis = "9ea4277c-da97-4c3a-afb0-537c066769de"
+SnoopCompileBot = "1d5e0e55-7d74-4714-b8d8-efa80e938cf7"
+SnoopCompileCore = "e2b509da-e806-4183-be48-004708413034"
 
 [compat]
-FilePathsBase = "0.9"
-YAML = "0.4"
-OrderedCollections = "1"
+SnoopCompileAnalysis = "1"
+SnoopCompileBot = "1"
+SnoopCompileCore = "1"
 julia = "1"
 
 [extras]

SnoopCompileAnalysis/Project.toml

@@ -0,0 +1,12 @@
+name = "SnoopCompileAnalysis"
+uuid = "9ea4277c-da97-4c3a-afb0-537c066769de"
+author = ["Tim Holy <tim.holy@gmail.com>"]
+version = "1.6.0"
+
+[deps]
+OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
+Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
+
+[compat]
+OrderedCollections = "1"
+julia = "1"

SnoopCompileAnalysis/src/SnoopCompileAnalysis.jl

@@ -0,0 +1,28 @@
+module SnoopCompileAnalysis
+
+using Core: MethodInstance, CodeInfo
+using Serialization, OrderedCollections
+
+# Parcel Regex
+const anonrex = r"#{1,2}\d+#{1,2}\d+"         # detect anonymous functions
+const kwrex = r"^#kw##(.*)$|^#([^#]*)##kw$"   # detect keyword-supplying functions
+const kwbodyrex = r"^##(\w[^#]*)#\d+"         # detect keyword body methods
+const genrex = r"^##s\d+#\d+$"                # detect generators for @generated functions
+const innerrex = r"^#[^#]+#\d+"               # detect inner functions
+
+# Parcel
+include("parcel_snoopc.jl")
+
+if VERSION >= v"1.2.0-DEV.573"
+    include("parcel_snoopi.jl")
+end
+
+if VERSION >= v"1.6.0-DEV.154"
+    include("invalidations.jl")
+end
+
+# Write
+include("write.jl")
+
+
+end # module

src/invalidations.jl → SnoopCompileAnalysis/src/invalidations.jl

@@ -1,4 +1,4 @@
-export @snoopr, invalidation_trees, filtermod, findcaller
+export invalidation_trees, filtermod, findcaller
 
 # Variable names:
 # - `node`, `root`, `leaf`, `parent`, `child`: all `InstanceNode`s, a.k.a. nodes in a MethodInstance tree
@@ -409,37 +409,3 @@ function findcaller(meth::Method, node::InstanceNode)
     end
     return nothing
 end
-
-"""
-    list = @snoopr expr
-
-Capture method cache invalidations triggered by evaluating `expr`.
-`list` is a sequence of invalidated `Core.MethodInstance`s together with "explanations," consisting
-of integers (encoding depth) and strings (documenting the source of an invalidation).
-
-Unless you are working at a low level, you essentially always want to pass `list`
-directly to [`invalidation_trees`](@ref).
-
-# Extended help
-
-`list` is in a format where the "reason" comes after the items.
-Method deletion results in the sequence
-
-    [zero or more (mi, "invalidate_mt_cache") pairs..., zero or more (depth1 tree, loctag) pairs..., method, loctag] with loctag = "jl_method_table_disable"
-
-where `mi` means a `MethodInstance`. `depth1` means a sequence starting at `depth=1`.
-
-Method insertion results in the sequence
-
-    [zero or more (depth0 tree, sig) pairs..., same info as with delete_method except loctag = "jl_method_table_insert"]
-"""
-macro snoopr(expr)
-    quote
-        local list = ccall(:jl_debug_method_invalidation, Any, (Cint,), 1)
-        Expr(:tryfinally,
-            $(esc(expr)),
-            ccall(:jl_debug_method_invalidation, Any, (Cint,), 0)
-        )
-        list
-    end
-end

SnoopCompileBot/Project.toml

@@ -0,0 +1,26 @@
+name = "SnoopCompileBot"
+uuid = "1d5e0e55-7d74-4714-b8d8-efa80e938cf7"
+author = ["Amin Yahyaabadi <aminyahyaabadi74@gmail.com>"]
+version = "1.6.0"
+
+[deps]
+FilePathsBase = "48062228-2e41-5def-b9a4-89aafe57970f"
+Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+YAML = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6"
+SnoopCompileAnalysis = "9ea4277c-da97-4c3a-afb0-537c066769de"
+SnoopCompileCore = "e2b509da-e806-4183-be48-004708413034"
+
+[compat]
+FilePathsBase = "0.9"
+SnoopCompileAnalysis = "1"
+SnoopCompileCore = "1"
+YAML = "0.4"
+julia = "1"
+
+[extras]
+MatLang = "05b439c0-bb3c-11e9-1d8d-1f0a9ebca87a" # only for pathof_noload
+Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[targets]
+test = ["Pkg", "Test", "MatLang"]

src/bot.jl → SnoopCompileBot/src/SnoopCompileBot.jl

@@ -1,5 +1,8 @@
+module SnoopCompileBot
+
 export BotConfig, snoop_bot, snoop_bench
 
+using Core: MethodInstance, CodeInfo
 using YAML
 
 if VERSION <=  v"1.1"
@@ -168,9 +171,12 @@ function BotConfig(
     return BotConfig(package_name, exclusions, check_eval, os, else_os, version, else_version, GoodPath(package_path), GoodPath(precompiles_rootpath), subst, tmin)
 end
 
-include("bot/botutils.jl")
-include("bot/precompile_include.jl")
-include("bot/precompile_activation.jl")
-include("bot/snoop_bot.jl")
-include("bot/snoop_bench.jl")
-include("bot/deprecations.jl")
+include("botutils.jl")
+include("precompile_include.jl")
+include("precompile_activation.jl")
+include("snoop_bot.jl")
+include("snoop_bench.jl")
+include("deprecations.jl")
+
+
+end # module

src/bot/botutils.jl → SnoopCompileBot/src/botutils.jl

@@ -1,3 +1,4 @@
+export timesum
 ################################################################
 """
     timesum(snoop::Vector{Tuple{Float64, Core.MethodInstance}}, unit = :s)
@@ -303,3 +304,8 @@ function JuliaVersionNumber(v::String)
         return VersionNumber(v)
     end
 end
+
+################################################################
+# TODO we need to remove begin and end manually
+# https://github.com/timholy/SnoopCompile.jl/pull/98#issuecomment-645969610
+toplevel_string(expr::Expr) = replace(string(expr), r"^begin([\s\S]*)end$"=>s"\1")

src/bot/snoop_bench.jl → SnoopCompileBot/src/snoop_bench.jl

@@ -1,37 +1,30 @@
 # Snooping functions
 function _snoopi_bench_cmd(snoop_script)
-    # quote end generates $ which doesn't work in commands
-    # TODO no escape is done for snoop_script!!!
-    # TODO
-    # data = Core.eval(  $test_modul, _snoopi($(esc(snoop_script)))  )
-    # TODO use code directly for now
-    # no filter in the benchmark
-    return """
-        using SnoopCompile
+    tmin = 0.0 # For benchmarking
+    return quote
         global SnoopCompile_ENV = true
 
-        empty!(SnoopCompile.__inf_timing__)
-        SnoopCompile.start_timing()
-        try
-            $(string(snoop_script));
-        finally
-            SnoopCompile.stop_timing()
+        using SnoopCompileCore
+
+        data = @snoopi tmin=$tmin begin
+            $snoop_script
         end
-        data = SnoopCompile.sort_timed_inf(0.0)
+
         global SnoopCompile_ENV = false
 
+        using SnoopCompileBot: timesum
         @info( "\nTotal inference time (ms): \t" * string(timesum(data, :ms)))
-    """
+    end
 end
 
 function _snoopv_bench_cmd(snoop_script, package_name)
-    return """
+    return quote
         using $package_name
         @info("Script execution time:")
         @timev begin
-            $(string(snoop_script));
+            $snoop_script
         end
-    """
+    end
 end
 
 function _snoop_bench(config::BotConfig, snoop_script::Expr, test_modul::Module = Main; snoop_mode::Symbol)
@@ -55,14 +48,15 @@ function _snoop_bench(config::BotConfig, snoop_script::Expr, test_modul::Module
     else
         error("snoop_mode $snoop_mode is unkown")
     end
+    snooping_code = toplevel_string(snooping_code)
 
     ################################################################
     julia_cmd = `julia --project=@. -e $snooping_code`
 
     out = quote
         package_sym = Symbol($package_name)
         ################################################################
-        using SnoopCompile
+        using SnoopCompileBot
         @info("""------------------------
         Benchmark Started
         ------------------------
@@ -72,15 +66,15 @@ function _snoop_bench(config::BotConfig, snoop_script::Expr, test_modul::Module
         Precompile Deactivated Benchmark
         ------------------------
         """)
-        SnoopCompile.precompile_deactivator($package_path);
+        SnoopCompileBot.precompile_deactivator($package_path);
         ### Log the compiles
         run($julia_cmd)
         ################################################################
         @info("""------------------------
         Precompile Activated Benchmark
         ------------------------
         """)
-        SnoopCompile.precompile_activator($package_path);
+        SnoopCompileBot.precompile_activator($package_path);
         ### Log the compiles
         run($julia_cmd)
         @info("""------------------------