Towards a semi-performant recursive interpreter

.gitignore

@@ -3,3 +3,6 @@
 *.jl.mem
 expected.out
 failed.out
+src/builtins.jl
+deps/build.log
+docs/build/

.travis.yml

@@ -9,5 +9,17 @@ julia:
   - 1.1
   - nightly
 
+script:
+  - julia --project -e 'using Pkg;
+                        Pkg.instantiate();
+                        Pkg.add([PackageSpec(name = "TerminalRegressionTests", rev = "master"),
+                                 PackageSpec(name = "VT100", rev = "master")]);
+                        Pkg.build();
+                        Pkg.test()'
+
+matrix:
+   allow_failures:
+   - julia: nightly
+
 notifications:
   email: false

Manifest.toml

@@ -7,21 +7,14 @@ uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 deps = ["REPL"]
 git-tree-sha1 = "e4a4693fc3fd3924d469bb0fa46215672fd6d4b8"
 repo-rev = "master"
-repo-url = "https://github.com/Keno/DebuggerFramework.jl.git"
+repo-url = "https://github.com/JuliaDebug/DebuggerFramework.jl.git"
 uuid = "67417a49-6d77-5db2-98c7-c13144130cd2"
 version = "0.1.2+"
 
-[[Distributed]]
-deps = ["Random", "Serialization", "Sockets"]
-uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"
-
 [[InteractiveUtils]]
 deps = ["Markdown"]
 uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
 
-[[Logging]]
-uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
-
 [[Markdown]]
 deps = ["Base64"]
 uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
@@ -30,16 +23,5 @@ uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
 deps = ["InteractiveUtils", "Markdown", "Sockets"]
 uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
 
-[[Random]]
-deps = ["Serialization"]
-uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
-
-[[Serialization]]
-uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
-
 [[Sockets]]
 uuid = "6462fe0b-24de-5631-8697-dd941f90decc"
-
-[[Test]]
-deps = ["Distributed", "InteractiveUtils", "Logging", "Random"]
-uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

Project.toml

@@ -4,11 +4,14 @@ version = "0.1.1"
 
 [deps]
 DebuggerFramework = "67417a49-6d77-5db2-98c7-c13144130cd2"
+InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
 Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a"
 REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
 
 [extras]
+TerminalRegressionTests = "98bfdc55-cc95-5876-a49a-74609291cbe0"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+VT100 = "7774df62-37c0-5c21-b34d-f6d7f98f54bc"
 
 [targets]
-test = ["Test"]
+test = ["Test", "TerminalRegressionTests", "VT100"]

deps/build.jl

@@ -0,0 +1,7 @@
+using InteractiveUtils
+
+const srcpath = joinpath(dirname(@__DIR__), "src")
+include(joinpath(srcpath, "generate_builtins.jl"))
+open(joinpath(srcpath, "builtins.jl"), "w") do io
+    generate_builtins(io)
+end

docs/make.jl

@@ -0,0 +1,20 @@
+using Documenter, ASTInterpreter2
+
+makedocs(
+    modules = [ASTInterpreter2],
+    clean = false,
+    format = Documenter.HTML(prettyurls = get(ENV, "CI", nothing) == "true"),
+    sitename = "ASTInterpreter2.jl",
+    authors = "Keno Fischer, Tim Holy, and others",
+    linkcheck = !("skiplinks" in ARGS),
+    pages = [
+        "Home" => "index.md",
+        "ast.md",
+        "internals.md",
+        "dev_reference.md",
+    ],
+)
+
+deploydocs(
+    repo = "github.com/JuliaDebug/ASTInterpreter2.jl.git",
+)

docs/src/ast.md

@@ -0,0 +1,93 @@
+# Lowered representation
+
+Let's start with a demonstration on simple function:
+
+```julia
+function summer(A::AbstractArray{T}) where T
+    s = zero(T)
+    for a in A
+        s += a
+    end
+    return s
+end
+
+A = [1, 2, 5]
+```
+
+ASTIntepreter2 uses the lowered representation of code:
+
+```julia
+julia> code = @code_lowered summer(A)
+CodeInfo(
+1 ─       s = (Main.zero)($(Expr(:static_parameter, 1)))
+│   %2  = A
+│         #temp# = (Base.iterate)(%2)
+│   %4  = #temp# === nothing
+│   %5  = (Base.not_int)(%4)
+└──       goto #4 if not %5
+2 ┄ %7  = #temp#
+│         a = (Core.getfield)(%7, 1)
+│   %9  = (Core.getfield)(%7, 2)
+│         s = s + a
+│         #temp# = (Base.iterate)(%2, %9)
+│   %12 = #temp# === nothing
+│   %13 = (Base.not_int)(%12)
+└──       goto #4 if not %13
+3 ─       goto #2
+4 ┄       return s
+)
+```
+
+To understand this package's internals, you need to familiarize yourself with these
+`CodeInfo` objects. The numbers on the left correspond to [basic blocks](https://en.wikipedia.org/wiki/Basic_block);
+when used in statements these are printed with a hash, e.g., in `goto #4 if not %6`, the
+`#4` refers to basic block 4.
+The numbers in the next column--e.g., `%1`, refer to [single static assignment (SSA) values](https://en.wikipedia.org/wiki/Static_single_assignment_form).
+Each statement (each line of this printout) corresponds to a single SSA value,
+but only those used later in the code are printed using assignment syntax.
+Wherever a previous SSA value is used, it's referenced by an `SSAValue` and printed as `%6`;
+for example, in `goto #4 if not %6`, the `%6` is the result of evaluating the 6th statement,
+which is `(Base.not_int)(%5)`, which in turn refers to the result of statement 5.
+Together lines 5 and 6 correspond to `!(#temp# === nothing)`.
+(The `#temp#` means that this was a generated variable name not present explicitly in the original source code.)
+
+Before diving into the details, let's first look at the statements themselves:
+
+```julia
+julia> code.code
+16-element Array{Any,1}:
+ :(_3 = (Main.zero)($(Expr(:static_parameter, 1))))
+ :(_2)
+ :(_4 = (Base.iterate)(%2))
+ :(_4 === nothing)
+ :((Base.not_int)(%4))
+ :(unless %5 goto %16)
+ :(_4)
+ :(_5 = (Core.getfield)(%7, 1))
+ :((Core.getfield)(%7, 2))
+ :(_3 = _3 + _5)
+ :(_4 = (Base.iterate)(%2, %9))
+ :(_4 === nothing)
+ :((Base.not_int)(%12))
+ :(unless %13 goto %16)
+ :(goto %7)
+ :(return _3)
+```
+
+You can see directly that the SSA assignments are implicit; they are not directly
+present in the statement list.
+The most noteworthy change here is the appearance of objects like `_3`, which are
+references that index into local variable slots:
+
+```julia
+julia> code.slotnames
+5-element Array{Any,1}:
+ Symbol("#self#")
+ :A
+ :s
+ Symbol("#temp#")
+ :a
+```
+
+When printing the whole `CodeInfo` object, these `slotnames` are substituted in.
+The types of objects that can be in `code.code` is well-described in the [Julia AST](https://docs.julialang.org/en/latest/devdocs/ast/) documentation.

docs/src/dev_reference.md

@@ -0,0 +1,55 @@
+# Function reference
+
+## Top-level
+
+```@docs
+@interpret
+```
+
+## Frame creation
+
+```@docs
+ASTInterpreter2.enter_call
+ASTInterpreter2.enter_call_expr
+ASTInterpreter2.build_frame
+ASTInterpreter2.determine_method_for_expr
+ASTInterpreter2.prepare_args
+ASTInterpreter2.prepare_call
+ASTInterpreter2.get_call_framecode
+ASTInterpreter2.optimize!
+```
+
+## Frame execution
+
+```@docs
+ASTInterpreter2.Compiled
+ASTInterpreter2.step_expr!
+ASTInterpreter2.finish!
+ASTInterpreter2.finish_and_return!
+ASTInterpreter2.next_until!
+ASTInterpreter2.evaluate_call!
+ASTInterpreter2.evaluate_foreigncall!
+ASTInterpreter2.maybe_evaluate_builtin
+ASTInterpreter2.@eval_rhs
+```
+
+## Types
+
+```@docs
+ASTInterpreter2.JuliaStackFrame
+ASTInterpreter2.JuliaFrameCode
+ASTInterpreter2.JuliaProgramCounter
+```
+
+## Internal storage
+
+```@docs
+ASTInterpreter2.framedict
+ASTInterpreter2.genframedict
+```
+
+## Utilities
+
+```@docs
+ASTInterpreter2.iswrappercall
+```

docs/src/index.md

@@ -0,0 +1,26 @@
+# ASTInterpreter2
+
+This package implements an [interpreter](https://en.wikipedia.org/wiki/Interpreter_(computing)) for Julia code.
+Normally, Julia compiles your code when you first execute it; using ASTInterpreter2 you can
+avoid compilation and execute the expressions that define your code directly.
+Interpreters have a number of applications, including support for stepping debuggers.
+
+At a pure user level, there is not much to know:
+
+```jldoctest
+julia> using ASTInterpreter2
+
+julia> a = [1, 2, 5]
+3-element Array{Int64,1}:
+ 1
+ 2
+ 5
+
+julia> sum(a)
+8
+
+julia> @interpret sum(a)
+8
+```
+
+Those who want to dive deeper should continue reading.