expand macros before checking for :toplevel/:module expressions (#…

…107)

* expand macros before checking for `:toplevel`/`:module` expressions

macros can arbitrarily generate `:toplevel` and `:module` expressions,
and if these expressions are passed to `partially_interpret!`, we will
fail to concretize them and their toplevel definitions

* fix wrong queuing order

src/virtualprocess.jl

@@ -103,15 +103,15 @@ function virtual_process!(toplevelex::Expr,
 
         return lwr
     end
-    # function macroexpand_err_handler(err, st)
-    #     # `4` corresponds to `with_err_handling`, `f`, `macroexpand` and its kwfunc
-    #     st = crop_stacktrace(st, 4)
-    #     push!(ret.toplevel_error_reports, ActualErrorWrapped(err, st, filename, lnn.line))
-    #     return nothing
-    # end
-    # macroexpand_with_err_handling(mod, x) = with_err_handling(macroexpand_err_handler) do
-    #     return macroexpand(mod, x)
-    # end
+    function macroexpand_err_handler(err, st)
+        # `4` corresponds to `with_err_handling`, `f`, `macroexpand` and its kwfunc
+        st = crop_stacktrace(st, 4)
+        push!(ret.toplevel_error_reports, ActualErrorWrapped(err, st, filename, lnn.line))
+        return nothing
+    end
+    macroexpand_with_err_handling(mod, x) = with_err_handling(macroexpand_err_handler) do
+        return macroexpand(mod, x; recursive = false)
+    end
     function eval_err_handler(err, st)
         # `3` corresponds to `with_err_handling`, `f` and `eval`
         st = crop_stacktrace(st, 3)
@@ -133,8 +133,14 @@ function virtual_process!(toplevelex::Expr,
             continue
         end
 
-        # XXX: expand macros at this point ?
-        # macro can essentially generate `:toplevel` and `:module` expressions
+        # we will end up lowering `x` later, but special case `macrocall`s and expand it here
+        # this is because macros can arbitrarily generate `:toplevel` and `:module` expressions
+        if @isexpr(x, :macrocall)
+            newx = macroexpand_with_err_handling(virtualmod, x)
+            # unless (toplevel) error happened during macro expansion, queue it and continue
+            isnothing(newx) || push!(exs, newx)
+            continue
+        end
 
         # flatten container expression
         if @isexpr(x, :toplevel)

test/test_virtualprocess.jl

@@ -272,6 +272,38 @@ end
         @test_broken isempty(res.toplevel_error_reports)
         @test isempty(res.inference_error_reports)
     end
+
+    # macros that can general :module or :toplevel expressions
+    let
+        # if we don't expand macros before we check `:toplevel` or `:module` expressions,
+        # we may pass `:toplevel` or `:module` expressions to `partially_interpret!` and
+        # eventually we will fail to concretize them and their toplevel definitions
+        vmod = gen_virtual_module()
+        res, interp = @profile_toplevel vmod begin
+            macro wrap_in_mod(blk)
+                ex = Expr(:module, true, esc(:foo), esc(blk))
+                return Expr(:toplevel, ex)
+            end
+
+            @wrap_in_mod begin
+                bar() = nothing
+            end
+        end
+        @test isdefined(vmod, :foo) && isdefined(vmod.foo, :bar)
+
+        vmod = gen_virtual_module()
+        res, interp = @profile_toplevel vmod begin
+            """
+                foo
+
+            Docstring for module `foo`
+            """
+            module foo
+            bar() = nothing
+            end
+        end
+        @test isdefined(vmod, :foo) && isdefined(vmod.foo, :bar)
+    end
 end
 
 @testset "remove `const`" begin