[JuliaLowering] get macro name in ctx.world; fix lowering iterator

JuliaLowering/src/eval.jl

@@ -33,50 +33,42 @@ end
 # how we end up putting this into Base.
 
 struct LoweringIterator{GraphType}
-    ctx::MacroExpansionContext{GraphType}
+    expr_compat_mode::Bool # later stored in module?
     todo::Vector{Tuple{SyntaxTree{GraphType}, Bool, Int}}
 end
 
-function lower_init(ex::SyntaxTree, mod::Module, macro_world::UInt; expr_compat_mode::Bool=false)
-    graph = ensure_macro_attributes(syntax_graph(ex))
-    ctx = MacroExpansionContext(graph, mod, expr_compat_mode, macro_world)
-    ex = reparent(ctx, ex)
-    LoweringIterator{typeof(graph)}(ctx, [(ex, false, 0)])
+function lower_init(ex::SyntaxTree{T};
+                    expr_compat_mode::Bool=false) where {T}
+    LoweringIterator{T}(expr_compat_mode, [(ex, false, 0)])
 end
 
-function lower_step(iter, push_mod=nothing)
-    if !isnothing(push_mod)
-        push_layer!(iter.ctx, push_mod, false)
-    end
-
+function lower_step(iter, mod, world=Base.get_world_counter())
     if isempty(iter.todo)
         return Core.svec(:done)
     end
 
-    ex, is_module_body, child_idx = pop!(iter.todo)
+    top_ex, is_module_body, child_idx = pop!(iter.todo)
     if child_idx > 0
-        next_child = child_idx + 1
-        if child_idx <= numchildren(ex)
-            push!(iter.todo, (ex, is_module_body, next_child))
-            ex = ex[child_idx]
+        if child_idx <= numchildren(top_ex)
+            push!(iter.todo, (top_ex, is_module_body, child_idx + 1))
+            ex = top_ex[child_idx]
+        elseif is_module_body
+            return Core.svec(:end_module)
         else
-            if is_module_body
-                pop_layer!(iter.ctx)
-                return Core.svec(:end_module)
-            else
-                return lower_step(iter)
-            end
+            return lower_step(iter, mod)
         end
+    else
+        ex = top_ex
     end
 
     k = kind(ex)
     if !(k in KSet"toplevel module")
-        ex = expand_forms_1(iter.ctx, ex)
+        ctx1, ex = expand_forms_1(mod, ex, iter.expr_compat_mode, world)
         k = kind(ex)
     end
     if k == K"toplevel"
         push!(iter.todo, (ex, false, 1))
-        return lower_step(iter)
+        return lower_step(iter, mod)
     elseif k == K"module"
         name = ex[1]
         if kind(name) != K"Identifier"
@@ -93,7 +85,7 @@ function lower_step(iter, push_mod=nothing)
         return Core.svec(:begin_module, newmod_name, std_defs, loc)
     else
         # Non macro expansion parts of lowering
-        ctx2, ex2 = expand_forms_2(iter.ctx, ex)
+        ctx2, ex2 = expand_forms_2(ctx1, ex)
         ctx3, ex3 = resolve_scopes(ctx2, ex2)
         ctx4, ex4 = convert_closures(ctx3, ex3)
         ctx5, ex5 = linearize_ir(ctx4, ex4)
@@ -457,7 +449,7 @@ end
 @fzone "JL: eval" function eval(mod::Module, ex::SyntaxTree;
                                 macro_world::UInt=Base.get_world_counter(),
                                 opts...)
-    iter = lower_init(ex, mod, macro_world; opts...)
+    iter = lower_init(ex; opts...)
     _eval(mod, iter)
 end
 
@@ -466,69 +458,32 @@ function eval(mod::Module, ex; opts...)
     eval(mod, expr_to_syntaxtree(ex); opts...)
 end
 
-if VERSION >= v"1.13.0-DEV.1199" # https://github.com/JuliaLang/julia/pull/59604
-
 function _eval(mod, iter)
-    modules = Module[]
-    new_mod = nothing
+    modules = Module[mod]
     result = nothing
     while true
-        thunk = lower_step(iter, new_mod)::Core.SimpleVector
-        new_mod = nothing
+        thunk = lower_step(iter, modules[end])::Core.SimpleVector
         type = thunk[1]::Symbol
         if type == :done
             break
         elseif type == :begin_module
-            push!(modules, mod)
             filename = something(thunk[4].file, :none)
-            mod = @ccall jl_begin_new_module(mod::Any, thunk[2]::Symbol, thunk[3]::Cint,
-                                             filename::Cstring, thunk[4].line::Cint)::Module
-            new_mod = mod
-        elseif type == :end_module
-            @ccall jl_end_new_module(mod::Module)::Cvoid
-            result = mod
-            mod = pop!(modules)
-        else
-            @assert type == :thunk
-            result = Core.eval(mod, thunk[2])
-        end
-    end
-    @assert isempty(modules)
-    return result
-end
-
-else
-
-function _eval(mod, iter, new_mod=nothing)
-    in_new_mod = !isnothing(new_mod)
-    result = nothing
-    while true
-        thunk = lower_step(iter, new_mod)::Core.SimpleVector
-        new_mod = nothing
-        type = thunk[1]::Symbol
-        if type == :done
-            @assert !in_new_mod
-            break
-        elseif type == :begin_module
-            name = thunk[2]::Symbol
-            std_defs = thunk[3]
-            result = Core.eval(mod,
-                Expr(:module, std_defs, name,
-                     Expr(:block, thunk[4], Expr(:call, m->_eval(m, iter, m), name)))
-            )
+            mod = @ccall jl_begin_new_module(
+                modules[end]::Any, thunk[2]::Symbol, thunk[3]::Cint,
+                filename::Cstring, thunk[4].line::Cint)::Module
+            push!(modules, mod)
         elseif type == :end_module
-            @assert in_new_mod
-            return mod
+            @ccall jl_end_new_module(modules[end]::Module)::Cvoid
+            result = pop!(modules)
         else
             @assert type == :thunk
-            result = Core.eval(mod, thunk[2])
+            result = Core.eval(modules[end], thunk[2])
         end
     end
+    @assert length(modules) === 1
     return result
 end
 
-end
-
 """
     include(mod::Module, path::AbstractString)
 

JuliaLowering/src/macro_expansion.jl

@@ -158,25 +158,49 @@ function fixup_macro_name(ctx::MacroExpansionContext, ex::SyntaxTree)
     end
 end
 
-function eval_macro_name(ctx::MacroExpansionContext, mctx::MacroContext, ex::SyntaxTree)
-    # `ex1` might contain a nontrivial mix of scope layers so we can't just
-    # `eval()` it, as it's already been partially lowered by this point.
-    # Instead, we repeat the latter parts of `lower()` here.
-    ex1 = expand_forms_1(ctx, fixup_macro_name(ctx, ex))
-    ctx2, ex2 = expand_forms_2(ctx, ex1)
-    ctx3, ex3 = resolve_scopes(ctx2, ex2)
-    ctx4, ex4 = convert_closures(ctx3, ex3)
-    ctx5, ex5 = linearize_ir(ctx4, ex4)
+function _eval_dot(world::UInt, mod, ex::SyntaxTree)
+    if kind(ex) === K"."
+        mod = _eval_dot(world, mod, ex[1])
+        ex = ex[2]
+    end
+    kind(ex) in KSet"Identifier Symbol" && mod isa Module ?
+        Base.invoke_in_world(world, getproperty, mod, Symbol(ex.name_val)) :
+        nothing
+end
+
+# If macroexpand(ex[1]) is an identifier or dot-expression, we can simply grab
+# it from the scope layer's module in ctx.macro_world.  Otherwise, we need to
+# eval arbitrary code (which, TODO: does not use the correct world age, and it
+# isn't clear the language is meant to support this).
+function eval_macro_name(ctx::MacroExpansionContext, mctx::MacroContext, ex0::SyntaxTree)
     mod = current_layer(ctx).mod
-    expr_form = to_lowered_expr(ex5)
+    ex = fixup_macro_name(ctx, expand_forms_1(ctx, ex0))
     try
-        # Using Core.eval here fails when precompiling packages since we hit the
-        # user-facing error (in `jl_check_top_level_effect`) that warns that
-        # effects won't persist when eval-ing into a closed module.
-        # `jl_invoke_julia_macro` bypasses this by calling `jl_toplevel_eval` on
-        # the macro name.  This is fine assuming the first argument to the
-        # macrocall is effect-free.
-        ccall(:jl_toplevel_eval, Any, (Any, Any), mod, expr_form)
+        if kind(ex) === K"Value"
+            !(ex.value isa GlobalRef) ? ex.value :
+                Base.invoke_in_world(ctx.macro_world, getglobal,
+                                     ex.value.mod, ex.value.name)
+        elseif kind(ex) === K"Identifier"
+            layer = get(ex, :scope_layer, nothing)
+            if !isnothing(layer)
+                mod = ctx.scope_layers[layer].mod
+            end
+            Base.invoke_in_world(ctx.macro_world, getproperty,
+                                 mod, Symbol(ex.name_val))
+        elseif kind(ex) === K"." &&
+                (ed = _eval_dot(ctx.macro_world, mod, ex); !isnothing(ed))
+            ed
+        else
+            # `ex` might contain a nontrivial mix of scope layers so we can't
+            # just `eval()` it, as it's already been partially lowered by this
+            # point.  Instead, we repeat the latter parts of `lower()` here.
+            ctx2, ex2 = expand_forms_2(ctx, ex)
+            ctx3, ex3 = resolve_scopes(ctx2, ex2)
+            ctx4, ex4 = convert_closures(ctx3, ex3)
+            ctx5, ex5 = linearize_ir(ctx4, ex4)
+            expr_form = to_lowered_expr(ex5)
+            ccall(:jl_toplevel_eval, Any, (Any, Any), mod, expr_form)
+        end
     catch err
         throw(MacroExpansionError(mctx, ex, "Macro not found", :all, err))
     end

JuliaLowering/test/macros.jl

@@ -515,5 +515,27 @@ end
 """)
 code = JuliaLowering.include_string(test_mod, """Mod1.@indirect_MODULE()""")
 @test JuliaLowering.eval(test_mod, code) === test_mod # !== test_mod.Mod1
+# the lowering/eval iterator needs to expand in the correct world age (currently
+# the only way to hit this from user code is macros producing toplevel)
+
+@testset "macros defining macros" begin
+    @eval test_mod macro make_and_use_macro_toplevel()
+        Expr(:toplevel,
+             esc(:(macro from_toplevel_expansion()
+                   :(123)
+               end)),
+             esc(:(@from_toplevel_expansion())))
+    end
+
+    @test JuliaLowering.include_string(
+        test_mod, "@make_and_use_macro_toplevel()"; expr_compat_mode=true) === 123
+
+    if isdefined(test_mod, Symbol("@from_toplevel_expansion"))
+        Base.delete_binding(test_mod, Symbol("@from_toplevel_expansion"))
+    end
+
+    @test JuliaLowering.include_string(
+        test_mod, "@make_and_use_macro_toplevel()"; expr_compat_mode=false) === 123
+end
 
 end

JuliaLowering/test/macros_ir.jl

@@ -147,7 +147,7 @@ _never_exist = @m_not_exist 42
 #---------------------
 MacroExpansionError while expanding @m_not_exist in module Main.TestMod:
 _never_exist = @m_not_exist 42
-#              └──────────┘ ── Macro not found
+#               └─────────┘ ── Macro not found
 Caused by:
 UndefVarError: `@m_not_exist` not defined in `Main.TestMod`
 Suggestion: check for spelling errors or missing imports.