RFC: Experimental API for may-happen in parallel parallelism

base/boot.jl

@@ -155,6 +155,20 @@
 #    name::Symbol
 #end
 
+#struct DetachNode
+#    syncregion
+#    label::Int
+#end
+
+#struct ReattachNode
+#    syncregion
+#    label::Int
+#end
+
+#struct SyncNode
+#    syncregion
+#end
+
 #mutable struct Task
 #    parent::Task
 #    storage::Any
@@ -414,6 +428,9 @@ eval(Core, :(PiNode(val, typ) = $(Expr(:new, :PiNode, :val, :typ))))
 eval(Core, :(PhiCNode(values::Array{Any, 1}) = $(Expr(:new, :PhiCNode, :values))))
 eval(Core, :(UpsilonNode(val) = $(Expr(:new, :UpsilonNode, :val))))
 eval(Core, :(UpsilonNode() = $(Expr(:new, :UpsilonNode))))
+eval(Core, :(SyncNode(token) = $(Expr(:new, :SyncNode, :token))))
+eval(Core, :(DetachNode(token, bb::Int) = $(Expr(:new, :DetachNode, :token, :bb))))
+eval(Core, :(ReattachNode(token, bb::Int) = $(Expr(:new, :ReattachNode, :token, :bb))))
 eval(Core, :(LineInfoNode(mod::Module, @nospecialize(method), file::Symbol, line::Int, inlined_at::Int) =
              $(Expr(:new, :LineInfoNode, :mod, :method, :file, :line, :inlined_at))))
 eval(Core, :(CodeInstance(mi::MethodInstance, @nospecialize(rettype), @nospecialize(inferred_const),
@@ -497,11 +514,13 @@ module IR
 export CodeInfo, MethodInstance, CodeInstance, GotoNode, GotoIfNot, ReturnNode,
     NewvarNode, SSAValue, Slot, SlotNumber, TypedSlot, Argument,
     PiNode, PhiNode, PhiCNode, UpsilonNode, LineInfoNode,
+    DetachNode, ReattachNode, SyncNode,
     Const, PartialStruct
 
 import Core: CodeInfo, MethodInstance, CodeInstance, GotoNode, GotoIfNot, ReturnNode,
     NewvarNode, SSAValue, Slot, SlotNumber, TypedSlot, Argument,
     PiNode, PhiNode, PhiCNode, UpsilonNode, LineInfoNode,
+    DetachNode, ReattachNode, SyncNode,
     Const, PartialStruct
 
 end

base/compiler/abstractinterpretation.jl

@@ -1763,6 +1763,8 @@ function abstract_eval_statement(interp::AbstractInterpreter, @nospecialize(e),
                 end
             end
         end
+    elseif e.head === :syncregion
+        t = abstract_eval_statement(interp, e.args[1], vtypes, sv)
     else
         t = abstract_eval_value_expr(interp, e, vtypes, sv)
     end
@@ -1908,6 +1910,20 @@ function typeinf_local(interp::AbstractInterpreter, frame::InferenceState)
                 changes[sn] = VarState(Bottom, true)
             elseif isa(stmt, GotoNode)
                 pc´ = (stmt::GotoNode).label
+            elseif isa(stmt, DetachNode)
+                # A detach node has two edges we need to add
+                # the reattach edge to the work queue
+                l = (stmt::DetachNode).label
+                newstate_reattach = stupdate!(states[l], changes)
+                if newstate_reattach !== nothing
+                    if l < frame.pc´´
+                        frame.pc´´ = l
+                    end
+                    push!(W, l)
+                    states[l] = newstate_reattach
+                end
+            elseif isa(stmt, ReattachNode)
+                pc´ = (stmt::ReattachNode).label
             elseif isa(stmt, GotoIfNot)
                 condx = stmt.cond
                 condt = abstract_eval_value(interp, condx, changes, frame)
@@ -2025,6 +2041,7 @@ function typeinf_local(interp::AbstractInterpreter, frame::InferenceState)
                         changes = StateUpdate(fname, VarState(Any, false), changes, false)
                     end
                 elseif hd === :code_coverage_effect ||
+                       (stmt isa SyncNode) ||
                        (hd !== :boundscheck && # :boundscheck can be narrowed to Bool
                         hd !== nothing && is_meta_expr_head(hd))
                     # these do not generate code

base/compiler/compiler.jl

@@ -91,6 +91,30 @@ ntuple(f, ::Val{3}) = (@inline; (f(1), f(2), f(3)))
 ntuple(f, ::Val{n}) where {n} = ntuple(f, n::Int)
 ntuple(f, n) = (Any[f(i) for i = 1:n]...,)
 
+function foldl(rf, xs; init)
+    acc = init
+    for x in xs
+        acc = rf(acc, x)
+    end
+    return acc
+end
+
+all(xs) = all(identity, xs)
+function all(f, xs)
+    for x in xs
+        f(x) || return false
+    end
+    return true
+end
+
+any(xs) = any(identity, xs)
+function any(f, xs)
+    for x in xs
+        f(x) && return true
+    end
+    return false
+end
+
 # core docsystem
 include("docs/core.jl")
 
@@ -130,9 +154,11 @@ include("compiler/stmtinfo.jl")
 include("compiler/abstractinterpretation.jl")
 include("compiler/typeinfer.jl")
 include("compiler/optimize.jl") # TODO: break this up further + extract utilities
+include("compiler/tapirpasses.jl")
 
 include("compiler/bootstrap.jl")
 ccall(:jl_set_typeinf_func, Cvoid, (Any,), typeinf_ext_toplevel)
+ccall(:jl_set_lower_tapir_func, Cvoid, (Any,), lower_tapir)
 
 include("compiler/parsing.jl")
 Core.eval(Core, :(_parse = Compiler.fl_parse))

base/compiler/optimize.jl

@@ -178,7 +178,8 @@ is_stmt_throw_block(stmt_flag::UInt8) = stmt_flag & IR_FLAG_THROW_BLOCK ≠ 0
 # if there is no usage within the function), but don't affect the purity
 # of the function as a whole.
 function stmt_affects_purity(@nospecialize(stmt), ir)
-    if isa(stmt, GotoNode) || isa(stmt, ReturnNode)
+    if isa(stmt, GotoNode) || isa(stmt, ReturnNode) ||
+      isa(stmt, DetachNode) || isa(stmt, ReattachNode) || isa(stmt, SyncNode)
         return false
     end
     if isa(stmt, GotoIfNot)
@@ -202,6 +203,9 @@ function stmt_effect_free(@nospecialize(stmt), @nospecialize(rt), src::Union{IRC
     isa(stmt, ReturnNode) && return false
     isa(stmt, GotoNode) && return false
     isa(stmt, GotoIfNot) && return false
+    isa(stmt, DetachNode) && return false
+    isa(stmt, ReattachNode) && return false
+    isa(stmt, SyncNode) && return false
     isa(stmt, Slot) && return false # Slots shouldn't occur in the IR at this point, but let's be defensive here
     isa(stmt, GlobalRef) && return isdefined(stmt.mod, stmt.name)
     if isa(stmt, Expr)
@@ -423,14 +427,31 @@ function run_passes(ci::CodeInfo, sv::OptimizationState)
     @timeit "convert"   ir = convert_to_ircode(ci, sv)
     @timeit "slot2reg"  ir = slot2reg(ir, ci, sv)
     # TODO: Domsorting can produce an updated domtree - no need to recompute here
+    tapir = has_tapir(ir)
+    if tapir
+        tapir_remark_begin(sv)
+        # This must be run just after `slot2reg`:
+        @timeit "Early tapir pass" ir, racy = early_tapir_pass!(ir)
+        if racy
+            # type_lift_pass! is required as a fixup of slot2reg
+            @timeit "post-racy: type lift" ir = type_lift_pass!(ir)
+            @timeit "post-racy: compact" ir = compact!(ir)
+            if JLOptions().debug_level == 2
+                @timeit "post-racy: verify" (verify_ir(ir); verify_linetable(ir.linetable))
+            end
+            return ir
+        end
+    end
     @timeit "compact 1" ir = compact!(ir)
     @timeit "Inlining"  ir = ssa_inlining_pass!(ir, ir.linetable, sv.inlining, ci.propagate_inbounds)
     # @timeit "verify 2" verify_ir(ir)
     @timeit "compact 2" ir = compact!(ir)
     @timeit "SROA"      ir = sroa_pass!(ir)
     @timeit "ADCE"      ir = adce_pass!(ir)
+    @timeit "Tapir DSE" ir = tapir_dead_store_elimination_pass!(ir)
     @timeit "type lift" ir = type_lift_pass!(ir)
     @timeit "compact 3" ir = compact!(ir)
+    @timeit "Late tapir pass" ir = late_tapir_pass!(ir)
     if JLOptions().debug_level == 2
         @timeit "verify 3" (verify_ir(ir); verify_linetable(ir.linetable))
     end
@@ -451,6 +472,13 @@ function convert_to_ircode(ci::CodeInfo, sv::OptimizationState)
     codelocs = ci.codelocs
     ssavaluetypes = ci.ssavaluetypes::Vector{Any}
     ssaflags = ci.ssaflags
+    function requires_reattach(idx)
+        for i in idx+1:length(code)
+            code[i] isa ReattachNode && return true
+            isterminator(code[i]) && break
+        end
+        return false
+    end
     while idx <= length(code)
         codeloc = codelocs[idx]
         if coverage && codeloc != prevloc && codeloc != 0
@@ -468,7 +496,11 @@ function convert_to_ircode(ci::CodeInfo, sv::OptimizationState)
             prevloc = codeloc
         end
         if code[idx] isa Expr && ssavaluetypes[idx] === Union{}
-            if !(idx < length(code) && isa(code[idx + 1], ReturnNode) && !isdefined((code[idx + 1]::ReturnNode), :val))
+            if !(
+                idx < length(code) &&
+                isa(code[idx+1], ReturnNode) &&
+                !isdefined((code[idx+1]::ReturnNode), :val)
+            ) && !requires_reattach(idx)
                 # insert unreachable in the same basic block after the current instruction (splitting it)
                 insert!(code, idx + 1, ReturnNode())
                 insert!(codelocs, idx + 1, codelocs[idx])
@@ -725,6 +757,39 @@ function renumber_ir_elements!(body::Vector{Any}, ssachangemap::Vector{Int}, lab
                     body[i] = ReturnNode(SSAValue(val.id + ssachangemap[val.id]))
                 end
             end
+        elseif isa(el, DetachNode)
+            if labelchangemap[el.label] == typemin(Int)
+                body[i] = nothing
+            else
+                label = el.label + labelchangemap[el.label]
+                syncregion = el.syncregion
+                if isa(syncregion, SSAValue)
+                    syncregion = SSAValue(syncregion.id + ssachangemap[syncregion.id])
+                    body[i] = DetachNode(syncregion, label)
+                else
+                    body[i] = DetachNode(syncregion, label)
+                end
+            end
+        elseif isa(el, ReattachNode)
+            if labelchangemap[el.label] == typemin(Int)
+                body[i] = nothing
+                # TODO: Make this always valid by using fallthrough in ReattachNode
+            else
+                label = el.label + labelchangemap[el.label]
+                syncregion = el.syncregion
+                if isa(syncregion, SSAValue)
+                    syncregion = SSAValue(syncregion.id + ssachangemap[syncregion.id])
+                    body[i] = ReattachNode(syncregion, label)
+                else
+                    body[i] = ReattachNode(syncregion, label)
+                end
+            end
+        elseif isa(el, SyncNode)
+            syncregion = el.syncregion
+            if isa(syncregion, SSAValue)
+                syncregion = SSAValue(syncregion.id + ssachangemap[syncregion.id])
+            end
+            body[i] = SyncNode(syncregion)
         elseif isa(el, SSAValue)
             body[i] = SSAValue(el.id + ssachangemap[el.id])
         elseif isa(el, PhiNode)

base/compiler/ssair/inlining.jl

@@ -412,6 +412,10 @@ function ir_inline_item!(compact::IncrementalCompact, idx::Int, argexprs::Vector
                 end
             elseif isa(stmt′, GotoNode)
                 stmt′ = GotoNode(stmt′.label + bb_offset)
+            elseif isa(stmt′, DetachNode)
+                stmt′ = DetachNode(stmt′.syncregion, stmt′.label + bb_offset)
+            elseif isa(stmt′, ReattachNode)
+                stmt′ = ReattachNode(stmt′.syncregion, stmt′.label + bb_offset)
             elseif isa(stmt′, Expr) && stmt′.head === :enter
                 stmt′ = Expr(:enter, stmt′.args[1]::Int + bb_offset)
             elseif isa(stmt′, GotoIfNot)
@@ -598,6 +602,10 @@ function batch_inline!(todo::Vector{Pair{Int, Any}}, ir::IRCode, linetable::Vect
                 end
             elseif isa(stmt, GotoNode)
                 compact[idx] = GotoNode(state.bb_rename[stmt.label])
+            elseif isa(stmt, DetachNode)
+                compact[idx] = DetachNode(stmt.syncregion, state.bb_rename[stmt.label])
+            elseif isa(stmt, ReattachNode)
+                compact[idx] = ReattachNode(stmt.syncregion, state.bb_rename[stmt.label])
             elseif isa(stmt, Expr) && stmt.head === :enter
                 compact[idx] = Expr(:enter, state.bb_rename[stmt.args[1]::Int])
             elseif isa(stmt, GotoIfNot)

base/compiler/ssair/ir.jl

@@ -2,7 +2,13 @@
 
 Core.PhiNode() = Core.PhiNode(Int32[], Any[])
 
-isterminator(@nospecialize(stmt)) = isa(stmt, GotoNode) || isa(stmt, GotoIfNot) || isa(stmt, ReturnNode)
+isterminator(@nospecialize(stmt)) =
+    isa(stmt, GotoNode) ||
+    isa(stmt, GotoIfNot) ||
+    isa(stmt, ReturnNode) ||
+    isa(stmt, DetachNode) ||
+    isa(stmt, ReattachNode) ||
+    isa(stmt, SyncNode)
 
 struct CFG
     blocks::Vector{BasicBlock}
@@ -49,6 +55,13 @@ function basic_blocks_starts(stmts::Vector{Any})
             # This is a fake dest to force the next stmt to start a bb
             idx < length(stmts) && push!(jump_dests, idx+1)
             push!(jump_dests, stmt.label)
+        elseif isa(stmt, DetachNode)
+            push!(jump_dests, idx+1)
+            push!(jump_dests, stmt.label)
+        elseif isa(stmt, ReattachNode)
+            push!(jump_dests, stmt.label)
+        elseif isa(stmt, SyncNode)
+            push!(jump_dests, idx+1)
         elseif isa(stmt, Expr)
             if stmt.head === :leave
                 # :leave terminates a BB
@@ -99,12 +112,17 @@ function compute_basic_blocks(stmts::Vector{Any})
             # return never has any successors
             continue
         end
-        if isa(terminator, GotoNode)
+        if isa(terminator, GotoNode) || isa(terminator, ReattachNode)
             block′ = block_for_inst(basic_block_index, terminator.label)
             push!(blocks[block′].preds, num)
             push!(b.succs, block′)
             continue
         end
+        if isa(terminator, DetachNode)
+            block′ = block_for_inst(basic_block_index, terminator.label)
+            push!(blocks[block′].preds, num)
+            push!(b.succs, block′)
+        end
         # Conditional Branch
         if isa(terminator, GotoIfNot)
             block′ = block_for_inst(basic_block_index, terminator.dest)
@@ -319,6 +337,9 @@ function setindex!(x::IRCode, @nospecialize(repl), s::SSAValue)
     return x
 end
 
+first_insert_for_bb(ir::IRCode, block::Int) =
+    first_insert_for_bb(ir.stmts.inst, ir.cfg, block)
+
 # SSA values that need renaming
 struct OldSSAValue
     id::Int
@@ -372,6 +393,9 @@ function getindex(x::UseRef)
     elseif isa(stmt, GotoIfNot)
         x.op == 1 || return OOB_TOKEN
         return stmt.cond
+    elseif isa(stmt, DetachNode) || isa(stmt, ReattachNode) || isa(stmt, SyncNode)
+        x.op == 1 || return OOBToken()
+        return stmt.syncregion
     elseif isa(stmt, ReturnNode)
         isdefined(stmt, :val) || return OOB_TOKEN
         x.op == 1 || return OOB_TOKEN
@@ -404,6 +428,7 @@ function is_relevant_expr(e::Expr)
                       :foreigncall, :isdefined, :copyast,
                       :undefcheck, :throw_undef_if_not,
                       :cfunction, :method, :pop_exception,
+                      :syncregion,
                       :new_opaque_closure)
 end
 
@@ -426,6 +451,15 @@ function setindex!(x::UseRef, @nospecialize(v))
     elseif isa(stmt, GotoIfNot)
         x.op == 1 || throw(BoundsError())
         x.stmt = GotoIfNot(v, stmt.dest)
+    elseif isa(stmt, DetachNode)
+        x.op == 1 || throw(BoundsError())
+        x.stmt = DetachNode(v, stmt.label)
+    elseif isa(stmt, ReattachNode)
+        x.op == 1 || throw(BoundsError())
+        x.stmt = ReattachNode(v, stmt.label)
+    elseif isa(stmt, SyncNode)
+        x.op == 1 || throw(BoundsError())
+        x.stmt = SyncNode(v)
     elseif isa(stmt, ReturnNode)
         x.op == 1 || throw(BoundsError())
         x.stmt = typeof(stmt)(v)
@@ -452,7 +486,8 @@ end
 function userefs(@nospecialize(x))
     relevant = (isa(x, Expr) && is_relevant_expr(x)) ||
         isa(x, GotoIfNot) || isa(x, ReturnNode) ||
-        isa(x, PiNode) || isa(x, PhiNode) || isa(x, PhiCNode) || isa(x, UpsilonNode)
+        isa(x, PiNode) || isa(x, PhiNode) || isa(x, PhiCNode) || isa(x, UpsilonNode) ||
+        isa(x, DetachNode) || isa(x, ReattachNode) || isa(x, SyncNode)
     return UseRefIterator(x, relevant)
 end
 
@@ -1003,6 +1038,20 @@ function process_node!(compact::IncrementalCompact, result_idx::Int, inst::Instr
     elseif isa(stmt, GlobalRef) || isa(stmt, GotoNode)
         result[result_idx][:inst] = stmt
         result_idx += 1
+    elseif isa(stmt, DetachNode) || isa(stmt, ReattachNode) || isa(stmt, SyncNode)
+        stmt = renumber_ssa2!(stmt, ssa_rename, used_ssas, late_fixup, result_idx, do_rename_ssa)
+        if compact.cfg_transforms_enabled
+            if isa(stmt, DetachNode)
+                stmt = DetachNode(
+                    stmt.syncregion,
+                    compact.bb_rename_succ[stmt.label],
+                )
+            elseif isa(stmt, ReattachNode)
+                stmt = ReattachNode(stmt.syncregion, compact.bb_rename_succ[stmt.label])
+            end
+        end
+        result[result_idx][:inst] = stmt
+        result_idx += 1
     elseif isa(stmt, GotoIfNot) && compact.cfg_transforms_enabled
         stmt = renumber_ssa2!(stmt, ssa_rename, used_ssas, late_fixup, result_idx, do_rename_ssa)::GotoIfNot
         result[result_idx][:inst] = stmt