Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

augmented forward for jl_f__apply_iterate #599

Closed
vchuravy opened this issue Feb 3, 2023 · 6 comments · Fixed by #601
Closed

augmented forward for jl_f__apply_iterate #599

vchuravy opened this issue Feb 3, 2023 · 6 comments · Fixed by #601
Labels
DJ4EARTH

Comments

@vchuravy
Copy link
Member

vchuravy commented Feb 3, 2023 

julia> autodiff(diffuse1d_model!,Duplicated(j,bj), Duplicated(c,bc) )
ERROR: AssertionError: ("jl_call calling convention not implemented yet",   %9 = call nonnull {} addrspace(10)* ({} addrspace(10)* ({} addrspace(10)*, {} addrspace(10)**, i32)*, {} addrspace(10)*, ...) @julia.call({} addrspace(10)* ({} addrspace(10)*, {} addrspace(10)**, i32)* noundef nonnull @jl_f__apply_iterate, {} addrspace(10)* noundef null, {} addrspace(10)* addrspacecast ({}* inttoptr (i64 140005600269808 to {}*) to {} addrspace(10)*), {} addrspace(10)* addrspacecast ({}* inttoptr (i64 140005611258896 to {}*) to {} addrspace(10)*), {} addrspace(10)* nonnull %4) #14, !dbg !48)
Stacktrace:
  [1] jlcall_augfwd(B::Ptr{LLVM.API.LLVMOpaqueBuilder}, OrigCI::Ptr{LLVM.API.LLVMOpaqueValue}, gutils::Ptr{Nothing}, normalR::Ptr{Ptr{LLVM.API.LLVMOpaqueValue}}, shadowR::Ptr{Ptr{LLVM.API.LLVMOpaqueValue}}, tapeR::Ptr{Ptr{LLVM.API.LLVMOpaqueValue}})
    @ Enzyme.Compiler ~/src/Enzyme/src/compiler.jl:1607
  [2] EnzymeCreateAugmentedPrimal(logic::Enzyme.Logic, todiff::LLVM.Function, retType::Enzyme.API.CDIFFE_TYPE, constant_args::Vector{Enzyme.API.CDIFFE_TYPE}, TA::Enzyme.TypeAnalysis, returnUsed::Bool, shadowReturnUsed::Bool, typeInfo::Enzyme.FnTypeInfo, uncacheable_args::Vector{Bool}, forceAnonymousTape::Bool, width::Int64, atomicAdd::Bool)
    @ Enzyme.API ~/src/Enzyme/src/api.jl:15

encountered by @christophernhill

using Enzyme
using Oceananigans
using KernelAbstractions:  @index, @kernel
using Oceananigans.Utils: launch!
using Oceananigans.Architectures: device

arch=CPU()
FT=Float64

N = 100
topo = (Periodic, Flat, Flat)
grid = RectilinearGrid(arch, FT, topology=topo, size=(N), halo=2, x=(-1, 1), y=(-1, 1), z=(-1, 1))

function del21d!(d2buf,fld::Field)
 g  = fld.grid
 Nx = g.Nx
 for i=1:Nx
  d2buf[i] = fld[i-1,1,1]+fld[i+1,1,1]-2fld[i,1,1]
 end
 return
end

@kernel function del21d_k!(d2buf_k,fld)
  i,j,k = @index(Global,NTuple)
  @inbounds d2buf_k[i,j,k] = fld[i-1,j,k]+fld[i+1,j,k]-2fld[i,j,k]
end

# 2. halo
function halo1d!(fld::Field)
        g  = fld.grid
        Hx = g.Hx
        Nx = g.Nx
        for i=1:Hx
         fld[i-Hx,1,1]=fld[Nx-Hx+i,1,1]
         fld[Nx+i,1,1]=fld[i      ,1,1]
        end
        return nothing
end
# 3. simple model
function diffuse1d_model!(jcost,fld)
  grid=fld.grid
  d2buf=ones(grid.Nx)
  arch=grid.architecture
  d2buf_k = CenterField(grid)
  k=1.0
  dt=0.1
  nsteps=50
  for i in 1:nsteps
    del21d!(d2buf,fld)
    ### kernel style
    event=launch!(arch,grid,:xyz,del21d_k!,d2buf_k,fld)
    wait(device(arch), event) 
    for j in 1:fld.grid.Nx
     d2buf[i] = d2buf_k[i]
    end
    for j in 1:fld.grid.Nx
      fld[j,1,1] = fld[j,1,1] + k*d2buf[j]*dt
    end
    halo1d!(fld)
  end
  jcost[1]=fld[15,1,1].*fld[15,1,1]
  return nothing
end

c     = CenterField(grid)
c2    = CenterField(grid)
f(x, y, z) = exp( -50((x-grid.xᶠᵃᵃ[1])/grid.Lx-0.5)^2 )
set!(c,f)
halo1d!(c)
set!(c2,f)
halo1d!(c2)
j=[0.]
diffuse1d_model!(j,c)

bc=CenterField(grid)
set!(c,f)
set!(bc,0)
j=[0.]
bj=[1.]
autodiff(diffuse1d_model!,Duplicated(j,bj), Duplicated(c,bc) )
@wsmoses wsmoses mentioned this issue Feb 3, 2023
Merged
@vchuravy vchuravy changed the title jl_call calling convention not implemented yet for @jl_f__apply_iterate augmented forward for jl_f__apply_iterate Feb 4, 2023
@vchuravy vchuravy reopened this Feb 4, 2023
@vchuravy
Copy link
Member Author

vchuravy commented Feb 4, 2023

Now:

ERROR: LoadError: Enzyme: Not yet implemented augmented forward for jl_f__apply_iterate
Stacktrace:
 [1] _totuple
   @ ./tuple.jl:401
 [2] Tuple
   @ ./tuple.jl:369
 [3] default_indices
   @ ~/.julia/packages/Oceananigans/DH3N3/src/Grids/new_data.jl:5
 [4] Field
   @ ~/.julia/packages/Oceananigans/DH3N3/src/Fields/field.jl:166
 [5] Field
   @ ~/.julia/packages/Oceananigans/DH3N3/src/Fields/field.jl:0
Stacktrace:
  [1] macro expansion
    @ ~/src/Enzyme/src/compiler.jl:7074 [inlined]
  [2] enzyme_call(::Val{false}, ::Ptr{Nothing}, ::Type{Enzyme.Compiler.AugmentedForwardThunk}, ::Type{Val{1}}, ::Val{true}, ::Type{Tuple{Const{Tuple{DataType, DataType, DataType}}, Duplicated{RectilinearGrid{Float64, Periodic, Flat, Flat, Float64, Float64, Float64, OffsetArrays.OffsetVector{Float64, StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}}, StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}, StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}, CPU}}, Const{Type{Float64}}}}, ::Type{Duplicated{Field{_A, _B, _C, Nothing, RectilinearGrid{Float64, Periodic, Flat, Flat, Float64, Float64, Float64, OffsetArrays.OffsetVector{Float64, StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}}, StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}, StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}, CPU}, _D, _E, _F, _G, Nothing} where {_A, _B, _C, _D, _E, _F, _G}}}, ::Type{Field}, ::Nothing, ::Type{NamedTuple{(Symbol("1"), Symbol("2"), Symbol("3"), Symbol("4"), Symbol("5"), Symbol("6"), Symbol("7"), Symbol("8"), Symbol("9"), Symbol("10"), Symbol("11"), Symbol("12"), Symbol("13"), Symbol("14"), Symbol("15"), Symbol("16"), Symbol("17"), Symbol("18"), Symbol("19"), Symbol("20"), Symbol("21"), Symbol("22"), Symbol("23"), Symbol("24"), Symbol("25"), Symbol("26"), Symbol("27"), Symbol("28"), Symbol("29"), Symbol("30"), Symbol("31"), Symbol("32"), Symbol("33"), Symbol("34"), Symbol("35"), Symbol("36"), Symbol("37"), Symbol("38")), NTuple{38, Any}}}, ::Const{Tuple{DataType, DataType, DataType}}, ::Duplicated{RectilinearGrid{Float64, Periodic, Flat, Flat, Float64, Float64, Float64, OffsetArrays.OffsetVector{Float64, StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}}, StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}, StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}, CPU}}, ::Const{Type{Float64}})
    @ Enzyme.Compiler ~/src/Enzyme/src/compiler.jl:6781
  [3] (::Enzyme.Compiler.AugmentedForwardThunk{Type{Field}, Duplicated{Field{_A, _B, _C, Nothing, RectilinearGrid{Float64, Periodic, Flat, Flat, Float64, Float64, Float64, OffsetArrays.OffsetVector{Float64, StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}}, StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}, StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}, CPU}, _D, _E, _F, _G, Nothing} where {_A, _B, _C, _D, _E, _F, _G}}, Tuple{Const{Tuple{DataType, DataType, DataType}}, Duplicated{RectilinearGrid{Float64, Periodic, Flat, Flat, Float64, Float64, Float64, OffsetArrays.OffsetVector{Float64, StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}}, StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}, StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}, CPU}}, Const{Type{Float64}}}, Val{1}, Nothing, Val{true}(), NamedTuple{(Symbol("1"), Symbol("2"), Symbol("3"), Symbol("4"), Symbol("5"), Symbol("6"), Symbol("7"), Symbol("8"), Symbol("9"), Symbol("10"), Symbol("11"), Symbol("12"), Symbol("13"), Symbol("14"), Symbol("15"), Symbol("16"), Symbol("17"), Symbol("18"), Symbol("19"), Symbol("20"), Symbol("21"), Symbol("22"), Symbol("23"), Symbol("24"), Symbol("25"), Symbol("26"), Symbol("27"), Symbol("28"), Symbol("29"), Symbol("30"), Symbol("31"), Symbol("32"), Symbol("33"), Symbol("34"), Symbol("35"), Symbol("36"), Symbol("37"), Symbol("38")), NTuple{38, Any}}})(::Const{Tuple{DataType, DataType, DataType}}, ::Vararg{Any})
    @ Enzyme.Compiler ~/src/Enzyme/src/compiler.jl:6755
  [4] runtime_generic_augfwd(activity::Val{(false, false, true, false)}, width::Val{1}, RT::Val{NamedTuple{(Symbol("1"), Symbol("2"), Symbol("3")), Tuple{Any, Any, Any}}}, f::Type{Field}, df::Nothing, primal_1::Tuple{DataType, DataType, DataType}, shadow_1_1::Nothing, primal_2::RectilinearGrid{Float64, Periodic, Flat, Flat, Float64, Float64, Float64, OffsetArrays.OffsetVector{Float64, StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}}, StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}, StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}, CPU}, shadow_2_1::RectilinearGrid{Float64, Periodic, Flat, Flat, Float64, Float64, Float64, OffsetArrays.OffsetVector{Float64, StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}}, StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}, StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}, CPU}, primal_3::Type{Float64}, shadow_3_1::Nothing)
    @ Enzyme.Compiler ~/src/Enzyme/src/compiler.jl:1020
  [5] macro expansion
    @ ~/src/Enzyme/src/compiler.jl:7081 [inlined]
  [6] enzyme_call
    @ ~/src/Enzyme/src/compiler.jl:6781 [inlined]
  [7] CombinedAdjointThunk
    @ ~/src/Enzyme/src/compiler.jl:6744 [inlined]
  [8] autodiff
    @ ~/src/Enzyme/src/Enzyme.jl:217 [inlined]
  [9] autodiff
    @ ~/src/Enzyme/src/Enzyme.jl:248 [inlined]
 [10] autodiff(::typeof(diffuse1d_model!), ::Duplicated{Vector{Float64}}, ::Duplicated{Field{Center, Center, Center, Nothing, RectilinearGrid{Float64, Periodic, Flat, Flat, Float64, Float64, Float64, OffsetArrays.OffsetVector{Float64, StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}}, StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}, StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}, CPU}, Tuple{Colon, Colon, Colon}, OffsetArrays.OffsetArray{Float64, 3, Array{Float64, 3}}, Float64, FieldBoundaryConditions{BoundaryCondition{Oceananigans.BoundaryConditions.Periodic, Nothing}, BoundaryCondition{Oceananigans.BoundaryConditions.Periodic, Nothing}, Nothing, Nothing, Nothing, Nothing, BoundaryCondition{Oceananigans.BoundaryConditions.Flux, Nothing}}, Nothing, Oceananigans.Fields.FieldBoundaryBuffers{Nothing, Nothing, Nothing, Nothing}}})
    @ Enzyme ~/src/Enzyme/src/Enzyme.jl:368
 [11] top-level scope
    @ ~/src/Enzyme/ocean.jl:83
in expression starting at /home/vchuravy/src/Enzyme/ocean.jl:83

@wsmoses
Copy link
Member

wsmoses commented Feb 5, 2023

Reduced issue:

julia> default_indices(n) = Tuple(Colon() for i=1:n)
default_indices (generic function with 1 method)

julia> default_indices(10)
(Colon(), Colon(), Colon(), Colon(), Colon(), Colon(), Colon(), Colon(), Colon(), Colon())

julia> Enzyme.autodiff(Reverse, default_indices, Const, Const(10))
┌ Warning: Type does not have a definite number of fields
│   T = Tuple{Vararg{Colon}}
└ @ Enzyme ~/.julia/packages/GPUCompiler/qdoh1/src/utils.jl:50
ERROR: Enzyme: Not yet implemented augmented forward for jl_f__apply_iterate
Stacktrace:
 [1] _totuple
   @ ./tuple.jl:349
 [2] Tuple
   @ ./tuple.jl:317
 [3] default_indices
   @ ./REPL[2]:1
 [4] default_indices
   @ ./REPL[2]:0
Stacktrace:
 [1] macro expansion
   @ ~/git/Enzyme.jl/src/compiler.jl:7677 [inlined]
 [2] enzyme_call
   @ ~/git/Enzyme.jl/src/compiler.jl:7377 [inlined]
 [3] CombinedAdjointThunk
   @ ~/git/Enzyme.jl/src/compiler.jl:7340 [inlined]
 [4] autodiff(#unused#::EnzymeCore.ReverseMode{false, false}, f::typeof(default_indices), #unused#::Type{Const}, args::Const{Int64})
   @ Enzyme ~/git/Enzyme.jl/src/Enzyme.jl:238
 [5] top-level scope
   @ REPL[4]:1

@wsmoses
Copy link
Member

wsmoses commented Feb 11, 2023 

[2870627] signal (11.1): Segmentation fault
in expression starting at /home/wmoses/git/KernelAbstractions.jl/cnh/ch.jl:82
gc_try_claim_and_push at /home/wmoses/git/Enzyme.jl/julia/src/gc.c:1853
gc_mark_obj16 at /home/wmoses/git/Enzyme.jl/julia/src/gc.c:1904
gc_mark_outrefs at /home/wmoses/git/Enzyme.jl/julia/src/gc.c:2502 [inlined]
gc_mark_loop_ at /home/wmoses/git/Enzyme.jl/julia/src/gc.c:2547
gc_mark_loop at /home/wmoses/git/Enzyme.jl/julia/src/gc.c:2570
_jl_gc_collect at /home/wmoses/git/Enzyme.jl/julia/src/gc.c:2852
ijl_gc_collect at /home/wmoses/git/Enzyme.jl/julia/src/gc.c:3136
maybe_collect at /home/wmoses/git/Enzyme.jl/julia/src/gc.c:936
jl_gc_pool_alloc_inner at /home/wmoses/git/Enzyme.jl/julia/src/gc.c:1299
jl_gc_pool_alloc_noinline at /home/wmoses/git/Enzyme.jl/julia/src/gc.c:1359
jl_gc_alloc_ at /home/wmoses/git/Enzyme.jl/julia/src/julia_internal.h:460
jl_gc_alloc at /home/wmoses/git/Enzyme.jl/julia/src/gc.c:3183
ijl_alloc_svec_uninit at /home/wmoses/git/Enzyme.jl/julia/src/simplevector.c:60
ijl_alloc_svec at /home/wmoses/git/Enzyme.jl/julia/src/simplevector.c:69
save_env at /home/wmoses/git/Enzyme.jl/julia/src/subtype.c:177
_forall_exists_subtype at /home/wmoses/git/Enzyme.jl/julia/src/subtype.c:1537
local_forall_exists_subtype at /home/wmoses/git/Enzyme.jl/julia/src/subtype.c:1454
forall_exists_equal at /home/wmoses/git/Enzyme.jl/julia/src/subtype.c:1497
subtype at /home/wmoses/git/Enzyme.jl/julia/src/subtype.c:1375
subtype_unionall at /home/wmoses/git/Enzyme.jl/julia/src/subtype.c:871
subtype at /home/wmoses/git/Enzyme.jl/julia/src/subtype.c:1325
subtype_unionall at /home/wmoses/git/Enzyme.jl/julia/src/subtype.c:871
subtype at /home/wmoses/git/Enzyme.jl/julia/src/subtype.c:1325
exists_subtype at /home/wmoses/git/Enzyme.jl/julia/src/subtype.c:1512
_forall_exists_subtype at /home/wmoses/git/Enzyme.jl/julia/src/subtype.c:1544
forall_exists_subtype at /home/wmoses/git/Enzyme.jl/julia/src/subtype.c:1560
ijl_subtype_env at /home/wmoses/git/Enzyme.jl/julia/src/subtype.c:2008
ijl_subtype at /home/wmoses/git/Enzyme.jl/julia/src/subtype.c:2054
subtype_tuple_tail at /home/wmoses/git/Enzyme.jl/julia/src/subtype.c:1150
subtype_tuple at /home/wmoses/git/Enzyme.jl/julia/src/subtype.c:1232
subtype at /home/wmoses/git/Enzyme.jl/julia/src/subtype.c:1368
exists_subtype at /home/wmoses/git/Enzyme.jl/julia/src/subtype.c:1512
_forall_exists_subtype at /home/wmoses/git/Enzyme.jl/julia/src/subtype.c:1544
forall_exists_subtype at /home/wmoses/git/Enzyme.jl/julia/src/subtype.c:1560
ijl_types_equal at /home/wmoses/git/Enzyme.jl/julia/src/subtype.c:2110
jl_specializations_get_linfo_ at /home/wmoses/git/Enzyme.jl/julia/src/gf.c:129
ijl_specializations_get_linfo at /home/wmoses/git/Enzyme.jl/julia/src/gf.c:189
#specialize_method#284 at ./compiler/utilities.jl:221
specialize_method at ./compiler/utilities.jl:202 [inlined]
typeinf_edge at ./compiler/typeinfer.jl:861
abstract_call_method at ./compiler/abstractinterpretation.jl:657
abstract_call_gf_by_type at ./compiler/abstractinterpretation.jl:154
abstract_call_known at ./compiler/abstractinterpretation.jl:2060
abstract_call at ./compiler/abstractinterpretation.jl:2136
abstract_apply at ./compiler/abstractinterpretation.jl:1606
abstract_call_known at ./compiler/abstractinterpretation.jl:1971
abstract_call at ./compiler/abstractinterpretation.jl:2136
abstract_call at ./compiler/abstractinterpretation.jl:2110
abstract_eval_statement_expr at ./compiler/abstractinterpretation.jl:2287
abstract_eval_statement at ./compiler/abstractinterpretation.jl:2541
abstract_eval_basic_statement at ./compiler/abstractinterpretation.jl:2832
typeinf_local at ./compiler/abstractinterpretation.jl:3017
typeinf_nocycle at ./compiler/abstractinterpretation.jl:3105
_typeinf at ./compiler/typeinfer.jl:244
typeinf at ./compiler/typeinfer.jl:215
typeinf_edge at ./compiler/typeinfer.jl:920
abstract_call_method at ./compiler/abstractinterpretation.jl:657
abstract_call_gf_by_type at ./compiler/abstractinterpretation.jl:154
abstract_call_known at ./compiler/abstractinterpretation.jl:2060
abstract_call at ./compiler/abstractinterpretation.jl:2136
abstract_apply at ./compiler/abstractinterpretation.jl:1606
abstract_call_known at ./compiler/abstractinterpretation.jl:1971
abstract_call at ./compiler/abstractinterpretation.jl:2136
abstract_call at ./compiler/abstractinterpretation.jl:2110
abstract_eval_statement_expr at ./compiler/abstractinterpretation.jl:2287
abstract_eval_statement at ./compiler/abstractinterpretation.jl:2541
abstract_eval_basic_statement at ./compiler/abstractinterpretation.jl:2808
typeinf_local at ./compiler/abstractinterpretation.jl:3017
typeinf_nocycle at ./compiler/abstractinterpretation.jl:3105
_typeinf at ./compiler/typeinfer.jl:244
typeinf at ./compiler/typeinfer.jl:215
typeinf_edge at ./compiler/typeinfer.jl:920
abstract_call_method at ./compiler/abstractinterpretation.jl:657
abstract_call_gf_by_type at ./compiler/abstractinterpretation.jl:154
abstract_call_known at ./compiler/abstractinterpretation.jl:2060
abstract_call at ./compiler/abstractinterpretation.jl:2136
abstract_apply at ./compiler/abstractinterpretation.jl:1606
abstract_call_known at ./compiler/abstractinterpretation.jl:1971
abstract_call at ./compiler/abstractinterpretation.jl:2136
abstract_call at ./compiler/abstractinterpretation.jl:2110
abstract_eval_statement_expr at ./compiler/abstractinterpretation.jl:2287
abstract_eval_statement at ./compiler/abstractinterpretation.jl:2541
abstract_eval_basic_statement at ./compiler/abstractinterpretation.jl:2832
typeinf_local at ./compiler/abstractinterpretation.jl:3017
typeinf_nocycle at ./compiler/abstractinterpretation.jl:3105
_typeinf at ./compiler/typeinfer.jl:244
typeinf at ./compiler/typeinfer.jl:215
typeinf_edge at ./compiler/typeinfer.jl:920
abstract_call_method at ./compiler/abstractinterpretation.jl:657
abstract_call_gf_by_type at ./compiler/abstractinterpretation.jl:154
abstract_call_known at ./compiler/abstractinterpretation.jl:2060
abstract_call at ./compiler/abstractinterpretation.jl:2136
return_type_tfunc at ./compiler/tfuncs.jl:2487
abstract_call_known at ./compiler/abstractinterpretation.jl:2011
abstract_call at ./compiler/abstractinterpretation.jl:2136
abstract_call at ./compiler/abstractinterpretation.jl:2110
abstract_eval_statement_expr at ./compiler/abstractinterpretation.jl:2287
abstract_eval_statement at ./compiler/abstractinterpretation.jl:2541
abstract_eval_basic_statement at ./compiler/abstractinterpretation.jl:2808
typeinf_local at ./compiler/abstractinterpretation.jl:3017
typeinf_nocycle at ./compiler/abstractinterpretation.jl:3105
_typeinf at ./compiler/typeinfer.jl:244
typeinf at ./compiler/typeinfer.jl:215
typeinf_ext at ./compiler/typeinfer.jl:1045
typeinf_ext_toplevel at ./compiler/typeinfer.jl:1078
typeinf_ext_toplevel at ./compiler/typeinfer.jl:1074
jfptr_typeinf_ext_toplevel_20316 at /home/wmoses/git/Enzyme.jl/julia/usr/lib/julia/sys-debug.so (unknown line)
_jl_invoke at /home/wmoses/git/Enzyme.jl/julia/src/gf.c:2672
ijl_apply_generic at /home/wmoses/git/Enzyme.jl/julia/src/gf.c:2873
jl_apply at /home/wmoses/git/Enzyme.jl/julia/src/julia.h:1880
jl_type_infer at /home/wmoses/git/Enzyme.jl/julia/src/gf.c:317
jl_generate_fptr_impl at /home/wmoses/git/Enzyme.jl/julia/src/jitlayers.cpp:431
jl_compile_method_internal at /home/wmoses/git/Enzyme.jl/julia/src/gf.c:2287
_jl_invoke at /home/wmoses/git/Enzyme.jl/julia/src/gf.c:2683
ijl_apply_generic at /home/wmoses/git/Enzyme.jl/julia/src/gf.c:2873
unknown function (ip: 0x7f636c168aaa)
macro expansion at /home/wmoses/git/Enzyme.jl/src/compiler.jl:7807 [inlined]
enzyme_call at /home/wmoses/git/Enzyme.jl/src/compiler.jl:7507 [inlined]
CombinedAdjointThunk at /home/wmoses/git/Enzyme.jl/src/compiler.jl:7470 [inlined]
autodiff at /home/wmoses/git/Enzyme.jl/src/Enzyme.jl:187 [inlined]
autodiff at /home/wmoses/git/Enzyme.jl/src/Enzyme.jl:218
unknown function (ip: 0x7f635bf217fb)
_jl_invoke at /home/wmoses/git/Enzyme.jl/julia/src/gf.c:2691
ijl_apply_generic at /home/wmoses/git/Enzyme.jl/julia/src/gf.c:2873
jl_apply at /home/wmoses/git/Enzyme.jl/julia/src/julia.h:1880
do_call at /home/wmoses/git/Enzyme.jl/julia/src/interpreter.c:125
eval_value at /home/wmoses/git/Enzyme.jl/julia/src/interpreter.c:222
eval_stmt_value at /home/wmoses/git/Enzyme.jl/julia/src/interpreter.c:173
eval_body at /home/wmoses/git/Enzyme.jl/julia/src/interpreter.c:602
jl_interpret_toplevel_thunk at /home/wmoses/git/Enzyme.jl/julia/src/interpreter.c:758
jl_toplevel_eval_flex at /home/wmoses/git/Enzyme.jl/julia/src/toplevel.c:910
jl_toplevel_eval_flex at /home/wmoses/git/Enzyme.jl/julia/src/toplevel.c:853
ijl_toplevel_eval at /home/wmoses/git/Enzyme.jl/julia/src/toplevel.c:919
ijl_toplevel_eval_in at /home/wmoses/git/Enzyme.jl/julia/src/toplevel.c:969
eval at ./boot.jl:370 [inlined]
include_string at ./loading.jl:1880
jl_fptr_args at /home/wmoses/git/Enzyme.jl/julia/src/gf.c:2333
_jl_invoke at /home/wmoses/git/Enzyme.jl/julia/src/gf.c:2672
ijl_apply_generic at /home/wmoses/git/Enzyme.jl/julia/src/gf.c:2873
_include at ./loading.jl:1940
include at ./Base.jl:457
jfptr_include_27081 at /home/wmoses/git/Enzyme.jl/julia/usr/lib/julia/sys-debug.so (unknown line)
_jl_invoke at /home/wmoses/git/Enzyme.jl/julia/src/gf.c:2672
ijl_apply_generic at /home/wmoses/git/Enzyme.jl/julia/src/gf.c:2873
exec_options at ./client.jl:307
_start at ./client.jl:522
jfptr__start_55894 at /home/wmoses/git/Enzyme.jl/julia/usr/lib/julia/sys-debug.so (unknown line)
_jl_invoke at /home/wmoses/git/Enzyme.jl/julia/src/gf.c:2672
ijl_apply_generic at /home/wmoses/git/Enzyme.jl/julia/src/gf.c:2873
jl_apply at /home/wmoses/git/Enzyme.jl/julia/src/julia.h:1880
true_main at /home/wmoses/git/Enzyme.jl/julia/src/jlapi.c:573
jl_repl_entrypoint at /home/wmoses/git/Enzyme.jl/julia/src/jlapi.c:717
jl_load_repl at /home/wmoses/git/Enzyme.jl/julia/cli/loader_lib.c:543
main at /home/wmoses/git/Enzyme.jl/julia/cli/loader_exe.c:58
unknown function (ip: 0x7f6388198d8f)
__libc_start_main at /lib/x86_64-linux-gnu/libc.so.6 (unknown line)
_start at ../Enzyme.jl/julia/julia (unknown line)
Allocations: 117352873 (Pool: 0; Other: 117352873); GC: 142
Allocations: 117352873 (Pool: 0; Other: 117352873); GC: 142

Thread 1 received signal SIGKILL, Killed.
0x00007f63875f8203 in gc_try_claim_and_push (mq=0x55e6727b0f58, _obj=0x32, nptr=0x7ffe93cddd88) at /home/wmoses/git/Enzyme.jl/julia/src/gc.c:1853
warning: Source file is more recent than executable.
1853	    if (!gc_old(o->header) && nptr)
(rr) bt
#0  0x00007f63875f8203 in gc_try_claim_and_push (mq=0x55e6727b0f58, _obj=0x32, nptr=0x7ffe93cddd88) at /home/wmoses/git/Enzyme.jl/julia/src/gc.c:1853
#1  0x00007f63875f8aa5 in gc_mark_obj16 (ptls=0x55e6727af3f0, obj16_parent=0x7f62b154ed90 "\270\355T\261b\177", obj16_begin=0x7f62c233848a, obj16_end=0x7f62c23384ec, nptr=200) at /home/wmoses/git/Enzyme.jl/julia/src/gc.c:1904
#2  0x00007f63875fbe60 in gc_mark_outrefs (meta_updated=0, _new_obj=0x7f62b154ed90, mq=0x55e6727b0f58, ptls=0x55e6727af3f0) at /home/wmoses/git/Enzyme.jl/julia/src/gc.c:2502
#3  gc_mark_loop_ (ptls=0x55e6727af3f0, mq=0x55e6727b0f58) at /home/wmoses/git/Enzyme.jl/julia/src/gc.c:2547
#4  0x00007f63875fc17c in gc_mark_loop (ptls=0x55e6727af3f0) at /home/wmoses/git/Enzyme.jl/julia/src/gc.c:2570
#5  0x00007f63875fe927 in _jl_gc_collect (ptls=0x55e6727af3f0, collection=JL_GC_AUTO) at /home/wmoses/git/Enzyme.jl/julia/src/gc.c:2852
#6  0x00007f63875ff471 in ijl_gc_collect (collection=JL_GC_AUTO) at /home/wmoses/git/Enzyme.jl/julia/src/gc.c:3136
#7  0x00007f63875f5762 in maybe_collect (ptls=0x55e6727af3f0) at /home/wmoses/git/Enzyme.jl/julia/src/gc.c:936
#8  0x00007f63875f68b6 in jl_gc_pool_alloc_inner (ptls=0x55e6727af3f0, pool_offset=1280, osize=64) at /home/wmoses/git/Enzyme.jl/julia/src/gc.c:1299
#9  0x00007f63875f6c1c in jl_gc_pool_alloc_noinline (ptls=0x55e6727af3f0, pool_offset=1280, osize=64) at /home/wmoses/git/Enzyme.jl/julia/src/gc.c:1359
#10 0x00007f63875f2cf4 in jl_gc_alloc_ (ptls=0x55e6727af3f0, sz=56, ty=0x7f6376e79040 <jl_system_image_data+85703936>) at /home/wmoses/git/Enzyme.jl/julia/src/julia_internal.h:460
#11 0x00007f63875ff639 in jl_gc_alloc (ptls=0x55e6727af3f0, sz=56, ty=0x7f6376e79040 <jl_system_image_data+85703936>) at /home/wmoses/git/Enzyme.jl/julia/src/gc.c:3183
#12 0x00007f63875e21a4 in ijl_alloc_svec_uninit (n=6) at /home/wmoses/git/Enzyme.jl/julia/src/simplevector.c:60
#13 0x00007f63875e21e8 in ijl_alloc_svec (n=6) at /home/wmoses/git/Enzyme.jl/julia/src/simplevector.c:69
#14 0x00007f63876146df in save_env (e=0x7ffe93cdeca0, root=0x7ffe93cde548, se=0x7ffe93cde570) at /home/wmoses/git/Enzyme.jl/julia/src/subtype.c:177
#15 0x00007f638761a646 in _forall_exists_subtype (x=0x7f635fbe9b30, y=0x7f637cba37d0, e=0x7ffe93cdeca0, param=2, count=0x7ffe93cde5f4, noRmore=0x7ffe93cde5f8) at /home/wmoses/git/Enzyme.jl/julia/src/subtype.c:1537
#16 0x00007f638761a01e in local_forall_exists_subtype (x=0x7f635fbe9b30, y=0x7f637cba37d0, e=0x7ffe93cdeca0, param=2, limit_slow=1) at /home/wmoses/git/Enzyme.jl/julia/src/subtype.c:1454
#17 0x00007f638761a39f in forall_exists_equal (x=0x7f635fbe9b30, y=0x7f637cba37d0, e=0x7ffe93cdeca0) at /home/wmoses/git/Enzyme.jl/julia/src/subtype.c:1497
^C#18 0x00007f63876199d8 in subtype (x=Quit
(rr) f 0
#0  0x00007f63875f8203 in gc_try_claim_and_push (mq=0x55e6727b0f58, _obj=0x32, nptr=0x7ffe93cddd88) at /home/wmoses/git/Enzyme.jl/julia/src/gc.c:1853
1853	    if (!gc_old(o->header) && nptr)
(rr) l
1848	{
1849	    if (_obj == NULL)
1850	        return;
1851	    jl_value_t *obj = (jl_value_t *)jl_assume(_obj);
1852	    jl_taggedvalue_t *o = jl_astaggedvalue(obj);
1853	    if (!gc_old(o->header) && nptr)
1854	        *nptr |= 1;
1855	    if (gc_try_setmark_tag(o, GC_MARKED))
1856	        gc_markqueue_push(mq, obj);
1857	}
(rr) p o
$1 = (jl_taggedvalue_t *) 0x2a
(rr) p obj
$2 = (jl_value_t *) 0x32
(rr) up 1
#1  0x00007f63875f8aa5 in gc_mark_obj16 (ptls=0x55e6727af3f0, obj16_parent=0x7f62b154ed90 "\270\355T\261b\177", obj16_begin=0x7f62c233848a, obj16_end=0x7f62c23384ec, nptr=200) at /home/wmoses/git/Enzyme.jl/julia/src/gc.c:1904
1904	            gc_try_claim_and_push(mq, new_obj, &nptr);
(rr) l
1899	        slot = &((jl_value_t **)obj16_parent)[*obj16_begin];
1900	        new_obj = *slot;
1901	        if (new_obj != NULL) {
1902	            verify_parent2("object", obj16_parent, slot, "field(%d)",
1903	                            gc_slot_to_fieldidx(obj16_parent, slot, (jl_datatype_t*)jl_typeof(obj16_parent)));
1904	            gc_try_claim_and_push(mq, new_obj, &nptr);
1905	            if (obj16_begin + 1 != obj16_end) {
1906	                gc_try_claim_and_push(mq, new_obj, &nptr);
1907	            }
1908	            else {
(rr) p new_obj
$3 = (jl_value_t *) 0x32
(rr) p slot
$4 = (jl_value_t **) 0x7f62b154ed98
(rr) p *(jl_value_t **) 0x7f62b154ed98
$5 = (jl_value_t *) 0x32
(rr) watch *(jl_value_t **) 0x7f62b154ed98
Hardware watchpoint 1: *(jl_value_t **) 0x7f62b154ed98
(rr) reverse-continue
Continuing.

@wsmoses
Copy link
Member

wsmoses commented Feb 11, 2023 

(rr) reverse-continue
Continuing.
[New Thread 2870627.2870629]

Thread 1 hit Hardware watchpoint 1: *(jl_value_t **) 0x7f62b154ed98

Old value = (jl_value_t *) 0x32
New value = (jl_value_t *) 0x2f
0x00007f63875b606d in _new_array_ (atype=0x7f6376e76450 <jl_system_image_data+85692688>, ndims=1, dims=0x7f62b154ebe8, isunboxed=0 '\000', hasptr=0 '\000', isunion=0 '\000', zeroinit=0 '\000', elsz=8) at /home/wmoses/git/Enzyme.jl/julia/src/array.c:156
156	    a->length = nel;
(rr) p a
$6 = (jl_array_t *) 0x7f62b154ed90
(rr) p &a->length
$7 = (size_t *) 0x7f62b154ed98
(rr) bt
#0  0x00007f63875b606d in _new_array_ (atype=0x7f6376e76450 <jl_system_image_data+85692688>, ndims=1, dims=0x7f62b154ebe8, isunboxed=0 '\000', hasptr=0 '\000', isunion=0 '\000', zeroinit=0 '\000', elsz=8)
    at /home/wmoses/git/Enzyme.jl/julia/src/array.c:156
#1  0x00007f63875b9f7f in ijl_array_copy (ary=0x7f62b154ebd0) at /home/wmoses/git/Enzyme.jl/julia/src/array.c:1181
#2  0x00007f6371598589 in copy () at array.jl:365
#3  julia_construct_ssa!_20496 (ci=..., ir=..., domtree=..., defuses=0x7f62b0b548f8, slottypes=0x7f62b0b54738, 𝕃ₒ=...) at compiler/ssair/slot2ssa.jl:823
#4  0x00007f637179412a in slot2reg () at compiler/optimize.jl:636
#5  julia_run_passes_21506 (ci=..., sv=..., caller=..., optimize_until=...) at compiler/optimize.jl:502
#6  0x00007f63715ac3a9 in run_passes () at compiler/optimize.jl:517
#7  optimize () at compiler/optimize.jl:466
#8  julia__typeinf_20182 (interp=..., frame=...) at compiler/typeinfer.jl:271
#9  0x00007f63715e7386 in julia_typeinf_20165 (interp=..., frame=...) at compiler/typeinfer.jl:215
#10 0x00007f6371789311 in julia_typeinf_edge_20271 (interp=..., method=..., atype=0x7f636138e4d0, sparams=0x0, caller=...) at compiler/typeinfer.jl:920
#11 0x00007f6371753660 in julia_abstract_call_method_19635 (interp=..., method=..., sig=0x7f636138e4d0, sparams=0x0, hardlimit=<optimized out>, si=..., sv=...) at compiler/abstractinterpretation.jl:657
#12 0x00007f63719ef351 in julia_abstract_call_gf_by_type_19554 (interp=<error reading variable: Cannot access memory at address 0x32>, f=0x7f63618071c0, arginfo=..., si=..., atype=0x7f636138e4d0, sv=..., max_methods=<optimized out>)
    at compiler/abstractinterpretation.jl:154
#13 0x00007f6371566273 in julia_abstract_call_known_19885 (interp=<error reading variable: Cannot access memory at address 0x32>, f=0x7f63618071c0, arginfo=<error reading variable: Cannot access memory at address 0x8>, 
    si=<error reading variable: Cannot access memory at address 0x0>, sv=..., max_methods=<optimized out>) at compiler/abstractinterpretation.jl:2060
#14 0x00007f63716be917 in julia_abstract_call_19934 (interp=..., arginfo=..., si=<error reading variable: Cannot access memory at address 0x0>, sv=..., max_methods=...) at compiler/abstractinterpretation.jl:2136
#15 0x00007f63717dfcec in julia_abstract_call_19959 (interp=..., arginfo=..., si=..., sv=...) at compiler/abstractinterpretation.jl:2110
#16 0x00007f63717e4cae in julia_abstract_eval_statement_expr_20008 (interp=<error reading variable: Cannot access memory at address 0x32>, e=..., vtypes=0x55e67bda7180, sv=..., mi=...) at compiler/abstractinterpretation.jl:2287
#17 0x00007f637160a231 in julia_abstract_eval_statement_20091 (interp=..., e=0x7f635fc732d0, vtypes=0x55e67bda7180, sv=...) at compiler/abstractinterpretation.jl:2541
#18 0x00007f637160996a in julia_abstract_eval_basic_statement_20130 (interp=..., stmt=0x7f635fc732d0, pc_vartable=0x55e67bda7180, frame=...) at compiler/abstractinterpretation.jl:2832
#19 0x00007f637175c331 in julia_typeinf_local_20137 (interp=..., frame=...) at compiler/abstractinterpretation.jl:3017
#20 0x00007f6371795d27 in julia_typeinf_nocycle_20161 (interp=..., frame=...) at compiler/abstractinterpretation.jl:3105
#21 0x00007f63715abde2 in julia__typeinf_20182 (interp=..., frame=...) at compiler/typeinfer.jl:244
#22 0x00007f63715e7386 in julia_typeinf_20165 (interp=..., frame=...) at compiler/typeinfer.jl:215
#23 0x00007f6371789311 in julia_typeinf_edge_20271 (interp=..., method=..., atype=0x7f637d3ebd50, sparams=0x0, caller=...) at compiler/typeinfer.jl:920
#24 0x00007f6371753660 in julia_abstract_call_method_19635 (interp=..., method=..., sig=0x7f637d3ebd50, sparams=0x0, hardlimit=<optimized out>, si=..., sv=...) at compiler/abstractinterpretation.jl:657
#25 0x00007f63719ef351 in julia_abstract_call_gf_by_type_19554 (interp=<error reading variable: Cannot access memory at address 0x32>, f=0x7f63617f9080, arginfo=..., si=..., atype=0x7f637d3ebd50, sv=..., max_methods=<optimized out>)
    at compiler/abstractinterpretation.jl:154
#26 0x00007f6371566273 in julia_abstract_call_known_19885 (interp=<error reading variable: Cannot access memory at address 0x32>, f=0x7f63617f9080, arginfo=<error reading variable: Cannot access memory at address 0x8>, 
    si=<error reading variable: Cannot access memory at address 0x0>, sv=..., max_methods=<optimized out>) at compiler/abstractinterpretation.jl:2060
#27 0x00007f63716be917 in julia_abstract_call_19934 (interp=..., arginfo=..., si=<error reading variable: Cannot access memory at address 0x0>, sv=..., max_methods=...) at compiler/abstractinterpretation.jl:2136
#28 0x00007f63717dfcec in julia_abstract_call_19959 (interp=..., arginfo=..., si=..., sv=...) at compiler/abstractinterpretation.jl:2110
#29 0x00007f63717e4cae in julia_abstract_eval_statement_expr_20008 (interp=<error reading variable: Cannot access memory at address 0x32>, e=..., vtypes=0x7f62b1842b00, sv=..., mi=...) at compiler/abstractinterpretation.jl:2287
#30 0x00007f637160a231 in julia_abstract_eval_statement_20091 (interp=..., e=0x7f635fc174b0, vtypes=0x7f62b1842b00, sv=...) at compiler/abstractinterpretation.jl:2541
#31 0x00007f637160996a in julia_abstract_eval_basic_statement_20130 (interp=..., stmt=0x7f635fc174b0, pc_vartable=0x7f62b1842b00, frame=...) at compiler/abstractinterpretation.jl:2832
#32 0x00007f637175c331 in julia_typeinf_local_20137 (interp=..., frame=...) at compiler/abstractinterpretation.jl:3017
#33 0x00007f6371795d27 in julia_typeinf_nocycle_20161 (interp=..., frame=...) at compiler/abstractinterpretation.jl:3105
#34 0x00007f63715abde2 in julia__typeinf_20182 (interp=..., frame=...) at compiler/typeinfer.jl:244
#35 0x00007f63715e7386 in julia_typeinf_20165 (interp=..., frame=...) at compiler/typeinfer.jl:215
#36 0x00007f6371789311 in julia_typeinf_edge_20271 (interp=..., method=..., atype=0x7f637d3b3810, sparams=0x0, caller=...) at compiler/typeinfer.jl:920
#37 0x00007f6371753660 in julia_abstract_call_method_19635 (interp=..., method=..., sig=0x7f637d3b3810, sparams=0x0, hardlimit=<optimized out>, si=..., sv=...) at compiler/abstractinterpretation.jl:657
#38 0x00007f63719ef351 in julia_abstract_call_gf_by_type_19554 (interp=<error reading variable: Cannot access memory at address 0x32>, f=0x7f6376422b00 <jl_system_image_data+74864576>, arginfo=..., si=..., atype=0x7f637d3b3810, sv=..., 
    max_methods=<optimized out>) at compiler/abstractinterpretation.jl:154
#39 0x00007f6371566273 in julia_abstract_call_known_19885 (interp=<error reading variable: Cannot access memory at address 0x32>, f=0x7f6376422b00 <jl_system_image_data+74864576>, arginfo=<error reading variable: Cannot access memory at address 0x8>, 
    si=<error reading variable: Cannot access memory at address 0x0>, sv=..., max_methods=<optimized out>) at compiler/abstractinterpretation.jl:2060
#40 0x00007f63716be917 in julia_abstract_call_19934 (interp=..., arginfo=..., si=<error reading variable: Cannot access memory at address 0x0>, sv=..., max_methods=...) at compiler/abstractinterpretation.jl:2136
#41 0x00007f63717dfcec in julia_abstract_call_19959 (interp=..., arginfo=..., si=..., sv=...) at compiler/abstractinterpretation.jl:2110
#42 0x00007f63717e4cae in julia_abstract_eval_statement_expr_20008 (interp=<error reading variable: Cannot access memory at address 0x32>, e=..., vtypes=0x7f637baa5f40, sv=..., mi=...) at compiler/abstractinterpretation.jl:2287
#43 0x00007f637160a231 in julia_abstract_eval_statement_20091 (interp=..., e=0x7f635fbfa8f0, vtypes=0x7f637baa5f40, sv=...) at compiler/abstractinterpretation.jl:2541
#44 0x00007f637160996a in julia_abstract_eval_basic_statement_20130 (interp=..., stmt=0x7f635fbfa8f0, pc_vartable=0x7f637baa5f40, frame=...) at compiler/abstractinterpretation.jl:2832
#45 0x00007f637175c331 in julia_typeinf_local_20137 (interp=..., frame=...) at compiler/abstractinterpretation.jl:3017
#46 0x00007f6371795d27 in julia_typeinf_nocycle_20161 (interp=..., frame=...) at compiler/abstractinterpretation.jl:3105
#47 0x00007f63715abde2 in julia__typeinf_20182 (interp=..., frame=...) at compiler/typeinfer.jl:244
#48 0x00007f63715e7386 in julia_typeinf_20165 (interp=..., frame=...) at compiler/typeinfer.jl:215
#49 0x00007f6371789311 in julia_typeinf_edge_20271 (interp=..., method=..., atype=0x7f637d37b8d0, sparams=0x0, caller=...) at compiler/typeinfer.jl:920
#50 0x00007f6371753660 in julia_abstract_call_method_19635 (interp=..., method=..., sig=0x7f637d37b8d0, sparams=0x0, hardlimit=<optimized out>, si=..., sv=...) at compiler/abstractinterpretation.jl:657
#51 0x00007f63719ef351 in julia_abstract_call_gf_by_type_19554 (interp=<error reading variable: Cannot access memory at address 0x32>, f=0x7f63617ef530, arginfo=..., si=..., atype=0x7f637d37b8d0, sv=..., max_methods=<optimized out>)
    at compiler/abstractinterpretation.jl:154
#52 0x00007f6371566273 in julia_abstract_call_known_19885 (interp=<error reading variable: Cannot access memory at address 0x32>, f=0x7f63617ef530, arginfo=<error reading variable: Cannot access memory at address 0x8>, 
    si=<error reading variable: Cannot access memory at address 0x0>, sv=..., max_methods=<optimized out>) at compiler/abstractinterpretation.jl:2060
#53 0x00007f63716be917 in julia_abstract_call_19934 (interp=..., arginfo=..., si=<error reading variable: Cannot access memory at address 0x0>, sv=..., max_methods=...) at compiler/abstractinterpretation.jl:2136
#54 0x00007f63717dfcec in julia_abstract_call_19959 (interp=..., arginfo=..., si=..., sv=...) at compiler/abstractinterpretation.jl:2110
#55 0x00007f63717e4cae in julia_abstract_eval_statement_expr_20008 (interp=<error reading variable: Cannot access memory at address 0x32>, e=..., vtypes=0x7f635ee061a0, sv=..., mi=...) at compiler/abstractinterpretation.jl:2287
#56 0x00007f637160a231 in julia_abstract_eval_statement_20091 (interp=..., e=0x7f635fbfa550, vtypes=0x7f635ee061a0, sv=...) at compiler/abstractinterpretation.jl:2541
#57 0x00007f637160996a in julia_abstract_eval_basic_statement_20130 (interp=..., stmt=0x7f635fbfa550, pc_vartable=0x7f635ee061a0, frame=...) at compiler/abstractinterpretation.jl:2832
#58 0x00007f637175c331 in julia_typeinf_local_20137 (interp=..., frame=...) at compiler/abstractinterpretation.jl:3017
#59 0x00007f6371795d27 in julia_typeinf_nocycle_20161 (interp=..., frame=...) at compiler/abstractinterpretation.jl:3105
#60 0x00007f63715abde2 in julia__typeinf_20182 (interp=..., frame=...) at compiler/typeinfer.jl:244
#61 0x00007f63715e7386 in julia_typeinf_20165 (interp=..., frame=...) at compiler/typeinfer.jl:215
#62 0x00007f63719759e1 in julia_typeinf_ext_20286 (interp=..., mi=...) at compiler/typeinfer.jl:1045
#63 0x00007f6371618466 in julia_typeinf_ext_toplevel_20319 (interp=..., linfo=...) at compiler/typeinfer.jl:1078
#64 0x00007f63716188de in julia_typeinf_ext_toplevel_20315 (mi=..., world=<optimized out>) at compiler/typeinfer.jl:1074
#65 0x00007f6371618933 in jfptr_typeinf_ext_toplevel_20316 () from /home/wmoses/git/Enzyme.jl/julia/usr/lib/julia/sys-debug.so
#66 0x00007f6387589418 in _jl_invoke (F=0x7f63768e40c0 <jl_system_image_data+79850880>, args=0x7ffe93ce3628, nargs=2, mfunc=0x7f637257f2c0 <jl_system_image_data+9183104>, world=5401) at /home/wmoses/git/Enzyme.jl/julia/src/gf.c:2672

@wsmoses
Copy link
Member

wsmoses commented Feb 11, 2023

Current minimization (getting rid of the for loop lets it succeed)

using Enzyme
using Oceananigans
using KernelAbstractions:  @index, @kernel
using Oceananigans.Utils: launch!
using Oceananigans.Architectures: device

Enzyme.API.looseTypeAnalysis!(true)
Enzyme.API.printall!(true)
EnzymeRules.inactive(::typeof(Oceananigans.Utils.flatten_reduced_dimensions), x...) = nothing
EnzymeRules.inactive(::typeof(Oceananigans.Grids.default_indices), x...) = nothing

arch=CPU()
FT=Float64

N = 100
topo = (Periodic, Flat, Flat)
grid = RectilinearGrid(arch, FT, topology=topo, size=(N), halo=2, x=(-1, 1), y=(-1, 1), z=(-1, 1))

@kernel function del21d_k!(d2buf_k,fld)
  i,j,k = @index(Global,NTuple)
  @inbounds d2buf_k[i,j,k] = fld[i-1,j,k]+fld[i+1,j,k]-2fld[i,j,k]
end

# 2. halo
function halo1d!(fld::Field)
        g  = fld.grid
        Hx = g.Hx
        Nx = g.Nx
        for i=1:Hx
         fld[i-Hx,1,1]=fld[Nx-Hx+i,1,1]
         fld[Nx+i,1,1]=fld[i      ,1,1]
        end
        return nothing
end
# 3. simple model
function diffuse1d_model!(jcost,fld)
  grid=fld.grid
  d2buf=ones(grid.Nx)
  arch=grid.architecture
  d2buf_k = CenterField(grid)
  k=1.0
  dt=0.1
  nsteps=50
  for i in 1:nsteps
    # i = 1
    # del21d!(d2buf,fld)
    for i=1:fld.grid.Nx
      d2buf[i] = fld[i-1,1,1]+fld[i+1,1,1]-2fld[i,1,1]
    end
    ### kernel style 
    workgroup, worksize = Oceananigans.Utils.work_layout(grid, :xyz;
                                      include_right_boundaries=false,
                                      reduced_dimensions=(),
                                      location=nothing, 
                                      only_active_cells=false)

    loop! = del21d_k!(Oceananigans.Architectures.device(arch), workgroup, worksize)
    event = loop!(d2buf_k, fld) #; dependencies=dependencies)
    
    wait(device(arch), event) 
     
    for j in 1:fld.grid.Nx
      d2buf[i] = d2buf_k[i]
    end
    # for j in 1:fld.grid.Nx
    #   fld[j,1,1] = fld[j,1,1] + k*d2buf[j]*dt
    # end
     # halo1d!(fld)
  end
  # jcost[1]=fld[15,1,1].*fld[15,1,1]
  return nothing
end

c     = CenterField(grid)
c2    = CenterField(grid)
f(x, y, z) = exp( -50((x-grid.xᶠᵃᵃ[1])/grid.Lx-0.5)^2 )
set!(c,f)
halo1d!(c)
set!(c2,f)
halo1d!(c2)
j=[0.]
diffuse1d_model!(j,c)

bc=CenterField(grid)
set!(c,f)
set!(bc,0)
j=[0.]
bj=[1.]
autodiff(Reverse, diffuse1d_model!,Duplicated(j,bj), Duplicated(c,bc) )

@wsmoses
Copy link
Member

wsmoses commented Feb 11, 2023 •
edited 

using Enzyme
using Oceananigans
using KernelAbstractions:  @index, @kernel
using Oceananigans.Architectures: device

Enzyme.API.looseTypeAnalysis!(true)
Enzyme.API.printall!(true)
EnzymeRules.inactive(::typeof(Oceananigans.Grids.default_indices), x...) = nothing

arch=CPU()
FT=Float64

N = 100
topo = (Periodic, Flat, Flat)
grid = RectilinearGrid(arch, FT, topology=topo, size=(N), halo=2, x=(-1, 1), y=(-1, 1), z=(-1, 1))

@kernel function del21d_k!(d2buf_k)
  # i,j,k = @index(Global,NTuple)
  # @inbounds d2buf_k[i,j,k] = fld[i-1,j,k]+fld[i+1,j,k]-2fld[i,j,k]
end

function mwork_layout(grid)

    workgroup = (100, 1) # heuristic_workgroup(Nx′, Ny′, Nz′)

    # Drop omitted dimemsions
    worksize = (100, 1, 1)

    return workgroup, worksize
end
# 3. simple model
function diffuse1d_model!(grid)
  d2buf=ones(grid.Nx)
  arch=grid.architecture
  d2buf_k = CenterField(grid)
  nsteps=50
  for i in 1:nsteps
    ### kernel style 
    workgroup, worksize = mwork_layout(grid)

    loop! = del21d_k!(Oceananigans.Architectures.device(arch), workgroup, worksize)
    event = loop!(d2buf_k) #; dependencies=dependencies)
    
    wait(device(arch), event) 
     
    for j in 1:grid.Nx
      d2buf[i] = d2buf_k[i]
    end
  end
  return nothing
end

diffuse1d_model!(grid)

autodiff(Reverse, diffuse1d_model!,Const(grid))

@wsmoses wsmoses mentioned this issue Aug 9, 2023
Closed
@wsmoses wsmoses closed this as completed Nov 17, 2023
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Assignees
No one assigned
Labels
DJ4EARTH
Projects
None yet
Milestone
No milestone
Development

Successfully merging a pull request may close this issue.

Add apply iterate fallback for 1.9 EnzymeAD/Enzyme.jl
2 participants
@vchuravy @wsmoses