Skip to content

Commit

Permalink
make it possible to serialize SnoopCompile results (#39695)
Browse files Browse the repository at this point in the history 
- use `nothing` instead of NULL as a sentinel in specializations
- remove some Serializer restrictions around saving Methods
  • Loading branch information
@JeffBezanson
JeffBezanson committed Mar 31, 2021
1 parent 619fd24 commit a5758aa
Show file tree
Hide file tree
Showing 5 changed files with 27 additions and 24 deletions.
2 changes: 1 addition & 1 deletion src/dump.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -910,7 +910,7 @@ static int jl_collect_methcache_from_mod(jl_typemap_entry_t *ml, void *closure)
size_t i, l = jl_svec_len(specializations);
for (i = 0; i < l; i++) {
jl_method_instance_t *callee = (jl_method_instance_t*)jl_svecref(specializations, i);
if (callee != NULL)
if ((jl_value_t*)callee != jl_nothing)
collect_backedges(callee);
}
}
Expand Down
17 changes: 9 additions & 8 deletions src/gf.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ JL_DLLEXPORT jl_method_instance_t *jl_specializations_get_linfo(jl_method_t *m J
JL_GC_PUSH1(&specializations); // clang-sa doesn't realize this loop uses specializations
for (i = cl; i > 0; i--) {
jl_method_instance_t *mi = jl_atomic_load_relaxed(&data[i - 1]);
if (mi == NULL)
if ((jl_value_t*)mi == jl_nothing)
break;
if (jl_types_equal(mi->specTypes, type)) {
if (locked)
Expand All @@ -143,19 +143,20 @@ JL_DLLEXPORT jl_method_instance_t *jl_specializations_get_linfo(jl_method_t *m J
jl_method_instance_t **data = (jl_method_instance_t**)jl_svec_data(specializations);
for (i = 0; i < cl; i++) {
jl_method_instance_t *mi = jl_atomic_load_relaxed(&data[i]);
if (mi == NULL)
if ((jl_value_t*)mi == jl_nothing)
break;
assert(!jl_types_equal(mi->specTypes, type));
}
}
jl_method_instance_t *mi = jl_get_specialized(m, type, sparams);
JL_GC_PUSH1(&mi);
if (hv ? (i + 1 >= cl || jl_svecref(specializations, i + 1) != NULL) : (i <= 1 || jl_svecref(specializations, i - 2) != NULL)) {
if (hv ? (i + 1 >= cl || jl_svecref(specializations, i + 1) != jl_nothing) : (i <= 1 || jl_svecref(specializations, i - 2) != jl_nothing)) {
size_t ncl = cl < 8 ? 8 : (cl*3)>>1;
jl_svec_t *nc = jl_alloc_svec_uninit(ncl);
if (i > 0)
memcpy((char*)jl_svec_data(nc), jl_svec_data(specializations), sizeof(void*) * i);
memset((char*)jl_svec_data(nc) + sizeof(void*) * i, 0, sizeof(void*) * (ncl - cl));
for (int j = 0; j < ncl - cl; j++)
jl_svecset(nc, j+i, jl_nothing);
if (i < cl)
memcpy((char*)jl_svec_data(nc) + sizeof(void*) * (i + ncl - cl),
(char*)jl_svec_data(specializations) + sizeof(void*) * i,
Expand All @@ -168,7 +169,7 @@ JL_DLLEXPORT jl_method_instance_t *jl_specializations_get_linfo(jl_method_t *m J
}
if (!hv)
i -= 1;
assert(jl_svecref(specializations, i) == NULL);
assert(jl_svecref(specializations, i) == jl_nothing);
jl_svecset(specializations, i, mi); // jl_atomic_store_release?
if (hv) {
// TODO: fuse lookup and insert steps?
Expand Down Expand Up @@ -413,7 +414,7 @@ static int get_method_unspec_list(jl_typemap_entry_t *def, void *closure)
size_t i, l = jl_svec_len(specializations);
for (i = 0; i < l; i++) {
jl_method_instance_t *mi = (jl_method_instance_t*)jl_svecref(specializations, i);
if (mi) {
if ((jl_value_t*)mi != jl_nothing) {
assert(jl_is_method_instance(mi));
if (jl_rettype_inferred(mi, jl_world_counter, jl_world_counter) == jl_nothing)
jl_array_ptr_1d_push((jl_array_t*)closure, (jl_value_t*)mi);
Expand Down Expand Up @@ -1561,7 +1562,7 @@ static void jl_method_table_invalidate(jl_methtable_t *mt, jl_typemap_entry_t *m
l = jl_svec_len(specializations);
for (i = 0; i < l; i++) {
jl_method_instance_t *mi = (jl_method_instance_t*)jl_svecref(specializations, i);
if (mi) {
if ((jl_value_t*)mi != jl_nothing) {
invalidated = 1;
invalidate_external(mi, methodentry->max_world);
invalidate_backedges(mi, methodentry->max_world, "jl_method_table_disable");
Expand Down Expand Up @@ -1728,7 +1729,7 @@ JL_DLLEXPORT void jl_method_table_insert(jl_methtable_t *mt, jl_method_t *method
enum morespec_options ambig = morespec_unknown;
for (i = 0; i < l; i++) {
jl_method_instance_t *mi = jl_atomic_load_relaxed(&data[i]);
if (mi == NULL)
if ((jl_value_t*)mi == jl_nothing)
continue;
isect3 = jl_type_intersection(m->sig, (jl_value_t*)mi->specTypes);
if (jl_type_intersection2(type, isect3, &isect, &isect2)) {
Expand Down
6 changes: 3 additions & 3 deletions src/precompile.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -353,9 +353,9 @@ static int precompile_enq_all_specializations__(jl_typemap_entry_t *def, void *c
jl_svec_t *specializations = def->func.method->specializations;
size_t i, l = jl_svec_len(specializations);
for (i = 0; i < l; i++) {
jl_method_instance_t *mi = (jl_method_instance_t*)jl_svecref(specializations, i);
if (mi != NULL)
precompile_enq_specialization_(mi, closure);
jl_value_t *mi = jl_svecref(specializations, i);
if (mi != jl_nothing)
precompile_enq_specialization_((jl_method_instance_t*)mi, closure);
}
}
if (m->ccallable)
Expand Down
22 changes: 12 additions & 10 deletions stdlib/Serialization/src/Serialization.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -425,7 +425,7 @@ function serialize(s::AbstractSerializer, meth::Method)
serialize(s, nothing)
end
if isdefined(meth, :generator)
serialize(s, Base._uncompressed_ast(meth, meth.generator.inferred)) # XXX: what was this supposed to do?
serialize(s, meth.generator)
else
serialize(s, nothing)
end
Expand All @@ -439,9 +439,12 @@ end

function serialize(s::AbstractSerializer, linfo::Core.MethodInstance)
serialize_cycle(s, linfo) && return
isa(linfo.def, Module) || error("can only serialize toplevel MethodInstance objects")
writetag(s.io, METHODINSTANCE_TAG)
serialize(s, linfo.uninferred)
if isdefined(linfo, :uninferred)
serialize(s, linfo.uninferred)
else
writetag(s.io, UNDEFREF_TAG)
end
serialize(s, nothing) # for backwards compat
serialize(s, linfo.sparam_vals)
serialize(s, Any) # for backwards compat
Expand Down Expand Up @@ -1036,11 +1039,7 @@ function deserialize(s::AbstractSerializer, ::Type{Method})
end
meth.slot_syms = slot_syms
if generator !== nothing
linfo = ccall(:jl_new_method_instance_uninit, Ref{Core.MethodInstance}, ())
linfo.specTypes = Tuple
linfo.inferred = generator
linfo.def = meth
meth.generator = linfo
meth.generator = generator
end
if recursion_relation !== nothing
meth.recursion_relation = recursion_relation
Expand All @@ -1059,7 +1058,10 @@ end
function deserialize(s::AbstractSerializer, ::Type{Core.MethodInstance})
linfo = ccall(:jl_new_method_instance_uninit, Ref{Core.MethodInstance}, (Ptr{Cvoid},), C_NULL)
deserialize_cycle(s, linfo)
linfo.uninferred = deserialize(s)::CodeInfo
tag = Int32(read(s.io, UInt8)::UInt8)
if tag != UNDEFREF_TAG
linfo.uninferred = handle_deserialize(s, tag)::CodeInfo
end
tag = Int32(read(s.io, UInt8)::UInt8)
if tag != UNDEFREF_TAG
# for reading files prior to v1.2
Expand All @@ -1068,7 +1070,7 @@ function deserialize(s::AbstractSerializer, ::Type{Core.MethodInstance})
linfo.sparam_vals = deserialize(s)::SimpleVector
_rettype = deserialize(s) # for backwards compat
linfo.specTypes = deserialize(s)
linfo.def = deserialize(s)::Module
linfo.def = deserialize(s)
return linfo
end

Expand Down
4 changes: 2 additions & 2 deletions test/worlds.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -219,8 +219,8 @@ function instance(f, types)
if isa(specs, Nothing)
elseif isa(specs, Core.SimpleVector)
for i = 1:length(specs)
if isassigned(specs, i)
mi = specs[i]::Core.MethodInstance
mi = specs[i]
if mi isa Core.MethodInstance
if mi.specTypes <: tt && tt <: mi.specTypes
inst = mi
break
Expand Down

2 comments on commit a5758aa

@nanosoldier
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Executing the daily package evaluation, I will reply here when finished:

@nanosoldier runtests(ALL, isdaily = true)

@nanosoldier
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Something went wrong when running your job:

NanosoldierError: failed to run tests: TaskFailedException:
failed process: Process(`docker stop Percival-WDVmlJXs`, ProcessExited(1)) [1]

pipeline_error at ./process.jl:525 [inlined]
#run#596 at ./process.jl:440
run at ./process.jl:438 [inlined]
kill_container at /storage/pkgeval/dev/PkgEval/src/run.jl:364 [inlined]
stop at /storage/pkgeval/dev/PkgEval/src/run.jl:167 [inlined]
#36 at /storage/pkgeval/dev/PkgEval/src/run.jl:234 [inlined]
lock at ./lock.jl:161
macro expansion at /storage/pkgeval/dev/PkgEval/src/run.jl:231 [inlined]
#35 at ./task.jl:356
wait at ./task.jl:267 [inlined]
fetch at ./task.jl:282 [inlined]
#run_sandboxed_test#30 at /storage/pkgeval/dev/PkgEval/src/run.jl:258
macro expansion at /storage/pkgeval/dev/PkgEval/src/run.jl:584 [inlined]
#50 at ./task.jl:356

Logs and partial data can be found here
cc @maleadt

Please sign in to comment.