From 4c6d30687caa210cdfcfb5e9e5c167b0cbc8c73b Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Thu, 25 May 2023 14:37:20 -0300 Subject: [PATCH] Don't permalloc the pkgimgs, but with an option (#49940) (cherry picked from commit 229269badb899cfcb52ad3e2103058560cec5573) --- base/options.jl | 1 + src/jloptions.c | 12 ++++++++++++ src/jloptions.h | 1 + src/staticdata.c | 29 +++++++++++++++++++---------- test/precompile.jl | 25 ++++++++++++++----------- 5 files changed, 47 insertions(+), 21 deletions(-) diff --git a/base/options.jl b/base/options.jl index dda0e8b377076..b9fa5b6ec5508 100644 --- a/base/options.jl +++ b/base/options.jl @@ -53,6 +53,7 @@ struct JLOptions rr_detach::Int8 strip_metadata::Int8 strip_ir::Int8 + permalloc_pkgimg::Int8 heap_size_hint::UInt64 end diff --git a/src/jloptions.c b/src/jloptions.c index 5d2dee81ddc79..988078c7c58d9 100644 --- a/src/jloptions.c +++ b/src/jloptions.c @@ -86,6 +86,7 @@ JL_DLLEXPORT void jl_init_options(void) 0, // rr-detach 0, // strip-metadata 0, // strip-ir + 0, // permalloc_pkgimg 0, // heap-size-hint }; jl_options_initialized = 1; @@ -207,6 +208,7 @@ static const char opts_hidden[] = " --trace-compile={stderr,name}\n" " Print precompile statements for methods compiled during execution or save to a path\n" " --image-codegen Force generate code in imaging mode\n" + " --permalloc-pkgimg={yes|no*} Copy the data section of package images into memory\n" ; JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) @@ -251,6 +253,7 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) opt_strip_metadata, opt_strip_ir, opt_heap_size_hint, + opt_permalloc_pkgimg }; static const char* const shortopts = "+vhqH:e:E:L:J:C:it:p:O:g:"; static const struct option longopts[] = { @@ -309,6 +312,7 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) { "rr-detach", no_argument, 0, opt_rr_detach }, { "strip-metadata", no_argument, 0, opt_strip_metadata }, { "strip-ir", no_argument, 0, opt_strip_ir }, + { "permalloc-pkgimg",required_argument, 0, opt_permalloc_pkgimg }, { "heap-size-hint", required_argument, 0, opt_heap_size_hint }, { 0, 0, 0, 0 } }; @@ -815,6 +819,14 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) jl_errorf("julia: invalid argument to --heap-size-hint without memory size specified"); break; + case opt_permalloc_pkgimg: + if (!strcmp(optarg,"yes")) + jl_options.permalloc_pkgimg = 1; + else if (!strcmp(optarg,"no")) + jl_options.permalloc_pkgimg = 0; + else + jl_errorf("julia: invalid argument to --permalloc-pkgimg={yes|no} (%s)", optarg); + break; default: jl_errorf("julia: unhandled option -- %c\n" "This is a bug, please report it.", c); diff --git a/src/jloptions.h b/src/jloptions.h index d0aba777027e7..0b72b4c3be062 100644 --- a/src/jloptions.h +++ b/src/jloptions.h @@ -57,6 +57,7 @@ typedef struct { int8_t rr_detach; int8_t strip_metadata; int8_t strip_ir; + int8_t permalloc_pkgimg; uint64_t heap_size_hint; } jl_options_t; diff --git a/src/staticdata.c b/src/staticdata.c index 2e84e72df3b42..c12c9b4b58562 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -71,6 +71,7 @@ External links: */ #include #include +#include #include // printf #include // PRIxPTR @@ -3297,7 +3298,7 @@ static jl_value_t *jl_validate_cache_file(ios_t *f, jl_array_t *depmods, uint64_ } // TODO?: refactor to make it easier to create the "package inspector" -static jl_value_t *jl_restore_package_image_from_stream(ios_t *f, jl_image_t *image, jl_array_t *depmods, int completeinfo) +static jl_value_t *jl_restore_package_image_from_stream(ios_t *f, jl_image_t *image, jl_array_t *depmods, int completeinfo, bool needs_permalloc) { uint64_t checksum = 0; int64_t dataendpos = 0; @@ -3308,7 +3309,7 @@ static jl_value_t *jl_restore_package_image_from_stream(ios_t *f, jl_image_t *im return verify_fail; assert(datastartpos > 0 && datastartpos < dataendpos); - + needs_permalloc = jl_options.permalloc_pkgimg || needs_permalloc; jl_value_t *restored = NULL; jl_array_t *init_order = NULL, *extext_methods = NULL, *new_specializations = NULL, *method_roots_list = NULL, *ext_targets = NULL, *edges = NULL; jl_svec_t *cachesizes_sv = NULL; @@ -3320,14 +3321,22 @@ static jl_value_t *jl_restore_package_image_from_stream(ios_t *f, jl_image_t *im ios_bufmode(f, bm_none); JL_SIGATOMIC_BEGIN(); size_t len = dataendpos - datastartpos; - char *sysimg = (char*)jl_gc_perm_alloc(len, 0, 64, 0); + char *sysimg; + bool success = !needs_permalloc; ios_seek(f, datastartpos); - if (ios_readall(f, sysimg, len) != len || jl_crc32c(0, sysimg, len) != (uint32_t)checksum) { - restored = jl_get_exceptionf(jl_errorexception_type, "Error reading system image file."); + if (needs_permalloc) + sysimg = (char*)jl_gc_perm_alloc(len, 0, 64, 0); + else + sysimg = &f->buf[f->bpos]; + if (needs_permalloc) + success = ios_readall(f, sysimg, len) == len; + if (!success || jl_crc32c(0, sysimg, len) != (uint32_t)checksum) { + restored = jl_get_exceptionf(jl_errorexception_type, "Error reading package image file."); JL_SIGATOMIC_END(); } else { - ios_close(f); + if (needs_permalloc) + ios_close(f); ios_static_buffer(f, sysimg, len); pkgcachesizes cachesizes; jl_restore_system_image_from_stream_(f, image, depmods, checksum, (jl_array_t**)&restored, &init_order, &extext_methods, &new_specializations, &method_roots_list, &ext_targets, &edges, &base, &ccallable_list, &cachesizes); @@ -3372,11 +3381,11 @@ static void jl_restore_system_image_from_stream(ios_t *f, jl_image_t *image, uin jl_restore_system_image_from_stream_(f, image, NULL, checksum | ((uint64_t)0xfdfcfbfa << 32), NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); } -JL_DLLEXPORT jl_value_t *jl_restore_incremental_from_buf(const char *buf, jl_image_t *image, size_t sz, jl_array_t *depmods, int completeinfo) +JL_DLLEXPORT jl_value_t *jl_restore_incremental_from_buf(const char *buf, jl_image_t *image, size_t sz, jl_array_t *depmods, int completeinfo, bool needs_permalloc) { ios_t f; ios_static_buffer(&f, (char*)buf, sz); - jl_value_t *ret = jl_restore_package_image_from_stream(&f, image, depmods, completeinfo); + jl_value_t *ret = jl_restore_package_image_from_stream(&f, image, depmods, completeinfo, needs_permalloc); ios_close(&f); return ret; } @@ -3389,7 +3398,7 @@ JL_DLLEXPORT jl_value_t *jl_restore_incremental(const char *fname, jl_array_t *d "Cache file \"%s\" not found.\n", fname); } jl_image_t pkgimage = {}; - jl_value_t *ret = jl_restore_package_image_from_stream(&f, &pkgimage, depmods, completeinfo); + jl_value_t *ret = jl_restore_package_image_from_stream(&f, &pkgimage, depmods, completeinfo, true); ios_close(&f); return ret; } @@ -3491,7 +3500,7 @@ JL_DLLEXPORT jl_value_t *jl_restore_package_image_from_file(const char *fname, j } #endif - jl_value_t* mod = jl_restore_incremental_from_buf(pkgimg_data, &pkgimage, *plen, depmods, completeinfo); + jl_value_t* mod = jl_restore_incremental_from_buf(pkgimg_data, &pkgimage, *plen, depmods, completeinfo, false); return mod; } diff --git a/test/precompile.jl b/test/precompile.jl index 1477001386f6b..f465c32496ad7 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -328,17 +328,20 @@ precompile_test_harness(false) do dir cachedir = joinpath(dir, "compiled", "v$(VERSION.major).$(VERSION.minor)") cachedir2 = joinpath(dir2, "compiled", "v$(VERSION.major).$(VERSION.minor)") cachefile = joinpath(cachedir, "$Foo_module.ji") - if Base.JLOptions().use_pkgimages == 1 - ocachefile = Base.ocachefile_from_cachefile(cachefile) - else - ocachefile = nothing - end - # use _require_from_serialized to ensure that the test fails if - # the module doesn't reload from the image: - @test_warn "@ccallable was already defined for this method name" begin - @test_logs (:warn, "Replacing module `$Foo_module`") begin - m = Base._require_from_serialized(Base.PkgId(Foo), cachefile, ocachefile) - @test isa(m, Module) + do_pkgimg = Base.JLOptions().use_pkgimages == 1 && Base.JLOptions().permalloc_pkgimg == 1 + if do_pkgimg || Base.JLOptions().use_pkgimages == 0 + if do_pkgimg + ocachefile = Base.ocachefile_from_cachefile(cachefile) + else + ocachefile = nothing + end + # use _require_from_serialized to ensure that the test fails if + # the module doesn't reload from the image: + @test_warn "@ccallable was already defined for this method name" begin + @test_logs (:warn, "Replacing module `$Foo_module`") begin + m = Base._require_from_serialized(Base.PkgId(Foo), cachefile, ocachefile) + @test isa(m, Module) + end end end