-
-
Notifications
You must be signed in to change notification settings - Fork 189
/
PackageCompiler.jl
794 lines (700 loc) · 29.9 KB
/
PackageCompiler.jl
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
module PackageCompiler
using Base: active_project
using Libdl: Libdl
using Pkg: Pkg
using UUIDs: UUID, uuid1
export create_sysimage, create_app, audit_app, restore_default_sysimage
include("juliaconfig.jl")
const NATIVE_CPU_TARGET = "native"
# See https://github.com/JuliaCI/julia-buildbot/blob/489ad6dee5f1e8f2ad341397dc15bb4fce436b26/master/inventory.py
function default_app_cpu_target()
if Sys.ARCH === :i686
return "pentium4;sandybridge,-xsaveopt,clone_all"
elseif Sys.ARCH === :x86_64
return "generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1)"
elseif Sys.ARCH === :arm
return "armv7-a;armv7-a,neon;armv7-a,neon,vfp4"
elseif Sys.ARCH === :aarch64
return "generic" # is this really the best here?
elseif Sys.ARCH === :powerpc64le
return "pwr8"
else
return "generic"
end
end
current_process_sysimage_path() = unsafe_string(Base.JLOptions().image_file)
all_stdlibs() = readdir(Sys.STDLIB)
@static if VERSION >= v"1.6.0-DEV.1673"
sysimage_modules() = map(x->x.name, Base._sysimage_modules)
else
sysimage_modules() = all_stdlibs()
end
stdlibs_in_sysimage() = intersect(all_stdlibs(), sysimage_modules())
stdlibs_not_in_sysimage() = setdiff(all_stdlibs(), sysimage_modules())
yesno(b::Bool) = b ? "yes" : "no"
function load_all_deps(ctx)
if isdefined(Pkg.Operations, :load_all_deps!)
pkgs = Pkg.Types.PackageSpec[]
Pkg.Operations.load_all_deps!(ctx, pkgs)
else
pkgs = Pkg.Operations.load_all_deps(ctx)
end
return pkgs
end
function source_path(ctx, pkg)
if VERSION <= v"1.4.0-rc1"
Pkg.Operations.source_path(pkg)
else
Pkg.Operations.source_path(ctx, pkg)
end
end
function bitflag()
if Sys.ARCH == :i686
return `-m32`
elseif Sys.ARCH == :x86_64
return `-m64`
else
return ``
end
end
function march()
if Sys.ARCH === :i686
return "-march=pentium4"
elseif Sys.ARCH === :x86_64
return "-march=x86-64"
elseif Sys.ARCH === :arm
return "-march=armv7-a+simd"
elseif Sys.ARCH === :aarch64
return "-march=armv8-a+crypto+simd"
elseif Sys.ARCH === :powerpc64le
return nothing
else
return nothing
end
end
# Overwriting an open file is problematic in Windows
# so move it out of the way first
function move_default_sysimage_if_windows()
if Sys.iswindows() && isfile(default_sysimg_path())
mv(default_sysimg_path(), tempname())
end
end
function run_with_env(cmd, compiler)
if Sys.iswindows()
env = copy(ENV)
env["PATH"] = string(env["PATH"], ";", dirname(compiler))
run(Cmd(cmd; env=env))
else
run(cmd)
end
end
function get_compiler()
cc = get(ENV, "JULIA_CC", nothing)
if cc !== nothing
return cc
end
@static if Sys.iswindows()
return joinpath(Pkg.Artifacts.artifact"x86_64-w64-mingw32", "mingw64", "bin", "gcc.exe")
end
if Sys.which("gcc") !== nothing
return "gcc"
elseif Sys.which("clang") !== nothing
return "clang"
end
error("could not find a compiler, looked for `gcc` and `clang`")
end
function get_julia_cmd()
julia_path = joinpath(Sys.BINDIR, Base.julia_exename())
cmd = `$julia_path --color=yes --startup-file=no`
end
function rewrite_sysimg_jl_only_needed_stdlibs(stdlibs::Vector{String})
sysimg_source_path = Base.find_source_file("sysimg.jl")
sysimg_content = read(sysimg_source_path, String)
# replaces the hardcoded list of stdlibs in sysimg.jl with
# the stdlibs that is given as argument
content = replace(sysimg_content,
r"stdlibs = \[(.*?)\]"s => string("stdlibs = [", join(":" .* stdlibs, ",\n"), "]"))
# Also replace a maximum call which fails for empty collections,
# see https://github.com/JuliaLang/julia/pull/34727
content = replace(content, "maximum(textwidth.(string.(stdlibs)))" =>
"reduce(max, textwidth.(string.(stdlibs)); init=0)")
return content
end
function create_fresh_base_sysimage(stdlibs::Vector{String}; cpu_target::String)
tmp = mktempdir()
sysimg_source_path = Base.find_source_file("sysimg.jl")
base_dir = dirname(sysimg_source_path)
tmp_corecompiler_ji = joinpath(tmp, "corecompiler.ji")
tmp_sys_ji = joinpath(tmp, "sys.ji")
compiler_source_path = joinpath(base_dir, "compiler", "compiler.jl")
@info "PackageCompiler: creating base system image (incremental=false)..."
cd(base_dir) do
# Create corecompiler.ji
cmd = `$(get_julia_cmd()) --cpu-target $cpu_target --output-ji $tmp_corecompiler_ji
-g0 -O0 $compiler_source_path`
@debug "running $cmd"
read(cmd)
# Use that to create sys.ji
new_sysimage_content = rewrite_sysimg_jl_only_needed_stdlibs(stdlibs)
new_sysimage_content *= "\nempty!(Base.atexit_hooks)\n"
new_sysimage_source_path = joinpath(tmp, "sysimage_packagecompiler_$(uuid1()).jl")
write(new_sysimage_source_path, new_sysimage_content)
try
cmd = `$(get_julia_cmd()) --cpu-target $cpu_target
--sysimage=$tmp_corecompiler_ji
-g1 -O0 --output-ji=$tmp_sys_ji $new_sysimage_source_path`
@debug "running $cmd"
read(cmd)
finally
rm(new_sysimage_source_path; force=true)
end
end
return tmp_sys_ji
end
function run_precompilation_script(project::String, sysimg::String, precompile_file::Union{String, Nothing})
tracefile = tempname()
if precompile_file === nothing
arg = `-e ''`
else
arg = `$precompile_file`
end
touch(tracefile)
cmd = `$(get_julia_cmd()) --sysimage=$(sysimg) --project=$project
--compile=all --trace-compile=$tracefile $arg`
@debug "run_precompilation_script: running $cmd"
precompile_file === nothing || @info "===== Start precompile execution ====="
run(cmd) # `Run` this command so that we'll display stdout from the user's script.
precompile_file === nothing || @info "===== End precompile execution ====="
return tracefile
end
function do_ensurecompiled(project, packages, sysimage)
if VERSION >= v"1.6.0"
run(`$(get_julia_cmd()) --sysimage=$sysimage --project=$project -e 'using Pkg; Pkg.precompile(; strict=true)'`)
else
# Pkg.precompile is buggy pre 1.6 (and slow).
use = join("import " .* packages, '\n')
cmd = `$(get_julia_cmd()) --sysimage=$sysimage --project=$project -e $use`
@debug "running $cmd"
read(cmd, String)
end
return nothing
end
function create_sysimg_object_file(object_file::String, packages::Vector{String};
project::String,
base_sysimage::String,
precompile_execution_file::Vector{String},
precompile_statements_file::Vector{String},
cpu_target::String,
script::Union{Nothing, String},
isapp::Bool)
# Handle precompilation
precompile_files = String[]
@debug "running precompilation execution script..."
tracefiles = String[]
for file in (isempty(precompile_execution_file) ? (nothing,) : precompile_execution_file)
tracefile = run_precompilation_script(project, base_sysimage, file)
push!(precompile_files, tracefile)
end
append!(precompile_files, precompile_statements_file)
precompile_code = """
# This @eval prevents symbols from being put into Main
@eval Module() begin
PrecompileStagingArea = Module()
for (_pkgid, _mod) in Base.loaded_modules
if !(_pkgid.name in ("Main", "Core", "Base"))
eval(PrecompileStagingArea, :(const \$(Symbol(_mod)) = \$_mod))
end
end
precompile_files = String[
$(join(map(repr, precompile_files), "\n" * " " ^ 8))
]
for file in precompile_files, statement in eachline(file)
# println(statement)
# The compiler has problem caching signatures with `Vararg{?, N}`. Replacing
# N with a large number seems to work around it.
statement = replace(statement, r"Vararg{(.*?), N} where N" => s"Vararg{\\1, 100}")
try
Base.include_string(PrecompileStagingArea, statement)
catch
# See julia issue #28808
@debug "failed to execute \$statement"
end
end
end # module
"""
# include all packages into the sysimg
julia_code = """
Base.reinit_stdio()
@eval Sys BINDIR = ccall(:jl_get_julia_bindir, Any, ())::String
@eval Sys STDLIB = $(repr(abspath(Sys.BINDIR, "../share/julia/stdlib", string('v', VERSION.major, '.', VERSION.minor))))
Base.init_load_path()
if isdefined(Base, :init_active_project)
Base.init_active_project()
end
Base.init_depot_path()
"""
# Ensure packages to be put into sysimage are precompiled
if !isempty(packages)
do_ensurecompiled(project, packages, base_sysimage)
end
for pkg in packages
julia_code *= """
import $pkg
"""
end
julia_code *= precompile_code
if script !== nothing
julia_code *= """
include($(repr(abspath(script))))
"""
end
if isapp
# If it is an app, there is only one packages
@assert length(packages) == 1
packages[1]
app_start_code = """
Base.@ccallable function julia_main()::Cint
try
$(packages[1]).julia_main()
catch
Core.print("julia_main() threw an unhandled exception")
return 1
end
end
"""
julia_code *= app_start_code
end
julia_code *= """
empty!(LOAD_PATH)
empty!(DEPOT_PATH)
"""
# finally, make julia output the resulting object file
@debug "creating object file at $object_file"
@info "PackageCompiler: creating system image object file, this might take a while..."
cmd = `$(get_julia_cmd()) --cpu-target=$cpu_target
--sysimage=$base_sysimage --project=$project --output-o=$(object_file) -e $julia_code`
@debug "running $cmd"
run(cmd)
end
default_sysimg_path() = abspath(Sys.BINDIR, "..", "lib", "julia", "sys." * Libdl.dlext)
default_sysimg_name() = basename(default_sysimg_path())
backup_default_sysimg_path() = default_sysimg_path() * ".backup"
backup_default_sysimg_name() = basename(backup_default_sysimg_path())
# TODO: Also check UUIDs for stdlibs, not only names
gather_stdlibs_project(project::String) = gather_stdlibs_project(create_pkg_context(project))
function gather_stdlibs_project(ctx)
@assert ctx.env.manifest !== nothing
sysimage_stdlibs = stdlibs_in_sysimage()
non_sysimage_stdlibs = stdlibs_not_in_sysimage()
sysimage_stdlibs_project = String[]
non_sysimage_stdlibs_project = String[]
for (uuid, pkg) in ctx.env.manifest
if pkg.name in sysimage_stdlibs
push!(sysimage_stdlibs_project, pkg.name)
elseif pkg.name in non_sysimage_stdlibs
push!(non_sysimage_stdlibs_project, pkg.name)
end
end
return sysimage_stdlibs_project, non_sysimage_stdlibs_project
end
function check_packages_in_project(ctx, packages)
packages_in_project = collect(keys(ctx.env.project.deps))
if ctx.env.pkg !== nothing
push!(packages_in_project, ctx.env.pkg.name)
end
packages_not_in_project = setdiff(string.(packages), packages_in_project)
if !isempty(packages_not_in_project)
error("package(s) $(join(packages_not_in_project, ", ")) not in project")
end
end
"""
create_sysimage(packages::Union{Symbol, Vector{Symbol}}; kwargs...)
Create a system image that includes the package(s) in `packages`. An attempt
to automatically find a compiler will be done but can also be given explicitly
by setting the environment variable `JULIA_CC` to a path to a compiler
### Keyword arguments:
- `sysimage_path::Union{String,Nothing}`: The path to where
the resulting sysimage should be saved. If set to `nothing` the keyword argument
`replace_default` needs to be set to `true`.
- `project::String`: The project that should be active when the sysimage is created,
defaults to the current active project.
- `precompile_execution_file::Union{String, Vector{String}}`: A file or list of
files that contain code which precompilation statements should be recorded from.
- `precompile_statements_file::Union{String, Vector{String}}`: A file or list of
files that contains precompilation statements that should be included in the sysimage.
- `incremental::Bool`: If `true`, build the new sysimage on top of the sysimage
of the current process otherwise build a new sysimage from scratch. Defaults to `true`.
- `filter_stdlibs::Bool`: If `true`, only include stdlibs that are in the project file.
Defaults to `false`, only set to `true` if you know the potential pitfalls.
- `replace_default::Bool`: If `true`, replaces the default system image which is automatically
used when Julia starts. To replace with the one Julia ships with, use [`restore_default_sysimage()`](@ref)
### Advanced keyword arguments
- `cpu_target::String`: The value to use for `JULIA_CPU_TARGET` when building the system image.
- `script::String`: Path to a file that gets executed in the `--output-o` process.
"""
function create_sysimage(packages::Union{Symbol, Vector{Symbol}}=Symbol[];
sysimage_path::Union{String,Nothing}=nothing,
project::String=dirname(active_project()),
precompile_execution_file::Union{String, Vector{String}}=String[],
precompile_statements_file::Union{String, Vector{String}}=String[],
incremental::Bool=true,
filter_stdlibs=false,
replace_default::Bool=false,
cpu_target::String=NATIVE_CPU_TARGET,
script::Union{Nothing, String}=nothing,
base_sysimage::Union{Nothing, String}=nothing,
isapp::Bool=false)
precompile_statements_file = abspath.(precompile_statements_file)
precompile_execution_file = abspath.(precompile_execution_file)
if replace_default==true
if sysimage_path !== nothing
error("cannot specify `sysimage_path` when `replace_default` is `true`")
end
end
if sysimage_path === nothing
if replace_default == false
error("`sysimage_path` cannot be `nothing` if `replace_default` is `false`")
end
# We will replace the default sysimage so just put it somewhere for now
tmp = mktempdir()
sysimage_path = joinpath(tmp, string("sys.", Libdl.dlext))
end
if filter_stdlibs && incremental
error("must use `incremental=false` to use `filter_stdlibs=true`")
end
# Functions lower down handles `packages` and precompilation file as arrays so convert here
packages = string.(vcat(packages)) # Package names are often used as string inside Julia
precompile_execution_file = vcat(precompile_execution_file)
precompile_statements_file = vcat(precompile_statements_file)
# Instantiate the project
ctx = create_pkg_context(project)
@debug "instantiating project at $(repr(project))"
Pkg.instantiate(ctx)
check_packages_in_project(ctx, packages)
if !incremental
if base_sysimage !== nothing
error("cannot specify `base_sysimage` when `incremental=false`")
end
if filter_stdlibs
sysimage_stdlibs, non_sysimage_stdlibs = gather_stdlibs_project(ctx)
else
sysimage_stdlibs = stdlibs_in_sysimage()
non_sysimage_stdlibs = stdlibs_not_in_sysimage()
end
base_sysimage = create_fresh_base_sysimage(sysimage_stdlibs; cpu_target=cpu_target)
else
if base_sysimage == nothing
base_sysimage = current_process_sysimage_path()
end
end
# Create the sysimage
object_file = tempname() * ".o"
create_sysimg_object_file(object_file, packages;
project=project,
base_sysimage=base_sysimage,
precompile_execution_file=precompile_execution_file,
precompile_statements_file=precompile_statements_file,
cpu_target=cpu_target,
script=script,
isapp=isapp)
create_sysimg_from_object_file(object_file, sysimage_path)
# Maybe replace default sysimage
if replace_default
if !isfile(backup_default_sysimg_path())
@debug "making a backup of default sysimg"
cp(default_sysimg_path(), backup_default_sysimg_path())
end
move_default_sysimage_if_windows()
mv(sysimage_path, default_sysimg_path(); force=true)
@info "PackageCompiler: default sysimg replaced, restart Julia for the new sysimg to be in effect"
end
rm(object_file; force=true)
return nothing
end
function create_sysimg_from_object_file(input_object::String, sysimage_path::String)
julia_libdir = dirname(Libdl.dlpath("libjulia"))
# Prevent compiler from stripping all symbols from the shared lib.
# TODO: On clang on windows this is called something else
if Sys.isapple()
o_file = `-Wl,-all_load $input_object`
else
o_file = `-Wl,--whole-archive $input_object -Wl,--no-whole-archive`
end
extra = Sys.iswindows() ? `-Wl,--export-all-symbols` : ``
compiler = get_compiler()
m = something(march(), ``)
cmd = if VERSION >= v"1.6.0-DEV.1673"
private_libdir = if Base.DARWIN_FRAMEWORK # taken from Libdl tests
if ccall(:jl_is_debugbuild, Cint, ()) != 0
dirname(abspath(Libdl.dlpath(Base.DARWIN_FRAMEWORK_NAME * "_debug")))
else
joinpath(dirname(abspath(Libdl.dlpath(Base.DARWIN_FRAMEWORK_NAME))),"Frameworks")
end
elseif ccall(:jl_is_debugbuild, Cint, ()) != 0
dirname(abspath(Libdl.dlpath("libjulia-internal-debug")))
else
dirname(abspath(Libdl.dlpath("libjulia-internal")))
end
`$compiler $(bitflag()) $m -shared -L$(julia_libdir) -L$(private_libdir) -o $sysimage_path $o_file -ljulia-internal -ljulia $extra`
else
`$compiler $(bitflag()) $m -shared -L$(julia_libdir) -o $sysimage_path $o_file -ljulia $extra`
end
@debug "running $cmd"
run_with_env(cmd, compiler)
return nothing
end
"""
restore_default_sysimage()
Restores the default system image to the one that Julia shipped with.
Useful after running [`create_sysimage`](@ref) with `replace_default=true`.
"""
function restore_default_sysimage()
if !isfile(backup_default_sysimg_path())
error("did not find a backup sysimg")
end
move_default_sysimage_if_windows()
mv(backup_default_sysimg_path(), default_sysimg_path(); force=true)
@info "PackageCompiler: default sysimg restored, restart Julia for the new sysimg to be in effect"
return nothing
end
const REQUIRES = "Requires" => UUID("ae029012-a4dd-5104-9daa-d747884805df")
function create_pkg_context(project)
project_toml_path = Pkg.Types.projectfile_path(project; strict=true)
if project_toml_path === nothing
error("could not find project at $(repr(project))")
end
return Pkg.Types.Context(env=Pkg.Types.EnvCache(project_toml_path))
end
"""
audit_app(app_dir::String)
Check for possible problems with regards to relocatability for
the project at `app_dir`.
!!! warning
This cannot guarantee that the project is free of relocatability problems,
it can only detect some known bad cases and warn about those.
"""
audit_app(app_dir::String) = audit_app(create_pkg_context(app_dir))
function audit_app(ctx::Pkg.Types.Context)
# Check for Requires.jl usage
if REQUIRES in ctx.env.project.deps
@warn "Project has a dependency on Requires.jl, code in `@require` will not be run"
end
for (uuid, pkg) in ctx.env.manifest
if REQUIRES in pkg.deps
@warn "$(pkg.name) has a dependency on Requires.jl, code in `@require` will not be run"
end
end
# Check for build script usage
if isfile(joinpath(dirname(ctx.env.project_file), "deps", "build.jl"))
@warn "Project has a build script, this might indicate that it is not relocatable"
end
pkgs = load_all_deps(ctx)
for pkg in pkgs
pkg_source = source_path(ctx, pkg)
pkg_source === nothing && continue
if isfile(joinpath(pkg_source, "deps", "build.jl"))
@warn "Package $(pkg.name) has a build script, this might indicate that it is not relocatable"
end
end
return
end
"""
create_app(app_source::String, compiled_app::String; kwargs...)
Compile an app with the source in `app_source` to the folder `compiled_app`.
The folder `app_source` needs to contain a package where the package include a
function with the signature
```
julia_main()::Cint
# Perhaps do something based on ARGS
...
end
```
The executable will be placed in a folder called `bin` in `compiled_app` and
when the executabl run the `julia_main` function is called.
An attempt to automatically find a compiler will be done but can also be given
explicitly by setting the envirnment variable `JULIA_CC` to a path to a
compiler.
### Keyword arguments:
- `app_name::String`: an alternative name for the compiled app. If not provided,
the name of the package (as specified in Project.toml) is used.
- `precompile_execution_file::Union{String, Vector{String}}`: A file or list of
files that contain code which precompilation statements should be recorded from.
- `precompile_statements_file::Union{String, Vector{String}}`: A file or list of
files that contains precompilation statements that should be included in the sysimage
for the app.
- `incremental::Bool`: If `true`, build the new sysimage on top of the sysimage
of the current process otherwise build a new sysimage from scratch. Defaults to `false`.
- `filter_stdlibs::Bool`: If `true`, only include stdlibs that are in the project file.
Defaults to `false`, only set to `true` if you know the potential pitfalls.
- `audit::Bool`: Warn about eventual relocatability problems with the app, defaults
to `true`.
- `force::Bool`: Remove the folder `compiled_app` if it exists before creating the app.
### Advanced keyword arguments
- `cpu_target::String`: The value to use for `JULIA_CPU_TARGET` when building the system image.
"""
function create_app(package_dir::String,
app_dir::String;
app_name=nothing,
precompile_execution_file::Union{String, Vector{String}}=String[],
precompile_statements_file::Union{String, Vector{String}}=String[],
incremental=false,
filter_stdlibs=false,
audit=true,
force=false,
c_driver_program::String=joinpath(@__DIR__, "embedding_wrapper.c"),
cpu_target::String=default_app_cpu_target())
precompile_statements_file = abspath.(precompile_statements_file)
precompile_execution_file = abspath.(precompile_execution_file)
package_dir = abspath(package_dir)
ctx = create_pkg_context(package_dir)
if VERSION >= v"1.6.0-DEV.1673"
Pkg.instantiate(ctx, allow_autoprecomp = false)
else
Pkg.instantiate(ctx)
end
if isempty(ctx.env.manifest)
@warn "it is not recommended to create an app without a preexisting manifest"
end
if ctx.env.pkg === nothing
error("expected package to have a `name`-entry")
end
sysimg_name = ctx.env.pkg.name
if app_name === nothing
app_name = sysimg_name
end
sysimg_file = app_name * "." * Libdl.dlext
if isdir(app_dir)
if !force
error("directory $(repr(app_dir)) already exists, use `force=true` to overwrite (will completely",
" remove the directory)")
end
rm(app_dir; force=true, recursive=true)
end
c_driver_program = abspath(c_driver_program)
audit && audit_app(ctx)
mkpath(app_dir)
bundle_julia_libraries(app_dir)
bundle_artifacts(ctx, app_dir)
# TODO: Create in a temp dir and then move it into place?
binpath = joinpath(app_dir, "bin")
mkpath(binpath)
cd(binpath) do
if !incremental
tmp = mktempdir()
# Use workaround at https://github.com/JuliaLang/julia/issues/34064#issuecomment-563950633
# by first creating a normal "empty" sysimage and then use that to finally create the one
# with the @ccallable function
tmp_base_sysimage = joinpath(tmp, "tmp_sys.so")
create_sysimage(Symbol[]; sysimage_path=tmp_base_sysimage, project=package_dir,
incremental=false, filter_stdlibs=filter_stdlibs,
cpu_target=cpu_target)
create_sysimage(Symbol(sysimg_name); sysimage_path=sysimg_file, project=package_dir,
incremental=true,
precompile_execution_file=precompile_execution_file,
precompile_statements_file=precompile_statements_file,
cpu_target=cpu_target,
base_sysimage=tmp_base_sysimage,
isapp=true)
else
create_sysimage(Symbol(sysimg_name); sysimage_path=sysimg_file, project=package_dir,
incremental=incremental, filter_stdlibs=filter_stdlibs,
precompile_execution_file=precompile_execution_file,
precompile_statements_file=precompile_statements_file,
cpu_target=cpu_target,
isapp=true)
end
create_executable_from_sysimg(; sysimage_path=sysimg_file, executable_path=app_name,
c_driver_program_path=c_driver_program,)
if Sys.isapple()
cmd = `install_name_tool -change $sysimg_file @rpath/$sysimg_file $app_name`
@debug "running $cmd"
run(cmd)
end
end
return
end
function create_executable_from_sysimg(;sysimage_path::String,
executable_path::String,
c_driver_program_path::String,)
flags = join((cflags(), ldflags(), ldlibs()), " ")
flags = Base.shell_split(flags)
wrapper = c_driver_program_path
rpath = if VERSION >= v"1.6.0-DEV.1673"
if Sys.iswindows()
``
elseif Sys.isapple()
`-Wl,-rpath,'@executable_path' -Wl,-rpath,'@executable_path/../lib' -Wl,-rpath,'@executable_path/../lib/julia'`
else
`-Wl,-rpath,\$ORIGIN:\$ORIGIN/../lib -Wl,-rpath,\$ORIGIN:\$ORIGIN/../lib/julia`
end
else
if Sys.iswindows()
``
elseif Sys.isapple()
`-Wl,-rpath,'@executable_path' -Wl,-rpath,'@executable_path/../lib'`
else
`-Wl,-rpath,\$ORIGIN:\$ORIGIN/../lib`
end
end
compiler = get_compiler()
m = something(march(), ``)
cmd = `$compiler -DJULIAC_PROGRAM_LIBNAME=$(repr(sysimage_path)) $(bitflag()) $m -o $(executable_path) $(wrapper) $(sysimage_path) -O2 $rpath $flags`
@debug "running $cmd"
run_with_env(cmd, compiler)
return nothing
end
function bundle_julia_libraries(app_dir)
app_libdir = joinpath(app_dir, Sys.isunix() ? "lib" : "bin")
cp(julia_libdir(), app_libdir; force=true)
# We do not want to bundle the sysimg (nor the backup sysimage):
rm(joinpath(app_libdir, "julia", default_sysimg_name()); force=true)
rm(joinpath(app_libdir, "julia", backup_default_sysimg_name()); force=true)
# Remove debug symbol libraries
if Sys.isapple()
v = string(VERSION.major, ".", VERSION.minor)
rm(joinpath(app_libdir, "libjulia.$v.dylib.dSYM"); force=true, recursive=true)
rm(joinpath(app_libdir, "julia", "sys.dylib.dSYM"); force=true, recursive=true)
end
return
end
function bundle_artifacts(ctx, app_dir)
@debug "bundling artifacts..."
pkgs = load_all_deps(ctx)
# Also want artifacts for the project itself
@assert ctx.env.pkg !== nothing
# This is kinda ugly...
ctx.env.pkg.path = dirname(ctx.env.project_file)
push!(pkgs, ctx.env.pkg)
# Collect all artifacts needed for the project
artifact_paths = Set{String}()
for pkg in pkgs
pkg_source_path = source_path(ctx, pkg)
pkg_source_path === nothing && continue
# Check to see if this package has an (Julia)Artifacts.toml
for f in Pkg.Artifacts.artifact_names
artifacts_toml_path = joinpath(pkg_source_path, f)
if isfile(artifacts_toml_path)
@debug "bundling artifacts for $(pkg.name)"
artifact_dict = Pkg.Artifacts.load_artifacts_toml(artifacts_toml_path)
for name in keys(artifact_dict)
meta = Pkg.Artifacts.artifact_meta(name, artifacts_toml_path)
meta == nothing && continue
@debug " \"$name\""
push!(artifact_paths, Pkg.Artifacts.ensure_artifact_installed(name, artifacts_toml_path))
end
break
end
end
end
# Copy the artifacts needed to the app directory
artifact_app_path = joinpath(app_dir, "artifacts")
if !isempty(artifact_paths)
mkpath(artifact_app_path)
end
for artifact_path in artifact_paths
artifact_name = basename(artifact_path)
cp(artifact_path, joinpath(artifact_app_path, artifact_name))
end
return
end
end # module