From 53abea0ca017c11ad0afde904b58ab148c0e6734 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Wed, 24 May 2023 02:48:32 +0800 Subject: [PATCH 1/8] Support `CRYSTAL_LIBRARY_RPATH` on Windows --- src/crystal/system/win32/delay_load.cr | 185 +++++++++++++++++- src/lib_c/x86_64-windows-msvc/c/heapapi.cr | 8 + .../x86_64-windows-msvc/c/libloaderapi.cr | 3 +- .../x86_64-windows-msvc/c/stringapiset.cr | 9 +- src/lib_c/x86_64-windows-msvc/c/winbase.cr | 2 +- src/process/executable_path.cr | 6 +- 6 files changed, 207 insertions(+), 6 deletions(-) create mode 100644 src/lib_c/x86_64-windows-msvc/c/heapapi.cr diff --git a/src/crystal/system/win32/delay_load.cr b/src/crystal/system/win32/delay_load.cr index 3f596822e928..ac5dfcc2ad27 100644 --- a/src/crystal/system/win32/delay_load.cr +++ b/src/crystal/system/win32/delay_load.cr @@ -1,4 +1,5 @@ require "c/delayimp" +require "c/heapapi" lib LibC $image_base = __ImageBase : IMAGE_DOS_HEADER @@ -44,6 +45,187 @@ module Crystal::System::DelayLoad def self.interlocked_exchange(atomic : LibC::HMODULE*, value : LibC::HMODULE) Atomic::Ops.atomicrmw(:xchg, atomic, value, :sequentially_consistent, false) end + + # the functions below work on null-terminated strings; they must not use any C + # runtime features nor the GC! bang methods may allocate memory + + # returns the length in character units of the null-terminated string *str* + private def self.strlen(str : LibC::WCHAR*) : Int32 + len = 0 + while str.value != 0 + len &+= 1 + str += 1 + end + len + end + + # assigns the concatenation of *args* to the buffer at *buf* with the given + # *size*, possibly reallocating it, and returns the new buffer + private def self.strcat(buf : LibC::WCHAR*, size : Int32, *args : *T) : {LibC::WCHAR*, Int32} forall T + new_size = 1 + {% for i in 0...T.size %} + %len{i} = strlen(args[{{ i }}]) + new_size &+= %len{i} + {% end %} + if new_size > size + size = new_size + buf = LibC.HeapReAlloc(LibC.GetProcessHeap, 0, buf, size &* 2).as(LibC::WCHAR*) + end + + ptr = buf + {% for i in 0...T.size %} + src = args[{{ i }}] + while src.value != 0 + ptr.value = src.value + ptr += 1 + src += 1 + end + {% end %} + ptr.value = 0 + + {buf, size} + end + + # if *str* starts with *prefix*, returns the substring with *prefix* removed, + # otherwise returns *str* unmodified + private def self.str_lchop(str : LibC::WCHAR*, prefix : LibC::WCHAR*) : LibC::WCHAR* + src = str + + while prefix.value != 0 + return src unless prefix.value == str.value + prefix += 1 + str += 1 + end + + str + end + + # given *str*, a normalized absolute path of *size* UTF-16 code units, returns + # its parent directory by replacing the last directory separator with a null + # character + private def self.dirname(str : LibC::WCHAR*, size : Int32) + ptr = str + size - 1 + + # C:\foo.exe -> C: + # C:\foo\bar.exe -> C:\foo + # C:\foo\bar\baz.exe -> C:\foo\bar + while ptr != str + if ptr.value === '\\' + ptr.value = 0 + return {str, (ptr - str).to_i32!} + end + ptr -= 1 + end + + {str, size} + end + + # effective returns `::File.dirname(::Process.executable_path).to_utf16` + private def self.get_origin! : {LibC::WCHAR*, Int32} + buf = LibC.HeapAlloc(LibC.GetProcessHeap, 0, LibC::MAX_PATH &* 2).as(LibC::WCHAR*) + len = LibC.GetModuleFileNameW(nil, buf, LibC::MAX_PATH) + return dirname(buf, len.to_i32!) unless WinError.value.error_insufficient_buffer? + + buf = LibC.HeapReAlloc(LibC.GetProcessHeap, 0, buf, 65534).as(LibC::WCHAR*) + len = LibC.GetModuleFileNameW(nil, buf, 32767) + return dirname(buf, len.to_i32!) unless WinError.value.error_insufficient_buffer? + + print_error("FATAL: Failed to get current executable path\n") + LibC.ExitProcess(1) + end + + # converts *utf8_str* to a UTF-16 string + private def self.to_utf16!(utf8_str : LibC::Char*) : LibC::WCHAR* + utf16_size = LibC.MultiByteToWideChar(LibC::CP_UTF8, 0, utf8_str, -1, nil, 0) + utf16_str = LibC.HeapAlloc(LibC.GetProcessHeap, 0, utf16_size &* 2).as(LibC::WCHAR*) + LibC.MultiByteToWideChar(LibC::CP_UTF8, 0, utf8_str, -1, utf16_str, utf16_size) + utf16_str + end + + # replaces all instances of "$ORIGIN" in *str* with the directory containing + # the running executable + # if "$ORIGIN" is not found, returns *str* unmodified without allocating + # memory + private def self.expand_origin!(str : LibC::WCHAR*) : LibC::WCHAR* + origin_prefix = UInt16.static_array(0x24, 0x4F, 0x52, 0x49, 0x47, 0x49, 0x4E, 0x00) # "$ORIGIN".to_utf16 + ptr = str + origin = Pointer(LibC::WCHAR).null + origin_size = 0 + output_size = 1 + + while ptr.value != 0 + new_ptr = str_lchop(ptr, origin_prefix.to_unsafe) + if new_ptr != ptr + origin, origin_size = get_origin! unless origin + output_size &+= origin_size + ptr = new_ptr + next + end + output_size &+= 1 + ptr += 1 + end + + return str unless origin + output = LibC.HeapAlloc(LibC.GetProcessHeap, 0, output_size &* 2).as(LibC::WCHAR*) + dst = output + ptr = str + + while ptr.value != 0 + new_ptr = str_lchop(ptr, origin_prefix.to_unsafe) + if new_ptr != ptr + src = origin + while src.value != 0 + dst.value = src.value + dst += 1 + src += 1 + end + ptr = new_ptr + next + end + dst.value = ptr.value + dst += 1 + ptr += 1 + end + dst.value = 0 + + LibC.HeapFree(LibC.GetProcessHeap, 0, origin) + output + end + + # `dll` is an ASCII base name without directory separators, e.g. `WS2_32.dll` + def self.load_library(dll : LibC::Char*) : LibC::HMODULE + utf16_dll = to_utf16!(dll) + + {% if paths = env("CRYSTAL_LIBRARY_RPATH") %} + {% paths = paths.gsub(/\$\{ORIGIN\}/, "$ORIGIN").split(::Process::PATH_DELIMITER).reject(&.empty?) %} + {% unless paths.empty? %} + size = 0x40 + buf = LibC.HeapAlloc(LibC.GetProcessHeap, 0, size &* 2).as(LibC::WCHAR*) + + {% for path, i in paths %} + # TODO: can this `to_utf16` be done at compilation time? + root = to_utf16!({{ path.ends_with?("\\") ? path : path + "\\" }}.to_unsafe) + root_expanded = expand_origin!(root) + buf, size = strcat(buf, size, root_expanded, utf16_dll) + handle = LibC.LoadLibraryExW(buf, nil, LibC::LOAD_WITH_ALTERED_SEARCH_PATH) + LibC.HeapFree(LibC.GetProcessHeap, 0, root_expanded) if root_expanded != root + LibC.HeapFree(LibC.GetProcessHeap, 0, root) + + if handle + LibC.HeapFree(LibC.GetProcessHeap, 0, buf) + LibC.HeapFree(LibC.GetProcessHeap, 0, utf16_dll) + return handle + end + {% end %} + + LibC.HeapFree(LibC.GetProcessHeap, 0, buf) + {% end %} + {% end %} + + handle = LibC.LoadLibraryExW(utf16_dll, nil, 0) + LibC.HeapFree(LibC.GetProcessHeap, 0, utf16_dll) + handle + end end # This is a port of the default delay-load helper function in the `DelayHlp.cpp` @@ -123,8 +305,7 @@ fun __delayLoadHelper2(pidd : LibC::ImgDelayDescr*, ppfnIATEntry : LibC::FARPROC # Check to see if we need to try to load the library. if !hmod - # note: ANSI variant used here - unless hmod = LibC.LoadLibraryExA(dli.szDll, nil, 0) + unless hmod = Crystal::System::DelayLoad.load_library(dli.szDll) # DloadReleaseSectionWriteAccess print_error("FATAL: Cannot find the DLL named `%1`, exiting\n", dli.szDll) LibC.ExitProcess(1) diff --git a/src/lib_c/x86_64-windows-msvc/c/heapapi.cr b/src/lib_c/x86_64-windows-msvc/c/heapapi.cr new file mode 100644 index 000000000000..1738cf774cac --- /dev/null +++ b/src/lib_c/x86_64-windows-msvc/c/heapapi.cr @@ -0,0 +1,8 @@ +require "c/winnt" + +lib LibC + fun GetProcessHeap : HANDLE + fun HeapAlloc(hHeap : HANDLE, dwFlags : DWORD, dwBytes : SizeT) : Void* + fun HeapReAlloc(hHeap : HANDLE, dwFlags : DWORD, lpMem : Void*, dwBytes : SizeT) : Void* + fun HeapFree(hHeap : HANDLE, dwFlags : DWORD, lpMem : Void*) : BOOL +end diff --git a/src/lib_c/x86_64-windows-msvc/c/libloaderapi.cr b/src/lib_c/x86_64-windows-msvc/c/libloaderapi.cr index 2b6259bf7015..37a95f3fa089 100644 --- a/src/lib_c/x86_64-windows-msvc/c/libloaderapi.cr +++ b/src/lib_c/x86_64-windows-msvc/c/libloaderapi.cr @@ -4,7 +4,8 @@ require "c/winnt" lib LibC alias FARPROC = Void* - fun LoadLibraryExA(lpLibFileName : LPSTR, hFile : HANDLE, dwFlags : DWORD) : HMODULE + LOAD_WITH_ALTERED_SEARCH_PATH = 0x00000008 + fun LoadLibraryExW(lpLibFileName : LPWSTR, hFile : HANDLE, dwFlags : DWORD) : HMODULE fun FreeLibrary(hLibModule : HMODULE) : BOOL diff --git a/src/lib_c/x86_64-windows-msvc/c/stringapiset.cr b/src/lib_c/x86_64-windows-msvc/c/stringapiset.cr index 060049841819..971e96fa9eb5 100644 --- a/src/lib_c/x86_64-windows-msvc/c/stringapiset.cr +++ b/src/lib_c/x86_64-windows-msvc/c/stringapiset.cr @@ -6,8 +6,15 @@ lib LibC # this is only for the `wmain` entry point where Crystal's standard library is # unusable, all other code should use `String.from_utf16` instead fun WideCharToMultiByte( - codePage : DWORD, dwFlags : DWORD, lpWideCharStr : WCHAR*, + codePage : UInt, dwFlags : DWORD, lpWideCharStr : LPWSTR, cchWideChar : Int, lpMultiByteStr : LPSTR, cbMultiByte : Int, lpDefaultChar : CHAR*, lpUsedDefaultChar : BOOL* ) : Int + + # this is only for the delay-load helper, all other code should use + # `String#to_utf16` instead + fun MultiByteToWideChar( + codePage : UInt, dwFlags : DWORD, lpMultiByteStr : LPSTR, + cbMultiByte : Int, lpWideCharStr : LPWSTR, cchWideChar : Int + ) : Int end diff --git a/src/lib_c/x86_64-windows-msvc/c/winbase.cr b/src/lib_c/x86_64-windows-msvc/c/winbase.cr index 96ca0710bc2d..40ef8a695d20 100644 --- a/src/lib_c/x86_64-windows-msvc/c/winbase.cr +++ b/src/lib_c/x86_64-windows-msvc/c/winbase.cr @@ -12,7 +12,7 @@ lib LibC FORMAT_MESSAGE_ARGUMENT_ARRAY = 0x00002000_u32 FORMAT_MESSAGE_MAX_WIDTH_MASK = 0x000000FF_u32 - STD_ERROR_HANDLE = DWORD.new!(-12) + STD_ERROR_HANDLE = 0xFFFFFFF4_u32 fun FormatMessageA(dwFlags : DWORD, lpSource : Void*, dwMessageId : DWORD, dwLanguageId : DWORD, lpBuffer : LPSTR, nSize : DWORD, arguments : Void*) : DWORD diff --git a/src/process/executable_path.cr b/src/process/executable_path.cr index cf2dd091c6ab..20c78b77e232 100644 --- a/src/process/executable_path.cr +++ b/src/process/executable_path.cr @@ -3,7 +3,11 @@ # - http://stackoverflow.com/questions/1023306/finding-current-executables-path-without-proc-self-exe class Process - PATH_DELIMITER = {% if flag?(:windows) %} ';' {% else %} ':' {% end %} + {% if flag?(:windows) %} + PATH_DELIMITER = ';' + {% else %} + PATH_DELIMITER = ':' + {% end %} # :nodoc: INITIAL_PATH = ENV["PATH"]? From 1a36ac90d207c9486515284bd65c304fa8fcf507 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Wed, 24 May 2023 03:15:17 +0800 Subject: [PATCH 2/8] define `Crystal::LIBRARY_RPATH` --- spec/std/spec_helper.cr | 7 ++++--- src/compiler/crystal/codegen/link.cr | 6 ++++++ src/compiler/crystal/command/env.cr | 11 ++++++----- src/compiler/crystal/program.cr | 1 + src/compiler/crystal/tools/doc/generator.cr | 2 +- src/crystal/main.cr | 4 ++++ src/crystal/system/win32/delay_load.cr | 4 ++-- 7 files changed, 24 insertions(+), 11 deletions(-) diff --git a/spec/std/spec_helper.cr b/spec/std/spec_helper.cr index 4ea8452af7c4..75f8a1fe36d3 100644 --- a/spec/std/spec_helper.cr +++ b/spec/std/spec_helper.cr @@ -81,9 +81,10 @@ def compile_file(source_file, *, bin_name = "executable_file", flags = %w(), fil args = ["build"] + flags + ["-o", executable_file, source_file] output = IO::Memory.new status = Process.run(compiler, args, env: { - "CRYSTAL_PATH" => Crystal::PATH, - "CRYSTAL_LIBRARY_PATH" => Crystal::LIBRARY_PATH, - "CRYSTAL_CACHE_DIR" => Crystal::CACHE_DIR, + "CRYSTAL_PATH" => Crystal::PATH, + "CRYSTAL_LIBRARY_PATH" => Crystal::LIBRARY_PATH, + "CRYSTAL_LIBRARY_RPATH" => Crystal::LIBRARY_RPATH, + "CRYSTAL_CACHE_DIR" => Crystal::CACHE_DIR, }, output: output, error: output) unless status.success? diff --git a/src/compiler/crystal/codegen/link.cr b/src/compiler/crystal/codegen/link.cr index dbda33033695..d4b81c7de664 100644 --- a/src/compiler/crystal/codegen/link.cr +++ b/src/compiler/crystal/codegen/link.cr @@ -98,6 +98,12 @@ module Crystal default_paths.join(Process::PATH_DELIMITER) end + def self.default_rpath : String + # do not call `CrystalPath.expand_paths`, as `$ORIGIN` inside this env + # variable is always expanded at run time + ENV.fetch("CRYSTAL_LIBRARY_RPATH", "") + end + class_getter paths : Array(String) do default_paths end diff --git a/src/compiler/crystal/command/env.cr b/src/compiler/crystal/command/env.cr index c4605860bf36..cf450ec55f3d 100644 --- a/src/compiler/crystal/command/env.cr +++ b/src/compiler/crystal/command/env.cr @@ -18,11 +18,12 @@ class Crystal::Command end vars = { - "CRYSTAL_CACHE_DIR" => CacheDir.instance.dir, - "CRYSTAL_PATH" => CrystalPath.default_path, - "CRYSTAL_VERSION" => Config.version || "", - "CRYSTAL_LIBRARY_PATH" => CrystalLibraryPath.default_path, - "CRYSTAL_OPTS" => ENV.fetch("CRYSTAL_OPTS", ""), + "CRYSTAL_CACHE_DIR" => CacheDir.instance.dir, + "CRYSTAL_PATH" => CrystalPath.default_path, + "CRYSTAL_VERSION" => Config.version || "", + "CRYSTAL_LIBRARY_PATH" => CrystalLibraryPath.default_path, + "CRYSTAL_LIBRARY_RPATH" => CrystalLibraryPath.default_rpath, + "CRYSTAL_OPTS" => ENV.fetch("CRYSTAL_OPTS", ""), } if var_names.empty? diff --git a/src/compiler/crystal/program.cr b/src/compiler/crystal/program.cr index 1c0a70430965..3fa7aa37f45a 100644 --- a/src/compiler/crystal/program.cr +++ b/src/compiler/crystal/program.cr @@ -278,6 +278,7 @@ module Crystal define_crystal_string_constant "DESCRIPTION", Crystal::Config.description define_crystal_string_constant "PATH", Crystal::CrystalPath.default_path define_crystal_string_constant "LIBRARY_PATH", Crystal::CrystalLibraryPath.default_path + define_crystal_string_constant "LIBRARY_RPATH", Crystal::CrystalLibraryPath.default_rpath define_crystal_string_constant "VERSION", Crystal::Config.version define_crystal_string_constant "LLVM_VERSION", Crystal::Config.llvm_version end diff --git a/src/compiler/crystal/tools/doc/generator.cr b/src/compiler/crystal/tools/doc/generator.cr index 9e2a22a25506..1299737e8f1f 100644 --- a/src/compiler/crystal/tools/doc/generator.cr +++ b/src/compiler/crystal/tools/doc/generator.cr @@ -227,7 +227,7 @@ class Crystal::Doc::Generator {"BUILD_COMMIT", "BUILD_DATE", "CACHE_DIR", "DEFAULT_PATH", "DESCRIPTION", "PATH", "VERSION", "LLVM_VERSION", - "LIBRARY_PATH"}.each do |name| + "LIBRARY_PATH", "LIBRARY_RPATH"}.each do |name| return true if type == crystal_type.types[name]? end diff --git a/src/crystal/main.cr b/src/crystal/main.cr index 6a63d838d234..223ada81b061 100644 --- a/src/crystal/main.cr +++ b/src/crystal/main.cr @@ -4,6 +4,10 @@ lib LibCrystalMain end module Crystal + {% unless Crystal.has_constant?("LIBRARY_RPATH") %} + LIBRARY_RPATH = {{ env("CRYSTAL_LIBRARY_RPATH") || "" }} + {% end %} + # Defines the main routine run by normal Crystal programs: # # - Initializes the GC diff --git a/src/crystal/system/win32/delay_load.cr b/src/crystal/system/win32/delay_load.cr index ac5dfcc2ad27..89d1d4af2ce1 100644 --- a/src/crystal/system/win32/delay_load.cr +++ b/src/crystal/system/win32/delay_load.cr @@ -196,8 +196,8 @@ module Crystal::System::DelayLoad def self.load_library(dll : LibC::Char*) : LibC::HMODULE utf16_dll = to_utf16!(dll) - {% if paths = env("CRYSTAL_LIBRARY_RPATH") %} - {% paths = paths.gsub(/\$\{ORIGIN\}/, "$ORIGIN").split(::Process::PATH_DELIMITER).reject(&.empty?) %} + {% begin %} + {% paths = Crystal::LIBRARY_RPATH.gsub(/\$\{ORIGIN\}/, "$ORIGIN").split(::Process::PATH_DELIMITER).reject(&.empty?) %} {% unless paths.empty? %} size = 0x40 buf = LibC.HeapAlloc(LibC.GetProcessHeap, 0, size &* 2).as(LibC::WCHAR*) From 98a27ee3ae12f0dd5fdd13b955af3d53880dd38f Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Wed, 24 May 2023 03:37:55 +0800 Subject: [PATCH 3/8] support CRYSTAL_LIBRARY_RPATH on most unix-like systems --- src/crystal/main.cr | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/crystal/main.cr b/src/crystal/main.cr index 223ada81b061..480571d06fb4 100644 --- a/src/crystal/main.cr +++ b/src/crystal/main.cr @@ -1,13 +1,23 @@ -lib LibCrystalMain - @[Raises] - fun __crystal_main(argc : Int32, argv : UInt8**) -end +require "process/executable_path" # Process::PATH_DELIMITER module Crystal {% unless Crystal.has_constant?("LIBRARY_RPATH") %} LIBRARY_RPATH = {{ env("CRYSTAL_LIBRARY_RPATH") || "" }} {% end %} +end +{% if flag?(:unix) && !flag?(:darwin) %} + {% unless Crystal::LIBRARY_RPATH.empty? %} + # TODO: is there a better way to quote this? + @[Link(ldflags: {{ "'-Wl,-rpath,#{Crystal::LIBRARY_RPATH.id}'" }})] + {% end %} +{% end %} +lib LibCrystalMain + @[Raises] + fun __crystal_main(argc : Int32, argv : UInt8**) +end + +module Crystal # Defines the main routine run by normal Crystal programs: # # - Initializes the GC From fcf11c6c771ade784577c51f754bd5bdfc3db1ed Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Wed, 24 May 2023 20:22:13 +0800 Subject: [PATCH 4/8] pass compiler's own RPATH to program being run --- src/compiler/crystal/codegen/link.cr | 49 ++++++++++++++++++++++++++- src/compiler/crystal/command.cr | 8 +++-- src/compiler/crystal/command/eval.cr | 2 +- src/compiler/crystal/command/spec.cr | 2 +- src/compiler/crystal/compiler.cr | 11 +++++- src/compiler/crystal/macros/macros.cr | 2 +- 6 files changed, 66 insertions(+), 8 deletions(-) diff --git a/src/compiler/crystal/codegen/link.cr b/src/compiler/crystal/codegen/link.cr index d4b81c7de664..231b8a81ef76 100644 --- a/src/compiler/crystal/codegen/link.cr +++ b/src/compiler/crystal/codegen/link.cr @@ -85,7 +85,7 @@ module Crystal end end - class CrystalLibraryPath + module CrystalLibraryPath def self.default_paths : Array(String) paths = ENV.fetch("CRYSTAL_LIBRARY_PATH", Crystal::Config.library_path).split(Process::PATH_DELIMITER, remove_empty: true) @@ -104,6 +104,53 @@ module Crystal ENV.fetch("CRYSTAL_LIBRARY_RPATH", "") end + # Adds the compiler itself's RPATH to the environment for the duration of + # the block. `$ORIGIN` in the compiler's RPATH is expanded immediately, but + # `$ORIGIN`s in the existing environment variable are not expanded. For + # example, on Linux: + # + # - CRYSTAL_LIBRARY_RPATH of the compiler: `$ORIGIN/so` + # - Current $CRYSTAL_LIBRARY_RPATH: `/home/foo:$ORIGIN/so` + # - Compiler's full path: `/opt/crystal` + # - Generated executable's Crystal::LIBRARY_RPATH: `/home/foo:$ORIGIN/so:/opt/so` + # + # On Windows we additionally append the compiler's parent directory to the + # list, as if by appending `$ORIGIN` to the compiler's RPATH. This directory + # is effectively the first search entry on any Windows executable. Example: + # + # - CRYSTAL_LIBRARY_RPATH of the compiler: `$ORIGIN\dll` + # - Current %CRYSTAL_LIBRARY_RPATH%: `C:\bar;$ORIGIN\dll` + # - Compiler's full path: `C:\foo\crystal.exe` + # - Generated executable's Crystal::LIBRARY_RPATH: `C:\bar;$ORIGIN\dll;C:\foo\dll;C:\foo` + # + # Combining RPATHs multiple times has no effect; the `CRYSTAL_LIBRARY_RPATH` + # environment variable at compiler startup is used, not really the "current" + # one. This can happen when running a program that also uses macro `run`s. + def self.add_compiler_rpath(&) + executable_path = Process.executable_path + compiler_origin = File.dirname(executable_path) if executable_path + + current_rpaths = ORIGINAL_CRYSTAL_LIBRARY_RPATH.try &.split(Process::PATH_DELIMITER, remove_empty: true) + compiler_rpaths = Crystal::LIBRARY_RPATH.split(Process::PATH_DELIMITER, remove_empty: true) + CrystalPath.expand_paths(compiler_rpaths, compiler_origin) + + rpaths = compiler_rpaths + rpaths.concat(current_rpaths) if current_rpaths + {% if flag?(:win32) %} + rpaths << compiler_origin if compiler_origin + {% end %} + + old_env = ENV["CRYSTAL_LIBRARY_RPATH"]? + ENV["CRYSTAL_LIBRARY_RPATH"] = rpaths.join(Process::PATH_DELIMITER) + begin + yield + ensure + ENV["CRYSTAL_LIBRARY_RPATH"] = old_env + end + end + + private ORIGINAL_CRYSTAL_LIBRARY_RPATH = ENV["CRYSTAL_LIBRARY_RPATH"]? + class_getter paths : Array(String) do default_paths end diff --git a/src/compiler/crystal/command.cr b/src/compiler/crystal/command.cr index 67b82425fad9..92a0cf270f1e 100644 --- a/src/compiler/crystal/command.cr +++ b/src/compiler/crystal/command.cr @@ -332,10 +332,11 @@ class Crystal::Command specified_output : Bool, hierarchy_exp : String?, cursor_location : String?, - output_format : String? do + output_format : String?, + combine_rpath : Bool do def compile(output_filename = self.output_filename) compiler.emit_base_filename = original_output_filename - compiler.compile sources, output_filename + compiler.compile sources, output_filename, combine_rpath: combine_rpath end def top_level_semantic @@ -543,7 +544,8 @@ class Crystal::Command error "can't use `#{output_filename}` as output filename because it's a directory" end - @config = CompilerConfig.new compiler, sources, output_filename, original_output_filename, arguments, specified_output, hierarchy_exp, cursor_location, output_format + combine_rpath = run && !no_codegen + @config = CompilerConfig.new compiler, sources, output_filename, original_output_filename, arguments, specified_output, hierarchy_exp, cursor_location, output_format, combine_rpath end private def gather_sources(filenames) diff --git a/src/compiler/crystal/command/eval.cr b/src/compiler/crystal/command/eval.cr index 507c4cbe9750..5db37cb0cd27 100644 --- a/src/compiler/crystal/command/eval.cr +++ b/src/compiler/crystal/command/eval.cr @@ -26,7 +26,7 @@ class Crystal::Command output_filename = Crystal.temp_executable "eval" - compiler.compile sources, output_filename + compiler.compile sources, output_filename, combine_rpath: true execute output_filename, program_args, compiler end end diff --git a/src/compiler/crystal/command/spec.cr b/src/compiler/crystal/command/spec.cr index 613d13cb9a63..3d4663e1cfe0 100644 --- a/src/compiler/crystal/command/spec.cr +++ b/src/compiler/crystal/command/spec.cr @@ -95,7 +95,7 @@ class Crystal::Command output_filename = Crystal.temp_executable "spec" ENV["CRYSTAL_SPEC_COMPILER_BIN"] ||= Process.executable_path - compiler.compile sources, output_filename + compiler.compile sources, output_filename, combine_rpath: true report_warnings execute output_filename, options, compiler, error_on_exit: warnings_fail_on_exit? end diff --git a/src/compiler/crystal/compiler.cr b/src/compiler/crystal/compiler.cr index 9d8e3b156975..ec49845c7d2d 100644 --- a/src/compiler/crystal/compiler.cr +++ b/src/compiler/crystal/compiler.cr @@ -150,12 +150,21 @@ module Crystal # Compiles the given *source*, with *output_filename* as the name # of the generated executable. # + # If *combine_rpath* is true, add the compiler itself's RPATH to the + # generated executable via `CrystalLibraryPath.add_compiler_rpath`. This is + # used by the `run` / `eval` / `spec` commands as well as the macro `run` + # (via `Crystal::Program#macro_compile`), and never during cross-compiling. + # # Raises `Crystal::CodeError` if there's an error in the # source code. # # Raises `InvalidByteSequenceError` if the source code is not # valid UTF-8. - def compile(source : Source | Array(Source), output_filename : String) : Result + def compile(source : Source | Array(Source), output_filename : String, *, combine_rpath : Bool = false) : Result + if combine_rpath + return CrystalLibraryPath.add_compiler_rpath { compile(source, output_filename, combine_rpath: false) } + end + source = [source] unless source.is_a?(Array) program = new_program(source) node = parse program, source diff --git a/src/compiler/crystal/macros/macros.cr b/src/compiler/crystal/macros/macros.cr index c74a482f9d03..5dc52218fdd8 100644 --- a/src/compiler/crystal/macros/macros.cr +++ b/src/compiler/crystal/macros/macros.cr @@ -93,7 +93,7 @@ class Crystal::Program return CompiledMacroRun.new(executable_path, elapsed_time, true) end - result = host_compiler.compile Compiler::Source.new(filename, source), executable_path + result = host_compiler.compile Compiler::Source.new(filename, source), executable_path, combine_rpath: true # Write the new files from which 'filename' depends into the cache dir # (here we store how to obtain these files, because a require might use From 7363e1650c7dd89133f93e77584957f853f718bd Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Wed, 24 May 2023 20:39:36 +0800 Subject: [PATCH 5/8] add interpreter todos --- src/compiler/crystal/loader/msvc.cr | 1 + src/compiler/crystal/loader/unix.cr | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/src/compiler/crystal/loader/msvc.cr b/src/compiler/crystal/loader/msvc.cr index 9f9a592477f2..01b8e6ce7b06 100644 --- a/src/compiler/crystal/loader/msvc.cr +++ b/src/compiler/crystal/loader/msvc.cr @@ -149,6 +149,7 @@ class Crystal::Loader end private def open_library(path : String) + # TODO: respect Crystal::LIBRARY_RPATH (#13490) LibC.LoadLibraryExW(System.to_wstr(path), nil, 0) end diff --git a/src/compiler/crystal/loader/unix.cr b/src/compiler/crystal/loader/unix.cr index 928c646ceb75..4c202231d7e6 100644 --- a/src/compiler/crystal/loader/unix.cr +++ b/src/compiler/crystal/loader/unix.cr @@ -137,11 +137,16 @@ class Crystal::Loader def self.default_search_paths : Array(String) default_search_paths = [] of String + # TODO: respect the compiler's DT_RPATH (#13490) + if env_library_path = ENV[{{ flag?(:darwin) ? "DYLD_LIBRARY_PATH" : "LD_LIBRARY_PATH" }}]? # TODO: Expand tokens $ORIGIN, $LIB, $PLATFORM default_search_paths.concat env_library_path.split(Process::PATH_DELIMITER, remove_empty: true) end + # TODO: respect the compiler's DT_RUNPATH + # TODO: respect $DYLD_FALLBACK_LIBRARY_PATH and the compiler's LC_RPATH on darwin + {% if (flag?(:linux) && !flag?(:android)) || flag?(:bsd) %} read_ld_conf(default_search_paths) {% end %} From 6b83b0b5c4a9b8a8673786ed1b8da9aa55acb858 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 25 May 2023 22:27:26 +0800 Subject: [PATCH 6/8] strcpy --- src/crystal/system/win32/delay_load.cr | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/crystal/system/win32/delay_load.cr b/src/crystal/system/win32/delay_load.cr index 89d1d4af2ce1..49ed467156cb 100644 --- a/src/crystal/system/win32/delay_load.cr +++ b/src/crystal/system/win32/delay_load.cr @@ -59,6 +59,16 @@ module Crystal::System::DelayLoad len end + # assigns *src* to *dst*, and returns the end of the new string in *dst* + private def self.strcpy(dst : LibC::WCHAR*, src : LibC::WCHAR*) : LibC::WCHAR* + while src.value != 0 + dst.value = src.value + dst += 1 + src += 1 + end + dst + end + # assigns the concatenation of *args* to the buffer at *buf* with the given # *size*, possibly reallocating it, and returns the new buffer private def self.strcat(buf : LibC::WCHAR*, size : Int32, *args : *T) : {LibC::WCHAR*, Int32} forall T @@ -74,12 +84,7 @@ module Crystal::System::DelayLoad ptr = buf {% for i in 0...T.size %} - src = args[{{ i }}] - while src.value != 0 - ptr.value = src.value - ptr += 1 - src += 1 - end + ptr = strcpy(ptr, args[{{ i }}]) {% end %} ptr.value = 0 @@ -173,12 +178,7 @@ module Crystal::System::DelayLoad while ptr.value != 0 new_ptr = str_lchop(ptr, origin_prefix.to_unsafe) if new_ptr != ptr - src = origin - while src.value != 0 - dst.value = src.value - dst += 1 - src += 1 - end + dst = strcpy(dst, origin) ptr = new_ptr next end From 250e175bdada8f29f3e5a2893f26e4f6433cdb8a Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Thu, 25 May 2023 22:28:34 +0800 Subject: [PATCH 7/8] docs --- src/compiler/crystal/codegen/link.cr | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/compiler/crystal/codegen/link.cr b/src/compiler/crystal/codegen/link.cr index 231b8a81ef76..dee4087f1e3b 100644 --- a/src/compiler/crystal/codegen/link.cr +++ b/src/compiler/crystal/codegen/link.cr @@ -110,18 +110,18 @@ module Crystal # example, on Linux: # # - CRYSTAL_LIBRARY_RPATH of the compiler: `$ORIGIN/so` - # - Current $CRYSTAL_LIBRARY_RPATH: `/home/foo:$ORIGIN/so` + # - Current $CRYSTAL_LIBRARY_RPATH: `/home/foo:$ORIGIN/mylibs` # - Compiler's full path: `/opt/crystal` - # - Generated executable's Crystal::LIBRARY_RPATH: `/home/foo:$ORIGIN/so:/opt/so` + # - Generated executable's Crystal::LIBRARY_RPATH: `/home/foo:$ORIGIN/mylibs:/opt/so` # # On Windows we additionally append the compiler's parent directory to the # list, as if by appending `$ORIGIN` to the compiler's RPATH. This directory # is effectively the first search entry on any Windows executable. Example: # # - CRYSTAL_LIBRARY_RPATH of the compiler: `$ORIGIN\dll` - # - Current %CRYSTAL_LIBRARY_RPATH%: `C:\bar;$ORIGIN\dll` + # - Current %CRYSTAL_LIBRARY_RPATH%: `C:\bar;$ORIGIN\mylibs` # - Compiler's full path: `C:\foo\crystal.exe` - # - Generated executable's Crystal::LIBRARY_RPATH: `C:\bar;$ORIGIN\dll;C:\foo\dll;C:\foo` + # - Generated executable's Crystal::LIBRARY_RPATH: `C:\bar;$ORIGIN\mylibs;C:\foo\dll;C:\foo` # # Combining RPATHs multiple times has no effect; the `CRYSTAL_LIBRARY_RPATH` # environment variable at compiler startup is used, not really the "current" From abc6dd9d3e58ef296908477bb220d1a568de3197 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Fri, 26 May 2023 05:01:33 +0800 Subject: [PATCH 8/8] fixup --- src/compiler/crystal/command.cr | 1 + 1 file changed, 1 insertion(+) diff --git a/src/compiler/crystal/command.cr b/src/compiler/crystal/command.cr index 95f5f9191386..273554728f66 100644 --- a/src/compiler/crystal/command.cr +++ b/src/compiler/crystal/command.cr @@ -549,6 +549,7 @@ class Crystal::Command error "can't use `#{output_filename}` as output filename because it's a directory" end + combine_rpath = run && !no_codegen @config = CompilerConfig.new compiler, sources, output_filename, arguments, specified_output, hierarchy_exp, cursor_location, output_format, combine_rpath end