475 changes: 475 additions & 0 deletions compiler-rt/lib/orc/macho_platform.cpp

Large diffs are not rendered by default.

129 changes: 129 additions & 0 deletions compiler-rt/lib/orc/macho_platform.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
//===- macho_platform.h -----------------------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// ORC Runtime support for Darwin dynamic loading features.
//
//===----------------------------------------------------------------------===//

#ifndef ORC_RT_MACHO_PLATFORM_H
#define ORC_RT_MACHO_PLATFORM_H

#include "common.h"
#include "executor_address.h"

// Atexit functions.
ORC_RT_INTERFACE int __orc_rt_macho_cxa_atexit(void (*func)(void *), void *arg,
void *dso_handle);
ORC_RT_INTERFACE void __orc_rt_macho_cxa_finalize(void *dso_handle);

// dlfcn functions.
ORC_RT_INTERFACE const char *__orc_rt_macho_jit_dlerror();
ORC_RT_INTERFACE void *__orc_rt_macho_jit_dlopen(const char *path, int mode);
ORC_RT_INTERFACE int __orc_rt_macho_jit_dlclose(void *dso_handle);
ORC_RT_INTERFACE void *__orc_rt_macho_jit_dlsym(void *dso_handle,
const char *symbol);

namespace __orc_rt {
namespace macho {

struct MachOPerObjectSectionsToRegister {
ExecutorAddressRange EHFrameSection;
};

struct MachOJITDylibInitializers {
using SectionList = std::vector<ExecutorAddressRange>;

MachOJITDylibInitializers() = default;
MachOJITDylibInitializers(std::string Name,
ExecutorAddress MachOHeaderAddress)
: Name(std::move(Name)),
MachOHeaderAddress(std::move(MachOHeaderAddress)) {}

std::string Name;
ExecutorAddress MachOHeaderAddress;

std::unordered_map<std::string, SectionList> InitSections;
};

class MachOJITDylibDeinitializers {};

using MachOJITDylibInitializerSequence = std::vector<MachOJITDylibInitializers>;

using MachOJITDylibDeinitializerSequence =
std::vector<MachOJITDylibDeinitializers>;

enum dlopen_mode : int {
ORC_RT_RTLD_LAZY = 0x1,
ORC_RT_RTLD_NOW = 0x2,
ORC_RT_RTLD_LOCAL = 0x4,
ORC_RT_RTLD_GLOBAL = 0x8
};

} // end namespace macho

using SPSMachOPerObjectSectionsToRegister = SPSTuple<SPSExecutorAddressRange>;

template <>
class SPSSerializationTraits<SPSMachOPerObjectSectionsToRegister,
macho::MachOPerObjectSectionsToRegister> {

public:
static size_t size(const macho::MachOPerObjectSectionsToRegister &MOPOSR) {
return SPSMachOPerObjectSectionsToRegister::AsArgList::size(
MOPOSR.EHFrameSection);
}

static bool serialize(SPSOutputBuffer &OB,
const macho::MachOPerObjectSectionsToRegister &MOPOSR) {
return SPSMachOPerObjectSectionsToRegister::AsArgList::serialize(
OB, MOPOSR.EHFrameSection);
}

static bool deserialize(SPSInputBuffer &IB,
macho::MachOPerObjectSectionsToRegister &MOPOSR) {
return SPSMachOPerObjectSectionsToRegister::AsArgList::deserialize(
IB, MOPOSR.EHFrameSection);
}
};

using SPSNamedExecutorAddressRangeSequenceMap =
SPSSequence<SPSTuple<SPSString, SPSExecutorAddressRangeSequence>>;

using SPSMachOJITDylibInitializers =
SPSTuple<SPSString, SPSExecutorAddress,
SPSNamedExecutorAddressRangeSequenceMap>;

using SPSMachOJITDylibInitializerSequence =
SPSSequence<SPSMachOJITDylibInitializers>;

/// Serialization traits for MachOJITDylibInitializers.
template <>
class SPSSerializationTraits<SPSMachOJITDylibInitializers,
macho::MachOJITDylibInitializers> {
public:
static size_t size(const macho::MachOJITDylibInitializers &MOJDIs) {
return SPSMachOJITDylibInitializers::AsArgList::size(
MOJDIs.Name, MOJDIs.MachOHeaderAddress, MOJDIs.InitSections);
}

static bool serialize(SPSOutputBuffer &OB,
const macho::MachOJITDylibInitializers &MOJDIs) {
return SPSMachOJITDylibInitializers::AsArgList::serialize(
OB, MOJDIs.Name, MOJDIs.MachOHeaderAddress, MOJDIs.InitSections);
}

static bool deserialize(SPSInputBuffer &IB,
macho::MachOJITDylibInitializers &MOJDIs) {
return SPSMachOJITDylibInitializers::AsArgList::deserialize(
IB, MOJDIs.Name, MOJDIs.MachOHeaderAddress, MOJDIs.InitSections);
}
};

} // end namespace __orc_rt

#endif // ORC_RT_MACHO_PLATFORM_H
51 changes: 51 additions & 0 deletions compiler-rt/lib/orc/run_program_wrapper.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
//===- run_program_wrapper.cpp --------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file is a part of the ORC runtime support library.
//
//===----------------------------------------------------------------------===//

#include "adt.h"
#include "common.h"
#include "wrapper_function_utils.h"

#include <vector>

using namespace __orc_rt;

extern "C" int64_t __orc_rt_run_program(const char *JITDylibName,
const char *EntrySymbolName, int argc,
char *argv[]);

ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
__orc_rt_run_program_wrapper(const char *ArgData, size_t ArgSize) {
return WrapperFunction<int64_t(SPSString, SPSString,
SPSSequence<SPSString>)>::
handle(ArgData, ArgSize,
[](const std::string &JITDylibName,
const std::string &EntrySymbolName,
const std::vector<string_view> &Args) {
std::vector<std::unique_ptr<char[]>> ArgVStorage;
ArgVStorage.reserve(Args.size());
for (auto &Arg : Args) {
ArgVStorage.push_back(
std::make_unique<char[]>(Arg.size() + 1));
memcpy(ArgVStorage.back().get(), Arg.data(), Arg.size());
ArgVStorage.back()[Arg.size()] = '\0';
}
std::vector<char *> ArgV;
ArgV.reserve(ArgVStorage.size() + 1);
for (auto &ArgStorage : ArgVStorage)
ArgV.push_back(ArgStorage.get());
ArgV.push_back(nullptr);
return __orc_rt_run_program(JITDylibName.c_str(),
EntrySymbolName.c_str(),
ArgV.size() - 1, ArgV.data());
})
.release();
}
41 changes: 32 additions & 9 deletions compiler-rt/lib/orc/simple_packed_serialization.h
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
#include <string>
#include <tuple>
#include <type_traits>
#include <unordered_map>
#include <utility>
#include <vector>

Expand Down Expand Up @@ -85,6 +86,7 @@ class SPSInputBuffer {
bool skip(size_t Size) {
if (Size > Remaining)
return false;
Buffer += Size;
Remaining -= Size;
return true;
}
Expand Down Expand Up @@ -170,17 +172,11 @@ class SPSSerializationTraits<
}
};

// Any empty placeholder suitable as a substitute for void when deserializing
/// Any empty placeholder suitable as a substitute for void when deserializing
class SPSEmpty {};

/// SPS tag type for target addresses.
///
/// SPSTagTargetAddresses should be serialized as a uint64_t value.
class SPSTagTargetAddress;

template <>
class SPSSerializationTraits<SPSTagTargetAddress, uint64_t>
: public SPSSerializationTraits<uint64_t, uint64_t> {};
/// Represents an address in the executor.
class SPSExecutorAddress {};

/// SPS tag type for tuples.
///
Expand Down Expand Up @@ -290,6 +286,33 @@ class TrivialSPSSequenceDeserialization<SPSElementTagT, std::vector<T>> {
}
};

/// Trivial std::unordered_map<K, V> -> SPSSequence<SPSTuple<SPSKey, SPSValue>>
/// serialization.
template <typename SPSKeyTagT, typename SPSValueTagT, typename K, typename V>
class TrivialSPSSequenceSerialization<SPSTuple<SPSKeyTagT, SPSValueTagT>,
std::unordered_map<K, V>> {
public:
static constexpr bool available = true;
};

/// Trivial SPSSequence<SPSTuple<SPSKey, SPSValue>> -> std::unordered_map<K, V>
/// deserialization.
template <typename SPSKeyTagT, typename SPSValueTagT, typename K, typename V>
class TrivialSPSSequenceDeserialization<SPSTuple<SPSKeyTagT, SPSValueTagT>,
std::unordered_map<K, V>> {
public:
static constexpr bool available = true;

using element_type = std::pair<K, V>;

static void reserve(std::unordered_map<K, V> &M, uint64_t Size) {
M.reserve(Size);
}
static bool append(std::unordered_map<K, V> &M, element_type E) {
return M.insert(std::move(E)).second;
}
};

/// 'Trivial' sequence serialization: Sequence is serialized as a uint64_t size
/// followed by a for-earch loop over the elements of the sequence to serialize
/// each of them.
Expand Down
2 changes: 1 addition & 1 deletion compiler-rt/lib/orc/unittests/adt_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ TEST(ADTTest, StringViewConstructFromCharPtr) {

EXPECT_FALSE(S.empty()) << "Span should be non-empty";
EXPECT_EQ(S.size(), StrLen) << "Span has unexpected size";
EXPECT_EQ(std::distance(S.begin(), S.end()), StrLen)
EXPECT_EQ(static_cast<size_t>(std::distance(S.begin(), S.end())), StrLen)
<< "Unexpected iterator range size";
EXPECT_EQ(S.data(), &Str[0]) << "Span data has unexpected value";
for (unsigned I = 0; I != S.size(); ++I)
Expand Down
4 changes: 2 additions & 2 deletions compiler-rt/lib/orc/wrapper_function_utils.h
Original file line number Diff line number Diff line change
Expand Up @@ -305,9 +305,9 @@ class WrapperFunction<SPSRetTagT(SPSTagTs...)> {
detail::ResultDeserializer<SPSRetTagT, RetT>::makeSafe(Result);

if (ORC_RT_UNLIKELY(!&__orc_rt_jit_dispatch_ctx))
return make_error<StringError>("__orc_jtjit_dispatch_ctx not set");
return make_error<StringError>("__orc_rt_jit_dispatch_ctx not set");
if (ORC_RT_UNLIKELY(!&__orc_rt_jit_dispatch))
return make_error<StringError>("__orc_jtjit_dispatch not set");
return make_error<StringError>("__orc_rt_jit_dispatch not set");

auto ArgBuffer =
detail::serializeViaSPSToWrapperFunctionResult<SPSArgList<SPSTagTs...>>(
Expand Down
13 changes: 0 additions & 13 deletions compiler-rt/test/orc/TestCases/Darwin/x86-64/placeholder_test.S

This file was deleted.

38 changes: 38 additions & 0 deletions compiler-rt/test/orc/TestCases/Darwin/x86-64/trivial-cxa-atexit.S
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
// Test that the runtime correctly interposes ___cxa_atexit.
//
// RUN: %clang -c -o %t %s
// RUN: %llvm_jitlink %t

.section __TEXT,__text,regular,pure_instructions
.build_version macos, 11, 0 sdk_version 11, 3

// OnExit destructor resets the test result override to zero.
.globl __ZN6OnExitD1Ev
.weak_def_can_be_hidden __ZN6OnExitD1Ev
.p2align 4, 0x90
__ZN6OnExitD1Ev:
xorl %edi, %edi
jmp _llvm_jitlink_setTestResultOverride

// main registers the atexit and sets the test result to one.
.globl _main
.p2align 4, 0x90
_main:
pushq %rbp
movq %rsp, %rbp

movq __ZN6OnExitD1Ev@GOTPCREL(%rip), %rdi
leaq _onExit(%rip), %rsi
leaq ___dso_handle(%rip), %rdx
callq ___cxa_atexit

movl $1, %edi
callq _llvm_jitlink_setTestResultOverride
xorl %eax, %eax
popq %rbp
retq

.globl _onExit
.zerofill __DATA,__common,_onExit,1,0

.subsections_via_symbols
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
// Test that basic MachO static initializers work. The main function in this
// test returns the value of 'x', which is initially 1 in the data section,
// and reset to 0 if the _static_init function is run. If the static initializer
// does not run then main will return 1, causing the test to be treated as a
// failure.
//
// RUN: %clang -c -o %t %s
// RUN: %llvm_jitlink %t

.section __TEXT,__text,regular,pure_instructions
.build_version macos, 11, 0

# main returns the value of 'x', which is defined as 1 in the data section..
.globl _main
.p2align 4, 0x90
_main:
movl _x(%rip), %eax
retq

# static initializer sets the value of 'x' to zero.
.section __TEXT,__StaticInit,regular,pure_instructions
.p2align 4, 0x90
_static_init:
movl $0, _x(%rip)
retq

.section __DATA,__data
.globl _x
.p2align 2
_x:
.long 1

.section __DATA,__mod_init_func,mod_init_funcs
.p2align 3
.quad _static_init

.subsections_via_symbols
6 changes: 6 additions & 0 deletions llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h
Original file line number Diff line number Diff line change
Expand Up @@ -916,6 +916,12 @@ class LinkGraph {

const char *getEdgeKindName(Edge::Kind K) const { return GetEdgeKindName(K); }

/// Allocate a mutable buffer of the given size using the LinkGraph's
/// allocator.
MutableArrayRef<char> allocateBuffer(size_t Size) {
return {Allocator.Allocate<char>(Size), Size};
}

/// Allocate a copy of the given string using the LinkGraph's allocator.
/// This can be useful when renaming symbols or adding new content to the
/// graph.
Expand Down
10 changes: 10 additions & 0 deletions llvm/include/llvm/ExecutionEngine/Orc/Core.h
Original file line number Diff line number Diff line change
Expand Up @@ -1224,9 +1224,19 @@ class Platform {

/// A utility function for looking up initializer symbols. Performs a blocking
/// lookup for the given symbols in each of the given JITDylibs.
///
/// Note: This function is deprecated and will be removed in the near future.
static Expected<DenseMap<JITDylib *, SymbolMap>>
lookupInitSymbols(ExecutionSession &ES,
const DenseMap<JITDylib *, SymbolLookupSet> &InitSyms);

/// Performs an async lookup for the the given symbols in each of the given
/// JITDylibs, calling the given handler with the compound result map once
/// all lookups have completed.
static void
lookupInitSymbolsAsync(unique_function<void(Error)> OnComplete,
ExecutionSession &ES,
const DenseMap<JITDylib *, SymbolLookupSet> &InitSyms);
};

/// Represents an abstract task for ORC to run.
Expand Down
36 changes: 32 additions & 4 deletions llvm/include/llvm/ExecutionEngine/Orc/ExecutorProcessControl.h
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
#include "llvm/ADT/Triple.h"
#include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h"
#include "llvm/ExecutionEngine/Orc/Core.h"
#include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h"
#include "llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h"
#include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h"
#include "llvm/Support/DynamicLibrary.h"
Expand Down Expand Up @@ -113,6 +114,13 @@ class ExecutorProcessControl {
const SymbolLookupSet &Symbols;
};

/// Contains the address of the dispatch function and context that the ORC
/// runtime can use to call functions in the JIT.
struct JITDispatchInfo {
ExecutorAddress JITDispatchFunctionAddress;
ExecutorAddress JITDispatchContextAddress;
};

virtual ~ExecutorProcessControl();

/// Intern a symbol name in the SymbolStringPool.
Expand All @@ -127,6 +135,9 @@ class ExecutorProcessControl {
/// Get the page size for the target process.
unsigned getPageSize() const { return PageSize; }

/// Get the JIT dispatch function and context address for the executor.
const JITDispatchInfo &getJITDispatchInfo() const { return JDI; }

/// Return a MemoryAccess object for the target process.
MemoryAccess &getMemoryAccess() const { return *MemAccess; }

Expand Down Expand Up @@ -198,14 +209,17 @@ class ExecutorProcessControl {

/// Run a wrapper function using SPS to serialize the arguments and
/// deserialize the results.
template <typename SPSSignature, typename RetT, typename... ArgTs>
Error runSPSWrapper(JITTargetAddress WrapperFnAddr, RetT &RetVal,
const ArgTs &...Args) {
///
/// If SPSSignature is a non-void function signature then the second argument
/// (the first in the Args list) should be a reference to a return value.
template <typename SPSSignature, typename... WrapperCallArgTs>
Error runSPSWrapper(JITTargetAddress WrapperFnAddr,
WrapperCallArgTs &&...WrapperCallArgs) {
return shared::WrapperFunction<SPSSignature>::call(
[this, WrapperFnAddr](const char *ArgData, size_t ArgSize) {
return runWrapper(WrapperFnAddr, ArrayRef<char>(ArgData, ArgSize));
},
RetVal, Args...);
std::forward<WrapperCallArgTs>(WrapperCallArgs)...);
}

/// Wrap a handler that takes concrete argument types (and a sender for a
Expand All @@ -223,6 +237,15 @@ class ExecutorProcessControl {
};
}

template <typename SPSSignature, typename ClassT, typename... MethodArgTs>
static AsyncWrapperFunction
wrapAsyncWithSPS(ClassT *Instance, void (ClassT::*Method)(MethodArgTs...)) {
return wrapAsyncWithSPS<SPSSignature>(
[Instance, Method](MethodArgTs &&...MethodArgs) {
(Instance->*Method)(std::forward<MethodArgTs>(MethodArgs)...);
});
}

/// For each symbol name, associate the AsyncWrapperFunction implementation
/// value with the address of that symbol.
///
Expand Down Expand Up @@ -250,6 +273,7 @@ class ExecutorProcessControl {
std::shared_ptr<SymbolStringPool> SSP;
Triple TargetTriple;
unsigned PageSize = 0;
JITDispatchInfo JDI;
MemoryAccess *MemAccess = nullptr;
jitlink::JITLinkMemoryManager *MemMgr = nullptr;

Expand Down Expand Up @@ -318,6 +342,10 @@ class SelfExecutorProcessControl
void writeBuffers(ArrayRef<tpctypes::BufferWrite> Ws,
WriteResultFn OnWriteComplete) override;

static shared::detail::CWrapperFunctionResult
jitDispatchViaWrapperFunctionManager(void *Ctx, const void *FnTag,
const char *Data, size_t Size);

std::unique_ptr<jitlink::JITLinkMemoryManager> OwnedMemMgr;
char GlobalManglingPrefix = 0;
std::vector<std::unique_ptr<sys::DynamicLibrary>> DynamicLibraries;
Expand Down
69 changes: 69 additions & 0 deletions llvm/include/llvm/ExecutionEngine/Orc/LLVMSPSSerializers.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
//===-- LLVMSPSSerializers.h - SPS serialization for LLVM types -*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// SPS Serialization for common LLVM types.
//
//===----------------------------------------------------------------------===//

#ifndef LLVM_EXECUTIONENGINE_ORC_LLVMSPSSERIALIZERS_H
#define LLVM_EXECUTIONENGINE_ORC_LLVMSPSSERIALIZERS_H

#include "llvm/ADT/StringMap.h"
#include "llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h"

namespace llvm {
namespace orc {
namespace shared {

template <typename SPSValueT, typename ValueT>
class SPSSerializationTraits<SPSSequence<SPSTuple<SPSString, SPSValueT>>,
StringMap<ValueT>> {
public:
static size_t size(const StringMap<ValueT> &M) {
size_t Sz = SPSArgList<uint64_t>::size(static_cast<uint64_t>(M.size()));
for (auto &E : M)
Sz += SPSArgList<SPSString, SPSValueT>::size(E.first(), E.second);
return Sz;
}

static bool serialize(SPSOutputBuffer &OB, const StringMap<ValueT> &M) {
if (!SPSArgList<uint64_t>::serialize(OB, static_cast<uint64_t>(M.size())))
return false;

for (auto &E : M)
if (!SPSArgList<SPSString, SPSValueT>::serialize(OB, E.first(), E.second))
return false;

return true;
}

static bool deserialize(SPSInputBuffer &IB, StringMap<ValueT> &M) {
uint64_t Size;
assert(M.empty() && "M already contains elements");

if (!SPSArgList<uint64_t>::deserialize(IB, Size))
return false;

while (Size--) {
StringRef S;
ValueT V;
if (!SPSArgList<SPSString, SPSValueT>::deserialize(IB, S, V))
return false;
if (!M.insert(std::make_pair(S, V)).second)
return false;
}

return true;
}
};

} // end namespace shared
} // end namespace orc
} // end namespace llvm

#endif // LLVM_EXECUTIONENGINE_ORC_LLVMSPSSERIALIZERS_H
295 changes: 228 additions & 67 deletions llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@

#include "llvm/ADT/StringRef.h"
#include "llvm/ExecutionEngine/Orc/Core.h"
#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
#include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
#include "llvm/ExecutionEngine/Orc/LLVMSPSSerializers.h"
#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
#include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h"

Expand All @@ -26,83 +27,106 @@
namespace llvm {
namespace orc {

struct MachOPerObjectSectionsToRegister {
ExecutorAddressRange EHFrameSection;
};

class MachOJITDylibInitializers {
public:
using RawPointerSectionList = std::vector<ExecutorAddressRange>;

void setObjCImageInfoAddr(JITTargetAddress ObjCImageInfoAddr) {
this->ObjCImageInfoAddr = ObjCImageInfoAddr;
}

void addModInitsSection(ExecutorAddressRange ModInit) {
ModInitSections.push_back(std::move(ModInit));
}

const RawPointerSectionList &getModInitsSections() const {
return ModInitSections;
}

void addObjCSelRefsSection(ExecutorAddressRange ObjCSelRefs) {
ObjCSelRefsSections.push_back(std::move(ObjCSelRefs));
}

const RawPointerSectionList &getObjCSelRefsSections() const {
return ObjCSelRefsSections;
}
struct MachOJITDylibInitializers {
using SectionList = std::vector<ExecutorAddressRange>;

void addObjCClassListSection(ExecutorAddressRange ObjCClassList) {
ObjCClassListSections.push_back(std::move(ObjCClassList));
}

const RawPointerSectionList &getObjCClassListSections() const {
return ObjCClassListSections;
}
MachOJITDylibInitializers(std::string Name,
ExecutorAddress MachOHeaderAddress)
: Name(std::move(Name)),
MachOHeaderAddress(std::move(MachOHeaderAddress)) {}

void runModInits() const;
void registerObjCSelectors() const;
Error registerObjCClasses() const;
std::string Name;
ExecutorAddress MachOHeaderAddress;

private:

JITTargetAddress ObjCImageInfoAddr;
RawPointerSectionList ModInitSections;
RawPointerSectionList ObjCSelRefsSections;
RawPointerSectionList ObjCClassListSections;
StringMap<SectionList> InitSections;
};

class MachOJITDylibDeinitializers {};

using MachOJITDylibInitializerSequence = std::vector<MachOJITDylibInitializers>;

using MachOJITDylibDeinitializerSequence =
std::vector<MachOJITDylibDeinitializers>;

/// Mediates between MachO initialization and ExecutionSession state.
class MachOPlatform : public Platform {
public:
using InitializerSequence =
std::vector<std::pair<JITDylib *, MachOJITDylibInitializers>>;

using DeinitializerSequence =
std::vector<std::pair<JITDylib *, MachOJITDylibDeinitializers>>;

MachOPlatform(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
std::unique_ptr<MemoryBuffer> StandardSymbolsObject);
/// Try to create a MachOPlatform instance, adding the ORC runtime to the
/// given JITDylib.
///
/// The ORC runtime requires access to a number of symbols in libc++, and
/// requires access to symbols in libobjc, and libswiftCore to support
/// Objective-C and Swift code. It is up to the caller to ensure that the
/// requried symbols can be referenced by code added to PlatformJD. The
/// standard way to achieve this is to first attach dynamic library search
/// generators for either the given process, or for the specific required
/// libraries, to PlatformJD, then to create the platform instance:
///
/// \code{.cpp}
/// auto &PlatformJD = ES.createBareJITDylib("stdlib");
/// PlatformJD.addGenerator(
/// ExitOnErr(EPCDynamicLibrarySearchGenerator
/// ::GetForTargetProcess(EPC)));
/// ES.setPlatform(
/// ExitOnErr(MachOPlatform::Create(ES, ObjLayer, EPC, PlatformJD,
/// "/path/to/orc/runtime")));
/// \endcode
///
/// Alternatively, these symbols could be added to another JITDylib that
/// PlatformJD links against.
///
/// Clients are also responsible for ensuring that any JIT'd code that
/// depends on runtime functions (including any code using TLV or static
/// destructors) can reference the runtime symbols. This is usually achieved
/// by linking any JITDylibs containing regular code against
/// PlatformJD.
///
/// By default, MachOPlatform will add the set of aliases returned by the
/// standardPlatformAliases function. This includes both required aliases
/// (e.g. __cxa_atexit -> __orc_rt_macho_cxa_atexit for static destructor
/// support), and optional aliases that provide JIT versions of common
/// functions (e.g. dlopen -> __orc_rt_macho_jit_dlopen). Clients can
/// override these defaults by passing a non-None value for the
/// RuntimeAliases function, in which case the client is responsible for
/// setting up all aliases (including the required ones).
static Expected<std::unique_ptr<MachOPlatform>>
Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
ExecutorProcessControl &EPC, JITDylib &PlatformJD,
const char *OrcRuntimePath,
Optional<SymbolAliasMap> RuntimeAliases = None);

ExecutionSession &getExecutionSession() const { return ES; }
ObjectLinkingLayer &getObjectLinkingLayer() const { return ObjLinkingLayer; }
ExecutorProcessControl &getExecutorProcessControl() const { return EPC; }

Error setupJITDylib(JITDylib &JD) override;
Error notifyAdding(ResourceTracker &RT,
const MaterializationUnit &MU) override;
Error notifyRemoving(ResourceTracker &RT) override;

Expected<InitializerSequence> getInitializerSequence(JITDylib &JD);
/// Returns an AliasMap containing the default aliases for the MachOPlatform.
/// This can be modified by clients when constructing the platform to add
/// or remove aliases.
static SymbolAliasMap standardPlatformAliases(ExecutionSession &ES);

Expected<DeinitializerSequence> getDeinitializerSequence(JITDylib &JD);
/// Returns the array of required CXX aliases.
static ArrayRef<std::pair<const char *, const char *>> requiredCXXAliases();

/// Returns the array of standard runtime utility aliases for MachO.
static ArrayRef<std::pair<const char *, const char *>>
standardRuntimeUtilityAliases();

private:
// This ObjectLinkingLayer plugin scans JITLink graphs for __mod_init_func,
// __objc_classlist and __sel_ref sections and records their extents so that
// they can be run in the target process.
class InitScraperPlugin : public ObjectLinkingLayer::Plugin {
// The MachOPlatformPlugin scans/modifies LinkGraphs to support MachO
// platform features including initializers, exceptions, TLV, and language
// runtime registration.
class MachOPlatformPlugin : public ObjectLinkingLayer::Plugin {
public:
InitScraperPlugin(MachOPlatform &MP) : MP(MP) {}
MachOPlatformPlugin(MachOPlatform &MP) : MP(MP) {}

void modifyPassConfig(MaterializationResponsibility &MR,
jitlink::LinkGraph &G,
Expand All @@ -128,36 +152,173 @@ class MachOPlatform : public Platform {
using InitSymbolDepMap =
DenseMap<MaterializationResponsibility *, JITLinkSymbolSet>;

void preserveInitSectionIfPresent(JITLinkSymbolSet &Symbols,
jitlink::LinkGraph &G,
StringRef SectionName);
void addInitializerSupportPasses(MaterializationResponsibility &MR,
jitlink::PassConfiguration &Config);

void addMachOHeaderSupportPasses(MaterializationResponsibility &MR,
jitlink::PassConfiguration &Config);

Error processObjCImageInfo(jitlink::LinkGraph &G,
void addEHSupportPasses(MaterializationResponsibility &MR,
jitlink::PassConfiguration &Config);

Error preserveInitSections(jitlink::LinkGraph &G,
MaterializationResponsibility &MR);

std::mutex InitScraperMutex;
Error registerInitSections(jitlink::LinkGraph &G, JITDylib &JD);

std::mutex PluginMutex;
MachOPlatform &MP;
DenseMap<JITDylib *, std::pair<uint32_t, uint32_t>> ObjCImageInfos;
InitSymbolDepMap InitSymbolDeps;
};

void registerInitInfo(JITDylib &JD, JITTargetAddress ObjCImageInfoAddr,
ExecutorAddressRange ModInits,
ExecutorAddressRange ObjCSelRefs,
ExecutorAddressRange ObjCClassList);
using SendInitializerSequenceFn =
unique_function<void(Expected<MachOJITDylibInitializerSequence>)>;

using SendDeinitializerSequenceFn =
unique_function<void(Expected<MachOJITDylibDeinitializerSequence>)>;

using SendSymbolAddressFn = unique_function<void(Expected<ExecutorAddress>)>;

static bool supportedTarget(const Triple &TT);

MachOPlatform(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
ExecutorProcessControl &EPC, JITDylib &PlatformJD,
std::unique_ptr<DefinitionGenerator> OrcRuntimeGenerator,
Error &Err);

// Associate MachOPlatform JIT-side runtime support functions with handlers.
Error associateRuntimeSupportFunctions(JITDylib &PlatformJD);

void getInitializersBuildSequencePhase(SendInitializerSequenceFn SendResult,
JITDylib &JD,
std::vector<JITDylibSP> DFSLinkOrder);

void getInitializersLookupPhase(SendInitializerSequenceFn SendResult,
JITDylib &JD);

void rt_getInitializers(SendInitializerSequenceFn SendResult,
StringRef JDName);

void rt_getDeinitializers(SendDeinitializerSequenceFn SendResult,
ExecutorAddress Handle);

void rt_lookupSymbol(SendSymbolAddressFn SendResult, ExecutorAddress Handle,
StringRef SymbolName);

// Records the addresses of runtime symbols used by the platform.
Error bootstrapMachORuntime(JITDylib &PlatformJD);

Error registerInitInfo(JITDylib &JD,
ArrayRef<jitlink::Section *> InitSections);

Error registerPerObjectSections(const MachOPerObjectSectionsToRegister &POSR);

ExecutionSession &ES;
ObjectLinkingLayer &ObjLinkingLayer;
std::unique_ptr<MemoryBuffer> StandardSymbolsObject;
ExecutorProcessControl &EPC;

SymbolStringPtr MachOHeaderStartSymbol;
std::atomic<bool> RuntimeBootstrapped{false};

ExecutorAddress orc_rt_macho_platform_bootstrap;
ExecutorAddress orc_rt_macho_platform_shutdown;
ExecutorAddress orc_rt_macho_register_object_sections;

DenseMap<JITDylib *, SymbolLookupSet> RegisteredInitSymbols;

// InitSeqs gets its own mutex to avoid locking the whole session when
// aggregating data from the jitlink.
std::mutex InitSeqsMutex;
std::mutex PlatformMutex;
DenseMap<JITDylib *, MachOJITDylibInitializers> InitSeqs;
std::vector<MachOPerObjectSectionsToRegister> BootstrapPOSRs;

DenseMap<JITTargetAddress, JITDylib *> HeaderAddrToJITDylib;
};

namespace shared {

using SPSMachOPerObjectSectionsToRegister = SPSTuple<SPSExecutorAddressRange>;

template <>
class SPSSerializationTraits<SPSMachOPerObjectSectionsToRegister,
MachOPerObjectSectionsToRegister> {

public:
static size_t size(const MachOPerObjectSectionsToRegister &MOPOSR) {
return SPSMachOPerObjectSectionsToRegister::AsArgList::size(
MOPOSR.EHFrameSection);
}

static bool serialize(SPSOutputBuffer &OB,
const MachOPerObjectSectionsToRegister &MOPOSR) {
return SPSMachOPerObjectSectionsToRegister::AsArgList::serialize(
OB, MOPOSR.EHFrameSection);
}

static bool deserialize(SPSInputBuffer &IB,
MachOPerObjectSectionsToRegister &MOPOSR) {
return SPSMachOPerObjectSectionsToRegister::AsArgList::deserialize(
IB, MOPOSR.EHFrameSection);
}
};

using SPSNamedExecutorAddressRangeSequenceMap =
SPSSequence<SPSTuple<SPSString, SPSExecutorAddressRangeSequence>>;

using SPSMachOJITDylibInitializers =
SPSTuple<SPSString, SPSExecutorAddress,
SPSNamedExecutorAddressRangeSequenceMap>;

using SPSMachOJITDylibInitializerSequence =
SPSSequence<SPSMachOJITDylibInitializers>;

/// Serialization traits for MachOJITDylibInitializers.
template <>
class SPSSerializationTraits<SPSMachOJITDylibInitializers,
MachOJITDylibInitializers> {
public:
static size_t size(const MachOJITDylibInitializers &MOJDIs) {
return SPSMachOJITDylibInitializers::AsArgList::size(
MOJDIs.Name, MOJDIs.MachOHeaderAddress, MOJDIs.InitSections);
}

static bool serialize(SPSOutputBuffer &OB,
const MachOJITDylibInitializers &MOJDIs) {
return SPSMachOJITDylibInitializers::AsArgList::serialize(
OB, MOJDIs.Name, MOJDIs.MachOHeaderAddress, MOJDIs.InitSections);
}

static bool deserialize(SPSInputBuffer &IB,
MachOJITDylibInitializers &MOJDIs) {
return SPSMachOJITDylibInitializers::AsArgList::deserialize(
IB, MOJDIs.Name, MOJDIs.MachOHeaderAddress, MOJDIs.InitSections);
}
};

using SPSMachOJITDylibDeinitializers = SPSEmpty;

using SPSMachOJITDylibDeinitializerSequence =
SPSSequence<SPSMachOJITDylibDeinitializers>;

template <>
class SPSSerializationTraits<SPSMachOJITDylibDeinitializers,
MachOJITDylibDeinitializers> {
public:
static size_t size(const MachOJITDylibDeinitializers &MOJDDs) { return 0; }

static bool serialize(SPSOutputBuffer &OB,
const MachOJITDylibDeinitializers &MOJDDs) {
return true;
}

static bool deserialize(SPSInputBuffer &IB,
MachOJITDylibDeinitializers &MOJDDs) {
MOJDDs = MachOJITDylibDeinitializers();
return true;
}
};

} // end namespace shared
} // end namespace orc
} // end namespace llvm

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -283,7 +283,11 @@ class OrcRPCExecutorProcessControlBase : public ExecutorProcessControl {
OrcRPCExecutorProcessControlBase(std::shared_ptr<SymbolStringPool> SSP,
RPCEndpointT &EP, ErrorReporter ReportError)
: ExecutorProcessControl(std::move(SSP)),
ReportError(std::move(ReportError)), EP(EP) {}
ReportError(std::move(ReportError)), EP(EP) {
using ThisT = OrcRPCExecutorProcessControlBase<RPCEndpointT>;
EP.template addAsyncHandler<orcrpctpc::RunWrapper>(*this,
&ThisT::runWrapperInJIT);
}

void reportError(Error Err) { ReportError(std::move(Err)); }

Expand Down Expand Up @@ -396,20 +400,32 @@ class OrcRPCExecutorProcessControlBase : public ExecutorProcessControl {
/// Subclasses must call this during construction to initialize the
/// TargetTriple and PageSize members.
Error initializeORCRPCEPCBase() {
if (auto TripleOrErr = EP.template callB<orcrpctpc::GetTargetTriple>())
TargetTriple = Triple(*TripleOrErr);
else
return TripleOrErr.takeError();

if (auto PageSizeOrErr = EP.template callB<orcrpctpc::GetPageSize>())
PageSize = *PageSizeOrErr;
else
return PageSizeOrErr.takeError();
if (auto EPI = EP.template callB<orcrpctpc::GetExecutorProcessInfo>()) {
this->TargetTriple = Triple(EPI->Triple);
this->PageSize = PageSize;
this->JDI = {ExecutorAddress(EPI->DispatchFuncAddr),
ExecutorAddress(EPI->DispatchCtxAddr)};
return Error::success();
} else
return EPI.takeError();
}

private:
Error runWrapperInJIT(
std::function<Error(Expected<shared::WrapperFunctionResult>)> SendResult,
JITTargetAddress FunctionTag, std::vector<uint8_t> ArgBuffer) {

runJITSideWrapperFunction(
[this, SendResult = std::move(SendResult)](
Expected<shared::WrapperFunctionResult> R) {
if (auto Err = SendResult(std::move(R)))
ReportError(std::move(Err));
},
FunctionTag,
{reinterpret_cast<const char *>(ArgBuffer.data()), ArgBuffer.size()});
return Error::success();
}

private:
ErrorReporter ReportError;
RPCEndpointT &EP;
};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@ class SPSInputBuffer {
bool skip(size_t Size) {
if (Size > Remaining)
return false;
Buffer += Size;
Remaining -= Size;
return true;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -298,7 +298,7 @@ class WrapperFunctionAsyncHandlerHelper<RetT(SendResultT, ArgTs...),
SendWFR(ResultSerializer<ResultT>::serialize(std::move(Result)));
};

callAsync(std::forward<HandlerT>(H), std::move(SendResult), Args,
callAsync(std::forward<HandlerT>(H), std::move(SendResult), std::move(Args),
ArgIndices{});
}

Expand All @@ -314,9 +314,9 @@ class WrapperFunctionAsyncHandlerHelper<RetT(SendResultT, ArgTs...),
typename ArgTupleT, std::size_t... I>
static void callAsync(HandlerT &&H,
SerializeAndSendResultT &&SerializeAndSendResult,
ArgTupleT &Args, std::index_sequence<I...>) {
ArgTupleT Args, std::index_sequence<I...>) {
return std::forward<HandlerT>(H)(std::move(SerializeAndSendResult),
std::get<I>(Args)...);
std::move(std::get<I>(Args))...);
}
};

Expand Down
126 changes: 90 additions & 36 deletions llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/OrcRPCTPCServer.h
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,13 @@ enum WireProtectionFlags : uint8_t {
LLVM_MARK_AS_BITMASK_ENUM(WPF_Exec)
};

struct ExecutorProcessInfo {
std::string Triple;
unsigned PageSize;
JITTargetAddress DispatchFuncAddr;
JITTargetAddress DispatchCtxAddr;
};

/// Convert from sys::Memory::ProtectionFlags
inline WireProtectionFlags
toWireProtectionFlags(sys::Memory::ProtectionFlags PF) {
Expand Down Expand Up @@ -95,6 +102,41 @@ using ReleaseOrFinalizeMemRequest =

namespace shared {

template <> class SerializationTypeName<WrapperFunctionResult> {
public:
static const char *getName() { return "WrapperFunctionResult"; }
};

template <typename ChannelT>
class SerializationTraits<
ChannelT, WrapperFunctionResult, WrapperFunctionResult,
std::enable_if_t<std::is_base_of<RawByteChannel, ChannelT>::value>> {
public:
static Error serialize(ChannelT &C, const WrapperFunctionResult &E) {
if (auto Err = serializeSeq(C, static_cast<uint64_t>(E.size())))
return Err;
if (E.size() == 0)
return Error::success();
return C.appendBytes(E.data(), E.size());
}

static Error deserialize(ChannelT &C, WrapperFunctionResult &E) {
uint64_t Size;
if (auto Err = deserializeSeq(C, Size))
return Err;

WrapperFunctionResult Tmp;
char *Data = WrapperFunctionResult::allocate(Tmp, Size);

if (auto Err = C.readBytes(Data, Size))
return Err;

E = std::move(Tmp);

return Error::success();
}
};

template <> class SerializationTypeName<tpctypes::UInt8Write> {
public:
static const char *getName() { return "UInt8Write"; }
Expand Down Expand Up @@ -136,9 +178,9 @@ class SerializationTypeName<orcrpctpc::ReleaseOrFinalizeMemRequestElement> {
static const char *getName() { return "ReleaseOrFinalizeMemRequestElement"; }
};

template <> class SerializationTypeName<shared::WrapperFunctionResult> {
template <> class SerializationTypeName<orcrpctpc::ExecutorProcessInfo> {
public:
static const char *getName() { return "WrapperFunctionResult"; }
static const char *getName() { return "ExecutorProcessInfo"; }
};

template <typename ChannelT, typename WriteT>
Expand Down Expand Up @@ -234,26 +276,17 @@ class SerializationTraits<ChannelT,
};

template <typename ChannelT>
class SerializationTraits<
ChannelT, shared::WrapperFunctionResult, shared::WrapperFunctionResult,
std::enable_if_t<std::is_base_of<RawByteChannel, ChannelT>::value>> {
class SerializationTraits<ChannelT, orcrpctpc::ExecutorProcessInfo> {
public:
static Error serialize(ChannelT &C, const shared::WrapperFunctionResult &E) {
if (auto Err = serializeSeq(C, static_cast<uint64_t>(E.size())))
return Err;
if (E.size() == 0)
return Error::success();
return C.appendBytes(E.data(), E.size());
static Error serialize(ChannelT &C,
const orcrpctpc::ExecutorProcessInfo &EPI) {
return serializeSeq(C, EPI.Triple, EPI.PageSize, EPI.DispatchFuncAddr,
EPI.DispatchCtxAddr);
}

static Error deserialize(ChannelT &C, shared::WrapperFunctionResult &E) {

uint64_t Size;
if (auto Err = deserializeSeq(C, Size))
return Err;

char *DataPtr = shared::WrapperFunctionResult::allocate(E, Size);
return C.readBytes(DataPtr, E.size());
static Error deserialize(ChannelT &C, orcrpctpc::ExecutorProcessInfo &EPI) {
return deserializeSeq(C, EPI.Triple, EPI.PageSize, EPI.DispatchFuncAddr,
EPI.DispatchCtxAddr);
}
};

Expand All @@ -265,15 +298,11 @@ using RemoteSymbolLookupSet = std::vector<std::pair<std::string, bool>>;
using RemoteLookupRequest =
std::pair<tpctypes::DylibHandle, RemoteSymbolLookupSet>;

class GetTargetTriple
: public shared::RPCFunction<GetTargetTriple, std::string()> {
public:
static const char *getName() { return "GetTargetTriple"; }
};

class GetPageSize : public shared::RPCFunction<GetPageSize, uint64_t()> {
class GetExecutorProcessInfo
: public shared::RPCFunction<GetExecutorProcessInfo,
orcrpctpc::ExecutorProcessInfo()> {
public:
static const char *getName() { return "GetPageSize"; }
static const char *getName() { return "GetJITDispatchInfo"; }
};

class ReserveMem
Expand Down Expand Up @@ -349,7 +378,7 @@ class LookupSymbols

class RunMain
: public shared::RPCFunction<RunMain,
int32_t(JITTargetAddress MainAddr,
int64_t(JITTargetAddress MainAddr,
std::vector<std::string> Args)> {
public:
static const char *getName() { return "RunMain"; }
Expand All @@ -372,18 +401,18 @@ class CloseConnection : public shared::RPCFunction<CloseConnection, void()> {

/// TargetProcessControl for a process connected via an ORC RPC Endpoint.
template <typename RPCEndpointT> class OrcRPCTPCServer {
private:
using ThisT = OrcRPCTPCServer<RPCEndpointT>;

public:
/// Create an OrcRPCTPCServer from the given endpoint.
OrcRPCTPCServer(RPCEndpointT &EP) : EP(EP) {
using ThisT = OrcRPCTPCServer<RPCEndpointT>;

TripleStr = sys::getProcessTriple();
PageSize = sys::Process::getPageSizeEstimate();

EP.template addHandler<orcrpctpc::GetTargetTriple>(*this,
&ThisT::getTargetTriple);
EP.template addHandler<orcrpctpc::GetPageSize>(*this, &ThisT::getPageSize);

EP.template addHandler<orcrpctpc::GetExecutorProcessInfo>(
*this, &ThisT::getExecutorProcessInfo);
EP.template addHandler<orcrpctpc::ReserveMem>(*this, &ThisT::reserveMemory);
EP.template addHandler<orcrpctpc::FinalizeMem>(*this,
&ThisT::finalizeMemory);
Expand Down Expand Up @@ -428,9 +457,34 @@ template <typename RPCEndpointT> class OrcRPCTPCServer {
return Error::success();
}

Expected<shared::WrapperFunctionResult>
runWrapperInJIT(JITTargetAddress FunctionId, ArrayRef<char> ArgBuffer) {
return EP.template callB<orcrpctpc::RunWrapper>(
FunctionId,
ArrayRef<uint8_t>(reinterpret_cast<const uint8_t *>(ArgBuffer.data()),
ArgBuffer.size()));
}

private:
std::string getTargetTriple() { return TripleStr; }
uint64_t getPageSize() { return PageSize; }
static shared::detail::CWrapperFunctionResult
jitDispatchViaOrcRPCTPCServer(void *Ctx, const void *FnTag, const char *Data,
size_t Size) {
assert(Ctx && "Attempt to dispatch with null context ptr");
auto R = static_cast<ThisT *>(Ctx)->runWrapperInJIT(
pointerToJITTargetAddress(FnTag), {Data, Size});
if (!R) {
auto ErrMsg = toString(R.takeError());
return shared::WrapperFunctionResult::createOutOfBandError(ErrMsg.data())
.release();
}
return R->release();
}

orcrpctpc::ExecutorProcessInfo getExecutorProcessInfo() {
return {TripleStr, static_cast<uint32_t>(PageSize),
pointerToJITTargetAddress(jitDispatchViaOrcRPCTPCServer),
pointerToJITTargetAddress(this)};
}

template <typename WriteT>
static void handleWriteUInt(const std::vector<WriteT> &Ws) {
Expand Down Expand Up @@ -569,7 +623,7 @@ template <typename RPCEndpointT> class OrcRPCTPCServer {
return Result;
}

int32_t runMain(JITTargetAddress MainFnAddr,
int64_t runMain(JITTargetAddress MainFnAddr,
const std::vector<std::string> &Args) {
Optional<StringRef> ProgramNameOverride;
if (ProgramName)
Expand Down
43 changes: 43 additions & 0 deletions llvm/lib/ExecutionEngine/Orc/Core.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1750,6 +1750,49 @@ Expected<DenseMap<JITDylib *, SymbolMap>> Platform::lookupInitSymbols(
return std::move(CompoundResult);
}

void Platform::lookupInitSymbolsAsync(
unique_function<void(Error)> OnComplete, ExecutionSession &ES,
const DenseMap<JITDylib *, SymbolLookupSet> &InitSyms) {

class TriggerOnComplete {
public:
using OnCompleteFn = unique_function<void(Error)>;
TriggerOnComplete(OnCompleteFn OnComplete)
: OnComplete(std::move(OnComplete)) {}
~TriggerOnComplete() { OnComplete(std::move(LookupResult)); }
void reportResult(Error Err) {
std::lock_guard<std::mutex> Lock(ResultMutex);
LookupResult = joinErrors(std::move(LookupResult), std::move(Err));
}

private:
std::mutex ResultMutex;
Error LookupResult{Error::success()};
OnCompleteFn OnComplete;
};

LLVM_DEBUG({
dbgs() << "Issuing init-symbol lookup:\n";
for (auto &KV : InitSyms)
dbgs() << " " << KV.first->getName() << ": " << KV.second << "\n";
});

auto TOC = std::make_shared<TriggerOnComplete>(std::move(OnComplete));

for (auto &KV : InitSyms) {
auto *JD = KV.first;
auto Names = std::move(KV.second);
ES.lookup(
LookupKind::Static,
JITDylibSearchOrder({{JD, JITDylibLookupFlags::MatchAllSymbols}}),
std::move(Names), SymbolState::Ready,
[TOC](Expected<SymbolMap> Result) {
TOC->reportResult(Result.takeError());
},
NoDependenciesToRegister);
}
}

void Task::anchor() {}

void MaterializationTask::printDescription(raw_ostream &OS) {
Expand Down
29 changes: 29 additions & 0 deletions llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@
#include "llvm/Support/Host.h"
#include "llvm/Support/Process.h"

#define DEBUG_TYPE "orc"

namespace llvm {
namespace orc {

Expand Down Expand Up @@ -47,6 +49,10 @@ Error ExecutorProcessControl::associateJITSideWrapperFunctions(
"AsyncWrapperFunction implementation missing");
TagToFunc[KV.second.getAddress()] =
std::make_shared<AsyncWrapperFunction>(std::move(I->second));
LLVM_DEBUG({
dbgs() << "Associated function tag \"" << *KV.first << "\" ("
<< formatv("{0:x}", KV.second.getAddress()) << ") with handler\n";
});
}
return Error::success();
}
Expand Down Expand Up @@ -84,6 +90,8 @@ SelfExecutorProcessControl::SelfExecutorProcessControl(
this->PageSize = PageSize;
this->MemMgr = OwnedMemMgr.get();
this->MemAccess = this;
this->JDI = {ExecutorAddress::fromPtr(jitDispatchViaWrapperFunctionManager),
ExecutorAddress::fromPtr(this)};
if (this->TargetTriple.isOSBinFormatMachO())
GlobalManglingPrefix = '_';
}
Expand Down Expand Up @@ -198,5 +206,26 @@ void SelfExecutorProcessControl::writeBuffers(
OnWriteComplete(Error::success());
}

shared::detail::CWrapperFunctionResult
SelfExecutorProcessControl::jitDispatchViaWrapperFunctionManager(
void *Ctx, const void *FnTag, const char *Data, size_t Size) {

LLVM_DEBUG({
dbgs() << "jit-dispatch call with tag " << FnTag << " and " << Size
<< " byte payload.\n";
});

std::promise<shared::WrapperFunctionResult> ResultP;
auto ResultF = ResultP.get_future();
static_cast<SelfExecutorProcessControl *>(Ctx)->runJITSideWrapperFunction(
[ResultP =
std::move(ResultP)](shared::WrapperFunctionResult Result) mutable {
ResultP.set_value(std::move(Result));
},
pointerToJITTargetAddress(FnTag), {Data, Size});

return ResultF.get().release();
}

} // end namespace orc
} // end namespace llvm
839 changes: 601 additions & 238 deletions llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ int main(int argc, char *argv[]) {
ExitOnErr.setBanner(std::string(argv[0]) + ":");

using JITLinkExecutorEndpoint =
shared::MultiThreadedRPCEndpoint<shared::FDRawByteChannel>;
shared::SingleThreadedRPCEndpoint<shared::FDRawByteChannel>;

shared::registerStringError<shared::FDRawByteChannel>();

Expand Down
231 changes: 179 additions & 52 deletions llvm/tools/llvm-jitlink/llvm-jitlink.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
#include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h"
#include "llvm/ExecutionEngine/Orc/EPCEHFrameRegistrar.h"
#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
#include "llvm/ExecutionEngine/Orc/MachOPlatform.h"
#include "llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.h"
#include "llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h"
#include "llvm/MC/MCAsmInfo.h"
Expand Down Expand Up @@ -159,6 +160,15 @@ static cl::opt<std::string> OutOfProcessExecutorConnect(
"oop-executor-connect",
cl::desc("Connect to an out-of-process executor via TCP"));

// TODO: Default to false if compiler-rt is not built.
static cl::opt<bool> UseOrcRuntime("use-orc-runtime",
cl::desc("Do not required/load ORC runtime"),
cl::init(true));

static cl::opt<std::string>
OrcRuntimePath("orc-runtime-path", cl::desc("Add orc runtime to session"),
cl::init(""));

ExitOnError ExitOnErr;

LLVM_ATTRIBUTE_USED void linkComponents() {
Expand All @@ -167,6 +177,14 @@ LLVM_ATTRIBUTE_USED void linkComponents() {
<< (void *)&llvm_orc_registerJITLoaderGDBWrapper;
}

static bool UseTestResultOverride = false;
static int64_t TestResultOverride = 0;

extern "C" void llvm_jitlink_setTestResultOverride(int64_t Value) {
TestResultOverride = Value;
UseTestResultOverride = true;
}

namespace llvm {

static raw_ostream &
Expand Down Expand Up @@ -588,6 +606,31 @@ Error LLVMJITLinkObjectLinkingLayer::add(ResourceTrackerSP RT,
return JD.define(std::move(MU), std::move(RT));
}

static Error loadProcessSymbols(Session &S) {
auto FilterMainEntryPoint =
[EPName = S.ES.intern(EntryPointName)](SymbolStringPtr Name) {
return Name != EPName;
};
S.MainJD->addGenerator(
ExitOnErr(orc::EPCDynamicLibrarySearchGenerator::GetForTargetProcess(
*S.EPC, std::move(FilterMainEntryPoint))));

return Error::success();
}

static Error loadDylibs(Session &S) {
LLVM_DEBUG(dbgs() << "Loading dylibs...\n");
for (const auto &Dylib : Dylibs) {
LLVM_DEBUG(dbgs() << " " << Dylib << "\n");
auto G = orc::EPCDynamicLibrarySearchGenerator::Load(*S.EPC, Dylib.c_str());
if (!G)
return G.takeError();
S.MainJD->addGenerator(std::move(*G));
}

return Error::success();
}

Expected<std::unique_ptr<ExecutorProcessControl>>
LLVMJITLinkRemoteExecutorProcessControl::LaunchExecutor() {
#ifndef LLVM_ON_UNIX
Expand Down Expand Up @@ -796,40 +839,46 @@ Expected<std::unique_ptr<Session>> Session::Create(Triple TT) {
if (!PageSize)
return PageSize.takeError();

/// If -oop-executor is passed then launch the executor.
std::unique_ptr<ExecutorProcessControl> EPC;
if (OutOfProcessExecutor.getNumOccurrences()) {
/// If -oop-executor is passed then launch the executor.
if (auto REPC = LLVMJITLinkRemoteExecutorProcessControl::LaunchExecutor())
EPC = std::move(*REPC);
else
return REPC.takeError();
} else if (OutOfProcessExecutorConnect.getNumOccurrences()) {
/// If -oop-executor-connect is passed then connect to the executor.
if (auto REPC =
LLVMJITLinkRemoteExecutorProcessControl::ConnectToExecutor())
EPC = std::move(*REPC);
else
return REPC.takeError();
} else
} else {
/// Otherwise use SelfExecutorProcessControl to target the current process.
EPC = std::make_unique<SelfExecutorProcessControl>(
std::make_shared<SymbolStringPool>(), std::move(TT), *PageSize,
createMemoryManager());
}

Error Err = Error::success();
std::unique_ptr<Session> S(new Session(std::move(EPC), Err));
if (Err)
return std::move(Err);
return std::move(S);

// FIXME: Errors destroy the session, leaving the SymbolStringPtrs dangling,
// so just exit here. We could fix this by having errors keep the pool alive.
ExitOnErr(std::move(Err));
return S;
}

Session::~Session() {
if (auto Err = ES.endSession())
ES.reportError(std::move(Err));
if (auto Err = EPC->disconnect())
ES.reportError(std::move(Err));
}

// FIXME: Move to createJITDylib if/when we start using Platform support in
// llvm-jitlink.
Session::Session(std::unique_ptr<ExecutorProcessControl> EPC, Error &Err)
: EPC(std::move(EPC)), ObjLayer(*this, this->EPC->getMemMgr()) {
: EPC(std::move(EPC)), ES(this->EPC->getSymbolStringPool()),
ObjLayer(*this, this->EPC->getMemMgr()) {

/// Local ObjectLinkingLayer::Plugin class to forward modifyPassConfig to the
/// Session.
Expand Down Expand Up @@ -863,7 +912,21 @@ Session::Session(std::unique_ptr<ExecutorProcessControl> EPC, Error &Err)
return;
}

if (!NoExec && !this->EPC->getTargetTriple().isOSWindows()) {
if (!NoProcessSymbols)
ExitOnErr(loadProcessSymbols(*this));
ExitOnErr(loadDylibs(*this));

// Set up the platform.
if (this->EPC->getTargetTriple().isOSBinFormatMachO() && UseOrcRuntime) {
if (auto P = MachOPlatform::Create(ES, ObjLayer, *this->EPC, *MainJD,
OrcRuntimePath.c_str()))
ES.setPlatform(std::move(*P));
else {
Err = P.takeError();
return;
}
} else if (!NoExec && !this->EPC->getTargetTriple().isOSWindows() &&
!this->EPC->getTargetTriple().isOSBinFormatMachO()) {
ObjLayer.addPlugin(std::make_unique<EHFrameRegistrationPlugin>(
ES, ExitOnErr(EPCEHFrameRegistrar::Create(*this->EPC))));
ObjLayer.addPlugin(std::make_unique<DebugObjectManagerPlugin>(
Expand Down Expand Up @@ -1044,18 +1107,41 @@ static Triple getFirstFileTriple() {
return FirstTT;
}

static Error sanitizeArguments(const Triple &TT, const char *ArgV0) {
// Set the entry point name if not specified.
if (EntryPointName.empty()) {
if (TT.getObjectFormat() == Triple::MachO)
EntryPointName = "_main";
else
EntryPointName = "main";
static bool isOrcRuntimeSupported(const Triple &TT) {
switch (TT.getObjectFormat()) {
case Triple::MachO:
switch (TT.getArch()) {
case Triple::x86_64:
return true;
default:
return false;
}
default:
return false;
}
}

static Error sanitizeArguments(const Triple &TT, const char *ArgV0) {

// If we're in noexec mode and the user didn't explicitly specify
// -use-orc-runtime then don't use it.
if (NoExec && UseOrcRuntime.getNumOccurrences() == 0)
UseOrcRuntime = false;

// -noexec and --args should not be used together.
if (NoExec && !InputArgv.empty())
outs() << "Warning: --args passed to -noexec run will be ignored.\n";
errs() << "Warning: --args passed to -noexec run will be ignored.\n";

// Turn off UseOrcRuntime on platforms where it's not supported.
if (UseOrcRuntime && !isOrcRuntimeSupported(TT)) {
errs() << "Warning: Orc runtime not available for target platform. "
"Use -use-orc-runtime=false to suppress this warning.\n";
UseOrcRuntime = false;
}

// Set the entry point name if not specified.
if (EntryPointName.empty())
EntryPointName = TT.getObjectFormat() == Triple::MachO ? "_main" : "main";

// If -slab-address is passed, require -slab-allocate and -noexec
if (SlabAddress != ~0ULL) {
Expand All @@ -1080,35 +1166,27 @@ static Error sanitizeArguments(const Triple &TT, const char *ArgV0) {
SmallString<256> OOPExecutorPath(sys::fs::getMainExecutable(
ArgV0, reinterpret_cast<void *>(&sanitizeArguments)));
sys::path::remove_filename(OOPExecutorPath);
if (OOPExecutorPath.back() != '/')
OOPExecutorPath += '/';
OOPExecutorPath += "llvm-jitlink-executor";
sys::path::append(OOPExecutorPath, "llvm-jitlink-executor");
OutOfProcessExecutor = OOPExecutorPath.str().str();
}

return Error::success();
}

static Error loadProcessSymbols(Session &S) {
auto FilterMainEntryPoint =
[EPName = S.ES.intern(EntryPointName)](SymbolStringPtr Name) {
return Name != EPName;
};
S.MainJD->addGenerator(
ExitOnErr(orc::EPCDynamicLibrarySearchGenerator::GetForTargetProcess(
*S.EPC, std::move(FilterMainEntryPoint))));

return Error::success();
}

static Error loadDylibs(Session &S) {
for (const auto &Dylib : Dylibs) {
auto G = orc::EPCDynamicLibrarySearchGenerator::Load(*S.EPC, Dylib.c_str());
if (!G)
return G.takeError();
S.MainJD->addGenerator(std::move(*G));
// If we're loading the Orc runtime then determine the path for it.
if (UseOrcRuntime) {
if (OrcRuntimePath.empty()) {
SmallString<256> DefaultOrcRuntimePath(sys::fs::getMainExecutable(
ArgV0, reinterpret_cast<void *>(&sanitizeArguments)));
sys::path::remove_filename(
DefaultOrcRuntimePath); // remove 'llvm-jitlink'
while (!DefaultOrcRuntimePath.empty() &&
DefaultOrcRuntimePath.back() == '/')
DefaultOrcRuntimePath.pop_back();
if (DefaultOrcRuntimePath.endswith("bin"))
sys::path::remove_filename(DefaultOrcRuntimePath); // remove 'bin'
sys::path::append(DefaultOrcRuntimePath,
"lib/clang/13.0.0/lib/darwin/libclang_rt.orc_osx.a");
OrcRuntimePath = DefaultOrcRuntimePath.str().str();
}
}

return Error::success();
}

Expand Down Expand Up @@ -1227,6 +1305,11 @@ static Error loadObjects(Session &S) {

static Error runChecks(Session &S) {

if (CheckFiles.empty())
return Error::success();

LLVM_DEBUG(dbgs() << "Running checks...\n");

auto TripleName = S.EPC->getTargetTriple().str();
std::string ErrorStr;
const Target *TheTarget = TargetRegistry::lookupTarget(TripleName, ErrorStr);
Expand Down Expand Up @@ -1310,16 +1393,50 @@ static Error runChecks(Session &S) {
}

static void dumpSessionStats(Session &S) {
if (!ShowSizes)
return;
if (UseOrcRuntime)
outs() << "Note: Session stats include runtime and entry point lookup, but "
"not JITDylib initialization/deinitialization.\n";
if (ShowSizes)
outs() << "Total size of all blocks before pruning: " << S.SizeBeforePruning
<< "\nTotal size of all blocks after fixups: " << S.SizeAfterFixups
outs() << " Total size of all blocks before pruning: "
<< S.SizeBeforePruning
<< "\n Total size of all blocks after fixups: " << S.SizeAfterFixups
<< "\n";
}

static Expected<JITEvaluatedSymbol> getMainEntryPoint(Session &S) {
return S.ES.lookup(S.JDSearchOrder, EntryPointName);
}

static Expected<JITEvaluatedSymbol> getOrcRuntimeEntryPoint(Session &S) {
std::string RuntimeEntryPoint = "__orc_rt_run_program_wrapper";
if (S.EPC->getTargetTriple().getObjectFormat() == Triple::MachO)
RuntimeEntryPoint = '_' + RuntimeEntryPoint;
return S.ES.lookup(S.JDSearchOrder, RuntimeEntryPoint);
}

static Expected<int> runWithRuntime(Session &S,
JITTargetAddress EntryPointAddress) {
StringRef DemangledEntryPoint = EntryPointName;
if (S.EPC->getTargetTriple().getObjectFormat() == Triple::MachO &&
DemangledEntryPoint.front() == '_')
DemangledEntryPoint = DemangledEntryPoint.drop_front();
using SPSRunProgramSig =
int64_t(SPSString, SPSString, SPSSequence<SPSString>);
int64_t Result;
if (auto Err = S.EPC->runSPSWrapper<SPSRunProgramSig>(
EntryPointAddress, Result, S.MainJD->getName(), DemangledEntryPoint,
static_cast<std::vector<std::string> &>(InputArgv)))
return std::move(Err);
return Result;
}

static Expected<int> runWithoutRuntime(Session &S,
JITTargetAddress EntryPointAddress) {
return S.EPC->runAsMain(EntryPointAddress, InputArgv);
}

namespace {
struct JITLinkTimers {
TimerGroup JITLinkTG{"llvm-jitlink timers", "timers for llvm-jitlink phases"};
Expand Down Expand Up @@ -1352,21 +1469,23 @@ int main(int argc, char *argv[]) {
ExitOnErr(loadObjects(*S));
}

if (!NoProcessSymbols)
ExitOnErr(loadProcessSymbols(*S));
ExitOnErr(loadDylibs(*S));

if (PhonyExternals)
addPhonyExternalsGenerator(*S);


if (ShowInitialExecutionSessionState)
S->ES.dump(outs());

JITEvaluatedSymbol EntryPoint = 0;
{
TimeRegion TR(Timers ? &Timers->LinkTimer : nullptr);
// Find the entry-point function unconditionally, since we want to force
// it to be materialized to collect stats.
EntryPoint = ExitOnErr(getMainEntryPoint(*S));

// If we're running with the ORC runtime then replace the entry-point
// with the __orc_rt_run_program symbol.
if (UseOrcRuntime)
EntryPoint = ExitOnErr(getOrcRuntimeEntryPoint(*S));
}

if (ShowAddrs)
Expand All @@ -1381,12 +1500,20 @@ int main(int argc, char *argv[]) {

int Result = 0;
{
LLVM_DEBUG(dbgs() << "Running \"" << EntryPointName << "\"...\n");
TimeRegion TR(Timers ? &Timers->RunTimer : nullptr);
Result = ExitOnErr(S->EPC->runAsMain(EntryPoint.getAddress(), InputArgv));
if (UseOrcRuntime)
Result = ExitOnErr(runWithRuntime(*S, EntryPoint.getAddress()));
else
Result = ExitOnErr(runWithoutRuntime(*S, EntryPoint.getAddress()));
}

ExitOnErr(S->ES.endSession());
ExitOnErr(S->EPC->disconnect());
// Destroy the session.
S.reset();

// If the executing code set a test result override then use that.
if (UseTestResultOverride)
Result = TestResultOverride;

return Result;
}
2 changes: 1 addition & 1 deletion llvm/tools/llvm-jitlink/llvm-jitlink.h
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ class LLVMJITLinkRemoteExecutorProcessControl
struct Session {
std::unique_ptr<orc::ExecutorProcessControl> EPC;
orc::ExecutionSession ES;
orc::JITDylib *MainJD;
orc::JITDylib *MainJD = nullptr;
LLVMJITLinkObjectLinkingLayer ObjLayer;
std::vector<orc::JITDylib *> JDSearchOrder;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
//===----------------------------------------------------------------------===//

#include "llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h"
#include "llvm/ExecutionEngine/Orc/LLVMSPSSerializers.h"
#include "gtest/gtest.h"

using namespace llvm;
Expand Down Expand Up @@ -158,3 +159,9 @@ TEST(SimplePackedSerializationTest, ArgListSerialization) {
EXPECT_EQ(Arg2, ArgOut2);
EXPECT_EQ(Arg3, ArgOut3);
}

TEST(SimplePackedSerialization, StringMap) {
StringMap<int32_t> M({{"A", 1}, {"B", 2}});
blobSerializationRoundTrip<SPSSequence<SPSTuple<SPSString, int32_t>>,
StringMap<int32_t>>(M);
}