72 changes: 72 additions & 0 deletions llvm/include/llvm/ExecutionEngine/Orc/JITLinkReentryTrampolines.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
//===- JITLinkReentryTrampolines.h -- JITLink-based trampolines -*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// Emit reentry trampolines via JITLink.
//
//===----------------------------------------------------------------------===//

#ifndef LLVM_EXECUTIONENGINE_ORC_JITLINKREENTRYTRAMPOLINES_H
#define LLVM_EXECUTIONENGINE_ORC_JITLINKREENTRYTRAMPOLINES_H

#include "llvm/ADT/FunctionExtras.h"
#include "llvm/ExecutionEngine/Orc/Core.h"
#include "llvm/ExecutionEngine/Orc/LazyReexports.h"
#include "llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h"
#include "llvm/Support/Error.h"

namespace llvm::jitlink {
class Block;
class LinkGraph;
class Section;
class Symbol;
} // namespace llvm::jitlink

namespace llvm::orc {

class ObjectLinkingLayer;
class RedirectableSymbolManager;

/// Produces trampolines on request using JITLink.
class JITLinkReentryTrampolines {
public:
using EmitTrampolineFn = unique_function<jitlink::Symbol &(
jitlink::LinkGraph &G, jitlink::Section &Sec,
jitlink::Symbol &ReentrySym)>;
using OnTrampolinesReadyFn = unique_function<void(
Expected<std::vector<ExecutorSymbolDef>> EntryAddrs)>;

/// Create trampolines using the default reentry trampoline function for
/// the session triple.
static Expected<std::unique_ptr<JITLinkReentryTrampolines>>
Create(ObjectLinkingLayer &ObjLinkingLayer);

JITLinkReentryTrampolines(ObjectLinkingLayer &ObjLinkingLayer,
EmitTrampolineFn EmitTrampoline);
JITLinkReentryTrampolines(JITLinkReentryTrampolines &&) = delete;
JITLinkReentryTrampolines &operator=(JITLinkReentryTrampolines &&) = delete;

void emit(ResourceTrackerSP RT, size_t NumTrampolines,
OnTrampolinesReadyFn OnTrampolinesReady);

private:
class TrampolineAddrScraperPlugin;

ObjectLinkingLayer &ObjLinkingLayer;
TrampolineAddrScraperPlugin *TrampolineAddrScraper = nullptr;
EmitTrampolineFn EmitTrampoline;
std::atomic<size_t> ReentryGraphIdx{0};
};

Expected<std::unique_ptr<LazyReexportsManager>>
createJITLinkLazyReexportsManager(ObjectLinkingLayer &ObjLinkingLayer,
RedirectableSymbolManager &RSMgr,
JITDylib &PlatformJD);

} // namespace llvm::orc

#endif // LLVM_EXECUTIONENGINE_ORC_JITLINKREENTRYTRAMPOLINES_H
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,13 @@
namespace llvm::orc {

class ObjectLinkingLayer;
class LazyCallThroughManager;
class LazyReexportsManager;
class RedirectableSymbolManager;

class LazyObjectLinkingLayer : public ObjectLayer {
public:
LazyObjectLinkingLayer(ObjectLinkingLayer &BaseLayer,
LazyCallThroughManager &LCTMgr,
RedirectableSymbolManager &RSMgr);
LazyReexportsManager &LRMgr);

llvm::Error add(llvm::orc::ResourceTrackerSP RT,
std::unique_ptr<llvm::MemoryBuffer> O,
Expand All @@ -38,8 +37,7 @@ class LazyObjectLinkingLayer : public ObjectLayer {
class RenamerPlugin;

ObjectLinkingLayer &BaseLayer;
LazyCallThroughManager &LCTMgr;
RedirectableSymbolManager &RSMgr;
LazyReexportsManager &LRMgr;
};

} // namespace llvm::orc
Expand Down
64 changes: 64 additions & 0 deletions llvm/include/llvm/ExecutionEngine/Orc/LazyReexports.h
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,70 @@ lazyReexports(LazyCallThroughManager &LCTManager,
LCTManager, RSManager, SourceJD, std::move(CallableAliases), SrcJDLoc);
}

class LazyReexportsManager {

friend std::unique_ptr<MaterializationUnit>
lazyReexports(LazyReexportsManager &, SymbolAliasMap);

public:
using OnTrampolinesReadyFn = unique_function<void(
Expected<std::vector<ExecutorSymbolDef>> EntryAddrs)>;
using EmitTrampolinesFn =
unique_function<void(ResourceTrackerSP RT, size_t NumTrampolines,
OnTrampolinesReadyFn OnTrampolinesReady)>;

/// Create a LazyReexportsManager that uses the ORC runtime for reentry.
/// This will work both in-process and out-of-process.
static Expected<std::unique_ptr<LazyReexportsManager>>
Create(EmitTrampolinesFn EmitTrampolines, RedirectableSymbolManager &RSMgr,
JITDylib &PlatformJD);

LazyReexportsManager(LazyReexportsManager &&) = delete;
LazyReexportsManager &operator=(LazyReexportsManager &&) = delete;

private:
struct CallThroughInfo {
SymbolStringPtr Name;
SymbolStringPtr BodyName;
JITDylibSP JD;
};

class MU;
class Plugin;

using ResolveSendResultFn =
unique_function<void(Expected<ExecutorSymbolDef>)>;

LazyReexportsManager(EmitTrampolinesFn EmitTrampolines,
RedirectableSymbolManager &RSMgr, JITDylib &PlatformJD,
Error &Err);

std::unique_ptr<MaterializationUnit>
createLazyReexports(SymbolAliasMap Reexports);

void emitReentryTrampolines(std::unique_ptr<MaterializationResponsibility> MR,
SymbolAliasMap Reexports);
void emitRedirectableSymbols(
std::unique_ptr<MaterializationResponsibility> MR,
SymbolAliasMap Reexports,
Expected<std::vector<ExecutorSymbolDef>> ReentryPoints);
void resolve(ResolveSendResultFn SendResult, ExecutorAddr ReentryStubAddr);

EmitTrampolinesFn EmitTrampolines;
RedirectableSymbolManager &RSMgr;

std::mutex M;
DenseMap<ExecutorAddr, CallThroughInfo> CallThroughs;
};

/// Define lazy-reexports based on the given SymbolAliasMap. Each lazy re-export
/// is a callable symbol that will look up and dispatch to the given aliasee on
/// first call. All subsequent calls will go directly to the aliasee.
inline std::unique_ptr<MaterializationUnit>
lazyReexports(LazyReexportsManager &LRM, SymbolAliasMap Reexports) {
return LRM.createLazyReexports(std::move(Reexports));
}

} // End namespace orc
} // End namespace llvm

Expand Down
5 changes: 5 additions & 0 deletions llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,11 @@ class MachOPlatform : public Platform {
static ArrayRef<std::pair<const char *, const char *>>
standardRuntimeUtilityAliases();

/// Returns a list of aliases required to enable lazy compilation via the
/// ORC runtime.
static ArrayRef<std::pair<const char *, const char *>>
standardLazyCompilationAliases();

private:
using SymbolTableVector = SmallVector<
std::tuple<ExecutorAddr, ExecutorAddr, MachOExecutorSymbolFlags>>;
Expand Down
1 change: 1 addition & 0 deletions llvm/include/llvm/ExecutionEngine/Orc/RedirectionManager.h
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ namespace orc {
class RedirectionManager {
public:
virtual ~RedirectionManager() = default;

/// Change the redirection destination of given symbols to new destination
/// symbols.
virtual Error redirect(JITDylib &JD, const SymbolMap &NewDests) = 0;
Expand Down
5 changes: 5 additions & 0 deletions llvm/lib/ExecutionEngine/JITLink/aarch64.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,11 @@ const char PointerJumpStubContent[12] = {
0x00, 0x02, 0x1f, (char)0xd6u // BR x16
};

const char ReentryTrampolineContent[8] = {
(char)0xfd, 0x7b, (char)0xbf, (char)0xa9, // STP x30, [sp, #-8]
0x00, 0x00, 0x00, (char)0x94 // BL
};

const char *getEdgeKindName(Edge::Kind R) {
switch (R) {
case Pointer64:
Expand Down
1 change: 1 addition & 0 deletions llvm/lib/ExecutionEngine/Orc/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ add_llvm_component_library(LLVMOrcJIT
IRTransformLayer.cpp
IRPartitionLayer.cpp
JITTargetMachineBuilder.cpp
JITLinkReentryTrampolines.cpp
LazyObjectLinkingLayer.cpp
LazyReexports.cpp
Layer.cpp
Expand Down
21 changes: 21 additions & 0 deletions llvm/lib/ExecutionEngine/Orc/Core.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,27 @@ void UnexpectedSymbolDefinitions::log(raw_ostream &OS) const {
<< ": " << Symbols;
}

void SymbolInstance::lookupAsync(LookupAsyncOnCompleteFn OnComplete) const {
JD->getExecutionSession().lookup(
LookupKind::Static, {{JD.get(), JITDylibLookupFlags::MatchAllSymbols}},
SymbolLookupSet(Name), SymbolState::Ready,
[OnComplete = std::move(OnComplete)
#ifndef NDEBUG
,
Name = this->Name // Captured for the assert below only.
#endif // NDEBUG
](Expected<SymbolMap> Result) mutable {
if (Result) {
assert(Result->size() == 1 && "Unexpected number of results");
assert(Result->count(Name) &&
"Result does not contain expected symbol");
OnComplete(Result->begin()->second);
} else
OnComplete(Result.takeError());
},
NoDependenciesToRegister);
}

AsynchronousSymbolQuery::AsynchronousSymbolQuery(
const SymbolLookupSet &Symbols, SymbolState RequiredState,
SymbolsResolvedCallback NotifyComplete)
Expand Down
184 changes: 184 additions & 0 deletions llvm/lib/ExecutionEngine/Orc/JITLinkReentryTrampolines.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,184 @@
//===----- JITLinkReentryTrampolines.cpp -- JITLink-based trampoline- -----===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#include "llvm/ExecutionEngine/Orc/JITLinkReentryTrampolines.h"

#include "llvm/ExecutionEngine/JITLink/aarch64.h"
#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"

#include <memory>

#define DEBUG_TYPE "orc"

using namespace llvm;
using namespace llvm::jitlink;

namespace {
constexpr StringRef ReentryFnName = "__orc_rt_reentry";
constexpr StringRef ReentrySectionName = "__orc_stubs";
} // namespace

namespace llvm::orc {

class JITLinkReentryTrampolines::TrampolineAddrScraperPlugin
: public ObjectLinkingLayer::Plugin {
public:
void modifyPassConfig(MaterializationResponsibility &MR,
jitlink::LinkGraph &G,
jitlink::PassConfiguration &Config) override {
Config.PreFixupPasses.push_back(
[this](LinkGraph &G) { return recordTrampolineAddrs(G); });
}

Error notifyFailed(MaterializationResponsibility &MR) override {
return Error::success();
}

Error notifyRemovingResources(JITDylib &JD, ResourceKey K) override {
return Error::success();
}

void notifyTransferringResources(JITDylib &JD, ResourceKey DstKey,
ResourceKey SrcKey) override {}

void registerGraph(LinkGraph &G,
std::shared_ptr<std::vector<ExecutorSymbolDef>> Addrs) {
std::lock_guard<std::mutex> Lock(M);
assert(!PendingAddrs.count(&G) && "Duplicate registration");
PendingAddrs[&G] = std::move(Addrs);
}

Error recordTrampolineAddrs(LinkGraph &G) {
std::shared_ptr<std::vector<ExecutorSymbolDef>> Addrs;
{
std::lock_guard<std::mutex> Lock(M);
auto I = PendingAddrs.find(&G);
if (I == PendingAddrs.end())
return Error::success();
Addrs = std::move(I->second);
PendingAddrs.erase(I);
}

auto *Sec = G.findSectionByName(ReentrySectionName);
assert(Sec && "Reentry graph missing reentry section");
assert(!Sec->empty() && "Reentry graph is empty");

for (auto *Sym : Sec->symbols())
if (!Sym->hasName())
Addrs->push_back({Sym->getAddress(), JITSymbolFlags()});

return Error::success();
}

private:
std::mutex M;
DenseMap<LinkGraph *, std::shared_ptr<std::vector<ExecutorSymbolDef>>>
PendingAddrs;
};

Expected<std::unique_ptr<JITLinkReentryTrampolines>>
JITLinkReentryTrampolines::Create(ObjectLinkingLayer &ObjLinkingLayer) {

EmitTrampolineFn EmitTrampoline;

switch (ObjLinkingLayer.getExecutionSession().getTargetTriple().getArch()) {
case Triple::aarch64:
EmitTrampoline = aarch64::createAnonymousReentryTrampoline;
break;
default:
return make_error<StringError>("Architecture not supported",
inconvertibleErrorCode());
}

return std::make_unique<JITLinkReentryTrampolines>(ObjLinkingLayer,
std::move(EmitTrampoline));
}

JITLinkReentryTrampolines::JITLinkReentryTrampolines(
ObjectLinkingLayer &ObjLinkingLayer, EmitTrampolineFn EmitTrampoline)
: ObjLinkingLayer(ObjLinkingLayer),
EmitTrampoline(std::move(EmitTrampoline)) {
auto TAS = std::make_shared<TrampolineAddrScraperPlugin>();
TrampolineAddrScraper = TAS.get();
ObjLinkingLayer.addPlugin(std::move(TAS));
}

void JITLinkReentryTrampolines::emit(ResourceTrackerSP RT,
size_t NumTrampolines,
OnTrampolinesReadyFn OnTrampolinesReady) {

if (NumTrampolines == 0)
return OnTrampolinesReady(std::vector<ExecutorSymbolDef>());

JITDylibSP JD(&RT->getJITDylib());
auto &ES = ObjLinkingLayer.getExecutionSession();
Triple TT = ES.getTargetTriple();

auto ReentryGraphSym =
ES.intern(("__orc_reentry_graph_#" + Twine(++ReentryGraphIdx)).str());

auto G = std::make_unique<jitlink::LinkGraph>(
(*ReentryGraphSym).str(), ES.getSymbolStringPool(), TT,
TT.isArch64Bit() ? 8 : 4,
TT.isLittleEndian() ? endianness::little : endianness::big,
jitlink::getGenericEdgeKindName);

auto &ReentryFnSym = G->addExternalSymbol(ReentryFnName, 0, false);

auto &ReentrySection =
G->createSection(ReentrySectionName, MemProt::Exec | MemProt::Read);

for (size_t I = 0; I != NumTrampolines; ++I)
EmitTrampoline(*G, ReentrySection, ReentryFnSym).setLive(true);

auto &FirstBlock = **ReentrySection.blocks().begin();
G->addDefinedSymbol(FirstBlock, 0, *ReentryGraphSym, FirstBlock.getSize(),
Linkage::Strong, Scope::SideEffectsOnly, true, true);

auto TrampolineAddrs = std::make_shared<std::vector<ExecutorSymbolDef>>();
TrampolineAddrScraper->registerGraph(*G, TrampolineAddrs);

// Add Graph via object linking layer.
if (auto Err = ObjLinkingLayer.add(std::move(RT), std::move(G)))
return OnTrampolinesReady(std::move(Err));

// Trigger graph emission.
ES.lookup(
LookupKind::Static, {{JD.get(), JITDylibLookupFlags::MatchAllSymbols}},
SymbolLookupSet(ReentryGraphSym,
SymbolLookupFlags::WeaklyReferencedSymbol),
SymbolState::Ready,
[OnTrampolinesReady = std::move(OnTrampolinesReady),
TrampolineAddrs =
std::move(TrampolineAddrs)](Expected<SymbolMap> Result) mutable {
if (Result)
OnTrampolinesReady(std::move(*TrampolineAddrs));
else
OnTrampolinesReady(Result.takeError());
},
NoDependenciesToRegister);
}

Expected<std::unique_ptr<LazyReexportsManager>>
createJITLinkLazyReexportsManager(ObjectLinkingLayer &ObjLinkingLayer,
RedirectableSymbolManager &RSMgr,
JITDylib &PlatformJD) {
auto JLT = JITLinkReentryTrampolines::Create(ObjLinkingLayer);
if (!JLT)
return JLT.takeError();

return LazyReexportsManager::Create(
[JLT = std::move(*JLT)](ResourceTrackerSP RT, size_t NumTrampolines,
LazyReexportsManager::OnTrampolinesReadyFn
OnTrampolinesReady) mutable {
JLT->emit(std::move(RT), NumTrampolines, std::move(OnTrampolinesReady));
},
RSMgr, PlatformJD);
}

} // namespace llvm::orc
8 changes: 3 additions & 5 deletions llvm/lib/ExecutionEngine/Orc/LazyObjectLinkingLayer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -71,10 +71,9 @@ class LazyObjectLinkingLayer::RenamerPlugin
};

LazyObjectLinkingLayer::LazyObjectLinkingLayer(ObjectLinkingLayer &BaseLayer,
LazyCallThroughManager &LCTMgr,
RedirectableSymbolManager &RSMgr)
LazyReexportsManager &LRMgr)
: ObjectLayer(BaseLayer.getExecutionSession()), BaseLayer(BaseLayer),
LCTMgr(LCTMgr), RSMgr(RSMgr) {
LRMgr(LRMgr) {
BaseLayer.addPlugin(std::make_unique<RenamerPlugin>());
}

Expand All @@ -101,8 +100,7 @@ Error LazyObjectLinkingLayer::add(ResourceTrackerSP RT,
return Err;

auto &JD = RT->getJITDylib();
return JD.define(lazyReexports(LCTMgr, RSMgr, JD, std::move(LazySymbols)),
std::move(RT));
return JD.define(lazyReexports(LRMgr, std::move(LazySymbols)), std::move(RT));
}

void LazyObjectLinkingLayer::emit(
Expand Down
164 changes: 164 additions & 0 deletions llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,9 @@

#include "llvm/ExecutionEngine/Orc/LazyReexports.h"

#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
#include "llvm/ExecutionEngine/Orc/OrcABISupport.h"
#include "llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h"
#include "llvm/TargetParser/Triple.h"

#define DEBUG_TYPE "orc"
Expand Down Expand Up @@ -229,5 +231,167 @@ LazyReexportsMaterializationUnit::extractFlags(const SymbolAliasMap &Aliases) {
return MaterializationUnit::Interface(std::move(SymbolFlags), nullptr);
}

class LazyReexportsManager::MU : public MaterializationUnit {
public:
MU(LazyReexportsManager &LRMgr, SymbolAliasMap Reexports)
: MaterializationUnit(getInterface(Reexports)), LRMgr(LRMgr),
Reexports(std::move(Reexports)) {}

private:
Interface getInterface(const SymbolAliasMap &Reexports) {
SymbolFlagsMap SF;
for (auto &[Alias, AI] : Reexports)
SF[Alias] = AI.AliasFlags;
return {std::move(SF), nullptr};
}

StringRef getName() const override { return "LazyReexportsManager::MU"; }

void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
LRMgr.emitReentryTrampolines(std::move(R), std::move(Reexports));
}

void discard(const JITDylib &JD, const SymbolStringPtr &Name) override {
Reexports.erase(Name);
}

LazyReexportsManager &LRMgr;
SymbolAliasMap Reexports;
};

class LazyReexportsManager::Plugin : public ObjectLinkingLayer::Plugin {
public:
void modifyPassConfig(MaterializationResponsibility &MR,
jitlink::LinkGraph &G,
jitlink::PassConfiguration &Config) override {}

Error notifyFailed(MaterializationResponsibility &MR) override {
return Error::success();
}

Error notifyRemovingResources(JITDylib &JD, ResourceKey K) override {
return Error::success();
}

void notifyTransferringResources(JITDylib &JD, ResourceKey DstKey,
ResourceKey SrcKey) override {}

private:
std::mutex M;
};

Expected<std::unique_ptr<LazyReexportsManager>>
LazyReexportsManager::Create(EmitTrampolinesFn EmitTrampolines,
RedirectableSymbolManager &RSMgr,
JITDylib &PlatformJD) {
Error Err = Error::success();
std::unique_ptr<LazyReexportsManager> LRM(new LazyReexportsManager(
std::move(EmitTrampolines), RSMgr, PlatformJD, Err));
if (Err)
return std::move(Err);
return std::move(LRM);
}

LazyReexportsManager::LazyReexportsManager(EmitTrampolinesFn EmitTrampolines,
RedirectableSymbolManager &RSMgr,
JITDylib &PlatformJD, Error &Err)
: EmitTrampolines(std::move(EmitTrampolines)), RSMgr(RSMgr) {

using namespace shared;

ErrorAsOutParameter _(&Err);

auto &ES = PlatformJD.getExecutionSession();
ExecutionSession::JITDispatchHandlerAssociationMap WFs;

WFs[ES.intern("__orc_rt_resolve_tag")] =
ES.wrapAsyncWithSPS<SPSExpected<SPSExecutorSymbolDef>(SPSExecutorAddr)>(
this, &LazyReexportsManager::resolve);

Err = ES.registerJITDispatchHandlers(PlatformJD, std::move(WFs));
}

std::unique_ptr<MaterializationUnit>
LazyReexportsManager::createLazyReexports(SymbolAliasMap Reexports) {
return std::make_unique<MU>(*this, std::move(Reexports));
}

void LazyReexportsManager::emitReentryTrampolines(
std::unique_ptr<MaterializationResponsibility> MR,
SymbolAliasMap Reexports) {
size_t NumTrampolines = Reexports.size();
auto RT = MR->getResourceTracker();
EmitTrampolines(
std::move(RT), NumTrampolines,
[this, MR = std::move(MR), Reexports = std::move(Reexports)](
Expected<std::vector<ExecutorSymbolDef>> ReentryPoints) mutable {
emitRedirectableSymbols(std::move(MR), std::move(Reexports),
std::move(ReentryPoints));
});
}

void LazyReexportsManager::emitRedirectableSymbols(
std::unique_ptr<MaterializationResponsibility> MR, SymbolAliasMap Reexports,
Expected<std::vector<ExecutorSymbolDef>> ReentryPoints) {

if (!ReentryPoints) {
MR->getExecutionSession().reportError(ReentryPoints.takeError());
MR->failMaterialization();
return;
}

assert(Reexports.size() == ReentryPoints->size() &&
"Number of reentry points doesn't match number of reexports");

// Bind entry points to names.
SymbolMap Redirs;
{
std::lock_guard<std::mutex> Lock(M);
size_t I = 0;
for (auto &[Name, AI] : Reexports) {
const auto &ReentryPoint = (*ReentryPoints)[I++];
Redirs[Name] = ReentryPoint;
CallThroughs[ReentryPoint.getAddress()] = {Name, AI.Aliasee,
&MR->getTargetJITDylib()};
}
}

RSMgr.emitRedirectableSymbols(std::move(MR), std::move(Redirs));
}

void LazyReexportsManager::resolve(ResolveSendResultFn SendResult,
ExecutorAddr ReentryStubAddr) {

CallThroughInfo LandingInfo;

{
std::lock_guard<std::mutex> Lock(M);

auto I = CallThroughs.find(ReentryStubAddr);
if (I == CallThroughs.end())
return SendResult(make_error<StringError>(
"Reentry address " + formatv("{0:x}", ReentryStubAddr) +
" not registered",
inconvertibleErrorCode()));
LandingInfo = I->second;
}

SymbolInstance LandingSym(LandingInfo.JD, std::move(LandingInfo.BodyName));
LandingSym.lookupAsync([this, JD = std::move(LandingInfo.JD),
ReentryName = std::move(LandingInfo.Name),
SendResult = std::move(SendResult)](
Expected<ExecutorSymbolDef> Result) mutable {
if (Result) {
// FIXME: Make RedirectionManager operations async, then use the async
// APIs here.
if (auto Err = RSMgr.redirect(*JD, ReentryName, *Result))
SendResult(std::move(Err));
else
SendResult(std::move(Result));
} else
SendResult(std::move(Result));
});
}

} // End namespace orc.
} // End namespace llvm.
12 changes: 12 additions & 0 deletions llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -409,6 +409,7 @@ SymbolAliasMap MachOPlatform::standardPlatformAliases(ExecutionSession &ES) {
SymbolAliasMap Aliases;
addAliases(ES, Aliases, requiredCXXAliases());
addAliases(ES, Aliases, standardRuntimeUtilityAliases());
addAliases(ES, Aliases, standardLazyCompilationAliases());
return Aliases;
}

Expand Down Expand Up @@ -436,6 +437,17 @@ MachOPlatform::standardRuntimeUtilityAliases() {
StandardRuntimeUtilityAliases);
}

ArrayRef<std::pair<const char *, const char *>>
MachOPlatform::standardLazyCompilationAliases() {
static const std::pair<const char *, const char *>
StandardLazyCompilationAliases[] = {
{"__orc_rt_reentry", "__orc_rt_sysv_reentry"},
{"__orc_rt_resolve_tag", "___orc_rt_resolve_tag"}};

return ArrayRef<std::pair<const char *, const char *>>(
StandardLazyCompilationAliases);
}

bool MachOPlatform::supportedTarget(const Triple &TT) {
switch (TT.getArch()) {
case Triple::aarch64:
Expand Down
56 changes: 21 additions & 35 deletions llvm/tools/llvm-jitlink/llvm-jitlink.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
#include "llvm/ExecutionEngine/Orc/IndirectionUtils.h"
#include "llvm/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.h"
#include "llvm/ExecutionEngine/Orc/JITLinkReentryTrampolines.h"
#include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h"
#include "llvm/ExecutionEngine/Orc/LoadLinkableFile.h"
#include "llvm/ExecutionEngine/Orc/MachO.h"
Expand Down Expand Up @@ -949,41 +950,18 @@ class PhonyExternalsGenerator : public DefinitionGenerator {
}
};

static void handleLazyCallFailure() {
dbgs() << "ERROR: failure to materialize lazy call-through target.\n";
exit(1);
}

static void *reenter(void *Ctx, void *TrampolineAddr) {
std::promise<void *> LandingAddressP;
auto LandingAddressF = LandingAddressP.get_future();

auto *EPCIU = static_cast<EPCIndirectionUtils *>(Ctx);
EPCIU->getLazyCallThroughManager().resolveTrampolineLandingAddress(
ExecutorAddr::fromPtr(TrampolineAddr), [&](ExecutorAddr LandingAddress) {
LandingAddressP.set_value(LandingAddress.toPtr<void *>());
});
return LandingAddressF.get();
}

Expected<std::unique_ptr<Session::LazyLinkingSupport>>
createLazyLinkingSupport(ObjectLinkingLayer &OLL) {
auto EPCIU = EPCIndirectionUtils::Create(OLL.getExecutionSession());
if (!EPCIU)
return EPCIU.takeError();
if (auto Err = (*EPCIU)
->writeResolverBlock(ExecutorAddr::fromPtr(&reenter),
ExecutorAddr::fromPtr(EPCIU->get()))
.takeError())
return Err;
(*EPCIU)->createLazyCallThroughManager(
OLL.getExecutionSession(), ExecutorAddr::fromPtr(handleLazyCallFailure));
createLazyLinkingSupport(ObjectLinkingLayer &OLL, JITDylib &PlatformJD) {
auto RSMgr = JITLinkRedirectableSymbolManager::Create(OLL);
if (!RSMgr)
return RSMgr.takeError();

return std::make_unique<Session::LazyLinkingSupport>(std::move(*EPCIU),
std::move(*RSMgr), OLL);
auto LRMgr = createJITLinkLazyReexportsManager(OLL, **RSMgr, PlatformJD);
if (!LRMgr)
return LRMgr.takeError();

return std::make_unique<Session::LazyLinkingSupport>(std::move(*RSMgr),
std::move(*LRMgr), OLL);
}

Expected<std::unique_ptr<Session>> Session::Create(Triple TT,
Expand Down Expand Up @@ -1020,7 +998,8 @@ Expected<std::unique_ptr<Session>> Session::Create(Triple TT,
S->Features = std::move(Features);

if (lazyLinkingRequested()) {
if (auto LazyLinking = createLazyLinkingSupport(S->ObjLayer))
if (auto LazyLinking =
createLazyLinkingSupport(S->ObjLayer, *S->PlatformJD))
S->LazyLinking = std::move(*LazyLinking);
else
return LazyLinking.takeError();
Expand Down Expand Up @@ -1642,10 +1621,17 @@ static Error sanitizeArguments(const Triple &TT, const char *ArgV0) {
OutOfProcessExecutor = OOPExecutorPath.str().str();
}

if (lazyLinkingRequested() && !TestHarnesses.empty())
return make_error<StringError>(
"Lazy linking cannot be used with -harness mode",
inconvertibleErrorCode());
// If lazy linking is requested then check compatibility with other options.
if (lazyLinkingRequested()) {
if (OrcRuntime.empty())
return make_error<StringError>("Lazy linking requries the ORC runtime",
inconvertibleErrorCode());

if (!TestHarnesses.empty())
return make_error<StringError>(
"Lazy linking cannot be used with -harness mode",
inconvertibleErrorCode());
}

return Error::success();
}
Expand Down
20 changes: 7 additions & 13 deletions llvm/tools/llvm-jitlink/llvm-jitlink.h
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,9 @@

#include "llvm/ADT/StringSet.h"
#include "llvm/ExecutionEngine/Orc/Core.h"
#include "llvm/ExecutionEngine/Orc/EPCIndirectionUtils.h"
#include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
#include "llvm/ExecutionEngine/Orc/LazyObjectLinkingLayer.h"
#include "llvm/ExecutionEngine/Orc/LazyReexports.h"
#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
#include "llvm/ExecutionEngine/Orc/RedirectionManager.h"
#include "llvm/ExecutionEngine/Orc/SimpleRemoteEPC.h"
Expand All @@ -33,20 +33,14 @@ namespace llvm {
struct Session {

struct LazyLinkingSupport {
LazyLinkingSupport(std::unique_ptr<orc::EPCIndirectionUtils> EPCIU,
std::unique_ptr<orc::RedirectableSymbolManager> RSMgr,
LazyLinkingSupport(std::unique_ptr<orc::RedirectableSymbolManager> RSMgr,
std::unique_ptr<orc::LazyReexportsManager> LRMgr,
orc::ObjectLinkingLayer &ObjLinkingLayer)
: EPCIU(std::move(EPCIU)), RSMgr(std::move(RSMgr)),
LazyObjLinkingLayer(ObjLinkingLayer,
this->EPCIU->getLazyCallThroughManager(),
*this->RSMgr) {}
~LazyLinkingSupport() {
if (auto Err = EPCIU->cleanup())
LazyObjLinkingLayer.getExecutionSession().reportError(std::move(Err));
}

std::unique_ptr<orc::EPCIndirectionUtils> EPCIU;
: RSMgr(std::move(RSMgr)), LRMgr(std::move(LRMgr)),
LazyObjLinkingLayer(ObjLinkingLayer, *this->LRMgr) {}

std::unique_ptr<orc::RedirectableSymbolManager> RSMgr;
std::unique_ptr<orc::LazyReexportsManager> LRMgr;
orc::LazyObjectLinkingLayer LazyObjLinkingLayer;
};

Expand Down