Skip to content

Commit

Permalink
[lld-macho] Add --start-lib --end-lib
Browse files Browse the repository at this point in the history 
In ld.lld, when an ObjFile/BitcodeFile is read in --start-lib state, the file is
given archive semantics. --end-lib closes the previous --start-lib. A build
system can use this feature as an alternative to archives. This patch ports
the feature to lld-macho.

--start-lib and --end-lib are positional, unlike usual ld64 options.
I think the slight drawback does not matter as (a) reusing option names
make build systems convenient (b) `--start-lib a.o b.o --end-lib` conveys more
information than an alternative design: `-objlib a.o -objlib b.o` because
--start-lib makes it clear which objects are in the same conceptual archive.
This provides flexibility (c) `-objlib`/`-filelist` interaction may be weird.

Close #52931

Reviewed By: #lld-macho, Jez Ng, oontvoo

Differential Revision: https://reviews.llvm.org/D116913
  • Loading branch information
@MaskRay
MaskRay committed Jan 19, 2022
1 parent 4f61749 commit 0aae2bf
Show file tree
Hide file tree
Showing 12 changed files with 338 additions and 31 deletions.
51 changes: 38 additions & 13 deletions lld/MachO/Driver.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -249,7 +249,8 @@ static llvm::CachePruningPolicy getLTOCachePolicy(InputArgList &args) {
static DenseMap<StringRef, ArchiveFile *> loadedArchives;

static InputFile *addFile(StringRef path, ForceLoad forceLoadArchive,
bool isExplicit = true, bool isBundleLoader = false) {
bool isLazy = false, bool isExplicit = true,
bool isBundleLoader = false) {
Optional<MemoryBufferRef> buffer = readFile(path);
if (!buffer)
return nullptr;
Expand Down Expand Up @@ -319,7 +320,7 @@ static InputFile *addFile(StringRef path, ForceLoad forceLoadArchive,
break;
}
case file_magic::macho_object:
newFile = make<ObjFile>(mbref, getModTime(path), "");
newFile = make<ObjFile>(mbref, getModTime(path), "", isLazy);
break;
case file_magic::macho_dynamically_linked_shared_lib:
case file_magic::macho_dynamically_linked_shared_lib_stub:
Expand All @@ -331,7 +332,7 @@ static InputFile *addFile(StringRef path, ForceLoad forceLoadArchive,
}
break;
case file_magic::bitcode:
newFile = make<BitcodeFile>(mbref, "", 0);
newFile = make<BitcodeFile>(mbref, "", 0, isLazy);
break;
case file_magic::macho_executable:
case file_magic::macho_bundle:
Expand All @@ -346,9 +347,20 @@ static InputFile *addFile(StringRef path, ForceLoad forceLoadArchive,
error(path + ": unhandled file type");
}
if (newFile && !isa<DylibFile>(newFile)) {
if ((isa<ObjFile>(newFile) || isa<BitcodeFile>(newFile)) && newFile->lazy &&
config->forceLoadObjC) {
for (Symbol *sym : newFile->symbols)
if (sym && sym->getName().startswith(objc::klass)) {
extract(*newFile, "-ObjC");
break;
}
if (newFile->lazy && hasObjCSection(mbref))
extract(*newFile, "-ObjC");
}

// printArchiveMemberLoad() prints both .a and .o names, so no need to
// print the .a name here.
if (config->printEachFile && magic != file_magic::archive)
// print the .a name here. Similarly skip lazy files.
if (config->printEachFile && magic != file_magic::archive && !isLazy)
message(toString(newFile));
inputFiles.insert(newFile);
}
Expand All @@ -360,7 +372,7 @@ static void addLibrary(StringRef name, bool isNeeded, bool isWeak,
ForceLoad forceLoadArchive) {
if (Optional<StringRef> path = findLibrary(name)) {
if (auto *dylibFile = dyn_cast_or_null<DylibFile>(
addFile(*path, forceLoadArchive, isExplicit))) {
addFile(*path, forceLoadArchive, /*isLazy=*/false, isExplicit))) {
if (isNeeded)
dylibFile->forceNeeded = true;
if (isWeak)
Expand All @@ -380,7 +392,7 @@ static void addFramework(StringRef name, bool isNeeded, bool isWeak,
ForceLoad forceLoadArchive) {
if (Optional<StringRef> path = findFramework(name)) {
if (auto *dylibFile = dyn_cast_or_null<DylibFile>(
addFile(*path, forceLoadArchive, isExplicit))) {
addFile(*path, forceLoadArchive, /*isLazy=*/false, isExplicit))) {
if (isNeeded)
dylibFile->forceNeeded = true;
if (isWeak)
Expand Down Expand Up @@ -425,13 +437,13 @@ void macho::parseLCLinkerOption(InputFile *f, unsigned argc, StringRef data) {
}
}

static void addFileList(StringRef path) {
static void addFileList(StringRef path, bool isLazy) {
Optional<MemoryBufferRef> buffer = readFile(path);
if (!buffer)
return;
MemoryBufferRef mbref = *buffer;
for (StringRef path : args::getLines(mbref))
addFile(rerootPath(path), ForceLoad::Default);
addFile(rerootPath(path), ForceLoad::Default, isLazy);
}

// An order file has one entry per line, in the following format:
Expand Down Expand Up @@ -545,7 +557,8 @@ static void compileBitcodeFiles() {
auto *lto = make<BitcodeCompiler>();
for (InputFile *file : inputFiles)
if (auto *bitcodeFile = dyn_cast<BitcodeFile>(file))
lto->add(*bitcodeFile);
if (!file->lazy)
lto->add(*bitcodeFile);

for (ObjFile *file : lto->compile())
inputFiles.insert(file);
Expand Down Expand Up @@ -962,14 +975,15 @@ static void createFiles(const InputArgList &args) {
TimeTraceScope timeScope("Load input files");
// This loop should be reserved for options whose exact ordering matters.
// Other options should be handled via filtered() and/or getLastArg().
bool isLazy = false;
for (const Arg *arg : args) {
const Option &opt = arg->getOption();
warnIfDeprecatedOption(opt);
warnIfUnimplementedOption(opt);

switch (opt.getID()) {
case OPT_INPUT:
addFile(rerootPath(arg->getValue()), ForceLoad::Default);
addFile(rerootPath(arg->getValue()), ForceLoad::Default, isLazy);
break;
case OPT_needed_library:
if (auto *dylibFile = dyn_cast_or_null<DylibFile>(
Expand All @@ -989,7 +1003,7 @@ static void createFiles(const InputArgList &args) {
dylibFile->forceWeakImport = true;
break;
case OPT_filelist:
addFileList(arg->getValue());
addFileList(arg->getValue(), isLazy);
break;
case OPT_force_load:
addFile(rerootPath(arg->getValue()), ForceLoad::Yes);
Expand All @@ -1011,6 +1025,16 @@ static void createFiles(const InputArgList &args) {
opt.getID() == OPT_reexport_framework, /*isExplicit=*/true,
ForceLoad::Default);
break;
case OPT_start_lib:
if (isLazy)
error("nested --start-lib");
isLazy = true;
break;
case OPT_end_lib:
if (!isLazy)
error("stray --end-lib");
isLazy = false;
break;
default:
break;
}
Expand Down Expand Up @@ -1247,7 +1271,8 @@ bool macho::link(ArrayRef<const char *> argsArr, bool canExitEarly,
if (const Arg *arg = args.getLastArg(OPT_bundle_loader)) {
if (config->outputType != MH_BUNDLE)
error("-bundle_loader can only be used with MachO bundle output");
addFile(arg->getValue(), ForceLoad::Default, /*isExplicit=*/false,
addFile(arg->getValue(), ForceLoad::Default, /*isLazy=*/false,
/*isExplicit=*/false,
/*isBundleLoader=*/true);
}
if (const Arg *arg = args.getLastArg(OPT_umbrella)) {
Expand Down
85 changes: 77 additions & 8 deletions lld/MachO/InputFiles.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -836,13 +836,21 @@ OpaqueFile::OpaqueFile(MemoryBufferRef mb, StringRef segName,
sections.back().subsections.push_back({0, isec});
}

ObjFile::ObjFile(MemoryBufferRef mb, uint32_t modTime, StringRef archiveName)
: InputFile(ObjKind, mb), modTime(modTime) {
ObjFile::ObjFile(MemoryBufferRef mb, uint32_t modTime, StringRef archiveName,
bool lazy)
: InputFile(ObjKind, mb, lazy), modTime(modTime) {
this->archiveName = std::string(archiveName);
if (target->wordSize == 8)
parse<LP64>();
else
parse<ILP32>();
if (lazy) {
if (target->wordSize == 8)
parseLazy<LP64>();
else
parseLazy<ILP32>();
} else {
if (target->wordSize == 8)
parse<LP64>();
else
parse<ILP32>();
}
}

template <class LP> void ObjFile::parse() {
Expand Down Expand Up @@ -904,6 +912,32 @@ template <class LP> void ObjFile::parse() {
registerCompactUnwind();
}

template <class LP> void ObjFile::parseLazy() {
using Header = typename LP::mach_header;
using NList = typename LP::nlist;

auto *buf = reinterpret_cast<const uint8_t *>(mb.getBufferStart());
auto *hdr = reinterpret_cast<const Header *>(mb.getBufferStart());
const load_command *cmd = findCommand(hdr, LC_SYMTAB);
if (!cmd)
return;
auto *c = reinterpret_cast<const symtab_command *>(cmd);
ArrayRef<NList> nList(reinterpret_cast<const NList *>(buf + c->symoff),
c->nsyms);
const char *strtab = reinterpret_cast<const char *>(buf) + c->stroff;
symbols.resize(nList.size());
for (auto it : llvm::enumerate(nList)) {
const NList &sym = it.value();
if ((sym.n_type & N_EXT) && !isUndef(sym)) {
// TODO: Bound checking
StringRef name = strtab + sym.n_strx;
symbols[it.index()] = symtab->addLazyObject(name, *this);
if (!lazy)
break;
}
}
}

void ObjFile::parseDebugInfo() {
std::unique_ptr<DwarfObject> dObj = DwarfObject::create(this);
if (!dObj)
Expand Down Expand Up @@ -1548,8 +1582,8 @@ static macho::Symbol *createBitcodeSymbol(const lto::InputFile::Symbol &objSym,
}

BitcodeFile::BitcodeFile(MemoryBufferRef mb, StringRef archiveName,
uint64_t offsetInArchive)
: InputFile(BitcodeKind, mb) {
uint64_t offsetInArchive, bool lazy)
: InputFile(BitcodeKind, mb, lazy) {
this->archiveName = std::string(archiveName);
std::string path = mb.getBufferIdentifier().str();
// ThinLTO assumes that all MemoryBufferRefs given to it have a unique
Expand All @@ -1565,12 +1599,47 @@ BitcodeFile::BitcodeFile(MemoryBufferRef mb, StringRef archiveName,
utostr(offsetInArchive)));

obj = check(lto::InputFile::create(mbref));
if (lazy)
parseLazy();
else
parse();
}

void BitcodeFile::parse() {
// Convert LTO Symbols to LLD Symbols in order to perform resolution. The
// "winning" symbol will then be marked as Prevailing at LTO compilation
// time.
symbols.clear();
for (const lto::InputFile::Symbol &objSym : obj->symbols())
symbols.push_back(createBitcodeSymbol(objSym, *this));
}

void BitcodeFile::parseLazy() {
symbols.resize(obj->symbols().size());
for (auto it : llvm::enumerate(obj->symbols())) {
const lto::InputFile::Symbol &objSym = it.value();
if (!objSym.isUndefined()) {
symbols[it.index()] =
symtab->addLazyObject(saver.save(objSym.getName()), *this);
if (!lazy)
break;
}
}
}

void macho::extract(InputFile &file, StringRef reason) {
assert(file.lazy);
file.lazy = false;
printArchiveMemberLoad(reason, &file);
if (auto *bitcode = dyn_cast<BitcodeFile>(&file)) {
bitcode->parse();
} else {
auto &f = cast<ObjFile>(file);
if (target->wordSize == 8)
f.parse<LP64>();
else
f.parse<ILP32>();
}
}

template void ObjFile::parse<LP64>();
27 changes: 20 additions & 7 deletions lld/MachO/InputFiles.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,16 +94,21 @@ class InputFile {

std::vector<Symbol *> symbols;
std::vector<Section> sections;
// Provides an easy way to sort InputFiles deterministically.
const int id;

// If not empty, this stores the name of the archive containing this file.
// We use this string for creating error messages.
std::string archiveName;

// Provides an easy way to sort InputFiles deterministically.
const int id;

// True if this is a lazy ObjFile or BitcodeFile.
bool lazy = false;

protected:
InputFile(Kind kind, MemoryBufferRef mb)
: mb(mb), id(idCount++), fileKind(kind), name(mb.getBufferIdentifier()) {}
InputFile(Kind kind, MemoryBufferRef mb, bool lazy = false)
: mb(mb), id(idCount++), lazy(lazy), fileKind(kind),
name(mb.getBufferIdentifier()) {}

InputFile(Kind, const llvm::MachO::InterfaceFile &);

Expand All @@ -117,8 +122,10 @@ class InputFile {
// .o file
class ObjFile final : public InputFile {
public:
ObjFile(MemoryBufferRef mb, uint32_t modTime, StringRef archiveName);
ObjFile(MemoryBufferRef mb, uint32_t modTime, StringRef archiveName,
bool lazy = false);
ArrayRef<llvm::MachO::data_in_code_entry> getDataInCode() const;
template <class LP> void parse();

static bool classof(const InputFile *f) { return f->kind() == ObjKind; }

Expand All @@ -130,7 +137,7 @@ class ObjFile final : public InputFile {
private:
Section *compactUnwindSection = nullptr;

template <class LP> void parse();
template <class LP> void parseLazy();
template <class SectionHeader> void parseSections(ArrayRef<SectionHeader>);
template <class LP>
void parseSymbols(ArrayRef<typename LP::section> sectionHeaders,
Expand Down Expand Up @@ -229,17 +236,23 @@ class ArchiveFile final : public InputFile {
class BitcodeFile final : public InputFile {
public:
explicit BitcodeFile(MemoryBufferRef mb, StringRef archiveName,
uint64_t offsetInArchive);
uint64_t offsetInArchive, bool lazy = false);
static bool classof(const InputFile *f) { return f->kind() == BitcodeKind; }
void parse();

std::unique_ptr<llvm::lto::InputFile> obj;

private:
void parseLazy();
};

extern llvm::SetVector<InputFile *> inputFiles;
extern llvm::DenseMap<llvm::CachedHashStringRef, MemoryBufferRef> cachedReads;

llvm::Optional<MemoryBufferRef> readFile(StringRef path);

void extract(InputFile &file, StringRef reason);

namespace detail {

template <class CommandType, class... Types>
Expand Down
4 changes: 4 additions & 0 deletions lld/MachO/Options.td
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,10 @@ def thinlto_cache_policy: Joined<["--"], "thinlto-cache-policy=">,
Group<grp_lld>;
def O : JoinedOrSeparate<["-"], "O">,
HelpText<"Optimize output file size">;
def start_lib: Flag<["--"], "start-lib">,
HelpText<"Start a grouping of objects that should be treated as if they were together in an archive">;
def end_lib: Flag<["--"], "end-lib">,
HelpText<"End a grouping of objects that should be treated as if they were together in an archive">;
def no_warn_dylib_install_name: Joined<["--"], "no-warn-dylib-install-name">,
HelpText<"Do not warn on -install-name if -dylib is not passed (default)">,
Group<grp_lld>;
Expand Down
22 changes: 22 additions & 0 deletions lld/MachO/SymbolTable.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,8 @@ Symbol *SymbolTable::addUndefined(StringRef name, InputFile *file,
replaceSymbol<Undefined>(s, name, file, refState);
else if (auto *lazy = dyn_cast<LazyArchive>(s))
lazy->fetchArchiveMember();
else if (isa<LazyObject>(s))
extract(*s->getFile(), s->getName());
else if (auto *dynsym = dyn_cast<DylibSymbol>(s))
dynsym->reference(refState);
else if (auto *undefined = dyn_cast<Undefined>(s))
Expand Down Expand Up @@ -199,6 +201,26 @@ Symbol *SymbolTable::addLazyArchive(StringRef name, ArchiveFile *file,
return s;
}

Symbol *SymbolTable::addLazyObject(StringRef name, InputFile &file) {
Symbol *s;
bool wasInserted;
std::tie(s, wasInserted) = insert(name, &file);

if (wasInserted) {
replaceSymbol<LazyObject>(s, file, name);
} else if (isa<Undefined>(s)) {
extract(file, name);
} else if (auto *dysym = dyn_cast<DylibSymbol>(s)) {
if (dysym->isWeakDef()) {
if (dysym->getRefState() != RefState::Unreferenced)
extract(file, name);
else
replaceSymbol<LazyObject>(s, file, name);
}
}
return s;
}

Defined *SymbolTable::addSynthetic(StringRef name, InputSection *isec,
uint64_t value, bool isPrivateExtern,
bool includeInSymtab,
Expand Down
1 change: 1 addition & 0 deletions lld/MachO/SymbolTable.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ class SymbolTable {

Symbol *addLazyArchive(StringRef name, ArchiveFile *file,
const llvm::object::Archive::Symbol &sym);
Symbol *addLazyObject(StringRef name, InputFile &file);

Defined *addSynthetic(StringRef name, InputSection *, uint64_t value,
bool isPrivateExtern, bool includeInSymtab,
Expand Down
Loading

0 comments on commit 0aae2bf

Please sign in to comment.