Skip to content

Commit b70eb86

Browse files
sbc100sbc100
authored
[lld][WebAssemlby] Implement --thinlto-object-suffix-replace/--thinlto-prefix-replace (llvm#114625)
Fixes: llvm#79604
1 parent 7ec682b commit b70eb86

9 files changed

+252
-3
lines changed

Diff for: ‎lld/test/wasm/lto/thinlto-emit-index.ll

+108
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
;; Copied from ELF/lto/thinlto-index-only.ll
2+
;; First ensure that the ThinLTO handling in lld handles
3+
;; bitcode without summary sections gracefully and generates index file.
4+
; RUN: rm -rf %t && mkdir %t && cd %t
5+
; RUN: mkdir d
6+
; RUN: llvm-as %s -o 1.o
7+
; RUN: llvm-as %p/Inputs/thinlto.ll -o d/2.o
8+
; RUN: wasm-ld --thinlto-emit-index-files -shared 1.o d/2.o -o 3
9+
; RUN: ls d/2.o.thinlto.bc
10+
; RUN: ls 3
11+
; RUN: wasm-ld -shared 1.o d/2.o -o 3
12+
; RUN: llvm-nm 3 | FileCheck %s --check-prefix=NM
13+
14+
;; Basic ThinLTO tests.
15+
; RUN: opt -module-summary %s -o 1.o
16+
; RUN: opt -module-summary %p/Inputs/thinlto.ll -o d/2.o
17+
; RUN: opt -module-summary %p/Inputs/thinlto_empty.ll -o 3.o
18+
; RUN: cp 3.o 4.o
19+
20+
;; Ensure lld generates an index and also a binary if requested.
21+
; RUN: wasm-ld --thinlto-emit-index-files -shared 1.o --start-lib d/2.o 3.o --end-lib 4.o -o 4
22+
; RUN: ls 4
23+
; RUN: llvm-bcanalyzer -dump 1.o.thinlto.bc | FileCheck %s --check-prefix=BACKEND1
24+
; RUN: llvm-bcanalyzer -dump d/2.o.thinlto.bc | FileCheck %s --check-prefix=BACKEND2
25+
; RUN: llvm-dis < 3.o.thinlto.bc | FileCheck %s --check-prefix=BACKEND3
26+
; RUN: llvm-dis < 4.o.thinlto.bc | FileCheck %s --check-prefix=BACKEND4
27+
28+
; IMPORTS1: d/2.o
29+
30+
;; Ensure lld generates an index and not a binary if both emit-index and index-only are present.
31+
; RUN: wasm-ld --thinlto-emit-index-files --thinlto-index-only -shared 1.o d/2.o -o 5
32+
; RUN: not ls 5
33+
34+
;; Test that LLD generates an empty index even for lazy object file that is not added to link.
35+
;; Test that LLD also generates empty imports file with the --thinlto-emit-imports-files option.
36+
; RUN: rm -f 1.o.thinlto.bc 1.o.imports
37+
; RUN: wasm-ld --thinlto-emit-index-files -shared d/2.o --start-lib 1.o --end-lib \
38+
; RUN: --thinlto-emit-imports-files -o 7
39+
; RUN: ls 7
40+
; RUN: ls 1.o.thinlto.bc
41+
; RUN: ls 1.o.imports
42+
43+
;; Ensure LLD generates an empty index for each bitcode file even if all bitcode files are lazy.
44+
; RUN: rm -f 1.o.thinlto.bc
45+
; RUN: llvm-mc -filetype=obj -triple=wasm32-unknown-linux /dev/null -o dummy.o
46+
; RUN: wasm-ld --thinlto-emit-index-files -shared dummy.o --start-lib 1.o --end-lib -o 8
47+
; RUN: ls 8
48+
; RUN: ls 1.o.thinlto.bc
49+
50+
;; Test that LLD errors out when run with suffix replacement, or prefix replacement
51+
; RUN: not wasm-ld --thinlto-emit-index-files -shared d/2.o --start-lib 1.o --end-lib \
52+
; RUN: --thinlto-prefix-replace="abc;xyz" 2>&1 | FileCheck %s --check-prefix=ERR1
53+
; ERR1: --thinlto-prefix-replace is not supported with --thinlto-emit-index-files
54+
55+
; RUN: not wasm-ld --thinlto-emit-index-files -shared d/2.o --start-lib 1.o --end-lib \
56+
; RUN: --thinlto-object-suffix-replace="abc;xyz" 2>&1 | FileCheck %s --check-prefix=ERR2
57+
; ERR2: --thinlto-object-suffix-replace is not supported with --thinlto-emit-index-files
58+
59+
;; But not when passed with index only as well
60+
; RUN: wasm-ld --thinlto-emit-index-files -shared d/2.o --start-lib 1.o --end-lib \
61+
; RUN: --thinlto-prefix-replace="abc;xyz" --thinlto-index-only
62+
63+
; RUN: wasm-ld --thinlto-emit-index-files -shared d/2.o --start-lib 1.o --end-lib \
64+
; RUN: --thinlto-object-suffix-replace="abc;xyz" --thinlto-index-only
65+
66+
; NM: T f
67+
68+
;; The backend index for this module contains summaries from itself and
69+
;; Inputs/thinlto.ll, as it imports from the latter.
70+
; BACKEND1: <MODULE_STRTAB_BLOCK
71+
; BACKEND1-NEXT: <ENTRY {{.*}} record string = '1.o'
72+
; BACKEND1-NEXT: <ENTRY {{.*}} record string = 'd/2.o'
73+
; BACKEND1-NEXT: </MODULE_STRTAB_BLOCK
74+
; BACKEND1: <GLOBALVAL_SUMMARY_BLOCK
75+
; BACKEND1: <VERSION
76+
; BACKEND1: <FLAGS
77+
; BACKEND1: <VALUE_GUID {{.*}} op0={{1|2}} {{op1=3060885059 op2=1207956914|op1=3432075125 op2=3712786831}}
78+
; BACKEND1: <VALUE_GUID {{.*}} op0={{1|2}} {{op1=3060885059 op2=1207956914|op1=3432075125 op2=3712786831}}
79+
; BACKEND1: <COMBINED
80+
; BACKEND1: <COMBINED
81+
; BACKEND1: </GLOBALVAL_SUMMARY_BLOCK
82+
83+
;; The backend index for Input/thinlto.ll contains summaries from itself only,
84+
;; as it does not import anything.
85+
; BACKEND2: <MODULE_STRTAB_BLOCK
86+
; BACKEND2-NEXT: <ENTRY {{.*}} record string = 'd/2.o'
87+
; BACKEND2-NEXT: </MODULE_STRTAB_BLOCK
88+
; BACKEND2-NEXT: <GLOBALVAL_SUMMARY_BLOCK
89+
; BACKEND2-NEXT: <VERSION
90+
; BACKEND2-NEXT: <FLAGS
91+
; BACKEND2-NEXT: <VALUE_GUID {{.*}} op0=1 op1=3060885059 op2=1207956914
92+
; BACKEND2-NEXT: <COMBINED
93+
; BACKEND2-NEXT: </GLOBALVAL_SUMMARY_BLOCK
94+
95+
; BACKEND3: ^0 = flags:
96+
97+
; BACKEND4: ^0 = module: (path: "4.o", hash: (0, 0, 0, 0, 0))
98+
99+
target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
100+
target triple = "wasm32-unknown-unknown"
101+
102+
declare void @g(...)
103+
104+
define void @f() {
105+
entry:
106+
call void (...) @g()
107+
ret void
108+
}

Diff for: ‎lld/test/wasm/lto/thinlto-object-suffix-replace.ll

+47
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
;; Copied from ELF/lto/thinlto-object-suffix-replace.ll
2+
;; Test to make sure the thinlto-object-suffix-replace option is handled
3+
;; correctly.
4+
; RUN: rm -rf %t && mkdir %t && cd %t
5+
6+
;; Generate bitcode file with summary, as well as a minimized bitcode without
7+
; the debug metadata for the thin link.
8+
; RUN: opt --thinlto-bc %s -thin-link-bitcode-file=1.thinlink.bc -o 1.o
9+
10+
;; First perform the thin link on the normal bitcode file, and save the
11+
;; resulting index.
12+
; RUN: wasm-ld --thinlto-index-only -shared 1.o -o 3
13+
; RUN: cp 1.o.thinlto.bc 1.o.thinlto.bc.orig
14+
15+
;; Next perform the thin link on the minimized bitcode file, and compare dump
16+
;; of the resulting index to the above dump to ensure they are identical.
17+
; RUN: rm -f 1.o.thinlto.bc
18+
;; Make sure it isn't inadvertently using the regular bitcode file.
19+
; RUN: rm -f 1.o
20+
; RUN: wasm-ld --thinlto-index-only --thinlto-object-suffix-replace=".thinlink.bc;.o" \
21+
; RUN: -shared 1.thinlink.bc -o 3
22+
; RUN: cmp 1.o.thinlto.bc.orig 1.o.thinlto.bc
23+
24+
;; Ensure lld generates error if object suffix replace option does not have 'old;new' format
25+
; RUN: rm -f 1.o.thinlto.bc
26+
; RUN: not wasm-ld --thinlto-index-only --thinlto-object-suffix-replace="abc:def" -shared 1.thinlink.bc \
27+
; RUN: -o 3 2>&1 | FileCheck %s --check-prefix=ERR1
28+
; ERR1: --thinlto-object-suffix-replace= expects 'old;new' format, but got abc:def
29+
30+
;; If filename does not end with old suffix, no suffix change should occur,
31+
;; so ".thinlto.bc" will simply be appended to the input file name.
32+
; RUN: rm -f 1.thinlink.bc.thinlto.bc
33+
; RUN: wasm-ld --thinlto-index-only --thinlto-object-suffix-replace=".abc;.o" -shared 1.thinlink.bc -o /dev/null
34+
; RUN: ls 1.thinlink.bc.thinlto.bc
35+
36+
target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
37+
target triple = "wasm32-unknown-unknown"
38+
39+
define void @f() {
40+
entry:
41+
ret void
42+
}
43+
44+
!llvm.dbg.cu = !{}
45+
46+
!1 = !{i32 2, !"Debug Info Version", i32 3}
47+
!llvm.module.flags = !{!1}

Diff for: ‎lld/test/wasm/lto/thinlto-prefix-replace.ll

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
; Copied from ELF/lto/thinlto-prefix-replace.ll
2+
; Check that changing the output path via thinlto-prefix-replace works
3+
; RUN: mkdir -p %t/oldpath
4+
; RUN: opt -module-summary %s -o %t/oldpath/thinlto_prefix_replace.o
5+
6+
; Ensure that there is no existing file at the new path, so we properly
7+
; test the creation of the new file there.
8+
; RUN: rm -f %t/newpath/thinlto_prefix_replace.o.thinlto.bc
9+
; RUN: wasm-ld --thinlto-index-only --thinlto-prefix-replace="%t/oldpath/;%t/newpath/" -shared %t/oldpath/thinlto_prefix_replace.o -o %t/thinlto_prefix_replace
10+
; RUN: ls %t/newpath/thinlto_prefix_replace.o.thinlto.bc
11+
12+
; Ensure that lld generates error if prefix replace option does not have 'old;new' format.
13+
; RUN: rm -f %t/newpath/thinlto_prefix_replace.o.thinlto.bc
14+
; RUN: not wasm-ld --thinlto-index-only --thinlto-prefix-replace=abc:def -shared %t/oldpath/thinlto_prefix_replace.o -o /dev/null 2>&1 | FileCheck %s --check-prefix=ERR
15+
; ERR: --thinlto-prefix-replace= expects 'old;new' format, but got abc:def
16+
17+
target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
18+
target triple = "wasm32-unknown-unknown"
19+
20+
define void @f() {
21+
entry:
22+
ret void
23+
}

Diff for: ‎lld/wasm/Config.h

+4
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,10 @@ struct Configuration {
110110
llvm::StringRef thinLTOCacheDir;
111111
llvm::StringRef thinLTOJobs;
112112
llvm::StringRef thinLTOIndexOnlyArg;
113+
std::pair<llvm::StringRef, llvm::StringRef> thinLTOObjectSuffixReplace;
114+
llvm::StringRef thinLTOPrefixReplaceOld;
115+
llvm::StringRef thinLTOPrefixReplaceNew;
116+
llvm::StringRef thinLTOPrefixReplaceNativeObject;
113117
llvm::StringRef whyExtract;
114118

115119
llvm::StringSet<> allowUndefinedSymbols;

Diff for: ‎lld/wasm/Driver.cpp

+46-1
Original file line numberDiff line numberDiff line change
@@ -425,6 +425,33 @@ void LinkerDriver::createFiles(opt::InputArgList &args) {
425425
error("no input files");
426426
}
427427

428+
static StringRef getAliasSpelling(opt::Arg *arg) {
429+
if (const opt::Arg *alias = arg->getAlias())
430+
return alias->getSpelling();
431+
return arg->getSpelling();
432+
}
433+
434+
static std::pair<StringRef, StringRef> getOldNewOptions(opt::InputArgList &args,
435+
unsigned id) {
436+
auto *arg = args.getLastArg(id);
437+
if (!arg)
438+
return {"", ""};
439+
440+
StringRef s = arg->getValue();
441+
std::pair<StringRef, StringRef> ret = s.split(';');
442+
if (ret.second.empty())
443+
error(getAliasSpelling(arg) + " expects 'old;new' format, but got " + s);
444+
return ret;
445+
}
446+
447+
// Parse options of the form "old;new[;extra]".
448+
static std::tuple<StringRef, StringRef, StringRef>
449+
getOldNewOptionsExtra(opt::InputArgList &args, unsigned id) {
450+
auto [oldDir, second] = getOldNewOptions(args, id);
451+
auto [newDir, extraDir] = second.split(';');
452+
return {oldDir, newDir, extraDir};
453+
}
454+
428455
static StringRef getEntry(opt::InputArgList &args) {
429456
auto *arg = args.getLastArg(OPT_entry, OPT_no_entry);
430457
if (!arg) {
@@ -577,6 +604,24 @@ static void readConfigs(opt::InputArgList &args) {
577604
config->thinLTOIndexOnly = args.hasArg(OPT_thinlto_index_only) ||
578605
args.hasArg(OPT_thinlto_index_only_eq);
579606
config->thinLTOIndexOnlyArg = args.getLastArgValue(OPT_thinlto_index_only_eq);
607+
config->thinLTOObjectSuffixReplace =
608+
getOldNewOptions(args, OPT_thinlto_object_suffix_replace_eq);
609+
std::tie(config->thinLTOPrefixReplaceOld, config->thinLTOPrefixReplaceNew,
610+
config->thinLTOPrefixReplaceNativeObject) =
611+
getOldNewOptionsExtra(args, OPT_thinlto_prefix_replace_eq);
612+
if (config->thinLTOEmitIndexFiles && !config->thinLTOIndexOnly) {
613+
if (args.hasArg(OPT_thinlto_object_suffix_replace_eq))
614+
error("--thinlto-object-suffix-replace is not supported with "
615+
"--thinlto-emit-index-files");
616+
else if (args.hasArg(OPT_thinlto_prefix_replace_eq))
617+
error("--thinlto-prefix-replace is not supported with "
618+
"--thinlto-emit-index-files");
619+
}
620+
if (!config->thinLTOPrefixReplaceNativeObject.empty() &&
621+
config->thinLTOIndexOnlyArg.empty()) {
622+
error("--thinlto-prefix-replace=old_dir;new_dir;obj_dir must be used with "
623+
"--thinlto-index-only=");
624+
}
580625
config->unresolvedSymbols = getUnresolvedSymbolPolicy(args);
581626
config->whyExtract = args.getLastArgValue(OPT_why_extract);
582627
errorHandler().verbose = args.hasArg(OPT_verbose);
@@ -721,7 +766,7 @@ static void checkOptions(opt::InputArgList &args) {
721766
if (config->pie && config->shared)
722767
error("-shared and -pie may not be used together");
723768

724-
if (config->outputFile.empty())
769+
if (config->outputFile.empty() && !config->thinLTOIndexOnly)
725770
error("no output file specified");
726771

727772
if (config->importTable && config->exportTable)

Diff for: ‎lld/wasm/InputFiles.cpp

+9
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,13 @@ std::string toString(const wasm::InputFile *file) {
4646

4747
namespace wasm {
4848

49+
std::string replaceThinLTOSuffix(StringRef path) {
50+
auto [suffix, repl] = config->thinLTOObjectSuffixReplace;
51+
if (path.consume_back(suffix))
52+
return (path + repl).str();
53+
return std::string(path);
54+
}
55+
4956
void InputFile::checkArch(Triple::ArchType arch) const {
5057
bool is64 = arch == Triple::wasm64;
5158
if (is64 && !config->is64) {
@@ -837,6 +844,8 @@ BitcodeFile::BitcodeFile(MemoryBufferRef m, StringRef archiveName,
837844
this->archiveName = std::string(archiveName);
838845

839846
std::string path = mb.getBufferIdentifier().str();
847+
if (config->thinLTOIndexOnly)
848+
path = replaceThinLTOSuffix(mb.getBufferIdentifier());
840849

841850
// ThinLTO assumes that all MemoryBufferRefs given to it have a unique
842851
// name. If two archives define two members with the same name, this

Diff for: ‎lld/wasm/InputFiles.h

+2
Original file line numberDiff line numberDiff line change
@@ -195,6 +195,8 @@ InputFile *createObjectFile(MemoryBufferRef mb, StringRef archiveName = "",
195195
// Opens a given file.
196196
std::optional<MemoryBufferRef> readFile(StringRef path);
197197

198+
std::string replaceThinLTOSuffix(StringRef path);
199+
198200
} // namespace wasm
199201

200202
std::string toString(const wasm::InputFile *file);

Diff for: ‎lld/wasm/LTO.cpp

+11-2
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,11 @@ using namespace llvm;
4343
using namespace lld::wasm;
4444
using namespace lld;
4545

46+
static std::string getThinLTOOutputFile(StringRef modulePath) {
47+
return lto::getThinLTOOutputFile(modulePath, config->thinLTOPrefixReplaceOld,
48+
config->thinLTOPrefixReplaceNew);
49+
}
50+
4651
static lto::Config createConfig() {
4752
lto::Config c;
4853
c.Options = initTargetOptionsFromCodeGenFlags();
@@ -84,7 +89,10 @@ BitcodeCompiler::BitcodeCompiler() {
8489
auto onIndexWrite = [&](StringRef s) { thinIndices.erase(s); };
8590
if (config->thinLTOIndexOnly) {
8691
backend = lto::createWriteIndexesThinBackend(
87-
llvm::hardware_concurrency(config->thinLTOJobs), "", "", "",
92+
llvm::hardware_concurrency(config->thinLTOJobs),
93+
std::string(config->thinLTOPrefixReplaceOld),
94+
std::string(config->thinLTOPrefixReplaceNew),
95+
std::string(config->thinLTOPrefixReplaceNativeObject),
8896
config->thinLTOEmitImportsFiles, indexFile.get(), onIndexWrite);
8997
} else {
9098
backend = lto::createInProcessThinBackend(
@@ -158,7 +166,8 @@ static void thinLTOCreateEmptyIndexFiles() {
158166
continue;
159167
if (linkedBitCodeFiles.contains(f->getName()))
160168
continue;
161-
std::string path(f->obj->getName());
169+
std::string path =
170+
replaceThinLTOSuffix(getThinLTOOutputFile(f->obj->getName()));
162171
std::unique_ptr<raw_fd_ostream> os = openFile(path + ".thinlto.bc");
163172
if (!os)
164173
continue;

Diff for: ‎lld/wasm/Options.td

+2
Original file line numberDiff line numberDiff line change
@@ -309,6 +309,8 @@ def thinlto_index_only: FF<"thinlto-index-only">;
309309
def thinlto_index_only_eq: JJ<"thinlto-index-only=">;
310310
def thinlto_jobs: JJ<"thinlto-jobs=">,
311311
HelpText<"Number of ThinLTO jobs. Default to --threads=">;
312+
def thinlto_object_suffix_replace_eq: JJ<"thinlto-object-suffix-replace=">;
313+
def thinlto_prefix_replace_eq: JJ<"thinlto-prefix-replace=">;
312314
def lto_debug_pass_manager: FF<"lto-debug-pass-manager">,
313315
HelpText<"Debug new pass manager">;
314316

0 commit comments

Comments
 (0)