diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp index ac96b836ed579..b6e5283355078 100644 --- a/bolt/lib/Core/BinaryContext.cpp +++ b/bolt/lib/Core/BinaryContext.cpp @@ -2020,6 +2020,10 @@ BinaryContext::getBaseAddressForMapping(uint64_t MMapAddress, // Only consider executable segments. if (!SegInfo.IsExecutable) continue; + // For Linux kernel perf files, SegInfo.FileOffset and FileOffset are + // irrelvent. + if (IsLinuxKernel) + return MMapAddress - SegInfo.Address; // FileOffset is got from perf event, // and it is equal to alignDown(SegInfo.FileOffset, pagesize). // If the pagesize is not equal to SegInfo.Alignment. diff --git a/bolt/lib/Profile/DataAggregator.cpp b/bolt/lib/Profile/DataAggregator.cpp index 2b02086e3e0c9..3ee80db6f4e08 100644 --- a/bolt/lib/Profile/DataAggregator.cpp +++ b/bolt/lib/Profile/DataAggregator.cpp @@ -544,26 +544,18 @@ Error DataAggregator::preprocessProfile(BinaryContext &BC) { }; if (BC.IsLinuxKernel) { - // Current MMap parsing logic does not work with linux kernel. - // MMap entries for linux kernel uses PERF_RECORD_MMAP - // format instead of typical PERF_RECORD_MMAP2 format. - // Since linux kernel address mapping is absolute (same as - // in the ELF file), we avoid parsing MMap in linux kernel mode. - // While generating optimized linux kernel binary, we may need - // to parse MMap entries. - // In linux kernel mode, we analyze and optimize // all linux kernel binary instructions, irrespective // of whether they are due to system calls or due to // interrupts. Therefore, we cannot ignore interrupt // in Linux kernel mode. opts::IgnoreInterruptLBR = false; - } else { - prepareToParse("mmap events", MMapEventsPPI, ErrorCallback); - if (parseMMapEvents()) - errs() << "PERF2BOLT: failed to parse mmap events\n"; } + prepareToParse("mmap events", MMapEventsPPI, ErrorCallback); + if (parseMMapEvents()) + errs() << "PERF2BOLT: failed to parse mmap events\n"; + prepareToParse("task events", TaskEventsPPI, ErrorCallback); if (parseTaskEvents()) errs() << "PERF2BOLT: failed to parse task events\n"; @@ -1158,6 +1150,11 @@ ErrorOr DataAggregator::parseBranchSample() { return make_error_code(errc::no_such_process); } + if (BC->IsLinuxKernel) { + // "-1" is the pid for the Linux kernel + MMapInfoIter = BinaryMMapInfo.find(-1); + } + while (checkAndConsumeFS()) { } @@ -1993,7 +1990,8 @@ DataAggregator::parseMMapEvent() { } StringRef Line = ParsingBuf.substr(0, LineEnd); - size_t Pos = Line.find("PERF_RECORD_MMAP2"); + // This would match both PERF_RECORD_MMAP and PERF_RECORD_MMAP2 + size_t Pos = Line.find("PERF_RECORD_MMAP"); if (Pos == StringRef::npos) { consumeRestOfLine(); return std::make_pair(StringRef(), ParsedInfo); @@ -2001,6 +1999,9 @@ DataAggregator::parseMMapEvent() { // Line: // { .* .: }PERF_RECORD_MMAP2 /: .* + // Or: + // { .* .: }PERF_RECORD_MMAP <-1 | pid>/: .* + // const StringRef TimeStr = Line.substr(0, Pos).rsplit(':').first.rsplit(FieldSeparator).second; @@ -2011,9 +2012,14 @@ DataAggregator::parseMMapEvent() { // Line: // PERF_RECORD_MMAP2 /: [() .*]: .* + // Or: + // PERF_RECORD_MMAP <-1 | pid>/: [() .*]: .* + // StringRef FileName = Line.rsplit(FieldSeparator).second; - if (FileName.starts_with("//") || FileName.starts_with("[")) { + if (FileName == "[kernel.kallsyms]_text") + FileName = "[kernel.kallsyms]"; + else if (FileName.starts_with("//") || FileName.starts_with("[")) { consumeRestOfLine(); return std::make_pair(StringRef(), ParsedInfo); } @@ -2040,8 +2046,11 @@ DataAggregator::parseMMapEvent() { return make_error_code(llvm::errc::io_error); } - const StringRef OffsetStr = - Line.split('@').second.ltrim().split(FieldSeparator).first; + const StringRef OffsetStr = Line.split('@') + .second.ltrim() + .split(FieldSeparator) + .first.split(']') + .first; if (OffsetStr.getAsInteger(0, ParsedInfo.Offset)) { reportError("expected mmaped page-aligned offset"); Diag << "Found: " << OffsetStr << "in '" << Line << "'\n"; @@ -2065,7 +2074,8 @@ std::error_code DataAggregator::parseMMapEvents() { return EC; std::pair FileMMapInfo = FileMMapInfoRes.get(); - if (FileMMapInfo.second.PID == -1) + if (FileMMapInfo.first != "[kernel.kallsyms]" && + FileMMapInfo.second.PID == -1) continue; if (FileMMapInfo.first == "(deleted)") continue; @@ -2087,6 +2097,8 @@ std::error_code DataAggregator::parseMMapEvents() { << "\" for profile matching\n"; NameToUse = BuildIDBinaryName; } + if (BC->IsLinuxKernel) + NameToUse = "[kernel.kallsyms]"; auto Range = GlobalMMapInfo.equal_range(NameToUse); for (MMapInfo &MMapInfo : llvm::make_second_range(make_range(Range))) { @@ -2135,7 +2147,7 @@ std::error_code DataAggregator::parseMMapEvents() { // Update mapping size. const uint64_t EndAddress = MMapInfo.MMapAddress + MMapInfo.Size; const uint64_t Size = EndAddress - BinaryMMapInfo[MMapInfo.PID].BaseAddress; - if (Size > BinaryMMapInfo[MMapInfo.PID].Size) + if (!BC->IsLinuxKernel && Size > BinaryMMapInfo[MMapInfo.PID].Size) BinaryMMapInfo[MMapInfo.PID].Size = Size; } diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp index 4329235d47049..5b6975465a021 100644 --- a/bolt/lib/Rewrite/RewriteInstance.cpp +++ b/bolt/lib/Rewrite/RewriteInstance.cpp @@ -534,8 +534,11 @@ Error RewriteInstance::discoverStorage() { Phdr.p_vaddr, Phdr.p_memsz, Phdr.p_offset, Phdr.p_filesz, Phdr.p_align, ((Phdr.p_flags & ELF::PF_X) != 0)}; if (BC->TheTriple->getArch() == llvm::Triple::x86_64 && - Phdr.p_vaddr >= BinaryContext::KernelStartX86_64) + Phdr.p_vaddr >= BinaryContext::KernelStartX86_64) { BC->IsLinuxKernel = true; + BC->HasFixedLoadAddress = false; + } + break; case ELF::PT_INTERP: BC->HasInterpHeader = true; @@ -1039,8 +1042,13 @@ void RewriteInstance::discoverFileObjects() { } if (!Section->isText()) { - assert(SymbolType != SymbolRef::ST_Function && - "unexpected function inside non-code section"); + // In kernel, a function can live in a non-text section. For Example, + // lkdtm_rodata_do_nothing() in ./drivers/misc/lkdtm/rodata.c is in + // the rodata section. + if (!BC->IsLinuxKernel) { + assert(SymbolType != SymbolRef::ST_Function && + "unexpected function inside non-code section"); + } LLVM_DEBUG(dbgs() << "BOLT-DEBUG: rejecting as symbol is not in code\n"); registerName(SymbolSize); continue;