From ce7ab7652cf29469a8addea8ebe67f408b4b03af Mon Sep 17 00:00:00 2001 From: "Plyakhin, Yury" Date: Tue, 25 Nov 2025 00:40:45 +0100 Subject: [PATCH 1/8] [Offloading] Extend OffloadBinary format to support multiple metadata entries --- llvm/include/llvm/Object/OffloadBinary.h | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/llvm/include/llvm/Object/OffloadBinary.h b/llvm/include/llvm/Object/OffloadBinary.h index f3847c1624977..cf25c02bc43fd 100644 --- a/llvm/include/llvm/Object/OffloadBinary.h +++ b/llvm/include/llvm/Object/OffloadBinary.h @@ -67,7 +67,7 @@ class OffloadBinary : public Binary { using string_iterator_range = iterator_range; /// The current version of the binary used for backwards compatibility. - static const uint32_t Version = 1; + static const uint32_t Version = 2; /// The offloading metadata that will be serialized to a memory buffer. struct OffloadingImage { @@ -109,9 +109,12 @@ class OffloadBinary : public Binary { struct Header { uint8_t Magic[4] = {0x10, 0xFF, 0x10, 0xAD}; // 0x10FF10AD magic bytes. uint32_t Version = OffloadBinary::Version; // Version identifier. - uint64_t Size; // Size in bytes of this entire binary. - uint64_t EntryOffset; // Offset of the metadata entry in bytes. - uint64_t EntrySize; // Size of the metadata entry in bytes. + uint64_t Size; // Size in bytes of this entire binary. + uint64_t EntriesCount; // Number of metadata entries in the binary. + uint64_t EntriesOffset; // Offset in bytes to the start of entries block. + uint64_t EntriesSize; // Size of the entries block in bytes. + uint64_t StringOffset; // Offset in bytes to the global string map + uint64_t NumStrings; // Number of entries in the global string map. }; struct Entry { @@ -127,6 +130,7 @@ class OffloadBinary : public Binary { struct StringEntry { uint64_t KeyOffset; uint64_t ValueOffset; + uint64_t ValueSize; // Size of the value in bytes. }; private: From f66ae8cca8d7678ba900c15eef1fa5fdb83a70dc Mon Sep 17 00:00:00 2001 From: "Plyakhin, Yury" Date: Tue, 25 Nov 2025 21:04:43 +0100 Subject: [PATCH 2/8] updated offloadbinary per discussion --- llvm/include/llvm/Object/OffloadBinary.h | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/llvm/include/llvm/Object/OffloadBinary.h b/llvm/include/llvm/Object/OffloadBinary.h index cf25c02bc43fd..bdcacce204966 100644 --- a/llvm/include/llvm/Object/OffloadBinary.h +++ b/llvm/include/llvm/Object/OffloadBinary.h @@ -52,6 +52,13 @@ enum ImageKind : uint16_t { IMG_LAST, }; +/// Flags associated with the Entry. +enum OffloadEntryFlags : uint32_t { + OIF_None = 0, + // Entry doesn't contain image. Used to keep metadata only entries. + OIF_NoImage = (1 << 0), +}; + /// A simple binary serialization of an offloading file. We use this format to /// embed the offloading image into the host executable so it can be extracted /// and used by the linker. @@ -110,17 +117,14 @@ class OffloadBinary : public Binary { uint8_t Magic[4] = {0x10, 0xFF, 0x10, 0xAD}; // 0x10FF10AD magic bytes. uint32_t Version = OffloadBinary::Version; // Version identifier. uint64_t Size; // Size in bytes of this entire binary. - uint64_t EntriesCount; // Number of metadata entries in the binary. uint64_t EntriesOffset; // Offset in bytes to the start of entries block. - uint64_t EntriesSize; // Size of the entries block in bytes. - uint64_t StringOffset; // Offset in bytes to the global string map - uint64_t NumStrings; // Number of entries in the global string map. + uint64_t EntriesCount; // Number of metadata entries in the binary. }; struct Entry { ImageKind TheImageKind; // The kind of the image stored. OffloadKind TheOffloadKind; // The producer of this image. - uint32_t Flags; // Additional flags associated with the image. + uint32_t Flags; // Additional flags associated with the entry. uint64_t StringOffset; // Offset in bytes to the string map. uint64_t NumStrings; // Number of entries in the string map. uint64_t ImageOffset; // Offset in bytes of the actual binary image. From 98ad7e9b818a68029b4f8d1b274d5165d8dd1bdf Mon Sep 17 00:00:00 2001 From: "Plyakhin, Yury" Date: Wed, 26 Nov 2025 03:16:48 +0100 Subject: [PATCH 3/8] Minimal changes to make build and lit tests pass with new OffloadBinary format. --- clang/test/Driver/linker-wrapper-image.c | 2 +- llvm/include/llvm/Object/OffloadBinary.h | 2 +- llvm/include/llvm/ObjectYAML/OffloadYAML.h | 4 ++-- .../Frontend/Offloading/OffloadWrapper.cpp | 2 +- llvm/lib/Object/OffloadBinary.cpp | 24 +++++++++++-------- llvm/lib/ObjectYAML/OffloadEmitter.cpp | 8 +++---- llvm/lib/ObjectYAML/OffloadYAML.cpp | 4 ++-- ...size.yaml => malformed-entries-count.yaml} | 2 +- .../ObjectYAML/Offload/malformed-offset.yaml | 2 +- .../ObjectYAML/Offload/malformed-version.yaml | 2 +- 10 files changed, 28 insertions(+), 24 deletions(-) rename llvm/test/ObjectYAML/Offload/{malformed-entry-size.yaml => malformed-entries-count.yaml} (94%) diff --git a/clang/test/Driver/linker-wrapper-image.c b/clang/test/Driver/linker-wrapper-image.c index b9327121edcf9..2c0df8c6be925 100644 --- a/clang/test/Driver/linker-wrapper-image.c +++ b/clang/test/Driver/linker-wrapper-image.c @@ -25,7 +25,7 @@ // OPENMP-REL: @.omp_offloading.device_image = internal unnamed_addr constant [[[SIZE:[0-9]+]] x i8] c"\10\FF\10\AD{{.*}}", section ".llvm.offloading.relocatable", align 8 // OPENMP: @.omp_offloading.device_image = internal unnamed_addr constant [[[SIZE:[0-9]+]] x i8] c"\10\FF\10\AD{{.*}}", section ".llvm.offloading", align 8 -// OPENMP-NEXT: @.omp_offloading.device_images = internal unnamed_addr constant [1 x %__tgt_device_image] [%__tgt_device_image { ptr getelementptr ([[[BEGIN:[0-9]+]] x i8], ptr @.omp_offloading.device_image, i64 0, i64 144), ptr getelementptr ([[[END:[0-9]+]] x i8], ptr @.omp_offloading.device_image, i64 0, i64 144), ptr @__start_llvm_offload_entries, ptr @__stop_llvm_offload_entries }] +// OPENMP-NEXT: @.omp_offloading.device_images = internal unnamed_addr constant [1 x %__tgt_device_image] [%__tgt_device_image { ptr getelementptr ([[[IMG_OFF:[0-9]+]] x i8], ptr @.omp_offloading.device_image, i64 0, i64 [[IMG_OFF]]), ptr getelementptr ([[[IMG_OFF]] x i8], ptr @.omp_offloading.device_image, i64 0, i64 [[IMG_OFF]]), ptr @__start_llvm_offload_entries, ptr @__stop_llvm_offload_entries }] // OPENMP-NEXT: @.omp_offloading.descriptor = internal constant %__tgt_bin_desc { i32 1, ptr @.omp_offloading.device_images, ptr @__start_llvm_offload_entries, ptr @__stop_llvm_offload_entries } // OPENMP-NEXT: @llvm.global_ctors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 101, ptr @.omp_offloading.descriptor_reg, ptr null }] diff --git a/llvm/include/llvm/Object/OffloadBinary.h b/llvm/include/llvm/Object/OffloadBinary.h index bdcacce204966..74dcd5cc7384b 100644 --- a/llvm/include/llvm/Object/OffloadBinary.h +++ b/llvm/include/llvm/Object/OffloadBinary.h @@ -134,7 +134,7 @@ class OffloadBinary : public Binary { struct StringEntry { uint64_t KeyOffset; uint64_t ValueOffset; - uint64_t ValueSize; // Size of the value in bytes. + uint64_t ValueSize; // Size of the value in bytes. }; private: diff --git a/llvm/include/llvm/ObjectYAML/OffloadYAML.h b/llvm/include/llvm/ObjectYAML/OffloadYAML.h index f897b52aa8b0e..63ff561f3fcbf 100644 --- a/llvm/include/llvm/ObjectYAML/OffloadYAML.h +++ b/llvm/include/llvm/ObjectYAML/OffloadYAML.h @@ -39,8 +39,8 @@ struct Binary { std::optional Version; std::optional Size; - std::optional EntryOffset; - std::optional EntrySize; + std::optional EntriesOffset; + std::optional EntriesCount; std::vector Members; }; diff --git a/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp b/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp index 288fa10fc04bb..f1602765dbaa7 100644 --- a/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp +++ b/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp @@ -161,7 +161,7 @@ GlobalVariable *createBinDesc(Module &M, ArrayRef> Bufs, Binary.bytes_begin()); const auto *Entry = reinterpret_cast( - Binary.bytes_begin() + Header->EntryOffset); + Binary.bytes_begin() + Header->EntriesOffset); BeginOffset = Entry->ImageOffset; EndOffset = Entry->ImageOffset + Entry->ImageSize; } diff --git a/llvm/lib/Object/OffloadBinary.cpp b/llvm/lib/Object/OffloadBinary.cpp index 3fff6b6a09e08..8e51b94b92951 100644 --- a/llvm/lib/Object/OffloadBinary.cpp +++ b/llvm/lib/Object/OffloadBinary.cpp @@ -189,19 +189,23 @@ OffloadBinary::create(MemoryBufferRef Buf) { TheHeader->Size < sizeof(Entry) || TheHeader->Size < sizeof(Header)) return errorCodeToError(object_error::unexpected_eof); - if (TheHeader->EntryOffset > TheHeader->Size - sizeof(Entry) || - TheHeader->EntrySize > TheHeader->Size - sizeof(Header)) + uint64_t EntriesSize = sizeof(Entry) * TheHeader->EntriesCount; + if (TheHeader->EntriesOffset > TheHeader->Size - EntriesSize || + EntriesSize > TheHeader->Size - sizeof(Header)) return errorCodeToError(object_error::unexpected_eof); - const Entry *TheEntry = - reinterpret_cast(&Start[TheHeader->EntryOffset]); + const Entry *Entries = + reinterpret_cast(&Start[TheHeader->EntriesOffset]); + for (uint32_t I = 0; I < TheHeader->EntriesCount; ++I) { + const Entry *TheEntry = &Entries[I]; - if (TheEntry->ImageOffset > Buf.getBufferSize() || - TheEntry->StringOffset > Buf.getBufferSize()) - return errorCodeToError(object_error::unexpected_eof); + if (TheEntry->ImageOffset > Buf.getBufferSize() || + TheEntry->StringOffset > Buf.getBufferSize()) + return errorCodeToError(object_error::unexpected_eof); + } return std::unique_ptr( - new OffloadBinary(Buf, TheHeader, TheEntry)); + new OffloadBinary(Buf, TheHeader, Entries)); } SmallString<0> OffloadBinary::write(const OffloadingImage &OffloadingData) { @@ -227,8 +231,8 @@ SmallString<0> OffloadBinary::write(const OffloadingImage &OffloadingData) { Header TheHeader; TheHeader.Size = alignTo( BinaryDataSize + OffloadingData.Image->getBufferSize(), getAlignment()); - TheHeader.EntryOffset = sizeof(Header); - TheHeader.EntrySize = sizeof(Entry); + TheHeader.EntriesOffset = sizeof(Header); + TheHeader.EntriesCount = 1; // Create the entry using the string table offsets. The string table will be // placed directly after the entry in memory, and the image after that. diff --git a/llvm/lib/ObjectYAML/OffloadEmitter.cpp b/llvm/lib/ObjectYAML/OffloadEmitter.cpp index 131da68d77506..34eca4d61a410 100644 --- a/llvm/lib/ObjectYAML/OffloadEmitter.cpp +++ b/llvm/lib/ObjectYAML/OffloadEmitter.cpp @@ -45,10 +45,10 @@ bool yaml2offload(Binary &Doc, raw_ostream &Out, ErrorHandler EH) { TheHeader->Version = *Doc.Version; if (Doc.Size) TheHeader->Size = *Doc.Size; - if (Doc.EntryOffset) - TheHeader->EntryOffset = *Doc.EntryOffset; - if (Doc.EntrySize) - TheHeader->EntrySize = *Doc.EntrySize; + if (Doc.EntriesOffset) + TheHeader->EntriesOffset = *Doc.EntriesOffset; + if (Doc.EntriesCount) + TheHeader->EntriesCount = *Doc.EntriesCount; Out.write(Buffer.begin(), Buffer.size()); } diff --git a/llvm/lib/ObjectYAML/OffloadYAML.cpp b/llvm/lib/ObjectYAML/OffloadYAML.cpp index d5a0edde2179f..cab5a39741294 100644 --- a/llvm/lib/ObjectYAML/OffloadYAML.cpp +++ b/llvm/lib/ObjectYAML/OffloadYAML.cpp @@ -50,8 +50,8 @@ void MappingTraits::mapping(IO &IO, IO.mapTag("!Offload", true); IO.mapOptional("Version", O.Version); IO.mapOptional("Size", O.Size); - IO.mapOptional("EntryOffset", O.EntryOffset); - IO.mapOptional("EntrySize", O.EntrySize); + IO.mapOptional("EntriesOffset", O.EntriesOffset); + IO.mapOptional("EntriesCount", O.EntriesCount); IO.mapRequired("Members", O.Members); IO.setContext(nullptr); } diff --git a/llvm/test/ObjectYAML/Offload/malformed-entry-size.yaml b/llvm/test/ObjectYAML/Offload/malformed-entries-count.yaml similarity index 94% rename from llvm/test/ObjectYAML/Offload/malformed-entry-size.yaml rename to llvm/test/ObjectYAML/Offload/malformed-entries-count.yaml index 3194607ae39a5..bb2a34277963f 100644 --- a/llvm/test/ObjectYAML/Offload/malformed-entry-size.yaml +++ b/llvm/test/ObjectYAML/Offload/malformed-entries-count.yaml @@ -1,6 +1,6 @@ # RUN: yaml2obj %s | not obj2yaml 2>&1 | FileCheck %s !Offload -EntrySize: 999999999 +EntriesCount: 999999999 Members: - ImageKind: IMG_Cubin OffloadKind: OFK_OpenMP diff --git a/llvm/test/ObjectYAML/Offload/malformed-offset.yaml b/llvm/test/ObjectYAML/Offload/malformed-offset.yaml index 03c0431053cce..5aecfffd937bf 100644 --- a/llvm/test/ObjectYAML/Offload/malformed-offset.yaml +++ b/llvm/test/ObjectYAML/Offload/malformed-offset.yaml @@ -1,6 +1,6 @@ # RUN: yaml2obj %s | not obj2yaml 2>&1 | FileCheck %s !Offload -EntryOffset: 999999999 +EntriesOffset: 999999999 Members: - ImageKind: IMG_Cubin OffloadKind: OFK_OpenMP diff --git a/llvm/test/ObjectYAML/Offload/malformed-version.yaml b/llvm/test/ObjectYAML/Offload/malformed-version.yaml index f9279a52e2764..99383491acce0 100644 --- a/llvm/test/ObjectYAML/Offload/malformed-version.yaml +++ b/llvm/test/ObjectYAML/Offload/malformed-version.yaml @@ -1,6 +1,6 @@ # RUN: yaml2obj %s | not obj2yaml 2>&1 | FileCheck %s !Offload -Version: 2 +Version: 3 Members: - ImageKind: IMG_Cubin OffloadKind: OFK_OpenMP From a28aa79becd0b7dfec37cb16bcd522f366c56e85 Mon Sep 17 00:00:00 2001 From: "Plyakhin, Yury" Date: Thu, 27 Nov 2025 02:33:41 +0100 Subject: [PATCH 4/8] Just a snapshot of work in progress. --- llvm/include/llvm/Object/OffloadBinary.h | 117 +++++++++++------- llvm/lib/Object/OffloadBinary.cpp | 92 +++++++++----- llvm/tools/llvm-objdump/OffloadDump.cpp | 5 +- .../llvm-offload-binary.cpp | 10 +- llvm/tools/obj2yaml/offload2yaml.cpp | 28 +++-- 5 files changed, 157 insertions(+), 95 deletions(-) diff --git a/llvm/include/llvm/Object/OffloadBinary.h b/llvm/include/llvm/Object/OffloadBinary.h index 74dcd5cc7384b..84ebf51db3813 100644 --- a/llvm/include/llvm/Object/OffloadBinary.h +++ b/llvm/include/llvm/Object/OffloadBinary.h @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// // -// This file contains the binary format used for budingling device metadata with +// This file contains the binary format used for bundling device metadata with // an associated device image. The data can then be stored inside a host object // file to create a fat binary and read by the linker. This is intended to be a // thin wrapper around the image itself. If this format becomes sufficiently @@ -70,6 +70,33 @@ enum OffloadEntryFlags : uint32_t { /// offsets from the beginning of the file. class OffloadBinary : public Binary { public: + struct Header { + uint8_t Magic[4] = {0x10, 0xFF, 0x10, 0xAD}; // 0x10FF10AD magic bytes. + uint32_t Version = OffloadBinary::Version; // Version identifier. + uint64_t Size; // Size in bytes of this entire binary. + uint64_t EntriesOffset; // Offset in bytes to the start of entries block. + uint64_t EntriesCount; // Number of metadata entries in the binary. + }; + + struct Entry { + ImageKind TheImageKind; // The kind of the image stored. + OffloadKind TheOffloadKind; // The producer of this image. + uint32_t Flags; // Additional flags associated with the entry. + uint64_t StringOffset; // Offset in bytes to the string map. + uint64_t NumStrings; // Number of entries in the string map. + uint64_t ImageOffset; // Offset in bytes of the actual binary image. + uint64_t ImageSize; // Size in bytes of the binary image. + }; + + struct StringEntry { + uint64_t KeyOffset; + uint64_t ValueOffset; + uint64_t ValueSize; // Size of the value in bytes. + }; + + using StringMap = MapVector; + using entry_iterator = SmallVector, 1>::const_iterator; + using entry_iterator_range = iterator_range; using string_iterator = MapVector::const_iterator; using string_iterator_range = iterator_range; @@ -89,77 +116,79 @@ class OffloadBinary : public Binary { LLVM_ABI static Expected> create(MemoryBufferRef); - /// Serialize the contents of \p File to a binary buffer to be read later. - LLVM_ABI static SmallString<0> write(const OffloadingImage &); + /// Serialize the contents of \p OffloadingData to a binary buffer to be read + /// later. + LLVM_ABI static SmallString<0> + write(ArrayRef OffloadingData); static uint64_t getAlignment() { return 8; } - ImageKind getImageKind() const { return TheEntry->TheImageKind; } - OffloadKind getOffloadKind() const { return TheEntry->TheOffloadKind; } + ImageKind getOnlyImageKind() const { + assert(getEntriesCount() == 1 && "Expected exactly one entry."); + return Entries[0].first->TheImageKind; + } + + OffloadKind getOffloadKind() const { return Entries[0].first->TheOffloadKind; } uint32_t getVersion() const { return TheHeader->Version; } - uint32_t getFlags() const { return TheEntry->Flags; } + uint32_t getFlags() const { return Entries[0].first->Flags; } uint64_t getSize() const { return TheHeader->Size; } + uint64_t getEntriesCount() const { return TheHeader->EntriesCount; } StringRef getTriple() const { return getString("triple"); } StringRef getArch() const { return getString("arch"); } StringRef getImage() const { - return StringRef(&Buffer[TheEntry->ImageOffset], TheEntry->ImageSize); + return StringRef(&Buffer[Entries[0].first->ImageOffset], Entries[0].first->ImageSize); } - // Iterator over all the key and value pairs in the binary. - string_iterator_range strings() const { return StringData; } - - StringRef getString(StringRef Key) const { return StringData.lookup(Key); } + // Iterator access to all entries in the binary + entry_iterator_range entries() const { + return make_range(Entries.begin(), Entries.end()); + } + entry_iterator entries_begin() const { return Entries.begin(); } + entry_iterator entries_end() const { return Entries.end(); } - static bool classof(const Binary *V) { return V->isOffloadFile(); } + // Access specific entry by index. + const std::pair &getEntry(size_t Index) const { + assert(Index < Entries.size() && "Entry index out of bounds"); + return Entries[Index]; + } - struct Header { - uint8_t Magic[4] = {0x10, 0xFF, 0x10, 0xAD}; // 0x10FF10AD magic bytes. - uint32_t Version = OffloadBinary::Version; // Version identifier. - uint64_t Size; // Size in bytes of this entire binary. - uint64_t EntriesOffset; // Offset in bytes to the start of entries block. - uint64_t EntriesCount; // Number of metadata entries in the binary. - }; + // Iterator over all the key and value pairs in the binary. + string_iterator_range strings() const { return Entries[0].second; } - struct Entry { - ImageKind TheImageKind; // The kind of the image stored. - OffloadKind TheOffloadKind; // The producer of this image. - uint32_t Flags; // Additional flags associated with the entry. - uint64_t StringOffset; // Offset in bytes to the string map. - uint64_t NumStrings; // Number of entries in the string map. - uint64_t ImageOffset; // Offset in bytes of the actual binary image. - uint64_t ImageSize; // Size in bytes of the binary image. - }; + StringRef getString(StringRef Key) const { return Entries[0].second.lookup(Key); } - struct StringEntry { - uint64_t KeyOffset; - uint64_t ValueOffset; - uint64_t ValueSize; // Size of the value in bytes. - }; + static bool classof(const Binary *V) { return V->isOffloadFile(); } private: OffloadBinary(MemoryBufferRef Source, const Header *TheHeader, - const Entry *TheEntry) + const Entry *EntriesBegin) : Binary(Binary::ID_Offload, Source), Buffer(Source.getBufferStart()), - TheHeader(TheHeader), TheEntry(TheEntry) { - const StringEntry *StringMapBegin = - reinterpret_cast(&Buffer[TheEntry->StringOffset]); - for (uint64_t I = 0, E = TheEntry->NumStrings; I != E; ++I) { - StringRef Key = &Buffer[StringMapBegin[I].KeyOffset]; - StringData[Key] = &Buffer[StringMapBegin[I].ValueOffset]; + TheHeader(TheHeader) { + for (uint64_t EI = 0, EE = TheHeader->EntriesCount; EI != EE; ++EI) { + const Entry *TheEntry = &EntriesBegin[EI]; + const StringEntry *StringMapBegin = reinterpret_cast( + &Buffer[TheEntry->StringOffset]); + StringMap Strings; + for (uint64_t SI = 0, SE = TheEntry->NumStrings; SI != SE; ++SI) { + StringRef Key = &Buffer[StringMapBegin[SI].KeyOffset]; + StringRef Value = StringRef( + &Buffer[StringMapBegin[SI].ValueOffset], StringMapBegin[SI].ValueSize); + Strings.insert({Key, Value}); + } + Entries.push_back(std::make_pair(TheEntry, std::move(Strings))); } } OffloadBinary(const OffloadBinary &Other) = delete; - /// Map from keys to offsets in the binary. - MapVector StringData; + /// Location of the metadata entries within the binary mapped to + /// the key-value string data. + SmallVector, 1> Entries; /// Raw pointer to the MemoryBufferRef for convenience. const char *Buffer; /// Location of the header within the binary. const Header *TheHeader; - /// Location of the metadata entries within the binary. - const Entry *TheEntry; }; /// A class to contain the binary information for a single OffloadBinary that diff --git a/llvm/lib/Object/OffloadBinary.cpp b/llvm/lib/Object/OffloadBinary.cpp index 8e51b94b92951..609b0896a0ba7 100644 --- a/llvm/lib/Object/OffloadBinary.cpp +++ b/llvm/lib/Object/OffloadBinary.cpp @@ -208,59 +208,85 @@ OffloadBinary::create(MemoryBufferRef Buf) { new OffloadBinary(Buf, TheHeader, Entries)); } -SmallString<0> OffloadBinary::write(const OffloadingImage &OffloadingData) { +SmallString<0> OffloadBinary::write(ArrayRef OffloadingData) { + uint64_t EntriesCount = OffloadingData.size(); + assert(EntriesCount > 0 && "At least one offloading image is required"); + // Create a null-terminated string table with all the used strings. + // Also calculate total size of images. StringTableBuilder StrTab(StringTableBuilder::ELF); - for (auto &KeyAndValue : OffloadingData.StringData) { - StrTab.add(KeyAndValue.first); - StrTab.add(KeyAndValue.second); + uint64_t TotalStringEntries = 0; + uint64_t TotalImagesSize = 0; + for (const OffloadingImage &Img : OffloadingData) { + for (auto &KeyAndValue : Img.StringData) { + StrTab.add(KeyAndValue.first); + StrTab.add(KeyAndValue.second); + } + TotalStringEntries += Img.StringData.size(); + TotalImagesSize += Img.Image->getBufferSize(); } StrTab.finalize(); - uint64_t StringEntrySize = - sizeof(StringEntry) * OffloadingData.StringData.size(); + uint64_t StringEntrySize = sizeof(StringEntry) * TotalStringEntries; + uint64_t EntriesSize = sizeof(Entry) * EntriesCount; + uint64_t StrTabOffset = sizeof(Header) + EntriesSize + StringEntrySize; // Make sure the image we're wrapping around is aligned as well. - uint64_t BinaryDataSize = alignTo(sizeof(Header) + sizeof(Entry) + - StringEntrySize + StrTab.getSize(), - getAlignment()); + uint64_t BinaryDataSize = + alignTo(StrTabOffset + StrTab.getSize(), getAlignment()); - // Create the header and fill in the offsets. The entry will be directly + // Create the header and fill in the offsets. The entries will be directly // placed after the header in memory. Align the size to the alignment of the // header so this can be placed contiguously in a single section. Header TheHeader; - TheHeader.Size = alignTo( - BinaryDataSize + OffloadingData.Image->getBufferSize(), getAlignment()); + TheHeader.Size = alignTo(BinaryDataSize + TotalImagesSize, getAlignment()); TheHeader.EntriesOffset = sizeof(Header); - TheHeader.EntriesCount = 1; - - // Create the entry using the string table offsets. The string table will be - // placed directly after the entry in memory, and the image after that. - Entry TheEntry; - TheEntry.TheImageKind = OffloadingData.TheImageKind; - TheEntry.TheOffloadKind = OffloadingData.TheOffloadKind; - TheEntry.Flags = OffloadingData.Flags; - TheEntry.StringOffset = sizeof(Header) + sizeof(Entry); - TheEntry.NumStrings = OffloadingData.StringData.size(); - - TheEntry.ImageOffset = BinaryDataSize; - TheEntry.ImageSize = OffloadingData.Image->getBufferSize(); + TheHeader.EntriesCount = EntriesCount; SmallString<0> Data; Data.reserve(TheHeader.Size); raw_svector_ostream OS(Data); OS << StringRef(reinterpret_cast(&TheHeader), sizeof(Header)); - OS << StringRef(reinterpret_cast(&TheEntry), sizeof(Entry)); - for (auto &KeyAndValue : OffloadingData.StringData) { - uint64_t Offset = sizeof(Header) + sizeof(Entry) + StringEntrySize; - StringEntry Map{Offset + StrTab.getOffset(KeyAndValue.first), - Offset + StrTab.getOffset(KeyAndValue.second)}; - OS << StringRef(reinterpret_cast(&Map), sizeof(StringEntry)); + + // Create the entries using the string table offsets. The string table will be + // placed directly after the set of entries in memory, and all the images are + // after that. + uint64_t StringEntryOffset = sizeof(Header) + EntriesSize; + uint64_t ImageOffset = BinaryDataSize; + for (const OffloadingImage &Img : OffloadingData) { + Entry TheEntry; + + TheEntry.TheImageKind = Img.TheImageKind; + TheEntry.TheOffloadKind = Img.TheOffloadKind; + TheEntry.Flags = Img.Flags; + + TheEntry.StringOffset = StringEntryOffset; + StringEntryOffset += sizeof(StringEntry) * Img.StringData.size(); + TheEntry.NumStrings = Img.StringData.size(); + + TheEntry.ImageOffset = ImageOffset; + ImageOffset += Img.Image->getBufferSize(); + TheEntry.ImageSize = Img.Image->getBufferSize(); + + OS << StringRef(reinterpret_cast(&TheEntry), sizeof(Entry)); } + + // Create the string map entries. + for (const OffloadingImage &Img : OffloadingData) { + for (auto &KeyAndValue : Img.StringData) { + StringEntry Map{StrTabOffset + StrTab.getOffset(KeyAndValue.first), + StrTabOffset + StrTab.getOffset(KeyAndValue.second), + KeyAndValue.second.size()}; + OS << StringRef(reinterpret_cast(&Map), sizeof(StringEntry)); + } + } + StrTab.write(OS); // Add padding to required image alignment. - OS.write_zeros(TheEntry.ImageOffset - OS.tell()); - OS << OffloadingData.Image->getBuffer(); + OS.write_zeros(BinaryDataSize - OS.tell()); + + for (const OffloadingImage &Img : OffloadingData) + OS << Img.Image->getBuffer(); // Add final padding to required alignment. assert(TheHeader.Size >= OS.tell() && "Too much data written?"); diff --git a/llvm/tools/llvm-objdump/OffloadDump.cpp b/llvm/tools/llvm-objdump/OffloadDump.cpp index a77537dd90eeb..35003fa9c8cc5 100644 --- a/llvm/tools/llvm-objdump/OffloadDump.cpp +++ b/llvm/tools/llvm-objdump/OffloadDump.cpp @@ -25,7 +25,10 @@ void disassembleObject(llvm::object::ObjectFile *, bool InlineRelocs); /// Get the printable name of the image kind. static StringRef getImageName(const OffloadBinary &OB) { - switch (OB.getImageKind()) { + if (OB.getEntriesCount() > 1) + return "bundle"; + + switch (OB.getOnlyImageKind()) { case IMG_Object: return "elf"; case IMG_Bitcode: diff --git a/llvm/tools/llvm-offload-binary/llvm-offload-binary.cpp b/llvm/tools/llvm-offload-binary/llvm-offload-binary.cpp index e22d13b946651..811daf24c2af5 100644 --- a/llvm/tools/llvm-offload-binary/llvm-offload-binary.cpp +++ b/llvm/tools/llvm-offload-binary/llvm-offload-binary.cpp @@ -202,10 +202,12 @@ static Error unbundleImages() { } else { uint64_t Idx = 0; for (const OffloadBinary *Binary : Extracted) { - StringRef Filename = - Saver.save(sys::path::stem(InputFile) + "-" + Binary->getTriple() + - "-" + Binary->getArch() + "." + std::to_string(Idx++) + - "." + getImageKindName(Binary->getImageKind())); + StringRef FileExt = Binary->getEntriesCount() == 1 + ? getImageKindName(Binary->getOnlyImageKind()) + : "bundle"; + StringRef Filename = Saver.save( + sys::path::stem(InputFile) + "-" + Binary->getTriple() + "-" + + Binary->getArch() + "." + std::to_string(Idx++) + "." + FileExt); if (Error E = writeFile(Filename, Binary->getImage())) return E; } diff --git a/llvm/tools/obj2yaml/offload2yaml.cpp b/llvm/tools/obj2yaml/offload2yaml.cpp index 2b63e1278cd22..f9c6369b4e484 100644 --- a/llvm/tools/obj2yaml/offload2yaml.cpp +++ b/llvm/tools/obj2yaml/offload2yaml.cpp @@ -18,20 +18,22 @@ namespace { void populateYAML(OffloadYAML::Binary &YAMLBinary, object::OffloadBinary &OB, UniqueStringSaver Saver) { - YAMLBinary.Members.emplace_back(); - auto &Member = YAMLBinary.Members.back(); - Member.ImageKind = OB.getImageKind(); - Member.OffloadKind = OB.getOffloadKind(); - Member.Flags = OB.getFlags(); - if (!OB.strings().empty()) { - Member.StringEntries = std::vector(); - for (const auto &Entry : OB.strings()) - Member.StringEntries->emplace_back(OffloadYAML::Binary::StringEntry( - {Saver.save(Entry.first), Saver.save(Entry.second)})); + for (const auto &Entry : OB.entries()) { + YAMLBinary.Members.emplace_back(); + auto &Member = YAMLBinary.Members.back(); + Member.ImageKind = Entry.first->TheImageKind; + Member.OffloadKind = Entry.first->TheOffloadKind; + Member.Flags = Entry.first->Flags; + if (!Entry.second.empty()) { + Member.StringEntries = std::vector(); + for (const auto &StringEntry : Entry.second) + Member.StringEntries->emplace_back(OffloadYAML::Binary::StringEntry( + {Saver.save(StringEntry.first), Saver.save(StringEntry.second)})); + } + + if (!OB.getImage().empty()) + Member.Content = arrayRefFromStringRef(OB.getImage()); } - - if (!OB.getImage().empty()) - Member.Content = arrayRefFromStringRef(OB.getImage()); } Expected dump(MemoryBufferRef Source, From df65ae903d3b66c0570d10d04925c287ad4d6163 Mon Sep 17 00:00:00 2001 From: "Plyakhin, Yury" Date: Thu, 27 Nov 2025 02:46:00 +0100 Subject: [PATCH 5/8] one more minor change to make build pass for now --- llvm/lib/Frontend/Offloading/OffloadWrapper.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp b/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp index f1602765dbaa7..744aae947f8af 100644 --- a/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp +++ b/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp @@ -952,7 +952,7 @@ class SYCLWrapper { Constant *OffloadKindConstant = ConstantInt::get( Type::getInt8Ty(C), static_cast(OB.getOffloadKind())); Constant *ImageKindConstant = ConstantInt::get( - Type::getInt8Ty(C), static_cast(OB.getImageKind())); + Type::getInt8Ty(C), static_cast(OB.getOnlyImageKind())); StringRef Triple = OB.getString("triple"); Constant *TripleConstant = addStringToModule(Triple, Twine(OffloadKindTag) + "target." + ImageID); From ac6b00c7ec50b185d231223152ec8fb2b76d359d Mon Sep 17 00:00:00 2001 From: "Plyakhin, Yury" Date: Tue, 2 Dec 2025 04:08:04 +0100 Subject: [PATCH 6/8] Use Index to keep getters to work with single Image --- llvm/include/llvm/Object/OffloadBinary.h | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/llvm/include/llvm/Object/OffloadBinary.h b/llvm/include/llvm/Object/OffloadBinary.h index 84ebf51db3813..15b9c1229e93e 100644 --- a/llvm/include/llvm/Object/OffloadBinary.h +++ b/llvm/include/llvm/Object/OffloadBinary.h @@ -123,21 +123,16 @@ class OffloadBinary : public Binary { static uint64_t getAlignment() { return 8; } - ImageKind getOnlyImageKind() const { - assert(getEntriesCount() == 1 && "Expected exactly one entry."); - return Entries[0].first->TheImageKind; - } - - OffloadKind getOffloadKind() const { return Entries[0].first->TheOffloadKind; } + ImageKind getImageKind() const { return Entries[Index].first->TheImageKind; } + OffloadKind getOffloadKind() const { return Entries[Index].first->TheOffloadKind; } uint32_t getVersion() const { return TheHeader->Version; } - uint32_t getFlags() const { return Entries[0].first->Flags; } uint64_t getSize() const { return TheHeader->Size; } uint64_t getEntriesCount() const { return TheHeader->EntriesCount; } StringRef getTriple() const { return getString("triple"); } StringRef getArch() const { return getString("arch"); } StringRef getImage() const { - return StringRef(&Buffer[Entries[0].first->ImageOffset], Entries[0].first->ImageSize); + return StringRef(&Buffer[Entries[Index].first->ImageOffset], Entries[Index].first->ImageSize); } // Iterator access to all entries in the binary @@ -154,9 +149,9 @@ class OffloadBinary : public Binary { } // Iterator over all the key and value pairs in the binary. - string_iterator_range strings() const { return Entries[0].second; } + string_iterator_range strings() const { return Entries[Index].second; } - StringRef getString(StringRef Key) const { return Entries[0].second.lookup(Key); } + StringRef getString(StringRef Key) const { return Entries[Index].second.lookup(Key); } static bool classof(const Binary *V) { return V->isOffloadFile(); } @@ -189,6 +184,8 @@ class OffloadBinary : public Binary { const char *Buffer; /// Location of the header within the binary. const Header *TheHeader; + /// Index of Entry represented by the current object. + const uint64_t Index; }; /// A class to contain the binary information for a single OffloadBinary that From f95e7a07b3951b3f42124536089606c79a44e994 Mon Sep 17 00:00:00 2001 From: "Plyakhin, Yury" Date: Wed, 3 Dec 2025 03:06:32 +0100 Subject: [PATCH 7/8] back to original interface --- llvm/include/llvm/Object/OffloadBinary.h | 106 +++++++++-------------- 1 file changed, 41 insertions(+), 65 deletions(-) diff --git a/llvm/include/llvm/Object/OffloadBinary.h b/llvm/include/llvm/Object/OffloadBinary.h index 15b9c1229e93e..4911b040d6886 100644 --- a/llvm/include/llvm/Object/OffloadBinary.h +++ b/llvm/include/llvm/Object/OffloadBinary.h @@ -70,33 +70,6 @@ enum OffloadEntryFlags : uint32_t { /// offsets from the beginning of the file. class OffloadBinary : public Binary { public: - struct Header { - uint8_t Magic[4] = {0x10, 0xFF, 0x10, 0xAD}; // 0x10FF10AD magic bytes. - uint32_t Version = OffloadBinary::Version; // Version identifier. - uint64_t Size; // Size in bytes of this entire binary. - uint64_t EntriesOffset; // Offset in bytes to the start of entries block. - uint64_t EntriesCount; // Number of metadata entries in the binary. - }; - - struct Entry { - ImageKind TheImageKind; // The kind of the image stored. - OffloadKind TheOffloadKind; // The producer of this image. - uint32_t Flags; // Additional flags associated with the entry. - uint64_t StringOffset; // Offset in bytes to the string map. - uint64_t NumStrings; // Number of entries in the string map. - uint64_t ImageOffset; // Offset in bytes of the actual binary image. - uint64_t ImageSize; // Size in bytes of the binary image. - }; - - struct StringEntry { - uint64_t KeyOffset; - uint64_t ValueOffset; - uint64_t ValueSize; // Size of the value in bytes. - }; - - using StringMap = MapVector; - using entry_iterator = SmallVector, 1>::const_iterator; - using entry_iterator_range = iterator_range; using string_iterator = MapVector::const_iterator; using string_iterator_range = iterator_range; @@ -123,69 +96,72 @@ class OffloadBinary : public Binary { static uint64_t getAlignment() { return 8; } - ImageKind getImageKind() const { return Entries[Index].first->TheImageKind; } - OffloadKind getOffloadKind() const { return Entries[Index].first->TheOffloadKind; } + ImageKind getImageKind() const { return TheEntry->TheImageKind; } + OffloadKind getOffloadKind() const { return TheEntry->TheOffloadKind; } uint32_t getVersion() const { return TheHeader->Version; } + uint32_t getFlags() const { return TheEntry->Flags; } uint64_t getSize() const { return TheHeader->Size; } - uint64_t getEntriesCount() const { return TheHeader->EntriesCount; } StringRef getTriple() const { return getString("triple"); } StringRef getArch() const { return getString("arch"); } StringRef getImage() const { - return StringRef(&Buffer[Entries[Index].first->ImageOffset], Entries[Index].first->ImageSize); - } - - // Iterator access to all entries in the binary - entry_iterator_range entries() const { - return make_range(Entries.begin(), Entries.end()); - } - entry_iterator entries_begin() const { return Entries.begin(); } - entry_iterator entries_end() const { return Entries.end(); } - - // Access specific entry by index. - const std::pair &getEntry(size_t Index) const { - assert(Index < Entries.size() && "Entry index out of bounds"); - return Entries[Index]; + return StringRef(&Buffer[TheEntry->ImageOffset], TheEntry->ImageSize); } // Iterator over all the key and value pairs in the binary. - string_iterator_range strings() const { return Entries[Index].second; } + string_iterator_range strings() const { return StringData; } - StringRef getString(StringRef Key) const { return Entries[Index].second.lookup(Key); } + StringRef getString(StringRef Key) const { return StringData.lookup(Key); } static bool classof(const Binary *V) { return V->isOffloadFile(); } + struct Header { + uint8_t Magic[4] = {0x10, 0xFF, 0x10, 0xAD}; // 0x10FF10AD magic bytes. + uint32_t Version = OffloadBinary::Version; // Version identifier. + uint64_t Size; // Size in bytes of this entire binary. + uint64_t EntriesOffset; // Offset in bytes to the start of entries block. + uint64_t EntriesCount; // Number of metadata entries in the binary. + }; + + struct Entry { + ImageKind TheImageKind; // The kind of the image stored. + OffloadKind TheOffloadKind; // The producer of this image. + uint32_t Flags; // Additional flags associated with the entry. + uint64_t StringOffset; // Offset in bytes to the string map. + uint64_t NumStrings; // Number of entries in the string map. + uint64_t ImageOffset; // Offset in bytes of the actual binary image. + uint64_t ImageSize; // Size in bytes of the binary image. + }; + + struct StringEntry { + uint64_t KeyOffset; + uint64_t ValueOffset; + uint64_t ValueSize; // Size of the value in bytes. + }; + private: OffloadBinary(MemoryBufferRef Source, const Header *TheHeader, - const Entry *EntriesBegin) + const Entry *TheEntry) : Binary(Binary::ID_Offload, Source), Buffer(Source.getBufferStart()), - TheHeader(TheHeader) { - for (uint64_t EI = 0, EE = TheHeader->EntriesCount; EI != EE; ++EI) { - const Entry *TheEntry = &EntriesBegin[EI]; - const StringEntry *StringMapBegin = reinterpret_cast( - &Buffer[TheEntry->StringOffset]); - StringMap Strings; - for (uint64_t SI = 0, SE = TheEntry->NumStrings; SI != SE; ++SI) { - StringRef Key = &Buffer[StringMapBegin[SI].KeyOffset]; - StringRef Value = StringRef( - &Buffer[StringMapBegin[SI].ValueOffset], StringMapBegin[SI].ValueSize); - Strings.insert({Key, Value}); - } - Entries.push_back(std::make_pair(TheEntry, std::move(Strings))); + TheHeader(TheHeader), TheEntry(TheEntry) { + const StringEntry *StringMapBegin = + reinterpret_cast(&Buffer[TheEntry->StringOffset]); + for (uint64_t I = 0, E = TheEntry->NumStrings; I != E; ++I) { + StringRef Key = &Buffer[StringMapBegin[I].KeyOffset]; + StringData[Key] = StringRef(&Buffer[StringMapBegin[I].ValueOffset], StringMapBegin[I].ValueSize); } } OffloadBinary(const OffloadBinary &Other) = delete; - /// Location of the metadata entries within the binary mapped to - /// the key-value string data. - SmallVector, 1> Entries; + /// Map from keys to offsets in the binary. + MapVector StringData; /// Raw pointer to the MemoryBufferRef for convenience. const char *Buffer; /// Location of the header within the binary. const Header *TheHeader; - /// Index of Entry represented by the current object. - const uint64_t Index; + /// Location of the metadata entries within the binary. + const Entry *TheEntry; }; /// A class to contain the binary information for a single OffloadBinary that From f72c1d4e433fbb81cf79fb13609a9dda24201a7d Mon Sep 17 00:00:00 2001 From: "Plyakhin, Yury" Date: Thu, 4 Dec 2025 03:26:54 +0100 Subject: [PATCH 8/8] work in progress --- llvm/include/llvm/Object/OffloadBinary.h | 66 ++++++----- .../Frontend/Offloading/OffloadWrapper.cpp | 2 +- llvm/lib/Object/Binary.cpp | 2 +- llvm/lib/Object/OffloadBinary.cpp | 103 +++++++++++++----- llvm/tools/llvm-objdump/OffloadDump.cpp | 5 +- .../llvm-offload-binary.cpp | 10 +- llvm/tools/obj2yaml/offload2yaml.cpp | 27 +++-- 7 files changed, 137 insertions(+), 78 deletions(-) diff --git a/llvm/include/llvm/Object/OffloadBinary.h b/llvm/include/llvm/Object/OffloadBinary.h index 4911b040d6886..e7db4a71ce059 100644 --- a/llvm/include/llvm/Object/OffloadBinary.h +++ b/llvm/include/llvm/Object/OffloadBinary.h @@ -85,9 +85,47 @@ class OffloadBinary : public Binary { std::unique_ptr Image; }; + struct Header { + uint8_t Magic[4] = {0x10, 0xFF, 0x10, 0xAD}; // 0x10FF10AD magic bytes. + uint32_t Version = OffloadBinary::Version; // Version identifier. + uint64_t Size; // Size in bytes of this entire binary. + uint64_t EntriesOffset; // Offset in bytes to the start of entries block. + uint64_t EntriesCount; // Number of metadata entries in the binary. + }; + + struct Entry { + ImageKind TheImageKind; // The kind of the image stored. + OffloadKind TheOffloadKind; // The producer of this image. + uint32_t Flags; // Additional flags associated with the entry. + uint64_t StringOffset; // Offset in bytes to the string map. + uint64_t NumStrings; // Number of entries in the string map. + uint64_t ImageOffset; // Offset in bytes of the actual binary image. + uint64_t ImageSize; // Size in bytes of the binary image. + }; + + struct StringEntry { + uint64_t KeyOffset; + uint64_t ValueOffset; + uint64_t ValueSize; // Size of the value in bytes. + }; + + struct StringEntryV1 { + uint64_t KeyOffset; + uint64_t ValueOffset; + }; + + /// Attempt to extract and validate the header from the offloading binary in + /// \p Buf. + LLVM_ABI + static Expected extractHeader(MemoryBufferRef Buf); + + /// Attempt to parse the offloading binary stored in \p Data. + LLVM_ABI static Expected> + createV1(MemoryBufferRef); + /// Attempt to parse the offloading binary stored in \p Data. LLVM_ABI static Expected> - create(MemoryBufferRef); + createV2(MemoryBufferRef); /// Serialize the contents of \p OffloadingData to a binary buffer to be read /// later. @@ -115,30 +153,6 @@ class OffloadBinary : public Binary { static bool classof(const Binary *V) { return V->isOffloadFile(); } - struct Header { - uint8_t Magic[4] = {0x10, 0xFF, 0x10, 0xAD}; // 0x10FF10AD magic bytes. - uint32_t Version = OffloadBinary::Version; // Version identifier. - uint64_t Size; // Size in bytes of this entire binary. - uint64_t EntriesOffset; // Offset in bytes to the start of entries block. - uint64_t EntriesCount; // Number of metadata entries in the binary. - }; - - struct Entry { - ImageKind TheImageKind; // The kind of the image stored. - OffloadKind TheOffloadKind; // The producer of this image. - uint32_t Flags; // Additional flags associated with the entry. - uint64_t StringOffset; // Offset in bytes to the string map. - uint64_t NumStrings; // Number of entries in the string map. - uint64_t ImageOffset; // Offset in bytes of the actual binary image. - uint64_t ImageSize; // Size in bytes of the binary image. - }; - - struct StringEntry { - uint64_t KeyOffset; - uint64_t ValueOffset; - uint64_t ValueSize; // Size of the value in bytes. - }; - private: OffloadBinary(MemoryBufferRef Source, const Header *TheHeader, const Entry *TheEntry) @@ -181,7 +195,7 @@ class OffloadFile : public OwningBinary { getBinary()->getMemoryBufferRef().getBufferIdentifier()); // This parsing should never fail because it has already been parsed. - auto NewBinaryOrErr = OffloadBinary::create(*Buffer); + auto NewBinaryOrErr = OffloadBinary::createV1(*Buffer); assert(NewBinaryOrErr && "Failed to parse a copy of the binary?"); if (!NewBinaryOrErr) llvm::consumeError(NewBinaryOrErr.takeError()); diff --git a/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp b/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp index 744aae947f8af..f1602765dbaa7 100644 --- a/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp +++ b/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp @@ -952,7 +952,7 @@ class SYCLWrapper { Constant *OffloadKindConstant = ConstantInt::get( Type::getInt8Ty(C), static_cast(OB.getOffloadKind())); Constant *ImageKindConstant = ConstantInt::get( - Type::getInt8Ty(C), static_cast(OB.getOnlyImageKind())); + Type::getInt8Ty(C), static_cast(OB.getImageKind())); StringRef Triple = OB.getString("triple"); Constant *TripleConstant = addStringToModule(Triple, Twine(OffloadKindTag) + "target." + ImageID); diff --git a/llvm/lib/Object/Binary.cpp b/llvm/lib/Object/Binary.cpp index da2a7bb0a19da..ff15df564356e 100644 --- a/llvm/lib/Object/Binary.cpp +++ b/llvm/lib/Object/Binary.cpp @@ -94,7 +94,7 @@ Expected> object::createBinary(MemoryBufferRef Buffer, // Unrecognized object file format. return errorCodeToError(object_error::invalid_file_type); case file_magic::offload_binary: - return OffloadBinary::create(Buffer); + return OffloadBinary::createV1(Buffer); case file_magic::minidump: return MinidumpFile::create(Buffer); case file_magic::tapi_file: diff --git a/llvm/lib/Object/OffloadBinary.cpp b/llvm/lib/Object/OffloadBinary.cpp index 609b0896a0ba7..3ff9dc4b12b25 100644 --- a/llvm/lib/Object/OffloadBinary.cpp +++ b/llvm/lib/Object/OffloadBinary.cpp @@ -33,31 +33,47 @@ namespace { /// binary format. Error extractOffloadFiles(MemoryBufferRef Contents, SmallVectorImpl &Binaries) { - uint64_t Offset = 0; - // There could be multiple offloading binaries stored at this section. - while (Offset < Contents.getBuffer().size()) { - std::unique_ptr Buffer = - MemoryBuffer::getMemBuffer(Contents.getBuffer().drop_front(Offset), "", - /*RequiresNullTerminator*/ false); - if (!isAddrAligned(Align(OffloadBinary::getAlignment()), - Buffer->getBufferStart())) - Buffer = MemoryBuffer::getMemBufferCopy(Buffer->getBuffer(), - Buffer->getBufferIdentifier()); - auto BinaryOrErr = OffloadBinary::create(*Buffer); - if (!BinaryOrErr) - return BinaryOrErr.takeError(); - OffloadBinary &Binary = **BinaryOrErr; - - // Create a new owned binary with a copy of the original memory. - std::unique_ptr BufferCopy = MemoryBuffer::getMemBufferCopy( - Binary.getData().take_front(Binary.getSize()), - Contents.getBufferIdentifier()); - auto NewBinaryOrErr = OffloadBinary::create(*BufferCopy); - if (!NewBinaryOrErr) - return NewBinaryOrErr.takeError(); - Binaries.emplace_back(std::move(*NewBinaryOrErr), std::move(BufferCopy)); - - Offset += Binary.getSize(); + if (Contents.getBuffer().size() == 0) + return Error::success(); + + std::unique_ptr Buffer = + MemoryBuffer::getMemBuffer(Contents.getBuffer(), "", + /*RequiresNullTerminator*/ false); + auto HeaderOrErr = OffloadBinary::extractHeader(*Buffer); + if (!HeaderOrErr) + return HeaderOrErr.takeError(); + const OffloadBinary::Header *Header = *HeaderOrErr; + + switch (Header->Version) { + case 1: { + uint64_t Offset = 0; + // There could be multiple offloading binaries stored at this section. + while (Offset < Contents.getBuffer().size()) { + std::unique_ptr Buffer = MemoryBuffer::getMemBuffer( + Contents.getBuffer().drop_front(Offset), "", + /*RequiresNullTerminator*/ false); + if (!isAddrAligned(Align(OffloadBinary::getAlignment()), + Buffer->getBufferStart())) + Buffer = MemoryBuffer::getMemBufferCopy(Buffer->getBuffer(), + Buffer->getBufferIdentifier()); + auto BinaryOrErr = OffloadBinary::createV1(*Buffer); + if (!BinaryOrErr) + return BinaryOrErr.takeError(); + OffloadBinary &Binary = **BinaryOrErr; + // Create a new owned binary with a copy of the original memory. + std::unique_ptr BufferCopy = MemoryBuffer::getMemBufferCopy( + Binary.getData().take_front(Binary.getSize()), + Contents.getBufferIdentifier()); + auto NewBinaryOrErr = OffloadBinary::createV1(*BufferCopy); + if (!NewBinaryOrErr) + return NewBinaryOrErr.takeError(); + Binaries.emplace_back(std::move(*NewBinaryOrErr), std::move(BufferCopy)); + Offset += Binary.getSize(); + } + } break; + case 2: { + + } break; } return Error::success(); @@ -167,8 +183,8 @@ Error extractFromArchive(const Archive &Library, } // namespace -Expected> -OffloadBinary::create(MemoryBufferRef Buf) { +Expected +OffloadBinary::extractHeader(MemoryBufferRef Buf) { if (Buf.getBufferSize() < sizeof(Header) + sizeof(Entry)) return errorCodeToError(object_error::parse_failed); @@ -182,7 +198,7 @@ OffloadBinary::create(MemoryBufferRef Buf) { const char *Start = Buf.getBufferStart(); const Header *TheHeader = reinterpret_cast(Start); - if (TheHeader->Version != OffloadBinary::Version) + if (TheHeader->Version == 0 || TheHeader->Version > OffloadBinary::Version) return errorCodeToError(object_error::parse_failed); if (TheHeader->Size > Buf.getBufferSize() || @@ -194,6 +210,37 @@ OffloadBinary::create(MemoryBufferRef Buf) { EntriesSize > TheHeader->Size - sizeof(Header)) return errorCodeToError(object_error::unexpected_eof); + return TheHeader; +} + +Expected> +OffloadBinary::createV1(MemoryBufferRef Buf) { + auto HeaderOrErr = extractHeader(Buf); + if (!HeaderOrErr) + return HeaderOrErr.takeError(); + const Header *TheHeader = *HeaderOrErr; + + const char *Start = Buf.getBufferStart(); + const Entry *TheEntry = + reinterpret_cast(&Start[TheHeader->EntriesOffset]); + + if (TheEntry->ImageOffset > Buf.getBufferSize() || + TheEntry->StringOffset > Buf.getBufferSize()) + return errorCodeToError(object_error::unexpected_eof); + + return std::unique_ptr( + new OffloadBinary(Buf, TheHeader, TheEntry)); +} + +Expected> +OffloadBinary::createV2(MemoryBufferRef Buf) { + auto HeaderOrErr = extractHeader(Buf); + if (!HeaderOrErr) + return HeaderOrErr.takeError(); + const Header *TheHeader = *HeaderOrErr; + + const char *Start = Buf.getBufferStart(); + const Entry *Entries = reinterpret_cast(&Start[TheHeader->EntriesOffset]); for (uint32_t I = 0; I < TheHeader->EntriesCount; ++I) { diff --git a/llvm/tools/llvm-objdump/OffloadDump.cpp b/llvm/tools/llvm-objdump/OffloadDump.cpp index 35003fa9c8cc5..a77537dd90eeb 100644 --- a/llvm/tools/llvm-objdump/OffloadDump.cpp +++ b/llvm/tools/llvm-objdump/OffloadDump.cpp @@ -25,10 +25,7 @@ void disassembleObject(llvm::object::ObjectFile *, bool InlineRelocs); /// Get the printable name of the image kind. static StringRef getImageName(const OffloadBinary &OB) { - if (OB.getEntriesCount() > 1) - return "bundle"; - - switch (OB.getOnlyImageKind()) { + switch (OB.getImageKind()) { case IMG_Object: return "elf"; case IMG_Bitcode: diff --git a/llvm/tools/llvm-offload-binary/llvm-offload-binary.cpp b/llvm/tools/llvm-offload-binary/llvm-offload-binary.cpp index 811daf24c2af5..e22d13b946651 100644 --- a/llvm/tools/llvm-offload-binary/llvm-offload-binary.cpp +++ b/llvm/tools/llvm-offload-binary/llvm-offload-binary.cpp @@ -202,12 +202,10 @@ static Error unbundleImages() { } else { uint64_t Idx = 0; for (const OffloadBinary *Binary : Extracted) { - StringRef FileExt = Binary->getEntriesCount() == 1 - ? getImageKindName(Binary->getOnlyImageKind()) - : "bundle"; - StringRef Filename = Saver.save( - sys::path::stem(InputFile) + "-" + Binary->getTriple() + "-" + - Binary->getArch() + "." + std::to_string(Idx++) + "." + FileExt); + StringRef Filename = + Saver.save(sys::path::stem(InputFile) + "-" + Binary->getTriple() + + "-" + Binary->getArch() + "." + std::to_string(Idx++) + + "." + getImageKindName(Binary->getImageKind())); if (Error E = writeFile(Filename, Binary->getImage())) return E; } diff --git a/llvm/tools/obj2yaml/offload2yaml.cpp b/llvm/tools/obj2yaml/offload2yaml.cpp index f9c6369b4e484..07fdefbe5a9b5 100644 --- a/llvm/tools/obj2yaml/offload2yaml.cpp +++ b/llvm/tools/obj2yaml/offload2yaml.cpp @@ -16,17 +16,20 @@ using namespace llvm; namespace { -void populateYAML(OffloadYAML::Binary &YAMLBinary, object::OffloadBinary &OB, +void populateYAML(OffloadYAML::Binary &YAMLBinary, + ArrayRef> OBinaries, UniqueStringSaver Saver) { - for (const auto &Entry : OB.entries()) { + for (const auto &OBinaryPtr : OBinaries) { + object::OffloadBinary &OB = *OBinaryPtr; + YAMLBinary.Members.emplace_back(); auto &Member = YAMLBinary.Members.back(); - Member.ImageKind = Entry.first->TheImageKind; - Member.OffloadKind = Entry.first->TheOffloadKind; - Member.Flags = Entry.first->Flags; - if (!Entry.second.empty()) { + Member.ImageKind = OB.getImageKind(); + Member.OffloadKind = OB.getOffloadKind(); + Member.Flags = OB.getFlags(); + if (!OB.strings().empty()) { Member.StringEntries = std::vector(); - for (const auto &StringEntry : Entry.second) + for (const auto &StringEntry : OB.strings()) Member.StringEntries->emplace_back(OffloadYAML::Binary::StringEntry( {Saver.save(StringEntry.first), Saver.save(StringEntry.second)})); } @@ -39,7 +42,7 @@ void populateYAML(OffloadYAML::Binary &YAMLBinary, object::OffloadBinary &OB, Expected dump(MemoryBufferRef Source, UniqueStringSaver Saver) { Expected> OB = - object::OffloadBinary::create(Source); + object::OffloadBinary::createV1(Source); if (!OB) return OB.takeError(); @@ -52,15 +55,15 @@ Expected dump(MemoryBufferRef Source, while (Offset < (*OB)->getMemoryBufferRef().getBufferSize()) { MemoryBufferRef Buffer = MemoryBufferRef( (*OB)->getData().drop_front(Offset), (*OB)->getFileName()); - auto BinaryOrErr = object::OffloadBinary::create(Buffer); + auto BinaryOrErr = object::OffloadBinary::createV1(Buffer); if (!BinaryOrErr) return BinaryOrErr.takeError(); - object::OffloadBinary &Binary = **BinaryOrErr; + std::unique_ptr BinaryPtr = std::move(*BinaryOrErr); - populateYAML(*YAMLBinary, Binary, Saver); + populateYAML(*YAMLBinary, BinaryPtr, Saver); - Offset += Binary.getSize(); + Offset += BinaryPtr->getSize(); } return YAMLBinary.release();