Skip to content

Fix compress/decompress in LLVM Offloading API #150064

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 11 commits into
base: main
Choose a base branch
from
Open

Fix compress/decompress in LLVM Offloading API #150064

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
8fd5968
SWDEV-52811 - fix compress/decompress in LLVM Offloading API
Jun 17, 2025
5a70f7d
Clean-up from review comments.
Aug 12, 2025
bd67241
Resolve PR comments.
Aug 19, 2025
98f59d4
Resolve PR comments part 2
david-salinas Aug 28, 2025
856bb01
Merge branch 'main' into correct-decompress-offloading-api
david-salinas Aug 29, 2025
d3c9a79
Change Offloading API to require a output stream to be passed for dec…
david-salinas Sep 4, 2025
de45503
Refactor API change for user specified output stream
david-salinas Sep 10, 2025
a375975
Merge branch 'main' into correct-decompress-offloading-api
david-salinas Sep 10, 2025
589e898
Resolve build issue and address PR comments.
david-salinas Sep 15, 2025
7ce94fe
Address PR comment.
david-salinas Sep 18, 2025
442a6bd
resolve PR comments
david-salinas Sep 22, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 50 additions & 24 deletions llvm/include/llvm/Object/OffloadBundle.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,42 +32,54 @@ namespace llvm {

namespace object {

// CompressedOffloadBundle represents the format for the compressed offload
// bundles.
//
// The format is as follows:
// - Magic Number (4 bytes) - A constant "CCOB".
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is probably more easily explained as a struct.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is actually pulled from "clang/include/clang/Driver/OffloadBundler.h". The intent here is to migrate the offloading API in clang/lib/Driver (used by the clang-offload-bundler) into this LLVM API. In this case, I need to add (copy) this code from the Clang API, so that the llvm-* tools can decompress the offload bundles that the Clang Driver API created. For now we would/will have this code duplicated, which is not ideal, until we can have the clang-offload-bundler use this LLVM API to handle compress/decompress.

// - Version (2 bytes)
// - Compression Method (2 bytes) - Uses the values from
// llvm::compression::Format.
// - Total file size (4 bytes in V2, 8 bytes in V3).
// - Uncompressed Size (4 bytes in V1/V2, 8 bytes in V3).
// - Truncated MD5 Hash (8 bytes).
// - Compressed Data (variable length).
class CompressedOffloadBundle {
private:
static inline const size_t MagicSize = 4;
static inline const size_t VersionFieldSize = sizeof(uint16_t);
static inline const size_t MethodFieldSize = sizeof(uint16_t);
static inline const size_t FileSizeFieldSize = sizeof(uint32_t);
static inline const size_t UncompressedSizeFieldSize = sizeof(uint32_t);
static inline const size_t HashFieldSize = sizeof(uint64_t);
static inline const size_t V1HeaderSize =
MagicSize + VersionFieldSize + MethodFieldSize +
UncompressedSizeFieldSize + HashFieldSize;
static inline const size_t V2HeaderSize =
MagicSize + VersionFieldSize + FileSizeFieldSize + MethodFieldSize +
UncompressedSizeFieldSize + HashFieldSize;
static inline const llvm::StringRef MagicNumber = "CCOB";
static inline const uint16_t Version = 2;

public:
LLVM_ABI static llvm::Expected<std::unique_ptr<llvm::MemoryBuffer>>
struct CompressedBundleHeader {
unsigned Version;
llvm::compression::Format CompressionFormat;
std::optional<size_t> FileSize;
size_t UncompressedFileSize;
uint64_t Hash;

static llvm::Expected<CompressedBundleHeader> tryParse(llvm::StringRef);
};

static inline const uint16_t DefaultVersion = 3;

static llvm::Expected<std::unique_ptr<llvm::MemoryBuffer>>
compress(llvm::compression::Params P, const llvm::MemoryBuffer &Input,
bool Verbose = false);
LLVM_ABI static llvm::Expected<std::unique_ptr<llvm::MemoryBuffer>>
decompress(llvm::MemoryBufferRef &Input, bool Verbose = false);
uint16_t Version, raw_ostream *VerboseStream = nullptr);
static llvm::Expected<std::unique_ptr<llvm::MemoryBuffer>>
decompress(const llvm::MemoryBuffer &Input,
raw_ostream *VerboseStream = nullptr);
};

/// Bundle entry in binary clang-offload-bundler format.
struct OffloadBundleEntry {
uint64_t Offset = 0u;
uint64_t Size = 0u;
uint64_t IDLength = 0u;
StringRef ID;
std::string ID;
OffloadBundleEntry(uint64_t O, uint64_t S, uint64_t I, StringRef T)
: Offset(O), Size(S), IDLength(I), ID(T) {}
: Offset(O), Size(S), IDLength(I), ID(T.str()) {}
void dumpInfo(raw_ostream &OS) {
OS << "Offset = " << Offset << ", Size = " << Size
<< ", ID Length = " << IDLength << ", ID = " << ID;
<< ", ID Length = " << IDLength << ", ID = " << ID << "\n";
}
void dumpURI(raw_ostream &OS, StringRef FilePath) {
OS << ID.data() << "\tfile://" << FilePath << "#offset=" << Offset
Expand All @@ -81,16 +93,21 @@ class OffloadBundleFatBin {
uint64_t Size = 0u;
StringRef FileName;
uint64_t NumberOfEntries;
bool Decompressed;
SmallVector<OffloadBundleEntry> Entries;

public:
std::unique_ptr<MemoryBuffer> DecompressedBuffer;

SmallVector<OffloadBundleEntry> getEntries() { return Entries; }
uint64_t getSize() const { return Size; }
StringRef getFileName() const { return FileName; }
uint64_t getNumEntries() const { return NumberOfEntries; }
bool isDecompressed() const { return Decompressed; }

LLVM_ABI static Expected<std::unique_ptr<OffloadBundleFatBin>>
create(MemoryBufferRef, uint64_t SectionOffset, StringRef FileName);
create(MemoryBufferRef, uint64_t SectionOffset, StringRef FileName,
bool Decompress = false);
LLVM_ABI Error extractBundle(const ObjectFile &Source);

LLVM_ABI Error dumpEntryToCodeObject();
Expand All @@ -106,9 +123,14 @@ class OffloadBundleFatBin {
Entry.dumpURI(outs(), FileName);
}

OffloadBundleFatBin(MemoryBufferRef Source, StringRef File)
: FileName(File), NumberOfEntries(0),
Entries(SmallVector<OffloadBundleEntry>()) {}
OffloadBundleFatBin(MemoryBufferRef Source, StringRef File,
bool Decompress = false)
: FileName(File), NumberOfEntries(0), Decompressed(Decompress),
Entries(SmallVector<OffloadBundleEntry>()) {
if (Decompress)
DecompressedBuffer =
MemoryBuffer::getMemBufferCopy(Source.getBuffer(), File);
}
};

enum UriTypeT { FILE_URI, MEMORY_URI };
Expand Down Expand Up @@ -191,6 +213,10 @@ LLVM_ABI Error extractOffloadBundleFatBinary(
LLVM_ABI Error extractCodeObject(const ObjectFile &Source, int64_t Offset,
int64_t Size, StringRef OutputFileName);

/// Extract code object memory from the given \p Source object file at \p Offset
/// and of \p Size, and copy into \p OutputFileName.
LLVM_ABI Error extractCodeObject(MemoryBufferRef Buffer, int64_t Offset,
int64_t Size, StringRef OutputFileName);
/// Extracts an Offload Bundle Entry given by URI
LLVM_ABI Error extractOffloadBundleByURI(StringRef URIstr);

Expand Down
Loading