Skip to content

[NFC][IR2Vec] Moving parseVocabSection() to VocabStorage #161711

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 2, 2025
Merged

[NFC][IR2Vec] Moving parseVocabSection() to VocabStorage #161711

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions llvm/include/llvm/Analysis/IR2Vec.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,13 @@ class VocabStorage {
const_iterator end() const {
return const_iterator(this, getNumSections(), 0);
}

using VocabMap = std::map<std::string, Embedding>;
/// Parse a vocabulary section from JSON and populate the target vocabulary
/// map.
static Error parseVocabSection(StringRef Key,
const json::Value &ParsedVocabValue,
VocabMap &TargetVocab, unsigned &Dim);
};

/// Class for storing and accessing the IR2Vec vocabulary.
Expand Down Expand Up @@ -600,8 +607,6 @@ class IR2VecVocabAnalysis : public AnalysisInfoMixin<IR2VecVocabAnalysis> {

Error readVocabulary(VocabMap &OpcVocab, VocabMap &TypeVocab,
VocabMap &ArgVocab);
Error parseVocabSection(StringRef Key, const json::Value &ParsedVocabValue,
VocabMap &TargetVocab, unsigned &Dim);
void generateVocabStorage(VocabMap &OpcVocab, VocabMap &TypeVocab,
VocabMap &ArgVocab);
void emitError(Error Err, LLVMContext &Ctx);
Expand Down
86 changes: 43 additions & 43 deletions llvm/lib/Analysis/IR2Vec.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -330,6 +330,43 @@ bool VocabStorage::const_iterator::operator!=(
return !(*this == Other);
}

Error VocabStorage::parseVocabSection(StringRef Key,
const json::Value &ParsedVocabValue,
VocabMap &TargetVocab, unsigned &Dim) {
json::Path::Root Path("");
const json::Object *RootObj = ParsedVocabValue.getAsObject();
if (!RootObj)
return createStringError(errc::invalid_argument,
"JSON root is not an object");

const json::Value *SectionValue = RootObj->get(Key);
if (!SectionValue)
return createStringError(errc::invalid_argument,
"Missing '" + std::string(Key) +
"' section in vocabulary file");
if (!json::fromJSON(*SectionValue, TargetVocab, Path))
return createStringError(errc::illegal_byte_sequence,
"Unable to parse '" + std::string(Key) +
"' section from vocabulary");

Dim = TargetVocab.begin()->second.size();
if (Dim == 0)
return createStringError(errc::illegal_byte_sequence,
"Dimension of '" + std::string(Key) +
"' section of the vocabulary is zero");

if (!std::all_of(TargetVocab.begin(), TargetVocab.end(),
[Dim](const std::pair<StringRef, Embedding> &Entry) {
return Entry.second.size() == Dim;
}))
return createStringError(
errc::illegal_byte_sequence,
"All vectors in the '" + std::string(Key) +
"' section of the vocabulary are not of the same dimension");

return Error::success();
}

// ==----------------------------------------------------------------------===//
// Vocabulary
//===----------------------------------------------------------------------===//
Expand Down Expand Up @@ -460,43 +497,6 @@ VocabStorage Vocabulary::createDummyVocabForTest(unsigned Dim) {
// IR2VecVocabAnalysis
//===----------------------------------------------------------------------===//

Error IR2VecVocabAnalysis::parseVocabSection(
StringRef Key, const json::Value &ParsedVocabValue, VocabMap &TargetVocab,
unsigned &Dim) {
json::Path::Root Path("");
const json::Object *RootObj = ParsedVocabValue.getAsObject();
if (!RootObj)
return createStringError(errc::invalid_argument,
"JSON root is not an object");

const json::Value *SectionValue = RootObj->get(Key);
if (!SectionValue)
return createStringError(errc::invalid_argument,
"Missing '" + std::string(Key) +
"' section in vocabulary file");
if (!json::fromJSON(*SectionValue, TargetVocab, Path))
return createStringError(errc::illegal_byte_sequence,
"Unable to parse '" + std::string(Key) +
"' section from vocabulary");

Dim = TargetVocab.begin()->second.size();
if (Dim == 0)
return createStringError(errc::illegal_byte_sequence,
"Dimension of '" + std::string(Key) +
"' section of the vocabulary is zero");

if (!std::all_of(TargetVocab.begin(), TargetVocab.end(),
[Dim](const std::pair<StringRef, Embedding> &Entry) {
return Entry.second.size() == Dim;
}))
return createStringError(
errc::illegal_byte_sequence,
"All vectors in the '" + std::string(Key) +
"' section of the vocabulary are not of the same dimension");

return Error::success();
}

// FIXME: Make this optional. We can avoid file reads
// by auto-generating a default vocabulary during the build time.
Error IR2VecVocabAnalysis::readVocabulary(VocabMap &OpcVocab,
Expand All @@ -513,16 +513,16 @@ Error IR2VecVocabAnalysis::readVocabulary(VocabMap &OpcVocab,
return ParsedVocabValue.takeError();

unsigned OpcodeDim = 0, TypeDim = 0, ArgDim = 0;
if (auto Err =
parseVocabSection("Opcodes", *ParsedVocabValue, OpcVocab, OpcodeDim))
if (auto Err = VocabStorage::parseVocabSection("Opcodes", *ParsedVocabValue,
OpcVocab, OpcodeDim))
return Err;

if (auto Err =
parseVocabSection("Types", *ParsedVocabValue, TypeVocab, TypeDim))
if (auto Err = VocabStorage::parseVocabSection("Types", *ParsedVocabValue,
TypeVocab, TypeDim))
return Err;

if (auto Err =
parseVocabSection("Arguments", *ParsedVocabValue, ArgVocab, ArgDim))
if (auto Err = VocabStorage::parseVocabSection("Arguments", *ParsedVocabValue,
ArgVocab, ArgDim))
return Err;

if (!(OpcodeDim == TypeDim && TypeDim == ArgDim))
Expand Down