Skip to content

Commit

Permalink
⚡ [Lex] Better reservations for improved performance/memory usage.
Browse files Browse the repository at this point in the history
  • Loading branch information
@ThePhD
ThePhD committed Oct 9, 2023
1 parent 357bda5 commit dfd6383
Show file tree
Hide file tree
Showing 2 changed files with 93 additions and 21 deletions.
4 changes: 2 additions & 2 deletions clang/include/clang/Lex/Preprocessor.h
View file
Expand Up @@ -2729,14 +2729,14 @@ class Preprocessor {
const FileEntry *LookupFromFile = nullptr);
void HandleEmbedDirectiveNaive(SourceLocation HashLoc,
SourceLocation FilenameTok,
LexEmbedParametersResult &Params,
const LexEmbedParametersResult &Params,
StringRef BinaryContents,
const size_t TargetCharWidth);
void HandleEmbedDirectiveBuiltin(SourceLocation HashLoc,
const Token &FilenameTok,
StringRef ResolvedFilename,
StringRef SearchPath, StringRef RelativePath,
LexEmbedParametersResult &Params,
const LexEmbedParametersResult &Params,
StringRef BinaryContents,
const size_t TargetCharWidth);

Expand Down
110 changes: 91 additions & 19 deletions clang/lib/Lex/PPDirectives.cpp
View file
Expand Up @@ -3854,11 +3854,59 @@ inline constexpr const char *IntegerLiterals[] = {
"242", "243", "244", "245", "246", "247", "248", "249", "250", "251", "252",
"253", "254", "255"};

void Preprocessor::HandleEmbedDirectiveNaive(SourceLocation HashLoc,
SourceLocation FilenameLoc,
LexEmbedParametersResult &Params,
StringRef BinaryContents,
const size_t TargetCharWidth) {
static size_t
ComputeNaiveReserveSize(const Preprocessor::LexEmbedParametersResult &Params,
StringRef TypeName, StringRef BinaryContents,
SmallVectorImpl<char> &TokSpellingBuffer) {
size_t ReserveSize = 0;
if (BinaryContents.empty()) {
if (Params.MaybeIfEmptyParam) {
for (const auto &Tok : Params.MaybeIfEmptyParam->Tokens) {
const size_t TokLen = Tok.getLength();
if (TokLen > TokSpellingBuffer.size()) {
TokSpellingBuffer.resize(TokLen);
}
ReserveSize += TokLen;
}
}
} else {
if (Params.MaybePrefixParam) {
for (const auto &Tok : Params.MaybePrefixParam->Tokens) {
const size_t TokLen = Tok.getLength();
if (TokLen > TokSpellingBuffer.size()) {
TokSpellingBuffer.resize(TokLen);
}
ReserveSize += TokLen;
}
}
for (const auto &Byte : BinaryContents) {
ReserveSize += 3 + TypeName.size(); // ((type-name)
if (Byte > 99) {
ReserveSize += 3; // ###
} else if (Byte > 9) {
ReserveSize += 2; // ##
} else {
ReserveSize += 1; // #
}
ReserveSize += 2; // ),
}
if (Params.MaybePrefixParam) {
for (const auto &Tok : Params.MaybePrefixParam->Tokens) {
const size_t TokLen = Tok.getLength();
if (TokLen > TokSpellingBuffer.size()) {
TokSpellingBuffer.resize(TokLen);
}
ReserveSize += TokLen;
}
}
}
return ReserveSize;
}

void Preprocessor::HandleEmbedDirectiveNaive(
SourceLocation HashLoc, SourceLocation FilenameLoc,
const LexEmbedParametersResult &Params, StringRef BinaryContents,
const size_t TargetCharWidth) {
// Load up a new embed buffer for this file and set of parameters in
// particular.
EmbedBuffers.push_back("");
Expand All @@ -3869,30 +3917,37 @@ void Preprocessor::HandleEmbedDirectiveNaive(SourceLocation HashLoc,
return PrefixNumber.concat(">");
}(EmbedBufferNumberVal);
std::string &TargetEmbedBuffer = EmbedBuffers.back();
const size_t TotalSize = BinaryContents.size();
// In the future, this might change/improve.
const StringRef TypeName = "unsigned char";

// In the future, this might improve.
const StringRef SmallestType = "unsigned char";
SmallVector<char, 32> TokSpellingBuffer(32, 0);
const size_t ReserveSize = ComputeNaiveReserveSize(
Params, TypeName, BinaryContents, TokSpellingBuffer);
TargetEmbedBuffer.reserve(ReserveSize);

// Generate the look-alike source file
if (BinaryContents.empty()) {
if (Params.MaybeIfEmptyParam) {
PPEmbedParameterIfEmpty &EmptyParam = *Params.MaybeIfEmptyParam;
const PPEmbedParameterIfEmpty &EmptyParam = *Params.MaybeIfEmptyParam;
for (const auto &Tok : EmptyParam.Tokens) {
TargetEmbedBuffer.append(this->getSpelling(Tok));
StringRef Spelling = this->getSpelling(Tok, TokSpellingBuffer);
TargetEmbedBuffer.append(Spelling.data(), Spelling.size());
}
}
} else {
if (Params.MaybePrefixParam) {
PPEmbedParameterPrefix &PrefixParam = *Params.MaybePrefixParam;
const PPEmbedParameterPrefix &PrefixParam = *Params.MaybePrefixParam;
for (const auto &Tok : PrefixParam.Tokens) {
TargetEmbedBuffer.append(this->getSpelling(Tok));
StringRef Spelling = this->getSpelling(Tok, TokSpellingBuffer);
TargetEmbedBuffer.append(Spelling.data(), Spelling.size());
}
}
for (size_t I = 0; I < BinaryContents.size(); ++I) {
unsigned char ByteValue = BinaryContents[I];
StringRef ByteRepresentation = IntegerLiterals[ByteValue];
TargetEmbedBuffer.append(2, '(');
TargetEmbedBuffer.append(SmallestType.data(), SmallestType.size());
TargetEmbedBuffer.append(TypeName.data(), TypeName.size());
TargetEmbedBuffer.append(1, ')');
TargetEmbedBuffer.append(ByteRepresentation.data(),
ByteRepresentation.size());
Expand All @@ -3903,9 +3958,10 @@ void Preprocessor::HandleEmbedDirectiveNaive(SourceLocation HashLoc,
}
}
if (Params.MaybeSuffixParam) {
PPEmbedParameterSuffix &SuffixParam = *Params.MaybeSuffixParam;
const PPEmbedParameterSuffix &SuffixParam = *Params.MaybeSuffixParam;
for (const auto &Tok : SuffixParam.Tokens) {
TargetEmbedBuffer.append(this->getSpelling(Tok));
StringRef Spelling = this->getSpelling(Tok, TokSpellingBuffer);
TargetEmbedBuffer.append(Spelling.data(), Spelling.size());
}
}
}
Expand All @@ -3926,7 +3982,7 @@ void Preprocessor::HandleEmbedDirectiveNaive(SourceLocation HashLoc,
static bool TokenListIsCharacterArray(Preprocessor &PP,
const size_t TargetCharWidth,
bool IsPrefix,
SmallVectorImpl<Token> &Tokens,
const SmallVectorImpl<Token> &Tokens,
llvm::SmallVectorImpl<char> &Output) {
const bool IsSuffix = !IsPrefix;
size_t MaxValue =
Expand Down Expand Up @@ -4032,7 +4088,7 @@ static void TripleEncodeBase64(StringRef Bytes0, StringRef Bytes1,
void Preprocessor::HandleEmbedDirectiveBuiltin(
SourceLocation HashLoc, const Token &FilenameTok,
StringRef ResolvedFilename, StringRef SearchPath, StringRef RelativePath,
LexEmbedParametersResult &Params, StringRef BinaryContents,
const LexEmbedParametersResult &Params, StringRef BinaryContents,
const size_t TargetCharWidth) {
// if it's empty, just process it like a normal expanded token stream
if (BinaryContents.empty()) {
Expand All @@ -4045,7 +4101,7 @@ void Preprocessor::HandleEmbedDirectiveBuiltin(
if (Params.MaybePrefixParam) {
// If we ahve a prefix, validate that it's a good fit for direct data
// embedded (and prepare to prepend it)
PPEmbedParameterPrefix &PrefixParam = *Params.MaybePrefixParam;
const PPEmbedParameterPrefix &PrefixParam = *Params.MaybePrefixParam;
if (!TokenListIsCharacterArray(*this, TargetCharWidth, true,
PrefixParam.Tokens, BinaryPrefix)) {
HandleEmbedDirectiveNaive(HashLoc, FilenameTok.getLocation(), Params,
Expand All @@ -4056,7 +4112,7 @@ void Preprocessor::HandleEmbedDirectiveBuiltin(
if (Params.MaybeSuffixParam) {
// If we ahve a prefix, validate that it's a good fit for direct data
// embedding (and prepare to append it)
PPEmbedParameterSuffix &SuffixParam = *Params.MaybeSuffixParam;
const PPEmbedParameterSuffix &SuffixParam = *Params.MaybeSuffixParam;
if (!TokenListIsCharacterArray(*this, TargetCharWidth, false,
SuffixParam.Tokens, BinarySuffix)) {
HandleEmbedDirectiveNaive(HashLoc, FilenameTok.getLocation(), Params,
Expand All @@ -4075,9 +4131,25 @@ void Preprocessor::HandleEmbedDirectiveBuiltin(
return PrefixNumber.concat(">");
}(EmbedBufferNumberVal);
std::string &TargetEmbedBuffer = EmbedBuffers.back();
StringRef TypeName = "unsigned char";
const size_t TotalSize =
BinaryPrefix.size() + BinaryContents.size() + BinarySuffix.size();
const size_t ReserveSize = // add up for necessary size:
19 // __builtin_pp_embed(
+ TypeName.size() // type-name
+ 2 // ,"
+ ResolvedFilename.size() // file-name
+ 3 // ","
+ (((TotalSize + 2) / 3) * 4) // base64-string
+ 2 // ");
;
// Reserve appropriate size
TargetEmbedBuffer.reserve(ReserveSize);

// Generate the look-alike source file
TargetEmbedBuffer.append("__builtin_pp_embed(unsigned char,\"");
TargetEmbedBuffer.append("__builtin_pp_embed(");
TargetEmbedBuffer.append(TypeName.data(), TypeName.size());
TargetEmbedBuffer.append(",\"");
TargetEmbedBuffer.append(ResolvedFilename.data(), ResolvedFilename.size());
TargetEmbedBuffer.append("\",\"");
// include the prefix(...) and suffix(...) binary data in the total contents
Expand Down

0 comments on commit dfd6383

Please sign in to comment.