Skip to content

Commit

Permalink
[clang-format] Support python-style comments in text protos
Browse files Browse the repository at this point in the history 
Summary: This patch adds support for python-style comments in text protos.

Reviewers: djasper

Reviewed By: djasper

Subscribers: bkramer, cfe-commits, klimek

Differential Revision: https://reviews.llvm.org/D39806

llvm-svn: 317886
  • Loading branch information
@krasimirgg
krasimirgg committed Nov 10, 2017
1 parent a9d58fa commit 410ed24
Show file tree
Hide file tree
Showing 5 changed files with 128 additions and 7 deletions.
23 changes: 17 additions & 6 deletions clang/lib/Format/BreakableToken.cpp
View file
Expand Up @@ -40,9 +40,15 @@ static bool IsBlank(char C) {
}
}

static StringRef getLineCommentIndentPrefix(StringRef Comment) {
static const char *const KnownPrefixes[] = {"///<", "//!<", "///", "//",
"//!"};
static StringRef getLineCommentIndentPrefix(StringRef Comment,
const FormatStyle &Style) {
static const char *const KnownCStylePrefixes[] = {"///<", "//!<", "///", "//",
"//!"};
static const char *const KnownTextProtoPrefixes[] = {"//", "#"};
ArrayRef<const char *> KnownPrefixes(KnownCStylePrefixes);
if (Style.Language == FormatStyle::LK_TextProto)
KnownPrefixes = KnownTextProtoPrefixes;

StringRef LongestPrefix;
for (StringRef KnownPrefix : KnownPrefixes) {
if (Comment.startswith(KnownPrefix)) {
Expand Down Expand Up @@ -732,7 +738,8 @@ BreakableLineCommentSection::BreakableLineCommentSection(
CurrentTok = CurrentTok->Next) {
LastLineTok = LineTok;
StringRef TokenText(CurrentTok->TokenText);
assert(TokenText.startswith("//"));
assert((TokenText.startswith("//") || TokenText.startswith("#")) &&
"unsupported line comment prefix, '//' and '#' are supported");
size_t FirstLineIndex = Lines.size();
TokenText.split(Lines, "\n");
Content.resize(Lines.size());
Expand All @@ -745,8 +752,9 @@ BreakableLineCommentSection::BreakableLineCommentSection(
// We need to trim the blanks in case this is not the first line in a
// multiline comment. Then the indent is included in Lines[i].
StringRef IndentPrefix =
getLineCommentIndentPrefix(Lines[i].ltrim(Blanks));
assert(IndentPrefix.startswith("//"));
getLineCommentIndentPrefix(Lines[i].ltrim(Blanks), Style);
assert((TokenText.startswith("//") || TokenText.startswith("#")) &&
"unsupported line comment prefix, '//' and '#' are supported");
OriginalPrefix[i] = Prefix[i] = IndentPrefix;
if (Lines[i].size() > Prefix[i].size() &&
isAlphanumeric(Lines[i][Prefix[i].size()])) {
Expand All @@ -760,6 +768,9 @@ BreakableLineCommentSection::BreakableLineCommentSection(
Prefix[i] = "///< ";
else if (Prefix[i] == "//!<")
Prefix[i] = "//!< ";
else if (Prefix[i] == "#" &&
Style.Language == FormatStyle::LK_TextProto)
Prefix[i] = "# ";
}

Tokens[i] = LineTok;
Expand Down
23 changes: 23 additions & 0 deletions clang/lib/Format/FormatTokenLexer.cpp
View file
Expand Up @@ -50,6 +50,8 @@ ArrayRef<FormatToken *> FormatTokenLexer::lex() {
tryParseJSRegexLiteral();
handleTemplateStrings();
}
if (Style.Language == FormatStyle::LK_TextProto)
tryParsePythonComment();
tryMergePreviousTokens();
if (Tokens.back()->NewlinesBefore > 0 || Tokens.back()->IsMultiline)
FirstInLineIndex = Tokens.size() - 1;
Expand Down Expand Up @@ -330,6 +332,27 @@ void FormatTokenLexer::handleTemplateStrings() {
resetLexer(SourceMgr.getFileOffset(loc));
}

void FormatTokenLexer::tryParsePythonComment() {
FormatToken *HashToken = Tokens.back();
if (HashToken->isNot(tok::hash))
return;
// Turn the remainder of this line into a comment.
const char *CommentBegin =
Lex->getBufferLocation() - HashToken->TokenText.size(); // at "#"
size_t From = CommentBegin - Lex->getBuffer().begin();
size_t To = Lex->getBuffer().find_first_of('\n', From);
if (To == StringRef::npos)
To = Lex->getBuffer().size();
size_t Len = To - From;
HashToken->Type = TT_LineComment;
HashToken->Tok.setKind(tok::comment);
HashToken->TokenText = Lex->getBuffer().substr(From, Len);
SourceLocation Loc = To < Lex->getBuffer().size()
? Lex->getSourceLocation(CommentBegin + Len)
: SourceMgr.getLocForEndOfFile(ID);
resetLexer(SourceMgr.getFileOffset(Loc));
}

bool FormatTokenLexer::tryMerge_TMacro() {
if (Tokens.size() < 4)
return false;
Expand Down
2 changes: 2 additions & 0 deletions clang/lib/Format/FormatTokenLexer.h
View file
Expand Up @@ -73,6 +73,8 @@ class FormatTokenLexer {
// nested template parts by balancing curly braces.
void handleTemplateStrings();

void tryParsePythonComment();

bool tryMerge_TMacro();

bool tryMergeConflictMarkers();
Expand Down
2 changes: 1 addition & 1 deletion clang/lib/Format/UnwrappedLineParser.cpp
View file
Expand Up @@ -56,7 +56,7 @@ class ScopedDeclarationState {
};

static bool isLineComment(const FormatToken &FormatTok) {
return FormatTok.is(tok::comment) && FormatTok.TokenText.startswith("//");
return FormatTok.is(tok::comment) && !FormatTok.TokenText.startswith("/*");
}

// Checks if \p FormatTok is a line comment that continues the line comment
Expand Down
85 changes: 85 additions & 0 deletions clang/unittests/Format/FormatTestComments.cpp
View file
Expand Up @@ -62,6 +62,12 @@ class FormatTestComments : public ::testing::Test {
return Style;
}

FormatStyle getTextProtoStyleWithColumns(unsigned ColumnLimit) {
FormatStyle Style = getGoogleStyle(FormatStyle::FormatStyle::LK_TextProto);
Style.ColumnLimit = ColumnLimit;
return Style;
}

void verifyFormat(llvm::StringRef Code,
const FormatStyle &Style = getLLVMStyle()) {
EXPECT_EQ(Code.str(), format(test::messUp(Code), Style));
Expand Down Expand Up @@ -2872,6 +2878,85 @@ TEST_F(FormatTestComments, NonTrailingBlockComments) {
" A = B;",
getLLVMStyleWithColumns(40)));
}

TEST_F(FormatTestComments, PythonStyleComments) {
// Keeps a space after '#'.
EXPECT_EQ("# comment\n"
"key: value",
format("#comment\n"
"key:value",
getTextProtoStyleWithColumns(20)));
EXPECT_EQ("# comment\n"
"key: value",
format("# comment\n"
"key:value",
getTextProtoStyleWithColumns(20)));
// Breaks long comment.
EXPECT_EQ("# comment comment\n"
"# comment\n"
"key: value",
format("# comment comment comment\n"
"key:value",
getTextProtoStyleWithColumns(20)));
// Indents comments.
EXPECT_EQ("data {\n"
" # comment comment\n"
" # comment\n"
" key: value\n"
"}",
format("data {\n"
"# comment comment comment\n"
"key: value}",
getTextProtoStyleWithColumns(20)));
EXPECT_EQ("data {\n"
" # comment comment\n"
" # comment\n"
" key: value\n"
"}",
format("data {# comment comment comment\n"
"key: value}",
getTextProtoStyleWithColumns(20)));
// Reflows long comments.
EXPECT_EQ("# comment comment\n"
"# comment comment\n"
"key: value",
format("# comment comment comment\n"
"# comment\n"
"key:value",
getTextProtoStyleWithColumns(20)));
// Breaks trailing comments.
EXPECT_EQ("k: val # comment\n"
" # comment\n"
"a: 1",
format("k:val#comment comment\n"
"a:1",
getTextProtoStyleWithColumns(20)));
EXPECT_EQ("id {\n"
" k: val # comment\n"
" # comment\n"
" # line line\n"
" a: 1\n"
"}",
format("id {k:val#comment comment\n"
"# line line\n"
"a:1}",
getTextProtoStyleWithColumns(20)));
// Aligns trailing comments.
EXPECT_EQ("k: val # commen1\n"
" # commen2\n"
" # commen3\n"
"# commen4\n"
"a: 1 # commen5\n"
" # commen6\n"
" # commen7",
format("k:val#commen1 commen2\n"
" # commen3\n"
"# commen4\n"
"a:1#commen5 commen6\n"
" #commen7",
getTextProtoStyleWithColumns(20)));
}

} // end namespace
} // end namespace format
} // end namespace clang

0 comments on commit 410ed24

Please sign in to comment.