Skip to content

Commit

Permalink
Issue checkstyle#14195: Support Java 21 String Template Syntax (Text …
Browse files Browse the repository at this point in the history 
…Blocks)
  • Loading branch information
@nrmancuso
nrmancuso committed Feb 1, 2024
1 parent e714291 commit 21b0496
Show file tree
Hide file tree
Showing 13 changed files with 764 additions and 37 deletions.
150 changes: 150 additions & 0 deletions src/main/java/com/puppycrawl/tools/checkstyle/JavaAstVisitor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,9 @@ public final class JavaAstVisitor extends JavaLanguageParserBaseVisitor<DetailAs
/** String representation of the double quote character. */
private static final String QUOTE = "\"";

/** String representation of the triple quote character. */
private static final String TRIPLE_QUOTE = "\"\"\"";

/** String representation of the string template embedded expression starting delimiter. */
private static final String EMBEDDED_EXPRESSION_BEGIN = "\\{";

Expand Down Expand Up @@ -1830,6 +1833,153 @@ public DetailAstImpl visitStringTemplate(JavaLanguageParser.StringTemplateContex
return begin;
}

@Override
public DetailAstImpl visitTextBlockTemplate(JavaLanguageParser.TextBlockTemplateContext ctx) {
final DetailAstImpl begin = buildTextBlockTemplateBeginning(ctx);

final Optional<DetailAstImpl> startExpression = Optional.ofNullable(ctx.expr())
.map(this::visit);

if (startExpression.isPresent()) {
final DetailAstImpl imaginaryExpr =
createImaginary(TokenTypes.EMBEDDED_EXPRESSION);
imaginaryExpr.addChild(startExpression.orElseThrow());
begin.addChild(imaginaryExpr);
}

ctx.textBlockTemplateMiddle().stream()
.map(this::buildTextBlockTemplateMiddle)
.collect(Collectors.toUnmodifiableList())
.forEach(begin::addChild);

final DetailAstImpl end = buildTextBlockTemplateEnd(ctx);
begin.addChild(end);
return begin;
}

private static DetailAstImpl buildTextBlockTemplateEnd(
JavaLanguageParser.TextBlockTemplateContext ctx) {

// token looks like '}' StringFragment '"""'
final TerminalNode context = ctx.TEXT_BLOCK_TEMPLATE_END();
final Token token = context.getSymbol();
final String tokenText = context.getText();
final int tokenStartIndex = token.getCharPositionInLine();
final int tokenLineNumber = token.getLine();
final int tokenTextLength = tokenText.length();

final DetailAstImpl embeddedExpressionEnd = createImaginary(
TokenTypes.EMBEDDED_EXPRESSION_END, EMBEDDED_EXPRESSION_END,
tokenLineNumber, tokenStartIndex
);

// remove delimiters '}' and '"'
final String stringFragment = tokenText.substring(
EMBEDDED_EXPRESSION_END.length(),
tokenTextLength - TRIPLE_QUOTE.length()
);

final DetailAstImpl endContent = createImaginary(
TokenTypes.TEXT_BLOCK_TEMPLATE_CONTENT, stringFragment,
tokenLineNumber,
tokenStartIndex + EMBEDDED_EXPRESSION_END.length()
);
embeddedExpressionEnd.addNextSibling(endContent);

final DetailAstImpl stringTemplateEnd = createImaginary(
TokenTypes.STRING_TEMPLATE_END, TRIPLE_QUOTE,
tokenLineNumber,
tokenStartIndex + tokenTextLength - TRIPLE_QUOTE.length()
);
endContent.addNextSibling(stringTemplateEnd);
return embeddedExpressionEnd;
}

private DetailAstImpl buildTextBlockTemplateMiddle(
JavaLanguageParser.TextBlockTemplateMiddleContext middleContext) {

// token looks like '}' TextBlockFragment '\{'
final TerminalNode context = middleContext.TEXT_BLOCK_TEMPLATE_MID();
final Token token = context.getSymbol();
final int tokenStartIndex = token.getCharPositionInLine();
final int tokenLineNumber = token.getLine();
final String tokenText = context.getText();
final int tokenTextLength = tokenText.length();

final DetailAstImpl embeddedExpressionEnd = createImaginary(
TokenTypes.EMBEDDED_EXPRESSION_END, EMBEDDED_EXPRESSION_END,
tokenLineNumber, tokenStartIndex
);

// remove delimiters '}' and '\\' '{'
final String stringFragment = tokenText.substring(
EMBEDDED_EXPRESSION_END.length(),
tokenTextLength - EMBEDDED_EXPRESSION_BEGIN.length()
);

final DetailAstImpl content = createImaginary(
TokenTypes.TEXT_BLOCK_TEMPLATE_CONTENT, stringFragment,
tokenLineNumber, tokenStartIndex + EMBEDDED_EXPRESSION_END.length()
);
embeddedExpressionEnd.addNextSibling(content);

final DetailAstImpl embeddedBegin = createImaginary(
TokenTypes.EMBEDDED_EXPRESSION_BEGIN,
EMBEDDED_EXPRESSION_BEGIN,
tokenLineNumber,
tokenStartIndex + tokenTextLength - EMBEDDED_EXPRESSION_BEGIN.length()
);
content.addNextSibling(embeddedBegin);

final Optional<DetailAstImpl> embeddedExpression = Optional.ofNullable(middleContext.expr())
.map(this::visit);

if (embeddedExpression.isPresent()) {
final DetailAstImpl imaginaryExpr =
createImaginary(TokenTypes.EMBEDDED_EXPRESSION);
imaginaryExpr.addChild(embeddedExpression.orElseThrow());
embeddedExpressionEnd.addNextSibling(imaginaryExpr);
}

return embeddedExpressionEnd;
}

private static DetailAstImpl buildTextBlockTemplateBeginning(
JavaLanguageParser.TextBlockTemplateContext ctx) {

// token looks like '"' StringFragment '\{'
final TerminalNode context = ctx.TEXT_BLOCK_TEMPLATE_BEGIN();
final Token token = context.getSymbol();
final String tokenText = context.getText();
final int tokenStartIndex = token.getCharPositionInLine();
final int tokenLineNumber = token.getLine();
final int tokenTextLength = tokenText.length();

final DetailAstImpl textBlockTemplateBegin = createImaginary(
TokenTypes.TEXT_BLOCK_TEMPLATE_BEGIN, TRIPLE_QUOTE,
tokenLineNumber, tokenStartIndex
);

// remove delimiters '"' and '\{'
final String stringFragment = tokenText.substring(
TRIPLE_QUOTE.length(), tokenTextLength - EMBEDDED_EXPRESSION_BEGIN.length());

final DetailAstImpl stringTemplateContent = createImaginary(
TokenTypes.TEXT_BLOCK_TEMPLATE_CONTENT, stringFragment,
tokenLineNumber, tokenStartIndex + TRIPLE_QUOTE.length()
);
textBlockTemplateBegin.addChild(stringTemplateContent);

final DetailAstImpl embeddedBegin = createImaginary(
TokenTypes.EMBEDDED_EXPRESSION_BEGIN,
EMBEDDED_EXPRESSION_BEGIN,
tokenLineNumber,
tokenStartIndex + tokenTextLength - EMBEDDED_EXPRESSION_BEGIN.length()
);
textBlockTemplateBegin.addChild(embeddedBegin);
return textBlockTemplateBegin;
}

/**
* Builds the beginning of a string template AST.
*
Expand Down
18 changes: 18 additions & 0 deletions src/main/java/com/puppycrawl/tools/checkstyle/api/TokenTypes.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6783,6 +6783,24 @@ public final class TokenTypes {
public static final int UNNAMED_PATTERN_DEF =
JavaLanguageLexer.UNNAMED_PATTERN_DEF;

/**
*
*/
public static final int TEXT_BLOCK_TEMPLATE_BEGIN =
JavaLanguageLexer.TEXT_BLOCK_TEMPLATE_BEGIN;

/**
*
*/
public static final int TEXT_BLOCK_TEMPLATE_END =
JavaLanguageLexer.TEXT_BLOCK_TEMPLATE_END;

/**
*
*/
public static final int TEXT_BLOCK_TEMPLATE_CONTENT =
JavaLanguageLexer.TEXT_BLOCK_TEMPLATE_CONTENT;

/** Prevent instantiation. */
private TokenTypes() {
}
Expand Down
78 changes: 52 additions & 26 deletions src/main/resources/com/puppycrawl/tools/checkstyle/grammar/java/JavaLanguageLexer.g4
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,8 @@ tokens {
STRING_TEMPLATE_CONTENT, EMBEDDED_EXPRESSION_BEGIN, EMBEDDED_EXPRESSION,
EMBEDDED_EXPRESSION_END,
LITERAL_UNDERSCORE, UNNAMED_PATTERN_DEF
LITERAL_UNDERSCORE, UNNAMED_PATTERN_DEF, TEXT_BLOCK_TEMPLATE_BEGIN,
TEXT_BLOCK_TEMPLATE_MID, TEXT_BLOCK_TEMPLATE_END, TEXT_BLOCK_TEMPLATE_CONTENT
}

@header {
Expand Down Expand Up @@ -156,6 +157,8 @@ import com.puppycrawl.tools.checkstyle.grammar.CrAwareLexerSimulator;
int startCol = -1;

private int stringTemplateDepth = 0;

private int textBlockTemplateDepth = 0;
}

// Keywords and restricted identifiers
Expand Down Expand Up @@ -336,6 +339,52 @@ STRING_TEMPLATE_END: {stringTemplateDepth > 0}?
{stringTemplateDepth--;}
;
// Text block Templates
TEXT_BLOCK_TEMPLATE_BEGIN: '"' '"' '"' TextBlockTemplateContent ~'\\' '\\' '{'
{textBlockTemplateDepth++;}
;
// TDO: try to remove ~'\\'from here and above
TEXT_BLOCK_TEMPLATE_MID: {textBlockTemplateDepth > 0}?
'}' TextBlockTemplateContent? ~'\\' '\\' '{'
;
TEXT_BLOCK_TEMPLATE_END: {textBlockTemplateDepth > 0}?
'}' TextBlockTemplateContent? '"' '"' '"'
{textBlockTemplateDepth--;}
;
// Text block fragments
fragment TextBlockTemplateContent
: ( TwoDoubleQuotes
| OneDoubleQuote
| Newline
| Text
| TextBlockStandardEscape
)+
;
fragment Text
: ~[\t\n\r\f"\\]
;
fragment TextBlockStandardEscape
: '\\' [btnfrs"'\\]
;

fragment Newline
: '\n' | '\r' ('\n')?
;

fragment TwoDoubleQuotes
: '"''"' ( Newline | ~'"' )
;

fragment OneDoubleQuote
: '"' ( Newline | ~'"' )
;

// Whitespace and comments

WS: [ \t\r\n\u000C]+ -> skip;
Expand Down Expand Up @@ -427,32 +476,9 @@ fragment Letter
// Text block lexical mode
mode TextBlock;
TEXT_BLOCK_CONTENT
: ( TwoDoubleQuotes
| OneDoubleQuote
| Newline
| ~'"'
| TextBlockStandardEscape
)+
;
TEXT_BLOCK_CONTENT: TextBlockTemplateContent;
TEXT_BLOCK_LITERAL_END
: '"' '"' '"' -> popMode
;
// Text block fragment rules
fragment TextBlockStandardEscape
: '\\' [btnfrs"'\\]
;
fragment Newline
: '\n' | '\r' ('\n')?
;
fragment TwoDoubleQuotes
: '"''"' ( Newline | ~'"' )
;
fragment OneDoubleQuote
: '"' ( Newline | ~'"' )
;
12 changes: 7 additions & 5 deletions src/main/resources/com/puppycrawl/tools/checkstyle/grammar/java/JavaLanguageParser.g4
View file
Original file line number Diff line number Diff line change
Expand Up @@ -760,21 +760,23 @@ primary

templateArgument
: template
| textBlockLiteral
| STRING_LITERAL
;

template
: stringTemplate
;

stringTemplate
: STRING_TEMPLATE_BEGIN expr? stringTemplateMiddle* STRING_TEMPLATE_END
: STRING_TEMPLATE_BEGIN expr? stringTemplateMiddle* STRING_TEMPLATE_END #stringTemplate
| TEXT_BLOCK_TEMPLATE_BEGIN expr? textBlockTemplateMiddle* TEXT_BLOCK_TEMPLATE_END #textBlockTemplate
;

stringTemplateMiddle
: STRING_TEMPLATE_MID expr?
;

textBlockTemplateMiddle
: TEXT_BLOCK_TEMPLATE_MID expr?
;

classType
: (classOrInterfaceType[false] DOT)? annotations[false] id typeArguments?
;
Expand Down
5 changes: 3 additions & 2 deletions src/test/java/com/puppycrawl/tools/checkstyle/checks/coding/IllegalTokenTextCheckTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,7 @@ public void testOrderOfProperties() {

@Test
public void testAcceptableTokensMakeSense() {
final int expectedTokenTypesTotalNumber = 195;
final int expectedTokenTypesTotalNumber = 198;
assertWithMessage("Total number of TokenTypes has changed, acceptable tokens in"
+ " IllegalTokenTextCheck need to be reconsidered.")
.that(TokenUtil.getTokenTypesTotalNumber())
Expand All @@ -197,7 +197,8 @@ public void testAcceptableTokensMakeSense() {
TokenTypes.STRING_LITERAL,
TokenTypes.CHAR_LITERAL,
TokenTypes.TEXT_BLOCK_CONTENT,
TokenTypes.STRING_TEMPLATE_CONTENT
TokenTypes.STRING_TEMPLATE_CONTENT,
TokenTypes.TEXT_BLOCK_TEMPLATE_CONTENT
);
for (int tokenType : allowedTokens) {
assertWithMessage(TokenUtil.getTokenName(tokenType) + " should not be allowed"
Expand Down
14 changes: 13 additions & 1 deletion src/test/java/com/puppycrawl/tools/checkstyle/grammar/GeneratedJavaTokenTypesTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -742,6 +742,18 @@ public void testTokenNumbering() {
assertWithMessage(message)
.that(JavaLanguageLexer.UNNAMED_PATTERN_DEF)
.isEqualTo(225);
assertWithMessage(message)
.that(JavaLanguageLexer.TEXT_BLOCK_TEMPLATE_BEGIN)
.isEqualTo(226);
assertWithMessage(message)
.that(JavaLanguageLexer.TEXT_BLOCK_TEMPLATE_MID)
.isEqualTo(227);
assertWithMessage(message)
.that(JavaLanguageLexer.TEXT_BLOCK_TEMPLATE_END)
.isEqualTo(228);
assertWithMessage(message)
.that(JavaLanguageLexer.TEXT_BLOCK_TEMPLATE_CONTENT)
.isEqualTo(229);

final int tokenCount = (int) Arrays.stream(JavaLanguageLexer.class.getDeclaredFields())
.filter(GeneratedJavaTokenTypesTest::isPublicStaticFinalInt)
Expand All @@ -752,7 +764,7 @@ public void testTokenNumbering() {
+ " 'GeneratedJavaTokenTypesTest' and verified"
+ " that their old numbering didn't change")
.that(tokenCount)
.isEqualTo(237);
.isEqualTo(241);
}

/**
Expand Down
Loading

0 comments on commit 21b0496

Please sign in to comment.