Skip to content

Commit

Permalink
Issue checkstyle#14195: Support Java 21 String Template Syntax (Text …
Browse files Browse the repository at this point in the history 
…Blocks)
  • Loading branch information
@nrmancuso
nrmancuso committed Feb 8, 2024
1 parent c3104f1 commit c497d9a
Show file tree
Hide file tree
Showing 15 changed files with 813 additions and 49 deletions.
8 changes: 0 additions & 8 deletions config/projects-to-test/openjdk21-excluded.files
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,14 +29,6 @@
<property name="fileNamePattern" value="[\\/]test[\\/]langtools[\\/]tools[\\/]javac[\\/]unicode[\\/]FirstChar2.java$"/>
</module>

<!-- until https://github.com/checkstyle/checkstyle/issues/14195 (text block templates) -->
<module name="BeforeExecutionExclusionFileFilter">
<property name="fileNamePattern" value="[\\/]test[\\/]jdk[\\/]java[\\/]lang[\\/]template[\\/]StringTemplateTest.java$"/>
</module>
<module name="BeforeExecutionExclusionFileFilter">
<property name="fileNamePattern" value="[\\/]test[\\/]jdk[\\/]java[\\/]lang[\\/]template[\\/]Basic.java$"/>
</module>

<!-- until https://github.com/checkstyle/checkstyle/issues/13987 -->
<module name="BeforeExecutionExclusionFileFilter">
<property name="fileNamePattern" value="[\\/]test[\\/]langtools[\\/]tools[\\/]javac[\\/]diags[\\/]examples[\\/]UnnamedClass.java$"/>
Expand Down
150 changes: 150 additions & 0 deletions src/main/java/com/puppycrawl/tools/checkstyle/JavaAstVisitor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,9 @@ public final class JavaAstVisitor extends JavaLanguageParserBaseVisitor<DetailAs
/** String representation of the double quote character. */
private static final String QUOTE = "\"";

/** String representation of the triple quote character. */
private static final String TRIPLE_QUOTE = "\"\"\"";

/** String representation of the string template embedded expression starting delimiter. */
private static final String EMBEDDED_EXPRESSION_BEGIN = "\\{";

Expand Down Expand Up @@ -1830,6 +1833,153 @@ public DetailAstImpl visitStringTemplate(JavaLanguageParser.StringTemplateContex
return begin;
}

@Override
public DetailAstImpl visitTextBlockTemplate(JavaLanguageParser.TextBlockTemplateContext ctx) {
final DetailAstImpl begin = buildTextBlockTemplateBeginning(ctx);

final Optional<DetailAstImpl> startExpression = Optional.ofNullable(ctx.expr())
.map(this::visit);

if (startExpression.isPresent()) {
final DetailAstImpl imaginaryExpr =
createImaginary(TokenTypes.EMBEDDED_EXPRESSION);
imaginaryExpr.addChild(startExpression.orElseThrow());
begin.addChild(imaginaryExpr);
}

ctx.textBlockTemplateMiddle().stream()
.map(this::buildTextBlockTemplateMiddle)
.collect(Collectors.toUnmodifiableList())
.forEach(begin::addChild);

final DetailAstImpl end = buildTextBlockTemplateEnd(ctx);
begin.addChild(end);
return begin;
}

private static DetailAstImpl buildTextBlockTemplateEnd(
JavaLanguageParser.TextBlockTemplateContext ctx) {

// token looks like '}' StringFragment '"""'
final TerminalNode context = ctx.TEXT_BLOCK_TEMPLATE_END();
final Token token = context.getSymbol();
final String tokenText = context.getText();
final int tokenStartIndex = token.getCharPositionInLine();
final int tokenLineNumber = token.getLine();
final int tokenTextLength = tokenText.length();

final DetailAstImpl embeddedExpressionEnd = createImaginary(
TokenTypes.EMBEDDED_EXPRESSION_END, EMBEDDED_EXPRESSION_END,
tokenLineNumber, tokenStartIndex
);

// remove delimiters '}' and '"'
final String stringFragment = tokenText.substring(
EMBEDDED_EXPRESSION_END.length(),
tokenTextLength - TRIPLE_QUOTE.length()
);

final DetailAstImpl endContent = createImaginary(
TokenTypes.TEXT_BLOCK_TEMPLATE_CONTENT, stringFragment,
tokenLineNumber,
tokenStartIndex + EMBEDDED_EXPRESSION_END.length()
);
embeddedExpressionEnd.addNextSibling(endContent);

final DetailAstImpl stringTemplateEnd = createImaginary(
TokenTypes.STRING_TEMPLATE_END, TRIPLE_QUOTE,
tokenLineNumber,
tokenStartIndex + tokenTextLength - TRIPLE_QUOTE.length()
);
endContent.addNextSibling(stringTemplateEnd);
return embeddedExpressionEnd;
}

private DetailAstImpl buildTextBlockTemplateMiddle(
JavaLanguageParser.TextBlockTemplateMiddleContext middleContext) {

// token looks like '}' TextBlockFragment '\{'
final Token token = middleContext.middle;
final int tokenStartIndex = token.getCharPositionInLine();
final int tokenLineNumber = token.getLine();
final String tokenText = token.getText();
final int tokenTextLength = tokenText.length();

final DetailAstImpl embeddedExpressionEnd = createImaginary(
TokenTypes.EMBEDDED_EXPRESSION_END, EMBEDDED_EXPRESSION_END,
tokenLineNumber, tokenStartIndex
);

// remove delimiters '}' and '\\' '{'
final String stringFragment = tokenText.substring(
EMBEDDED_EXPRESSION_END.length(),
tokenTextLength - EMBEDDED_EXPRESSION_BEGIN.length()
);

final DetailAstImpl content = createImaginary(
TokenTypes.TEXT_BLOCK_TEMPLATE_CONTENT, stringFragment,
tokenLineNumber, tokenStartIndex + EMBEDDED_EXPRESSION_END.length()
);
embeddedExpressionEnd.addNextSibling(content);

final DetailAstImpl embeddedBegin = createImaginary(
TokenTypes.EMBEDDED_EXPRESSION_BEGIN,
EMBEDDED_EXPRESSION_BEGIN,
tokenLineNumber,
tokenStartIndex + tokenTextLength - EMBEDDED_EXPRESSION_BEGIN.length()
);
content.addNextSibling(embeddedBegin);

final Optional<DetailAstImpl> embeddedExpression =
Optional.ofNullable(middleContext.getChild(1))
.map(this::visit);

if (embeddedExpression.isPresent()) {
final DetailAstImpl imaginaryExpr =
createImaginary(TokenTypes.EMBEDDED_EXPRESSION);
imaginaryExpr.addChild(embeddedExpression.orElseThrow());
embeddedExpressionEnd.addNextSibling(imaginaryExpr);
}

return embeddedExpressionEnd;
}

private static DetailAstImpl buildTextBlockTemplateBeginning(
JavaLanguageParser.TextBlockTemplateContext ctx) {

// token looks like '"' StringFragment '\{'
final TerminalNode context = ctx.TEXT_BLOCK_TEMPLATE_BEGIN();
final Token token = context.getSymbol();
final String tokenText = context.getText();
final int tokenStartIndex = token.getCharPositionInLine();
final int tokenLineNumber = token.getLine();
final int tokenTextLength = tokenText.length();

final DetailAstImpl textBlockTemplateBegin = createImaginary(
TokenTypes.TEXT_BLOCK_TEMPLATE_BEGIN, TRIPLE_QUOTE,
tokenLineNumber, tokenStartIndex
);

// remove delimiters '"' and '\{'
final String stringFragment = tokenText.substring(
TRIPLE_QUOTE.length(), tokenTextLength - EMBEDDED_EXPRESSION_BEGIN.length());

final DetailAstImpl stringTemplateContent = createImaginary(
TokenTypes.TEXT_BLOCK_TEMPLATE_CONTENT, stringFragment,
tokenLineNumber, tokenStartIndex + TRIPLE_QUOTE.length()
);
textBlockTemplateBegin.addChild(stringTemplateContent);

final DetailAstImpl embeddedBegin = createImaginary(
TokenTypes.EMBEDDED_EXPRESSION_BEGIN,
EMBEDDED_EXPRESSION_BEGIN,
tokenLineNumber,
tokenStartIndex + tokenTextLength - EMBEDDED_EXPRESSION_BEGIN.length()
);
textBlockTemplateBegin.addChild(embeddedBegin);
return textBlockTemplateBegin;
}

/**
* Builds the beginning of a string template AST.
*
Expand Down
18 changes: 18 additions & 0 deletions src/main/java/com/puppycrawl/tools/checkstyle/api/TokenTypes.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6783,6 +6783,24 @@ public final class TokenTypes {
public static final int UNNAMED_PATTERN_DEF =
JavaLanguageLexer.UNNAMED_PATTERN_DEF;

/**
*
*/
public static final int TEXT_BLOCK_TEMPLATE_BEGIN =
JavaLanguageLexer.TEXT_BLOCK_TEMPLATE_BEGIN;

/**
*
*/
public static final int TEXT_BLOCK_TEMPLATE_END =
JavaLanguageLexer.TEXT_BLOCK_TEMPLATE_END;

/**
*
*/
public static final int TEXT_BLOCK_TEMPLATE_CONTENT =
JavaLanguageLexer.TEXT_BLOCK_TEMPLATE_CONTENT;

/** Prevent instantiation. */
private TokenTypes() {
}
Expand Down
95 changes: 67 additions & 28 deletions src/main/resources/com/puppycrawl/tools/checkstyle/grammar/java/JavaLanguageLexer.g4
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,8 @@ tokens {
STRING_TEMPLATE_CONTENT, EMBEDDED_EXPRESSION_BEGIN, EMBEDDED_EXPRESSION,
EMBEDDED_EXPRESSION_END,
LITERAL_UNDERSCORE, UNNAMED_PATTERN_DEF
LITERAL_UNDERSCORE, UNNAMED_PATTERN_DEF, TEXT_BLOCK_TEMPLATE_BEGIN,
TEXT_BLOCK_TEMPLATE_MID, TEXT_BLOCK_TEMPLATE_END, TEXT_BLOCK_TEMPLATE_CONTENT
}

@header {
Expand Down Expand Up @@ -156,6 +157,8 @@ import com.puppycrawl.tools.checkstyle.grammar.CrAwareLexerSimulator;
int startCol = -1;

private int stringTemplateDepth = 0;

private int textBlockTemplateDepth = 0;
}

// Keywords and restricted identifiers
Expand Down Expand Up @@ -321,11 +324,12 @@ AT: '@';
ELLIPSIS: '...';
// String templates
STRING_TEMPLATE_BEGIN: '"' StringFragment '\\' '{'
STRING_TEMPLATE_BEGIN: '"' StringFragment '\\' '{'
{stringTemplateDepth++;}
;
STRING_TEMPLATE_MID: {stringTemplateDepth > 0}?
// this is also used for text block template middles that have no content
STRING_TEMPLATE_MID: {stringTemplateDepth > 0 || textBlockTemplateDepth > 0}?
'}' StringFragment '\\' '{'
;
Expand All @@ -334,6 +338,64 @@ STRING_TEMPLATE_END: {stringTemplateDepth > 0}?
{stringTemplateDepth--;}
;
// Text Block Templates
TEXT_BLOCK_TEMPLATE_BEGIN: '"' '"' '"' TextBlockContent ~'\\' '\\' '{'
{textBlockTemplateDepth++;}
;
// We do not make TextBlockTemplateContent optional here, because this token
// would then match the STRING_TEMPLATE_MID token above when there is no content.
//
// Example:
// String s = STR."""
// \{}\{}""";
// ^ this (`}\{`) is matched by STRING_TEMPLATE_MID
//
// In order to make this token work for these lexemes, we would need to add
// more semantic predicates to both tokens,
// which would be more complex than just reusing the STRING_TEMPLATE_MID token.
TEXT_BLOCK_TEMPLATE_MID: {textBlockTemplateDepth > 0}?
'}' TextBlockContent ~'\\' '\\' '{'
;

TEXT_BLOCK_TEMPLATE_END: {textBlockTemplateDepth > 0}?
'}' TextBlockContent? '"' '"' '"'
{textBlockTemplateDepth--;}
;

// Text block fragments

fragment TextBlockContent
: ( TwoDoubleQuotes
| OneDoubleQuote
| Newline
| TextBlockCharacter
)+
;

fragment TextBlockCharacter
: ~["\\]
| TextBlockStandardEscape
| EscapeSequence
;
fragment TextBlockStandardEscape
: '\\' ( [btnfrs"'\\] | Newline | OneDoubleQuote )
;
fragment Newline
: '\n' | '\r' ('\n')?
;
fragment TwoDoubleQuotes
: '"''"' ( Newline | ~'"' )
;
fragment OneDoubleQuote
: '"' ( Newline | ~'"' )
;
// Whitespace and comments
WS: [ \t\r\n\u000C]+ -> skip;
Expand Down Expand Up @@ -425,32 +487,9 @@ fragment Letter
// Text block lexical mode
mode TextBlock;
TEXT_BLOCK_CONTENT
: ( TwoDoubleQuotes
| OneDoubleQuote
| Newline
| ~'"'
| TextBlockStandardEscape
)+
;
TEXT_BLOCK_CONTENT: TextBlockContent;
TEXT_BLOCK_LITERAL_END
: '"' '"' '"' -> popMode
;
// Text block fragment rules
fragment TextBlockStandardEscape
: '\\' [btnfrs"'\\]
;
fragment Newline
: '\n' | '\r' ('\n')?
;
fragment TwoDoubleQuotes
: '"''"' ( Newline | ~'"' )
;
fragment OneDoubleQuote
: '"' ( Newline | ~'"' )
;
18 changes: 14 additions & 4 deletions src/main/resources/com/puppycrawl/tools/checkstyle/grammar/java/JavaLanguageParser.g4
View file
Original file line number Diff line number Diff line change
Expand Up @@ -760,21 +760,31 @@ primary

templateArgument
: template
| textBlockLiteral
| STRING_LITERAL
;

template
: stringTemplate
;
: STRING_TEMPLATE_BEGIN
expr? stringTemplateMiddle*
STRING_TEMPLATE_END #stringTemplate

stringTemplate
: STRING_TEMPLATE_BEGIN expr? stringTemplateMiddle* STRING_TEMPLATE_END
| TEXT_BLOCK_TEMPLATE_BEGIN
expr? textBlockTemplateMiddle*
TEXT_BLOCK_TEMPLATE_END #textBlockTemplate
;

// 'isEmptyTextBlockMiddle' is used to distinguish between empty string and empty text block
// middle. This is necessary because the lexer cannot distinguish between the two.
stringTemplateMiddle
: STRING_TEMPLATE_MID expr?
;

textBlockTemplateMiddle
: middle=STRING_TEMPLATE_MID expr?
| middle=TEXT_BLOCK_TEMPLATE_MID expr?
;

classType
: (classOrInterfaceType[false] DOT)? annotations[false] id typeArguments?
;
Expand Down
5 changes: 2 additions & 3 deletions src/test/java/com/puppycrawl/tools/checkstyle/JavaAstVisitorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,10 +88,9 @@ public class JavaAstVisitorTest extends AbstractModuleTestSupport {
"visitQualifiedNameExtended",
"visitGuard",

// until https://github.com/checkstyle/checkstyle/issues/14195
"visitTemplate",
// handled as a list in the parent rule
"visitStringTemplateMiddle"
"visitStringTemplateMiddle",
"visitTextBlockTemplateMiddle"
);

@Override
Expand Down
Loading

0 comments on commit c497d9a

Please sign in to comment.