Issue checkstyle#14195: Support Java 21 String Template Syntax (Text …

…Blocks)
nrmancuso · Mar 4, 2024 · d2fb18b · d2fb18b
1 parent 5d5f5b2
commit d2fb18b
Show file tree

Hide file tree

Showing 15 changed files with 729 additions and 27 deletions.
diff --git a/config/projects-to-test/openjdk21-excluded.files b/config/projects-to-test/openjdk21-excluded.files
@@ -29,14 +29,6 @@
   <property name="fileNamePattern" value="[\\/]test[\\/]langtools[\\/]tools[\\/]javac[\\/]unicode[\\/]FirstChar2.java$"/>
 </module>
 
-<!-- until https://github.com/checkstyle/checkstyle/issues/14195 (text block templates) -->
-<module name="BeforeExecutionExclusionFileFilter">
-  <property name="fileNamePattern" value="[\\/]test[\\/]jdk[\\/]java[\\/]lang[\\/]template[\\/]StringTemplateTest.java$"/>
-</module>
-<module name="BeforeExecutionExclusionFileFilter">
-  <property name="fileNamePattern" value="[\\/]test[\\/]jdk[\\/]java[\\/]lang[\\/]template[\\/]Basic.java$"/>
-</module>
-
 <!-- until https://github.com/checkstyle/checkstyle/issues/13987 -->
 <module name="BeforeExecutionExclusionFileFilter">
   <property name="fileNamePattern" value="[\\/]test[\\/]langtools[\\/]tools[\\/]javac[\\/]diags[\\/]examples[\\/]UnnamedClass.java$"/>

diff --git a/src/main/java/com/puppycrawl/tools/checkstyle/JavaAstVisitor.java b/src/main/java/com/puppycrawl/tools/checkstyle/JavaAstVisitor.java
@@ -107,6 +107,9 @@ public final class JavaAstVisitor extends JavaLanguageParserBaseVisitor<DetailAs
     /** String representation of the double quote character. */
     private static final String QUOTE = "\"";
 
+    /** String representation of the triple quote character. */
+    private static final String TRIPLE_QUOTE = "\"\"\"";
+
     /** String representation of the string template embedded expression starting delimiter. */
     private static final String EMBEDDED_EXPRESSION_BEGIN = "\\{";
 
@@ -1830,6 +1833,153 @@ public DetailAstImpl visitStringTemplate(JavaLanguageParser.StringTemplateContex
         return begin;
     }
 
+    @Override
+    public DetailAstImpl visitTextBlockTemplate(JavaLanguageParser.TextBlockTemplateContext ctx) {
+        final DetailAstImpl begin = buildTextBlockTemplateBeginning(ctx);
+
+        final Optional<DetailAstImpl> startExpression = Optional.ofNullable(ctx.expr())
+            .map(this::visit);
+
+        if (startExpression.isPresent()) {
+            final DetailAstImpl imaginaryExpr =
+                    createImaginary(TokenTypes.EMBEDDED_EXPRESSION);
+            imaginaryExpr.addChild(startExpression.orElseThrow());
+            begin.addChild(imaginaryExpr);
+        }
+
+        ctx.textBlockTemplateMiddle().stream()
+                .map(this::buildTextBlockTemplateMiddle)
+                .collect(Collectors.toUnmodifiableList())
+                .forEach(begin::addChild);
+
+        final DetailAstImpl end = buildTextBlockTemplateEnd(ctx);
+        begin.addChild(end);
+        return begin;
+    }
+
+    private static DetailAstImpl buildTextBlockTemplateEnd(
+            JavaLanguageParser.TextBlockTemplateContext ctx) {
+
+        // token looks like '}' StringFragment '"""'
+        final TerminalNode context = ctx.TEXT_BLOCK_TEMPLATE_END();
+        final Token token = context.getSymbol();
+        final String tokenText = context.getText();
+        final int tokenStartIndex = token.getCharPositionInLine();
+        final int tokenLineNumber = token.getLine();
+        final int tokenTextLength = tokenText.length();
+
+        final DetailAstImpl embeddedExpressionEnd = createImaginary(
+                TokenTypes.EMBEDDED_EXPRESSION_END, EMBEDDED_EXPRESSION_END,
+                tokenLineNumber, tokenStartIndex
+        );
+
+        // remove delimiters '}' and '"'
+        final String stringFragment = tokenText.substring(
+                EMBEDDED_EXPRESSION_END.length(),
+                tokenTextLength - TRIPLE_QUOTE.length()
+        );
+
+        final DetailAstImpl endContent = createImaginary(
+                TokenTypes.TEXT_BLOCK_TEMPLATE_CONTENT, stringFragment,
+                tokenLineNumber,
+                tokenStartIndex + EMBEDDED_EXPRESSION_END.length()
+        );
+        embeddedExpressionEnd.addNextSibling(endContent);
+
+        final DetailAstImpl stringTemplateEnd = createImaginary(
+                TokenTypes.STRING_TEMPLATE_END, TRIPLE_QUOTE,
+                tokenLineNumber,
+                tokenStartIndex + tokenTextLength - TRIPLE_QUOTE.length()
+        );
+        endContent.addNextSibling(stringTemplateEnd);
+        return embeddedExpressionEnd;
+    }
+
+    private DetailAstImpl buildTextBlockTemplateMiddle(
+            JavaLanguageParser.TextBlockTemplateMiddleContext middleContext) {
+
+        // token looks like '}' TextBlockFragment '\{'
+        final Token token = middleContext.middle;
+        final int tokenStartIndex = token.getCharPositionInLine();
+        final int tokenLineNumber = token.getLine();
+        final String tokenText = token.getText();
+        final int tokenTextLength = tokenText.length();
+
+        final DetailAstImpl embeddedExpressionEnd = createImaginary(
+                TokenTypes.EMBEDDED_EXPRESSION_END, EMBEDDED_EXPRESSION_END,
+                tokenLineNumber, tokenStartIndex
+        );
+
+        // remove delimiters '}' and '\\' '{'
+        final String stringFragment = tokenText.substring(
+                EMBEDDED_EXPRESSION_END.length(),
+                tokenTextLength - EMBEDDED_EXPRESSION_BEGIN.length()
+        );
+
+        final DetailAstImpl content = createImaginary(
+                TokenTypes.TEXT_BLOCK_TEMPLATE_CONTENT, stringFragment,
+                tokenLineNumber, tokenStartIndex + EMBEDDED_EXPRESSION_END.length()
+        );
+        embeddedExpressionEnd.addNextSibling(content);
+
+        final DetailAstImpl embeddedBegin = createImaginary(
+                TokenTypes.EMBEDDED_EXPRESSION_BEGIN,
+                EMBEDDED_EXPRESSION_BEGIN,
+                tokenLineNumber,
+                tokenStartIndex + tokenTextLength - EMBEDDED_EXPRESSION_BEGIN.length()
+        );
+        content.addNextSibling(embeddedBegin);
+
+        final Optional<DetailAstImpl> embeddedExpression =
+                Optional.ofNullable(middleContext.getChild(1))
+                .map(this::visit);
+
+        if (embeddedExpression.isPresent()) {
+            final DetailAstImpl imaginaryExpr =
+                    createImaginary(TokenTypes.EMBEDDED_EXPRESSION);
+            imaginaryExpr.addChild(embeddedExpression.orElseThrow());
+            embeddedExpressionEnd.addNextSibling(imaginaryExpr);
+        }
+
+        return embeddedExpressionEnd;
+    }
+
+    private static DetailAstImpl buildTextBlockTemplateBeginning(
+            JavaLanguageParser.TextBlockTemplateContext ctx) {
+
+        // token looks like '"' StringFragment '\{'
+        final TerminalNode context = ctx.TEXT_BLOCK_TEMPLATE_BEGIN();
+        final Token token = context.getSymbol();
+        final String tokenText = context.getText();
+        final int tokenStartIndex = token.getCharPositionInLine();
+        final int tokenLineNumber = token.getLine();
+        final int tokenTextLength = tokenText.length();
+
+        final DetailAstImpl textBlockTemplateBegin = createImaginary(
+                TokenTypes.TEXT_BLOCK_TEMPLATE_BEGIN, TRIPLE_QUOTE,
+                tokenLineNumber, tokenStartIndex
+        );
+
+        // remove delimiters '"' and '\{'
+        final String stringFragment = tokenText.substring(
+                TRIPLE_QUOTE.length(), tokenTextLength - EMBEDDED_EXPRESSION_BEGIN.length());
+
+        final DetailAstImpl stringTemplateContent = createImaginary(
+                TokenTypes.TEXT_BLOCK_TEMPLATE_CONTENT, stringFragment,
+                tokenLineNumber, tokenStartIndex + TRIPLE_QUOTE.length()
+        );
+        textBlockTemplateBegin.addChild(stringTemplateContent);
+
+        final DetailAstImpl embeddedBegin = createImaginary(
+                TokenTypes.EMBEDDED_EXPRESSION_BEGIN,
+                EMBEDDED_EXPRESSION_BEGIN,
+                tokenLineNumber,
+                tokenStartIndex + tokenTextLength - EMBEDDED_EXPRESSION_BEGIN.length()
+        );
+        textBlockTemplateBegin.addChild(embeddedBegin);
+        return textBlockTemplateBegin;
+    }
+
     /**
      * Builds the beginning of a string template AST.
      *

diff --git a/src/main/java/com/puppycrawl/tools/checkstyle/api/TokenTypes.java b/src/main/java/com/puppycrawl/tools/checkstyle/api/TokenTypes.java
@@ -6786,6 +6786,24 @@ public final class TokenTypes {
     public static final int UNNAMED_PATTERN_DEF =
             JavaLanguageLexer.UNNAMED_PATTERN_DEF;
 
+    /**
+     *
+     */
+    public static final int TEXT_BLOCK_TEMPLATE_BEGIN =
+            JavaLanguageLexer.TEXT_BLOCK_TEMPLATE_BEGIN;
+
+    /**
+     *
+     */
+    public static final int TEXT_BLOCK_TEMPLATE_END =
+            JavaLanguageLexer.TEXT_BLOCK_TEMPLATE_END;
+
+    /**
+     *
+     */
+    public static final int TEXT_BLOCK_TEMPLATE_CONTENT =
+            JavaLanguageLexer.TEXT_BLOCK_TEMPLATE_CONTENT;
+
     /** Prevent instantiation. */
     private TokenTypes() {
     }

diff --git a/src/main/resources/com/puppycrawl/tools/checkstyle/grammar/java/JavaLanguageLexer.g4 b/src/main/resources/com/puppycrawl/tools/checkstyle/grammar/java/JavaLanguageLexer.g4
@@ -116,7 +116,8 @@ tokens {
     STRING_TEMPLATE_CONTENT, EMBEDDED_EXPRESSION_BEGIN, EMBEDDED_EXPRESSION,
     EMBEDDED_EXPRESSION_END,
 
-    LITERAL_UNDERSCORE, UNNAMED_PATTERN_DEF
+    LITERAL_UNDERSCORE, UNNAMED_PATTERN_DEF, TEXT_BLOCK_TEMPLATE_BEGIN,
+    TEXT_BLOCK_TEMPLATE_MID, TEXT_BLOCK_TEMPLATE_END, TEXT_BLOCK_TEMPLATE_CONTENT
 }
 
 @header {
@@ -264,6 +265,10 @@ STRING_TEMPLATE_BEGIN:  '"' StringFragment '\\' '{'
 
 TEXT_BLOCK_LITERAL_BEGIN: '"' '"' '"' -> pushMode(TextBlock);
 
+TEXT_BLOCK_TEMPLATE_BEGIN: '"' '"' '"' TextBlockContent  '\\' '{'
+                           { contextCache.enterTemplateContext(TextBlockTemplate); }
+                           ;
+
 LITERAL_NULL:            'null';
 
 // Separators
@@ -333,9 +338,6 @@ DOUBLE_COLON:            '::';
 AT:                      '@';
 ELLIPSIS:                '...';
 
-// String templates
-
-
 // Text block fragments
 
 fragment TextBlockContent
@@ -476,3 +478,12 @@ mode StringTemplate;
     STRING_TEMPLATE_END: StringFragment '"'
                          { contextCache.exitTemplateContext(); }
                          -> popMode, type(STRING_TEMPLATE_END);
+
+mode TextBlockTemplate;
+
+    TEXT_BLOCK_TEMPLATE_MID: TextBlockContent? '\\' '{'
+                             -> pushMode(DEFAULT_MODE), type(TEXT_BLOCK_TEMPLATE_MID);
+
+    TEXT_BLOCK_TEMPLATE_END: TextBlockContent? '"' '"' '"'
+                             { contextCache.exitTemplateContext(); }
+                             -> popMode, type(TEXT_BLOCK_TEMPLATE_END);
diff --git a/src/main/resources/com/puppycrawl/tools/checkstyle/grammar/java/JavaLanguageParser.g4 b/src/main/resources/com/puppycrawl/tools/checkstyle/grammar/java/JavaLanguageParser.g4
@@ -760,21 +760,31 @@ primary
 
 templateArgument
     : template
+    | textBlockLiteral
     | STRING_LITERAL
     ;
 
 template
-    : stringTemplate
-    ;
+    : STRING_TEMPLATE_BEGIN
+      expr? stringTemplateMiddle*
+      STRING_TEMPLATE_END                                                  #stringTemplate
 
-stringTemplate
-    : STRING_TEMPLATE_BEGIN expr? stringTemplateMiddle* STRING_TEMPLATE_END
+    | TEXT_BLOCK_TEMPLATE_BEGIN
+      expr? textBlockTemplateMiddle*
+      TEXT_BLOCK_TEMPLATE_END                                              #textBlockTemplate
     ;
 
+// 'isEmptyTextBlockMiddle' is used to distinguish between empty string and empty text block
+// middle. This is necessary because the lexer cannot distinguish between the two.
 stringTemplateMiddle
     : STRING_TEMPLATE_MID expr?
     ;
 
+textBlockTemplateMiddle
+    : middle=STRING_TEMPLATE_MID expr?
+    | middle=TEXT_BLOCK_TEMPLATE_MID expr?
+    ;
+
 classType
     : (classOrInterfaceType[false] DOT)? annotations[false] id typeArguments?
     ;

diff --git a/src/test/java/com/puppycrawl/tools/checkstyle/JavaAstVisitorTest.java b/src/test/java/com/puppycrawl/tools/checkstyle/JavaAstVisitorTest.java
@@ -90,10 +90,9 @@ public class JavaAstVisitorTest extends AbstractModuleTestSupport {
             "visitQualifiedNameExtended",
             "visitGuard",
 
-            // until https://github.com/checkstyle/checkstyle/issues/14195
-            "visitTemplate",
             // handled as a list in the parent rule
-            "visitStringTemplateMiddle"
+            "visitStringTemplateMiddle",
+            "visitTextBlockTemplateMiddle"
     );
 
     @Override

diff --git a/src/test/java/com/puppycrawl/tools/checkstyle/checks/coding/IllegalTokenTextCheckTest.java b/src/test/java/com/puppycrawl/tools/checkstyle/checks/coding/IllegalTokenTextCheckTest.java
@@ -179,7 +179,7 @@ public void testOrderOfProperties() {
 
     @Test
     public void testAcceptableTokensMakeSense() {
-        final int expectedTokenTypesTotalNumber = 195;
+        final int expectedTokenTypesTotalNumber = 198;
         assertWithMessage("Total number of TokenTypes has changed, acceptable tokens in"
                 + " IllegalTokenTextCheck need to be reconsidered.")
             .that(TokenUtil.getTokenTypesTotalNumber())
@@ -197,7 +197,8 @@ public void testAcceptableTokensMakeSense() {
             TokenTypes.STRING_LITERAL,
             TokenTypes.CHAR_LITERAL,
             TokenTypes.TEXT_BLOCK_CONTENT,
-            TokenTypes.STRING_TEMPLATE_CONTENT
+            TokenTypes.STRING_TEMPLATE_CONTENT,
+            TokenTypes.TEXT_BLOCK_TEMPLATE_CONTENT
         );
         for (int tokenType : allowedTokens) {
             assertWithMessage(TokenUtil.getTokenName(tokenType) + " should not be allowed"

diff --git a/src/test/java/com/puppycrawl/tools/checkstyle/grammar/GeneratedJavaTokenTypesTest.java b/src/test/java/com/puppycrawl/tools/checkstyle/grammar/GeneratedJavaTokenTypesTest.java
@@ -763,6 +763,18 @@ public void testTokenNumbering() {
         assertWithMessage(message)
             .that(JavaLanguageLexer.UNNAMED_PATTERN_DEF)
             .isEqualTo(225);
+        assertWithMessage(message)
+            .that(JavaLanguageLexer.TEXT_BLOCK_TEMPLATE_BEGIN)
+            .isEqualTo(226);
+        assertWithMessage(message)
+            .that(JavaLanguageLexer.TEXT_BLOCK_TEMPLATE_MID)
+            .isEqualTo(227);
+        assertWithMessage(message)
+            .that(JavaLanguageLexer.TEXT_BLOCK_TEMPLATE_END)
+            .isEqualTo(228);
+        assertWithMessage(message)
+            .that(JavaLanguageLexer.TEXT_BLOCK_TEMPLATE_CONTENT)
+            .isEqualTo(229);
 
         final Set<String> modeNames = Set.of(JavaLanguageLexer.modeNames);
         final Set<String> channelNames = Set.of(JavaLanguageLexer.channelNames);
@@ -780,7 +792,7 @@ public void testTokenNumbering() {
                         + " 'GeneratedJavaTokenTypesTest' and verified"
                         + " that their old numbering didn't change")
             .that(tokenCount)
-            .isEqualTo(225);
+            .isEqualTo(229);
     }
 
     /**

diff --git a/src/test/java/com/puppycrawl/tools/checkstyle/grammar/java21/Java21AstRegressionTest.java b/src/test/java/com/puppycrawl/tools/checkstyle/grammar/java21/Java21AstRegressionTest.java
@@ -128,8 +128,25 @@ public void testTextBlockParsingFail() throws Exception {
 
         assertWithMessage(incorrectParsingFailureMessage)
                 .that(throwable.getCause().getMessage())
-                .contains("13:14: mismatched input '}\\n"
-                        + "            ' expecting TEXT_BLOCK_LITERAL_END");
+                .contains("12:15: no viable alternative at input"
+                        + " '\"\"\"\\n            \\{'");
 
     }
+    @Test
+    public void testTextBlockTemplateBasic() throws Exception {
+        verifyAst(
+                getNonCompilablePath(
+                        "ExpectedTextBlockTemplateBasic.txt"),
+                getNonCompilablePath(
+                        "InputTextBlockTemplateBasic.java"));
+    }
+
+    @Test
+    public void testTextBlockTemplateInlineCodeTricky() throws Exception {
+        verifyAst(
+                getNonCompilablePath(
+                        "ExpectedTextBlockTemplateInlineCodeTricky.txt"),
+                getNonCompilablePath(
+                        "InputTextBlockTemplateInlineCodeTricky.java"));
+    }
 }
diff --git a/src/test/java/com/puppycrawl/tools/checkstyle/utils/TokenUtilTest.java b/src/test/java/com/puppycrawl/tools/checkstyle/utils/TokenUtilTest.java
@@ -228,7 +228,7 @@ public void testGetTokenTypesTotalNumber() {
 
         assertWithMessage("Invalid token total number")
             .that(tokenTypesTotalNumber)
-            .isEqualTo(195);
+            .isEqualTo(198);
     }
 
     @Test
@@ -238,10 +238,10 @@ public void testGetAllTokenIds() {
 
         assertWithMessage("Invalid token length")
             .that(allTokenIds.length)
-            .isEqualTo(195);
+            .isEqualTo(198);
         assertWithMessage("invalid sum")
             .that(sum)
-            .isEqualTo(21142);
+            .isEqualTo(21825);
     }
 
     @Test