Skip to content

Commit

Permalink
Pull #14615: rework AST building/parsing
Browse files Browse the repository at this point in the history
  • Loading branch information
@nrmancuso @romani
nrmancuso authored and romani committed Mar 22, 2024
1 parent 781182d commit 7c50484
Show file tree
Hide file tree
Showing 15 changed files with 939 additions and 347 deletions.
66 changes: 0 additions & 66 deletions config/checker-framework-suppressions/checker-index-suppressions.xml
View file
Expand Up @@ -33,28 +33,6 @@
</details>
</checkerFrameworkError>

<checkerFrameworkError unstable="false">
<fileName>src/main/java/com/puppycrawl/tools/checkstyle/JavaAstVisitor.java</fileName>
<specifier>argument</specifier>
<message>incompatible argument for parameter beginIndex of String.substring.</message>
<lineContent>EMBEDDED_EXPRESSION_END.length(),</lineContent>
<details>
found : @LTEqLengthOf(&quot;com.puppycrawl.tools.checkstyle.JavaAstVisitor.EMBEDDED_EXPRESSION_END&quot;) int
required: @LTEqLengthOf(&quot;tokenText&quot;) int
</details>
</checkerFrameworkError>

<checkerFrameworkError unstable="false">
<fileName>src/main/java/com/puppycrawl/tools/checkstyle/JavaAstVisitor.java</fileName>
<specifier>argument</specifier>
<message>incompatible argument for parameter beginIndex of String.substring.</message>
<lineContent>EMBEDDED_EXPRESSION_END.length(),</lineContent>
<details>
found : @LTEqLengthOf(&quot;com.puppycrawl.tools.checkstyle.JavaAstVisitor.EMBEDDED_EXPRESSION_END&quot;) int
required: @LTEqLengthOf(&quot;tokenText&quot;) int
</details>
</checkerFrameworkError>

<checkerFrameworkError unstable="false">
<fileName>src/main/java/com/puppycrawl/tools/checkstyle/JavaAstVisitor.java</fileName>
<specifier>argument</specifier>
Expand All @@ -66,17 +44,6 @@
</details>
</checkerFrameworkError>

<checkerFrameworkError unstable="false">
<fileName>src/main/java/com/puppycrawl/tools/checkstyle/JavaAstVisitor.java</fileName>
<specifier>argument</specifier>
<message>incompatible argument for parameter beginIndex of String.substring.</message>
<lineContent>QUOTE.length(), tokenTextLength - EMBEDDED_EXPRESSION_BEGIN.length());</lineContent>
<details>
found : @LTEqLengthOf(&quot;com.puppycrawl.tools.checkstyle.JavaAstVisitor.QUOTE&quot;) int
required: @LTEqLengthOf(&quot;tokenText&quot;) int
</details>
</checkerFrameworkError>

<checkerFrameworkError unstable="false">
<fileName>src/main/java/com/puppycrawl/tools/checkstyle/JavaAstVisitor.java</fileName>
<specifier>argument</specifier>
Expand All @@ -88,39 +55,6 @@
</details>
</checkerFrameworkError>

<checkerFrameworkError unstable="false">
<fileName>src/main/java/com/puppycrawl/tools/checkstyle/JavaAstVisitor.java</fileName>
<specifier>argument</specifier>
<message>incompatible argument for parameter endIndex of String.substring.</message>
<lineContent>tokenTextLength - QUOTE.length()</lineContent>
<details>
found : @GTENegativeOne int
required: @NonNegative int
</details>
</checkerFrameworkError>

<checkerFrameworkError unstable="false">
<fileName>src/main/java/com/puppycrawl/tools/checkstyle/JavaAstVisitor.java</fileName>
<specifier>argument</specifier>
<message>incompatible argument for parameter endIndex of String.substring.</message>
<lineContent>QUOTE.length(), tokenTextLength - EMBEDDED_EXPRESSION_BEGIN.length());</lineContent>
<details>
found : int
required: @NonNegative int
</details>
</checkerFrameworkError>

<checkerFrameworkError unstable="false">
<fileName>src/main/java/com/puppycrawl/tools/checkstyle/JavaAstVisitor.java</fileName>
<specifier>argument</specifier>
<message>incompatible argument for parameter endIndex of String.substring.</message>
<lineContent>tokenTextLength - EMBEDDED_EXPRESSION_BEGIN.length()</lineContent>
<details>
found : int
required: @NonNegative int
</details>
</checkerFrameworkError>

<checkerFrameworkError unstable="false">
<fileName>src/main/java/com/puppycrawl/tools/checkstyle/JavadocDetailNodeParser.java</fileName>
<specifier>argument</specifier>
Expand Down
7 changes: 0 additions & 7 deletions config/checker-framework-suppressions/checker-nullness-optional-interning-suppressions.xml
View file
Expand Up @@ -5369,13 +5369,6 @@
</details>
</checkerFrameworkError>

<checkerFrameworkError unstable="false">
<fileName>src/main/java/com/puppycrawl/tools/checkstyle/grammar/CompositeLexerContextCache.java</fileName>
<specifier>dereference.of.nullable</specifier>
<message>dereference of possibly-null reference currentContext</message>
<lineContent>if (currentContext.getCurlyBraceDepth() == 0) {</lineContent>
</checkerFrameworkError>

<checkerFrameworkError unstable="false">
<fileName>src/main/java/com/puppycrawl/tools/checkstyle/gui/BaseCellEditor.java</fileName>
<specifier>return</specifier>
Expand Down
1 change: 1 addition & 0 deletions config/import-control.xml
View file
Expand Up @@ -309,6 +309,7 @@
</file>
<file name="CompositeLexerContextCache">
<allow pkg="org.antlr.v4.runtime"/>
<allow class="com.puppycrawl.tools.checkstyle.grammar.java.JavaLanguageLexer"/>
</file>
</subpackage>

Expand Down
190 changes: 53 additions & 137 deletions src/main/java/com/puppycrawl/tools/checkstyle/JavaAstVisitor.java
View file
Expand Up @@ -107,12 +107,6 @@ public final class JavaAstVisitor extends JavaLanguageParserBaseVisitor<DetailAs
/** String representation of the double quote character. */
private static final String QUOTE = "\"";

/** String representation of the string template embedded expression starting delimiter. */
private static final String EMBEDDED_EXPRESSION_BEGIN = "\\{";

/** String representation of the string template embedded expression ending delimiter. */
private static final String EMBEDDED_EXPRESSION_END = "}";

/**
* The tokens here are technically expressions, but should
* not return an EXPR token as their root.
Expand Down Expand Up @@ -1808,164 +1802,86 @@ private static DetailAstImpl buildSimpleStringTemplateArgument(

@Override
public DetailAstImpl visitStringTemplate(JavaLanguageParser.StringTemplateContext ctx) {
final DetailAstImpl begin = buildStringTemplateBeginning(ctx);
final DetailAstImpl stringTemplateBegin = visit(ctx.stringTemplateBegin());

final Optional<DetailAstImpl> startExpression = Optional.ofNullable(ctx.expr())
final Optional<DetailAstImpl> expression = Optional.ofNullable(ctx.expr())
.map(this::visit);

if (startExpression.isPresent()) {
final DetailAstImpl imaginaryExpr =
if (expression.isPresent()) {
final DetailAstImpl imaginaryExpression =
createImaginary(TokenTypes.EMBEDDED_EXPRESSION);
imaginaryExpr.addChild(startExpression.orElseThrow());
begin.addChild(imaginaryExpr);
imaginaryExpression.addChild(expression.orElseThrow());
stringTemplateBegin.addChild(imaginaryExpression);
}

ctx.stringTemplateMiddle().stream()
.map(this::buildStringTemplateMiddle)
.collect(Collectors.toUnmodifiableList())
.forEach(begin::addChild);

final DetailAstImpl end = buildStringTemplateEnd(ctx);
begin.addChild(end);
return begin;
}

/**
* Builds the beginning of a string template AST.
*
* @param ctx the StringTemplateContext to build AST from
* @return string template AST
*/
private static DetailAstImpl buildStringTemplateBeginning(
JavaLanguageParser.StringTemplateContext ctx) {

// token looks like '"' StringFragment '\{'
final TerminalNode context = ctx.STRING_TEMPLATE_BEGIN();
final Token token = context.getSymbol();
final String tokenText = context.getText();
final int tokenStartIndex = token.getCharPositionInLine();
final int tokenLineNumber = token.getLine();
final int tokenTextLength = tokenText.length();

final DetailAstImpl stringTemplateBegin = createImaginary(
TokenTypes.STRING_TEMPLATE_BEGIN, QUOTE,
tokenLineNumber, tokenStartIndex
);
.forEach(stringTemplateBegin::addChild);

// remove delimiters '"' and '\{'
final String stringFragment = tokenText.substring(
QUOTE.length(), tokenTextLength - EMBEDDED_EXPRESSION_BEGIN.length());

final DetailAstImpl stringTemplateContent = createImaginary(
TokenTypes.STRING_TEMPLATE_CONTENT, stringFragment,
tokenLineNumber, tokenStartIndex + QUOTE.length()
);
stringTemplateBegin.addChild(stringTemplateContent);

final DetailAstImpl embeddedBegin = createImaginary(
TokenTypes.EMBEDDED_EXPRESSION_BEGIN, EMBEDDED_EXPRESSION_BEGIN,
tokenLineNumber,
tokenStartIndex + tokenTextLength - EMBEDDED_EXPRESSION_BEGIN.length()
);
stringTemplateBegin.addChild(embeddedBegin);
final DetailAstImpl stringTemplateEnd = visit(ctx.stringTemplateEnd());
stringTemplateBegin.addChild(stringTemplateEnd);
return stringTemplateBegin;
}

/**
* Builds the middle of a string template AST.
* Builds a string template middle AST.
*
* @param middleContext the StringTemplateMiddleContext to build AST from
* @param ctx the StringTemplateMiddleContext to build AST from
* @return DetailAstImpl of string template middle
*/
private DetailAstImpl buildStringTemplateMiddle(
JavaLanguageParser.StringTemplateMiddleContext middleContext) {

// token looks like '}' StringFragment '\{'
final TerminalNode context = middleContext.STRING_TEMPLATE_MID();
final Token token = context.getSymbol();
final int tokenStartIndex = token.getCharPositionInLine();
final int tokenLineNumber = token.getLine();
final String tokenText = context.getText();
final int tokenTextLength = tokenText.length();

final DetailAstImpl embeddedExpressionEnd = createImaginary(
TokenTypes.EMBEDDED_EXPRESSION_END, EMBEDDED_EXPRESSION_END,
tokenLineNumber, tokenStartIndex
);

// remove delimiters '}' and '\\' '{'
final String stringFragment = tokenText.substring(
EMBEDDED_EXPRESSION_END.length(),
tokenTextLength - EMBEDDED_EXPRESSION_BEGIN.length()
);

final DetailAstImpl content = createImaginary(
TokenTypes.STRING_TEMPLATE_CONTENT, stringFragment,
tokenLineNumber, tokenStartIndex + EMBEDDED_EXPRESSION_END.length()
);
embeddedExpressionEnd.addNextSibling(content);
JavaLanguageParser.StringTemplateMiddleContext ctx) {
final DetailAstImpl stringTemplateMiddle =
visit(ctx.stringTemplateMid());

final DetailAstImpl embeddedBegin = createImaginary(
TokenTypes.EMBEDDED_EXPRESSION_BEGIN, EMBEDDED_EXPRESSION_BEGIN,
tokenLineNumber,
tokenStartIndex + tokenTextLength - EMBEDDED_EXPRESSION_BEGIN.length()
);
content.addNextSibling(embeddedBegin);

final Optional<DetailAstImpl> embeddedExpression = Optional.ofNullable(middleContext.expr())
final Optional<DetailAstImpl> expression = Optional.ofNullable(ctx.expr())
.map(this::visit);

if (embeddedExpression.isPresent()) {
final DetailAstImpl imaginaryExpr =
if (expression.isPresent()) {
final DetailAstImpl imaginaryExpression =
createImaginary(TokenTypes.EMBEDDED_EXPRESSION);
imaginaryExpr.addChild(embeddedExpression.orElseThrow());
embeddedExpressionEnd.addNextSibling(imaginaryExpr);
imaginaryExpression.addChild(expression.orElseThrow());
addLastSibling(stringTemplateMiddle, imaginaryExpression);
}

return embeddedExpressionEnd;
return stringTemplateMiddle;
}

/**
* Builds the end of a string template AST.
*
* @param ctx the StringTemplateContext to build AST from
* @return DetailAstImpl of string template end
*/
private static DetailAstImpl buildStringTemplateEnd(
JavaLanguageParser.StringTemplateContext ctx) {

// token looks like '}' StringFragment '"'
final TerminalNode context = ctx.STRING_TEMPLATE_END();
final Token token = context.getSymbol();
final String tokenText = context.getText();
final int tokenStartIndex = token.getCharPositionInLine();
final int tokenLineNumber = token.getLine();
final int tokenTextLength = tokenText.length();

final DetailAstImpl embeddedExpressionEnd = createImaginary(
TokenTypes.EMBEDDED_EXPRESSION_END, EMBEDDED_EXPRESSION_END,
tokenLineNumber, tokenStartIndex
);

// remove delimiters '}' and '"'
final String stringFragment = tokenText.substring(
EMBEDDED_EXPRESSION_END.length(),
tokenTextLength - QUOTE.length()
);
@Override
public DetailAstImpl visitStringTemplateBegin(
JavaLanguageParser.StringTemplateBeginContext ctx) {
final DetailAstImpl stringTemplateBegin =
create(ctx.STRING_TEMPLATE_BEGIN());
final Optional<DetailAstImpl> stringTemplateContent =
Optional.ofNullable(ctx.STRING_TEMPLATE_CONTENT())
.map(this::create);
stringTemplateContent.ifPresent(stringTemplateBegin::addChild);
final DetailAstImpl embeddedExpressionBegin = create(ctx.EMBEDDED_EXPRESSION_BEGIN());
stringTemplateBegin.addChild(embeddedExpressionBegin);
return stringTemplateBegin;
}

final DetailAstImpl endContent = createImaginary(
TokenTypes.STRING_TEMPLATE_CONTENT, stringFragment,
tokenLineNumber,
tokenStartIndex + EMBEDDED_EXPRESSION_END.length()
);
embeddedExpressionEnd.addNextSibling(endContent);
@Override
public DetailAstImpl visitStringTemplateMid(JavaLanguageParser.StringTemplateMidContext ctx) {
final DetailAstImpl embeddedExpressionEnd = create(ctx.EMBEDDED_EXPRESSION_END());
final Optional<DetailAstImpl> stringTemplateContent =
Optional.ofNullable(ctx.STRING_TEMPLATE_CONTENT())
.map(this::create);
stringTemplateContent.ifPresent(self -> addLastSibling(embeddedExpressionEnd, self));
final DetailAstImpl embeddedExpressionBegin = create(ctx.EMBEDDED_EXPRESSION_BEGIN());
addLastSibling(embeddedExpressionEnd, embeddedExpressionBegin);
return embeddedExpressionEnd;
}

final DetailAstImpl stringTemplateEnd = createImaginary(
TokenTypes.STRING_TEMPLATE_END, QUOTE,
tokenLineNumber,
tokenStartIndex + tokenTextLength - QUOTE.length()
);
endContent.addNextSibling(stringTemplateEnd);
@Override
public DetailAstImpl visitStringTemplateEnd(JavaLanguageParser.StringTemplateEndContext ctx) {
final DetailAstImpl embeddedExpressionEnd = create(ctx.EMBEDDED_EXPRESSION_END());
final Optional<DetailAstImpl> stringTemplateContent =
Optional.ofNullable(ctx.STRING_TEMPLATE_CONTENT())
.map(this::create);
stringTemplateContent.ifPresent(self -> addLastSibling(embeddedExpressionEnd, self));
final DetailAstImpl stringTemplateEnd = create(ctx.STRING_TEMPLATE_END());
addLastSibling(embeddedExpressionEnd, stringTemplateEnd);
return embeddedExpressionEnd;
}

Expand Down
3 changes: 1 addition & 2 deletions src/main/java/com/puppycrawl/tools/checkstyle/api/TokenTypes.java
View file
Expand Up @@ -6554,7 +6554,7 @@ public final class TokenTypes {
* template may have more than one node of this type.
* <p>For example:</p>
* <pre>
* String s = STR."Hello, \{firstName + " " + lastName}!";
* String s = STR."Hello, \{firstName + " " + lastName}";
* </pre>
* <p>parses as:</p>
* <pre>
Expand All @@ -6577,7 +6577,6 @@ public final class TokenTypes {
* | | `--STRING_LITERAL -&gt; " "
* | `--IDENT -&gt; lastName
* |--EMBEDDED_EXPRESSION_END -&gt; }
* |--STRING_TEMPLATE_CONTENT -&gt; !
* `--STRING_TEMPLATE_END -&gt; "
* </pre>
*
Expand Down

0 comments on commit 7c50484

Please sign in to comment.