Skip to content

SONARPY-407 Ensure nodes in the AST have the rightfirstToken and lastToken #317

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
benzonico merged 4 commits into from
Sep 12, 2019
Merged

SONARPY-407 Ensure nodes in the AST have the rightfirstToken and lastToken #317

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2579439
SONARPY-407 Fix firstToken and lastToken for nested PyQualifiedExpres…
andreaguarino Sep 11, 2019
be582e0
SONARPY-407 ClassDef: argList should be null if there are no arguments
andreaguarino Sep 11, 2019
e6aacd8
SONARPY-407 Fix firstToken and lastToken for If, Else, Except, Finall…
andreaguarino Sep 11, 2019
9831015
Fixes after review
andreaguarino Sep 11, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 1 addition & 4 deletions python-squid/src/main/java/org/sonar/python/api/tree/PyClassDefTree.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,11 +34,8 @@ public interface PyClassDefTree extends PyStatementTree {
@CheckForNull
Token leftPar();


/**
* null if class is defined without args : class Foo:
* empty list if class defined with empty parentheses : class Foo():
* @return
* null if class is defined without args {@code class Foo:...} or {@code class Foo():...}
*/
Copy link
Copy Markdown
Contributor

@benzonico benzonico Sep 11, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I believe we lack quite some documentation around tree interfaces, what about updating it explaining in which cases this part of the tree is null rather than simply dropping it ?

@CheckForNull
PyArgListTree args();
Expand Down
5 changes: 2 additions & 3 deletions python-squid/src/main/java/org/sonar/python/tree/PyElseStatementTreeImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
*/
package org.sonar.python.tree;

import com.sonar.sslr.api.AstNode;
import com.sonar.sslr.api.Token;
import java.util.Collections;
import java.util.List;
Expand All @@ -32,8 +31,8 @@ public class PyElseStatementTreeImpl extends PyTree implements PyElseStatementTr
private final Token elseKeyword;
private final PyStatementListTree body;

public PyElseStatementTreeImpl(AstNode astNode, Token elseKeyword, PyStatementListTree body) {
super(astNode);
public PyElseStatementTreeImpl(Token elseKeyword, PyStatementListTree body) {
super(elseKeyword, body.lastToken());
this.elseKeyword = elseKeyword;
this.body = body;
}
Expand Down
12 changes: 6 additions & 6 deletions python-squid/src/main/java/org/sonar/python/tree/PyExceptClauseTreeImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,8 @@ public class PyExceptClauseTreeImpl extends PyTree implements PyExceptClauseTree
private final Token commaToken;
private final PyExpressionTree exceptionInstance;

public PyExceptClauseTreeImpl(AstNode astNode, Token exceptKeyword, PyStatementListTree body) {
super(astNode);
public PyExceptClauseTreeImpl(Token exceptKeyword, PyStatementListTree body) {
super(exceptKeyword, body.lastToken());
this.exceptKeyword = exceptKeyword;
this.body = body;
this.exception = null;
Expand All @@ -48,8 +48,8 @@ public PyExceptClauseTreeImpl(AstNode astNode, Token exceptKeyword, PyStatementL
this.exceptionInstance = null;
}

public PyExceptClauseTreeImpl(AstNode astNode, Token exceptKeyword, PyStatementListTree body, PyExpressionTree exception, AstNode asNode, AstNode commaNode, PyExpressionTree exceptionInstance) {
super(astNode);
public PyExceptClauseTreeImpl(Token exceptKeyword, PyStatementListTree body, PyExpressionTree exception, AstNode asNode, AstNode commaNode, PyExpressionTree exceptionInstance) {
super(exceptKeyword, body.lastToken());
this.exceptKeyword = exceptKeyword;
this.body = body;
this.exception = exception;
Expand All @@ -58,8 +58,8 @@ public PyExceptClauseTreeImpl(AstNode astNode, Token exceptKeyword, PyStatementL
this.exceptionInstance = exceptionInstance;
}

public PyExceptClauseTreeImpl(AstNode except, Token exceptKeyword, PyStatementListTree body, PyExpressionTree exception) {
super(except);
public PyExceptClauseTreeImpl(Token exceptKeyword, PyStatementListTree body, PyExpressionTree exception) {
super(exceptKeyword, body.lastToken());
this.exceptKeyword = exceptKeyword;
this.body = body;
this.exception = exception;
Expand Down
5 changes: 2 additions & 3 deletions python-squid/src/main/java/org/sonar/python/tree/PyFinallyClauseTreeImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
*/
package org.sonar.python.tree;

import com.sonar.sslr.api.AstNode;
import com.sonar.sslr.api.Token;
import java.util.Collections;
import java.util.List;
Expand All @@ -32,8 +31,8 @@ public class PyFinallyClauseTreeImpl extends PyTree implements PyFinallyClauseTr
private final Token finallyKeyword;
private final PyStatementListTree body;

public PyFinallyClauseTreeImpl(AstNode astNode, Token finallyKeyword, PyStatementListTree body) {
super(astNode);
public PyFinallyClauseTreeImpl(Token finallyKeyword, PyStatementListTree body) {
super(finallyKeyword, body.lastToken());
this.finallyKeyword = finallyKeyword;
this.body = body;
}
Expand Down
9 changes: 4 additions & 5 deletions python-squid/src/main/java/org/sonar/python/tree/PyIfStatementTreeImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
*/
package org.sonar.python.tree;

import com.sonar.sslr.api.AstNode;
import com.sonar.sslr.api.Token;
import java.util.Arrays;
import java.util.Collections;
Expand Down Expand Up @@ -48,8 +47,8 @@ public class PyIfStatementTreeImpl extends PyTree implements PyIfStatementTree {
*
* If statement constructor
*/
public PyIfStatementTreeImpl(AstNode node, Token ifKeyword, PyExpressionTree condition, PyStatementListTree statements, List<PyIfStatementTree> elifBranches, @CheckForNull PyElseStatementTree elseStatement) {
super(node);
public PyIfStatementTreeImpl(Token ifKeyword, PyExpressionTree condition, PyStatementListTree statements, List<PyIfStatementTree> elifBranches, @CheckForNull PyElseStatementTree elseStatement) {
super(ifKeyword, statements.lastToken());
this.keyword = ifKeyword;
this.condition = condition;
this.statements = statements;
Expand All @@ -61,8 +60,8 @@ public PyIfStatementTreeImpl(AstNode node, Token ifKeyword, PyExpressionTree con
/**
* Elif statement constructor
*/
public PyIfStatementTreeImpl(AstNode node, Token elifKeyword, PyExpressionTree condition, PyStatementListTree statements) {
super(node);
public PyIfStatementTreeImpl(Token elifKeyword, PyExpressionTree condition, PyStatementListTree statements) {
super(elifKeyword, statements.lastToken());
this.keyword = elifKeyword;
this.condition = condition;
this.statements = statements;
Expand Down
8 changes: 0 additions & 8 deletions python-squid/src/main/java/org/sonar/python/tree/PyQualifiedExpressionTreeImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
*/
package org.sonar.python.tree;

import com.sonar.sslr.api.AstNode;
import com.sonar.sslr.api.Token;
import java.util.Arrays;
import java.util.List;
Expand All @@ -34,13 +33,6 @@ public class PyQualifiedExpressionTreeImpl extends PyTree implements PyQualified
private final PyExpressionTree qualifier;
private final Token dotToken;

public PyQualifiedExpressionTreeImpl(AstNode astNode, PyNameTree name, PyExpressionTree qualifier, Token dotToken) {
super(astNode);
this.name = name;
this.qualifier = qualifier;
this.dotToken = dotToken;
}

public PyQualifiedExpressionTreeImpl(PyNameTree name, PyExpressionTree qualifier, Token dotToken) {
super(qualifier.firstToken(), name.lastToken());
this.name = name;
Expand Down
25 changes: 10 additions & 15 deletions python-squid/src/main/java/org/sonar/python/tree/PythonTreeMaker.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -416,23 +416,21 @@ public PyIfStatementTree ifStatement(AstNode astNode) {
.map(this::elifStatement)
.collect(Collectors.toList());

return new PyIfStatementTreeImpl(
astNode, ifToken, expression(condition), statements, elifBranches, elseStatement);
return new PyIfStatementTreeImpl(ifToken, expression(condition), statements, elifBranches, elseStatement);
}

private PyIfStatementTree elifStatement(AstNode astNode) {
Token elifToken = astNode.getToken();
AstNode suite = astNode.getNextSibling().getNextSibling().getNextSibling();
AstNode condition = astNode.getNextSibling();
PyStatementListTree statements = getStatementListFromSuite(suite);
return new PyIfStatementTreeImpl(
astNode, elifToken, expression(condition), statements);
return new PyIfStatementTreeImpl(elifToken, expression(condition), statements);
}

private PyElseStatementTree elseStatement(AstNode astNode) {
Token elseToken = astNode.getPreviousSibling().getPreviousSibling().getToken();
PyStatementListTree statements = getStatementListFromSuite(astNode);
return new PyElseStatementTreeImpl(astNode, elseToken, statements);
return new PyElseStatementTreeImpl(elseToken, statements);
}

public PyFunctionDefTree funcDefStatement(AstNode astNode) {
Expand Down Expand Up @@ -505,9 +503,7 @@ public PyClassDefTree classDefStatement(AstNode astNode) {
PyArgListTree args = null;
AstNode leftPar = astNode.getFirstChild(PythonPunctuator.LPARENTHESIS);
if (leftPar != null) {
AstNode argList = astNode.getFirstChild(PythonGrammar.ARGLIST);
// FIXME: PyArgList should never have null firstToken and lastToken
args = argList != null ? argList(argList) : new PyArgListTreeImpl(argList, Collections.emptyList());
args = argList(astNode.getFirstChild(PythonGrammar.ARGLIST));
}
PyStatementListTree body = getStatementListFromSuite(astNode.getFirstChild(PythonGrammar.SUITE));
Token classToken = astNode.getFirstChild(PythonKeyword.CLASS).getToken();
Expand Down Expand Up @@ -622,7 +618,7 @@ public PyTryStatementTree tryStatement(AstNode astNode) {
if (finallyNode != null) {
AstNode finallySuite = finallyNode.getNextSibling().getNextSibling();
PyStatementListTree body = getStatementListFromSuite(finallySuite);
finallyClause = new PyFinallyClauseTreeImpl(finallySuite, finallyNode.getToken(), body);
finallyClause = new PyFinallyClauseTreeImpl(finallyNode.getToken(), body);
}
PyElseStatementTree elseStatementTree = null;
AstNode elseNode = astNode.getFirstChild(PythonKeyword.ELSE);
Expand Down Expand Up @@ -667,15 +663,15 @@ private PyExceptClauseTree exceptClause(AstNode except, PyStatementListTree body
Token exceptKeyword = except.getFirstChild(PythonKeyword.EXCEPT).getToken();
AstNode exceptionNode = except.getFirstChild(PythonGrammar.TEST);
if (exceptionNode == null) {
return new PyExceptClauseTreeImpl(except, exceptKeyword, body);
return new PyExceptClauseTreeImpl(exceptKeyword, body);
}
AstNode asNode = except.getFirstChild(PythonKeyword.AS);
AstNode commaNode = except.getFirstChild(PythonPunctuator.COMMA);
if (asNode != null || commaNode != null) {
PyExpressionTree exceptionInstance = expression(except.getLastChild(PythonGrammar.TEST));
return new PyExceptClauseTreeImpl(except, exceptKeyword, body, expression(exceptionNode), asNode, commaNode, exceptionInstance);
return new PyExceptClauseTreeImpl(exceptKeyword, body, expression(exceptionNode), asNode, commaNode, exceptionInstance);
}
return new PyExceptClauseTreeImpl(except, exceptKeyword, body, expression(exceptionNode));
return new PyExceptClauseTreeImpl(exceptKeyword, body, expression(exceptionNode));
}

// expressions
Expand Down Expand Up @@ -1001,11 +997,10 @@ public PyQualifiedExpressionTree qualifiedExpression(AstNode astNode) {
AstNode lastNameNode = astNode.getLastChild();
for (AstNode nameNode : names) {
if (nameNode != lastNameNode) {
// FIXME: there is no corresponding astNode, parseTree and strongly typed AST are structurally different
qualifier = new PyQualifiedExpressionTreeImpl(astNode, name(nameNode), qualifier, nameNode.getPreviousSibling().getToken());
qualifier = new PyQualifiedExpressionTreeImpl(name(nameNode), qualifier, nameNode.getPreviousSibling().getToken());
}
}
return new PyQualifiedExpressionTreeImpl(astNode, name(lastNameNode), qualifier, lastNameNode.getPreviousSibling().getToken());
return new PyQualifiedExpressionTreeImpl(name(lastNameNode), qualifier, lastNameNode.getPreviousSibling().getToken());
}

public PyCallExpressionTree callExpression(AstNode astNode) {
Expand Down
Loading