Test and fix token order

its/ruling/src/test/resources/expected/python-S107.json

@@ -196,6 +196,9 @@
 12,
 52,
 ],
+'project:django-2.2.3/django/contrib/gis/db/models/fields.py':[
+207,
+],
 'project:django-2.2.3/django/contrib/gis/utils/layermapping.py':[
 80,
 476,
@@ -263,6 +266,7 @@
 ],
 'project:django-2.2.3/django/forms/fields.py':[
 57,
+1079,
 ],
 'project:django-2.2.3/django/forms/forms.py':[
 74,

python-squid/src/main/java/org/sonar/python/api/tree/Parameter.java

@@ -26,6 +26,10 @@ public interface Parameter extends AnyParameter {
   @CheckForNull
   Token starToken();
 
+  /**
+   * @return null only in case of star parameter
+   */
+  @CheckForNull
   Name name();
 
   @CheckForNull

python-squid/src/main/java/org/sonar/python/semantic/SymbolTableBuilder.java

@@ -27,12 +27,12 @@
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import javax.annotation.CheckForNull;
 import javax.annotation.Nullable;
-import org.sonar.python.api.tree.HasSymbol;
 import org.sonar.python.api.tree.AliasedName;
 import org.sonar.python.api.tree.AnnotatedAssignment;
 import org.sonar.python.api.tree.AnyParameter;
@@ -48,17 +48,18 @@
 import org.sonar.python.api.tree.FunctionDef;
 import org.sonar.python.api.tree.FunctionLike;
 import org.sonar.python.api.tree.GlobalStatement;
+import org.sonar.python.api.tree.HasSymbol;
 import org.sonar.python.api.tree.ImportFrom;
 import org.sonar.python.api.tree.ImportName;
 import org.sonar.python.api.tree.LambdaExpression;
 import org.sonar.python.api.tree.Name;
 import org.sonar.python.api.tree.NonlocalStatement;
-import org.sonar.python.api.tree.ParameterList;
 import org.sonar.python.api.tree.Parameter;
+import org.sonar.python.api.tree.ParameterList;
 import org.sonar.python.api.tree.QualifiedExpression;
-import org.sonar.python.api.tree.Tuple;
 import org.sonar.python.api.tree.Tree;
 import org.sonar.python.api.tree.Tree.Kind;
+import org.sonar.python.api.tree.Tuple;
 import org.sonar.python.api.tree.TupleParameter;
 import org.sonar.python.tree.BaseTreeVisitor;
 import org.sonar.python.tree.ClassDefImpl;
@@ -223,7 +224,9 @@ private void createParameters(FunctionLike function) {
       parameterList.nonTuple()
         .stream()
         .skip(hasSelf ? 1 : 0)
-        .forEach(param -> addBindingUsage(param.name(), Usage.Kind.PARAMETER));
+        .map(Parameter::name)
+        .filter(Objects::nonNull)
+        .forEach(param -> addBindingUsage(param, Usage.Kind.PARAMETER));
 
       parameterList.all().stream()
         .filter(param -> param.is(Kind.TUPLE_PARAMETER))
@@ -340,11 +343,13 @@ private Set<Symbol> symbols() {
 
     private void createSelfParameter(Parameter parameter) {
       Name nameTree = parameter.name();
-      String symbolName = nameTree.name();
-      SymbolImpl symbol = new SelfSymbolImpl(symbolName, parent);
-      symbols.add(symbol);
-      symbolsByName.put(symbolName, symbol);
-      symbol.addUsage(nameTree, Usage.Kind.PARAMETER);
+      if (nameTree != null) {
+        String symbolName = nameTree.name();
+        SymbolImpl symbol = new SelfSymbolImpl(symbolName, parent);
+        symbols.add(symbol);
+        symbolsByName.put(symbolName, symbol);
+        symbol.addUsage(nameTree, Usage.Kind.PARAMETER);
+      }
     }
 
     void addBindingUsage(Name nameTree, Usage.Kind kind, @Nullable String fullyQualifiedName) {

python-squid/src/main/java/org/sonar/python/tree/ArgumentImpl.java

@@ -96,6 +96,6 @@ public Kind getKind() {
 
   @Override
   public List<Tree> computeChildren() {
-    return Stream.of(keywordArgument, equalToken, expression, star, starStar).filter(Objects::nonNull).collect(Collectors.toList());
+    return Stream.of(keywordArgument, equalToken, star, starStar, expression).filter(Objects::nonNull).collect(Collectors.toList());
   }
 }

python-squid/src/main/java/org/sonar/python/tree/ComprehensionExpressionImpl.java

@@ -23,6 +23,7 @@
 import java.util.Objects;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
+import javax.annotation.Nullable;
 import org.sonar.python.api.tree.ComprehensionExpression;
 import org.sonar.python.api.tree.ComprehensionFor;
 import org.sonar.python.api.tree.Expression;
@@ -38,8 +39,8 @@ public class ComprehensionExpressionImpl extends PyTree implements Comprehension
   private final ComprehensionFor comprehensionFor;
   private final Token closingToken;
 
-  public ComprehensionExpressionImpl(Kind kind, Token openingToken, Expression resultExpression,
-                                     ComprehensionFor compFor, Token closingToken) {
+  public ComprehensionExpressionImpl(Kind kind, @Nullable Token openingToken, Expression resultExpression,
+                                     ComprehensionFor compFor, @Nullable Token closingToken) {
     this.kind = kind;
     this.resultExpression = resultExpression;
     this.comprehensionFor = compFor;

python-squid/src/main/java/org/sonar/python/tree/ExceptClauseImpl.java

@@ -130,7 +130,7 @@ public void accept(TreeVisitor visitor) {
 
   @Override
   public List<Tree> computeChildren() {
-    return Stream.of(exceptKeyword, exception, asKeyword, exceptionInstance, commaToken, colon, newLine, indent, body, dedent)
+    return Stream.of(exceptKeyword, exception, asKeyword, commaToken, exceptionInstance, colon, newLine, indent, body, dedent)
       .filter(Objects::nonNull).collect(Collectors.toList());
   }
 }

python-squid/src/main/java/org/sonar/python/tree/KeyValuePairImpl.java

@@ -91,7 +91,7 @@ public void accept(TreeVisitor visitor) {
 
   @Override
   public List<Tree> computeChildren() {
-    return Stream.of(expression, key, colon, value, starStarToken).filter(Objects::nonNull).collect(Collectors.toList());
+    return Stream.of(starStarToken, expression, key, colon, value).filter(Objects::nonNull).collect(Collectors.toList());
   }
 
   @Override

python-squid/src/main/java/org/sonar/python/tree/ParameterImpl.java

@@ -50,12 +50,25 @@ public ParameterImpl(@Nullable Token starToken, Name name, @Nullable TypeAnnotat
     this.defaultValue = defaultValue;
   }
 
+  /**
+   * constructor for star parameter syntax.
+   * def fun(arg1, *, arg2)
+   */
+  public ParameterImpl(Token starToken) {
+    this.starToken = starToken;
+    this.name = null;
+    this.annotation = null;
+    this.equalToken = null;
+    this.defaultValue = null;
+  }
+
   @CheckForNull
   @Override
   public Token starToken() {
     return starToken;
   }
 
+  @CheckForNull
   @Override
   public Name name() {
     return name;

python-squid/src/main/java/org/sonar/python/tree/PythonTreeMaker.java

@@ -25,6 +25,7 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Objects;
 import java.util.stream.Collectors;
 import javax.annotation.CheckForNull;
 import javax.annotation.Nullable;
@@ -516,8 +517,8 @@ public FunctionDef funcDefStatement(AstNode astNode) {
     ParameterList parameterList = null;
     AstNode typedArgListNode = astNode.getFirstChild(PythonGrammar.TYPEDARGSLIST);
     if (typedArgListNode != null) {
-      List<AnyParameter> arguments = typedArgListNode.getChildren(PythonGrammar.TFPDEF).stream()
-        .map(this::parameter).collect(Collectors.toList());
+      List<AnyParameter> arguments = typedArgListNode.getChildren(PythonGrammar.TFPDEF, PythonPunctuator.MUL).stream()
+        .map(this::parameter).filter(Objects::nonNull).collect(Collectors.toList());
       List<Token> commas = punctuators(typedArgListNode, PythonPunctuator.COMMA);
       parameterList = new ParameterListImpl(arguments, commas);
     }
@@ -712,11 +713,12 @@ public WithStatement withStatement(AstNode astNode) {
       asyncKeyword = toPyToken(astNode.getFirstChild().getToken());
     }
     List<WithItem> withItems = withItems(withStmtNode.getChildren(PythonGrammar.WITH_ITEM));
+    List<Token> commas = punctuators(withStmtNode, PythonPunctuator.COMMA);
     AstNode suite = withStmtNode.getFirstChild(PythonGrammar.SUITE);
     Token withKeyword = toPyToken(withStmtNode.getToken());
     Token colon = toPyToken(suite.getPreviousSibling().getToken());
     StatementList body = getStatementListFromSuite(suite);
-    return new WithStatementImpl(withKeyword, withItems, colon, suiteNewLine(suite), suiteIndent(suite), body, suiteDedent(suite), asyncKeyword);
+    return new WithStatementImpl(withKeyword, withItems, commas, colon, suiteNewLine(suite), suiteIndent(suite), body, suiteDedent(suite), asyncKeyword);
   }
 
   private List<WithItem> withItems(List<AstNode> withItems) {
@@ -1136,7 +1138,7 @@ public Argument argument(AstNode astNode) {
     if (compFor != null) {
       Expression expression = expression(astNode.getFirstChild());
       ComprehensionExpression comprehension =
-        new ComprehensionExpressionImpl(Tree.Kind.GENERATOR_EXPR, expression.firstToken(), expression, compFor(compFor), toPyToken(compFor.getLastToken()));
+        new ComprehensionExpressionImpl(Tree.Kind.GENERATOR_EXPR, null, expression, compFor(compFor), null);
       return new ArgumentImpl(comprehension, null, null);
     }
     AstNode assign = astNode.getFirstChild(PythonPunctuator.ASSIGN);
@@ -1187,6 +1189,12 @@ public LambdaExpression lambdaExpression(AstNode astNode) {
   }
 
   private AnyParameter parameter(AstNode parameter) {
+    if(parameter.is(PythonPunctuator.MUL)) {
+      if(parameter.getNextSibling()==null || parameter.getNextSibling().is(PythonPunctuator.COMMA)) {
+        return new ParameterImpl(toPyToken(parameter.getToken()));
+      }
+      return null;
+    }
     AstNode prevSibling = parameter.getPreviousSibling();
 
     if (parameter.is(PythonGrammar.NAME)) {

python-squid/src/main/java/org/sonar/python/tree/TryStatementImpl.java

@@ -98,7 +98,7 @@ public void accept(TreeVisitor visitor) {
 
   @Override
   public List<Tree> computeChildren() {
-    return Stream.of(Arrays.asList(tryKeyword, colon, newLine, indent, tryBody, dedent), exceptClauses, Arrays.asList(finallyClause, elseClause))
+    return Stream.of(Arrays.asList(tryKeyword, colon, newLine, indent, tryBody, dedent), exceptClauses, Arrays.asList(elseClause, finallyClause))
       .flatMap(List::stream).filter(Objects::nonNull).collect(Collectors.toList());
   }
 }

python-squid/src/main/java/org/sonar/python/tree/WithStatementImpl.java

@@ -19,6 +19,7 @@
  */
 package org.sonar.python.tree;
 
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Objects;
@@ -38,6 +39,7 @@ public class WithStatementImpl extends PyTree implements WithStatement {
 
   private final Token withKeyword;
   private final List<WithItem> withItems;
+  private final List<Token> commas;
   private final Token newLine;
   private final Token indent;
   private final StatementList statements;
@@ -46,10 +48,11 @@ public class WithStatementImpl extends PyTree implements WithStatement {
   private final boolean isAsync;
   private final Token colon;
 
-  public WithStatementImpl(Token withKeyword, List<WithItem> withItems, Token colon, @Nullable Token newLine, @Nullable Token indent, StatementList statements,
+  public WithStatementImpl(Token withKeyword, List<WithItem> withItems, List<Token> commas, Token colon, @Nullable Token newLine, @Nullable Token indent, StatementList statements,
                            @Nullable Token dedent, @Nullable Token asyncKeyword) {
     this.withKeyword = withKeyword;
     this.withItems = withItems;
+    this.commas = commas;
     this.colon = colon;
     this.newLine = newLine;
     this.indent = indent;
@@ -102,8 +105,17 @@ public void accept(TreeVisitor visitor) {
 
   @Override
   public List<Tree> computeChildren() {
-    return Stream.of(Arrays.asList(asyncKeyword, withKeyword), withItems, Arrays.asList(colon, newLine, indent, statements, dedent))
-      .flatMap(List::stream).filter(Objects::nonNull).collect(Collectors.toList());
+    List<Tree> children = new ArrayList<>(Arrays.asList(asyncKeyword, withKeyword));
+    int i = 0;
+    for (Tree item : withItems) {
+      children.add(item);
+      if (i < commas.size()) {
+        children.add(commas.get(i));
+      }
+      i++;
+    }
+    children.addAll(Arrays.asList(colon, newLine, indent, statements, dedent));
+    return children.stream().filter(Objects::nonNull).collect(Collectors.toList());
   }
 
   public static class WithItemImpl extends PyTree implements WithItem {

python-squid/src/test/java/org/sonar/python/semantic/SymbolTableBuilderTreeTest.java → python-squid/src/test/java/org/sonar/python/semantic/SymbolTableBuilderTest.java

@@ -40,7 +40,7 @@
 
 import static org.assertj.core.api.Assertions.assertThat;
 
-public class SymbolTableBuilderTreeTest {
+public class SymbolTableBuilderTest {
   private static Map<String, FunctionDef> functionTreesByName = new HashMap<>();
   private static FileInput fileInput;
 
@@ -236,6 +236,17 @@ public void function_with_tuple_param() {
     assertThat(symbolByName).hasSize(4);
   }
 
+  @Test
+  public void function_with_star_param() {
+    FunctionDef functionTree = functionTreesByName.get("func_with_star_param");
+    Map<String, Symbol> symbolByName = getSymbolByName(functionTree);
+    assertThat(symbolByName).hasSize(2);
+
+    functionTree = functionTreesByName.get("method_with_star_param");
+    symbolByName = getSymbolByName(functionTree);
+    assertThat(symbolByName).hasSize(1);
+  }
+
   private static class TestVisitor extends BaseTreeVisitor {
     @Override
     public void visitFunctionDef(FunctionDef pyFunctionDefTree) {

python-squid/src/test/java/org/sonar/python/tree/PythonTreeMakerTest.java

@@ -22,6 +22,7 @@
 import com.sonar.sslr.api.AstNode;
 import com.sonar.sslr.api.GenericTokenType;
 import com.sonar.sslr.api.RecognitionException;
+import com.sonar.sslr.api.TokenType;
 import java.io.File;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
@@ -31,6 +32,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.function.Function;
+import java.util.stream.Collectors;
 import org.junit.Test;
 import org.sonar.python.api.PythonGrammar;
 import org.sonar.python.api.PythonPunctuator;
@@ -827,6 +829,11 @@ public void funcdef_statement() {
       "This is a function docstring\n" +
       "\"\"\"");
     assertThat(functionDef.children()).hasSize(10);
+
+    functionDef = parse("def __call__(self, *, manager):\n  pass", treeMaker::funcDefStatement);
+    assertThat(functionDef.parameters().all()).hasSize(3);
+    functionDef = parse("def __call__(*):\n  pass", treeMaker::funcDefStatement);
+    assertThat(functionDef.parameters().all()).hasSize(1);
   }
 
   @Test
@@ -1225,7 +1232,17 @@ public void try_statement() {
     assertThat(exceptClause.asKeyword()).isNull();
     assertThat(exceptClause.commaToken().value()).isEqualTo(",");
     assertThat(exceptClause.exceptionInstance()).isNotNull();
+    assertThat(exceptClause.children())
+      .containsExactly(exceptClause.exceptKeyword(), exceptClause.exception(), exceptClause.commaToken(), exceptClause.exceptionInstance(),
+        /*colon token is not accessible through API*/ exceptClause.children().get(4), exceptClause.body());
     assertThat(tryStatement.children()).hasSize(4);
+
+    astNode = p.parse("try:\n    pass\nexcept Error:\n    pass\nelse:\n    pass\nfinally:\n    pass");
+    tryStatement = treeMaker.tryStatement(astNode);
+    List<Tree> children = tryStatement.children();
+    assertThat(children.get(children.size() - 2)).isSameAs(tryStatement.elseClause());
+    assertThat(children.get(children.size() - 1)).isSameAs(tryStatement.finallyClause());
+
   }
 
   @Test
@@ -1288,7 +1305,7 @@ public void with_statement() {
     assertThat(withItem.expression()).isNull();
     assertThat(withStatement.statements().statements()).hasSize(1);
     assertThat(withStatement.statements().statements().get(0).is(Tree.Kind.PASS_STMT)).isTrue();
-    assertThat(withStatement.children()).hasSize(8);
+    assertThat(withStatement.children()).hasSize(9);
   }
 
   @Test
@@ -2205,6 +2222,17 @@ private <T extends Tree> T parse(String code, Function<AstNode, T> func) {
     // ensure every visit method of base tree visitor is called without errors
     BaseTreeVisitor visitor = new BaseTreeVisitor();
     tree.accept(visitor);
+    List<TokenType> ptt = Arrays.asList(PythonTokenType.NEWLINE, PythonTokenType.DEDENT, PythonTokenType.INDENT, GenericTokenType.EOF);
+    List<Token> tokenList = TreeUtils.tokens(tree);
+
+    String tokens = tokenList.stream().filter(t -> !ptt.contains(t.type())).map(token -> {
+      if(token.type() == PythonTokenType.STRING) {
+        return token.value().replaceAll("\n", "").replaceAll(" ", "");
+      }
+      return token.value();
+    }).collect(Collectors.joining(""));
+    String originalCode = code.replaceAll("#.*\\n", "").replaceAll("\\n", "").replaceAll(" ", "");
+    assertThat(tokens).isEqualTo(originalCode);
     return tree;
   }
 

python-squid/src/test/resources/semantic/symbols2.py

@@ -76,3 +76,11 @@ def binding_usages(param):
 
 def func_with_tuple_param((a, (b, c)), d):
   pass
+
+
+def func_with_star_param(a, *, d):
+  pass
+
+class a:
+  def method_with_star_param(*, d):
+    pass