diff --git a/its/ruling/src/test/resources/expected/python-S107.json b/its/ruling/src/test/resources/expected/python-S107.json index c985f135c..d1d131875 100644 --- a/its/ruling/src/test/resources/expected/python-S107.json +++ b/its/ruling/src/test/resources/expected/python-S107.json @@ -196,6 +196,9 @@ 12, 52, ], +'project:django-2.2.3/django/contrib/gis/db/models/fields.py':[ +207, +], 'project:django-2.2.3/django/contrib/gis/utils/layermapping.py':[ 80, 476, @@ -263,6 +266,7 @@ ], 'project:django-2.2.3/django/forms/fields.py':[ 57, +1079, ], 'project:django-2.2.3/django/forms/forms.py':[ 74, diff --git a/python-squid/src/main/java/org/sonar/python/api/tree/Parameter.java b/python-squid/src/main/java/org/sonar/python/api/tree/Parameter.java index ae1c076df..a297c5acc 100644 --- a/python-squid/src/main/java/org/sonar/python/api/tree/Parameter.java +++ b/python-squid/src/main/java/org/sonar/python/api/tree/Parameter.java @@ -26,6 +26,10 @@ public interface Parameter extends AnyParameter { @CheckForNull Token starToken(); + /** + * @return null only in case of star parameter + */ + @CheckForNull Name name(); @CheckForNull diff --git a/python-squid/src/main/java/org/sonar/python/semantic/SymbolTableBuilder.java b/python-squid/src/main/java/org/sonar/python/semantic/SymbolTableBuilder.java index 36fad200c..82f85d361 100644 --- a/python-squid/src/main/java/org/sonar/python/semantic/SymbolTableBuilder.java +++ b/python-squid/src/main/java/org/sonar/python/semantic/SymbolTableBuilder.java @@ -27,12 +27,12 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.CheckForNull; import javax.annotation.Nullable; -import org.sonar.python.api.tree.HasSymbol; import org.sonar.python.api.tree.AliasedName; import org.sonar.python.api.tree.AnnotatedAssignment; import org.sonar.python.api.tree.AnyParameter; @@ -48,17 +48,18 @@ import org.sonar.python.api.tree.FunctionDef; import org.sonar.python.api.tree.FunctionLike; import org.sonar.python.api.tree.GlobalStatement; +import org.sonar.python.api.tree.HasSymbol; import org.sonar.python.api.tree.ImportFrom; import org.sonar.python.api.tree.ImportName; import org.sonar.python.api.tree.LambdaExpression; import org.sonar.python.api.tree.Name; import org.sonar.python.api.tree.NonlocalStatement; -import org.sonar.python.api.tree.ParameterList; import org.sonar.python.api.tree.Parameter; +import org.sonar.python.api.tree.ParameterList; import org.sonar.python.api.tree.QualifiedExpression; -import org.sonar.python.api.tree.Tuple; import org.sonar.python.api.tree.Tree; import org.sonar.python.api.tree.Tree.Kind; +import org.sonar.python.api.tree.Tuple; import org.sonar.python.api.tree.TupleParameter; import org.sonar.python.tree.BaseTreeVisitor; import org.sonar.python.tree.ClassDefImpl; @@ -223,7 +224,9 @@ private void createParameters(FunctionLike function) { parameterList.nonTuple() .stream() .skip(hasSelf ? 1 : 0) - .forEach(param -> addBindingUsage(param.name(), Usage.Kind.PARAMETER)); + .map(Parameter::name) + .filter(Objects::nonNull) + .forEach(param -> addBindingUsage(param, Usage.Kind.PARAMETER)); parameterList.all().stream() .filter(param -> param.is(Kind.TUPLE_PARAMETER)) @@ -340,11 +343,13 @@ private Set symbols() { private void createSelfParameter(Parameter parameter) { Name nameTree = parameter.name(); - String symbolName = nameTree.name(); - SymbolImpl symbol = new SelfSymbolImpl(symbolName, parent); - symbols.add(symbol); - symbolsByName.put(symbolName, symbol); - symbol.addUsage(nameTree, Usage.Kind.PARAMETER); + if (nameTree != null) { + String symbolName = nameTree.name(); + SymbolImpl symbol = new SelfSymbolImpl(symbolName, parent); + symbols.add(symbol); + symbolsByName.put(symbolName, symbol); + symbol.addUsage(nameTree, Usage.Kind.PARAMETER); + } } void addBindingUsage(Name nameTree, Usage.Kind kind, @Nullable String fullyQualifiedName) { diff --git a/python-squid/src/main/java/org/sonar/python/tree/ArgumentImpl.java b/python-squid/src/main/java/org/sonar/python/tree/ArgumentImpl.java index 8b35d16e0..fecd4f5ad 100644 --- a/python-squid/src/main/java/org/sonar/python/tree/ArgumentImpl.java +++ b/python-squid/src/main/java/org/sonar/python/tree/ArgumentImpl.java @@ -96,6 +96,6 @@ public Kind getKind() { @Override public List computeChildren() { - return Stream.of(keywordArgument, equalToken, expression, star, starStar).filter(Objects::nonNull).collect(Collectors.toList()); + return Stream.of(keywordArgument, equalToken, star, starStar, expression).filter(Objects::nonNull).collect(Collectors.toList()); } } diff --git a/python-squid/src/main/java/org/sonar/python/tree/ComprehensionExpressionImpl.java b/python-squid/src/main/java/org/sonar/python/tree/ComprehensionExpressionImpl.java index 5b97c3b4b..953e7ceb9 100644 --- a/python-squid/src/main/java/org/sonar/python/tree/ComprehensionExpressionImpl.java +++ b/python-squid/src/main/java/org/sonar/python/tree/ComprehensionExpressionImpl.java @@ -23,6 +23,7 @@ import java.util.Objects; import java.util.stream.Collectors; import java.util.stream.Stream; +import javax.annotation.Nullable; import org.sonar.python.api.tree.ComprehensionExpression; import org.sonar.python.api.tree.ComprehensionFor; import org.sonar.python.api.tree.Expression; @@ -38,8 +39,8 @@ public class ComprehensionExpressionImpl extends PyTree implements Comprehension private final ComprehensionFor comprehensionFor; private final Token closingToken; - public ComprehensionExpressionImpl(Kind kind, Token openingToken, Expression resultExpression, - ComprehensionFor compFor, Token closingToken) { + public ComprehensionExpressionImpl(Kind kind, @Nullable Token openingToken, Expression resultExpression, + ComprehensionFor compFor, @Nullable Token closingToken) { this.kind = kind; this.resultExpression = resultExpression; this.comprehensionFor = compFor; diff --git a/python-squid/src/main/java/org/sonar/python/tree/ExceptClauseImpl.java b/python-squid/src/main/java/org/sonar/python/tree/ExceptClauseImpl.java index 7a33b6ee4..d8168a1d9 100644 --- a/python-squid/src/main/java/org/sonar/python/tree/ExceptClauseImpl.java +++ b/python-squid/src/main/java/org/sonar/python/tree/ExceptClauseImpl.java @@ -130,7 +130,7 @@ public void accept(TreeVisitor visitor) { @Override public List computeChildren() { - return Stream.of(exceptKeyword, exception, asKeyword, exceptionInstance, commaToken, colon, newLine, indent, body, dedent) + return Stream.of(exceptKeyword, exception, asKeyword, commaToken, exceptionInstance, colon, newLine, indent, body, dedent) .filter(Objects::nonNull).collect(Collectors.toList()); } } diff --git a/python-squid/src/main/java/org/sonar/python/tree/KeyValuePairImpl.java b/python-squid/src/main/java/org/sonar/python/tree/KeyValuePairImpl.java index 7ff431ff2..311d21ce3 100644 --- a/python-squid/src/main/java/org/sonar/python/tree/KeyValuePairImpl.java +++ b/python-squid/src/main/java/org/sonar/python/tree/KeyValuePairImpl.java @@ -91,7 +91,7 @@ public void accept(TreeVisitor visitor) { @Override public List computeChildren() { - return Stream.of(expression, key, colon, value, starStarToken).filter(Objects::nonNull).collect(Collectors.toList()); + return Stream.of(starStarToken, expression, key, colon, value).filter(Objects::nonNull).collect(Collectors.toList()); } @Override diff --git a/python-squid/src/main/java/org/sonar/python/tree/ParameterImpl.java b/python-squid/src/main/java/org/sonar/python/tree/ParameterImpl.java index 5bc6972ff..972fba69e 100644 --- a/python-squid/src/main/java/org/sonar/python/tree/ParameterImpl.java +++ b/python-squid/src/main/java/org/sonar/python/tree/ParameterImpl.java @@ -50,12 +50,25 @@ public ParameterImpl(@Nullable Token starToken, Name name, @Nullable TypeAnnotat this.defaultValue = defaultValue; } + /** + * constructor for star parameter syntax. + * def fun(arg1, *, arg2) + */ + public ParameterImpl(Token starToken) { + this.starToken = starToken; + this.name = null; + this.annotation = null; + this.equalToken = null; + this.defaultValue = null; + } + @CheckForNull @Override public Token starToken() { return starToken; } + @CheckForNull @Override public Name name() { return name; diff --git a/python-squid/src/main/java/org/sonar/python/tree/PythonTreeMaker.java b/python-squid/src/main/java/org/sonar/python/tree/PythonTreeMaker.java index a29204ab1..4777e8b6c 100644 --- a/python-squid/src/main/java/org/sonar/python/tree/PythonTreeMaker.java +++ b/python-squid/src/main/java/org/sonar/python/tree/PythonTreeMaker.java @@ -25,6 +25,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; import javax.annotation.CheckForNull; import javax.annotation.Nullable; @@ -516,8 +517,8 @@ public FunctionDef funcDefStatement(AstNode astNode) { ParameterList parameterList = null; AstNode typedArgListNode = astNode.getFirstChild(PythonGrammar.TYPEDARGSLIST); if (typedArgListNode != null) { - List arguments = typedArgListNode.getChildren(PythonGrammar.TFPDEF).stream() - .map(this::parameter).collect(Collectors.toList()); + List arguments = typedArgListNode.getChildren(PythonGrammar.TFPDEF, PythonPunctuator.MUL).stream() + .map(this::parameter).filter(Objects::nonNull).collect(Collectors.toList()); List commas = punctuators(typedArgListNode, PythonPunctuator.COMMA); parameterList = new ParameterListImpl(arguments, commas); } @@ -712,11 +713,12 @@ public WithStatement withStatement(AstNode astNode) { asyncKeyword = toPyToken(astNode.getFirstChild().getToken()); } List withItems = withItems(withStmtNode.getChildren(PythonGrammar.WITH_ITEM)); + List commas = punctuators(withStmtNode, PythonPunctuator.COMMA); AstNode suite = withStmtNode.getFirstChild(PythonGrammar.SUITE); Token withKeyword = toPyToken(withStmtNode.getToken()); Token colon = toPyToken(suite.getPreviousSibling().getToken()); StatementList body = getStatementListFromSuite(suite); - return new WithStatementImpl(withKeyword, withItems, colon, suiteNewLine(suite), suiteIndent(suite), body, suiteDedent(suite), asyncKeyword); + return new WithStatementImpl(withKeyword, withItems, commas, colon, suiteNewLine(suite), suiteIndent(suite), body, suiteDedent(suite), asyncKeyword); } private List withItems(List withItems) { @@ -1136,7 +1138,7 @@ public Argument argument(AstNode astNode) { if (compFor != null) { Expression expression = expression(astNode.getFirstChild()); ComprehensionExpression comprehension = - new ComprehensionExpressionImpl(Tree.Kind.GENERATOR_EXPR, expression.firstToken(), expression, compFor(compFor), toPyToken(compFor.getLastToken())); + new ComprehensionExpressionImpl(Tree.Kind.GENERATOR_EXPR, null, expression, compFor(compFor), null); return new ArgumentImpl(comprehension, null, null); } AstNode assign = astNode.getFirstChild(PythonPunctuator.ASSIGN); @@ -1187,6 +1189,12 @@ public LambdaExpression lambdaExpression(AstNode astNode) { } private AnyParameter parameter(AstNode parameter) { + if(parameter.is(PythonPunctuator.MUL)) { + if(parameter.getNextSibling()==null || parameter.getNextSibling().is(PythonPunctuator.COMMA)) { + return new ParameterImpl(toPyToken(parameter.getToken())); + } + return null; + } AstNode prevSibling = parameter.getPreviousSibling(); if (parameter.is(PythonGrammar.NAME)) { diff --git a/python-squid/src/main/java/org/sonar/python/tree/TryStatementImpl.java b/python-squid/src/main/java/org/sonar/python/tree/TryStatementImpl.java index 13921fc90..a7f6dab11 100644 --- a/python-squid/src/main/java/org/sonar/python/tree/TryStatementImpl.java +++ b/python-squid/src/main/java/org/sonar/python/tree/TryStatementImpl.java @@ -98,7 +98,7 @@ public void accept(TreeVisitor visitor) { @Override public List computeChildren() { - return Stream.of(Arrays.asList(tryKeyword, colon, newLine, indent, tryBody, dedent), exceptClauses, Arrays.asList(finallyClause, elseClause)) + return Stream.of(Arrays.asList(tryKeyword, colon, newLine, indent, tryBody, dedent), exceptClauses, Arrays.asList(elseClause, finallyClause)) .flatMap(List::stream).filter(Objects::nonNull).collect(Collectors.toList()); } } diff --git a/python-squid/src/main/java/org/sonar/python/tree/WithStatementImpl.java b/python-squid/src/main/java/org/sonar/python/tree/WithStatementImpl.java index 4dfed8abc..18234eba2 100644 --- a/python-squid/src/main/java/org/sonar/python/tree/WithStatementImpl.java +++ b/python-squid/src/main/java/org/sonar/python/tree/WithStatementImpl.java @@ -19,6 +19,7 @@ */ package org.sonar.python.tree; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Objects; @@ -38,6 +39,7 @@ public class WithStatementImpl extends PyTree implements WithStatement { private final Token withKeyword; private final List withItems; + private final List commas; private final Token newLine; private final Token indent; private final StatementList statements; @@ -46,10 +48,11 @@ public class WithStatementImpl extends PyTree implements WithStatement { private final boolean isAsync; private final Token colon; - public WithStatementImpl(Token withKeyword, List withItems, Token colon, @Nullable Token newLine, @Nullable Token indent, StatementList statements, + public WithStatementImpl(Token withKeyword, List withItems, List commas, Token colon, @Nullable Token newLine, @Nullable Token indent, StatementList statements, @Nullable Token dedent, @Nullable Token asyncKeyword) { this.withKeyword = withKeyword; this.withItems = withItems; + this.commas = commas; this.colon = colon; this.newLine = newLine; this.indent = indent; @@ -102,8 +105,17 @@ public void accept(TreeVisitor visitor) { @Override public List computeChildren() { - return Stream.of(Arrays.asList(asyncKeyword, withKeyword), withItems, Arrays.asList(colon, newLine, indent, statements, dedent)) - .flatMap(List::stream).filter(Objects::nonNull).collect(Collectors.toList()); + List children = new ArrayList<>(Arrays.asList(asyncKeyword, withKeyword)); + int i = 0; + for (Tree item : withItems) { + children.add(item); + if (i < commas.size()) { + children.add(commas.get(i)); + } + i++; + } + children.addAll(Arrays.asList(colon, newLine, indent, statements, dedent)); + return children.stream().filter(Objects::nonNull).collect(Collectors.toList()); } public static class WithItemImpl extends PyTree implements WithItem { diff --git a/python-squid/src/test/java/org/sonar/python/semantic/SymbolTableBuilderTreeTest.java b/python-squid/src/test/java/org/sonar/python/semantic/SymbolTableBuilderTest.java similarity index 96% rename from python-squid/src/test/java/org/sonar/python/semantic/SymbolTableBuilderTreeTest.java rename to python-squid/src/test/java/org/sonar/python/semantic/SymbolTableBuilderTest.java index b16486487..0c9f91ed0 100644 --- a/python-squid/src/test/java/org/sonar/python/semantic/SymbolTableBuilderTreeTest.java +++ b/python-squid/src/test/java/org/sonar/python/semantic/SymbolTableBuilderTest.java @@ -40,7 +40,7 @@ import static org.assertj.core.api.Assertions.assertThat; -public class SymbolTableBuilderTreeTest { +public class SymbolTableBuilderTest { private static Map functionTreesByName = new HashMap<>(); private static FileInput fileInput; @@ -236,6 +236,17 @@ public void function_with_tuple_param() { assertThat(symbolByName).hasSize(4); } + @Test + public void function_with_star_param() { + FunctionDef functionTree = functionTreesByName.get("func_with_star_param"); + Map symbolByName = getSymbolByName(functionTree); + assertThat(symbolByName).hasSize(2); + + functionTree = functionTreesByName.get("method_with_star_param"); + symbolByName = getSymbolByName(functionTree); + assertThat(symbolByName).hasSize(1); + } + private static class TestVisitor extends BaseTreeVisitor { @Override public void visitFunctionDef(FunctionDef pyFunctionDefTree) { diff --git a/python-squid/src/test/java/org/sonar/python/tree/PythonTreeMakerTest.java b/python-squid/src/test/java/org/sonar/python/tree/PythonTreeMakerTest.java index 13370683c..81d3c2e63 100644 --- a/python-squid/src/test/java/org/sonar/python/tree/PythonTreeMakerTest.java +++ b/python-squid/src/test/java/org/sonar/python/tree/PythonTreeMakerTest.java @@ -22,6 +22,7 @@ import com.sonar.sslr.api.AstNode; import com.sonar.sslr.api.GenericTokenType; import com.sonar.sslr.api.RecognitionException; +import com.sonar.sslr.api.TokenType; import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -31,6 +32,7 @@ import java.util.List; import java.util.Map; import java.util.function.Function; +import java.util.stream.Collectors; import org.junit.Test; import org.sonar.python.api.PythonGrammar; import org.sonar.python.api.PythonPunctuator; @@ -827,6 +829,11 @@ public void funcdef_statement() { "This is a function docstring\n" + "\"\"\""); assertThat(functionDef.children()).hasSize(10); + + functionDef = parse("def __call__(self, *, manager):\n pass", treeMaker::funcDefStatement); + assertThat(functionDef.parameters().all()).hasSize(3); + functionDef = parse("def __call__(*):\n pass", treeMaker::funcDefStatement); + assertThat(functionDef.parameters().all()).hasSize(1); } @Test @@ -1225,7 +1232,17 @@ public void try_statement() { assertThat(exceptClause.asKeyword()).isNull(); assertThat(exceptClause.commaToken().value()).isEqualTo(","); assertThat(exceptClause.exceptionInstance()).isNotNull(); + assertThat(exceptClause.children()) + .containsExactly(exceptClause.exceptKeyword(), exceptClause.exception(), exceptClause.commaToken(), exceptClause.exceptionInstance(), + /*colon token is not accessible through API*/ exceptClause.children().get(4), exceptClause.body()); assertThat(tryStatement.children()).hasSize(4); + + astNode = p.parse("try:\n pass\nexcept Error:\n pass\nelse:\n pass\nfinally:\n pass"); + tryStatement = treeMaker.tryStatement(astNode); + List children = tryStatement.children(); + assertThat(children.get(children.size() - 2)).isSameAs(tryStatement.elseClause()); + assertThat(children.get(children.size() - 1)).isSameAs(tryStatement.finallyClause()); + } @Test @@ -1288,7 +1305,7 @@ public void with_statement() { assertThat(withItem.expression()).isNull(); assertThat(withStatement.statements().statements()).hasSize(1); assertThat(withStatement.statements().statements().get(0).is(Tree.Kind.PASS_STMT)).isTrue(); - assertThat(withStatement.children()).hasSize(8); + assertThat(withStatement.children()).hasSize(9); } @Test @@ -2205,6 +2222,17 @@ private T parse(String code, Function func) { // ensure every visit method of base tree visitor is called without errors BaseTreeVisitor visitor = new BaseTreeVisitor(); tree.accept(visitor); + List ptt = Arrays.asList(PythonTokenType.NEWLINE, PythonTokenType.DEDENT, PythonTokenType.INDENT, GenericTokenType.EOF); + List tokenList = TreeUtils.tokens(tree); + + String tokens = tokenList.stream().filter(t -> !ptt.contains(t.type())).map(token -> { + if(token.type() == PythonTokenType.STRING) { + return token.value().replaceAll("\n", "").replaceAll(" ", ""); + } + return token.value(); + }).collect(Collectors.joining("")); + String originalCode = code.replaceAll("#.*\\n", "").replaceAll("\\n", "").replaceAll(" ", ""); + assertThat(tokens).isEqualTo(originalCode); return tree; } diff --git a/python-squid/src/test/resources/semantic/symbols2.py b/python-squid/src/test/resources/semantic/symbols2.py index 5ce907f30..09904df69 100644 --- a/python-squid/src/test/resources/semantic/symbols2.py +++ b/python-squid/src/test/resources/semantic/symbols2.py @@ -76,3 +76,11 @@ def binding_usages(param): def func_with_tuple_param((a, (b, c)), d): pass + + +def func_with_star_param(a, *, d): + pass + +class a: + def method_with_star_param(*, d): + pass