diff --git a/python-squid/src/main/java/org/sonar/python/api/tree/FileInput.java b/python-squid/src/main/java/org/sonar/python/api/tree/FileInput.java index bc6d62757..a0f1c8784 100644 --- a/python-squid/src/main/java/org/sonar/python/api/tree/FileInput.java +++ b/python-squid/src/main/java/org/sonar/python/api/tree/FileInput.java @@ -19,7 +19,9 @@ */ package org.sonar.python.api.tree; +import java.util.Set; import javax.annotation.CheckForNull; +import org.sonar.python.semantic.Symbol; public interface FileInput extends Tree { @CheckForNull @@ -27,4 +29,6 @@ public interface FileInput extends Tree { @CheckForNull Token docstring(); + + Set globalVariables(); } diff --git a/python-squid/src/main/java/org/sonar/python/semantic/SymbolTableBuilder.java b/python-squid/src/main/java/org/sonar/python/semantic/SymbolTableBuilder.java index b95610e2d..36fad200c 100644 --- a/python-squid/src/main/java/org/sonar/python/semantic/SymbolTableBuilder.java +++ b/python-squid/src/main/java/org/sonar/python/semantic/SymbolTableBuilder.java @@ -62,6 +62,7 @@ import org.sonar.python.api.tree.TupleParameter; import org.sonar.python.tree.BaseTreeVisitor; import org.sonar.python.tree.ClassDefImpl; +import org.sonar.python.tree.FileInputImpl; import org.sonar.python.tree.FunctionDefImpl; import org.sonar.python.tree.LambdaExpressionImpl; import org.sonar.python.tree.NameImpl; @@ -77,9 +78,8 @@ public void visitFileInput(FileInput fileInput) { scopesByRootTree = new HashMap<>(); fileInput.accept(new FirstPhaseVisitor()); fileInput.accept(new SecondPhaseVisitor()); - scopesByRootTree.values().stream() - .filter(scope -> scope.rootTree instanceof FunctionLike) - .forEach(scope -> { + for (Scope scope : scopesByRootTree.values()) { + if (scope.rootTree instanceof FunctionLike) { FunctionLike funcDef = (FunctionLike) scope.rootTree; for (Symbol symbol : scope.symbols()) { if (funcDef.is(Kind.LAMBDA)) { @@ -88,14 +88,14 @@ public void visitFileInput(FileInput fileInput) { ((FunctionDefImpl) funcDef).addLocalVariableSymbol(symbol); } } - }); - scopesByRootTree.values().stream() - .filter(scope -> scope.rootTree.is(Kind.CLASSDEF)) - .forEach(scope -> { + } else if (scope.rootTree.is(Kind.CLASSDEF)) { ClassDefImpl classDef = (ClassDefImpl) scope.rootTree; scope.symbols.forEach(classDef::addClassField); scope.instanceAttributesByName.values().forEach(classDef::addInstanceField); - }); + } else if (scope.rootTree.is(Kind.FILE_INPUT)) { + scope.symbols.forEach(((FileInputImpl) fileInput)::addGlobalVariables); + } + } } private static class ScopeVisitor extends BaseTreeVisitor { diff --git a/python-squid/src/main/java/org/sonar/python/tree/FileInputImpl.java b/python-squid/src/main/java/org/sonar/python/tree/FileInputImpl.java index 6d5ef43bb..fd018fd10 100644 --- a/python-squid/src/main/java/org/sonar/python/tree/FileInputImpl.java +++ b/python-squid/src/main/java/org/sonar/python/tree/FileInputImpl.java @@ -19,8 +19,10 @@ */ package org.sonar.python.tree; +import java.util.HashSet; import java.util.List; import java.util.Objects; +import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.CheckForNull; @@ -30,12 +32,14 @@ import org.sonar.python.api.tree.Token; import org.sonar.python.api.tree.Tree; import org.sonar.python.api.tree.TreeVisitor; +import org.sonar.python.semantic.Symbol; public class FileInputImpl extends PyTree implements FileInput { private final StatementList statements; private final Token endOfFile; private final Token docstring; + private final Set globalVariables = new HashSet<>(); public FileInputImpl(@Nullable StatementList statements, Token endOfFile, Token docstring) { super(statements == null ? endOfFile : statements.firstToken(), endOfFile); @@ -61,6 +65,15 @@ public Token docstring() { return docstring; } + @Override + public Set globalVariables() { + return globalVariables; + } + + public void addGlobalVariables(Symbol globalVariable) { + globalVariables.add(globalVariable); + } + @Override public void accept(TreeVisitor visitor) { visitor.visitFileInput(this); diff --git a/python-squid/src/test/java/org/sonar/python/semantic/SymbolTableBuilderTreeTest.java b/python-squid/src/test/java/org/sonar/python/semantic/SymbolTableBuilderTreeTest.java index 8534ea0fb..43931c36c 100644 --- a/python-squid/src/test/java/org/sonar/python/semantic/SymbolTableBuilderTreeTest.java +++ b/python-squid/src/test/java/org/sonar/python/semantic/SymbolTableBuilderTreeTest.java @@ -24,6 +24,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; import org.junit.BeforeClass; import org.junit.Test; @@ -40,6 +41,7 @@ public class SymbolTableBuilderTreeTest { private static Map functionTreesByName = new HashMap<>(); + private static FileInput fileInput; private Map getSymbolByName(FunctionDef functionTree) { @@ -49,10 +51,17 @@ private Map getSymbolByName(FunctionDef functionTree) { @BeforeClass public static void init() { PythonVisitorContext context = TestPythonVisitorRunner.createContext(new File("src/test/resources/semantic/symbols2.py")); - FileInput fileInput = context.rootTree(); + fileInput = context.rootTree(); fileInput.accept(new TestVisitor()); } + @Test + public void global_variable() { + Set moduleSymbols = fileInput.globalVariables(); + assertThat(moduleSymbols.size()).isEqualTo(2); + assertThat(moduleSymbols).extracting(Symbol::name).containsExactlyInAnyOrder("global_x", "global_var"); + } + @Test public void local_variable() { FunctionDef functionTree = functionTreesByName.get("function_with_local"); diff --git a/sonar-python-plugin/src/main/java/org/sonar/plugins/python/PythonScanner.java b/sonar-python-plugin/src/main/java/org/sonar/plugins/python/PythonScanner.java index 6539a77e8..4b3b61d30 100644 --- a/sonar-python-plugin/src/main/java/org/sonar/plugins/python/PythonScanner.java +++ b/sonar-python-plugin/src/main/java/org/sonar/plugins/python/PythonScanner.java @@ -120,6 +120,7 @@ private void scanFile(InputFile inputFile) { SubscriptionVisitor.analyze(checksBasedOnTree, visitorContext); saveIssues(inputFile, visitorContext.getIssues()); + new SymbolVisitor(context.newSymbolTable().onFile(inputFile)).visitFileInput(visitorContext.rootTree()); new PythonHighlighter(context, inputFile).scanFile(visitorContext); } diff --git a/sonar-python-plugin/src/main/java/org/sonar/plugins/python/SymbolVisitor.java b/sonar-python-plugin/src/main/java/org/sonar/plugins/python/SymbolVisitor.java new file mode 100644 index 000000000..86abb8d4e --- /dev/null +++ b/sonar-python-plugin/src/main/java/org/sonar/plugins/python/SymbolVisitor.java @@ -0,0 +1,82 @@ +/* + * SonarQube Python Plugin + * Copyright (C) 2011-2019 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.plugins.python; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import org.sonar.api.batch.sensor.symbol.NewSymbol; +import org.sonar.api.batch.sensor.symbol.NewSymbolTable; +import org.sonar.python.api.tree.ClassDef; +import org.sonar.python.api.tree.FileInput; +import org.sonar.python.api.tree.FunctionDef; +import org.sonar.python.api.tree.LambdaExpression; +import org.sonar.python.api.tree.Tree; +import org.sonar.python.semantic.Symbol; +import org.sonar.python.semantic.Usage; +import org.sonar.python.tree.BaseTreeVisitor; + +public class SymbolVisitor extends BaseTreeVisitor { + + private final NewSymbolTable newSymbolTable; + + public SymbolVisitor(NewSymbolTable newSymbolTable) { + this.newSymbolTable = newSymbolTable; + } + + @Override + public void visitClassDef(ClassDef classDef) { + classDef.classFields().forEach(this::handleSymbol); + classDef.instanceFields().forEach(this::handleSymbol); + super.visitClassDef(classDef); + } + + @Override + public void visitFunctionDef(FunctionDef functionDef) { + functionDef.localVariables().forEach(this::handleSymbol); + super.visitFunctionDef(functionDef); + } + + @Override + public void visitLambda(LambdaExpression lambdaExpression) { + lambdaExpression.localVariables().forEach(this::handleSymbol); + super.visitLambda(lambdaExpression); + } + + @Override + public void visitFileInput(FileInput fileInput) { + fileInput.globalVariables().forEach(this::handleSymbol); + super.visitFileInput(fileInput); + newSymbolTable.save(); + } + + private void handleSymbol(Symbol symbol) { + List usages = new ArrayList<>(symbol.usages()); + usages.sort(Comparator.comparingInt(u -> u.tree().firstToken().line())); + Tree firstUsageTree = usages.get(0).tree(); + NewSymbol newSymbol = newSymbolTable.newSymbol(firstUsageTree.firstToken().line(), firstUsageTree.firstToken().column(), + firstUsageTree.lastToken().line(), firstUsageTree.lastToken().column() + firstUsageTree.lastToken().value().length()); + for (int i = 1; i < usages.size(); i++) { + Tree usageTree = usages.get(i).tree(); + newSymbol.newReference(usageTree.firstToken().line(), usageTree.firstToken().column(), + usageTree.lastToken().line(), usageTree.lastToken().column() + usageTree.lastToken().value().length()); + } + } +} diff --git a/sonar-python-plugin/src/test/java/org/sonar/plugins/python/PythonSquidSensorTest.java b/sonar-python-plugin/src/test/java/org/sonar/plugins/python/PythonSquidSensorTest.java index 8d06f0dc2..c49a41a22 100644 --- a/sonar-python-plugin/src/test/java/org/sonar/plugins/python/PythonSquidSensorTest.java +++ b/sonar-python-plugin/src/test/java/org/sonar/plugins/python/PythonSquidSensorTest.java @@ -22,6 +22,7 @@ import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.Collection; import java.util.Iterator; import org.junit.Before; import org.junit.Test; @@ -30,7 +31,10 @@ import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.InputFile.Type; import org.sonar.api.batch.fs.TextPointer; +import org.sonar.api.batch.fs.TextRange; import org.sonar.api.batch.fs.internal.DefaultInputFile; +import org.sonar.api.batch.fs.internal.DefaultTextPointer; +import org.sonar.api.batch.fs.internal.DefaultTextRange; import org.sonar.api.batch.fs.internal.TestInputFileBuilder; import org.sonar.api.batch.rule.ActiveRules; import org.sonar.api.batch.rule.CheckFactory; @@ -60,6 +64,7 @@ public class PythonSquidSensorTest { private static final String FILE_1 = "file1.py"; + private static final String FILE_2 = "file2.py"; private static final String ONE_STATEMENT_PER_LINE_RULE_KEY = "OneStatementPerLine"; private static final String FILE_COMPLEXITY_RULE_KEY = "FileComplexity"; @@ -84,7 +89,7 @@ public void init() { @Test public void sensor_descriptor() { - activeRules = (new ActiveRulesBuilder()).build(); + activeRules = new ActiveRulesBuilder().build(); DefaultSensorDescriptor descriptor = new DefaultSensorDescriptor(); sensor().describe(descriptor); @@ -125,6 +130,27 @@ public void test_execute_on_sonarlint() { assertThat(context.allAnalysisErrors()).isEmpty(); } + @Test + public void test_symbol_visitor() { + activeRules = new ActiveRulesBuilder().build(); + inputFile(FILE_2); + inputFile("symbolVisitor.py"); + sensor().execute(context); + + String key = "moduleKey:file2.py"; + assertThat(context.referencesForSymbolAt(key, 1, 10)).isNull(); + verifyUsages(key, 3, 4, reference(4, 10, 4, 11), + reference(6, 15, 6, 16), reference(7, 19, 7, 20)); + verifyUsages(key, 5, 12, reference(6, 19, 6, 20)); + + key = "moduleKey:symbolVisitor.py"; + assertThat(context.referencesForSymbolAt(key, 1, 10)).isNull(); + verifyUsages(key, 1, 0); + verifyUsages(key, 2, 0, reference(10, 4, 10, 5)); + verifyUsages(key, 5, 4, reference(6, 4, 6, 5), reference(7, 4, 7, 5), + reference(8, 8, 8, 9), reference(13, 9, 13, 10)); + } + @Test public void test_issues() { activeRules = new ActiveRulesBuilder() @@ -140,7 +166,7 @@ public void test_issues() { .build()) .build(); - InputFile inputFile = inputFile("file2.py"); + InputFile inputFile = inputFile(FILE_2); sensor().execute(context); assertThat(context.allIssues()).hasSize(3); @@ -201,7 +227,7 @@ public void test_exception_does_not_fail_analysis() throws IOException { when(inputFile.contents()).thenThrow(RuntimeException.class); context.fileSystem().add(inputFile); - inputFile("file2.py"); + inputFile(FILE_2); sensor().execute(context); @@ -258,4 +284,12 @@ private InputFile inputFile(String name) { return inputFile; } + private void verifyUsages(String componentKey, int line, int offset, TextRange... trs) { + Collection textRanges = context.referencesForSymbolAt(componentKey, line, offset); + assertThat(textRanges).containsExactly(trs); + } + + private static TextRange reference(int lineStart, int columnStart, int lineEnd, int columnEnd) { + return new DefaultTextRange(new DefaultTextPointer(lineStart, columnStart), new DefaultTextPointer(lineEnd, columnEnd)); + } } diff --git a/sonar-python-plugin/src/test/java/org/sonar/plugins/python/SymbolVisitorTest.java b/sonar-python-plugin/src/test/java/org/sonar/plugins/python/SymbolVisitorTest.java new file mode 100644 index 000000000..bf38d8b78 --- /dev/null +++ b/sonar-python-plugin/src/test/java/org/sonar/plugins/python/SymbolVisitorTest.java @@ -0,0 +1,86 @@ +/* + * SonarQube Python Plugin + * Copyright (C) 2011-2019 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.plugins.python; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.util.Collection; +import org.junit.BeforeClass; +import org.junit.Test; +import org.sonar.api.batch.fs.TextRange; +import org.sonar.api.batch.fs.internal.DefaultInputFile; +import org.sonar.api.batch.fs.internal.DefaultTextPointer; +import org.sonar.api.batch.fs.internal.DefaultTextRange; +import org.sonar.api.batch.fs.internal.TestInputFileBuilder; +import org.sonar.api.batch.sensor.internal.SensorContextTester; +import org.sonar.python.PythonVisitorContext; +import org.sonar.python.TestPythonVisitorRunner; +import org.sonar.python.api.tree.FileInput; + +import static org.assertj.core.api.Assertions.assertThat; + +public class SymbolVisitorTest { + + private static SensorContextTester context; + private static String componentKey; + + @BeforeClass + public static void scanFile() { + File file = new File("src/test/resources/org/sonar/plugins/python/squid-sensor", "/symbolVisitor.py"); + DefaultInputFile inputFile = TestInputFileBuilder.create("moduleKey", file.getName()) + .initMetadata(TestUtils.fileContent(file, StandardCharsets.UTF_8)) + .build(); + + context = SensorContextTester.create(file); + context.fileSystem().add(inputFile); + componentKey = inputFile.key(); + + SymbolVisitor symbolVisitor = new SymbolVisitor(context.newSymbolTable().onFile(inputFile)); + PythonVisitorContext context = TestPythonVisitorRunner.createContext(file); + FileInput fileInput = context.rootTree(); + fileInput.accept(symbolVisitor); + } + + @Test + public void symbol_visitor() { + assertThat(context.referencesForSymbolAt(componentKey, 1, 10)).isNull(); + verifyUsages(1, 0); + verifyUsages(2, 0, reference(10, 4, 10, 5)); + verifyUsages(5, 4, reference(6, 4, 6, 5), reference(7, 4, 7, 5), + reference(8, 8, 8, 9), reference(13, 9, 13, 10)); + verifyUsages(9, 4); + verifyUsages(16, 4); + verifyUsages(23, 13); + verifyUsages(19, 13, reference(22, 13, 22, 14), reference(26, 13, 26, 14)); + verifyUsages(18, 17, reference(19, 8, 19, 12)); + verifyUsages(21, 18, reference(22, 8, 22, 12), reference(23, 8, 23, 12)); + verifyUsages(25, 19, reference(26, 8, 26, 12)); + verifyUsages(28, 11, reference(28, 15, 28, 16), reference(28, 17, 28, 18)); + } + + private void verifyUsages(int line, int offset, TextRange... trs) { + Collection textRanges = context.referencesForSymbolAt(componentKey, line, offset); + assertThat(textRanges).containsExactly(trs); + } + + private static TextRange reference(int lineStart, int columnStart, int lineEnd, int columnEnd) { + return new DefaultTextRange(new DefaultTextPointer(lineStart, columnStart), new DefaultTextPointer(lineEnd, columnEnd)); + } +} diff --git a/sonar-python-plugin/src/test/resources/org/sonar/plugins/python/squid-sensor/symbolVisitor.py b/sonar-python-plugin/src/test/resources/org/sonar/plugins/python/squid-sensor/symbolVisitor.py new file mode 100644 index 000000000..0783afd17 --- /dev/null +++ b/sonar-python-plugin/src/test/resources/org/sonar/plugins/python/squid-sensor/symbolVisitor.py @@ -0,0 +1,28 @@ +a = 3 +b = foo() + +def function_with_local(): + a = 11 + a += 1 + a.x = 1 + foo(a) + t2: str = "abc" + b.a = 1 + foo().x *= 1 + foo().x : int = 1 + toto(a) + +class clazz: + field = "a" + + def __init__(self): + self.a = "abc" + + def some_func(self): + self.a = "u" + self.field = "b" + + def other_func(self): + self.a = "c" + +l = lambda z : z*z