SONARPY-432 Support symbol highlighting

python-squid/src/main/java/org/sonar/python/api/tree/FileInput.java

@@ -19,12 +19,16 @@
  */
 package org.sonar.python.api.tree;
 
+import java.util.Set;
 import javax.annotation.CheckForNull;
+import org.sonar.python.semantic.Symbol;
 
 public interface FileInput extends Tree {
   @CheckForNull
   StatementList statements();
 
   @CheckForNull
   Token docstring();
+
+  Set<Symbol> globalVariables();
 }

python-squid/src/main/java/org/sonar/python/semantic/SymbolTableBuilder.java

@@ -62,6 +62,7 @@
 import org.sonar.python.api.tree.TupleParameter;
 import org.sonar.python.tree.BaseTreeVisitor;
 import org.sonar.python.tree.ClassDefImpl;
+import org.sonar.python.tree.FileInputImpl;
 import org.sonar.python.tree.FunctionDefImpl;
 import org.sonar.python.tree.LambdaExpressionImpl;
 import org.sonar.python.tree.NameImpl;
@@ -77,9 +78,8 @@ public void visitFileInput(FileInput fileInput) {
     scopesByRootTree = new HashMap<>();
     fileInput.accept(new FirstPhaseVisitor());
     fileInput.accept(new SecondPhaseVisitor());
-    scopesByRootTree.values().stream()
-      .filter(scope -> scope.rootTree instanceof FunctionLike)
-      .forEach(scope -> {
+    for (Scope scope : scopesByRootTree.values()) {
+      if (scope.rootTree instanceof FunctionLike) {
         FunctionLike funcDef = (FunctionLike) scope.rootTree;
         for (Symbol symbol : scope.symbols()) {
           if (funcDef.is(Kind.LAMBDA)) {
@@ -88,14 +88,14 @@ public void visitFileInput(FileInput fileInput) {
             ((FunctionDefImpl) funcDef).addLocalVariableSymbol(symbol);
           }
         }
-      });
-    scopesByRootTree.values().stream()
-      .filter(scope -> scope.rootTree.is(Kind.CLASSDEF))
-      .forEach(scope -> {
+      } else if (scope.rootTree.is(Kind.CLASSDEF)) {
         ClassDefImpl classDef = (ClassDefImpl) scope.rootTree;
         scope.symbols.forEach(classDef::addClassField);
         scope.instanceAttributesByName.values().forEach(classDef::addInstanceField);
-      });
+      } else if (scope.rootTree.is(Kind.FILE_INPUT)) {
+        scope.symbols.forEach(((FileInputImpl) fileInput)::addGlobalVariables);
+      }
+    }
   }
 
   private static class ScopeVisitor extends BaseTreeVisitor {

python-squid/src/main/java/org/sonar/python/tree/FileInputImpl.java

@@ -19,8 +19,10 @@
  */
 package org.sonar.python.tree;
 
+import java.util.HashSet;
 import java.util.List;
 import java.util.Objects;
+import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import javax.annotation.CheckForNull;
@@ -30,12 +32,14 @@
 import org.sonar.python.api.tree.Token;
 import org.sonar.python.api.tree.Tree;
 import org.sonar.python.api.tree.TreeVisitor;
+import org.sonar.python.semantic.Symbol;
 
 public class FileInputImpl extends PyTree implements FileInput {
 
   private final StatementList statements;
   private final Token endOfFile;
   private final Token docstring;
+  private final Set<Symbol> globalVariables = new HashSet<>();
 
   public FileInputImpl(@Nullable StatementList statements, Token endOfFile, Token docstring) {
     super(statements == null ? endOfFile : statements.firstToken(), endOfFile);
@@ -61,6 +65,15 @@ public Token docstring() {
     return docstring;
   }
 
+  @Override
+  public Set<Symbol> globalVariables() {
+    return globalVariables;
+  }
+
+  public void addGlobalVariables(Symbol globalVariable) {
+    globalVariables.add(globalVariable);
+  }
+
   @Override
   public void accept(TreeVisitor visitor) {
     visitor.visitFileInput(this);

python-squid/src/test/java/org/sonar/python/semantic/SymbolTableBuilderTreeTest.java

@@ -24,6 +24,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.stream.Collectors;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -40,6 +41,7 @@
 
 public class SymbolTableBuilderTreeTest {
   private static Map<String, FunctionDef> functionTreesByName = new HashMap<>();
+  private static FileInput fileInput;
 
 
   private Map<String, Symbol> getSymbolByName(FunctionDef functionTree) {
@@ -49,10 +51,17 @@ private Map<String, Symbol> getSymbolByName(FunctionDef functionTree) {
   @BeforeClass
   public static void init() {
     PythonVisitorContext context = TestPythonVisitorRunner.createContext(new File("src/test/resources/semantic/symbols2.py"));
-    FileInput fileInput = context.rootTree();
+    fileInput = context.rootTree();
     fileInput.accept(new TestVisitor());
   }
 
+  @Test
+  public void global_variable() {
+    Set<Symbol> moduleSymbols = fileInput.globalVariables();
+    assertThat(moduleSymbols.size()).isEqualTo(2);
+    assertThat(moduleSymbols).extracting(Symbol::name).containsExactlyInAnyOrder("global_x", "global_var");
+  }
+
   @Test
   public void local_variable() {
     FunctionDef functionTree = functionTreesByName.get("function_with_local");

sonar-python-plugin/src/main/java/org/sonar/plugins/python/PythonScanner.java

@@ -120,6 +120,7 @@ private void scanFile(InputFile inputFile) {
     SubscriptionVisitor.analyze(checksBasedOnTree, visitorContext);
     saveIssues(inputFile, visitorContext.getIssues());
 
+    new SymbolVisitor(context.newSymbolTable().onFile(inputFile)).visitFileInput(visitorContext.rootTree());
     new PythonHighlighter(context, inputFile).scanFile(visitorContext);
   }
 

sonar-python-plugin/src/main/java/org/sonar/plugins/python/SymbolVisitor.java

@@ -0,0 +1,82 @@
+/*
+ * SonarQube Python Plugin
+ * Copyright (C) 2011-2019 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.plugins.python;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import org.sonar.api.batch.sensor.symbol.NewSymbol;
+import org.sonar.api.batch.sensor.symbol.NewSymbolTable;
+import org.sonar.python.api.tree.ClassDef;
+import org.sonar.python.api.tree.FileInput;
+import org.sonar.python.api.tree.FunctionDef;
+import org.sonar.python.api.tree.LambdaExpression;
+import org.sonar.python.api.tree.Tree;
+import org.sonar.python.semantic.Symbol;
+import org.sonar.python.semantic.Usage;
+import org.sonar.python.tree.BaseTreeVisitor;
+
+public class SymbolVisitor extends BaseTreeVisitor {
+
+  private final NewSymbolTable newSymbolTable;
+
+  public SymbolVisitor(NewSymbolTable newSymbolTable) {
+    this.newSymbolTable = newSymbolTable;
+  }
+
+  @Override
+  public void visitClassDef(ClassDef classDef) {
+    classDef.classFields().forEach(this::handleSymbol);
+    classDef.instanceFields().forEach(this::handleSymbol);
+    super.visitClassDef(classDef);
+  }
+
+  @Override
+  public void visitFunctionDef(FunctionDef functionDef) {
+    functionDef.localVariables().forEach(this::handleSymbol);
+    super.visitFunctionDef(functionDef);
+  }
+
+  @Override
+  public void visitLambda(LambdaExpression lambdaExpression) {
+    lambdaExpression.localVariables().forEach(this::handleSymbol);
+    super.visitLambda(lambdaExpression);
+  }
+
+  @Override
+  public void visitFileInput(FileInput fileInput) {
+    fileInput.globalVariables().forEach(this::handleSymbol);
+    super.visitFileInput(fileInput);
+    newSymbolTable.save();
+  }
+
+  private void handleSymbol(Symbol symbol) {
+    List<Usage> usages = new ArrayList<>(symbol.usages());
+    usages.sort(Comparator.comparingInt(u -> u.tree().firstToken().line()));
+    Tree firstUsageTree = usages.get(0).tree();
+    NewSymbol newSymbol = newSymbolTable.newSymbol(firstUsageTree.firstToken().line(), firstUsageTree.firstToken().column(),
+      firstUsageTree.lastToken().line(), firstUsageTree.lastToken().column() + firstUsageTree.lastToken().value().length());
+    for (int i = 1; i < usages.size(); i++) {
+      Tree usageTree = usages.get(i).tree();
+      newSymbol.newReference(usageTree.firstToken().line(), usageTree.firstToken().column(),
+        usageTree.lastToken().line(), usageTree.lastToken().column() + usageTree.lastToken().value().length());
+    }
+  }
+}

sonar-python-plugin/src/test/java/org/sonar/plugins/python/PythonSquidSensorTest.java

@@ -22,6 +22,7 @@
 import java.io.File;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.util.Collection;
 import java.util.Iterator;
 import org.junit.Before;
 import org.junit.Test;
@@ -30,7 +31,10 @@
 import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.batch.fs.InputFile.Type;
 import org.sonar.api.batch.fs.TextPointer;
+import org.sonar.api.batch.fs.TextRange;
 import org.sonar.api.batch.fs.internal.DefaultInputFile;
+import org.sonar.api.batch.fs.internal.DefaultTextPointer;
+import org.sonar.api.batch.fs.internal.DefaultTextRange;
 import org.sonar.api.batch.fs.internal.TestInputFileBuilder;
 import org.sonar.api.batch.rule.ActiveRules;
 import org.sonar.api.batch.rule.CheckFactory;
@@ -60,6 +64,7 @@
 public class PythonSquidSensorTest {
 
   private static final String FILE_1 = "file1.py";
+  private static final String FILE_2 = "file2.py";
   private static final String ONE_STATEMENT_PER_LINE_RULE_KEY = "OneStatementPerLine";
   private static final String FILE_COMPLEXITY_RULE_KEY = "FileComplexity";
 
@@ -84,7 +89,7 @@ public void init() {
 
   @Test
   public void sensor_descriptor() {
-    activeRules = (new ActiveRulesBuilder()).build();
+    activeRules = new ActiveRulesBuilder().build();
     DefaultSensorDescriptor descriptor = new DefaultSensorDescriptor();
     sensor().describe(descriptor);
 
@@ -125,6 +130,27 @@ public void test_execute_on_sonarlint() {
     assertThat(context.allAnalysisErrors()).isEmpty();
   }
 
+  @Test
+  public void test_symbol_visitor() {
+    activeRules = new ActiveRulesBuilder().build();
+    inputFile(FILE_2);
+    inputFile("symbolVisitor.py");
+    sensor().execute(context);
+
+    String key = "moduleKey:file2.py";
+    assertThat(context.referencesForSymbolAt(key, 1, 10)).isNull();
+    verifyUsages(key, 3, 4, reference(4, 10, 4, 11),
+      reference(6, 15, 6, 16), reference(7, 19, 7, 20));
+    verifyUsages(key, 5, 12, reference(6, 19, 6, 20));
+
+    key = "moduleKey:symbolVisitor.py";
+    assertThat(context.referencesForSymbolAt(key, 1, 10)).isNull();
+    verifyUsages(key, 1, 0);
+    verifyUsages(key, 2, 0, reference(10, 4, 10, 5));
+    verifyUsages(key, 5, 4, reference(6, 4, 6, 5), reference(7, 4, 7, 5),
+      reference(8, 8, 8, 9), reference(13, 9, 13, 10));
+  }
+
   @Test
   public void test_issues() {
     activeRules = new ActiveRulesBuilder()
@@ -140,7 +166,7 @@ public void test_issues() {
         .build())
       .build();
 
-    InputFile inputFile = inputFile("file2.py");
+    InputFile inputFile = inputFile(FILE_2);
     sensor().execute(context);
 
     assertThat(context.allIssues()).hasSize(3);
@@ -201,7 +227,7 @@ public void test_exception_does_not_fail_analysis() throws IOException {
     when(inputFile.contents()).thenThrow(RuntimeException.class);
 
     context.fileSystem().add(inputFile);
-    inputFile("file2.py");
+    inputFile(FILE_2);
 
     sensor().execute(context);
 
@@ -258,4 +284,12 @@ private InputFile inputFile(String name) {
     return inputFile;
   }
 
+  private void verifyUsages(String componentKey, int line, int offset, TextRange... trs) {
+    Collection<TextRange> textRanges = context.referencesForSymbolAt(componentKey, line, offset);
+    assertThat(textRanges).containsExactly(trs);
+  }
+
+  private static TextRange reference(int lineStart, int columnStart, int lineEnd, int columnEnd) {
+    return new DefaultTextRange(new DefaultTextPointer(lineStart, columnStart), new DefaultTextPointer(lineEnd, columnEnd));
+  }
 }