SONARPY-870 Use serialized version of TypeShed core modules (builtins and its dependencies)

.gitattributes

@@ -1,3 +1,5 @@
 # force eol to LF so we don't have problem with tests on windows
 
 * text eol=lf
+
+*.protobuf binary

.gitignore

@@ -35,3 +35,4 @@ Desktop.ini
 # ---- Python
 venv
 __pycache__
+python-frontend/typeshed_serializer/serializer/proto_out

its/ruling/src/test/resources/expected/python-S2159.json

@@ -2,7 +2,4 @@
 'project:numpy-1.16.4/numpy/distutils/system_info.py':[
 259,
 ],
-'project:numpy-1.16.4/numpy/lib/utils.py':[
-571,
-],
 }

its/ruling/src/test/resources/expected/python-S5607.json

@@ -6,9 +6,16 @@
 'project:buildbot-0.8.6p1/buildbot/status/build.py':[
 104,
 ],
+'project:django-cms-3.7.1/cms/models/metaclasses.py':[
+14,
+],
 'project:numpy-1.16.4/tools/npy_tempita/__init__.py':[
 184,
 ],
+'project:tensorflow/python/autograph/pyct/common_transformers/anf_test.py':[
+37,
+42,
+],
 'project:tensorflow/python/keras/layers/wrappers.py':[
 515,
 ],
@@ -26,4 +33,7 @@
 104,
 105,
 ],
+'project:twisted-12.1.0/twisted/test/test_defer.py':[
+367,
+],
 }

its/ruling/src/test/resources/expected/python-S5644.json

@@ -7,4 +7,8 @@
 22,
 24,
 ],
+'project:twisted-12.1.0/twisted/test/test_paths.py':[
+166,
+167,
+],
 }

its/ruling/src/test/resources/expected/python-S5655.json

@@ -14,4 +14,7 @@
 'project:tensorflow/python/eager/function.py':[
 968,
 ],
+'project:twisted-12.1.0/twisted/python/components.py':[
+313,
+],
 }

its/ruling/src/test/resources/expected/python-S5727.json

@@ -5,10 +5,20 @@
 'project:buildbot-0.8.6p1/buildbot/status/web/feeds.py':[
 240,
 ],
+'project:buildbot-0.8.6p1/buildbot/steps/shell.py':[
+469,
+],
 'project:django-2.2.3/django/core/files/locks.py':[
 109,
 113,
 ],
+'project:numpy-1.16.4/numpy/distutils/misc_util.py':[
+340,
+341,
+342,
+343,
+344,
+],
 'project:tensorflow/python/keras/layers/normalization.py':[
 1131,
 ],

its/ruling/src/test/resources/expected/python-S5795.json

@@ -4,6 +4,11 @@
 217,
 ],
 'project:numpy-1.16.4/numpy/distutils/misc_util.py':[
+340,
+341,
+342,
+343,
+344,
 476,
 ],
 'project:numpy-1.16.4/numpy/ma/core.py':[

python-checks/src/test/resources/checks/incompatibleOperands/arithmetic.py

@@ -59,11 +59,14 @@ def __neg__(self):
 
 def builtin_noncompliant():
   1 + "1"  # Noncompliant
+  "1" + 1  # Noncompliant
   1 + [1]  # Noncompliant
   1 + {1}  # Noncompliant
   1 + (1,)  # Noncompliant
   1 + {'a': 1}  # Noncompliant
   [1] + (1,)  # Noncompliant
+  "foo " + "bar".encode('base64') # OK, FP in Python2
+  "bar".encode('base64') + "foo"  # OK, FP in Python2
 
 
   -'1'  # Noncompliant

python-checks/src/test/resources/checks/itemOperationsTypeCheck/itemOperations_delitem.py

@@ -16,9 +16,9 @@ def builtin_types_supporting_delitem():
 def builtin_types_not_supporting_delitem():
   # dictviews https://docs.python.org/3/library/stdtypes.html#dictionary-view-objects
   mydict = {'a': 1, 'b': 2}
-  del mydict.keys()[0]  # OK: FP in Python 2
-  del mydict.values()[0] # OK: FP in Python 2
-  del mydict.items()[0]  # OK: FP in Python 2
+  del mydict.keys()[0]  # Noncompliant
+  del mydict.values()[0] # Noncompliant
+  del mydict.items()[0]  # Noncompliant
 
   # iterators
   del iter(mylist)[0]  # Noncompliant
@@ -48,7 +48,7 @@ def builtin_types_not_supporting_delitem():
   var = None
   del var[0]  # Noncompliant
 
-  del bytes(b'123')[0]  # FN: unknown return type
+  del bytes(b'123')[0]  # Noncompliant
   del memoryview(bytearray(b'abc'))[0]  # Noncompliant
 
   del "abc"[0]  # Noncompliant

python-checks/src/test/resources/checks/itemOperationsTypeCheck/itemOperations_getitem.py

@@ -24,9 +24,9 @@ def builtins_supporting_getitem():
 def builtins_not_supporting_getitem():
   # dictviews https://docs.python.org/3/library/stdtypes.html#dictionary-view-objects
   mydict = {'a': 1, 'b': 2}
-  mydict.keys()[0] # OK: FP in Python 2
-  mydict.values()[0] # OK: FP in Python 2
-  mydict.items()[0] # OK: FP in Python 2
+  mydict.keys()[0] # Noncompliant
+  mydict.values()[0] # Noncompliant
+  mydict.items()[0] # Noncompliant
 
   # iterators
   iter(mylist)[0]  # Noncompliant

python-checks/src/test/resources/checks/itemOperationsTypeCheck/itemOperations_setitem.py

@@ -19,9 +19,9 @@ def builtin_types_supporting_setitem():
 def builtin_types_not_supporting_setitem():
   mydict = {'a': 1, 'b': 2}
   # dictviews https://docs.python.org/3/library/stdtypes.html#dictionary-view-objects
-  mydict.keys()[0] = 42 # OK: FP in Python 2
-  mydict.values()[0] = 42 # OK: FP in Python 2
-  mydict.items()[0] = 42  # OK: FP in Python 2
+  mydict.keys()[0] = 42 # Noncompliant
+  mydict.values()[0] = 42 # Noncompliant
+  mydict.items()[0] = 42  # Noncompliant
 
   # iterators
   iter(mylist)[0] = 42  # Noncompliant
@@ -50,7 +50,7 @@ def builtin_types_not_supporting_setitem():
   var = None
   var[0] = 42  # Noncompliant
 
-  bytes(b'123')[0] = 42  # FN: unknown return type
+  bytes(b'123')[0] = 42  # Noncompliant
 
   # String
   "abc"[0] = 42  # Noncompliant

python-checks/src/test/resources/checks/sillyEquality.py

@@ -66,7 +66,7 @@ def stdlib():
     if passwd == pwd.getpwuid(1): pass # OK
     if 42 == pwd.getpwuid(1): pass # FN, unresolved type hierarchy
     if pwd.getpwall() == 42: pass # Noncompliant
-    if zip(l1, l2) == 42: pass # Noncompliant
+    if zip(l1, l2) == 42: pass # FN due to missing Python 2 and usage of zip.__new__
     if platform.architecture() == '32bit': ... # Noncompliant
 
 def third_party():

python-checks/src/test/resources/checks/tooManyParametersCustom.py

@@ -4,9 +4,8 @@ def func(p1, p2, p3, p4): ...  # Noncompliant
 #        ^^^^^^^^^^^^^^
 
 class MyTextIOWrapper(TextIOWrapper):
-  # FP (ambiguous symbol in type hierarchy)
   def __init__(
-          self,  # Noncompliant
+          self,
           buffer: IO[bytes],
           encoding: Optional[str] = ...,
           errors: Optional[str] = ...,
@@ -24,9 +23,8 @@ def new_method_nok(self, p1, p2, p3, p4) -> str: ...  # Noncompliant
 class MyOtherTextIOWrapper(TextIOWrapper): ...
 
 class ChildWithComplexHierarchy(MyOtherTextIOWrapper):
-  # FP (ambiguous symbol in type hierarchy)
   def __init__(
-          self,  # Noncompliant
+          self,
           buffer: IO[bytes],
           encoding: Optional[str] = ...,
           errors: Optional[str] = ...,

python-checks/src/test/resources/checks/unreachableExcept.py

@@ -76,8 +76,7 @@ def fn_Exception_not_a_super_class():
     raise UnicodeDecodeError()
   except Exception as e:
     print(e)
-  except ValueError as e:  # FN as "Exception" super class is missing
-#        ^^^^^^^^^^>
+  except ValueError as e:  # Noncompliant
     print("Never executed")
   except UnicodeError as e:  # Noncompliant
 #        ^^^^^^^^^^^^

python-frontend/src/main/java/org/sonar/python/semantic/AmbiguousSymbolImpl.java

@@ -68,7 +68,7 @@ public Set<Symbol> alternatives() {
   }
 
   @Override
-  AmbiguousSymbolImpl copyWithoutUsages() {
+  public AmbiguousSymbolImpl copyWithoutUsages() {
     Set<SymbolImpl> copiedAlternativeSymbols = symbols.stream()
       .map(SymbolImpl.class::cast)
       .map(SymbolImpl::copyWithoutUsages)
@@ -81,4 +81,9 @@ public void removeUsages() {
     super.removeUsages();
     symbols.forEach(symbol -> ((SymbolImpl) symbol).removeUsages());
   }
+
+  @Override
+  public Set<String> validForPythonVersions() {
+    return alternatives().stream().flatMap(symbol -> ((SymbolImpl) symbol).validForPythonVersions().stream()).collect(Collectors.toSet());
+  }
 }

python-frontend/src/main/java/org/sonar/python/semantic/ClassSymbolImpl.java

@@ -24,6 +24,7 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashSet;
 import java.util.List;
@@ -45,6 +46,8 @@
 
 import static org.sonar.python.semantic.SymbolUtils.pathOf;
 import static org.sonar.python.tree.TreeUtils.locationInFile;
+import static org.sonar.python.types.TypeShed.normalizedFqn;
+import static org.sonar.python.types.TypeShed.symbolsFromDescriptor;
 
 public class ClassSymbolImpl extends SymbolImpl implements ClassSymbol {
 
@@ -77,49 +80,74 @@ public ClassSymbolImpl(ClassDef classDef, @Nullable String fullyQualifiedName, P
   }
 
   public ClassSymbolImpl(String name, @Nullable String fullyQualifiedName) {
-    this(name, fullyQualifiedName, null, false, false, null, false);
+    super(name, fullyQualifiedName);
+    classDefinitionLocation = null;
+    hasDecorators = false;
+    hasMetaClass = false;
+    metaclassFQN = null;
+    supportsGenerics = false;
+    setKind(Kind.CLASS);
   }
 
-  public ClassSymbolImpl(String name, @Nullable String fullyQualifiedName, @Nullable LocationInFile definitionLocation,
-                         boolean hasDecorators, boolean hasMetaClass, @Nullable String metaclassFQN, boolean supportsGenerics) {
+  public ClassSymbolImpl(String name, @Nullable String fullyQualifiedName, LocationInFile location) {
     super(name, fullyQualifiedName);
-    classDefinitionLocation = definitionLocation;
-    this.hasDecorators = hasDecorators;
-    this.hasMetaClass = hasMetaClass;
-    this.metaclassFQN = metaclassFQN;
-    this.supportsGenerics = supportsGenerics;
+    classDefinitionLocation = location;
+    hasDecorators = false;
+    hasMetaClass = false;
+    metaclassFQN = null;
+    supportsGenerics = false;
+    setKind(Kind.CLASS);
+  }
+
+  public static ClassSymbol copyFrom(String name, ClassSymbol classSymbol) {
+    return new ClassSymbolImpl(name, classSymbol);
+  }
+
+  private ClassSymbolImpl(String name, ClassSymbol classSymbol) {
+    super(name, classSymbol.fullyQualifiedName());
+    classDefinitionLocation = classSymbol.definitionLocation();
+    hasDecorators = classSymbol.hasDecorators();
+    hasMetaClass = ((ClassSymbolImpl) classSymbol).hasMetaClass();
+    metaclassFQN = ((ClassSymbolImpl) classSymbol).metaclassFQN;
+    supportsGenerics = ((ClassSymbolImpl) classSymbol).supportsGenerics;
+    validForPythonVersions = ((ClassSymbolImpl) classSymbol).validForPythonVersions;
+    superClassesFqns = ((ClassSymbolImpl) classSymbol).superClassesFqns;
     setKind(Kind.CLASS);
   }
 
   public ClassSymbolImpl(SymbolsProtos.ClassSymbol classSymbolProto) {
-    super(classSymbolProto.getName(), classSymbolProto.getFullyQualifiedName());
+    super(classSymbolProto.getName(), normalizedFqn(classSymbolProto.getFullyQualifiedName()));
     setKind(Kind.CLASS);
     classDefinitionLocation = null;
     hasDecorators = classSymbolProto.getHasDecorators();
     hasMetaClass = classSymbolProto.getHasMetaclass();
     metaclassFQN = classSymbolProto.getMetaclassName();
     supportsGenerics = classSymbolProto.getIsGeneric();
-    List<SymbolsProtos.FunctionSymbol> methodsList = classSymbolProto.getMethodsList();
-    Set<Symbol> methods = methodsList.stream().map(m -> new FunctionSymbolImpl(m, true)).collect(Collectors.toSet());
-    for (SymbolsProtos.OverloadedFunctionSymbol overloadedMethod : classSymbolProto.getOverloadedMethodsList()) {
-      methods.add(AmbiguousSymbolImpl.create(overloadedMethod.getDefinitionsList().stream().map(m -> new FunctionSymbolImpl(m, true)).collect(Collectors.toSet())));
+    Set<Symbol> methods = new HashSet<>();
+    Map<String, Set<Object>> descriptorsByFqn = new HashMap<>();
+    classSymbolProto.getMethodsList().forEach(proto -> descriptorsByFqn.computeIfAbsent(proto.getFullyQualifiedName(), d -> new HashSet<>()).add(proto));
+    classSymbolProto.getOverloadedMethodsList().forEach(proto -> descriptorsByFqn.computeIfAbsent(proto.getFullname(), d -> new HashSet<>()).add(proto));
+    for (Map.Entry<String, Set<Object>> entry : descriptorsByFqn.entrySet()) {
+      Set<Symbol> symbols = symbolsFromDescriptor(entry.getValue(), true);
+      methods.add(symbols.size() > 1 ? AmbiguousSymbolImpl.create(symbols) : symbols.iterator().next());
     }
     addMembers(methods);
-    superClassesFqns = classSymbolProto.getSuperClassesList();
+    superClassesFqns = classSymbolProto.getSuperClassesList().stream().map(TypeShed::normalizedFqn).collect(Collectors.toList());
+    validForPythonVersions = new HashSet<>(classSymbolProto.getValidForList());
   }
 
   @Override
-  ClassSymbolImpl copyWithoutUsages() {
-    ClassSymbolImpl copiedClassSymbol = new ClassSymbolImpl(name(), fullyQualifiedName(), definitionLocation(), hasDecorators, hasMetaClass, metaclassFQN, supportsGenerics);
-    for (Symbol superClass : superClasses()) {
-      if (superClass == this) {
-        copiedClassSymbol.superClasses.add(copiedClassSymbol);
-      } else if (superClass.kind() == Symbol.Kind.CLASS) {
-        copiedClassSymbol.superClasses.add(((ClassSymbolImpl) superClass).copyWithoutUsages());
-      } else if (superClass.is(Kind.AMBIGUOUS)) {
-        copiedClassSymbol.superClasses.add(((AmbiguousSymbolImpl) superClass).copyWithoutUsages());
-      } else {
-        copiedClassSymbol.superClasses.add(new SymbolImpl(superClass.name(), superClass.fullyQualifiedName()));
+  public ClassSymbolImpl copyWithoutUsages() {
+    ClassSymbolImpl copiedClassSymbol = new ClassSymbolImpl(name(), this);
+    if (hasEvaluatedSuperClasses()) {
+      for (Symbol superClass : superClasses()) {
+        if (superClass == this) {
+          copiedClassSymbol.superClasses.add(copiedClassSymbol);
+        } else if (superClass.is(Kind.CLASS, Kind.AMBIGUOUS)) {
+          copiedClassSymbol.superClasses.add(((SymbolImpl) superClass).copyWithoutUsages());
+        } else {
+          copiedClassSymbol.superClasses.add(new SymbolImpl(superClass.name(), superClass.fullyQualifiedName()));
+        }
       }
     }
     copiedClassSymbol.addMembers(members.stream().map(m -> ((SymbolImpl) m).copyWithoutUsages()).collect(Collectors.toList()));
@@ -340,4 +368,16 @@ public boolean supportsGenerics() {
   public void setSupportsGenerics(boolean supportsGenerics) {
     this.supportsGenerics = supportsGenerics;
   }
+
+  /**
+   * Precomputed typeshed class symbols might be "lazily evaluated", i.e. only information about super classes fqn is stored, without having created the actual
+   * type hierarchy.
+   * This method is used to know if super classes have been already created and added to {@link #superClasses}.
+   * This might happen in the following cases:
+   * - Super classes have been already read, hence class symbol is not lazy anymore
+   * - {@link #superClassesFqns} is empty, meaning either this isn't a precomputed typeshed symbol or the class have no superclass.
+   */
+  public boolean hasEvaluatedSuperClasses() {
+    return hasAlreadyReadSuperClasses || superClassesFqns.isEmpty();
+  }
 }