Merge dev into linter.

.coveragerc

@@ -1,6 +1,7 @@
 [run]
 omit =
     jedi/_compatibility.py
+    jedi/evaluate/site.py
 
 [report]
 # Regexes for lines to exclude from consideration

.travis.yml

@@ -1,4 +1,5 @@
 language: python
+sudo: false
 env:
     - TOXENV=py26
     - TOXENV=py27

AUTHORS.txt

@@ -33,5 +33,6 @@ Savor d'Isavano (@KenetJervet) <newelevenken@163.com>
 Phillip Berndt (@phillipberndt) <phillip.berndt@gmail.com>
 Ian Lee (@IanLee1521) <IanLee1521@gmail.com>
 Farkhad Khatamov (@hatamov) <comsgn@gmail.com>
+Kevin Kelley (@kelleyk) <kelleyk@kelleyk.net>
 
 Note: (@user) means a github user name.

conftest.py

@@ -1,8 +1,9 @@
 import tempfile
 import shutil
 
-import jedi
+import pytest
 
+import jedi
 
 collect_ignore = ["setup.py"]
 
@@ -47,3 +48,25 @@ def pytest_unconfigure(config):
     global jedi_cache_directory_orig, jedi_cache_directory_temp
     jedi.settings.cache_directory = jedi_cache_directory_orig
     shutil.rmtree(jedi_cache_directory_temp)
+
+
+@pytest.fixture(scope='session')
+def clean_jedi_cache(request):
+    """
+    Set `jedi.settings.cache_directory` to a temporary directory during test.
+
+    Note that you can't use built-in `tmpdir` and `monkeypatch`
+    fixture here because their scope is 'function', which is not used
+    in 'session' scope fixture.
+
+    This fixture is activated in ../pytest.ini.
+    """
+    from jedi import settings
+    old = settings.cache_directory
+    tmp = tempfile.mkdtemp(prefix='jedi-test-')
+    settings.cache_directory = tmp
+
+    @request.addfinalizer
+    def restore():
+        settings.cache_directory = old
+        shutil.rmtree(tmp)

jedi/_compatibility.py

@@ -25,7 +25,7 @@ def find_module_py33(string, path=None):
         except ValueError as e:
             # See #491. Importlib might raise a ValueError, to avoid this, we
             # just raise an ImportError to fix the issue.
-            raise ImportError("Originally ValueError: " + e.message)
+            raise ImportError("Originally ValueError: " + str(e))
 
     if loader is None:
         raise ImportError("Couldn't find a loader for {0}".format(string))

jedi/api/__init__.py

@@ -34,6 +34,7 @@
 from jedi.evaluate.cache import memoize_default
 from jedi.evaluate.helpers import FakeName, get_module_names
 from jedi.evaluate.finder import global_names_dict_generator, filter_definition_names
+from jedi.evaluate.sys_path import get_venv_path
 
 # Jedi uses lots and lots of recursion. By setting this a little bit higher, we
 # can remove some "maximum recursion depth" errors.
@@ -58,6 +59,18 @@ class Script(object):
     You can either use the ``source`` parameter or ``path`` to read a file.
     Usually you're going to want to use both of them (in an editor).
 
+    The script might be analyzed in a different ``sys.path`` than |jedi|:
+
+    - if `sys_path` parameter is not ``None``, it will be used as ``sys.path``
+      for the script;
+
+    - if `sys_path` parameter is ``None`` and ``VIRTUAL_ENV`` environment
+      variable is defined, ``sys.path`` for the specified environment will be
+      guessed (see :func:`jedi.evaluate.sys_path.get_venv_path`) and used for
+      the script;
+
+    - otherwise ``sys.path`` will match that of |jedi|.
+
     :param source: The source code of the current file, separated by newlines.
     :type source: str
     :param line: The line to perform actions on (starting with 1).
@@ -73,9 +86,13 @@ class Script(object):
     :param source_encoding: The encoding of ``source``, if it is not a
         ``unicode`` object (default ``'utf-8'``).
     :type encoding: str
+    :param sys_path: ``sys.path`` to use during analysis of the script
+    :type sys_path: list
+
     """
     def __init__(self, source=None, line=None, column=None, path=None,
-                 encoding='utf-8', source_path=None, source_encoding=None):
+                 encoding='utf-8', source_path=None, source_encoding=None,
+                 sys_path=None):
         if source_path is not None:
             warnings.warn("Use path instead of source_path.", DeprecationWarning)
             path = source_path
@@ -109,7 +126,11 @@ def __init__(self, source=None, line=None, column=None, path=None,
         self._parser = UserContextParser(self._grammar, self.source, path,
                                          self._pos, self._user_context,
                                          self._parsed_callback)
-        self._evaluator = Evaluator(self._grammar)
+        if sys_path is None:
+            venv = os.getenv('VIRTUAL_ENV')
+            if venv:
+                sys_path = list(get_venv_path(venv))
+        self._evaluator = Evaluator(self._grammar, sys_path=sys_path)
         debug.speed('init')
 
     def _parsed_callback(self, parser):

jedi/api/classes.py

@@ -336,7 +336,7 @@ def params(self):
             raise AttributeError()
         followed = followed[0]  # only check the first one.
 
-        if followed.type == 'funcdef':
+        if followed.type in ('funcdef', 'lambda'):
             if isinstance(followed, er.InstanceElement):
                 params = followed.params[1:]
             else:

jedi/evaluate/__init__.py

@@ -61,6 +61,7 @@
 """
 
 import copy
+import sys
 from itertools import chain
 
 from jedi.parser import tree
@@ -79,7 +80,7 @@
 
 
 class Evaluator(object):
-    def __init__(self, grammar):
+    def __init__(self, grammar, sys_path=None):
         self.grammar = grammar
         self.memoize_cache = {}  # for memoize decorators
         # To memorize modules -> equals `sys.modules`.
@@ -91,6 +92,14 @@ def __init__(self, grammar):
         self.predefined_if_name_dict_dict = {}
         self.is_analysis = False
 
+        if sys_path is None:
+            sys_path = sys.path
+        self.sys_path = copy.copy(sys_path)
+        try:
+            self.sys_path.remove('')
+        except ValueError:
+            pass
+
     def wrap(self, element):
         if isinstance(element, tree.Class):
             return er.Class(self, element)

jedi/evaluate/compiled/__init__.py

@@ -10,7 +10,6 @@
 from jedi._compatibility import builtins as _builtins, unicode
 from jedi import debug
 from jedi.cache import underscore_memoization, memoize_method
-from jedi.evaluate.sys_path import get_sys_path
 from jedi.parser.tree import Param, Base, Operator, zero_position_modifier
 from jedi.evaluate.helpers import FakeName
 from . import fake
@@ -309,15 +308,12 @@ def parent(self, value):
         pass  # Just ignore this, FakeName tries to overwrite the parent attribute.
 
 
-def dotted_from_fs_path(fs_path, sys_path=None):
+def dotted_from_fs_path(fs_path, sys_path):
     """
     Changes `/usr/lib/python3.4/email/utils.py` to `email.utils`.  I.e.
     compares the path with sys.path and then returns the dotted_path. If the
     path is not in the sys.path, just returns None.
     """
-    if sys_path is None:
-        sys_path = get_sys_path()
-
     if os.path.basename(fs_path).startswith('__init__.'):
         # We are calculating the path. __init__ files are not interesting.
         fs_path = os.path.dirname(fs_path)
@@ -341,13 +337,13 @@ def dotted_from_fs_path(fs_path, sys_path=None):
     return _path_re.sub('', fs_path[len(path):].lstrip(os.path.sep)).replace(os.path.sep, '.')
 
 
-def load_module(path=None, name=None):
+def load_module(evaluator, path=None, name=None):
+    sys_path = evaluator.sys_path
     if path is not None:
-        dotted_path = dotted_from_fs_path(path)
+        dotted_path = dotted_from_fs_path(path, sys_path=sys_path)
     else:
         dotted_path = name
 
-    sys_path = get_sys_path()
     if dotted_path is None:
         p, _, dotted_path = path.partition(os.path.sep)
         sys_path.insert(0, p)

jedi/evaluate/imports.py

@@ -342,7 +342,7 @@ def _do_import(self, import_path, sys_path):
             module_file.close()
 
         if module_file is None and not module_path.endswith('.py'):
-            module = compiled.load_module(module_path)
+            module = compiled.load_module(self._evaluator, module_path)
         else:
             module = _load_module(self._evaluator, module_path, source, sys_path)
 
@@ -440,12 +440,15 @@ def load(source):
                 with open(path, 'rb') as f:
                     source = f.read()
         else:
-            return compiled.load_module(path)
+            return compiled.load_module(evaluator, path)
         p = path
         p = fast.FastParser(evaluator.grammar, common.source_to_unicode(source), p)
         cache.save_parser(path, p)
         return p.module
 
+    if sys_path is None:
+        sys_path = evaluator.sys_path
+
     cached = cache.load_parser(path)
     module = load(source) if cached is None else cached.module
     module = evaluator.wrap(module)

jedi/evaluate/site.py

@@ -0,0 +1,110 @@
+"""An adapted copy of relevant site-packages functionality from Python stdlib.
+
+This file contains some functions related to handling site-packages in Python
+with jedi-specific modifications:
+
+- the functions operate on sys_path argument rather than global sys.path
+
+- in .pth files "import ..." lines that allow execution of arbitrary code are
+  skipped to prevent code injection into jedi interpreter
+
+"""
+
+# Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
+# 2011, 2012, 2013, 2014, 2015 Python Software Foundation; All Rights Reserved
+
+from __future__ import print_function
+
+import sys
+import os
+
+
+def makepath(*paths):
+    dir = os.path.join(*paths)
+    try:
+        dir = os.path.abspath(dir)
+    except OSError:
+        pass
+    return dir, os.path.normcase(dir)
+
+
+def _init_pathinfo(sys_path):
+    """Return a set containing all existing directory entries from sys_path"""
+    d = set()
+    for dir in sys_path:
+        try:
+            if os.path.isdir(dir):
+                dir, dircase = makepath(dir)
+                d.add(dircase)
+        except TypeError:
+            continue
+    return d
+
+
+def addpackage(sys_path, sitedir, name, known_paths):
+    """Process a .pth file within the site-packages directory:
+       For each line in the file, either combine it with sitedir to a path
+       and add that to known_paths, or execute it if it starts with 'import '.
+    """
+    if known_paths is None:
+        known_paths = _init_pathinfo(sys_path)
+        reset = 1
+    else:
+        reset = 0
+    fullname = os.path.join(sitedir, name)
+    try:
+        f = open(fullname, "r")
+    except OSError:
+        return
+    with f:
+        for n, line in enumerate(f):
+            if line.startswith("#"):
+                continue
+            try:
+                if line.startswith(("import ", "import\t")):
+                    # Change by immerrr: don't evaluate import lines to prevent
+                    # code injection into jedi through pth files.
+                    #
+                    # exec(line)
+                    continue
+                line = line.rstrip()
+                dir, dircase = makepath(sitedir, line)
+                if not dircase in known_paths and os.path.exists(dir):
+                    sys_path.append(dir)
+                    known_paths.add(dircase)
+            except Exception:
+                print("Error processing line {:d} of {}:\n".format(n+1, fullname),
+                      file=sys.stderr)
+                import traceback
+                for record in traceback.format_exception(*sys.exc_info()):
+                    for line in record.splitlines():
+                        print('  '+line, file=sys.stderr)
+                print("\nRemainder of file ignored", file=sys.stderr)
+                break
+    if reset:
+        known_paths = None
+    return known_paths
+
+
+def addsitedir(sys_path, sitedir, known_paths=None):
+    """Add 'sitedir' argument to sys_path if missing and handle .pth files in
+    'sitedir'"""
+    if known_paths is None:
+        known_paths = _init_pathinfo(sys_path)
+        reset = 1
+    else:
+        reset = 0
+    sitedir, sitedircase = makepath(sitedir)
+    if not sitedircase in known_paths:
+        sys_path.append(sitedir)        # Add path component
+        known_paths.add(sitedircase)
+    try:
+        names = os.listdir(sitedir)
+    except OSError:
+        return
+    names = [name for name in names if name.endswith(".pth")]
+    for name in sorted(names):
+        addpackage(sys_path, sitedir, name, known_paths)
+    if reset:
+        known_paths = None
+    return known_paths