Permalink
Browse files

developed star import caching. useful for numpy/pylab/scipy... fixes …

…jedi-vim issue 23
  • Loading branch information...
davidhalter committed Nov 21, 2012
1 parent 890748a commit 586dcc273cd51b7efc6a4dcb2c4fc886cb8b1455
Showing with 55 additions and 11 deletions.
  1. +5 −0 jedi/builtin.py
  2. +37 −5 jedi/imports.py
  3. +10 −0 jedi/settings.py
  4. +3 −6 test/regression.py
View
@@ -12,6 +12,7 @@
import debug
import parsing
+import imports
def get_sys_path():
@@ -50,6 +51,10 @@ def parser(self):
if not self.path or os.path.getmtime(self.path) <= timestamp:
self._parser = parser
else:
+ # In case there is already a module cached and this module
+ # has to be reparsed, we also need to invalidate the import
+ # caches.
+ imports.invalidate_star_import_cache(self._parser.module)
raise KeyError()
except KeyError:
self._load_module()
View
@@ -5,17 +5,21 @@
import imp
import sys
import weakref
+import time
import builtin
import modules
import debug
import parsing
import evaluate
import itertools
+import settings
# for debugging purposes only
imports_processed = 0
+star_import_cache = {}
+
class ModuleNotFound(Exception):
pass
@@ -264,12 +268,42 @@ def strip_imports(scopes):
return result
-def star_import_cache(func):
+def cache_star_import(func):
def wrapper(scope, *args, **kwargs):
- return func(scope, *args, **kwargs)
+ try:
+ mods = star_import_cache[scope]
+ if mods[0] + settings.star_import_cache_validity > time.time():
+ return mods[1]
+ except KeyError:
+ pass
+ # cache is too old and therefore invalid or not available
+ invalidate_star_import_cache(scope)
+ mods = func(scope, *args, **kwargs)
+ star_import_cache[scope] = time.time(), mods
+
+ return mods
return wrapper
-@star_import_cache
+
+def invalidate_star_import_cache(module, only_main=False):
+ """ Important if some new modules are being reparsed """
+ try:
+ mods = star_import_cache[module]
+
+ for t, m in mods:
+ invalidate_star_import_cache(m, only_main=True)
+
+ del star_import_cache[module]
+ except KeyError:
+ pass
+
+ if not only_main:
+ for key, (t, mods) in star_import_cache.items():
+ if module in mods:
+ invalidate_star_import_cache(key)
+
+
+@cache_star_import
def remove_star_imports(scope, ignored_modules=[]):
"""
Check a module for star imports:
@@ -285,6 +319,4 @@ def remove_star_imports(scope, ignored_modules=[]):
modules += new
# Filter duplicate modules.
- if len(modules) > 10:
- print scope, len(modules)
return set(modules)
View
@@ -75,3 +75,13 @@
# Size of the current code part, which is used to speed up parsing.
part_line_length = 20
+
+# ----------------
+# star import caching
+# ----------------
+
+# In huge packages like numpy, checking all star imports on every completion
+# might be slow, therefore we do a star import caching, that lasts a certain
+# time span (in seconds).
+
+star_import_cache_validity = 60.0
View
@@ -249,7 +249,7 @@ def wrapper(self):
for i in range(number):
func(self)
single_time = (time.time() - first) / number
- print(func, single_time)
+ print('speed', func, single_time)
assert single_time < time_per_run
return wrapper
return decorated
@@ -259,15 +259,12 @@ def test_os_path_join(self):
s = "from posixpath import join; join('', '')."
assert len(self.complete(s)) > 10 # is a str completion
- @_check_speed(0.2, number=1)
+ @_check_speed(0.1)
def test_scipy_speed(self):
s = 'import scipy.weave; scipy.weave.inline('
- #api.set_debug_function(api.debug.print_to_stdout)
script = api.Script(s, 1, len(s), '')
script.get_in_function_call()
- # self.get_in_function_call(s)
- #api.set_debug_function(None)
- print(api.imports.imports_processed)
+ #print(api.imports.imports_processed)
if __name__ == '__main__':

0 comments on commit 586dcc2

Please sign in to comment.