-
-
Notifications
You must be signed in to change notification settings - Fork 499
/
imports.py
564 lines (492 loc) · 20.9 KB
/
imports.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
"""
:mod:`jedi.inference.imports` is here to resolve import statements and return
the modules/classes/functions/whatever, which they stand for. However there's
not any actual importing done. This module is about finding modules in the
filesystem. This can be quite tricky sometimes, because Python imports are not
always that simple.
This module also supports import autocompletion, which means to complete
statements like ``from datetim`` (cursor at the end would return ``datetime``).
"""
import os
from pathlib import Path
from parso.python import tree
from parso.tree import search_ancestor
from jedi import debug
from jedi import settings
from jedi.file_io import FolderIO
from jedi.parser_utils import get_cached_code_lines
from jedi.inference import sys_path
from jedi.inference import helpers
from jedi.inference import compiled
from jedi.inference import analysis
from jedi.inference.utils import unite
from jedi.inference.cache import inference_state_method_cache
from jedi.inference.names import ImportName, SubModuleName
from jedi.inference.base_value import ValueSet, NO_VALUES
from jedi.inference.gradual.typeshed import import_module_decorator, \
create_stub_module, parse_stub_module
from jedi.inference.compiled.subprocess.functions import ImplicitNSInfo
from jedi.plugins import plugin_manager
class ModuleCache:
def __init__(self):
self._name_cache = {}
def add(self, string_names, value_set):
if string_names is not None:
self._name_cache[string_names] = value_set
def get(self, string_names):
return self._name_cache.get(string_names)
# This memoization is needed, because otherwise we will infinitely loop on
# certain imports.
@inference_state_method_cache(default=NO_VALUES)
def infer_import(context, tree_name):
module_context = context.get_root_context()
from_import_name, import_path, level, values = \
_prepare_infer_import(module_context, tree_name)
if values:
if from_import_name is not None:
values = values.py__getattribute__(
from_import_name,
name_context=context,
analysis_errors=False
)
if not values:
path = import_path + (from_import_name,)
importer = Importer(context.inference_state, path, module_context, level)
values = importer.follow()
debug.dbg('after import: %s', values)
return values
@inference_state_method_cache(default=[])
def goto_import(context, tree_name):
module_context = context.get_root_context()
from_import_name, import_path, level, values = \
_prepare_infer_import(module_context, tree_name)
if not values:
return []
if from_import_name is not None:
names = unite([
c.goto(
from_import_name,
name_context=context,
analysis_errors=False
) for c in values
])
# Avoid recursion on the same names.
if names and not any(n.tree_name is tree_name for n in names):
return names
path = import_path + (from_import_name,)
importer = Importer(context.inference_state, path, module_context, level)
values = importer.follow()
return set(s.name for s in values)
def _prepare_infer_import(module_context, tree_name):
import_node = search_ancestor(tree_name, 'import_name', 'import_from')
import_path = import_node.get_path_for_name(tree_name)
from_import_name = None
try:
from_names = import_node.get_from_names()
except AttributeError:
# Is an import_name
pass
else:
if len(from_names) + 1 == len(import_path):
# We have to fetch the from_names part first and then check
# if from_names exists in the modules.
from_import_name = import_path[-1]
import_path = from_names
importer = Importer(module_context.inference_state, tuple(import_path),
module_context, import_node.level)
return from_import_name, tuple(import_path), import_node.level, importer.follow()
def _add_error(value, name, message):
if hasattr(name, 'parent') and value is not None:
analysis.add(value, 'import-error', name, message)
else:
debug.warning('ImportError without origin: ' + message)
def _level_to_base_import_path(project_path, directory, level):
"""
In case the level is outside of the currently known package (something like
import .....foo), we can still try our best to help the user for
completions.
"""
for i in range(level - 1):
old = directory
directory = os.path.dirname(directory)
if old == directory:
return None, None
d = directory
level_import_paths = []
# Now that we are on the level that the user wants to be, calculate the
# import path for it.
while True:
if d == project_path:
return level_import_paths, d
dir_name = os.path.basename(d)
if dir_name:
level_import_paths.insert(0, dir_name)
d = os.path.dirname(d)
else:
return None, directory
class Importer:
def __init__(self, inference_state, import_path, module_context, level=0):
"""
An implementation similar to ``__import__``. Use `follow`
to actually follow the imports.
*level* specifies whether to use absolute or relative imports. 0 (the
default) means only perform absolute imports. Positive values for level
indicate the number of parent directories to search relative to the
directory of the module calling ``__import__()`` (see PEP 328 for the
details).
:param import_path: List of namespaces (strings or Names).
"""
debug.speed('import %s %s' % (import_path, module_context))
self._inference_state = inference_state
self.level = level
self._module_context = module_context
self._fixed_sys_path = None
self._infer_possible = True
if level:
base = module_context.get_value().py__package__()
# We need to care for two cases, the first one is if it's a valid
# Python import. This import has a properly defined module name
# chain like `foo.bar.baz` and an import in baz is made for
# `..lala.` It can then resolve to `foo.bar.lala`.
# The else here is a heuristic for all other cases, if for example
# in `foo` you search for `...bar`, it's obviously out of scope.
# However since Jedi tries to just do it's best, we help the user
# here, because he might have specified something wrong in his
# project.
if level <= len(base):
# Here we basically rewrite the level to 0.
base = tuple(base)
if level > 1:
base = base[:-level + 1]
import_path = base + tuple(import_path)
else:
path = module_context.py__file__()
project_path = self._inference_state.project.path
import_path = list(import_path)
if path is None:
# If no path is defined, our best guess is that the current
# file is edited by a user on the current working
# directory. We need to add an initial path, because it
# will get removed as the name of the current file.
directory = project_path
else:
directory = os.path.dirname(path)
base_import_path, base_directory = _level_to_base_import_path(
project_path, directory, level,
)
if base_directory is None:
# Everything is lost, the relative import does point
# somewhere out of the filesystem.
self._infer_possible = False
else:
self._fixed_sys_path = [base_directory]
if base_import_path is None:
if import_path:
_add_error(
module_context, import_path[0],
message='Attempted relative import beyond top-level package.'
)
else:
import_path = base_import_path + import_path
self.import_path = import_path
@property
def _str_import_path(self):
"""Returns the import path as pure strings instead of `Name`."""
return tuple(
name.value if isinstance(name, tree.Name) else name
for name in self.import_path
)
def _sys_path_with_modifications(self, is_completion):
if self._fixed_sys_path is not None:
return self._fixed_sys_path
return (
# For import completions we don't want to see init paths, but for
# inference we want to show the user as much as possible.
# See GH #1446.
self._inference_state.get_sys_path(add_init_paths=not is_completion)
+ [
str(p) for p
in sys_path.check_sys_path_modifications(self._module_context)
]
)
def follow(self):
if not self.import_path or not self._infer_possible:
return NO_VALUES
# Check caches first
from_cache = self._inference_state.stub_module_cache.get(self._str_import_path)
if from_cache is not None:
return ValueSet({from_cache})
from_cache = self._inference_state.module_cache.get(self._str_import_path)
if from_cache is not None:
return from_cache
sys_path = self._sys_path_with_modifications(is_completion=False)
return import_module_by_names(
self._inference_state, self.import_path, sys_path, self._module_context
)
def _get_module_names(self, search_path=None, in_module=None):
"""
Get the names of all modules in the search_path. This means file names
and not names defined in the files.
"""
if search_path is None:
sys_path = self._sys_path_with_modifications(is_completion=True)
else:
sys_path = search_path
return list(iter_module_names(
self._inference_state, self._module_context, sys_path,
module_cls=ImportName if in_module is None else SubModuleName,
add_builtin_modules=search_path is None and in_module is None,
))
def completion_names(self, inference_state, only_modules=False):
"""
:param only_modules: Indicates wheter it's possible to import a
definition that is not defined in a module.
"""
if not self._infer_possible:
return []
names = []
if self.import_path:
# flask
if self._str_import_path == ('flask', 'ext'):
# List Flask extensions like ``flask_foo``
for mod in self._get_module_names():
modname = mod.string_name
if modname.startswith('flask_'):
extname = modname[len('flask_'):]
names.append(ImportName(self._module_context, extname))
# Now the old style: ``flaskext.foo``
for dir in self._sys_path_with_modifications(is_completion=True):
flaskext = os.path.join(dir, 'flaskext')
if os.path.isdir(flaskext):
names += self._get_module_names([flaskext])
values = self.follow()
for value in values:
# Non-modules are not completable.
if value.api_type != 'module': # not a module
continue
if not value.is_compiled():
# sub_modules_dict is not implemented for compiled modules.
names += value.sub_modules_dict().values()
if not only_modules:
from jedi.inference.gradual.conversion import convert_values
both_values = values | convert_values(values)
for c in both_values:
for filter in c.get_filters():
names += filter.values()
else:
if self.level:
# We only get here if the level cannot be properly calculated.
names += self._get_module_names(self._fixed_sys_path)
else:
# This is just the list of global imports.
names += self._get_module_names()
return names
def import_module_by_names(inference_state, import_names, sys_path=None,
module_context=None, prefer_stubs=True):
if sys_path is None:
sys_path = inference_state.get_sys_path()
str_import_names = tuple(
i.value if isinstance(i, tree.Name) else i
for i in import_names
)
value_set = [None]
for i, name in enumerate(import_names):
value_set = ValueSet.from_sets([
import_module(
inference_state,
str_import_names[:i+1],
parent_module_value,
sys_path,
prefer_stubs=prefer_stubs,
) for parent_module_value in value_set
])
if not value_set:
message = 'No module named ' + '.'.join(str_import_names)
if module_context is not None:
_add_error(module_context, name, message)
else:
debug.warning(message)
return NO_VALUES
return value_set
@plugin_manager.decorate()
@import_module_decorator
def import_module(inference_state, import_names, parent_module_value, sys_path):
"""
This method is very similar to importlib's `_gcd_import`.
"""
if import_names[0] in settings.auto_import_modules:
module = _load_builtin_module(inference_state, import_names, sys_path)
if module is None:
return NO_VALUES
return ValueSet([module])
module_name = '.'.join(import_names)
if parent_module_value is None:
# Override the sys.path. It works only good that way.
# Injecting the path directly into `find_module` did not work.
file_io_or_ns, is_pkg = inference_state.compiled_subprocess.get_module_info(
string=import_names[-1],
full_name=module_name,
sys_path=sys_path,
is_global_search=True,
)
if is_pkg is None:
return NO_VALUES
else:
paths = parent_module_value.py__path__()
if paths is None:
# The module might not be a package.
return NO_VALUES
for path in paths:
# At the moment we are only using one path. So this is
# not important to be correct.
if not isinstance(path, list):
path = [path]
file_io_or_ns, is_pkg = inference_state.compiled_subprocess.get_module_info(
string=import_names[-1],
path=path,
full_name=module_name,
is_global_search=False,
)
if is_pkg is not None:
break
else:
return NO_VALUES
if isinstance(file_io_or_ns, ImplicitNSInfo):
from jedi.inference.value.namespace import ImplicitNamespaceValue
module = ImplicitNamespaceValue(
inference_state,
string_names=tuple(file_io_or_ns.name.split('.')),
paths=file_io_or_ns.paths,
)
elif file_io_or_ns is None:
module = _load_builtin_module(inference_state, import_names, sys_path)
if module is None:
return NO_VALUES
else:
module = _load_python_module(
inference_state, file_io_or_ns,
import_names=import_names,
is_package=is_pkg,
)
if parent_module_value is None:
debug.dbg('global search_module %s: %s', import_names[-1], module)
else:
debug.dbg('search_module %s in paths %s: %s', module_name, paths, module)
return ValueSet([module])
def _load_python_module(inference_state, file_io,
import_names=None, is_package=False):
module_node = inference_state.parse(
file_io=file_io,
cache=True,
diff_cache=settings.fast_parser,
cache_path=settings.cache_directory,
)
from jedi.inference.value import ModuleValue
return ModuleValue(
inference_state, module_node,
file_io=file_io,
string_names=import_names,
code_lines=get_cached_code_lines(inference_state.grammar, file_io.path),
is_package=is_package,
)
def _load_builtin_module(inference_state, import_names=None, sys_path=None):
project = inference_state.project
if sys_path is None:
sys_path = inference_state.get_sys_path()
if not project._load_unsafe_extensions:
safe_paths = project._get_base_sys_path(inference_state)
sys_path = [p for p in sys_path if p in safe_paths]
dotted_name = '.'.join(import_names)
assert dotted_name is not None
module = compiled.load_module(inference_state, dotted_name=dotted_name, sys_path=sys_path)
if module is None:
# The file might raise an ImportError e.g. and therefore not be
# importable.
return None
return module
def load_module_from_path(inference_state, file_io, import_names=None, is_package=None):
"""
This should pretty much only be used for get_modules_containing_name. It's
here to ensure that a random path is still properly loaded into the Jedi
module structure.
"""
path = Path(file_io.path)
if import_names is None:
e_sys_path = inference_state.get_sys_path()
import_names, is_package = sys_path.transform_path_to_dotted(e_sys_path, path)
else:
assert isinstance(is_package, bool)
is_stub = path.suffix == '.pyi'
if is_stub:
folder_io = file_io.get_parent_folder()
if folder_io.path.endswith('-stubs'):
folder_io = FolderIO(folder_io.path[:-6])
if path.name == '__init__.pyi':
python_file_io = folder_io.get_file_io('__init__.py')
else:
python_file_io = folder_io.get_file_io(import_names[-1] + '.py')
try:
v = load_module_from_path(
inference_state, python_file_io,
import_names, is_package=is_package
)
values = ValueSet([v])
except FileNotFoundError:
values = NO_VALUES
return create_stub_module(
inference_state, values, parse_stub_module(inference_state, file_io),
file_io, import_names
)
else:
module = _load_python_module(
inference_state, file_io,
import_names=import_names,
is_package=is_package,
)
inference_state.module_cache.add(import_names, ValueSet([module]))
return module
def load_namespace_from_path(inference_state, folder_io):
import_names, is_package = sys_path.transform_path_to_dotted(
inference_state.get_sys_path(),
Path(folder_io.path)
)
from jedi.inference.value.namespace import ImplicitNamespaceValue
return ImplicitNamespaceValue(inference_state, import_names, [folder_io.path])
def follow_error_node_imports_if_possible(context, name):
error_node = tree.search_ancestor(name, 'error_node')
if error_node is not None:
# Get the first command start of a started simple_stmt. The error
# node is sometimes a small_stmt and sometimes a simple_stmt. Check
# for ; leaves that start a new statements.
start_index = 0
for index, n in enumerate(error_node.children):
if n.start_pos > name.start_pos:
break
if n == ';':
start_index = index + 1
nodes = error_node.children[start_index:]
first_name = nodes[0].get_first_leaf().value
# Make it possible to infer stuff like `import foo.` or
# `from foo.bar`.
if first_name in ('from', 'import'):
is_import_from = first_name == 'from'
level, names = helpers.parse_dotted_names(
nodes,
is_import_from=is_import_from,
until_node=name,
)
return Importer(
context.inference_state, names, context.get_root_context(), level).follow()
return None
def iter_module_names(inference_state, module_context, search_path,
module_cls=ImportName, add_builtin_modules=True):
"""
Get the names of all modules in the search_path. This means file names
and not names defined in the files.
"""
# add builtin module names
if add_builtin_modules:
for name in inference_state.compiled_subprocess.get_builtin_module_names():
yield module_cls(module_context, name)
for name in inference_state.compiled_subprocess.iter_module_names(search_path):
yield module_cls(module_context, name)