forked from pyinstaller/pyinstaller
-
Notifications
You must be signed in to change notification settings - Fork 0
/
modulegraph.py
3293 lines (2765 loc) · 136 KB
/
modulegraph.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
"""
Find modules used by a script, using bytecode analysis.
Based on the stdlib modulefinder by Thomas Heller and Just van Rossum,
but uses a graph data structure and 2.3 features
XXX: Verify all calls to _import_hook (and variants) to ensure that
imports are done in the right way.
"""
#FIXME: To decrease the likelihood of ModuleGraph exceeding the recursion limit
#and hence unpredictably raising fatal exceptions, increase the recursion
#limit at PyInstaller startup (i.e., in the
#PyInstaller.building.build_main.build() function). For details, see:
# https://github.com/pyinstaller/pyinstaller/issues/1919#issuecomment-216016176
import pkg_resources
import ast
import codecs
import imp
import marshal
import os
import pkgutil
import sys
import re
from collections import deque, namedtuple
import warnings
from altgraph.ObjectGraph import ObjectGraph
from altgraph import GraphError
from . import util
from . import zipio
from ._compat import BytesIO, StringIO, pathname2url, _READ_MODE
BOM = codecs.BOM_UTF8.decode('utf-8')
class BUILTIN_MODULE:
def is_package(fqname):
return False
class NAMESPACE_PACKAGE:
def __init__(self, namespace_dirs):
self.namespace_dirs = namespace_dirs
def is_package(self, fqname):
return True
#FIXME: Leverage this rather than magic numbers below.
ABSOLUTE_OR_RELATIVE_IMPORT_LEVEL = -1
"""
Constant instructing the builtin `__import__()` function to attempt both
absolute and relative imports.
"""
#FIXME: Leverage this rather than magic numbers below.
ABSOLUTE_IMPORT_LEVEL = 0
"""
Constant instructing the builtin `__import__()` function to attempt only
absolute imports.
"""
#FIXME: Leverage this rather than magic numbers below.
DEFAULT_IMPORT_LEVEL = (
ABSOLUTE_OR_RELATIVE_IMPORT_LEVEL if sys.version_info[0] == 2 else
ABSOLUTE_IMPORT_LEVEL)
"""
Constant instructing the builtin `__import__()` function to attempt the default
import style specific to the active Python interpreter.
Specifically, under:
* Python 2, this defaults to attempting both absolute and relative imports.
* Python 3, this defaults to attempting only absolute imports.
"""
# TODO: Refactor all uses of explicit filetypes in this module *AND* of the
# imp.get_suffixes() function to use this dictionary instead. Unfortunately,
# tests for explicit filetypes (e.g., ".py") are non-portable. Under Windows,
# for example, both the ".py" *AND* ".pyw" filetypes signify valid uncompiled
# Python modules.
# TODO: The imp.get_suffixes() function (in fact, the entire "imp" package) has
# been deprecated as of Python 3.3 by the importlib.machinery.all_suffixes()
# function, which largely performs the same role. Unfortunately, the latter
# function was only introduced with Python 3.3. Since PyInstaller requires
# Python >= 3.3 when running under Python 3, refactor this as follows:
#
# * Under Python 2, continue calling imp.get_suffixes().
# * Under Python 3, call importlib.machinery.all_suffixes() instead.
_IMPORTABLE_FILETYPE_TO_METADATA = {
filetype: (filetype, open_mode, imp_type)
for filetype, open_mode, imp_type in imp.get_suffixes()
}
# Reverse sort by length so when comparing filenames the longest match first
_IMPORTABLE_FILETYPE_EXTS = sorted(_IMPORTABLE_FILETYPE_TO_METADATA,
key=lambda p: len(p), reverse=True)
"""
Dictionary mapping the filetypes of importable files to the 3-tuple of metadata
describing such files returned by the `imp.get_suffixes()` function whose first
element is that filetype.
This dictionary simplifies platform-portable importation of importable files,
including:
* Uncompiled modules suffixed by `.py` (as well as `.pyw` under Windows).
* Compiled modules suffixed by either `.pyc` or `.pyo`.
* C extensions suffixed by the platform-specific shared library filetype (e.g.,
`.so` under Linux, `.dll` under Windows).
The keys of this dictionary are `.`-prefixed filetypes (e.g., `.py`, `.so`) or
`-`-prefixed filetypes (e.g., `-cpython-37m.dll`[1]);
the values of this dictionary are 3-tuples whose:
1. First element is the same `.` or `-` prefixed filetype.
1. Second element is the mode to be passed to the `open()` built-in to open
files of that filetype under the current platform and Python interpreter
(e.g., `rU` for the `.py` filetype under Python 2, `r` for the same
filetype under Python 3).
1. Third element is a magic number specific to the `imp` module (e.g.,
`imp.C_EXTENSION` for filetypes corresponding to C extensions).
[1] For example of `-cpython-m37.dll` search on
https://packages.msys2.org/package/mingw-w64-x86_64-python3?repo=mingw64
"""
# Modulegraph does a good job at simulating Python's, but it can not
# handle packagepath modifications packages make at runtime. Therefore there
# is a mechanism whereby you can register extra paths in this map for a
# package, and it will be honored.
#
# Note this is a mapping is lists of paths.
_packagePathMap = {}
# Prefix used in magic .pth files used by setuptools to create namespace
# packages without an __init__.py file.
#
# The value is a list of such prefixes as the prefix varies with versions of
# setuptools.
_SETUPTOOLS_NAMESPACEPKG_PTHs=(
# setuptools 31.0.0
("import sys, types, os;has_mfs = sys.version_info > (3, 5);"
"p = os.path.join(sys._getframe(1).f_locals['sitedir'], *('"),
# distribute 0.6.10
("import sys,types,os; p = os.path.join("
"sys._getframe(1).f_locals['sitedir'], *('"),
# setuptools 0.6c9, distribute 0.6.12
("import sys,new,os; p = os.path.join(sys._getframe("
"1).f_locals['sitedir'], *('"),
# setuptools 28.1.0
("import sys, types, os;p = os.path.join("
"sys._getframe(1).f_locals['sitedir'], *('"),
# setuptools 28.7.0
("import sys, types, os;pep420 = sys.version_info > (3, 3);"
"p = os.path.join(sys._getframe(1).f_locals['sitedir'], *('"),
)
class InvalidRelativeImportError (ImportError):
pass
def _namespace_package_path(fqname, pathnames, path=None):
"""
Return the __path__ for the python package in *fqname*.
This function uses setuptools metadata to extract information
about namespace packages from installed eggs.
"""
working_set = pkg_resources.WorkingSet(path)
path = list(pathnames)
for dist in working_set:
if dist.has_metadata('namespace_packages.txt'):
namespaces = dist.get_metadata(
'namespace_packages.txt').splitlines()
if fqname in namespaces:
nspath = os.path.join(dist.location, *fqname.split('.'))
if nspath not in path:
path.append(nspath)
return path
_strs = re.compile(r'''^\s*["']([A-Za-z0-9_]+)["'],?\s*''') # "<- emacs happy
def _eval_str_tuple(value):
"""
Input is the repr of a tuple of strings, output
is that tuple.
This only works with a tuple where the members are
python identifiers.
"""
if not (value.startswith('(') and value.endswith(')')):
raise ValueError(value)
orig_value = value
value = value[1:-1]
result = []
while value:
m = _strs.match(value)
if m is None:
raise ValueError(orig_value)
result.append(m.group(1))
value = value[len(m.group(0)):]
return tuple(result)
def _path_from_importerror(exc, default):
# This is a hack, but sadly enough the necessary information
# isn't available otherwise.
m = re.match(r'^No module named (\S+)$', str(exc))
if m is not None:
return m.group(1)
return default
def os_listdir(path):
"""
Deprecated name
"""
warnings.warn(
"Use zipio.listdir instead of os_listdir",
DeprecationWarning)
return zipio.listdir(path)
def _code_to_file(co):
""" Convert code object to a .pyc pseudo-file """
if sys.version_info >= (3, 7):
header = imp.get_magic() + (b'\0' * 12)
elif sys.version_info >= (3, 4):
header = imp.get_magic() + (b'\0' * 8)
else:
header = imp.get_magic() + (b'\0' * 4)
return BytesIO(header + marshal.dumps(co))
def moduleInfoForPath(path):
for (ext, readmode, typ) in imp.get_suffixes():
if path.endswith(ext):
return os.path.basename(path)[:-len(ext)], readmode, typ
return None
def AddPackagePath(packagename, path):
warnings.warn(
"Use addPackagePath instead of AddPackagePath",
DeprecationWarning)
addPackagePath(packagename, path)
def addPackagePath(packagename, path):
paths = _packagePathMap.get(packagename, [])
paths.append(path)
_packagePathMap[packagename] = paths
_replacePackageMap = {}
# This ReplacePackage mechanism allows modulefinder to work around the
# way the _xmlplus package injects itself under the name "xml" into
# sys.modules at runtime by calling ReplacePackage("_xmlplus", "xml")
# before running ModuleGraph.
def ReplacePackage(oldname, newname):
warnings.warn("use replacePackage instead of ReplacePackage",
DeprecationWarning)
replacePackage(oldname, newname)
def replacePackage(oldname, newname):
_replacePackageMap[oldname] = newname
#FIXME: What is this? Do we actually need this? This appears to provide
#significantly more fine-grained metadata than PyInstaller will ever require.
#It consumes a great deal of space (slots or no slots), since we store an
#instance of this class for each edge of the graph.
class DependencyInfo (namedtuple("DependencyInfo",
["conditional", "function", "tryexcept", "fromlist"])):
__slots__ = ()
def _merged(self, other):
if (not self.conditional and not self.function and not self.tryexcept) \
or (not other.conditional and not other.function and not other.tryexcept):
return DependencyInfo(
conditional=False,
function=False,
tryexcept=False,
fromlist=self.fromlist and other.fromlist)
else:
return DependencyInfo(
conditional=self.conditional or other.conditional,
function=self.function or other.function,
tryexcept=self.tryexcept or other.tryexcept,
fromlist=self.fromlist and other.fromlist)
#FIXME: Shift the following Node class hierarchy into a new
#"PyInstaller.lib.modulegraph.node" module. This module is much too long.
#FIXME: Refactor "_deferred_imports" from a tuple into a proper lightweight
#class leveraging "__slots__". If not for backward compatibility, we'd just
#leverage a named tuple -- but this should do just as well.
#FIXME: Move the "packagepath" attribute into the "Package" class. Only
#packages define the "__path__" special attribute. The codebase currently
#erroneously tests whether "module.packagepath is not None" to determine
#whether a node is a package or not. However, "isinstance(module, Package)" is
#a significantly more reliable test. Refactor the former into the latter.
class Node(object):
"""
Abstract base class (ABC) of all objects added to a `ModuleGraph`.
Attributes
----------
code : codeobject
Code object of the pure-Python module corresponding to this graph node
if any _or_ `None` otherwise.
graphident : str
Synonym of `identifier` required by the `ObjectGraph` superclass of the
`ModuleGraph` class. For readability, the `identifier` attribute should
typically be used instead.
filename : str
Absolute path of this graph node's corresponding module, package, or C
extension if any _or_ `None` otherwise.
identifier : str
Fully-qualified name of this graph node's corresponding module,
package, or C extension.
packagepath : str
List of the absolute paths of all directories comprising this graph
node's corresponding package. If this is a:
* Non-namespace package, this list contains exactly one path.
* Namespace package, this list contains one or more paths.
_deferred_imports : list
List of all target modules imported by the source module corresponding
to this graph node whole importations have been deferred for subsequent
processing in between calls to the `_ModuleGraph._scan_code()` and
`_ModuleGraph._process_imports()` methods for this source module _or_
`None` otherwise. Each element of this list is a 3-tuple
`(have_star, _safe_import_hook_args, _safe_import_hook_kwargs)`
collecting the importation of a target module from this source module
for subsequent processing, where:
* `have_star` is a boolean `True` only if this is a `from`-style star
import (e.g., resembling `from {target_module_name} import *`).
* `_safe_import_hook_args` is a (typically non-empty) sequence of all
positional arguments to be passed to the `_safe_import_hook()` method
to add this importation to the graph.
* `_safe_import_hook_kwargs` is a (typically empty) dictionary of all
keyword arguments to be passed to the `_safe_import_hook()` method
to add this importation to the graph.
Unlike functional languages, Python imposes a maximum depth on the
interpreter stack (and hence recursion). On breaching this depth,
Python raises a fatal `RuntimeError` exception. Since `ModuleGraph`
parses imports recursively rather than iteratively, this depth _was_
commonly breached before the introduction of this list. Python
environments installing a large number of modules (e.g., Anaconda) were
particularly susceptible. Why? Because `ModuleGraph` concurrently
descended through both the abstract syntax trees (ASTs) of all source
modules being parsed _and_ the graph of all target modules imported by
these source modules being built. The stack thus consisted of
alternating layers of AST and graph traversal. To unwind such
alternation and effectively halve the stack depth, `ModuleGraph` now
descends through the abstract syntax tree (AST) of each source module
being parsed and adds all importations originating within this module
to this list _before_ descending into the graph of these importations.
See pyinstaller/pyinstaller/#1289 for further details.
_global_attr_names : set
Set of the unqualified names of all global attributes (e.g., classes,
variables) defined in the pure-Python module corresponding to this
graph node if any _or_ the empty set otherwise. This includes the names
of all attributes imported via `from`-style star imports from other
existing modules (e.g., `from {target_module_name} import *`). This
set is principally used to differentiate the non-ignorable importation
of non-existent submodules in a package from the ignorable importation
of existing global attributes defined in that package's pure-Python
`__init__` submodule in `from`-style imports (e.g., `bar` in
`from foo import bar`, which may be either a submodule or attribute of
`foo`), as such imports ambiguously allow both. This set is _not_ used
to differentiate submodules from attributes in `import`-style imports
(e.g., `bar` in `import foo.bar`, which _must_ be a submodule of
`foo`), as such imports unambiguously allow only submodules.
_starimported_ignored_module_names : set
Set of the fully-qualified names of all existing unparsable modules
that the existing parsable module corresponding to this graph node
attempted to perform one or more "star imports" from. If this module
either does _not_ exist or does but is unparsable, this is the empty
set. Equivalently, this set contains each fully-qualified name
`{trg_module_name}` for which:
* This module contains an import statement of the form
`from {trg_module_name} import *`.
* The module whose name is `{trg_module_name}` exists but is _not_
parsable by `ModuleGraph` (e.g., due to _not_ being pure-Python).
**This set is currently defined but otherwise ignored.**
_submodule_basename_to_node : dict
Dictionary mapping from the unqualified name of each submodule
contained by the parent module corresponding to this graph node to that
submodule's graph node. If this dictionary is non-empty, this parent
module is typically but _not_ always a package (e.g., the non-package
`os` module containing the `os.path` submodule).
"""
__slots__ = [
'code',
'filename',
'graphident',
'identifier',
'packagepath',
'_deferred_imports',
'_global_attr_names',
'_starimported_ignored_module_names',
'_submodule_basename_to_node',
]
def __init__(self, identifier):
"""
Initialize this graph node.
Parameters
----------
identifier : str
Fully-qualified name of this graph node's corresponding module,
package, or C extension.
"""
self.code = None
self.filename = None
self.graphident = identifier
self.identifier = identifier
self.packagepath = None
self._deferred_imports = None
self._global_attr_names = set()
self._starimported_ignored_module_names = set()
self._submodule_basename_to_node = dict()
def is_global_attr(self, attr_name):
"""
`True` only if the pure-Python module corresponding to this graph node
defines a global attribute (e.g., class, variable) with the passed
name.
If this module is actually a package, this method instead returns
`True` only if this package's pure-Python `__init__` submodule defines
such a global attribute. In this case, note that this package may still
contain an importable submodule of the same name. Callers should
attempt to import this attribute as a submodule of this package
_before_ assuming this attribute to be an ignorable global. See
"Examples" below for further details.
Parameters
----------
attr_name : str
Unqualified name of the attribute to be tested.
Returns
----------
bool
`True` only if this module defines this global attribute.
Examples
----------
Consider a hypothetical module `foo` containing submodules `bar` and
`__init__` where the latter assigns `bar` to be a global variable
(possibly star-exported via the special `__all__` global variable):
>>> # In "foo.__init__":
>>> bar = 3.1415
Python 2 and 3 both permissively permit this. This method returns
`True` in this case (i.e., when called on the `foo` package's graph
node, passed the attribute name `bar`) despite the importability of the
`foo.bar` submodule.
"""
return attr_name in self._global_attr_names
def is_submodule(self, submodule_basename):
"""
`True` only if the parent module corresponding to this graph node
contains the submodule with the passed name.
If `True`, this parent module is typically but _not_ always a package
(e.g., the non-package `os` module containing the `os.path` submodule).
Parameters
----------
submodule_basename : str
Unqualified name of the submodule to be tested.
Returns
----------
bool
`True` only if this parent module contains this submodule.
"""
return submodule_basename in self._submodule_basename_to_node
def add_global_attr(self, attr_name):
"""
Record the global attribute (e.g., class, variable) with the passed
name to be defined by the pure-Python module corresponding to this
graph node.
If this module is actually a package, this method instead records this
attribute to be defined by this package's pure-Python `__init__`
submodule.
Parameters
----------
attr_name : str
Unqualified name of the attribute to be added.
"""
self._global_attr_names.add(attr_name)
def add_global_attrs_from_module(self, target_module):
"""
Record all global attributes (e.g., classes, variables) defined by the
target module corresponding to the passed graph node to also be defined
by the source module corresponding to this graph node.
If the source module is actually a package, this method instead records
these attributes to be defined by this package's pure-Python `__init__`
submodule.
Parameters
----------
target_module : Node
Graph node of the target module to import attributes from.
"""
self._global_attr_names.update(target_module._global_attr_names)
def add_submodule(self, submodule_basename, submodule_node):
"""
Add the submodule with the passed name and previously imported graph
node to the parent module corresponding to this graph node.
This parent module is typically but _not_ always a package (e.g., the
non-package `os` module containing the `os.path` submodule).
Parameters
----------
submodule_basename : str
Unqualified name of the submodule to add to this parent module.
submodule_node : Node
Graph node of this submodule.
"""
self._submodule_basename_to_node[submodule_basename] = submodule_node
def get_submodule(self, submodule_basename):
"""
Graph node of the submodule with the passed name in the parent module
corresponding to this graph node.
If this parent module does _not_ contain this submodule, an exception
is raised. Else, this parent module is typically but _not_ always a
package (e.g., the non-package `os` module containing the `os.path`
submodule).
Parameters
----------
module_basename : str
Unqualified name of the submodule to retrieve.
Returns
----------
Node
Graph node of this submodule.
"""
return self._submodule_basename_to_node[submodule_basename]
def get_submodule_or_none(self, submodule_basename):
"""
Graph node of the submodule with the passed unqualified name in the
parent module corresponding to this graph node if this module contains
this submodule _or_ `None`.
This parent module is typically but _not_ always a package (e.g., the
non-package `os` module containing the `os.path` submodule).
Parameters
----------
submodule_basename : str
Unqualified name of the submodule to retrieve.
Returns
----------
Node
Graph node of this submodule if this parent module contains this
submodule _or_ `None`.
"""
return self._submodule_basename_to_node.get(submodule_basename)
def remove_global_attr_if_found(self, attr_name):
"""
Record the global attribute (e.g., class, variable) with the passed
name if previously recorded as defined by the pure-Python module
corresponding to this graph node to be subsequently undefined by the
same module.
If this module is actually a package, this method instead records this
attribute to be undefined by this package's pure-Python `__init__`
submodule.
This method is intended to be called on globals previously defined by
this module that are subsequently undefined via the `del` built-in by
this module, thus "forgetting" or "undoing" these globals.
For safety, there exists no corresponding `remove_global_attr()`
method. While defining this method is trivial, doing so would invite
`KeyError` exceptions on scanning valid Python that lexically deletes a
global in a scope under this module's top level (e.g., in a function)
_before_ defining this global at this top level. Since `ModuleGraph`
cannot and should not (re)implement a full-blown Python interpreter,
ignoring out-of-order deletions is the only sane policy.
Parameters
----------
attr_name : str
Unqualified name of the attribute to be removed.
"""
if self.is_global_attr(attr_name):
self._global_attr_names.remove(attr_name)
def __cmp__(self, other):
try:
otherIdent = getattr(other, 'graphident')
except AttributeError:
return NotImplemented
return cmp(self.graphident, otherIdent) # noqa: F821
def __eq__(self, other):
try:
otherIdent = getattr(other, 'graphident')
except AttributeError:
return False
return self.graphident == otherIdent
def __ne__(self, other):
try:
otherIdent = getattr(other, 'graphident')
except AttributeError:
return True
return self.graphident != otherIdent
def __lt__(self, other):
try:
otherIdent = getattr(other, 'graphident')
except AttributeError:
return NotImplemented
return self.graphident < otherIdent
def __le__(self, other):
try:
otherIdent = getattr(other, 'graphident')
except AttributeError:
return NotImplemented
return self.graphident <= otherIdent
def __gt__(self, other):
try:
otherIdent = getattr(other, 'graphident')
except AttributeError:
return NotImplemented
return self.graphident > otherIdent
def __ge__(self, other):
try:
otherIdent = getattr(other, 'graphident')
except AttributeError:
return NotImplemented
return self.graphident >= otherIdent
def __hash__(self):
return hash(self.graphident)
def infoTuple(self):
return (self.identifier,)
def __repr__(self):
return '%s%r' % (type(self).__name__, self.infoTuple())
# TODO: This indirection is, frankly, unnecessary. The
# ModuleGraph.alias_module() should directly add the desired AliasNode instance
# to the graph rather than indirectly adding an Alias instance to the
# "lazynodes" dictionary.
class Alias(str):
"""
Placeholder aliasing an existing source module to a non-existent target
module (i.e., the desired alias).
For obscure reasons, this class subclasses `str`. Each instance of this
class is the fully-qualified name of the existing source module being
aliased. Unlike the related `AliasNode` class, instances of this class are
_not_ actual nodes and hence _not_ added to the graph; they only facilitate
communication between the `ModuleGraph.alias_module()` and
`ModuleGraph.findNode()` methods.
"""
class AliasNode(Node):
"""
Graph node representing the aliasing of an existing source module under a
non-existent target module name (i.e., the desired alias).
"""
def __init__(self, name, node):
"""
Initialize this alias.
Parameters
----------
name : str
Fully-qualified name of the non-existent target module to be
created (as an alias of the existing source module).
node : Node
Graph node of the existing source module being aliased.
"""
super(AliasNode, self).__init__(name)
#FIXME: Why only some? Why not *EVERYTHING* except "graphident", which
#must remain equal to "name" for lookup purposes? This is, after all,
#an alias. The idea is for the two nodes to effectively be the same.
# Copy some attributes from this source module into this target alias.
for attr_name in (
'identifier', 'packagepath',
'_global_attr_names', '_starimported_ignored_module_names',
'_submodule_basename_to_node'):
if hasattr(node, attr_name):
setattr(self, attr_name, getattr(node, attr_name))
def infoTuple(self):
return (self.graphident, self.identifier)
class BadModule(Node):
pass
class ExcludedModule(BadModule):
pass
class MissingModule(BadModule):
pass
class InvalidRelativeImport (BadModule):
def __init__(self, relative_path, from_name):
identifier = relative_path
if relative_path.endswith('.'):
identifier += from_name
else:
identifier += '.' + from_name
super(InvalidRelativeImport, self).__init__(identifier)
self.relative_path = relative_path
self.from_name = from_name
def infoTuple(self):
return (self.relative_path, self.from_name)
class Script(Node):
def __init__(self, filename):
super(Script, self).__init__(filename)
self.filename = filename
def infoTuple(self):
return (self.filename,)
class BaseModule(Node):
def __init__(self, name, filename=None, path=None):
super(BaseModule, self).__init__(name)
self.filename = filename
self.packagepath = path
def infoTuple(self):
return tuple(filter(None, (self.identifier, self.filename, self.packagepath)))
class BuiltinModule(BaseModule):
pass
class SourceModule(BaseModule):
pass
class InvalidSourceModule(SourceModule):
pass
class CompiledModule(BaseModule):
pass
class InvalidCompiledModule(BaseModule):
pass
class Extension(BaseModule):
pass
class Package(BaseModule):
"""
Graph node representing a non-namespace package.
"""
pass
class ExtensionPackage(Extension, Package):
"""
Graph node representing a package where the __init__ module is an extension
module.
"""
pass
class NamespacePackage(Package):
"""
Graph node representing a namespace package.
"""
pass
class RuntimeModule(BaseModule):
"""
Graph node representing a non-package Python module dynamically defined at
runtime.
Most modules are statically defined on-disk as standard Python files.
Some modules, however, are dynamically defined in-memory at runtime
(e.g., `gi.repository.Gst`, dynamically defined by the statically
defined `gi.repository.__init__` module).
This node represents such a runtime module. Since this is _not_ a package,
all attempts to import submodules from this module in `from`-style import
statements (e.g., the `queue` submodule in `from six.moves import queue`)
will be silently ignored.
To ensure that the parent package of this module if any is also imported
and added to the graph, this node is typically added to the graph by
calling the `ModuleGraph.add_module()` method.
"""
pass
class RuntimePackage(Package):
"""
Graph node representing a non-namespace Python package dynamically defined
at runtime.
Most packages are statically defined on-disk as standard subdirectories
containing `__init__.py` files. Some packages, however, are dynamically
defined in-memory at runtime (e.g., `six.moves`, dynamically defined by
the statically defined `six` module).
This node represents such a runtime package. All attributes imported from
this package in `from`-style import statements that are submodules of this
package (e.g., the `queue` submodule in `from six.moves import queue`) will
be imported rather than ignored.
To ensure that the parent package of this package if any is also imported
and added to the graph, this node is typically added to the graph by
calling the `ModuleGraph.add_module()` method.
"""
pass
#FIXME: Safely removable. We don't actually use this anywhere. After removing
#this class, remove the corresponding entry from "compat".
class FlatPackage(BaseModule):
def __init__(self, *args, **kwds):
warnings.warn(
"This class will be removed in a future version of modulegraph",
DeprecationWarning)
super(FlatPackage, *args, **kwds)
#FIXME: Safely removable. We don't actually use this anywhere. After removing
#this class, remove the corresponding entry from "compat".
class ArchiveModule(BaseModule):
def __init__(self, *args, **kwds):
warnings.warn(
"This class will be removed in a future version of modulegraph",
DeprecationWarning)
super(FlatPackage, *args, **kwds)
# HTML templates for ModuleGraph generator
header = """\
<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
<title>%(TITLE)s</title>
<style>
.node { padding: 0.5em 0 0.5em; border-top: thin grey dotted; }
.moduletype { font: smaller italic }
.node a { text-decoration: none; color: #006699; }
.node a:visited { text-decoration: none; color: #2f0099; }
</style>
</head>
<body>
<h1>%(TITLE)s</h1>"""
entry = """
<div class="node">
<a name="%(NAME)s"></a>
%(CONTENT)s
</div>"""
contpl = """<tt>%(NAME)s</tt> <span class="moduletype">%(TYPE)s</span>"""
contpl_linked = """\
<a target="code" href="%(URL)s" type="text/plain"><tt>%(NAME)s</tt></a>
<span class="moduletype">%(TYPE)s</span>"""
imports = """\
<div class="import">
%(HEAD)s:
%(LINKS)s
</div>
"""
footer = """
</body>
</html>"""
def _ast_names(names):
result = []
for nm in names:
if isinstance(nm, ast.alias):
result.append(nm.name)
else:
result.append(nm)
result = [r for r in result if r != '__main__']
return result
def uniq(seq):
"""Remove duplicates from a list, preserving order"""
# Taken from https://stackoverflow.com/questions/480214
seen = set()
seen_add = seen.add
return [x for x in seq if not (x in seen or seen_add(x))]
if sys.version_info[0] == 2:
DEFAULT_IMPORT_LEVEL = -1
else:
DEFAULT_IMPORT_LEVEL = 0
class _Visitor(ast.NodeVisitor):
def __init__(self, graph, module):
self._graph = graph
self._module = module
self._level = DEFAULT_IMPORT_LEVEL
self._in_if = [False]
self._in_def = [False]
self._in_tryexcept = [False]
@property