Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Newer
Older
100644 342 lines (304 sloc) 13.086 kB
a96882e @scoder use explicit relative imports everywhere and enable absolute imports …
scoder authored
1 from __future__ import absolute_import
2
53e4c10 @dagss Refactored pipeline construction to Pipeline.py
dagss authored
3 import itertools
4 from time import time
5
a96882e @scoder use explicit relative imports everywhere and enable absolute imports …
scoder authored
6 from . import Errors
7 from . import DebugFlags
8 from . import Options
9 from .Visitor import CythonTransform
10 from .Errors import CompileError, InternalError, AbortError
11 from . import Naming
53e4c10 @dagss Refactored pipeline construction to Pipeline.py
dagss authored
12
13 #
14 # Really small pipeline stages
15 #
16 def dumptree(t):
17 # For quick debugging in pipelines
69139ea @scoder adapt some more syntax to Py2/Py3
scoder authored
18 print(t.dump())
53e4c10 @dagss Refactored pipeline construction to Pipeline.py
dagss authored
19 return t
20
21 def abort_on_errors(node):
22 # Stop the pipeline if there are any errors.
23 if Errors.num_errors != 0:
1fed785 @scoder fix some unnecessary Py2-isms in the code base
scoder authored
24 raise AbortError("pipeline break")
53e4c10 @dagss Refactored pipeline construction to Pipeline.py
dagss authored
25 return node
26
27 def parse_stage_factory(context):
28 def parse(compsrc):
29 source_desc = compsrc.source_desc
aae844f @robertwb Undo --embed -> __main__ conversion.
robertwb authored
30 full_module_name = compsrc.full_module_name
53e4c10 @dagss Refactored pipeline construction to Pipeline.py
dagss authored
31 initial_pos = (source_desc, 1, 0)
32 saved_cimport_from_pyx, Options.cimport_from_pyx = Options.cimport_from_pyx, False
e0fd2b3 @robertwb Rename module to __main__ in embed mode.
robertwb authored
33 scope = context.find_module(full_module_name, pos = initial_pos, need_pxd = 0)
53e4c10 @dagss Refactored pipeline construction to Pipeline.py
dagss authored
34 Options.cimport_from_pyx = saved_cimport_from_pyx
35 tree = context.parse(source_desc, scope, pxd = 0, full_module_name = full_module_name)
36 tree.compilation_source = compsrc
37 tree.scope = scope
38 tree.is_pxd = False
39 return tree
40 return parse
41
42 def parse_pxd_stage_factory(context, scope, module_name):
43 def parse(source_desc):
44 tree = context.parse(source_desc, scope, pxd=True,
45 full_module_name=module_name)
46 tree.scope = scope
47 tree.is_pxd = True
48 return tree
49 return parse
50
51 def generate_pyx_code_stage_factory(options, result):
52 def generate_pyx_code_stage(module_node):
53 module_node.process_implementation(options, result)
54 result.compilation_source = module_node.compilation_source
55 return result
56 return generate_pyx_code_stage
57
58 def inject_pxd_code_stage_factory(context):
59 def inject_pxd_code_stage(module_node):
38abf1b @scoder make dict iteration compatible with Py2/Py3
scoder authored
60 for name, (statlistnode, scope) in context.pxds.items():
53e4c10 @dagss Refactored pipeline construction to Pipeline.py
dagss authored
61 module_node.merge_in(statlistnode, scope)
62 return module_node
63 return inject_pxd_code_stage
64
8fbf6b8 @markflorisson Fix cython scope caching (it breaks about half of the test suite, but…
markflorisson authored
65 def use_utility_code_definitions(scope, target, seen=None):
66 if seen is None:
67 seen = set()
68
38abf1b @scoder make dict iteration compatible with Py2/Py3
scoder authored
69 for entry in scope.entries.values():
8fbf6b8 @markflorisson Fix cython scope caching (it breaks about half of the test suite, but…
markflorisson authored
70 if entry in seen:
71 continue
72
73 seen.add(entry)
31609ac @dagss Make it possible to construct sub-magic-modules of cython
dagss authored
74 if entry.used and entry.utility_code_definition:
75 target.use_utility_code(entry.utility_code_definition)
ab8b5f9 @markflorisson Cython Utility Code cname extclass decorator + extmethod prototypes
markflorisson authored
76 for required_utility in entry.utility_code_definition.requires:
77 target.use_utility_code(required_utility)
31609ac @dagss Make it possible to construct sub-magic-modules of cython
dagss authored
78 elif entry.as_module:
8fbf6b8 @markflorisson Fix cython scope caching (it breaks about half of the test suite, but…
markflorisson authored
79 use_utility_code_definitions(entry.as_module, target, seen)
31609ac @dagss Make it possible to construct sub-magic-modules of cython
dagss authored
80
81 def inject_utility_code_stage_factory(context):
82 def inject_utility_code_stage(module_node):
dc0e025 @markflorisson merge -- working fused types, broken cython testscope
markflorisson authored
83 use_utility_code_definitions(context.cython_scope, module_node.scope)
31609ac @dagss Make it possible to construct sub-magic-modules of cython
dagss authored
84 added = []
85 # Note: the list might be extended inside the loop (if some utility code
07a58cc @dagss use_utility_code vs. memoryviews fixup
dagss authored
86 # pulls in other utility code, explicitly or implicitly)
31609ac @dagss Make it possible to construct sub-magic-modules of cython
dagss authored
87 for utilcode in module_node.scope.utility_code_list:
88 if utilcode in added: continue
89 added.append(utilcode)
07a58cc @dagss use_utility_code vs. memoryviews fixup
dagss authored
90 if utilcode.requires:
91 for dep in utilcode.requires:
92 if not dep in added and not dep in module_node.scope.utility_code_list:
93 module_node.scope.utility_code_list.append(dep)
31609ac @dagss Make it possible to construct sub-magic-modules of cython
dagss authored
94 tree = utilcode.get_tree()
95 if tree:
96 module_node.merge_in(tree.body, tree.scope, merge_scope=True)
97 return module_node
98 return inject_utility_code_stage
f4e4484 @dagss Support utility code written in Cython
dagss authored
99
9ab6e44 @markflorisson Fix refnanny + test tags
markflorisson authored
100 class UseUtilityCodeDefinitions(CythonTransform):
101 # Temporary hack to use any utility code in nodes' "utility_code_definitions".
102 # This should be moved to the code generation phase of the relevant nodes once
103 # it is safe to generate CythonUtilityCode at code generation time.
104 def __call__(self, node):
105 self.scope = node.scope
106 return super(UseUtilityCodeDefinitions, self).__call__(node)
107
89c9297 @scoder move utility code inclusion from Name/AttributeNodes to the latest po…
scoder authored
108 def process_entry(self, entry):
109 if entry:
110 for utility_code in (entry.utility_code, entry.utility_code_definition):
111 if utility_code:
112 self.scope.use_utility_code(utility_code)
113
9ab6e44 @markflorisson Fix refnanny + test tags
markflorisson authored
114 def visit_AttributeNode(self, node):
89c9297 @scoder move utility code inclusion from Name/AttributeNodes to the latest po…
scoder authored
115 self.process_entry(node.entry)
9ab6e44 @markflorisson Fix refnanny + test tags
markflorisson authored
116 return node
117
118 def visit_NameNode(self, node):
89c9297 @scoder move utility code inclusion from Name/AttributeNodes to the latest po…
scoder authored
119 self.process_entry(node.entry)
120 self.process_entry(node.type_entry)
9ab6e44 @markflorisson Fix refnanny + test tags
markflorisson authored
121 return node
404c896 @robertwb Add support for external C++ template functions.
robertwb authored
122
53e4c10 @dagss Refactored pipeline construction to Pipeline.py
dagss authored
123 #
124 # Pipeline factories
125 #
126
f4e4484 @dagss Support utility code written in Cython
dagss authored
127 def create_pipeline(context, mode, exclude_classes=()):
53e4c10 @dagss Refactored pipeline construction to Pipeline.py
dagss authored
128 assert mode in ('pyx', 'py', 'pxd')
a96882e @scoder use explicit relative imports everywhere and enable absolute imports …
scoder authored
129 from .Visitor import PrintTree
130 from .ParseTreeTransforms import WithTransform, NormalizeTree, PostParse, PxdPostParse
131 from .ParseTreeTransforms import ForwardDeclareTypes, AnalyseDeclarationsTransform
132 from .ParseTreeTransforms import AnalyseExpressionsTransform, FindInvalidUseOfFusedTypes
133 from .ParseTreeTransforms import CreateClosureClasses, MarkClosureVisitor, DecoratorTransform
134 from .ParseTreeTransforms import InterpretCompilerDirectives, TransformBuiltinMethods
135 from .ParseTreeTransforms import ExpandInplaceOperators, ParallelRangeTransform
136 from .ParseTreeTransforms import CalculateQualifiedNamesTransform
137 from .TypeInference import MarkParallelAssignments, MarkOverflowingArithmetic
138 from .ParseTreeTransforms import AdjustDefByDirectives, AlignFunctionDefinitions
139 from .ParseTreeTransforms import RemoveUnreachableCode, GilCheck
140 from .FlowControl import ControlFlowAnalysis
141 from .AnalysedTreeTransforms import AutoTestDictTransform
142 from .AutoDocTransforms import EmbedSignature
143 from .Optimize import FlattenInListTransform, SwitchTransform, IterationTransform
144 from .Optimize import EarlyReplaceBuiltinCalls, OptimizeBuiltinCalls
145 from .Optimize import InlineDefNodeCalls
146 from .Optimize import ConstantFolding, FinalOptimizePhase
147 from .Optimize import DropRefcountingTransform
148 from .Optimize import ConsolidateOverflowCheck
149 from .Buffer import IntroduceBufferAuxiliaryVars
150 from .ModuleNode import check_c_declarations, check_c_declarations_pxd
3bed371 [mq]: memview-stuff
Kurt Smith authored
151
53e4c10 @dagss Refactored pipeline construction to Pipeline.py
dagss authored
152
153 if mode == 'pxd':
154 _check_c_declarations = check_c_declarations_pxd
155 _specific_post_parse = PxdPostParse(context)
156 else:
157 _check_c_declarations = check_c_declarations
158 _specific_post_parse = None
159
160 if mode == 'py':
161 _align_function_definitions = AlignFunctionDefinitions(context)
162 else:
163 _align_function_definitions = None
164
31609ac @dagss Make it possible to construct sub-magic-modules of cython
dagss authored
165 # NOTE: This is the "common" parts of the pipeline, which is also
166 # code in pxd files. So it will be run multiple times in a
167 # compilation stage.
f4e4484 @dagss Support utility code written in Cython
dagss authored
168 stages = [
53e4c10 @dagss Refactored pipeline construction to Pipeline.py
dagss authored
169 NormalizeTree(context),
170 PostParse(context),
171 _specific_post_parse,
172 InterpretCompilerDirectives(context, context.compiler_directives),
173 ParallelRangeTransform(context),
174 AdjustDefByDirectives(context),
d78673f @scoder implement "async with" (PEP 492)
scoder authored
175 WithTransform(context),
53e4c10 @dagss Refactored pipeline construction to Pipeline.py
dagss authored
176 MarkClosureVisitor(context),
177 _align_function_definitions,
178 RemoveUnreachableCode(context),
179 ConstantFolding(),
180 FlattenInListTransform(),
181 DecoratorTransform(context),
182 ForwardDeclareTypes(context),
183 AnalyseDeclarationsTransform(context),
184 AutoTestDictTransform(context),
185 EmbedSignature(context),
186 EarlyReplaceBuiltinCalls(context), ## Necessary?
d7453bd @scoder remove unclear comment and add docstring to explain why
scoder authored
187 TransformBuiltinMethods(context),
e090be9 @vitek Rename MarkAssignments to MarkParallelAssignments
vitek authored
188 MarkParallelAssignments(context),
5bac3a3 @vitek Rename CreateControlFlowGraph to ControlFlowAnalysis
vitek authored
189 ControlFlowAnalysis(context),
53e4c10 @dagss Refactored pipeline construction to Pipeline.py
dagss authored
190 RemoveUnreachableCode(context),
e090be9 @vitek Rename MarkAssignments to MarkParallelAssignments
vitek authored
191 # MarkParallelAssignments(context),
53e4c10 @dagss Refactored pipeline construction to Pipeline.py
dagss authored
192 MarkOverflowingArithmetic(context),
193 IntroduceBufferAuxiliaryVars(context),
194 _check_c_declarations,
1e94a83 @vitek Simple closure defnode call inlining
vitek authored
195 InlineDefNodeCalls(context),
53e4c10 @dagss Refactored pipeline construction to Pipeline.py
dagss authored
196 AnalyseExpressionsTransform(context),
6349253 @markflorisson Better fused types error detection, specialized fused C++ templates
markflorisson authored
197 FindInvalidUseOfFusedTypes(context),
53e4c10 @dagss Refactored pipeline construction to Pipeline.py
dagss authored
198 ExpandInplaceOperators(context),
5a2287c @scoder change pipeline order to include tree changes made by structural opti…
scoder authored
199 IterationTransform(context),
200 SwitchTransform(context),
53e4c10 @dagss Refactored pipeline construction to Pipeline.py
dagss authored
201 OptimizeBuiltinCalls(context), ## Necessary?
10d51a6 @scoder fix crash when looking up closure names in OptimizeBuiltinCalls trans…
scoder authored
202 CreateClosureClasses(context), ## After all lookups and type inference
265bbaa @scoder move qualname calculation transform a little later in the pipeline
scoder authored
203 CalculateQualifiedNamesTransform(context),
7e1b07c @robertwb Cheaper overflow checks for nested expressions.
robertwb authored
204 ConsolidateOverflowCheck(context),
53e4c10 @dagss Refactored pipeline construction to Pipeline.py
dagss authored
205 DropRefcountingTransform(),
206 FinalOptimizePhase(context),
207 GilCheck(),
9ab6e44 @markflorisson Fix refnanny + test tags
markflorisson authored
208 UseUtilityCodeDefinitions(context),
53e4c10 @dagss Refactored pipeline construction to Pipeline.py
dagss authored
209 ]
f4e4484 @dagss Support utility code written in Cython
dagss authored
210 filtered_stages = []
211 for s in stages:
212 if s.__class__ not in exclude_classes:
213 filtered_stages.append(s)
214 return filtered_stages
53e4c10 @dagss Refactored pipeline construction to Pipeline.py
dagss authored
215
f4e4484 @dagss Support utility code written in Cython
dagss authored
216 def create_pyx_pipeline(context, options, result, py=False, exclude_classes=()):
53e4c10 @dagss Refactored pipeline construction to Pipeline.py
dagss authored
217 if py:
218 mode = 'py'
219 else:
220 mode = 'pyx'
221 test_support = []
222 if options.evaluate_tree_assertions:
a96882e @scoder use explicit relative imports everywhere and enable absolute imports …
scoder authored
223 from ..TestUtils import TreeAssertVisitor
53e4c10 @dagss Refactored pipeline construction to Pipeline.py
dagss authored
224 test_support.append(TreeAssertVisitor())
225
226 if options.gdb_debug:
a96882e @scoder use explicit relative imports everywhere and enable absolute imports …
scoder authored
227 from ..Debugger import DebugWriter # requires Py2.5+
228 from .ParseTreeTransforms import DebugTransform
53e4c10 @dagss Refactored pipeline construction to Pipeline.py
dagss authored
229 context.gdb_debug_outputwriter = DebugWriter.CythonDebugWriter(
230 options.output_dir)
231 debug_transform = [DebugTransform(context, options, result)]
232 else:
233 debug_transform = []
234
235 return list(itertools.chain(
236 [parse_stage_factory(context)],
f4e4484 @dagss Support utility code written in Cython
dagss authored
237 create_pipeline(context, mode, exclude_classes=exclude_classes),
53e4c10 @dagss Refactored pipeline construction to Pipeline.py
dagss authored
238 test_support,
239 [inject_pxd_code_stage_factory(context),
31609ac @dagss Make it possible to construct sub-magic-modules of cython
dagss authored
240 inject_utility_code_stage_factory(context),
53e4c10 @dagss Refactored pipeline construction to Pipeline.py
dagss authored
241 abort_on_errors],
242 debug_transform,
243 [generate_pyx_code_stage_factory(options, result)]))
244
245 def create_pxd_pipeline(context, scope, module_name):
a96882e @scoder use explicit relative imports everywhere and enable absolute imports …
scoder authored
246 from .CodeGeneration import ExtractPxdCode
53e4c10 @dagss Refactored pipeline construction to Pipeline.py
dagss authored
247
248 # The pxd pipeline ends up with a CCodeWriter containing the
249 # code of the pxd, as well as a pxd scope.
250 return [
251 parse_pxd_stage_factory(context, scope, module_name)
252 ] + create_pipeline(context, 'pxd') + [
fa9c5bd fix compiler crash on .pxd file processing
Stefan Behnel authored
253 ExtractPxdCode()
53e4c10 @dagss Refactored pipeline construction to Pipeline.py
dagss authored
254 ]
255
256 def create_py_pipeline(context, options, result):
257 return create_pyx_pipeline(context, options, result, py=True)
258
259 def create_pyx_as_pxd_pipeline(context, result):
a96882e @scoder use explicit relative imports everywhere and enable absolute imports …
scoder authored
260 from .ParseTreeTransforms import AlignFunctionDefinitions, \
81bd573 @markflorisson py23 compatibility
markflorisson authored
261 MarkClosureVisitor, WithTransform, AnalyseDeclarationsTransform
a96882e @scoder use explicit relative imports everywhere and enable absolute imports …
scoder authored
262 from .Optimize import ConstantFolding, FlattenInListTransform
263 from .Nodes import StatListNode
53e4c10 @dagss Refactored pipeline construction to Pipeline.py
dagss authored
264 pipeline = []
f4e4484 @dagss Support utility code written in Cython
dagss authored
265 pyx_pipeline = create_pyx_pipeline(context, context.options, result,
266 exclude_classes=[
267 AlignFunctionDefinitions,
268 MarkClosureVisitor,
269 ConstantFolding,
270 FlattenInListTransform,
271 WithTransform
272 ])
53e4c10 @dagss Refactored pipeline construction to Pipeline.py
dagss authored
273 for stage in pyx_pipeline:
274 pipeline.append(stage)
275 if isinstance(stage, AnalyseDeclarationsTransform):
276 # This is the last stage we need.
277 break
278 def fake_pxd(root):
279 for entry in root.scope.entries.values():
16005a3 @nnemkin In cimport_from_pyx mode don't treat "cdef extern from" variables as …
nnemkin authored
280 if not entry.in_cinclude:
281 entry.defined_in_pxd = 1
b65f48f @robertwb Cimported name mangling.
robertwb authored
282 if entry.name == entry.cname and entry.visibility != 'extern':
283 # Always mangle non-extern cimported entries.
284 entry.cname = entry.scope.mangle(Naming.func_prefix, entry.name)
53e4c10 @dagss Refactored pipeline construction to Pipeline.py
dagss authored
285 return StatListNode(root.pos, stats=[]), root.scope
286 pipeline.append(fake_pxd)
287 return pipeline
288
0daf725 @markflorisson Support cname() decorators in CythonUtilityCode
markflorisson authored
289 def insert_into_pipeline(pipeline, transform, before=None, after=None):
290 """
291 Insert a new transform into the pipeline after or before an instance of
292 the given class. e.g.
293
294 pipeline = insert_into_pipeline(pipeline, transform,
295 after=AnalyseDeclarationsTransform)
296 """
297 assert before or after
298
299 cls = before or after
300 for i, t in enumerate(pipeline):
301 if isinstance(t, cls):
302 break
303
304 if after:
305 i += 1
306
307 return pipeline[:i] + [transform] + pipeline[i:]
308
53e4c10 @dagss Refactored pipeline construction to Pipeline.py
dagss authored
309 #
310 # Running a pipeline
311 #
312
155a4ef @markflorisson Support slicing memoryview slices
markflorisson authored
313 def run_pipeline(pipeline, source, printtree=True):
a96882e @scoder use explicit relative imports everywhere and enable absolute imports …
scoder authored
314 from .Visitor import PrintTree
155a4ef @markflorisson Support slicing memoryview slices
markflorisson authored
315
53e4c10 @dagss Refactored pipeline construction to Pipeline.py
dagss authored
316 error = None
317 data = source
318 try:
319 try:
320 for phase in pipeline:
321 if phase is not None:
322 if DebugFlags.debug_verbose_pipeline:
323 t = time()
69139ea @scoder adapt some more syntax to Py2/Py3
scoder authored
324 print("Entering pipeline phase %r" % phase)
155a4ef @markflorisson Support slicing memoryview slices
markflorisson authored
325 if not printtree and isinstance(phase, PrintTree):
326 continue
53e4c10 @dagss Refactored pipeline construction to Pipeline.py
dagss authored
327 data = phase(data)
328 if DebugFlags.debug_verbose_pipeline:
69139ea @scoder adapt some more syntax to Py2/Py3
scoder authored
329 print(" %.3f seconds" % (time() - t))
46b9991 @encukou 2to3: Apply the 'except' fixer
encukou authored
330 except CompileError as err:
53e4c10 @dagss Refactored pipeline construction to Pipeline.py
dagss authored
331 # err is set
332 Errors.report_error(err)
333 error = err
46b9991 @encukou 2to3: Apply the 'except' fixer
encukou authored
334 except InternalError as err:
53e4c10 @dagss Refactored pipeline construction to Pipeline.py
dagss authored
335 # Only raise if there was not an earlier error
336 if Errors.num_errors == 0:
337 raise
338 error = err
46b9991 @encukou 2to3: Apply the 'except' fixer
encukou authored
339 except AbortError as err:
53e4c10 @dagss Refactored pipeline construction to Pipeline.py
dagss authored
340 error = err
341 return (error, data)
Something went wrong with that request. Please try again.