Merge pull request cython#36 from vitek/_control_flow

Merged in Vitja's control flow analysis implementation.

Cython/Compiler/Buffer.py

@@ -1140,7 +1140,7 @@ def get_type_information_cname(code, dtype, maxdepth=None):
 }
 
 static CYTHON_INLINE int __Pyx_GetBufferAndValidate(Py_buffer* buf, PyObject* obj, __Pyx_TypeInfo* dtype, int flags, int nd, int cast, __Pyx_BufFmt_StackElem* stack) {
-  if (obj == Py_None) {
+  if (obj == Py_None || obj == NULL) {
     __Pyx_ZeroBuffer(buf);
     return 0;
   }

Cython/Compiler/CmdLine.py

@@ -39,6 +39,7 @@
   -3                             Compile based on Python-3 syntax and code semantics.
   --fast-fail                    Abort the compilation on the first error
   --warning-error, -Werror       Make all warnings into errors
+  --warning-extra, -Wextra       Enable extra warnings
   -X, --directive <name>=<value>[,<name=value,...] Overrides a compiler directive
 """
 
@@ -132,6 +133,8 @@ def get_param(option):
                 Options.fast_fail = True
             elif option in ('-Werror', '--warning-errors'):
                 Options.warning_errors = True
+            elif option in ('-Wextra', '--warning-extra'):
+                options.compiler_directives.update(Options.extra_warnings)
             elif option == "--disable-function-redefinition":
                 Options.disable_function_redefinition = True
             elif option == "--directive" or option.startswith('-X'):

Cython/Compiler/Code.py

@@ -1187,6 +1187,8 @@ def put_var_declaration(self, entry, storage_class="",
                 entry.cname, dll_linkage = dll_linkage))
         if entry.init is not None:
             self.put_safe(" = %s" % entry.type.literal_code(entry.init))
+        elif entry.type.is_pyobject:
+            self.put(" = NULL");
         self.putln(";")
 
     def put_temp_declarations(self, func_context):
@@ -1290,10 +1292,7 @@ def put_xdecref_clear(self, cname, type, nanny=True):
 
     def put_var_decref(self, entry):
         if entry.type.is_pyobject:
-            if entry.init_to_none is False:  # FIXME: 0 and False are treated differently???
-                self.putln("__Pyx_XDECREF(%s);" % self.entry_as_pyobject(entry))
-            else:
-                self.putln("__Pyx_DECREF(%s);" % self.entry_as_pyobject(entry))
+            self.putln("__Pyx_XDECREF(%s);" % self.entry_as_pyobject(entry))
 
     def put_var_decref_clear(self, entry):
         if entry.type.is_pyobject:
@@ -1420,6 +1419,19 @@ def put_error_if_neg(self, pos, value):
 #        return self.putln("if (unlikely(%s < 0)) %s" % (value, self.error_goto(pos)))  # TODO this path is almost _never_ taken, yet this macro makes is slower!
         return self.putln("if (%s < 0) %s" % (value, self.error_goto(pos)))
 
+    def put_error_if_unbound(self, pos, entry):
+        import ExprNodes
+        if entry.from_closure:
+            func = '__Pyx_RaiseClosureNameError'
+            self.globalstate.use_utility_code(
+                ExprNodes.raise_closure_name_error_utility_code)
+        else:
+            func = '__Pyx_RaiseUnboundLocalError'
+            self.globalstate.use_utility_code(
+                ExprNodes.raise_unbound_local_error_utility_code)
+        self.put('if (unlikely(!%s)) { %s("%s"); %s }' % (
+            entry.cname, func, entry.name, self.error_goto(pos)))
+
     def set_error_info(self, pos):
         self.funcstate.should_declare_error_indicator = True
         if self.c_line_in_traceback:

Cython/Compiler/ExprNodes.py

@@ -1267,14 +1267,20 @@ class NameNode(AtomicExprNode):
     #  name            string    Python name of the variable
     #  entry           Entry     Symbol table entry
     #  type_entry      Entry     For extension type names, the original type entry
+    #  cf_is_null      boolean   Is uninitialized before this node
+    #  cf_maybe_null   boolean   Maybe uninitialized before this node
+    #  allow_null      boolean   Don't raise UnboundLocalError
 
     is_name = True
     is_cython_module = False
     cython_attribute = None
-    lhs_of_first_assignment = False
+    lhs_of_first_assignment = False # TODO: remove me
     is_used_as_rvalue = 0
     entry = None
     type_entry = None
+    cf_maybe_null = True
+    cf_is_null = False
+    allow_null = False
 
     def create_analysed_rvalue(pos, env, entry):
         node = NameNode(pos)
@@ -1550,15 +1556,11 @@ def generate_result_code(self, code):
                 code.error_goto_if_null(self.result(), self.pos)))
             code.put_gotref(self.py_result())
 
-        elif entry.is_local and False:
-            # control flow not good enough yet
-            assigned = entry.scope.control_flow.get_state((entry.name, 'initialized'), self.pos)
-            if assigned is False:
-                error(self.pos, "local variable '%s' referenced before assignment" % entry.name)
-            elif not Options.init_local_none and assigned is None:
-                code.putln('if (%s == 0) { PyErr_SetString(PyExc_UnboundLocalError, "%s"); %s }' %
-                           (entry.cname, entry.name, code.error_goto(self.pos)))
-                entry.scope.control_flow.set_state(self.pos, (entry.name, 'initialized'), True)
+        elif entry.is_local or entry.in_closure or entry.from_closure:
+            if entry.type.is_pyobject:
+                if (self.cf_maybe_null or self.cf_is_null) \
+                       and not self.allow_null:
+                    code.put_error_if_unbound(self.pos, entry)
 
     def generate_assignment_code(self, rhs, code):
         #print "NameNode.generate_assignment_code:", self.name ###
@@ -1627,17 +1629,20 @@ def generate_assignment_code(self, rhs, code):
                 if self.use_managed_ref:
                     rhs.make_owned_reference(code)
                     is_external_ref = entry.is_cglobal or self.entry.in_closure or self.entry.from_closure
-                    if not self.lhs_of_first_assignment:
-                        if is_external_ref:
-                            code.put_gotref(self.py_result())
-                        if entry.is_local and not Options.init_local_none:
-                            initialized = entry.scope.control_flow.get_state((entry.name, 'initialized'), self.pos)
-                            if initialized is True:
-                                code.put_decref(self.result(), self.ctype())
-                            elif initialized is None:
+                    if is_external_ref:
+                        if not self.cf_is_null:
+                            if self.cf_maybe_null:
+                                code.put_xgotref(self.py_result())
+                            else:
+                                code.put_gotref(self.py_result())
+                    if entry.is_cglobal:
+                        code.put_decref(self.result(), self.ctype())
+                    else:
+                        if not self.cf_is_null:
+                            if self.cf_maybe_null:
                                 code.put_xdecref(self.result(), self.ctype())
-                        else:
-                            code.put_decref(self.result(), self.ctype())
+                            else:
+                                code.put_decref(self.result(), self.ctype())
                     if is_external_ref:
                         code.put_giveref(rhs.py_result())
 
@@ -1686,8 +1691,11 @@ def generate_deletion_code(self, code):
                     Naming.module_cname,
                     self.entry.name))
         elif self.entry.type.is_pyobject:
-            # Fake it until we can do it for real...
-            self.generate_assignment_code(NoneNode(self.pos), code)
+            if not self.cf_is_null:
+                if self.cf_maybe_null:
+                    code.put_error_if_unbound(self.pos, self.entry)
+                code.put_decref(self.result(), self.ctype())
+                code.putln('%s = NULL;' % self.result())
         else:
             error(self.pos, "Deletion of C names not supported")
 
@@ -4485,8 +4493,6 @@ def generate_evaluation_code(self, code):
             generate_inner_evaluation_code(code)
             code.putln('} /* exit inner scope */')
             return
-        for entry in py_entries:
-            code.put_init_var_to_py_none(entry)
 
         # must free all local Python references at each exit point
         old_loop_labels = tuple(code.new_loop_labels())
@@ -4731,12 +4737,14 @@ def generate_evaluation_code(self, code):
 class DictNode(ExprNode):
     #  Dictionary constructor.
     #
-    #  key_value_pairs  [DictItemNode]
+    #  key_value_pairs     [DictItemNode]
+    #  exclude_null_values [boolean]          Do not add NULL values to dict
     #
     # obj_conversion_errors    [PyrexError]   used internally
 
     subexprs = ['key_value_pairs']
     is_temp = 1
+    exclude_null_values = False
     type = dict_type
 
     obj_conversion_errors = []
@@ -4824,11 +4832,15 @@ def generate_evaluation_code(self, code):
         for item in self.key_value_pairs:
             item.generate_evaluation_code(code)
             if self.type.is_pyobject:
+                if self.exclude_null_values:
+                    code.putln('if (%s) {' % item.value.py_result())
                 code.put_error_if_neg(self.pos,
                     "PyDict_SetItem(%s, %s, %s)" % (
                         self.result(),
                         item.key.py_result(),
                         item.value.py_result()))
+                if self.exclude_null_values:
+                    code.putln('}')
             else:
                 code.putln("%s.%s = %s;" % (
                         self.result(),
@@ -8280,6 +8292,26 @@ def generate_result_code(self, code):
 }
 ''')
 
+raise_unbound_local_error_utility_code = UtilityCode(
+proto = """
+static CYTHON_INLINE void __Pyx_RaiseUnboundLocalError(const char *varname);
+""",
+impl = """
+static CYTHON_INLINE void __Pyx_RaiseUnboundLocalError(const char *varname) {
+    PyErr_Format(PyExc_UnboundLocalError, "local variable '%s' referenced before assignment", varname);
+}
+""")
+
+raise_closure_name_error_utility_code = UtilityCode(
+proto = """
+static CYTHON_INLINE void __Pyx_RaiseClosureNameError(const char *varname);
+""",
+impl = """
+static CYTHON_INLINE void __Pyx_RaiseClosureNameError(const char *varname) {
+    PyErr_Format(PyExc_NameError, "free variable '%s' referenced before assignment in enclosing scope", varname);
+}
+""")
+
 #------------------------------------------------------------------------------------
 
 getitem_dict_utility_code = UtilityCode(

Cython/Compiler/FlowControl.pxd

@@ -0,0 +1,77 @@
+cimport cython
+
+cdef class ControlBlock:
+     cdef public set children
+     cdef public set parents
+     cdef public set positions
+     cdef public list stats
+     cdef public dict gen
+     cdef public set bounded
+     cdef public dict input
+     cdef public dict output
+
+     # Big integer it bitsets
+     cdef public object i_input
+     cdef public object i_output
+     cdef public object i_gen
+     cdef public object i_kill
+     cdef public object i_state
+
+     cpdef bint empty(self)
+     cpdef detach(self)
+     cpdef add_child(self, block)
+
+cdef class ExitBlock(ControlBlock):
+     cpdef bint empty(self)
+
+cdef class NameAssignment:
+    cdef public bint is_arg
+    cdef public object lhs
+    cdef public object rhs
+    cdef public object entry
+    cdef public object pos
+    cdef public set refs
+    cdef public object bit
+
+cdef class AssignmentList:
+    cdef public object bit
+    cdef public object mask
+    cdef public list stats
+
+cdef class ControlFlow:
+     cdef public set blocks
+     cdef public set entries
+     cdef public list loops
+     cdef public list exceptions
+
+     cdef public ControlBlock entry_point
+     cdef public ExitBlock exit_point
+     cdef public ControlBlock block
+
+     cdef public dict assmts
+
+     cpdef newblock(self, parent=*)
+     cpdef nextblock(self, parent=*)
+     cpdef bint is_tracked(self, entry)
+     cpdef mark_position(self, node)
+     cpdef mark_assignment(self, lhs, rhs, entry=*)
+     cpdef mark_argument(self, lhs, rhs, entry)
+     cpdef mark_deletion(self, node, entry)
+     cpdef mark_reference(self, node, entry)
+     cpdef normalize(self)
+
+     @cython.locals(offset=object, assmts=AssignmentList,
+                    block=ControlBlock)
+     cpdef initialize(self)
+
+     @cython.locals(assmts=AssignmentList, assmt=NameAssignment)
+     cpdef set map_one(self, istate, entry)
+
+     @cython.locals(block=ControlBlock, parent=ControlBlock)
+     cdef reaching_definitions(self)
+
+cdef class Uninitialized:
+     pass
+
+@cython.locals(dirty=bint, block=ControlBlock, parent=ControlBlock)
+cdef check_definitions(ControlFlow flow, dict compiler_directives)