Merge pull request #177 from maximusron/eigen-reference-dev

Updates to Eigen and ptr/reference  support with ImplClassTypes and basic types

python/cppyy/numba_ext.py

@@ -16,7 +16,11 @@
 import numba.core.typing as nb_typing
 
 from llvmlite import ir
-
+from llvmlite.llvmpy.core import Type as llvmfnty
+from numba.extending import make_attribute_wrapper
+import itertools
+import re
+import inspect
 
 # setuptools entry point for Numba
 def _init_extension():
@@ -26,6 +30,7 @@ def _init_extension():
 class Qualified:
     default   = 0
     value     = 1
+    instance  = 2
 
 ir_byte     = ir.IntType(8)
 ir_voidptr  = ir.PointerType(ir_byte)                 # by convention
@@ -56,12 +61,30 @@ class Qualified:
     'unsigned long long'     : nb_types.ulonglong,
     'float'                  : nb_types.float32,
     'double'                 : nb_types.float64,
+    'char'                   : nb_types.char,
+    'unsigned char'          : nb_types.uchar,
+    'char*'                  : nb_types.unicode_type
 }
 
+def resolve_std_vector(val):
+    return re.match(r'std::vector<(.+?)>', val).group(1)
+
+def resolve_const_types(val):
+    return re.match(r'const\s+(.+)\s*\*', val).group(1)
+
 def cpp2numba(val):
     if type(val) != str:
         # TODO: distinguish ptr/ref/byval
+        # TODO: Only metaclasses/proxies end up here since
+        #  ref cases makes the RETURN_TYPE from reflex a string
         return typeof_scope(val, nb_typing.typeof.Purpose.argument, Qualified.value)
+    elif val.startswith("std::vector"):
+        type_arr = getattr(numba, str(cpp2numba(resolve_std_vector(val))))[:]
+        return type_arr
+    elif val[-1] == '*' or val[-1] == '&':
+        if val.startswith('const'):
+            return nb_types.CPointer(cpp2numba(resolve_const_types(val)))
+        return nb_types.CPointer(_cpp2numba[val[:-1]])
     return _cpp2numba[val]
 
 _numba2cpp = dict()
@@ -76,7 +99,39 @@ def numba2cpp(val):
         if val == nb_types.int64:      # Python int
             # TODO: this is only necessary until "best matching" is in place
             val = nb_types.intc        # more likely match candidate
-    return _numba2cpp[val]
+    elif isinstance(val, numba.types.CPointer):
+        return _numba2cpp[val.dtype]
+    elif isinstance(val, numba.types.RawPointer):
+        return _numba2cpp[nb_types.voidptr]
+    elif isinstance(val, numba.types.Array):
+        return "std::vector<" + _numba2cpp[val.dtype] + ">"
+    elif isinstance(val, CppClassNumbaType):
+        return val._scope.__cpp_name__
+    else:
+        try:
+            return _numba2cpp[val]
+        except:
+            raise RuntimeError("Type mapping failed from Numba to C++ for ", val)
+
+def numba_arg_convertor(args):
+    args_cpp = []
+    for i, arg in enumerate(list(args)):
+        # If the user explicitly passes an argument using numba CPointer, the regex match is used
+        # to detect the pass by reference since the dispatcher always returns typeref[val*]
+        match = re.search(r"typeref\[(.*?)\*\]", str(arg))
+        if match:
+            literal_val = match.group(1)
+            arg_type = numba.typeof(eval(literal_val))
+            args_cpp.append(to_ref(numba2cpp(arg_type)))
+        else:
+            args_cpp.append(numba2cpp(arg))
+    return tuple(args_cpp)
+
+def to_ref(type_list):
+    ref_list = []
+    for i, l in enumerate(type_list):
+        ref_list.append(l + '&')
+    return ref_list
 
 # TODO: looks like Numba treats unsigned types as signed when lowering,
 # which seems to work as they're just reinterpret_casts
@@ -101,7 +156,19 @@ def numba2cpp(val):
 }
 
 def cpp2ir(val):
-    return _cpp2ir[val]
+    try:
+        return _cpp2ir[val]
+    except KeyError:
+        if val.startswith("std::vector"):
+            ## TODO should be possible to obtain the vector length from the CPPDataMember val
+            type_arr = ir.VectorType(cpp2ir(resolve_std_vector(val)), 3)
+            return type_arr
+        elif val != "char*" and val[-1] == "*":
+            if val.startswith('const'):
+                return ir.PointerType(cpp2ir(resolve_const_types(val)))
+            else:
+                type_2 = _cpp2ir[val[:-1]]
+                return ir.PointerType(type_2)
 
 
 #
@@ -120,6 +187,8 @@ def __init__(self, func, is_method=False):
 
         self._signatures = list()
         self._impl_keys = dict()
+        self._arg_set_matched = tuple()
+        self.ret_type = None
 
     def is_precise(self):
         return True          # by definition
@@ -130,24 +199,28 @@ def get_call_type(self, context, args, kwds):
         except KeyError:
             pass
 
-        ol = CppFunctionNumbaType(self._func.__overload__(tuple(numba2cpp(x) for x in args)), self._is_method)
+        ol = CppFunctionNumbaType(self._func.__overload__(numba_arg_convertor(args)), self._is_method)
 
         thistype = None
         if self._is_method:
             thistype = nb_types.voidptr
 
+        self.ret_type = cpp2numba(ol._func.__cpp_reflex__(cpp_refl.RETURN_TYPE))
         ol.sig = nb_typing.Signature(
-            return_type=cpp2numba(ol._func.__cpp_reflex__(cpp_refl.RETURN_TYPE)),
+            return_type=self.ret_type,
             args=args,
             recvr=thistype)
 
         extsig = ol.sig
         if self._is_method:
+            self.ret_type = ol.sig.return_type
             args = (nb_types.voidptr, *args)
             extsig = nb_typing.Signature(
                 return_type=ol.sig.return_type, args=args, recvr=None)
 
         self._impl_keys[args] = ol
+        self._arg_set_matched = numba_arg_convertor(args)
+
 
         @nb_iutils.lower_builtin(ol, *args)
         def lower_external_call(context, builder, sig, args,
@@ -166,9 +239,12 @@ def get_call_signatures(self):
     def get_impl_key(self, sig):
         return self._impl_keys[sig.args]
 
+    #TODO : Remove the redundancy of __overload__ matching and use this function to only obtain the address given the matched overload
     def get_pointer(self, func):
         if func is None: func = self._func
-        ol = func.__overload__(tuple(numba2cpp(x) for x in self.sig.args))
+
+        ol = func.__overload__(numba_arg_convertor(self.sig.args))
+
         address = cppyy.addressof(ol)
         if not address:
             raise RuntimeError("unresolved address for %s" % str(ol))
@@ -222,10 +298,19 @@ def __init__(self, name, offset, cpptype):
 #
 class CppClassNumbaType(CppFunctionNumbaType):
     def __init__(self, scope, qualifier):
+        addr = None
+        cppinstance_val = None
+        if qualifier == Qualified.instance:
+            addr = cppyy.addressof(scope)
+            cppinstance_val = scope
+            scope = type(scope)
+            qualifier = Qualified.default
         super(CppClassNumbaType, self).__init__(scope.__init__)
         self.name = 'CppClass(%s)' % scope.__cpp_name__    # overrides value in Type
         self._scope     = scope
         self._qualifier = qualifier
+        self._cppinstanceval = cppinstance_val
+        self._addr = addr
 
     def get_scope(self):
         return self._scope
@@ -311,7 +396,6 @@ def cppclass_getattr_impl(context, builder, typ, val, attr):
         return builder.bitcast(val, ir_voidptr)
 
     elif q == Qualified.value:
-        # TODO: take address of by value returns
         return None
 
     assert not "unknown qualified type"
@@ -339,6 +423,7 @@ def get(self, builder, val, pos):
         return builder.load(pf)
 
 class ImplClassValueModel(ImplAggregateValueModel):
+  # TODO : Should the address have to be passed here and stored in meminfo
   # value: representation inside function body. Maybe stored in stack.
   #        The representation here are flexible.
     def get_value_type(self):
@@ -364,6 +449,14 @@ def get_return_type(self):
 
 @nb_ext.typeof_impl.register(cpp_types.Scope)
 def typeof_scope(val, c, q = Qualified.default):
+    is_instance = False
+    cppinstance_val = None
+    if q == Qualified.instance:
+        cppinstance_val = val
+        val = type(val)
+        q = Qualified.default
+        is_instance = True
+
     global scope_numbatypes
 
     try:
@@ -379,16 +472,24 @@ def typeof_scope(val, c, q = Qualified.default):
     class ImplClassType(CppClassNumbaType):
         pass
 
-    cnt = ImplClassType(val, q)
+    if is_instance:
+        cnt = ImplClassType(cppinstance_val, Qualified.instance)
+    else:
+        cnt = ImplClassType(val, q)
+
     scope_numbatypes[q][val] = cnt
 
   # declare data members to Numba
     data_members = list()
+    member_methods = dict()
+
     for name, field in val.__dict__.items():
         if type(field) == cpp_types.DataMember:
             data_members.append(CppDataMemberInfo(
                 name, field.__cpp_reflex__(cpp_refl.OFFSET), field.__cpp_reflex__(cpp_refl.TYPE))
             )
+        elif type(field) == cpp_types.Function:
+            member_methods[name] = field.__cpp_reflex__(cpp_refl.RETURN_TYPE)
 
   # TODO: this refresh is needed b/c the scope type is registered as a
   # callable after the tracing started; no idea of the side-effects ...
@@ -400,6 +501,7 @@ class ImplClassType(CppClassNumbaType):
         class ImplClassModel(nb_dm.models.StructModel):
             def __init__(self, dmm, fe_type):
                 self._data_members = data_members
+                self._member_methods = member_methods
 
               # TODO: eventually we need not derive from StructModel
                 members = [(dmi.f_name, dmi.f_nbtype) for dmi in data_members]
@@ -471,6 +573,7 @@ class ImplClassModel(ImplClassValueModel):
 
         def init(self, dmm, fe_type, sz = cppyy.sizeof(val)):
             self._data_members = data_members
+            self._member_methods = member_methods
             self._sizeof = sz
 
           # TODO: this code exists purely to be able to use the indexing and hierarchy
@@ -498,27 +601,48 @@ def unbox_instance(typ, obj, c):
 
         return nb_ext.NativeValue(pobj, is_error=None, cleanup=None)
 
+    def make_implclass(context, builder, typ, **kwargs):
+        return nb_cgu.create_struct_proxy(typ)(context, builder, **kwargs)
+
   # C++ object to Python proxy wrapping for returns from Numba trace
     @nb_ext.box(ImplClassType)
     def box_instance(typ, val, c):
-        assert not "requires object model and passing of intact object, not memberwise copy"
+
         global cppyy_from_voidptr
 
-        ir_pyobj = c.context.get_argument_type(nb_types.pyobject)
-        ir_int   = cpp2ir('int')
+        implclass = make_implclass(c.context, c.builder, typ)
+        classobj = c.pyapi.unserialize(c.pyapi.serialize_object(cpp_types.Instance))
+        pyobj = c.context.get_argument_type(nb_types.pyobject)
 
-        ptrty = ir.PointerType(ir.FunctionType(ir_pyobj, [ir_voidptr, cpp2ir('char*'), ir_int]))
-        ptrval = c.context.add_dynamic_addr(c.builder, cppyy_from_voidptr, info='Instance_FromVoidPtr')
-        fp = c.builder.bitcast(ptrval, ptrty)
+        if type(val) == ir.Constant:
+            if val.constant == ir.Undefined:
+                assert not "Value passed to instance boxing is undefined"
 
-        module = c.builder.basic_block.function.module
-        clname = c.context.insert_const_string(module, typ._scope.__cpp_name__)
+        box_list = []
 
-        NULL = c.context.get_constant_null(nb_types.voidptr)     # TODO: get the real thing
-        return c.context.call_function_pointer(c.builder, fp, [NULL, clname, ir_int(0)])
+        model = implclass._datamodel
+        cfr = CppClassFieldResolver(c.context)
 
-    return cnt
+        for i in typ._scope.__dict__:
+            if isinstance(cfr.generic_resolve(typ, i), CppFunctionNumbaType):
+                ret_type = CppFunctionNumbaType
+                fnty = llvmfnty.function(ir_voidptr, [pyobj, pyobj, pyobj])
+                fn = c.pyapi._get_function(fnty, name=i)
+                box_list.append(c.builder.call(fn, [c.pyapi.get_null_object()]*3))
+            elif isinstance(cfr.generic_resolve(typ, i), nb_types.Type):
+                box_list.append(c.box(cfr.generic_resolve(typ, i), getattr(implclass, i)))
+
+        box_res = c.pyapi.call_function_objargs(
+            classobj, tuple(box_list)
+        )
+
+        # Required for nopython mode, numba nrt requres each member box call to decref since it steals the reference
+        for i in box_list:
+            c.pyapi.decref(i)
 
+        return box_res
+
+    return cnt
 
 #
 # C++ instance -> Numba
@@ -531,5 +655,5 @@ def typeof_instance(val, c):
         return scope_numbatypes[Qualified.default][type(val)]
     except KeyError:
         pass
-
-    return typeof_scope(type(val), c, Qualified.default)
+    # Pass the val itself to obtain Cling address of the CPPInstance for reference to C++ objects
+    return typeof_scope(val, c, Qualified.instance)