Fixing really tough bridging errors.

java/libpython_clj/jna/JVMBridge.java

@@ -9,8 +9,6 @@ public interface JVMBridge extends AutoCloseable
   public Pointer getAttr(String name);
   public void setAttr(String name, Pointer val);
   public String[] dir();
-  // Next has to have special treatment; in this case it is more manually wrapped.
-  public Object nextFn ();
   public Object interpreter();
   public Object wrappedObject();
   // Called from python when the python mirror is deleted.

src/libpython_clj/python.clj

@@ -45,6 +45,8 @@
                 ->py-long
                 ->py-string
                 ->python
+                ;;Used when you are returning a value from a function.
+                ->python-incref
                 ->jvm
                 make-tuple-fn
                 make-tuple-instance-fn
@@ -118,6 +120,7 @@
                 afn
                 as-jvm
                 as-python
+                as-python-incref ;; Used when returning a value from a function to python.
                 ->numpy
                 as-numpy)
 

src/libpython_clj/python/bridge.clj

@@ -34,7 +34,8 @@
               ->py-tuple
               ->py-dict
               ->py-string
-              ->py-fn]]
+              ->py-fn]
+             :as pyobj]
             [libpython-clj.python.interop :as pyinterop
              :refer
              [expose-bridge-to-python!
@@ -200,6 +201,24 @@
   (->jvm pyobj))
 
 
+(defn- mostly-copy-arg
+  "This is the dirty boundary between the languages.  Copying as often faster
+  for simple things but we have to be careful not to attempt a copy of things that
+  are only iterable (and not random access)."
+  [arg]
+  (cond
+    (jna/as-ptr arg)
+    (jna/as-ptr arg)
+    (instance? RandomAccess arg)
+    (->python arg)
+    (instance? Map arg)
+    (->python arg)
+    (instance? Iterable arg)
+    (as-python arg)
+    :else
+    (->python arg)))
+
+
 (defmacro bridge-pyobject
   [pyobj interpreter & body]
   `(let [pyobj# ~pyobj
@@ -260,12 +279,14 @@
              (py-proto/att-type-map pyobj#)))
          py-proto/PyCall
          (do-call-fn [callable# arglist# kw-arg-map#]
-           (-> (py-proto/do-call-fn pyobj# (mapv as-python arglist#)
-                                    (->> kw-arg-map#
-                                         (map (fn [[k# v#]]
-                                                [k# (as-python v#)]))
-                                         (into {})))
-               (as-jvm)))
+           (with-interpreter interpreter#
+             (let [arglist# (mapv mostly-copy-arg arglist#)
+                   kw-arg-map# (->> kw-arg-map#
+                                    (map (fn [[k# v#]]
+                                           [k# (mostly-copy-arg v#)]))
+                                    (into {}))]
+               (-> (py-proto/do-call-fn pyobj# arglist# kw-arg-map#)
+                   (as-jvm)))))
          Object
          (toString [this#]
            (->jvm (py-proto/call-attr pyobj# "__str__")))
@@ -507,7 +528,8 @@
           long-addr (get-attr ctypes "data")
           hash-ary {:ctypes-map ctypes}
           ptr-val (-> (Pointer. long-addr)
-                      (resource/track #(get hash-ary :ctypes-map) [:gc]))]
+                      (resource/track #(get hash-ary :ctypes-map)
+                                      pyobj/*pyobject-tracking-flags*))]
       {:ptr ptr-val
        :datatype np-dtype
        :shape shape
@@ -682,47 +704,43 @@
                 (with-out-str
                   (st/print-stack-trace e#)))))))
 
-(defmacro impl-tuple-function
-  [& body]
-  `(reify CFunction$TupleFunction
-     (pyinvoke [this# ~'self ~'args]
-       (wrap-jvm-context
-        (-> (let [~'args (as-jvm ~'args)]
-              ~@body)
-            ;;as-python can create a bridge object or just a ptr
-            as-python
-            ;;convert any bridge objects to actual jna ptrs to match
-            ;;interface definitions
-            jna/->ptr-backing-store)))))
-
-
-(defn jvm-fn->iface
-  [jvm-fn]
-  (impl-tuple-function
-   (apply jvm-fn args)))
-
-
-(defn as-py-fn
-  [jvm-fn]
-  (-> (jvm-fn->iface jvm-fn)
-      ->py-fn))
-
 
 (defn jvm-iterator-as-python
   ^Pointer [^Iterator item]
   (with-gil nil
     (let [next-fn #(if (.hasNext item)
                      (-> (.next item)
+                         ;;As python tracks the object in a jvm context
                          (as-python)
-                         (jna/->ptr-backing-store))
+                         (jna/->ptr-backing-store)
+                         ;;But we are handing the object back to python which is expecting
+                         ;;a new reference.
+                         (pyobj/incref))
                      (do
                        (libpy/PyErr_SetNone
                         (err/PyExc_StopIteration))
                        nil))
           att-map
-          {"__next__" (->py-fn next-fn)}]
-      (create-bridge-from-att-map item att-map
-                                  :next-fn next-fn))))
+          {"__next__" (pyobj/make-tuple-fn next-fn
+                                           :arg-converter nil
+                                           :result-converter nil)}]
+      (create-bridge-from-att-map item att-map))))
+
+
+(defn as-python-incref
+  "Convert to python and add a reference.  Necessary for return values from
+  functions as python expects a new reference and the as-python pathway
+  ensures the jvm garbage collector also sees the reference."
+  [item]
+  (-> (as-python item)
+      (pyobj/incref)))
+
+
+(defn- as-py-fn
+  ^Pointer [obj-fn]
+  (pyobj/make-tuple-fn obj-fn
+                       :arg-converter as-jvm
+                       :result-converter as-python-incref))
 
 
 (defn jvm-map-as-python
@@ -763,13 +781,13 @@
            "append" (as-py-fn #(.add jvm-data %))
            "insert" (as-py-fn #(.add jvm-data (int %1) %2))
            "pop" (as-py-fn (fn [& args]
-                                   (let [index (int (if (first args)
-                                                      (first args)
-                                                      -1))
-                                         index (if (< index 0)
-                                                 (- (.size jvm-data) index)
-                                                 index)]
-                                     #(.remove jvm-data index))))}]
+                             (let [index (int (if (first args)
+                                                (first args)
+                                                -1))
+                                   index (if (< index 0)
+                                           (- (.size jvm-data) index)
+                                           index)]
+                               #(.remove jvm-data index))))}]
       (create-bridge-from-att-map jvm-data att-map))))
 
 
@@ -869,7 +887,7 @@
                             initial-buffer
                             (->py-tuple shape)
                             (->py-tuple strides))]
-      (resource/track retval #(get buffer-desc :ptr) [:gc]))))
+      (resource/track retval #(get buffer-desc :ptr) pyobj/*pyobject-tracking-flags*))))
 
 
 (extend-type Object

src/libpython_clj/python/interop.clj

@@ -215,7 +215,6 @@
                     (reify CFunction$tp_new
                       (pyinvoke [this self varargs kw_args]
                         (let [retval (libpy/PyType_GenericNew self varargs kw_args)]
-                          (println retval)
                           retval)
                         )))
          module-name (get-attr module "__name__")
@@ -274,7 +273,7 @@
 (defn pybridge->bridge
   ^JVMBridge [^Pointer pybridge]
   (let [bridge-type (JVMBridgeType. pybridge)]
-    (get-jvm-bridge (.jvm_handle bridge-type)
+    (get-jvm-bridge (Pointer/nativeValue pybridge)
                     (.jvm_interpreter_handle bridge-type))))
 
 
@@ -311,7 +310,7 @@
                                                e
                                                (with-out-str
                                                  (st/print-stack-trace e))))))
-                            (unregister-bridge! bridge))
+                            (unregister-bridge! bridge self))
                           (catch Throwable e
                             (log-error e)))
                         (try
@@ -343,17 +342,20 @@
                             0))))
         :tp_iter (reify CFunction$NoArgFunction
                    (pyinvoke [this self]
-                     (let [attr
-                           (-> (pybridge->bridge self)
-                               (.getAttr "__iter__"))]
-                       (py-proto/call attr))))
-
+                     (if-let [attr (-> (pybridge->bridge self)
+                                         (.getAttr "__iter__"))]
+                       (libpy/PyObject_CallObject (jna/as-ptr attr) nil)
+                       (do
+                         (libpy/PyErr_SetNone (libpy/PyExc_Exception))
+                         nil))))
         :tp_iternext (reify CFunction$NoArgFunction
                        (pyinvoke [this self]
-                         (when-let [next
-                                    (-> (pybridge->bridge self)
-                                        (.nextFn))]
-                           (next))))}))))
+                         (if-let [next (-> (pybridge->bridge self)
+                                           (.getAttr "__next__"))]
+                           (libpy/PyObject_CallObject (jna/as-ptr next) nil)
+                           (do
+                             (libpy/PyErr_SetNone (libpy/PyExc_Exception))
+                             nil))))}))))
 
 
 (defn expose-bridge-to-python!
@@ -368,23 +370,17 @@
               (throw (ex-info "Failed to find bridge type" {})))
           bridge-type (PyTypeObject. bridge-type-ptr)
           ^Pointer new-py-obj (libpy/_PyObject_New bridge-type)
-          pybridge (JVMBridgeType. new-py-obj)
-          ]
-      (println (format "Creating bridge: 0x%x -> %s:%s"
-                       (Pointer/nativeValue new-py-obj)
-                       (get-object-handle (.interpreter bridge))
-                       (get-object-handle (.wrappedObject bridge))))
+          pybridge (JVMBridgeType. new-py-obj)]
       (set! (.jvm_interpreter_handle pybridge) (get-object-handle
                                                 (.interpreter bridge)))
       (set! (.jvm_handle pybridge) (get-object-handle (.wrappedObject bridge)))
       (.write pybridge)
-      (register-bridge! bridge pybridge)
-      (-> (.getPointer pybridge)
-          wrap-pyobject))))
+      (register-bridge! bridge new-py-obj)
+      (wrap-pyobject new-py-obj))))
 
 
 (defn create-bridge-from-att-map
-  [src-item att-map & {:keys [next-fn]}]
+  [src-item att-map]
   (with-gil
     (let [interpreter (ensure-bound-interpreter)
           dir-data (->> (keys att-map)
@@ -402,21 +398,11 @@
             (dir [bridge] dir-data)
             (interpreter [bridge] interpreter)
             (wrappedObject [bridge] src-item)
-            (nextFn [bridge] next-fn)
-            libpy-base/PToPyObjectPtr
-            (->py-object-ptr [item]
-              (with-gil
-                (if-let [existing-bridge (find-jvm-bridge-entry
-                                          (get-object-handle src-item)
-                                          (ensure-bound-interpreter))]
-                  (:pyobject existing-bridge)
-                  (expose-bridge-to-python! item))))
             py-proto/PCopyToJVM
             (->jvm [item options] src-item)
             py-proto/PBridgeToJVM
             (as-jvm [item options] src-item))]
-      (expose-bridge-to-python! bridge)
-      (libpy-base/->py-object-ptr bridge))))
+      (expose-bridge-to-python! bridge))))
 
 
 (defmethod py-proto/pyobject->jvm :jvm-bridge

src/libpython_clj/python/interpreter.clj

@@ -86,28 +86,24 @@
 
 
 (defn register-bridge!
-  [^JVMBridge bridge ^PyObject bridge-pyobject]
+  [^JVMBridge bridge ^Pointer bridge-pyobject]
   (let [interpreter (.interpreter bridge)
-        bridge-handle (get-object-handle (.wrappedObject bridge))]
+        bridge-handle (Pointer/nativeValue bridge-pyobject)]
     (when (contains? (get-in @(:interpreter-state* interpreter)
                            [:bridge-objects])
                      bridge-handle)
-      (throw (Exception. (format "Bridge already exists!! - %s" bridge-handle))))
+      (throw (Exception. (format "Bridge already exists!! - 0x%x" bridge-handle))))
     (swap! (:interpreter-state* interpreter) assoc-in
            [:bridge-objects bridge-handle]
            {:jvm-bridge bridge
-            :pyobject bridge-pyobject})
-    (println "ADDING-BRIDGE" bridge-handle
-             (keys (get-in @(:interpreter-state* interpreter)
-                           [:bridge-objects])))
+            :pyobject bridge-handle})
     :ok))
 
 
 (defn unregister-bridge!
-  [^JVMBridge bridge]
+  [^JVMBridge bridge ^Pointer bridge-pyobject]
   (let [interpreter (.interpreter bridge)
-        bridge-handle (get-object-handle (.wrappedObject bridge))]
-    (println "REMOVING-BRIDGE:" bridge-handle)
+        bridge-handle (Pointer/nativeValue bridge-pyobject)]
     (swap! (:interpreter-state* interpreter)
            update :bridge-objects dissoc bridge-handle)
     :ok))

src/libpython_clj/python/object.clj

@@ -95,7 +95,7 @@
     (py-proto/->jvm item options)))
 
 
-(def ^:dynamic *object-reference-logging* true)
+(def ^:dynamic *object-reference-logging* false)
 
 
 (def ^:dynamic *object-reference-tracker* nil)
@@ -427,14 +427,14 @@
           :or {method-name "unnamed_function"
                documentation "not documented"}}]
   (with-gil
-    (let [py-self (or py-self (->python {}))]
+    ;;This is a nice little tidbit, cfunction_new
+    ;;steals the reference.
+    (let [py-self (when py-self (incref (->python py-self)))]
       (-> (libpy/PyCFunction_New (method-def-data->method-def
                                   {:name method-name
                                    :doc documentation
                                    :function cfunc})
-                                 ;;This is a nice little tidbit, cfunction_new
-                                 ;;steals the reference.
-                                 (libpy/Py_IncRef py-self))
+                                 py-self)
           (wrap-pyobject)))))