From 2440b0a88fd1bc5ddd4811de370402d22fc1b593 Mon Sep 17 00:00:00 2001 From: Karl Nelson Date: Thu, 1 Aug 2019 08:54:41 -0700 Subject: [PATCH 1/2] Fix for caller sensitive methods required for Java 12 --- .travis.yml | 3 - appveyor.yml | 10 +- appveyor/install.sh | 20 +-- doc/CHANGELOG.rst | 14 +- doc/caller_sensitive.rst | 96 +++++++++++ doc/develguide.rst | 161 ++++++++++++++---- doc/install.rst | 3 - doc/userguide.rst | 20 --- jpype/_core.py | 14 +- jpype/_jclass.py | 18 +- jpype/imports.py | 2 +- native/common/include/jp_baseclasses.h | 2 +- native/common/include/jp_boxedclasses.h | 2 +- native/common/include/jp_class.h | 5 +- native/common/include/jp_field.h | 2 +- native/common/include/jp_javaframe.h | 2 +- native/common/include/jp_methodoverload.h | 3 +- native/common/include/jp_primitivetype.h | 7 +- native/common/include/jp_typemanager.h | 4 + native/common/jp_array.cpp | 2 +- native/common/jp_baseclasses.cpp | 4 +- native/common/jp_booleantype.cpp | 2 + native/common/jp_bytetype.cpp | 2 + native/common/jp_class.cpp | 15 +- native/common/jp_classloader.cpp | 2 + native/common/jp_doubletype.cpp | 2 + native/common/jp_floattype.cpp | 2 + native/common/jp_inttype.cpp | 2 + native/common/jp_longtype.cpp | 2 + native/common/jp_methodoverload.cpp | 59 ++++++- native/common/jp_primitivetype.cpp | 15 +- native/common/jp_proxy.cpp | 2 +- native/common/jp_shorttype.cpp | 2 + native/common/jp_stringclass.cpp | 2 +- native/common/jp_typemanager.cpp | 43 ++++- native/common/jp_value.cpp | 2 +- native/common/jp_voidtype.cpp | 2 + native/java/org/jpype/Utility.java | 91 +++++++++- native/python/include/jpype_memory_view.h | 2 +- native/python/jp_pythontypes.cpp | 6 +- native/python/pyjp_class.cpp | 2 +- native/python/pyjp_module.cpp | 2 +- native/python/pyjp_monitor.cpp | 2 +- native/python/pyjp_value.cpp | 12 +- project/jpype_cpython/nbproject/project.xml | 5 +- .../jpype_java/nbproject/project.properties | 12 ++ test/build.xml | 25 ++- test/harness/java9/jpype/method/Caller.java | 77 +++++++++ test/jpypetest/test_caller_sensitive.py | 108 ++++++++++++ test/jpypetest/test_reflect.py | 2 + 50 files changed, 742 insertions(+), 154 deletions(-) create mode 100644 doc/caller_sensitive.rst create mode 100644 test/harness/java9/jpype/method/Caller.java create mode 100644 test/jpypetest/test_caller_sensitive.py diff --git a/.travis.yml b/.travis.yml index 5495d4a2b..cd342a7ef 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,7 +32,6 @@ matrix: - python: nightly - python: pypy - python: pypy3 - - os: osx include: - name: "Python 2.7 on Xenial Linux (w/o NUMPY)" @@ -82,8 +81,6 @@ matrix: - PIP=pip3 - JPYPE_STR_CONVERSION=False - ## This pattern is good, but something is broken in openJDK 12 - ## Deal with in another pull - name: "Python 2.7 on macOS" os: osx osx_image: xcode10.2 diff --git a/appveyor.yml b/appveyor.yml index 948f7a2b7..35c5cebcc 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -42,6 +42,11 @@ environment: # ARCH: x86_64 # CYGSH: C:\Cygwin64\bin\bash -c + - PYTHON: "python3.7" + CYGWIN: "C:\\cygwin64" + ARCH: x86_64 + CYGSH: C:\Cygwin64\bin\bash -c + - PYTHON: "C:\\Miniconda" ARCH: x86 CONDA_PY: "2-latest" @@ -50,11 +55,6 @@ environment: ARCH: x86_64 CONDA_PY: "2-latest" - - PYTHON: "python3" - CYGWIN: "C:\\cygwin64" - ARCH: x86_64 - CYGSH: C:\Cygwin64\bin\bash -c - - PYTHON: "C:\\Miniconda3" ARCH: x86 CONDA_PY: "3-latest" diff --git a/appveyor/install.sh b/appveyor/install.sh index a3ecdf3f5..4e60ecb5d 100755 --- a/appveyor/install.sh +++ b/appveyor/install.sh @@ -14,35 +14,31 @@ if [ ! -d "$JAVA_HOME" ]; then exit -1 fi + # Make sure the jvm.dll is where it should be find "$JAVA_HOME" -name "jvm.dll" # Define programs SETUP=/setup-$ARCH -if [ $PYTHON = "python3" ]; then - PIP=pip3 - EASYINSTALL=easy_install-3.6 -else - PIP=pip - EASYINSTALL=easy_install-2.7 -fi # Install prereqs echo "==== update gcc" $SETUP -q -P gcc-core,gcc-g++,libcrypt-devel echo "==== update python" -$SETUP -q -P $PYTHON,$PYTHON-numpy,$PYTHON-devel,$PYTHON,$PYTHON-setuptools +if [ $PYTHON = "python3.7" ]; then + $SETUP -q -P python37,python37-numpy,python37-devel +fi echo "==== get modules" -$EASYINSTALL pip -$EASYINSTALL mock -#$PIP install mock +curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py +$PYTHON get-pip.py +$PYTHON -m pip install setuptools +$PYTHON -m pip install mock git clone --depth=1 https://github.com/pypa/setuptools.git cd setuptools $PYTHON ./bootstrap.py $PYTHON -m pip install ./ cd .. rm -r ./setuptools - git clone --depth=1 https://github.com/pypa/wheel.git git clone --depth=1 https://github.com/pypa/pip.git git clone --depth=1 https://github.com/pypa/setuptools_scm.git diff --git a/doc/CHANGELOG.rst b/doc/CHANGELOG.rst index 077c96a4a..781af0797 100644 --- a/doc/CHANGELOG.rst +++ b/doc/CHANGELOG.rst @@ -4,12 +4,22 @@ Changelog This changelog *only* contains changes from the *first* pypi release (0.5.4.3) onwards. - **Next version - unreleased** + + - Updated getJVMVersion to work with JDK 9+. + - Added support for pickling of Java objects using optional module ``jpype.pickle`` - - Fixed incorrect string conversion on exceptions. + - Fixed incorrect string conversion on exceptions. `str()` was + incorrectly returning `getMessage` rather than `toString`. + + - Fixed an issue with JDK 12 regarding calling methods with reflection. - - Corrected segfault when converting null elements while accessing a slice + - Removed limitations having to do with CallerSensitive methods. Methods + affected are listed in :doc:`caller-sensitive`. Caller sensitive + methods now receive an internal JPype class as the desut + + - Fixed segfault when converting null elements while accessing a slice from a Java object array. - PyJPMethod now supports the FunctionType API. diff --git a/doc/caller_sensitive.rst b/doc/caller_sensitive.rst new file mode 100644 index 000000000..5f3dae54c --- /dev/null +++ b/doc/caller_sensitive.rst @@ -0,0 +1,96 @@ +The following methods use the caller sensitive (as of JDK 12): + + - ``java.io.ObjectStreamClass.forClass`` + - ``java.io.ObjectStreamField.getType`` + - ``java.lang.Class.forName`` + - ``java.lang.Class.newInstance`` + - ``java.lang.Class.getClassLoader`` + - ``java.lang.Class.getEnclosingMethod`` + - ``java.lang.Class.getEnclosingConstructor`` + - ``java.lang.Class.getDeclaringClass`` + - ``java.lang.Class.getEnclosingClass`` + - ``java.lang.Class.getClasses`` + - ``java.lang.Class.getFields`` + - ``java.lang.Class.getMethods`` + - ``java.lang.Class.getConstructor`` + - ``java.lang.Class.getConstructors`` + - ``java.lang.Class.getField`` + - ``java.lang.Class.getMethod`` + - ``java.lang.Class.getDeclaredClasses`` + - ``java.lang.Class.getDeclaredField`` + - ``java.lang.Class.getDeclaredFields`` + - ``java.lang.Class.getDeclaredMethod`` + - ``java.lang.Class.getDeclaredMethods`` + - ``java.lang.Class.getDeclaredConstructor`` + - ``java.lang.Class.getDeclaredConstructors`` + - ``java.lang.Class.getResource`` + - ``java.lang.Class.getResourceAsStream`` + - ``java.lang.Class.getNestHost`` + - ``java.lang.Class.getNestMembers`` + - ``java.lang.ClassLoader.getParent`` + - ``java.lang.ClassLoader.getPlatformClassLoader`` + - ``java.lang.invoke,MethodHandleProxies.asInterfaceInstance`` + - ``java.lang.invoke.MethodHandles.lookup`` + - ``java.lang.Module.addReads`` + - ``java.lang.Module.addExports`` + - ``java.lang.Module.addOpens`` + - ``java.lang.Module.addUses`` + - ``java.lang.Module.getResourceAsStream`` + - ``java.lang.Package.getPackage`` + - ``java.lang.Package.getPackages`` + - ``java.lang.reflect.AccessibleObject.setAccessible`` + - ``java.lang.reflect.AccessibleObject.setAccessible`` + - ``java.lang.reflect.AccessibleObject.trySetAccessible`` + - ``java.lang.reflect.AccessibleObject.canAccess`` + - ``java.lang.reflect.Constructor.setAccessible`` + - ``java.lang.reflect.Constructor.newInstance`` + - ``java.lang.reflect.Field.setAccessible`` + - ``java.lang.reflect.Field.get`` + - ``java.lang.reflect.Field.getBoolean`` + - ``java.lang.reflect.Field.getByte`` + - ``java.lang.reflect.Field.getChar`` + - ``java.lang.reflect.Field.getShort`` + - ``java.lang.reflect.Field.getInt`` + - ``java.lang.reflect.Field.getLong`` + - ``java.lang.reflect.Field.getFloat`` + - ``java.lang.reflect.Field.getDouble`` + - ``java.lang.reflect.Field.set`` + - ``java.lang.reflect.Field.setBoolean`` + - ``java.lang.reflect.Field.setByte`` + - ``java.lang.reflect.Field.setChar`` + - ``java.lang.reflect.Field.setShort`` + - ``java.lang.reflect.Field.setInt`` + - ``java.lang.reflect.Field.setLong`` + - ``java.lang.reflect.Field.setFloat`` + - ``java.lang.reflect.Field.setDouble`` + - ``java.lang.reflect.Method.setAccessible`` + - ``java.lang.reflect.Method.invoke`` + - ``java.lang.reflect.Proxy.getProxyClass`` + - ``java.lang.reflect.Proxy.newProxyInstance`` + - ``java.lang.reflect.Proxy.getInvocationHandler`` + - ``java.lang.Runtime.load`` + - ``java.lang.Runtime.loadLibrary`` + - ``java.lang.StackWalker.walk`` + - ``java.lang.StackWalker.forEach`` + - ``java.lang.StackWalker.getCallerClass`` + - ``java.lang.System.getLogger`` + - ``java.lang.System.getLogger`` + - ``java.lang.System.load`` + - ``java.lang.System.loadLibrary`` + - ``java.lang.Thread.getContextClassLoader`` + - ``java.security.AccessController.doPrivileged`` + - ``java.security.AccessController.doPrivilegedWithCombiner`` + - ``java.util.concurrent.atomic.AtomicIntegerFieldUpdater.newUpdater`` + - ``java.util.concurrent.atomic.AtomicLongFieldUpdater.newUpdater`` + - ``java.util.concurrent.atomic.AtomicReferenceFieldUpdater.newUpdater`` + - ``java.util.ResourceBundle.getBundle`` + - ``java.util.ResourceBundle.clearCache`` + - ``java.util.ServiceLoader.load`` + - ``java.util.ServiceLoader.loadInstalled`` + - ``java.util.logging.Logger.getLogger`` + - ``java.util.logging.Logger.getLogger`` + - ``java.util.logging.Logger.getAnonymousLogger`` + - ``java.sql.DriverManager.getConnection`` + - ``java.sql.DriverManager.getDriver`` + - ``java.sql.DriverManager.deregisterDriver`` + - ``java.sql.DriverManager.getDrivers`` diff --git a/doc/develguide.rst b/doc/develguide.rst index ac8620345..2d839e8b1 100644 --- a/doc/develguide.rst +++ b/doc/develguide.rst @@ -542,6 +542,44 @@ passed as arguments operate like this, that means much of the API accepts bare ``PyObject*`` as arguments. It is the job of the caller to hold the reference for its scope. +On CPython extensions +~~~~~~~~~~~~~~~~~~~~~ + +CPython is somewhat of a nightmare to program in. It is not that they did not +try to document the API, but it is darn complex. The problems extend well +beyond the reference counting system that we have worked around. In +particular, the object model though well developed is very complex, often to +get it to work you must follow letter for letter the example on the CPython +user guide, and even then it may all go into the ditch. + +The key problem is that there are a lot of very bad examples of how to write +CPython extension modules out there. Often the these examples bypass the +appropriate macro and just call the field, or skip the virtual table and try to +call the Python method directly. It is true that these things do not break +there example, but they are conditioned on these methods they are calling +directly to be the right one for the job, but depends a lot on what the +behavior of the object is supposed to be. Get it wrong and you get really nasty +segfault. + +CPython itself may be partly responsible for some of these problems. They +generally seem to trust the user and thus don't verify if the call makes sense. +It is true that it will cost a little speed to be aggressive about checking the +type flags and the allocator match, but not checking when the error happens, +means that it fails far from the original problem source. I would hope that we +have moved beyond the philosophy that the user should just to whatever they +want so it runs as fast as possible, but that never appears to be the case. Of +course, I am just opining from the outside of the tent and I am sure the issues +are much more complicated it appears superficially. Then again if I can manage +to provide a safe workspace while juggling the issues of multiple virtual +machines, I am free to have opinions on the value of trading performance and +safety. + +In short when working on the extension code, make sure you do everything by the +book, and check that book twice. Always go through the types virtual table and +use the propery macros to access the resources. Miss one line in some complex +pattern even once and you are in for a world of hurt. There are very few guard +rails in the CPython code. + C++ JNI layer ------------- @@ -636,17 +674,15 @@ you always store only the global reference. } } -But don't mistake this as an invitation to make global references -everywhere. Global reference are global, thus will hold the member -until the reference is destroyed. C++ exceptions can lead -to missing the unreference, thus global references should only -happen when you are placing the Java object into a class member -variable or a global variable. +But don't mistake this as an invitation to make global references everywhere. +Global reference are global, thus will hold the member until the reference is +destroyed. C++ exceptions can lead to missing the unreference, thus global +references should only happen when you are placing the Java object into a class +member variable or a global variable. -To help manage global references, we have ``JPRef<>`` which -holds a global reference for the duration of the C++ lifespace. -This is the base class for each of the global reference types -we use. +To help manage global references, we have ``JPRef<>`` which holds a global +reference for the duration of the C++ lifespace. This is the base class for +each of the global reference types we use. .. code-block:: cpp @@ -672,19 +708,19 @@ than creating a new one. return obj; } -Although the system we have set up is "safe by design", there are things that can -go wrong is misused. If the caller fails to create a frame prior to calling a -function that returns a local reference, the reference will go into the program -scoped local references and thus leak. Thus, it is usually best to force the user -to make a scope with the frame extension pattern. Second, if any JNI references -that are not kept or converted to global, it becomes invalid. Further, since JNI -recycles the reference pointer fairly quickly, it most likely will be pointed to -another object whose type may not be expected. Thus, best case is using the stale -reference will crash and burn. Worse case, the reference will be a live reference -to another object and it will produce an error which seems completely irrelevant -to anything that was being called. Horrible case, the live object does not object -to bad call and it all silently proceeds down the road another two miles before -coming to flaming death. +Although the system we have set up is "safe by design", there are things that +can go wrong is misused. If the caller fails to create a frame prior to +calling a function that returns a local reference, the reference will go into +the program scoped local references and thus leak. Thus, it is usually best to +force the user to make a scope with the frame extension pattern. Second, if any +JNI references that are not kept or converted to global, it becomes invalid. +Further, since JNI recycles the reference pointer fairly quickly, it most +likely will be pointed to another object whose type may not be expected. Thus, +best case is using the stale reference will crash and burn. Worse case, the +reference will be a live reference to another object and it will produce an +error which seems completely irrelevant to anything that was being called. +Horrible case, the live object does not object to bad call and it all silently +proceeds down the road another two miles before coming to flaming death. Moral of the story, always create a local frame even if you are handling a global reference. If passed or returned a reference of any kind, it is a borrowed reference @@ -852,8 +888,8 @@ which loads a class from the internal jar. Java concept of UTF is pretty much out of sync with the rest of the world. Java used 16 bits for its native characters. But this was inadequate for all of the -unicode characters, thus longer unicode character had to be encoded in the -16 bit space. Rather the directly providing methods to convert to a standard +unicode characters, thus longer unicode character had to be encoded in the 16 +bit space. Rather the directly providing methods to convert to a standard encoding such as UTF8, Java used UTF16 encoded in 8 bits which they dub Modified-UTF8. ``JPEncoding`` deals with converting this unusual encoding into something that Python can understand. @@ -878,13 +914,13 @@ converter only appears on ``java.io.DataInput`` and ``java.io.DataOutput``. Java native code ---------------- -At the lowest level of the onion is the native Java layer. Although this -layer is most remote from Python, ironically it is the easiest layer to communicate +At the lowest level of the onion is the native Java layer. Although this layer +is most remote from Python, ironically it is the easiest layer to communicate with. As the point of jpype is to communicate with Java, it is possible to -directly communicate with the jpype Java internals. These can be imported -from the package ``org.jpype``. The code for the Java layer is located in -``native/java``. It is compiled into a jar in the build directory and then converted -to a C++ header to be compiled into the ``_jpype`` module. +directly communicate with the jpype Java internals. These can be imported from +the package ``org.jpype``. The code for the Java layer is located in +``native/java``. It is compiled into a jar in the build directory and then +converted to a C++ header to be compiled into the ``_jpype`` module. The Java layer currently houses the reference queue, a classloader which can load a Java class from a bytestream source, the proxy code for implementing @@ -910,6 +946,69 @@ the fault to specific location where it failed. To use the logger in a function start the ``JP_TRACE_IN(function_name)`` which will open a ``try catch`` block. +The JPype tracer can be augmented with the Python tracing module to give +a very good picture of both JPype and Python states at the time of the crash. +To use the Python tracing, start Python with... :: + + python -m trace --trace myscript.py + + +Debugging issues +---------------- + +If the tracing function proves inadequate to identify a problem, we often need +to turn to a general purpose tool like gdb or valgrind. The JPype core is not +easy to debug. Python can be difficult to properly monitor especially with +tools like valgrind due to its memory handling. Java is also challenging to +debug. Put them together and you have the mother of all debugging issues. There +are a number of complicating factors. Let us start with how to debug with gdb. + +Gdb runs into two major issues, both tied to the signal handler. +First, Java installs its own signal handlers that take over the entire process +when a segfault occurs. This tends to cause very poor segfault stacktraces +when examining a core file, which often is corrupt after the first user frame. +Second, Java installs its signal handlers in such as way that attempting to run +under a debugger like gdb will often immediately crash preventing one from +catching the segfault before Java catches it. This makes for a catch 22, +you can't capture a meaningful non-interactively produced core file, and you +can't get an interactive session to work. + +Fortunately there are solutions to the interactive session issue. By disabling +the SIGSEGV handler, we can get past the initial failure and also we can catch +the stack before it is altered by the JVM. :: + + gdb -ex 'handle SIGSEGV nostop noprint pass' python + +Thus far I have not found any good solutions to prevent the JVM from altering +the stack frames when dumping the core. Thus interactive debugging appears +to be the best option. + +There are additional issues that one should be aware of. Open-JDK 1.8 has had a +number of problems with the debugger. Starting JPype under gdb may trigger, may +trigger the following error. :: + + gdb.error: No type named nmethod. + +There are supposed to be fixes for this problem, but none worked for me. +Upgrading to Open-JDK 9 appears to fix the problem. + +Another complexity with debugging memory problems is that Python tends to +hide the problem with its allocation pools. Rather than allocating memory +when a new object is request, it will often recycle and existing object +which was collect earlier. The result is that an object which turns out is +still live becomes recycled as a new object with a new type. Thus suddenly +a method which was expected to produce some result instead vectors into +the new type table, which may or may not send us into segfault land +depending on whether the old and new objects have similar memory layouts. + +This can be partially overcome by forcing Python to use a different memory +allocation scheme. This can avoid the recycling which means we are more likely +to catch the error, but at the same time means we will be excuting different +code paths so we may not reach a similar state. If the core dump is vectoring +off into code that just does not make sense it is likely caused by the memory +pools. Starting Python 3, it is possible to select the memory allocation policy +through an enviroment variable. See the ``PYTHONMALLOC`` setting for details. + Future directions ----------------- diff --git a/doc/install.rst b/doc/install.rst index e2a396b3e..f3fc7c7ee 100644 --- a/doc/install.rst +++ b/doc/install.rst @@ -192,7 +192,4 @@ Known Bugs/Limitations restart it. - Structural issues prevent managing objects from more than one JVM at a time. -- Some methods rely on the "current" class/caller. Since calls coming - directly from python code do not have a current class, these methods - do not work. The :doc:`userguide` lists all the known methods like that. - Mixing 64 bit Python with 32 bit Java and vice versa crashes on import jpype. diff --git a/doc/userguide.rst b/doc/userguide.rst index 5cd91b0f9..a9ad6e2da 100644 --- a/doc/userguide.rst +++ b/doc/userguide.rst @@ -645,26 +645,6 @@ advantage of it. As the time of writing, the latest stable Sun JVM was 1.4.2_04. -Methods dependent on "current" class -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -There are a few methods in the Java libraries that rely on finding -information on the calling class. So these methods, if called directly from -Python code, will fail because there is no calling Java class, and the JNI -API does not provide methods to simulate one. - -At the moment, the methods known to fail are : - - -java.sql.DriverManager.getConnection(...) -::::::::::::::::::::::::::::::::::::::::: - -For some reason, this class verifies that the driver class as loaded in the -"current" classloader is the same as previously registered. Since there is no -"current" classloader, it defaults to the internal classloader, which -typically does not find the driver. To remedy, simply instantiate the driver -yourself and call its ``connect(...)`` method. - Unsupported Python versions ~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/jpype/_core.py b/jpype/_core.py index b0ac189e6..36006b451 100644 --- a/jpype/_core.py +++ b/jpype/_core.py @@ -362,10 +362,14 @@ def getJVMVersion(): """ if not _jpype.isStarted(): return (0, 0, 0) - version = str(_jclass.JClass( - 'java.lang.Runtime').class_.getPackage().getImplementationVersion()) - if version.find('_') != -1: - parts = version.split('_') - version = parts[0] + + import re + runtime = _jclass.JClass('java.lang.Runtime') + version = runtime.class_.getPackage().getImplementationVersion() + + # Java 9+ has a version method + if not version: + version = runtime.version() + version = (re.match("([0-9.]+)", str(version)).group(1)) return tuple([int(i) for i in version.split('.')]) diff --git a/jpype/_jclass.py b/jpype/_jclass.py index 0e0971c07..94759d9ef 100644 --- a/jpype/_jclass.py +++ b/jpype/_jclass.py @@ -28,7 +28,6 @@ _JObject = None -_java_lang_throwable = None _java_lang_RuntimeException = None _java_ClassLoader = None _java_lang_Class = None @@ -50,9 +49,10 @@ def _initialize(): _java_lang_Class = JClass("java.lang.Class") _java_ClassLoader = JClass('java.lang.ClassLoader').getSystemClassLoader() + if not _java_ClassLoader: + raise RuntimeError("Unable to load Java SystemClassLoader") - global _java_lang_throwable, _java_lang_RuntimeException - _java_lang_throwable = JClass("java.lang.Throwable") + global _java_lang_RuntimeException _java_lang_RuntimeException = JClass("java.lang.RuntimeException") # Preload needed classes @@ -419,18 +419,6 @@ def _getDefaultJavaObject(obj): raise TypeError( "Unable to determine the default type of `{0}`".format(obj.__class__)) -# Patch for forName - - -@_jcustomizer.JImplementationFor('java.lang.Class') -class _JavaLangClass(object): - @classmethod - def forName(cls, *args): - if len(args) == 1 and isinstance(args[0], str): - return cls._forName(args[0], True, _java_ClassLoader) - else: - return cls._forName(*args) - def typeLookup(tp, name): """Fetch a descriptor from the inheritance tree. diff --git a/jpype/imports.py b/jpype/imports.py index 25974877c..efd8ef868 100644 --- a/jpype/imports.py +++ b/jpype/imports.py @@ -130,7 +130,7 @@ def _getJavaClass(javaname): # Otherwise!? except Exception as ex: - err = "Unable to import '%s' due to unexpected exception" + err = "Unable to import '%s' due to unexpected exception, '%s'"%(javaname, ex) raise ImportError(err) # FIXME imports of static fields not working for now. diff --git a/native/common/include/jp_baseclasses.h b/native/common/include/jp_baseclasses.h index 4ac046acc..867fccddd 100644 --- a/native/common/include/jp_baseclasses.h +++ b/native/common/include/jp_baseclasses.h @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright 2004 Steve M�nard + Copyright 2004 Steve Ménard Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/native/common/include/jp_boxedclasses.h b/native/common/include/jp_boxedclasses.h index 967a698bc..4810ed896 100644 --- a/native/common/include/jp_boxedclasses.h +++ b/native/common/include/jp_boxedclasses.h @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright 2004 Steve M�nard + Copyright 2004 Steve Ménard Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/native/common/include/jp_class.h b/native/common/include/jp_class.h index 8f3da8051..9e78dda6f 100644 --- a/native/common/include/jp_class.h +++ b/native/common/include/jp_class.h @@ -34,13 +34,14 @@ class JPClass { return m_Class.get(); } - + string toString() const; string getCanonicalName() const; bool isAbstract(); bool isFinal(); bool isThrowable(); bool isInterface(); + virtual bool isPrimitive() const; const MethodList& getMethods(); const FieldList& getFields(); @@ -143,7 +144,7 @@ class JPClass // Check if a value is an instance of this class bool isInstance(JPValue& val); - + virtual void postLoad(); private: void loadFields(); diff --git a/native/common/include/jp_field.h b/native/common/include/jp_field.h index 9a82c80a4..bb4d3b481 100644 --- a/native/common/include/jp_field.h +++ b/native/common/include/jp_field.h @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright 2004 Steve M�nard + Copyright 2004 Steve Ménard Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/native/common/include/jp_javaframe.h b/native/common/include/jp_javaframe.h index c15f6bbe7..9f8233407 100644 --- a/native/common/include/jp_javaframe.h +++ b/native/common/include/jp_javaframe.h @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright 2004 Steve Ménard + Copyright 2004 Steve Ménard Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/native/common/include/jp_methodoverload.h b/native/common/include/jp_methodoverload.h index 9f458efde..d21d7c2ba 100644 --- a/native/common/include/jp_methodoverload.h +++ b/native/common/include/jp_methodoverload.h @@ -37,7 +37,7 @@ class JPMatch JPMethodOverload* overload; char offset; char skip; - + JPMatch() { type = JPMatch::_none; @@ -153,6 +153,7 @@ class JPMethodOverload OverloadList m_MoreSpecificOverloads; bool m_Ordered; bool m_IsAbstract; + bool m_CallerSensitive; } ; #endif // _JPMETHODOVERLOAD_H_ diff --git a/native/common/include/jp_primitivetype.h b/native/common/include/jp_primitivetype.h index 027abf6e8..f1f4da73f 100644 --- a/native/common/include/jp_primitivetype.h +++ b/native/common/include/jp_primitivetype.h @@ -23,10 +23,8 @@ class JPPrimitiveType : public JPClass JPPrimitiveType(JPBoxedClass* cls); virtual ~JPPrimitiveType(); -protected: - JPClass* m_BoxedClass; - public: + virtual bool isPrimitive() const override; JPClass* getBoxedClass() const { @@ -35,6 +33,9 @@ class JPPrimitiveType : public JPClass void setBoxedClass(JPBoxedClass* boxedClass); +protected: + JPClass* m_BoxedClass; + } ; #endif diff --git a/native/common/include/jp_typemanager.h b/native/common/include/jp_typemanager.h index 4ade9b7dc..3d70d0f95 100644 --- a/native/common/include/jp_typemanager.h +++ b/native/common/include/jp_typemanager.h @@ -77,6 +77,10 @@ namespace JPTypeManager JPClass* registerClass(JPClass* classWrapper); jclass getClassFor(jobject obj); + + // Support for caller sensitive methods + bool isCallerSensitive(jobject obj); + jobject callMethod(jobject method, jobject obj, jobject args); } // namespace diff --git a/native/common/jp_array.cpp b/native/common/jp_array.cpp index 7d8910ee8..c5634fd7c 100644 --- a/native/common/jp_array.cpp +++ b/native/common/jp_array.cpp @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright 2004 Steve Ménard + Copyright 2004 Steve Ménard Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/native/common/jp_baseclasses.cpp b/native/common/jp_baseclasses.cpp index e69299452..9c106e2a0 100644 --- a/native/common/jp_baseclasses.cpp +++ b/native/common/jp_baseclasses.cpp @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright 2004 Steve M�nard + Copyright 2004 Steve Ménard Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -104,7 +104,7 @@ jvalue JPObjectBaseClass::convertToJava(PyObject* pyobj) JPValue *value = JPPythonEnv::getJavaValue(pyobj); if (value != NULL) { - if (dynamic_cast (value->getClass()) == NULL) + if (!(value->getClass())->isPrimitive()) { res.l = frame.NewLocalRef(value->getJavaObject()); res.l = frame.keep(res.l); diff --git a/native/common/jp_booleantype.cpp b/native/common/jp_booleantype.cpp index 63208f75c..030da2abb 100644 --- a/native/common/jp_booleantype.cpp +++ b/native/common/jp_booleantype.cpp @@ -93,6 +93,7 @@ JPMatch::Type JPBooleanType::canConvertToJava(PyObject* obj) jvalue JPBooleanType::convertToJava(PyObject* obj) { + JP_TRACE_IN("JPBooleanType::convertToJava"); jvalue res; JPValue* value = JPPythonEnv::getJavaValue(obj); @@ -112,6 +113,7 @@ jvalue JPBooleanType::convertToJava(PyObject* obj) res.z = (jboolean) JPPyLong::asLong(obj) != 0; } return res; + JP_TRACE_OUT; } jarray JPBooleanType::newArrayInstance(JPJavaFrame& frame, jsize sz) diff --git a/native/common/jp_bytetype.cpp b/native/common/jp_bytetype.cpp index 24c788060..52a1849bd 100644 --- a/native/common/jp_bytetype.cpp +++ b/native/common/jp_bytetype.cpp @@ -109,6 +109,7 @@ JPMatch::Type JPByteType::canConvertToJava(PyObject* obj) jvalue JPByteType::convertToJava(PyObject* obj) { + JP_TRACE_IN("JPByteType::convertToJava"); jvalue res; field(res) = 0; JPValue* value = JPPythonEnv::getJavaValue(obj); @@ -132,6 +133,7 @@ jvalue JPByteType::convertToJava(PyObject* obj) JP_RAISE_TYPE_ERROR("Cannot convert value to Java byte"); return res; + JP_TRACE_OUT; } jarray JPByteType::newArrayInstance(JPJavaFrame& frame, jsize sz) diff --git a/native/common/jp_class.cpp b/native/common/jp_class.cpp index 6dd2604c4..204329175 100644 --- a/native/common/jp_class.cpp +++ b/native/common/jp_class.cpp @@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - + *****************************************************************************/ #include @@ -68,6 +68,11 @@ jarray JPClass::newArrayInstance(JPJavaFrame& frame, jsize sz) // // +bool JPClass::isPrimitive() const +{ + return false; +} + string JPClass::toString() const { return JPJni::toString(m_Class.get()); @@ -364,6 +369,7 @@ jvalue JPClass::convertToJava(PyObject* obj) // assume it is convertible; if (JPPyObject::isNone(obj)) { + JP_TRACE("Null"); res.l = NULL; return res; } @@ -371,6 +377,7 @@ jvalue JPClass::convertToJava(PyObject* obj) JPValue* value = JPPythonEnv::getJavaValue(obj); if (value != NULL) { + JP_TRACE("Value"); res.l = frame.NewLocalRef(value->getJavaObject()); res.l = frame.keep(res.l); return res; @@ -379,10 +386,12 @@ jvalue JPClass::convertToJava(PyObject* obj) JPProxy* proxy = JPPythonEnv::getJavaProxy(obj); if (proxy != NULL) { + JP_TRACE("Proxy"); res.l = frame.keep(proxy->getProxy()); return res; } + JP_TRACE("Miss"); return res; JP_TRACE_OUT; } @@ -607,11 +616,11 @@ string JPClass::describe() bool JPClass::isInstance(JPValue& val) { JPClass* cls = val.getClass(); - if (dynamic_cast (cls) == cls) + if (cls->isPrimitive()) return false; JPJavaFrame frame; - return frame.IsInstanceOf(val.getValue().l, m_Class.get()); + return frame.IsInstanceOf(val.getValue().l, m_Class.get()) != 0; } diff --git a/native/common/jp_classloader.cpp b/native/common/jp_classloader.cpp index c3fbf44e5..14c1f34d0 100644 --- a/native/common/jp_classloader.cpp +++ b/native/common/jp_classloader.cpp @@ -62,8 +62,10 @@ void JPClassLoader::init() jclass JPClassLoader::findClass(string name) { + JP_TRACE_IN("JPClassLoader::findClass"); JPJavaFrame frame; jvalue v; v.l = frame.NewStringUTF(name.c_str()); return (jclass) frame.keep(frame.CallObjectMethodA(classLoader, findClassID, &v)); + JP_TRACE_OUT; } diff --git a/native/common/jp_doubletype.cpp b/native/common/jp_doubletype.cpp index 649634c7f..1ce030fcf 100644 --- a/native/common/jp_doubletype.cpp +++ b/native/common/jp_doubletype.cpp @@ -83,6 +83,7 @@ JPMatch::Type JPDoubleType::canConvertToJava(PyObject* obj) jvalue JPDoubleType::convertToJava(PyObject* obj) { + JP_TRACE_IN("JPDoubleType::convertToJava"); jvalue res; field(res) = 0; JPValue* value = JPPythonEnv::getJavaValue(obj); @@ -116,6 +117,7 @@ jvalue JPDoubleType::convertToJava(PyObject* obj) JP_RAISE_TYPE_ERROR("Cannot convert value to Java double"); return res; + JP_TRACE_OUT; } jarray JPDoubleType::newArrayInstance(JPJavaFrame& frame, jsize sz) diff --git a/native/common/jp_floattype.cpp b/native/common/jp_floattype.cpp index cdd7f5c72..7281f1cc7 100644 --- a/native/common/jp_floattype.cpp +++ b/native/common/jp_floattype.cpp @@ -84,6 +84,7 @@ JPMatch::Type JPFloatType::canConvertToJava(PyObject* obj) jvalue JPFloatType::convertToJava(PyObject* obj) { + JP_TRACE_IN("JPFloatType::convertToJava"); jvalue res; JPValue* value = JPPythonEnv::getJavaValue(obj); if (value != NULL) @@ -127,6 +128,7 @@ jvalue JPFloatType::convertToJava(PyObject* obj) JP_RAISE_TYPE_ERROR("Cannot convert value to Java float"); return res; + JP_TRACE_OUT; } jarray JPFloatType::newArrayInstance(JPJavaFrame& frame, jsize sz) diff --git a/native/common/jp_inttype.cpp b/native/common/jp_inttype.cpp index 2a8a901db..bb0077f1e 100644 --- a/native/common/jp_inttype.cpp +++ b/native/common/jp_inttype.cpp @@ -89,6 +89,7 @@ JPMatch::Type JPIntType::canConvertToJava(PyObject* obj) jvalue JPIntType::convertToJava(PyObject* obj) { + JP_TRACE_IN("JPIntType::convertToJava"); jvalue res; field(res) = 0; JPValue* value = JPPythonEnv::getJavaValue(obj); @@ -112,6 +113,7 @@ jvalue JPIntType::convertToJava(PyObject* obj) JP_RAISE_TYPE_ERROR("Cannot convert value to Java int"); return res; // never reached + JP_TRACE_OUT; } jarray JPIntType::newArrayInstance(JPJavaFrame& frame, jsize sz) diff --git a/native/common/jp_longtype.cpp b/native/common/jp_longtype.cpp index 9ec78f5b4..e4191ba9e 100644 --- a/native/common/jp_longtype.cpp +++ b/native/common/jp_longtype.cpp @@ -88,6 +88,7 @@ JPMatch::Type JPLongType::canConvertToJava(PyObject* obj) jvalue JPLongType::convertToJava(PyObject* obj) { + JP_TRACE_IN("JPLongType::convertToJava"); jvalue res; field(res) = 0; JPValue* value = JPPythonEnv::getJavaValue(obj); @@ -111,6 +112,7 @@ jvalue JPLongType::convertToJava(PyObject* obj) JP_RAISE_TYPE_ERROR("Cannot convert value to Java long"); return res; // never reached + JP_TRACE_OUT; } jarray JPLongType::newArrayInstance(JPJavaFrame& frame, jsize sz) diff --git a/native/common/jp_methodoverload.cpp b/native/common/jp_methodoverload.cpp index c4af487c4..ccd3784fa 100644 --- a/native/common/jp_methodoverload.cpp +++ b/native/common/jp_methodoverload.cpp @@ -1,5 +1,5 @@ /***************************************************************************** - Copyright 2004 Steve M�nard + Copyright 2004 Steve Ménard Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -46,6 +46,9 @@ JPMethodOverload::JPMethodOverload(JPClass* claz, jobject mth) : m_Method(mth) { m_Arguments.insert(m_Arguments.begin(), 1, claz->getJavaClass()); } + + m_CallerSensitive = JPTypeManager::isCallerSensitive(m_Method.get()); + } JPMethodOverload::~JPMethodOverload() @@ -173,7 +176,6 @@ JPMatch JPMethodOverload::matches(bool callInstance, JPPyObjectVector& arg) JP_TRACE("Match vargs direct", lastMatch); } } - else if (len > tlen) { // Must match the array type @@ -250,20 +252,67 @@ JPPyObject JPMethodOverload::invoke(JPMatch& match, JPPyObjectVector& arg, bool ensureTypeCache(); size_t alen = m_Arguments.size(); JPJavaFrame frame(8 + alen); - JPClass* retType = m_ReturnTypeCache; // Pack the arguments vector v(alen + 1); packArgs(match, v, arg); + // Check if it is caller sensitive + if (m_CallerSensitive) + { + JP_TRACE("Caller sensitive method"); + //public static Object callMethod(Method method, Object obj, Object[] args) + jobject self = NULL; + size_t len = alen; + if (!m_IsStatic) + { + JP_TRACE("Call instance"); + len--; + JPValue *selfObj = JPPythonEnv::getJavaValue(arg[0]); + self = selfObj->getJavaObject(); + } + + // Convert arguments + jobjectArray ja = frame.NewObjectArray(len, JPTypeManager::_java_lang_Object->getJavaClass(), NULL); + for (jsize i = 0; i < (jsize) len; ++i) + { + JPClass *cls = m_ArgumentsTypeCache[i + match.skip - match.offset]; + if (cls->isPrimitive()) + { + JPPrimitiveType* type = (JPPrimitiveType*) cls; + frame.SetObjectArrayElement(ja, i, type->getBoxedClass()->convertToJava(arg[i + match.skip]).l); + } else + { + frame.SetObjectArrayElement(ja, i, v[i].l); + } + } + + // Call the method + jobject o = JPTypeManager::callMethod(m_Method.get(), self, ja); + + // Deal with the return + if (retType->isPrimitive()) + { + JP_TRACE("Return primitive"); + JPValue out = retType->getValueFromObject(o); + return retType->convertToPythonObject(out.getValue()); + } + else + { + JP_TRACE("Return object"); + jvalue v; + v.l = o; + return retType->convertToPythonObject(v); + } + } + // Invoke the method (arg[0] = this) if (m_IsStatic) { jclass claz = m_Class->getJavaClass(); return retType->invokeStatic(frame, claz, m_MethodID, &v[0]); - } - else + } else { JPValue* selfObj = JPPythonEnv::getJavaValue(arg[0]); jobject c; diff --git a/native/common/jp_primitivetype.cpp b/native/common/jp_primitivetype.cpp index fe5c90e4b..d31e2209c 100644 --- a/native/common/jp_primitivetype.cpp +++ b/native/common/jp_primitivetype.cpp @@ -26,14 +26,7 @@ JPPrimitiveType::~JPPrimitiveType() { } -//jobject JPPrimitiveType::convertToJavaObject(PyObject obj) -//{ -// JPJavaFrame frame; -// -// JPPyTuple tuple(JPPyTuple::newTuple(1)); -// tuple.setItem(0, obj); -// -// JPValue o = m_BoxedClass->newInstance(tuple); -// jobject res = o.getJavaObject(); -// return frame.keep(res); -//} +bool JPPrimitiveType::isPrimitive() const +{ + return true; +} diff --git a/native/common/jp_proxy.cpp b/native/common/jp_proxy.cpp index 152c27000..6a898bb33 100644 --- a/native/common/jp_proxy.cpp +++ b/native/common/jp_proxy.cpp @@ -91,7 +91,7 @@ JNIEXPORT jobject JNICALL Java_jpype_JPypeInvocationHandler_hostInvoke( } // We must box here. - if (dynamic_cast (returnClass) == returnClass) + if (returnClass->isPrimitive()) { returnClass = ((JPPrimitiveType*) returnClass)->getBoxedClass(); } diff --git a/native/common/jp_shorttype.cpp b/native/common/jp_shorttype.cpp index ee64e82f1..0d6514efd 100644 --- a/native/common/jp_shorttype.cpp +++ b/native/common/jp_shorttype.cpp @@ -91,6 +91,7 @@ JPMatch::Type JPShortType::canConvertToJava(PyObject* obj) jvalue JPShortType::convertToJava(PyObject* obj) { + JP_TRACE_IN("JPShortType::convertToJava"); jvalue res; field(res) = 0; JPValue* value = JPPythonEnv::getJavaValue(obj); @@ -114,6 +115,7 @@ jvalue JPShortType::convertToJava(PyObject* obj) JP_RAISE_TYPE_ERROR("Cannot convert value to Java short"); return res; // never reached + JP_TRACE_OUT; } jarray JPShortType::newArrayInstance(JPJavaFrame& frame, jsize sz) diff --git a/native/common/jp_stringclass.cpp b/native/common/jp_stringclass.cpp index d986bc11c..fd6dd2462 100644 --- a/native/common/jp_stringclass.cpp +++ b/native/common/jp_stringclass.cpp @@ -26,7 +26,7 @@ JPStringClass::~JPStringClass() JPPyObject JPStringClass::convertToPythonObject(jvalue val) { - JP_TRACE_IN("JPStringType::asHostObject"); + JP_TRACE_IN("JPStringType::convertToPythonObject"); if (val.l == NULL) { diff --git a/native/common/jp_typemanager.cpp b/native/common/jp_typemanager.cpp index 0b843d290..6d700b0b3 100644 --- a/native/common/jp_typemanager.cpp +++ b/native/common/jp_typemanager.cpp @@ -28,6 +28,8 @@ namespace JavaClassMap javaClassMap; jclass utility; jmethodID getClassForID; + jmethodID callMethodID; + jmethodID isCallerSensitiveID; // TypeMap typeMap; // JavaClassMap javaClassMap; @@ -93,15 +95,39 @@ JPClass* registerObjectClass(string name, jclass jc) JP_TRACE_OUT; } - jclass JPTypeManager::getClassFor(jobject obj) { + if (getClassForID == 0) + return NULL; JPJavaFrame frame; jvalue v; v.l = obj; return (jclass) frame.keep(frame.CallStaticObjectMethodA(utility, getClassForID, &v)); } +bool JPTypeManager::isCallerSensitive(jobject obj) +{ + if (isCallerSensitiveID == 0) + return false; + JPJavaFrame frame; + jvalue v; + v.l = obj; + return frame.CallStaticBooleanMethodA(utility, isCallerSensitiveID, &v) != 0; +} + +jobject JPTypeManager::callMethod(jobject method, jobject obj, jobject args) +{ + JP_TRACE_IN("JPTypeManager::callMethod"); + if (callMethodID == 0) + return NULL; + JPJavaFrame frame; + jvalue v[3]; + v[0].l = method; + v[1].l = obj; + v[2].l = args; + return frame.keep(frame.CallStaticObjectMethodA(utility, callMethodID, v)); + JP_TRACE_OUT; +} void JPTypeManager::init() { @@ -109,8 +135,18 @@ void JPTypeManager::init() JPJavaFrame frame; JP_TRACE_IN("JPTypeManager::init"); + // Get utility class utility = (jclass) frame.NewGlobalRef(JPClassLoader::findClass("org.jpype.Utility")); - getClassForID = frame.GetStaticMethodID(utility, "getClassFor", "(Ljava/lang/Object;)Ljava/lang/Class;"); + + // Get support methods + callMethodID = frame.GetStaticMethodID(utility, "callMethod", + "(Ljava/lang/reflect/Method;Ljava/lang/Object;[Ljava/lang/Object;)Ljava/lang/Object;"); + + isCallerSensitiveID = frame.GetStaticMethodID(utility, "isCallerSensitive", + "(Ljava/lang/reflect/Method;)Z"); + + getClassForID = frame.GetStaticMethodID(utility, "getClassFor", + "(Ljava/lang/Object;)Ljava/lang/Class;"); registerClass(_java_lang_Object = new JPObjectBaseClass()); registerClass(_java_lang_Class = new JPClassBaseClass()); @@ -202,8 +238,7 @@ JPClass* JPTypeManager::findClass(jclass cls) if (JPJni::isArray(cls)) { return registerArrayClass(name, cls); - } - else + } else { return registerObjectClass(name, cls); } diff --git a/native/common/jp_value.cpp b/native/common/jp_value.cpp index 3096de8d0..ef6cfb098 100644 --- a/native/common/jp_value.cpp +++ b/native/common/jp_value.cpp @@ -18,7 +18,7 @@ jobject JPValue::getJavaObject() const { - if (dynamic_cast (m_Class) != m_Class) + if (!m_Class->isPrimitive()) return m_Value.l; // This method is only used internally, thus it requires a logical code diff --git a/native/common/jp_voidtype.cpp b/native/common/jp_voidtype.cpp index 3e3d81450..b4fab4247 100644 --- a/native/common/jp_voidtype.cpp +++ b/native/common/jp_voidtype.cpp @@ -51,9 +51,11 @@ JPMatch::Type JPVoidType::canConvertToJava(PyObject* obj) jvalue JPVoidType::convertToJava(PyObject* obj) { + JP_TRACE_IN("JPVoidType::convertToJava"); jvalue res; res.l = NULL; return res; + JP_TRACE_OUT; } JPPyObject JPVoidType::invokeStatic(JPJavaFrame& frame, jclass claz, jmethodID mth, jvalue* val) diff --git a/native/java/org/jpype/Utility.java b/native/java/org/jpype/Utility.java index c1ba841fd..43245a952 100644 --- a/native/java/org/jpype/Utility.java +++ b/native/java/org/jpype/Utility.java @@ -20,11 +20,97 @@ import org.jpype.proxy.JPypeInvocationHandler; /** + * Support for JPype TypeManager. + * + * This is an internal class containing backported functionality from JPype 0.8. + * All functionality in this class will be moved in JPype 0.8 when the type + * manager functionality moved into the Java layer. * - * @author Karl Einar Nelson */ public class Utility { + + static boolean hasCallerSensitive = false; + + static + { + try + { + java.lang.reflect.Method method = java.lang.Class.class.getDeclaredMethod("forName", String.class); + for (Annotation annotation : method.getAnnotations()) + { + if ("@jdk.internal.reflect.CallerSensitive()".equals(annotation.toString())) + { + hasCallerSensitive = true; + } + } + } catch (NoSuchMethodException | SecurityException ex) + { + } + } + + /** + * Checks to see if the method is caller sensitive. + * + * As the annotation is a private internal, we must check by name. + * + * @param method is the method to be probed. + * @return true if caller sensitive. + */ + public static boolean isCallerSensitive(Method method) + { + if (hasCallerSensitive) + { + for (Annotation annotation : method.getAnnotations()) + { + if ("@jdk.internal.reflect.CallerSensitive()".equals(annotation.toString())) + { + return true; + } + } + } else + { + // JDK prior versions prior to 9 do not annotate methods that + // require special handling, thus we will just blanket those + // classes known to have issues. + Class cls = method.getDeclaringClass(); + if (cls.equals(java.lang.Class.class) + || cls.equals(java.lang.ClassLoader.class) + || cls.equals(java.sql.DriverManager.class)) + { + return true; + } + } + return false; + } + + /** + * Call a method using reflection. + * + * This method creates a stackframe so that caller sensitive methods will + * execute properly. + * + * @param method is the method to call. + * @param obj is the object to operate on, it will be null if the method is + * static. + * @param args the arguments to method. + * @return the object that results form the invocation. + * @throws IllegalAccessException + * @throws IllegalArgumentException + * @throws InvocationTargetException + */ + public static Object callMethod(Method method, Object obj, Object[] args) + throws Throwable + { + try + { + return method.invoke(obj, args); + } catch (InvocationTargetException ex) + { + throw ex.getCause(); + } + } + /** * Get the class for an object. *

@@ -51,5 +137,6 @@ public static Class getClassFor(Object obj) return cls.getInterfaces()[0]; } return cls; - } + } + } diff --git a/native/python/include/jpype_memory_view.h b/native/python/include/jpype_memory_view.h index 2604079f7..16f7bd08b 100644 --- a/native/python/include/jpype_memory_view.h +++ b/native/python/include/jpype_memory_view.h @@ -84,7 +84,7 @@ class JPPyMemoryViewAccessor return py_buff != 0; } - int size() + Py_ssize_t size() { return py_buff->len; } diff --git a/native/python/jp_pythontypes.cpp b/native/python/jp_pythontypes.cpp index d8df79d23..b2e8dabdb 100644 --- a/native/python/jp_pythontypes.cpp +++ b/native/python/jp_pythontypes.cpp @@ -264,7 +264,7 @@ bool JPPyLong::checkConvertable(PyObject* obj) bool JPPyLong::checkIndexable(PyObject* obj) { - return PyObject_HasAttrString(obj, "__index__"); + return PyObject_HasAttrString(obj, "__index__")!=0; } jlong JPPyLong::asLong(PyObject* obj) @@ -334,7 +334,7 @@ JPPyObject JPPyString::fromCharUTF16(jchar c) #else if (c < 128) { - char c1 = c; + char c1 = (char)c; return JPPyObject(JPPyRef::_call, PyUnicode_FromStringAndSize(&c1, 1)); } JPPyObject buf(JPPyRef::_call, PyUnicode_New(1, 65535)); @@ -428,7 +428,7 @@ jchar JPPyString::asCharUTF16(PyObject* pyobj) #else if (PyBytes_Check(pyobj)) { - int sz = PyBytes_Size(pyobj); + Py_ssize_t sz = PyBytes_Size(pyobj); if (sz != 1) JP_RAISE_VALUE_ERROR("Char must be length 1"); diff --git a/native/python/pyjp_class.cpp b/native/python/pyjp_class.cpp index edecf1545..ed2175f40 100644 --- a/native/python/pyjp_class.cpp +++ b/native/python/pyjp_class.cpp @@ -371,7 +371,7 @@ PyObject* PyJPClass::isPrimitive(PyJPClass* self, PyObject* args) { ASSERT_JVM_RUNNING("PyJPClass::isPrimitive"); JPJavaFrame frame; - return PyBool_FromLong(dynamic_cast (self->m_Class) == self->m_Class); + return PyBool_FromLong((self->m_Class)->isPrimitive()); } PY_STANDARD_CATCH; return NULL; diff --git a/native/python/pyjp_module.cpp b/native/python/pyjp_module.cpp index 0fabfe11e..4a0d99c58 100644 --- a/native/python/pyjp_module.cpp +++ b/native/python/pyjp_module.cpp @@ -153,7 +153,7 @@ PyObject* PyJPModule::startup(PyObject* obj, PyObject* args) } } - JPEnv::loadJVM(cVmPath, args, ignoreUnrecognized, convertStrings); + JPEnv::loadJVM(cVmPath, args, ignoreUnrecognized!=0, convertStrings!=0); Py_RETURN_NONE; } PY_STANDARD_CATCH; diff --git a/native/python/pyjp_monitor.cpp b/native/python/pyjp_monitor.cpp index c160fe195..e8d7a3315 100644 --- a/native/python/pyjp_monitor.cpp +++ b/native/python/pyjp_monitor.cpp @@ -98,7 +98,7 @@ int PyJPMonitor::__init__(PyJPMonitor* self, PyObject* args) return -1; } - if (dynamic_cast (v1.getClass()) != 0) + if ((v1.getClass())->isPrimitive()) { PyErr_SetString(PyExc_TypeError, "Primitives cannot be used to synchronize."); return -1; diff --git a/native/python/pyjp_value.cpp b/native/python/pyjp_value.cpp index 657d9d277..7b066eb2b 100644 --- a/native/python/pyjp_value.cpp +++ b/native/python/pyjp_value.cpp @@ -96,7 +96,7 @@ JPPyObject PyJPValue::alloc(JPClass* cls, jvalue value) JP_PY_CHECK(); // If it is not a primitive we need to reference it - if (dynamic_cast (cls) != cls) + if (!cls->isPrimitive()) value.l = frame.NewGlobalRef(value.l); // New value instance @@ -160,7 +160,7 @@ int PyJPValue::__init__(PyJPValue* self, PyObject* args, PyObject* kwargs) } jvalue v = type->convertToJava(value); - if (dynamic_cast (type) != type) + if (!type->isPrimitive()) v.l = frame.NewGlobalRef(v.l); self->m_Value = JPValue(type, v); return 0; @@ -180,7 +180,7 @@ PyObject* PyJPValue::__str__(PyJPValue* self) sout << "m_Value.getClass()->toString(); // FIXME Remove these extra diagnostic values - if (dynamic_cast (self->m_Value.getClass()) != NULL) + if ((self->m_Value.getClass())->isPrimitive()) sout << endl << " value = primitive"; else { @@ -202,7 +202,7 @@ void PyJPValue::__dealloc__(PyJPValue* self) JPClass* cls = value.getClass(); // This one can't check for initialized because we may need to delete a stale // resource after shutdown. - if (cls != NULL && JPEnv::isInitialized() && dynamic_cast (cls) != cls) + if (cls != NULL && JPEnv::isInitialized() && !cls->isPrimitive()) { JP_TRACE("Value", cls->getCanonicalName(), &(value.getValue())); // If the JVM has shutdown then we don't need to free the resource @@ -264,7 +264,7 @@ PyObject* PyJPValue::toString(PyJPValue* self) return out; } - if (dynamic_cast (cls) != 0) + if (cls->isPrimitive()) JP_RAISE_VALUE_ERROR("toString requires a java object"); if (cls == NULL) JP_RAISE_VALUE_ERROR("toString called with null class"); @@ -303,7 +303,7 @@ PyObject* PyJPValue::toUnicode(PyJPValue* self) return out; } - if (dynamic_cast (cls) != 0) + if (cls->isPrimitive()) JP_RAISE_VALUE_ERROR("toUnicode requires a java object"); if (cls == NULL) JP_RAISE_VALUE_ERROR("toUnicode called with null class"); diff --git a/project/jpype_cpython/nbproject/project.xml b/project/jpype_cpython/nbproject/project.xml index bda6dd7cc..6270726ee 100755 --- a/project/jpype_cpython/nbproject/project.xml +++ b/project/jpype_cpython/nbproject/project.xml @@ -20,7 +20,10 @@ - false + true + ANSI|ANSI + Default_0|JPype + Default_1|JPype diff --git a/project/jpype_java/nbproject/project.properties b/project/jpype_java/nbproject/project.properties index dd07aac68..261872392 100755 --- a/project/jpype_java/nbproject/project.properties +++ b/project/jpype_java/nbproject/project.properties @@ -19,8 +19,12 @@ build.test.results.dir=${build.dir}/test/results #debug.transport=dt_socket debug.classpath=\ ${run.classpath} +debug.modulepath=\ + ${run.modulepath} debug.test.classpath=\ ${run.test.classpath} +debug.test.modulepath=\ + ${run.test.modulepath} # Files in build.classes.dir which should be excluded from distribution jar dist.archive.excludes= # This directory is removed when the project is cleaned: @@ -38,6 +42,8 @@ javac.classpath= javac.compilerargs= javac.deprecation=false javac.external.vm=true +javac.modulepath= +javac.processormodulepath= javac.processorpath=\ ${javac.classpath} javac.source=1.8 @@ -45,6 +51,8 @@ javac.target=1.8 javac.test.classpath=\ ${javac.classpath}:\ ${build.classes.dir} +javac.test.modulepath=\ + ${javac.modulepath} javac.test.processorpath=\ ${javac.test.classpath} javadoc.additionalparam= @@ -68,9 +76,13 @@ run.classpath=\ # You may also define separate properties like run-sys-prop.name=value instead of -Dname=value. # To set system properties for unit tests define test-sys-prop.name=value: run.jvmargs= +run.modulepath=\ + ${javac.modulepath} run.test.classpath=\ ${javac.test.classpath}:\ ${build.test.classes.dir} +run.test.modulepath=\ + ${javac.test.modulepath} source.encoding=UTF-8 src.java.dir=${file.reference.native-java} test.harness.dir=${file.reference.test-harness} diff --git a/test/build.xml b/test/build.xml index 8152297c1..f5cfbc57a 100644 --- a/test/build.xml +++ b/test/build.xml @@ -7,10 +7,31 @@ - + + + + + + + + + + + + + + + + + + + - + diff --git a/test/harness/java9/jpype/method/Caller.java b/test/harness/java9/jpype/method/Caller.java new file mode 100644 index 000000000..1ec72c39c --- /dev/null +++ b/test/harness/java9/jpype/method/Caller.java @@ -0,0 +1,77 @@ +package jpype.method; + +import jdk.internal.reflect.CallerSensitive; + +/** Caller sensitive methods need special handling. + * + * So we need to test each possible path + * (different return types and arguments). + * + * We will pretend our methods are also CallerSensitive. + */ +public class Caller +{ + @CallerSensitive + public static Object callObjectStatic() + { + return new Caller(); + } + + @CallerSensitive + public Object callObjectMember() + { + return new Caller(); + } + + @CallerSensitive + public static void callVoidStatic() + { + } + + @CallerSensitive + public void callVoidMember() + { + } + + @CallerSensitive + public static int callIntegerStatic() + { + return 123; + } + + @CallerSensitive + public int callIntegerMember() + { + return 123; + } + + @CallerSensitive + public Object callArgs(Object a, Object b) + { + return b; + } + + @CallerSensitive + public int callArg1(int a) + { + return a; + } + + @CallerSensitive + public Object callVarArgs(Object a, Object... b) + { + return b; + } + + public Class callStackWalker1() + { + return StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE).getCallerClass(); + } + + @CallerSensitive + public Class callStackWalker2() + { + return StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE).getCallerClass(); + } + +}; diff --git a/test/jpypetest/test_caller_sensitive.py b/test/jpypetest/test_caller_sensitive.py new file mode 100644 index 000000000..f95beba3a --- /dev/null +++ b/test/jpypetest/test_caller_sensitive.py @@ -0,0 +1,108 @@ +# ***************************************************************************** +# Copyright 2019 Karl Einar Nelson +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ***************************************************************************** +import jpype +import common + +class JCallerSensitiveCase(common.JPypeTestCase): + """ Test for caller sensitive methods. + + Java uses a special pattern to deal with the security manager. It + uses the stack to find the callers class and then decides if the + call is permitted. But when we call from JNI there is no class. + Thus, we need a special caller pattern which proxies to Java and + then calls the method. + + This alternative method has to be tested against all of the + different patterns (static, member), returning (void, primitive, object), + called with (nothing, object, primitive, many, varargs) + + Unfortunately, the actual problematic method in Java is private, + so we can't get to it directly. Thus will will perform indirect tests. + + For now we do not support caller sensitive constructors. + """ + + def setUp(self): + common.JPypeTestCase.setUp(self) + if not jpype.getJVMVersion()>(1,8,0): + raise common.unittest.SkipTest + + self.Class = jpype.JClass("jpype.method.Caller") + self.obj = self.Class() + + def testCallStatic(self): + self.assertIsInstance(self.Class.callObjectStatic(), self.Class) + + def testCallStaticAsMember(self): + self.assertIsInstance(self.obj.callObjectStatic(), self.Class) + + def testCallMember(self): + self.assertIsInstance(self.obj.callObjectMember(), self.Class) + + def testCallMemberFromClass(self): + self.assertIsInstance(self.Class.callObjectMember(self.obj), self.Class) + + def testCallVoidStatic(self): + self.assertEqual(self.Class.callVoidStatic(), None) + + def testCallVoidStaticAsMember(self): + self.assertEqual(self.obj.callVoidStatic(), None) + + def testCallVoidMemberFromClass(self): + self.assertEqual(self.Class.callVoidMember(self.obj), None) + + def testCallIntegerStatic(self): + self.assertEqual(self.Class.callIntegerStatic(), 123) + + def testCallIntegerMember(self): + self.assertEqual(self.obj.callIntegerMember(), 123) + + def testCallIntegerStaticAsMember(self): + self.assertEqual(self.obj.callIntegerStatic(), 123) + + def testCallIntegerMemberFromClass(self): + self.assertEqual(self.Class.callIntegerMember(self.obj), 123) + + def testArgs(self): + self.assertEqual(self.obj.callArgs(1, 2), 2) + + def testArgsFromClass(self): + self.assertEqual(self.Class.callArgs(self.obj, 1, 2), 2) + + def testPrimitive(self): + self.assertEqual(self.obj.callArg1(123), 123) + + def testPrimitiveFromClass(self): + self.assertEqual(self.Class.callArg1(self.obj, 125), 125) + + def testVarArgs(self): + self.assertEqual(tuple(self.obj.callVarArgs(1,2,3)), (2,3)) + + def testVarArgsFromClass(self): + self.assertEqual(tuple(self.Class.callVarArgs(self.obj, 1,2,3)), (2,3)) + + def testDeclaredMethod(self): + self.assertIsInstance(jpype.java.lang.Object.class_.getDeclaredMethod('wait'), jpype.java.lang.reflect.Method) + + def testStackWalker1(self): + with self.assertRaises(jpype.java.lang.IllegalCallerException): + self.obj.callStackWalker1() + + def testStackWalker2(self): + self.assertEqual(self.obj.callStackWalker2(), jpype.JClass(jpype.java.lang.Class.forName("org.jpype.Utility")).class_) + + diff --git a/test/jpypetest/test_reflect.py b/test/jpypetest/test_reflect.py index 1d68005cf..53cd56e7c 100644 --- a/test/jpypetest/test_reflect.py +++ b/test/jpypetest/test_reflect.py @@ -56,6 +56,7 @@ def testCallPublicMethod(self): def testCallPrivateMethod(self): method = self.Reflect.class_.getDeclaredMethod('privateMethod') + method.setAccessible(True) obj = self.Reflect() self.assertIsNotNone(obj) self.assertIsNotNone(method) @@ -71,6 +72,7 @@ def testAccessPublicField(self): def testAccessPrivateField(self): field = self.Reflect.class_.getDeclaredField('privateField') obj = self.Reflect() + field.setAccessible(True) self.assertIsNotNone(obj) self.assertIsNotNone(field) self.assertEqual('private', field.get(obj)) From b6b3fd956333b955a7a4277fc90b0ee44c0d5a76 Mon Sep 17 00:00:00 2001 From: Karl Nelson Date: Thu, 1 Aug 2019 10:03:15 -0700 Subject: [PATCH 2/2] Missing imports fixed --- native/java/org/jpype/Utility.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/native/java/org/jpype/Utility.java b/native/java/org/jpype/Utility.java index 43245a952..228b03693 100644 --- a/native/java/org/jpype/Utility.java +++ b/native/java/org/jpype/Utility.java @@ -16,7 +16,10 @@ */ package org.jpype; +import java.lang.annotation.Annotation; import java.lang.reflect.Proxy; +import java.lang.reflect.Method; +import java.lang.reflect.InvocationTargetException; import org.jpype.proxy.JPypeInvocationHandler; /**