From 6dfdc243bd612a73e588cbe45f9925ef58ef542c Mon Sep 17 00:00:00 2001 From: Craig Macdonald Date: Fri, 29 May 2020 10:26:49 +0100 Subject: [PATCH 1/7] refactoring of Java location bootstrap, addressing #537 --- jnius/__init__.py | 5 +- jnius/env.py | 411 ++++++++++++++++++++---------------- jnius/jnius_jvm_desktop.pxi | 5 +- jnius/jnius_jvm_dlopen.pxi | 46 +--- setup.py | 23 +- 5 files changed, 253 insertions(+), 237 deletions(-) diff --git a/jnius/__init__.py b/jnius/__init__.py index 55d1d380..b7207c7e 100644 --- a/jnius/__init__.py +++ b/jnius/__init__.py @@ -9,13 +9,14 @@ __version__ = '1.3.0' -from .env import get_jnius_lib_location, get_jdk_home +from .env import get_java_setup import os import sys if sys.platform == 'win32' and sys.version_info >= (3, 8): path = os.path.dirname(__file__) - jdk_home = get_jdk_home(sys.platform) + java = get_java_setup(sys.platform) + jdk_home = java.get_javahome() with os.add_dll_directory(path): for suffix in ( ('bin', 'client'), diff --git a/jnius/env.py b/jnius/env.py index 6031ff4f..0f5237a4 100644 --- a/jnius/env.py +++ b/jnius/env.py @@ -30,116 +30,268 @@ "sun4v": "sparcv9" } -JAVA_HOME = getenv('JAVA_HOME') +DEFAULT_PLATFORM = sys.platform + +def is_set(string): + return string is not None and len(string) > 0 + +def get_java_setup(platform=DEFAULT_PLATFORM): + ''' + Returns an instance of JavaLocation. + ''' + # prefer Env variables + JAVA_HOME = getenv('JAVA_HOME') + if not is_set(JAVA_HOME): + JAVA_HOME = getenv('JDK_HOME') + if not is_set(JAVA_HOME): + JAVA_HOME = getenv('JRE_HOME') + #TODO encodings + + # Use java_home program on Mac + if not is_set(JAVA_HOME) and platform == 'darwin': + JAVA_HOME = get_osx_framework() + if not is_set(JAVA_HOME): + raise Exception('You must install Java for Mac OSX') + + # go hunting for Javac and Java programs, in that order + if not is_set(JAVA_HOME): + JAVA_HOME = get_jdk_home(platform) + + if not is_set(JAVA_HOME): + JAVA_HOME = get_jre_home(platform) + + if JAVA_HOME is None: + raise RuntimeError("Could not find your Java installed. Please set the JAVA_HOME env var.") + + if isinstance(JAVA_HOME, bytes): + JAVA_HOME = JAVA_HOME.decode('utf-8') + + log.debug("Identified Java at %s" % JAVA_HOME) + + # Remove /bin if it's appended to JAVA_HOME + if JAVA_HOME[-3:] == 'bin': + JAVA_HOME = JAVA_HOME[:-4] + + if platform == "android": + return AndroidJavaLocation(platform, JAVA_HOME) + if platform == "win32": #only this? + return WindowsJavaLocation(platform, JAVA_HOME) + if platform == "darwin": #only this? + return MacOsXJavaLocation(platform, JAVA_HOME) + if platform in ('linux', 'linux2', 'sunos5'): #only this? + return UnixJavaLocation(platform, JAVA_HOME) + log.warning("warning: unknown platform %s assuming linux or sunOS" % platform) + return UnixJavaLocation(platform, JAVA_HOME) + +class JavaLocation: + + def __init__(self, platform, home): + self.platform = platform + self.home = home + + def get_javahome(self): + ''' + Returns the location of the identified JRE or JDK + ''' + return self.home + + + def is_jdk(self): + ''' + Returns true if the location is a JDK, based on existing of javac + ''' + return exists(self.get_javac()) + + def get_javac(self): + ''' + Returns absolute path of the javac executable + ''' + return join(self.home, "bin", "javac") + + def get_include_dirs(self): + ''' + Returns a list of absolute paths of JDK include directories, for compiling. + Calls _get_platform_include_dir() internally. + ''' + return [ + join(self.home, 'include'), + self._get_platform_include_dir() + ] + def _get_platform_include_dir(self): + ''' + Returns the platform-specific include directory, for setup.py + ''' + pass + + def get_library_dirs(self): + ''' + Returns a list of absolute paths of JDK lib directories, for setup.py + ''' + pass + + def get_libraries(self): + ''' + Returns the names of the libraries for this platform, for setup.py + ''' + pass + + def get_jnius_lib_location(self): + ''' + Returns the full path of the Java library for runtime binding with. + Can be overridden by using JVM_PATH env var to set absolute path of the Java library + ''' + libjvm_override_path = getenv('JVM_PATH') + if libjvm_override_path: + log.info( + dedent(""" + Using override env var JVM_PATH (%s) to load libjvm. + Please report your system information (os version, java + version, etc), and the path that works for you, to the + PyJNIus project, at https://github.com/kivy/pyjnius/issues. + so we can improve the automatic discovery. + """ + ), + libjvm_override_path + ) + return libjvm_override_path -def find_javac(platform, possible_homes): - '''Find javac in all possible locations.''' - name = "javac.exe" if platform == "win32" else "javac" - for home in possible_homes: - for javac in [join(home, name), join(home, 'bin', name)]: - if exists(javac): - if platform == "win32" and not PY2: # Avoid path space execution error - return '"%s"' % javac - return javac - return name # Fall back to "hope it's on the path" + cpu = get_cpu() + platform = self.platform + log.debug( + "looking for libjvm to initiate pyjnius, cpu is %s, platform is %s", + cpu, platform + ) + lib_locations = self._possible_lib_locations(cpu) + for location in lib_locations: + full_lib_location = join(self.home, location) + if exists(full_lib_location): + log.debug("found libjvm.so at %s", full_lib_location) + return full_lib_location + + raise RuntimeError( + """ + Unable to find libjvm.so, (tried %s) + you can use the JVM_PATH env variable with the absolute path + to libjvm.so to override this lookup, if you know + where pyjnius should look for it. + e.g: + export JAVA_HOME=/usr/lib/jvm/java-8-oracle/ + export JVM_PATH=/usr/lib/jvm/java-8-oracle/jre/lib/amd64/server/libjvm.so + # run your program + """ + % [join(self.home, loc) for loc in lib_locations] + ) -def get_include_dirs(platform): - if platform == 'darwin': - framework = get_osx_framework() - if '1.6' in framework: - return [join( - framework, ( - 'System/Library/Frameworks/' - 'JavaVM.framework/Versions/Current/Headers' - ) - )] - else: - # We want to favor Java installation declaring JAVA_HOME - if JAVA_HOME: - framework = JAVA_HOME - - return [ - '{0}/include'.format(framework), - '{0}/include/darwin'.format(framework) - ] - - else: - jdk_home = get_jdk_home(platform) - if platform == 'win32': - incl_dir = join(jdk_home, 'include', 'win32') - elif platform == 'sunos5': - incl_dir = join(jdk_home, 'include', 'solaris') + def _possible_lib_locations(self, cpu): + ''' + Returns a list of relative possible locations for the Java library. + Used by the default implementation of get_jnius_lib_location() + ''' + pass + + +class WindowsJavaLocation(JavaLocation): + + def get_javac(self): + super().get_javac() + ".exe" + + def get_libraries(self): + return ['jvm'] + + def _get_platform_include_dir(self): + return join(self.home, 'include', 'win32') + + +class UnixJavaLocation(JavaLocation): + + def _get_platform_include_dir(self): + if self.platform == 'sunos5': + return join(self.home, 'include', 'solaris') else: - incl_dir = join(jdk_home, 'include', 'linux') + return join(self.home, 'include', 'linux') + + def _possible_lib_locations(self, cpu): + + root = self.home + if root.endswith('jre'): + root = root[:-3] return [ - join(jdk_home, 'include'), - incl_dir + 'lib/server/libjvm.so', + 'jre/lib/{}/default/libjvm.so'.format(cpu), + 'jre/lib/{}/server/libjvm.so'.format(cpu), ] -def get_library_dirs(platform, arch=None): - if platform == 'win32': - jre_home = get_jre_home(platform) - jdk_home = JAVA_HOME - if isinstance(jre_home, bytes): - jre_home = jre_home.decode('utf-8') +class MacOsXJavaLocation(UnixJavaLocation): + + def __init__(): + self.platform = platform + + def _get_platform_include_dir(self): + return join(jdk_home, 'include', 'darwin') + + def _possible_lib_locations(self, cpu): + if '1.6' in self.root: + return ['../Libraries/libjvm.dylib'] # TODO what should this be resolved to? return [ - join(jdk_home, 'lib'), - join(jdk_home, 'bin', 'server') + 'jre/lib/jli/libjli.dylib', + # In that case, the Java version >=9. + 'lib/jli/libjli.dylib', + # adoptopenjdk12 doesn't have the jli subfolder either + 'lib/libjli.dylib', ] - elif platform == 'android': - return ['libs/{}'.format(arch)] - return [] + + + + # this is overridden due to the specifalities of version 1.6 + def get_include_dirs(self): + framework = self.home + if '1.6' in framework: + return [join( + framework, ( + 'System/Library/Frameworks/' + 'JavaVM.framework/Versions/Current/Headers' + ) + )] + return super().get_include_dirs() + +class AndroidJavaLocation(UnixJavaLocation): + + def get_libraries(platform): + #if platform == 'android': + # for android, we use SDL... + return ['sdl', 'log'] + + def get_library_dirs(self): + return [join(self.home, 'libs', arch)] def get_jre_home(platform): jre_home = None - if JAVA_HOME and exists(join(JAVA_HOME, 'jre')): - jre_home = join(JAVA_HOME, 'jre') - - if platform != 'win32' and not jre_home: + + if platform != 'win32': jre_home = realpath( check_output( split('which java') ).decode('utf-8').strip() ).replace('bin/java', '') - if platform == 'win32': - if isinstance(jre_home, bytes): - jre_home = jre_home.decode('utf-8') - return jre_home def get_jdk_home(platform): - jdk_home = getenv('JDK_HOME') - if not jdk_home: - if platform == 'win32': - TMP_JDK_HOME = getenv('JAVA_HOME') - if not TMP_JDK_HOME: - raise Exception('Unable to find JAVA_HOME') - - # Remove /bin if it's appended to JAVA_HOME - if TMP_JDK_HOME[-3:] == 'bin': - TMP_JDK_HOME = TMP_JDK_HOME[:-4] - - # Check whether it's JDK - if exists(join(TMP_JDK_HOME, 'bin', 'javac.exe')): - jdk_home = TMP_JDK_HOME - - else: - jdk_home = realpath( + jdk_home = realpath( check_output( ['which', 'javac'] ).decode('utf-8').strip() ).replace('bin/javac', '') if not jdk_home or not exists(jdk_home): - raise Exception('Unable to determine JDK_HOME') + return None return jdk_home @@ -156,24 +308,6 @@ def get_osx_framework(): return framework.strip() -def get_possible_homes(platform): - if platform == 'darwin': - if JAVA_HOME: - return JAVA_HOME - - FRAMEWORK = get_osx_framework() - if not FRAMEWORK: - raise Exception('You must install Java for Mac OSX') - - return FRAMEWORK - - else: - return ( - get_jdk_home(platform), - get_jre_home(platform), - ) - - def get_cpu(): try: return MACHINE2CPU[machine] @@ -185,90 +319,3 @@ def get_cpu(): print(" Using cpu = 'i386' instead!") return 'i386' - -def get_libraries(platform): - if platform == 'android': - # for android, we use SDL... - return ['sdl', 'log'] - - elif platform == 'win32': - return ['jvm'] - - -def get_jnius_lib_location(platform): - cpu = get_cpu() - libjvm_override_path = getenv('JVM_PATH') - - if libjvm_override_path: - log.info( - dedent(""" - Using override env var JVM_PATH (%s) to load libjvm. - Please report your system information (os version, java - version, etc), and the path that works for you, to the - PyJNIus project, at https://github.com/kivy/pyjnius/issues. - so we can improve the automatic discovery. - """ - ), - libjvm_override_path - ) - return libjvm_override_path - - log.debug( - "looing for libjvm to initiate pyjnius, cpu is %s, platform is %s", - cpu, platform - ) - - if platform == 'darwin': - root = get_osx_framework() - - if '1.6' in root: - return '../Libraries/libjvm.dylib' - - else: - # We want to favor Java installation declaring JAVA_HOME - if JAVA_HOME: - root = JAVA_HOME - - lib_locations = ( - 'jre/lib/jli/libjli.dylib', - # In that case, the Java version >=9. - 'lib/jli/libjli.dylib', - # adoptopenjdk12 doesn't have the jli subfolder either - 'lib/libjli.dylib', - ) - - else: - if platform not in ('linux', 'linux2', 'sunos5'): - log.warning("warning: unknown platform assuming linux or sunOS") - - root = dirname(get_jre_home(platform)) - if root.endswith('jre'): - root = root[:-3] - - lib_locations = ( - 'lib/server/libjvm.so', - 'jre/lib/{}/default/libjvm.so'.format(cpu), - 'jre/lib/{}/server/libjvm.so'.format(cpu), - ) - - for location in lib_locations: - full_lib_location = join(root, location) - - if exists(full_lib_location): - log.debug("found libjvm.so at %s", full_lib_location) - return location - - raise RuntimeError( - """ - Unable to find libjvm.so, (tried %s) - you can use the JVM_PATH env variable with the relative path - from JAVA_HOME to libjvm.so to override this lookup, if you know - where pyjnius should look for it. - - e.g: - export JAVA_HOME=/usr/lib/jvm/java-8-oracle/ - export JVM_PATH=./jre/lib/amd64/server/libjvm.so - # run your program - """ - % [join(root, loc) for loc in lib_locations] - ) diff --git a/jnius/jnius_jvm_desktop.pxi b/jnius/jnius_jvm_desktop.pxi index 444e188c..fbfbfc97 100644 --- a/jnius/jnius_jvm_desktop.pxi +++ b/jnius/jnius_jvm_desktop.pxi @@ -1,7 +1,7 @@ import sys import os from os.path import join -from jnius.env import get_jdk_home +from jnius.env import get_java_setup from cpython.version cimport PY_MAJOR_VERSION # on desktop, we need to create an env :) @@ -49,7 +49,8 @@ cdef void create_jnienv() except *: if sys.version_info >= (3, 8): # uh, let's see if this works and cleanup later - jdk_home = get_jdk_home('win32') + java = get_java_setup('win32') + jdk_home = java.get_javahome() for suffix in ( ('bin', 'client'), ('bin', 'server'), diff --git a/jnius/jnius_jvm_dlopen.pxi b/jnius/jnius_jvm_dlopen.pxi index ce1cf436..ff81ea69 100644 --- a/jnius/jnius_jvm_dlopen.pxi +++ b/jnius/jnius_jvm_dlopen.pxi @@ -5,7 +5,7 @@ from subprocess import check_output, CalledProcessError from os.path import dirname, join, exists from os import readlink from sys import platform -from .env import get_jnius_lib_location +from .env import get_java_setup cdef extern from 'dlfcn.h' nogil: @@ -42,36 +42,6 @@ cdef extern from "jni.h": cdef JNIEnv *_platform_default_env = NULL -cdef find_java_home(): - if platform in ('linux2', 'linux'): - java = check_output(split('which javac')).strip() - if not java: - java = check_output(split('which java')).strip() - if not java: - return - - while True: - try: - java = readlink(java) - except OSError: - break - return dirname(dirname(java)).decode('utf8') - - if platform == 'darwin': - MAC_JAVA_HOME='/usr/libexec/java_home' - # its a mac - if not exists(MAC_JAVA_HOME): - # I believe this always exists, but just in case - return - try: - java = check_output(MAC_JAVA_HOME).strip().decode('utf8') - return java - except CalledProcessError as exc: - # java_home return non-zero exit code if no Javas are installed - return - - - cdef void create_jnienv() except *: cdef JavaVM* jvm cdef JavaVMInitArgs args @@ -81,19 +51,17 @@ cdef void create_jnienv() except *: cdef void *handle import jnius_config - JAVA_HOME = os.getenv('JAVA_HOME') or find_java_home() - if JAVA_HOME is None or JAVA_HOME == '': - raise SystemError("JAVA_HOME is not set, and unable to guess JAVA_HOME") - cdef str JNIUS_LIB_SUFFIX = get_jnius_lib_location(JNIUS_PLATFORM) + JAVA_LOCATION = get_java_setup() + cdef str java_lib = JAVA_LOCATION.get_jnius_lib_location() IF JNIUS_PYTHON3: try: - jnius_lib_suffix = JNIUS_LIB_SUFFIX.decode("utf-8") + java_lib = java_lib.decode("utf-8") except AttributeError: - jnius_lib_suffix = JNIUS_LIB_SUFFIX - lib_path = str_for_c(os.path.join(JAVA_HOME, jnius_lib_suffix)) + java_lib = java_lib + lib_path = str_for_c(java_lib) ELSE: - lib_path = str_for_c(os.path.join(JAVA_HOME, JNIUS_LIB_SUFFIX)) + lib_path = str_for_c(java_lib) handle = dlopen(lib_path, RTLD_NOW | RTLD_GLOBAL) diff --git a/setup.py b/setup.py index 37670128..5e6de7ca 100644 --- a/setup.py +++ b/setup.py @@ -22,12 +22,8 @@ syspath = sys.path[:] sys.path.insert(0, 'jnius') from env import ( - get_possible_homes, - get_library_dirs, - get_include_dirs, - get_libraries, - find_javac, - PY2, + get_java_setup, + PY2 ) sys.path = syspath @@ -76,10 +72,13 @@ def getenv(key): else: FILES = [fn[:-3] + 'c' for fn in FILES if fn.endswith('pyx')] +JAVA=get_java_setup(PLATFORM) -def compile_native_invocation_handler(*possible_homes): +assert JAVA.is_jdk(), "You need a JDK, we only found a JRE. Try setting JAVA_HOME" + +def compile_native_invocation_handler(java): '''Find javac and compile NativeInvocationHandler.java.''' - javac = find_javac(PLATFORM, possible_homes) + javac = java.get_javac() source_level = '1.7' try: subprocess.check_call([ @@ -92,7 +91,7 @@ def compile_native_invocation_handler(*possible_homes): join('jnius', 'src', 'org', 'jnius', 'NativeInvocationHandler.java') ]) -compile_native_invocation_handler(*get_possible_homes(PLATFORM)) +compile_native_invocation_handler(JAVA) # generate the config.pxi @@ -116,9 +115,9 @@ def compile_native_invocation_handler(*possible_homes): ext_modules=[ Extension( 'jnius', [join('jnius', x) for x in FILES], - libraries=get_libraries(PLATFORM), - library_dirs=get_library_dirs(PLATFORM), - include_dirs=get_include_dirs(PLATFORM), + libraries=JAVA.get_libraries(), + library_dirs=JAVA.get_library_dirs(), + include_dirs=JAVA.get_include_dirs(), extra_link_args=EXTRA_LINK_ARGS, ) ], From e5889866908dd70eca6f14c1768e667cc59008c7 Mon Sep 17 00:00:00 2001 From: Craig Macdonald Date: Fri, 29 May 2020 10:34:55 +0100 Subject: [PATCH 2/7] fix for mac, todo or android --- jnius/env.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/jnius/env.py b/jnius/env.py index 0f5237a4..bc478c70 100644 --- a/jnius/env.py +++ b/jnius/env.py @@ -227,14 +227,11 @@ def _possible_lib_locations(self, cpu): class MacOsXJavaLocation(UnixJavaLocation): - def __init__(): - self.platform = platform - def _get_platform_include_dir(self): - return join(jdk_home, 'include', 'darwin') + return join(self.home, 'include', 'darwin') def _possible_lib_locations(self, cpu): - if '1.6' in self.root: + if '1.6' in self.home: return ['../Libraries/libjvm.dylib'] # TODO what should this be resolved to? return [ @@ -261,12 +258,14 @@ def get_include_dirs(self): class AndroidJavaLocation(UnixJavaLocation): - def get_libraries(platform): + def get_libraries(self): #if platform == 'android': # for android, we use SDL... return ['sdl', 'log'] def get_library_dirs(self): + raise RuntimeError("TODO: who sets arch?") + arch = None return [join(self.home, 'libs', arch)] From ab7df83a0a4e52481ecf45defc525c10340a60d5 Mon Sep 17 00:00:00 2001 From: Craig Macdonald Date: Thu, 4 Jun 2020 16:38:27 +0100 Subject: [PATCH 3/7] added some default *nix search paths --- jnius/env.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/jnius/env.py b/jnius/env.py index bc478c70..fdf5cf98 100644 --- a/jnius/env.py +++ b/jnius/env.py @@ -279,6 +279,16 @@ def get_jre_home(platform): ).decode('utf-8').strip() ).replace('bin/java', '') + if is_set(jre_home): + return jre_home + + # didnt find java command on the path, we can + # fallback to hunting in some default unix locations + for loc in ["/usr/java/latest/", "/usr/java/default/", "/usr/lib/jvm/default-java/"]: + if exists(loc + "bin/java"): + jre_home = loc + break + return jre_home From 4bda698042a7bad13b9c5b9e3474a7db8d5b6c8c Mon Sep 17 00:00:00 2001 From: Craig Macdonald Date: Mon, 22 Jun 2020 09:39:42 +0100 Subject: [PATCH 4/7] remove python 2.7 for this build --- .github/workflows/push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 410a7ef8..108cf9c9 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -6,7 +6,7 @@ jobs: strategy: matrix: python: - - '2.7' + # - '2.7' - '3.6' - '3.7' - '3.8' From 8afbdf1e470228c7278d9b3fe9931c6d70781241 Mon Sep 17 00:00:00 2001 From: Craig Macdonald Date: Mon, 22 Jun 2020 11:58:32 +0100 Subject: [PATCH 5/7] make sure javac location is correct on Windows --- jnius/env.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/jnius/env.py b/jnius/env.py index fdf5cf98..e4f2255e 100644 --- a/jnius/env.py +++ b/jnius/env.py @@ -100,7 +100,9 @@ def is_jdk(self): ''' Returns true if the location is a JDK, based on existing of javac ''' - return exists(self.get_javac()) + javac = self.get_javac() + print(javac) + return exists(javac) def get_javac(self): ''' @@ -195,7 +197,7 @@ def _possible_lib_locations(self, cpu): class WindowsJavaLocation(JavaLocation): def get_javac(self): - super().get_javac() + ".exe" + return super().get_javac() + ".exe" def get_libraries(self): return ['jvm'] From 9ad32cfce586051088b4ad9808c9a986f387213c Mon Sep 17 00:00:00 2001 From: Craig Macdonald Date: Mon, 22 Jun 2020 12:50:10 +0100 Subject: [PATCH 6/7] add library location for setup.py on Windows --- jnius/env.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/jnius/env.py b/jnius/env.py index e4f2255e..4a52b74c 100644 --- a/jnius/env.py +++ b/jnius/env.py @@ -201,6 +201,10 @@ def get_javac(self): def get_libraries(self): return ['jvm'] + + def get_library_dirs(self): + suffices = ['lib', join('bin', 'server')] + return [ join(self.home, suffix) for suffix in suffices ] def _get_platform_include_dir(self): return join(self.home, 'include', 'win32') From 640fa88ba307a10c1ef9fae7b335f899679d337a Mon Sep 17 00:00:00 2001 From: Craig Macdonald Date: Mon, 22 Jun 2020 13:10:45 +0100 Subject: [PATCH 7/7] remove debug print --- jnius/env.py | 1 - 1 file changed, 1 deletion(-) diff --git a/jnius/env.py b/jnius/env.py index 4a52b74c..c97d2403 100644 --- a/jnius/env.py +++ b/jnius/env.py @@ -101,7 +101,6 @@ def is_jdk(self): Returns true if the location is a JDK, based on existing of javac ''' javac = self.get_javac() - print(javac) return exists(javac) def get_javac(self):