[GR-19811] JVM_* fallback generation for native-image.

PullRequest: graal/5006

substratevm/mx.substratevm/mx_substratevm.py

@@ -32,6 +32,8 @@
 import time
 import re
 import tempfile
+from glob import glob
+import StringIO
 from contextlib import contextmanager
 from distutils.dir_util import mkpath, remove_tree  # pylint: disable=no-name-in-module
 from os.path import join, exists, basename, dirname
@@ -719,6 +721,144 @@ def _cinterfacetutorial(native_image, args=None):
     # Start the C executable
     mx.run([join(build_dir, 'cinterfacetutorial')])
 
+def gen_fallbacks():
+    native_project_dir = join(mx.dependency('substratevm:com.oracle.svm.native.jvm.' + ('windows' if mx.is_windows() else 'posix')).dir, 'src')
+
+    def collect_missing_symbols():
+        symbols = set()
+
+        def collect_symbols_fn(symbol_prefix):
+            def collector(line):
+                try:
+                    mx.logvv('Processing line: ' + line.rstrip())
+                    line_tokens = line.split()
+                    if mx.is_windows():
+                        # Windows dumpbin /SYMBOLS output
+                        # 030 00000000 UNDEF  notype ()    External     | JVM_GetArrayLength
+                        found_undef = line_tokens[2] == 'UNDEF'
+                    else:
+                        # Linux objdump objdump --wide --syms
+                        # 0000000000000000         *UND*	0000000000000000 JVM_InitStackTraceElement
+                        # Darwin objdump -t
+                        # 0000000000000000         *UND*  JVM_InitStackTraceElement
+                        found_undef = line_tokens[1] = '*UND*'
+                    if found_undef:
+                        symbol_candiate = line_tokens[-1]
+                        mx.logvv('Found undefined symbol: ' + symbol_candiate)
+                        platform_prefix = '_' if mx.is_darwin() else ''
+                        if symbol_candiate.startswith(platform_prefix + symbol_prefix):
+                            mx.logv('Pick symbol: ' + symbol_candiate)
+                            symbols.add(symbol_candiate[len(platform_prefix):])
+                except:
+                    mx.logv('Skipping line: ' + line.rstrip())
+            return collector
+
+        if mx.is_windows():
+            symbol_dump_command = 'dumpbin /SYMBOLS'
+        elif mx.is_darwin():
+            symbol_dump_command = 'objdump -t'
+        elif mx.is_linux():
+            symbol_dump_command = 'objdump --wide --syms'
+        else:
+            mx.abort('gen_fallbacks not supported on ' + sys.platform)
+
+        staticlib_wildcard = ['lib', mx_subst.path_substitutions.substitute('<staticlib:*>')]
+        if svm_java8():
+            staticlib_wildcard[0:0] = ['jre']
+        staticlib_wildcard_path = join(mx_compiler.jdk.home, *staticlib_wildcard)
+        for staticlib_path in glob(staticlib_wildcard_path):
+            mx.logv('Collect from : ' + staticlib_path)
+            mx.run(symbol_dump_command.split() + [staticlib_path], out=collect_symbols_fn('JVM_'))
+
+        if len(symbols) == 0:
+            mx.abort('Could not find any unresolved JVM_* symbols in static JDK libraries')
+        return symbols
+
+    def collect_implementations():
+        impls = set()
+
+        jvm_funcs_path = join(native_project_dir, 'JvmFuncs.c')
+
+        def collect_impls_fn(symbol_prefix):
+            def collector(line):
+                mx.logvv('Processing line: ' + line.rstrip())
+                # JNIEXPORT void JNICALL JVM_DefineModule(JNIEnv *env, jobject module, jboolean is_open, jstring version
+                tokens = line.split()
+                try:
+                    index = tokens.index('JNICALL')
+                    name_part = tokens[index + 1]
+                    if name_part.startswith(symbol_prefix):
+                        impl_name = name_part.split('(')[0].rstrip()
+                        mx.logv('Found matching implementation: ' + impl_name)
+                        impls.add(impl_name)
+                except:
+                    mx.logv('Skipping line: ' + line.rstrip())
+            return collector
+
+        with open(jvm_funcs_path) as f:
+            collector = collect_impls_fn('JVM_')
+            for line in f:
+                collector(line)
+
+        if len(impls) == 0:
+            mx.abort('Could not find any implementations for JVM_* symbols in JvmFuncs.c')
+        return impls
+
+    def write_fallbacks(required_fallbacks):
+        try:
+            new_fallback = StringIO.StringIO()
+            new_fallback.write('/* Fallback implementations autogenerated by mx_substratevm.py */\n\n')
+            new_fallback.write('#include <jni.h>\n')
+            jnienv_function_stub = '''
+JNIEXPORT jobject JNICALL {0}(JNIEnv *env) {{
+    (*env)->FatalError(env, "{0} called:  Unimplemented");
+    return NULL;
+}}
+'''
+            plain_function_stub = '''
+JNIEXPORT void JNICALL {0}() {{
+    fprintf(stderr, "{0} called:  Unimplemented\\n");
+    abort();
+}}
+'''
+            noJNIEnvParam = [
+                'JVM_GC',
+                'JVM_ActiveProcessorCount',
+                'JVM_GetInterfaceVersion',
+                'JVM_GetManagement',
+                'JVM_IsSupportedJNIVersion',
+                'JVM_MaxObjectInspectionAge',
+                'JVM_NativePath',
+                'JVM_ReleaseUTF',
+                'JVM_SupportsCX8',
+                'JVM_BeforeHalt', 'JVM_Halt',
+                'JVM_LoadLibrary', 'JVM_UnloadLibrary', 'JVM_FindLibraryEntry',
+                'JVM_FindSignal', 'JVM_RaiseSignal', 'JVM_RegisterSignal',
+                'JVM_FreeMemory', 'JVM_MaxMemory', 'JVM_TotalMemory',
+                'JVM_RawMonitorCreate', 'JVM_RawMonitorDestroy', 'JVM_RawMonitorEnter', 'JVM_RawMonitorExit'
+            ]
+
+            for name in required_fallbacks:
+                function_stub = plain_function_stub if name in noJNIEnvParam else jnienv_function_stub
+                new_fallback.write(function_stub.format(name))
+
+            native_project_src_gen_dir = join(native_project_dir, 'src_gen')
+            jvm_fallbacks_path = join(native_project_src_gen_dir, 'JvmFuncsFallbacks.c')
+            if exists(jvm_fallbacks_path):
+                with open(jvm_fallbacks_path) as old_fallback:
+                    if old_fallback.read() == new_fallback.getvalue():
+                        return
+
+            mx.ensure_dir_exists(native_project_src_gen_dir)
+            with open(jvm_fallbacks_path, mode='w') as new_fallback_file:
+                new_fallback_file.write(new_fallback.getvalue())
+                mx.log('Updated ' + jvm_fallbacks_path)
+        finally:
+            if new_fallback:
+                new_fallback.close()
+
+    required_fallbacks = collect_missing_symbols() - collect_implementations()
+    write_fallbacks(sorted(required_fallbacks))
 
 def _helloworld(native_image, javac_command, path, args):
     mkpath(path)
@@ -1129,6 +1269,8 @@ def update_if_needed(version_tag, graal_compiler_flags):
     for version_tag in GRAAL_COMPILER_FLAGS_MAP:
         update_if_needed(version_tag, GRAAL_COMPILER_FLAGS_BASE + GRAAL_COMPILER_FLAGS_MAP[version_tag])
 
+    gen_fallbacks()
+
     orig_command_build(args, vm)
 
 

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java

@@ -1494,7 +1494,7 @@ private void processNativeLibraryImports(NativeLibraries nativeLibs, MetaAccessP
             classInitializationSupport.initializeAtBuildTime(clazz, "classes annotated with " + CContext.class.getSimpleName() + " are always initialized");
         }
         for (CLibrary library : loader.findAnnotations(CLibrary.class)) {
-            nativeLibs.addLibrary(library.value(), library.requireStatic());
+            nativeLibs.addAnnotated(library);
         }
 
         nativeLibs.finish();

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/c/NativeLibraries.java

@@ -48,6 +48,7 @@
 import org.graalvm.nativeimage.c.constant.CConstant;
 import org.graalvm.nativeimage.c.constant.CEnum;
 import org.graalvm.nativeimage.c.function.CFunction;
+import org.graalvm.nativeimage.c.function.CLibrary;
 import org.graalvm.nativeimage.c.struct.CPointerTo;
 import org.graalvm.nativeimage.c.struct.CStruct;
 import org.graalvm.nativeimage.c.struct.RawStructure;
@@ -96,6 +97,7 @@ public final class NativeLibraries {
     private final ResolvedJavaType enumType;
     private final ResolvedJavaType locationIdentityType;
 
+    private final LinkedHashSet<CLibrary> annotated;
     private final List<String> libraries;
     private final List<String> staticLibraries;
     private final LinkedHashSet<String> libraryPaths;
@@ -129,6 +131,8 @@ public NativeLibraries(ConstantReflectionProvider constantReflection, MetaAccess
         enumType = metaAccess.lookupJavaType(Enum.class);
         locationIdentityType = metaAccess.lookupJavaType(LocationIdentity.class);
 
+        annotated = new LinkedHashSet<>();
+
         /*
          * Libraries can be added during the static analysis, which runs multi-threaded. So the
          * lists must be synchronized.
@@ -248,6 +252,10 @@ public void loadJavaType(ResolvedJavaType type) {
         }
     }
 
+    public void addAnnotated(CLibrary library) {
+        annotated.add(library);
+    }
+
     public void addLibrary(String library, boolean requireStatic) {
         (requireStatic ? staticLibraries : libraries).add(library);
     }
@@ -418,4 +426,15 @@ public ResolvedJavaType getLocationIdentityType() {
     public ConstantReflectionProvider getConstantReflection() {
         return constantReflection;
     }
+
+    public boolean processAnnotated() {
+        if (annotated.isEmpty()) {
+            return false;
+        }
+        for (CLibrary lib : annotated) {
+            addLibrary(lib.value(), lib.requireStatic());
+        }
+        annotated.clear();
+        return true;
+    }
 }

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/c/codegen/CCompilerInvoker.java

@@ -90,7 +90,7 @@ public Path compileAndParseError(List<String> options, Path source, Path target)
         } catch (InterruptedException ex) {
             throw new InterruptImageBuilding();
         } catch (IOException ex) {
-            throw UserError.abort("Unable to compile C-ABI query code. Make sure GCC toolchain is installed on your system.", ex);
+            throw UserError.abort("Unable to compile C-ABI query code. Make sure native software development toolchain is installed on your system.", ex);
         }
     }
 

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeBootImageViaCC.java

@@ -285,6 +285,12 @@ LinkerInvocation getLinkerInvocation(Path outputDirectory, Path tempDirectory, S
         inv.setOutputFile(outputFile);
         inv.setOutputKind(getOutputKind());
 
+        /*
+         * Libraries defined via @CLibrary annotations are added at the end of the list of libraries
+         * so that the written object file AND the static JDK libraries can depend on them.
+         */
+        nativeLibs.processAnnotated();
+
         inv.addLibPath(tempDirectory.toString());
         for (String libraryPath : nativeLibs.getLibraryPaths()) {
             inv.addLibPath(libraryPath);