Support loading native lib directly from FS (1.4)

CMakeLists.txt

@@ -560,10 +560,10 @@ file(GLOB_RECURSE JAVA_SRC_FILES src/main/java/org/duckdb/*.java)
 file(GLOB_RECURSE JAVA_TEST_FILES src/test/java/org/duckdb/*.java)
 set(CMAKE_JAVA_COMPILE_FLAGS -encoding utf-8 -g -Xlint:all)
 
-add_jar(duckdb_jdbc ${JAVA_SRC_FILES} META-INF/services/java.sql.Driver
+add_jar(duckdb_jdbc_nolib ${JAVA_SRC_FILES} META-INF/services/java.sql.Driver
         MANIFEST META-INF/MANIFEST.MF
         GENERATE_NATIVE_HEADERS duckdb-native)
-add_jar(duckdb_jdbc_tests ${JAVA_TEST_FILES} INCLUDE_JARS duckdb_jdbc)
+add_jar(duckdb_jdbc_tests ${JAVA_TEST_FILES} INCLUDE_JARS duckdb_jdbc_nolib)
 
 
 # main shared lib compilation
@@ -654,7 +654,10 @@ set_target_properties(duckdb_java PROPERTIES PREFIX "lib")
 
 add_custom_command(
   OUTPUT dummy_jdbc_target
-  DEPENDS duckdb_java duckdb_jdbc duckdb_jdbc_tests
+  DEPENDS duckdb_java duckdb_jdbc_nolib duckdb_jdbc_tests
+  COMMAND ${CMAKE_COMMAND} -E copy
+          duckdb_jdbc_nolib.jar
+          duckdb_jdbc.jar
   COMMAND ${Java_JAR_EXECUTABLE} uf duckdb_jdbc.jar -C
           $<TARGET_FILE_DIR:duckdb_java> $<TARGET_FILE_NAME:duckdb_java>)
 

CMakeLists.txt.in

@@ -86,10 +86,10 @@ file(GLOB_RECURSE JAVA_SRC_FILES src/main/java/org/duckdb/*.java)
 file(GLOB_RECURSE JAVA_TEST_FILES src/test/java/org/duckdb/*.java)
 set(CMAKE_JAVA_COMPILE_FLAGS -encoding utf-8 -g -Xlint:all)
 
-add_jar(duckdb_jdbc ${JAVA_SRC_FILES} META-INF/services/java.sql.Driver
+add_jar(duckdb_jdbc_nolib ${JAVA_SRC_FILES} META-INF/services/java.sql.Driver
         MANIFEST META-INF/MANIFEST.MF
         GENERATE_NATIVE_HEADERS duckdb-native)
-add_jar(duckdb_jdbc_tests ${JAVA_TEST_FILES} INCLUDE_JARS duckdb_jdbc)
+add_jar(duckdb_jdbc_tests ${JAVA_TEST_FILES} INCLUDE_JARS duckdb_jdbc_nolib)
 
 
 # main shared lib compilation
@@ -180,7 +180,10 @@ set_target_properties(duckdb_java PROPERTIES PREFIX "lib")
 
 add_custom_command(
   OUTPUT dummy_jdbc_target
-  DEPENDS duckdb_java duckdb_jdbc duckdb_jdbc_tests
+  DEPENDS duckdb_java duckdb_jdbc_nolib duckdb_jdbc_tests
+  COMMAND ${CMAKE_COMMAND} -E copy
+          duckdb_jdbc_nolib.jar
+          duckdb_jdbc.jar
   COMMAND ${Java_JAR_EXECUTABLE} uf duckdb_jdbc.jar -C
           $<TARGET_FILE_DIR:duckdb_java> $<TARGET_FILE_NAME:duckdb_java>)
 

src/main/java/org/duckdb/DuckDBNative.java

@@ -1,66 +1,134 @@
 package org.duckdb;
 
-import java.io.File;
+import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
+
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.math.BigDecimal;
+import java.net.URI;
 import java.net.URL;
 import java.nio.ByteBuffer;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.nio.file.StandardCopyOption;
+import java.security.CodeSource;
+import java.security.ProtectionDomain;
 import java.sql.SQLException;
 import java.util.Properties;
 
 final class DuckDBNative {
+
+    private static final String ARCH_X86_64 = "amd64";
+    private static final String ARCH_AARCH64 = "arm64";
+    private static final String ARCH_UNIVERSAL = "universal";
+
+    private static final String OS_WINDOWS = "windows";
+    private static final String OS_MACOS = "osx";
+    private static final String OS_LINUX = "linux";
+
     static {
         try {
-            String os_name = "";
-            String os_arch;
-            String os_name_detect = System.getProperty("os.name").toLowerCase().trim();
-            String os_arch_detect = System.getProperty("os.arch").toLowerCase().trim();
-            switch (os_arch_detect) {
-            case "x86_64":
-            case "amd64":
-                os_arch = "amd64";
-                break;
-            case "aarch64":
-            case "arm64":
-                os_arch = "arm64";
-                break;
-            case "i386":
-                os_arch = "i386";
-                break;
-            default:
-                throw new IllegalStateException("Unsupported system architecture");
-            }
-            if (os_name_detect.startsWith("windows")) {
-                os_name = "windows";
-            } else if (os_name_detect.startsWith("mac")) {
-                os_name = "osx";
-                os_arch = "universal";
-            } else if (os_name_detect.startsWith("linux")) {
-                os_name = "linux";
-            }
-            String lib_res_name = "/libduckdb_java.so"
-                                  + "_" + os_name + "_" + os_arch;
-
-            Path lib_file = Files.createTempFile("libduckdb_java", ".so");
-            URL lib_res = DuckDBNative.class.getResource(lib_res_name);
-            if (lib_res == null) {
-                System.load(Paths.get("build/debug", lib_res_name).normalize().toAbsolutePath().toString());
-            } else {
-                try (final InputStream lib_res_input_stream = lib_res.openStream()) {
-                    Files.copy(lib_res_input_stream, lib_file, StandardCopyOption.REPLACE_EXISTING);
-                }
-                new File(lib_file.toString()).deleteOnExit();
-                System.load(lib_file.toAbsolutePath().toString());
-            }
-        } catch (IOException e) {
+            loadNativeLibrary();
+        } catch (Exception e) {
             throw new RuntimeException(e);
         }
     }
+
+    private static void loadNativeLibrary() throws Exception {
+        String libName = nativeLibName();
+        URL libRes = DuckDBNative.class.getResource("/" + libName);
+
+        // The current JAR has a native library bundled, in this case we unpack and load it.
+        // There is no fallback if the unpacking or loading fails. We expect that only
+        // the '-nolib' JAR can be used with an external native lib
+        if (null != libRes) {
+            unpackAndLoad(libRes);
+            return;
+        }
+
+        // There is no native library inside the JAR file, so we try to load it by name
+        try {
+            System.loadLibrary("duckdb_java");
+        } catch (UnsatisfiedLinkError e) {
+            // Native library cannot be loaded by name using ordinary JVM mechanisms, we try to load it directly
+            // from FS - from the same directory where the current JAR resides
+            try {
+                loadFromCurrentJarDir(libName);
+            } catch (Throwable t) {
+                e.printStackTrace();
+                throw new IllegalStateException(t);
+            }
+        }
+    }
+
+    private static String cpuArch() {
+        String prop = System.getProperty("os.arch").toLowerCase().trim();
+        switch (prop) {
+        case "x86_64":
+        case "amd64":
+            return ARCH_X86_64;
+        case "aarch64":
+        case "arm64":
+            return ARCH_AARCH64;
+        default:
+            throw new IllegalStateException("Unsupported system architecture: '" + prop + "'");
+        }
+    }
+
+    static String osName() {
+        String prop = System.getProperty("os.name").toLowerCase().trim();
+        if (prop.startsWith("windows")) {
+            return OS_WINDOWS;
+        } else if (prop.startsWith("mac")) {
+            return OS_MACOS;
+        } else if (prop.startsWith("linux")) {
+            return OS_LINUX;
+        } else {
+            throw new IllegalStateException("Unsupported OS: '" + prop + "'");
+        }
+    }
+
+    static String nativeLibName() {
+        String os = osName();
+        final String arch;
+        if (OS_MACOS.equals(os)) {
+            arch = ARCH_UNIVERSAL;
+        } else {
+            arch = cpuArch();
+        }
+        return "libduckdb_java.so_" + os + "_" + arch;
+    }
+
+    static Path currentJarDir() throws Exception {
+        ProtectionDomain pd = DuckDBNative.class.getProtectionDomain();
+        CodeSource cs = pd.getCodeSource();
+        URL loc = cs.getLocation();
+        URI uri = loc.toURI();
+        Path jarPath = Paths.get(uri);
+        Path dirPath = jarPath.getParent();
+        return dirPath.toRealPath();
+    }
+
+    private static void unpackAndLoad(URL nativeLibRes) throws IOException {
+        Path tmpFile = Files.createTempFile("libduckdb_java", ".so");
+        try (InputStream is = nativeLibRes.openStream()) {
+            Files.copy(is, tmpFile, REPLACE_EXISTING);
+        }
+        tmpFile.toFile().deleteOnExit();
+        System.load(tmpFile.toAbsolutePath().toString());
+    }
+
+    private static void loadFromCurrentJarDir(String libName) throws Exception {
+        Path dir = currentJarDir();
+        Path libPath = dir.resolve(libName);
+        if (Files.exists(libPath)) {
+            System.load(libPath.toAbsolutePath().toString());
+        } else {
+            throw new FileNotFoundException("DuckDB JNI library not found, path: '" + libPath.toAbsolutePath() + "'");
+        }
+    }
+
     // We use zero-length ByteBuffer-s as a hacky but cheap way to pass C++ pointers
     // back and forth
 

src/test/java/org/duckdb/TestDuckDBJDBC.java

@@ -3155,11 +3155,12 @@ public static void main(String[] args) throws Exception {
             Class<?> clazz = Class.forName("org.duckdb." + arg1);
             statusCode = runTests(new String[0], clazz);
         } else {
-            statusCode = runTests(args, TestDuckDBJDBC.class, TestAppender.class, TestAppenderCollection.class,
-                                  TestAppenderCollection2D.class, TestAppenderComposite.class,
-                                  TestSingleValueAppender.class, TestBatch.class, TestBindings.class, TestClosure.class,
-                                  TestExtensionTypes.class, TestSpatial.class, TestParameterMetadata.class,
-                                  TestPrepare.class, TestResults.class, TestSessionInit.class, TestTimestamp.class);
+            statusCode =
+                runTests(args, TestDuckDBJDBC.class, TestAppender.class, TestAppenderCollection.class,
+                         TestAppenderCollection2D.class, TestAppenderComposite.class, TestSingleValueAppender.class,
+                         TestBatch.class, TestBindings.class, TestClosure.class, TestExtensionTypes.class,
+                         TestNoLib.class, TestSpatial.class, TestParameterMetadata.class, TestPrepare.class,
+                         TestResults.class, TestSessionInit.class, TestTimestamp.class);
         }
         System.exit(statusCode);
     }

src/test/java/org/duckdb/TestNoLib.java

@@ -0,0 +1,91 @@
+package org.duckdb;
+
+import static java.util.Arrays.asList;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.sql.SQLException;
+import org.duckdb.test.TempDirectory;
+
+public class TestNoLib {
+
+    private static Path javaExe() {
+        String javaHomeProp = System.getProperty("java.home");
+        Path javaHome = Paths.get(javaHomeProp);
+        boolean isWindows = "windows".equals(DuckDBNative.osName());
+        return isWindows ? javaHome.resolve("bin/java.exe") : javaHome.resolve("bin/java");
+    }
+
+    private static void runQuickTest(Path currentJarDir) throws Exception {
+        String dir = currentJarDir.toAbsolutePath().toString();
+        ProcessBuilder pb = new ProcessBuilder(javaExe().toAbsolutePath().toString(),
+                                               "-Djava.library.path=" + currentJarDir.toAbsolutePath(), "-cp",
+                                               dir + File.separator + "duckdb_jdbc_tests.jar" + File.pathSeparator +
+                                                   dir + File.separator + "duckdb_jdbc_nolib.jar",
+                                               "org.duckdb.TestDuckDBJDBC", "test_spatial_POINT_2D")
+                                .inheritIO();
+        int code = pb.start().waitFor();
+        if (0 != code) {
+            throw new RuntimeException("Spawned test failure, code: " + code);
+        }
+    }
+
+    private static String platformLibName() throws Exception {
+        String os = DuckDBNative.osName();
+        switch (os) {
+        case "windows":
+            return "duckdb_java.dll";
+        case "osx":
+            return "libduckdb_java.dylib";
+        case "linux":
+            return "libduckdb_java.so";
+        default:
+            throw new SQLException("Unsupported OS: " + os);
+        }
+    }
+    private static Path nativeLibPathInBuildTree(Path buildDir) throws SQLException {
+        String libName = DuckDBNative.nativeLibName();
+        Path libPath = buildDir.resolve(libName);
+        if (Files.exists(libPath)) {
+            return libPath;
+        }
+        for (String subdirName : asList("Release", "Debug", "RelWithDebInfo")) {
+            Path dir = buildDir.resolve(subdirName);
+            Path libPathSubdir = dir.resolve(libName);
+            if (Files.exists(libPathSubdir)) {
+                return libPathSubdir;
+            }
+        }
+        throw new SQLException("Native lib not found in build tree, name: '" + libName + "'");
+    }
+
+    public static void test_nolib_next_to_jar() throws Exception {
+        try (TempDirectory td = new TempDirectory()) {
+            Path dir = DuckDBNative.currentJarDir();
+            Path nativeLib = nativeLibPathInBuildTree(dir);
+            Files.copy(dir.resolve("duckdb_jdbc_nolib.jar"), td.path().resolve("duckdb_jdbc_nolib.jar"));
+            Files.copy(dir.resolve("duckdb_jdbc_tests.jar"), td.path().resolve("duckdb_jdbc_tests.jar"));
+            Files.copy(nativeLib, td.path().resolve(nativeLib.getFileName()));
+            System.out.println();
+            System.out.println("----");
+            runQuickTest(td.path());
+            System.out.println("----");
+        }
+    }
+
+    public static void test_nolib_by_name() throws Exception {
+        try (TempDirectory td = new TempDirectory()) {
+            Path dir = DuckDBNative.currentJarDir();
+            Path nativeLib = nativeLibPathInBuildTree(dir);
+            Files.copy(dir.resolve("duckdb_jdbc_nolib.jar"), td.path().resolve("duckdb_jdbc_nolib.jar"));
+            Files.copy(dir.resolve("duckdb_jdbc_tests.jar"), td.path().resolve("duckdb_jdbc_tests.jar"));
+            Files.copy(nativeLib, td.path().resolve(platformLibName()));
+            System.out.println();
+            System.out.println("----");
+            runQuickTest(td.path());
+            System.out.println("----");
+        }
+    }
+}

src/test/java/org/duckdb/test/Runner.java

@@ -40,7 +40,7 @@ public static int runTests(String[] args, Class<?>... testClasses) {
         boolean anyFailed = false;
         for (Method m : methods) {
             if (m.getName().startsWith("test_")) {
-                if (quick_run && m.getName().startsWith("test_lots_")) {
+                if (quick_run && (m.getName().startsWith("test_lots_") || m.getName().startsWith("test_nolib_"))) {
                     continue;
                 }
                 if (specific_test != null && !m.getName().contains(specific_test)) {