diff --git a/CMakeLists.txt b/CMakeLists.txt index 7153f8424..732b2dea3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -560,10 +560,10 @@ file(GLOB_RECURSE JAVA_SRC_FILES src/main/java/org/duckdb/*.java) file(GLOB_RECURSE JAVA_TEST_FILES src/test/java/org/duckdb/*.java) set(CMAKE_JAVA_COMPILE_FLAGS -encoding utf-8 -g -Xlint:all) -add_jar(duckdb_jdbc ${JAVA_SRC_FILES} META-INF/services/java.sql.Driver +add_jar(duckdb_jdbc_nolib ${JAVA_SRC_FILES} META-INF/services/java.sql.Driver MANIFEST META-INF/MANIFEST.MF GENERATE_NATIVE_HEADERS duckdb-native) -add_jar(duckdb_jdbc_tests ${JAVA_TEST_FILES} INCLUDE_JARS duckdb_jdbc) +add_jar(duckdb_jdbc_tests ${JAVA_TEST_FILES} INCLUDE_JARS duckdb_jdbc_nolib) # main shared lib compilation @@ -654,7 +654,10 @@ set_target_properties(duckdb_java PROPERTIES PREFIX "lib") add_custom_command( OUTPUT dummy_jdbc_target - DEPENDS duckdb_java duckdb_jdbc duckdb_jdbc_tests + DEPENDS duckdb_java duckdb_jdbc_nolib duckdb_jdbc_tests + COMMAND ${CMAKE_COMMAND} -E copy + duckdb_jdbc_nolib.jar + duckdb_jdbc.jar COMMAND ${Java_JAR_EXECUTABLE} uf duckdb_jdbc.jar -C $ $) diff --git a/CMakeLists.txt.in b/CMakeLists.txt.in index b5ec906bf..de3f157e6 100644 --- a/CMakeLists.txt.in +++ b/CMakeLists.txt.in @@ -86,10 +86,10 @@ file(GLOB_RECURSE JAVA_SRC_FILES src/main/java/org/duckdb/*.java) file(GLOB_RECURSE JAVA_TEST_FILES src/test/java/org/duckdb/*.java) set(CMAKE_JAVA_COMPILE_FLAGS -encoding utf-8 -g -Xlint:all) -add_jar(duckdb_jdbc ${JAVA_SRC_FILES} META-INF/services/java.sql.Driver +add_jar(duckdb_jdbc_nolib ${JAVA_SRC_FILES} META-INF/services/java.sql.Driver MANIFEST META-INF/MANIFEST.MF GENERATE_NATIVE_HEADERS duckdb-native) -add_jar(duckdb_jdbc_tests ${JAVA_TEST_FILES} INCLUDE_JARS duckdb_jdbc) +add_jar(duckdb_jdbc_tests ${JAVA_TEST_FILES} INCLUDE_JARS duckdb_jdbc_nolib) # main shared lib compilation @@ -180,7 +180,10 @@ set_target_properties(duckdb_java PROPERTIES PREFIX "lib") add_custom_command( OUTPUT dummy_jdbc_target - DEPENDS duckdb_java duckdb_jdbc duckdb_jdbc_tests + DEPENDS duckdb_java duckdb_jdbc_nolib duckdb_jdbc_tests + COMMAND ${CMAKE_COMMAND} -E copy + duckdb_jdbc_nolib.jar + duckdb_jdbc.jar COMMAND ${Java_JAR_EXECUTABLE} uf duckdb_jdbc.jar -C $ $) diff --git a/src/main/java/org/duckdb/DuckDBNative.java b/src/main/java/org/duckdb/DuckDBNative.java index 25bc92b5f..c334526bb 100644 --- a/src/main/java/org/duckdb/DuckDBNative.java +++ b/src/main/java/org/duckdb/DuckDBNative.java @@ -1,66 +1,134 @@ package org.duckdb; -import java.io.File; +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; + +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.math.BigDecimal; +import java.net.URI; import java.net.URL; import java.nio.ByteBuffer; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; +import java.security.CodeSource; +import java.security.ProtectionDomain; import java.sql.SQLException; import java.util.Properties; final class DuckDBNative { + + private static final String ARCH_X86_64 = "amd64"; + private static final String ARCH_AARCH64 = "arm64"; + private static final String ARCH_UNIVERSAL = "universal"; + + private static final String OS_WINDOWS = "windows"; + private static final String OS_MACOS = "osx"; + private static final String OS_LINUX = "linux"; + static { try { - String os_name = ""; - String os_arch; - String os_name_detect = System.getProperty("os.name").toLowerCase().trim(); - String os_arch_detect = System.getProperty("os.arch").toLowerCase().trim(); - switch (os_arch_detect) { - case "x86_64": - case "amd64": - os_arch = "amd64"; - break; - case "aarch64": - case "arm64": - os_arch = "arm64"; - break; - case "i386": - os_arch = "i386"; - break; - default: - throw new IllegalStateException("Unsupported system architecture"); - } - if (os_name_detect.startsWith("windows")) { - os_name = "windows"; - } else if (os_name_detect.startsWith("mac")) { - os_name = "osx"; - os_arch = "universal"; - } else if (os_name_detect.startsWith("linux")) { - os_name = "linux"; - } - String lib_res_name = "/libduckdb_java.so" - + "_" + os_name + "_" + os_arch; - - Path lib_file = Files.createTempFile("libduckdb_java", ".so"); - URL lib_res = DuckDBNative.class.getResource(lib_res_name); - if (lib_res == null) { - System.load(Paths.get("build/debug", lib_res_name).normalize().toAbsolutePath().toString()); - } else { - try (final InputStream lib_res_input_stream = lib_res.openStream()) { - Files.copy(lib_res_input_stream, lib_file, StandardCopyOption.REPLACE_EXISTING); - } - new File(lib_file.toString()).deleteOnExit(); - System.load(lib_file.toAbsolutePath().toString()); - } - } catch (IOException e) { + loadNativeLibrary(); + } catch (Exception e) { throw new RuntimeException(e); } } + + private static void loadNativeLibrary() throws Exception { + String libName = nativeLibName(); + URL libRes = DuckDBNative.class.getResource("/" + libName); + + // The current JAR has a native library bundled, in this case we unpack and load it. + // There is no fallback if the unpacking or loading fails. We expect that only + // the '-nolib' JAR can be used with an external native lib + if (null != libRes) { + unpackAndLoad(libRes); + return; + } + + // There is no native library inside the JAR file, so we try to load it by name + try { + System.loadLibrary("duckdb_java"); + } catch (UnsatisfiedLinkError e) { + // Native library cannot be loaded by name using ordinary JVM mechanisms, we try to load it directly + // from FS - from the same directory where the current JAR resides + try { + loadFromCurrentJarDir(libName); + } catch (Throwable t) { + e.printStackTrace(); + throw new IllegalStateException(t); + } + } + } + + private static String cpuArch() { + String prop = System.getProperty("os.arch").toLowerCase().trim(); + switch (prop) { + case "x86_64": + case "amd64": + return ARCH_X86_64; + case "aarch64": + case "arm64": + return ARCH_AARCH64; + default: + throw new IllegalStateException("Unsupported system architecture: '" + prop + "'"); + } + } + + static String osName() { + String prop = System.getProperty("os.name").toLowerCase().trim(); + if (prop.startsWith("windows")) { + return OS_WINDOWS; + } else if (prop.startsWith("mac")) { + return OS_MACOS; + } else if (prop.startsWith("linux")) { + return OS_LINUX; + } else { + throw new IllegalStateException("Unsupported OS: '" + prop + "'"); + } + } + + static String nativeLibName() { + String os = osName(); + final String arch; + if (OS_MACOS.equals(os)) { + arch = ARCH_UNIVERSAL; + } else { + arch = cpuArch(); + } + return "libduckdb_java.so_" + os + "_" + arch; + } + + static Path currentJarDir() throws Exception { + ProtectionDomain pd = DuckDBNative.class.getProtectionDomain(); + CodeSource cs = pd.getCodeSource(); + URL loc = cs.getLocation(); + URI uri = loc.toURI(); + Path jarPath = Paths.get(uri); + Path dirPath = jarPath.getParent(); + return dirPath.toRealPath(); + } + + private static void unpackAndLoad(URL nativeLibRes) throws IOException { + Path tmpFile = Files.createTempFile("libduckdb_java", ".so"); + try (InputStream is = nativeLibRes.openStream()) { + Files.copy(is, tmpFile, REPLACE_EXISTING); + } + tmpFile.toFile().deleteOnExit(); + System.load(tmpFile.toAbsolutePath().toString()); + } + + private static void loadFromCurrentJarDir(String libName) throws Exception { + Path dir = currentJarDir(); + Path libPath = dir.resolve(libName); + if (Files.exists(libPath)) { + System.load(libPath.toAbsolutePath().toString()); + } else { + throw new FileNotFoundException("DuckDB JNI library not found, path: '" + libPath.toAbsolutePath() + "'"); + } + } + // We use zero-length ByteBuffer-s as a hacky but cheap way to pass C++ pointers // back and forth diff --git a/src/test/java/org/duckdb/TestDuckDBJDBC.java b/src/test/java/org/duckdb/TestDuckDBJDBC.java index 674a90696..f09a45990 100644 --- a/src/test/java/org/duckdb/TestDuckDBJDBC.java +++ b/src/test/java/org/duckdb/TestDuckDBJDBC.java @@ -3155,11 +3155,12 @@ public static void main(String[] args) throws Exception { Class clazz = Class.forName("org.duckdb." + arg1); statusCode = runTests(new String[0], clazz); } else { - statusCode = runTests(args, TestDuckDBJDBC.class, TestAppender.class, TestAppenderCollection.class, - TestAppenderCollection2D.class, TestAppenderComposite.class, - TestSingleValueAppender.class, TestBatch.class, TestBindings.class, TestClosure.class, - TestExtensionTypes.class, TestSpatial.class, TestParameterMetadata.class, - TestPrepare.class, TestResults.class, TestSessionInit.class, TestTimestamp.class); + statusCode = + runTests(args, TestDuckDBJDBC.class, TestAppender.class, TestAppenderCollection.class, + TestAppenderCollection2D.class, TestAppenderComposite.class, TestSingleValueAppender.class, + TestBatch.class, TestBindings.class, TestClosure.class, TestExtensionTypes.class, + TestNoLib.class, TestSpatial.class, TestParameterMetadata.class, TestPrepare.class, + TestResults.class, TestSessionInit.class, TestTimestamp.class); } System.exit(statusCode); } diff --git a/src/test/java/org/duckdb/TestNoLib.java b/src/test/java/org/duckdb/TestNoLib.java new file mode 100644 index 000000000..442c3d2fe --- /dev/null +++ b/src/test/java/org/duckdb/TestNoLib.java @@ -0,0 +1,91 @@ +package org.duckdb; + +import static java.util.Arrays.asList; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.SQLException; +import org.duckdb.test.TempDirectory; + +public class TestNoLib { + + private static Path javaExe() { + String javaHomeProp = System.getProperty("java.home"); + Path javaHome = Paths.get(javaHomeProp); + boolean isWindows = "windows".equals(DuckDBNative.osName()); + return isWindows ? javaHome.resolve("bin/java.exe") : javaHome.resolve("bin/java"); + } + + private static void runQuickTest(Path currentJarDir) throws Exception { + String dir = currentJarDir.toAbsolutePath().toString(); + ProcessBuilder pb = new ProcessBuilder(javaExe().toAbsolutePath().toString(), + "-Djava.library.path=" + currentJarDir.toAbsolutePath(), "-cp", + dir + File.separator + "duckdb_jdbc_tests.jar" + File.pathSeparator + + dir + File.separator + "duckdb_jdbc_nolib.jar", + "org.duckdb.TestDuckDBJDBC", "test_spatial_POINT_2D") + .inheritIO(); + int code = pb.start().waitFor(); + if (0 != code) { + throw new RuntimeException("Spawned test failure, code: " + code); + } + } + + private static String platformLibName() throws Exception { + String os = DuckDBNative.osName(); + switch (os) { + case "windows": + return "duckdb_java.dll"; + case "osx": + return "libduckdb_java.dylib"; + case "linux": + return "libduckdb_java.so"; + default: + throw new SQLException("Unsupported OS: " + os); + } + } + private static Path nativeLibPathInBuildTree(Path buildDir) throws SQLException { + String libName = DuckDBNative.nativeLibName(); + Path libPath = buildDir.resolve(libName); + if (Files.exists(libPath)) { + return libPath; + } + for (String subdirName : asList("Release", "Debug", "RelWithDebInfo")) { + Path dir = buildDir.resolve(subdirName); + Path libPathSubdir = dir.resolve(libName); + if (Files.exists(libPathSubdir)) { + return libPathSubdir; + } + } + throw new SQLException("Native lib not found in build tree, name: '" + libName + "'"); + } + + public static void test_nolib_next_to_jar() throws Exception { + try (TempDirectory td = new TempDirectory()) { + Path dir = DuckDBNative.currentJarDir(); + Path nativeLib = nativeLibPathInBuildTree(dir); + Files.copy(dir.resolve("duckdb_jdbc_nolib.jar"), td.path().resolve("duckdb_jdbc_nolib.jar")); + Files.copy(dir.resolve("duckdb_jdbc_tests.jar"), td.path().resolve("duckdb_jdbc_tests.jar")); + Files.copy(nativeLib, td.path().resolve(nativeLib.getFileName())); + System.out.println(); + System.out.println("----"); + runQuickTest(td.path()); + System.out.println("----"); + } + } + + public static void test_nolib_by_name() throws Exception { + try (TempDirectory td = new TempDirectory()) { + Path dir = DuckDBNative.currentJarDir(); + Path nativeLib = nativeLibPathInBuildTree(dir); + Files.copy(dir.resolve("duckdb_jdbc_nolib.jar"), td.path().resolve("duckdb_jdbc_nolib.jar")); + Files.copy(dir.resolve("duckdb_jdbc_tests.jar"), td.path().resolve("duckdb_jdbc_tests.jar")); + Files.copy(nativeLib, td.path().resolve(platformLibName())); + System.out.println(); + System.out.println("----"); + runQuickTest(td.path()); + System.out.println("----"); + } + } +} diff --git a/src/test/java/org/duckdb/test/Runner.java b/src/test/java/org/duckdb/test/Runner.java index 49c853eca..a6e10b27b 100644 --- a/src/test/java/org/duckdb/test/Runner.java +++ b/src/test/java/org/duckdb/test/Runner.java @@ -40,7 +40,7 @@ public static int runTests(String[] args, Class... testClasses) { boolean anyFailed = false; for (Method m : methods) { if (m.getName().startsWith("test_")) { - if (quick_run && m.getName().startsWith("test_lots_")) { + if (quick_run && (m.getName().startsWith("test_lots_") || m.getName().startsWith("test_nolib_"))) { continue; } if (specific_test != null && !m.getName().contains(specific_test)) {