From 15f040e4b668c250ce6d1e2a169a71b8fc334647 Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Thu, 7 May 2026 11:52:56 +0200 Subject: [PATCH 1/4] Add a test for missing delvewheel --- .../integration/advanced/NativeExtTest.java | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/graalpython/com.oracle.graal.python.test.integration/src/com/oracle/graal/python/test/integration/advanced/NativeExtTest.java b/graalpython/com.oracle.graal.python.test.integration/src/com/oracle/graal/python/test/integration/advanced/NativeExtTest.java index c9322f6b0b..98f50bc176 100644 --- a/graalpython/com.oracle.graal.python.test.integration/src/com/oracle/graal/python/test/integration/advanced/NativeExtTest.java +++ b/graalpython/com.oracle.graal.python.test.integration/src/com/oracle/graal/python/test/integration/advanced/NativeExtTest.java @@ -40,11 +40,15 @@ */ package com.oracle.graal.python.test.integration.advanced; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.concurrent.CountDownLatch; import org.graalvm.polyglot.Context; import org.graalvm.polyglot.Engine; import org.graalvm.polyglot.PolyglotException; +import org.graalvm.polyglot.Value; import org.junit.Assert; import org.junit.Assume; import org.junit.BeforeClass; @@ -109,6 +113,33 @@ public void testSharingErrorWithCpythonSre() throws InterruptedException { } } + @Test + public void testMissingDelvewheelError() throws IOException { + Assume.assumeTrue(System.getProperty("os.name").toLowerCase().contains("win")); + + Path tempDir = Files.createTempDirectory("graalpy-no-delvewheel"); + try (Engine engine = Engine.create("python"); + Context context = newContext(engine) + .option("python.IsolateNativeModules", "true") + .option("python.Executable", tempDir.resolve("python.exe").toString()) + .environment("PATH", tempDir.toString()) + .build()) { + try { + context.eval("python", "import _sqlite3"); + Assert.fail("importing _sqlite3 with python.IsolateNativeModules=true should fail when delvewheel is not on PATH"); + } catch (PolyglotException ex) { + Assert.assertTrue(ex.getMessage(), ex.isGuestException()); + Value exception = ex.getGuestObject(); + Assert.assertTrue(exception.isException()); + Assert.assertEquals("SystemError", exception.getMetaObject().getMetaSimpleName()); + Assert.assertTrue(ex.getMessage(), ex.getMessage().contains("delvewheel` 1.9.0")); + Assert.assertTrue(ex.getMessage(), ex.getMessage().contains("installed")); + } + } finally { + Files.deleteIfExists(tempDir); + } + } + private static Context.Builder newContext(Engine engine) { return Context.newBuilder().allowExperimentalOptions(true).allowAllAccess(true).engine(engine); } From e9f95341ce9835e81f77ec00a0cacf67f9668918 Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Mon, 11 May 2026 16:12:58 +0200 Subject: [PATCH 2/4] Improve delvewheel error reporting --- .../integration/advanced/NativeExtTest.java | 6 +++-- .../cext/copying/NativeLibraryLocator.java | 12 +++------- .../builtins/objects/cext/copying/PEFile.java | 23 +++++++++++++++---- 3 files changed, 26 insertions(+), 15 deletions(-) diff --git a/graalpython/com.oracle.graal.python.test.integration/src/com/oracle/graal/python/test/integration/advanced/NativeExtTest.java b/graalpython/com.oracle.graal.python.test.integration/src/com/oracle/graal/python/test/integration/advanced/NativeExtTest.java index 98f50bc176..5b365a99af 100644 --- a/graalpython/com.oracle.graal.python.test.integration/src/com/oracle/graal/python/test/integration/advanced/NativeExtTest.java +++ b/graalpython/com.oracle.graal.python.test.integration/src/com/oracle/graal/python/test/integration/advanced/NativeExtTest.java @@ -59,6 +59,8 @@ * because we cannot create multiple contexts that would load native extensions. */ public class NativeExtTest { + private static final String DELVEWHEEL_VERSION = "1.9.0"; + @BeforeClass public static void setUpClass() { Assume.assumeFalse(System.getProperty("os.name").toLowerCase().contains("mac")); @@ -131,8 +133,8 @@ public void testMissingDelvewheelError() throws IOException { Assert.assertTrue(ex.getMessage(), ex.isGuestException()); Value exception = ex.getGuestObject(); Assert.assertTrue(exception.isException()); - Assert.assertEquals("SystemError", exception.getMetaObject().getMetaSimpleName()); - Assert.assertTrue(ex.getMessage(), ex.getMessage().contains("delvewheel` 1.9.0")); + Assert.assertEquals(ex.getMessage(), "SystemError", exception.getMetaObject().getMetaSimpleName()); + Assert.assertTrue(ex.getMessage(), ex.getMessage().contains("delvewheel` " + DELVEWHEEL_VERSION)); Assert.assertTrue(ex.getMessage(), ex.getMessage().contains("installed")); } } finally { diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/copying/NativeLibraryLocator.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/copying/NativeLibraryLocator.java index 4366027daf..f0a401f112 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/copying/NativeLibraryLocator.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/copying/NativeLibraryLocator.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -51,7 +51,6 @@ import com.oracle.graal.python.PythonLanguage; import com.oracle.graal.python.builtins.objects.cext.common.LoadCExtException.ApiInitException; -import com.oracle.graal.python.builtins.objects.cext.common.LoadCExtException.ImportException; import com.oracle.graal.python.nodes.ErrorMessages; import com.oracle.graal.python.nodes.util.CannotCastException; import com.oracle.graal.python.nodes.util.CastToTruffleStringNode; @@ -127,13 +126,8 @@ public NativeLibraryLocator(PythonContext context, TruffleFile capiLibrary, bool * * @see PythonOptions#IsolateNativeModules */ - public String resolve(PythonContext context, TruffleFile original) throws ImportException { - try { - return resolve(context, original, capiSlot, capiOriginal); - } catch (ApiInitException e) { - throw new ImportException(null, toTruffleStringUncached(original.getName()), toTruffleStringUncached(original.getPath()), - toTruffleStringUncached(e.getMessage() == null ? "" : e.getMessage())); - } + public String resolve(PythonContext context, TruffleFile original) throws ApiInitException { + return resolve(context, original, capiSlot, capiOriginal); } public String getCapiLibrary() { diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/copying/PEFile.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/copying/PEFile.java index 63cbd39ea3..57b475243e 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/copying/PEFile.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/copying/PEFile.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -49,6 +49,10 @@ import com.oracle.truffle.api.TruffleFile; final class PEFile extends SharedObject { + private static final String DELVEWHEEL_VERSION = "1.9.0"; + private static final String DELVEWHEEL_INSTALL_INSTRUCTION = "IsolateNativeModules option needs `delvewheel` tool to copy libraries. Make sure you have `delvewheel==" + DELVEWHEEL_VERSION + + "` available on PATH."; + private final PythonContext context; private final TruffleFile tempfile; @@ -65,7 +69,7 @@ public void setId(String newId) throws IOException { // TODO } - private String getDelvewheelPython() { + private String getDelvewheelPython() throws IOException { TruffleFile delvewheel = which(context, "delvewheel.exe"); if (!delvewheel.exists()) { delvewheel = which(context, "delvewheel.bat"); @@ -73,6 +77,9 @@ private String getDelvewheelPython() { if (!delvewheel.exists()) { delvewheel = which(context, "delvewheel.cmd"); } + if (!delvewheel.exists()) { + throw new IOException("Could not find `delvewheel` on PATH. " + DELVEWHEEL_INSTALL_INSTRUCTION); + } TruffleFile python = delvewheel.resolveSibling("python.exe"); if (!python.exists()) { python = delvewheel.resolveSibling("python.bat"); @@ -89,6 +96,9 @@ private String getDelvewheelPython() { if (!python.exists()) { python = delvewheel.getParent().resolveSibling("python.cmd"); } + if (!python.exists()) { + throw new IOException("Could not find Python executable next to `delvewheel` at '" + delvewheel + "'. " + DELVEWHEEL_INSTALL_INSTRUCTION); + } return python.toString(); } @@ -100,9 +110,14 @@ public void changeOrAddDependency(String oldName, String newName) throws IOExcep pb.command(pythonExe, "-c", String.format("from delvewheel import _dll_utils; _dll_utils.replace_needed('%s', ['%s'], {'%s': '%s'}, strip=True, verbose=2, test=[])", tempfileWithForwardSlashes, oldName, oldName, newName)); - var proc = pb.start(); + Process proc; + try { + proc = pb.start(); + } catch (IOException e) { + throw new IOException("Failed to start `delvewheel` to copy required DLL: " + e.getMessage() + ". " + DELVEWHEEL_INSTALL_INSTRUCTION, e); + } if (proc.waitFor() != 0) { - throw new IOException("Failed to run `delvewheel` 1.9.0 to copy required DLL. Make sure you have it installed in your venv."); + throw new IOException("Failed to run `delvewheel` to copy required DLL (exit code " + proc.exitValue() + "). " + DELVEWHEEL_INSTALL_INSTRUCTION); } } From 159247ed378cc4658ad7fc7b4df9a0a2f2cab312 Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Wed, 13 May 2026 16:41:55 +0200 Subject: [PATCH 3/4] Improve error handling for patchelf --- .../integration/advanced/NativeExtTest.java | 31 ++++++++++-- .../objects/cext/copying/ElfFile.java | 50 +++++++++++-------- 2 files changed, 56 insertions(+), 25 deletions(-) diff --git a/graalpython/com.oracle.graal.python.test.integration/src/com/oracle/graal/python/test/integration/advanced/NativeExtTest.java b/graalpython/com.oracle.graal.python.test.integration/src/com/oracle/graal/python/test/integration/advanced/NativeExtTest.java index 5b365a99af..ed7213ef9f 100644 --- a/graalpython/com.oracle.graal.python.test.integration/src/com/oracle/graal/python/test/integration/advanced/NativeExtTest.java +++ b/graalpython/com.oracle.graal.python.test.integration/src/com/oracle/graal/python/test/integration/advanced/NativeExtTest.java @@ -121,11 +121,8 @@ public void testMissingDelvewheelError() throws IOException { Path tempDir = Files.createTempDirectory("graalpy-no-delvewheel"); try (Engine engine = Engine.create("python"); - Context context = newContext(engine) - .option("python.IsolateNativeModules", "true") - .option("python.Executable", tempDir.resolve("python.exe").toString()) - .environment("PATH", tempDir.toString()) - .build()) { + Context context = newContext(engine).option("python.IsolateNativeModules", "true").option("python.Executable", tempDir.resolve("python.exe").toString()).environment("PATH", + tempDir.toString()).build()) { try { context.eval("python", "import _sqlite3"); Assert.fail("importing _sqlite3 with python.IsolateNativeModules=true should fail when delvewheel is not on PATH"); @@ -142,6 +139,30 @@ public void testMissingDelvewheelError() throws IOException { } } + @Test + public void testMissingPatchelfError() throws IOException { + Assume.assumeTrue(System.getProperty("os.name").toLowerCase().contains("linux")); + + Path tempDir = Files.createTempDirectory("graalpy-no-patchelf"); + try (Engine engine = Engine.create("python"); + Context context = newContext(engine).option("python.IsolateNativeModules", "true").option("python.Executable", tempDir.resolve("python").toString()).environment("PATH", + tempDir.toString()).build()) { + try { + context.eval("python", "import _sqlite3"); + Assert.fail("importing _sqlite3 with python.IsolateNativeModules=true should fail when patchelf is not on PATH"); + } catch (PolyglotException ex) { + Assert.assertTrue(ex.getMessage(), ex.isGuestException()); + Value exception = ex.getGuestObject(); + Assert.assertTrue(exception.isException()); + Assert.assertEquals(ex.getMessage(), "SystemError", exception.getMetaObject().getMetaSimpleName()); + Assert.assertTrue(ex.getMessage(), ex.getMessage().contains("patchelf`")); + Assert.assertTrue(ex.getMessage(), ex.getMessage().contains("installed")); + } + } finally { + Files.deleteIfExists(tempDir); + } + } + private static Context.Builder newContext(Engine engine) { return Context.newBuilder().allowExperimentalOptions(true).allowAllAccess(true).engine(engine); } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/copying/ElfFile.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/copying/ElfFile.java index ee6c5c0e65..c51de60ed7 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/copying/ElfFile.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/copying/ElfFile.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -49,11 +49,35 @@ import com.oracle.truffle.api.TruffleFile; final class ElfFile extends SharedObject { + private static final String PATCHELF_INSTALL_INSTRUCTION = "IsolateNativeModules option needs `patchelf` tool to copy libraries. Make sure you have it available " + + "on PATH or installed in your venv."; + private final PythonContext context; private final TruffleFile tempfile; - private String getPatchelf() { - return which(context, "patchelf").toString(); + private String getPatchelf() throws IOException { + TruffleFile patchelf = which(context, "patchelf"); + if (!patchelf.exists()) { + throw new IOException("Could not find `patchelf` on PATH. " + PATCHELF_INSTALL_INSTRUCTION); + } + return patchelf.toString(); + } + + private void runPatchelf(String action, String... arguments) throws IOException, InterruptedException { + var command = new String[arguments.length + 1]; + command[0] = getPatchelf(); + System.arraycopy(arguments, 0, command, 1, arguments.length); + var pb = newProcessBuilder(context); + pb.command(command); + Process proc; + try { + proc = pb.start(); + } catch (IOException e) { + throw new IOException("Failed to start `patchelf` to " + action + ": " + e.getMessage() + ". " + PATCHELF_INSTALL_INSTRUCTION, e); + } + if (proc.waitFor() != 0) { + throw new IOException("Failed to run `patchelf` to " + action + " (exit code " + proc.exitValue() + "). " + PATCHELF_INSTALL_INSTRUCTION); + } } ElfFile(byte[] b, PythonContext context) throws IOException { @@ -66,27 +90,13 @@ private String getPatchelf() { @Override public void setId(String newId) throws IOException, InterruptedException { - var pb = newProcessBuilder(context); - pb.command(getPatchelf(), "--debug", "--set-soname", newId, tempfile.toString()); - var proc = pb.start(); - if (proc.waitFor() != 0) { - throw new IOException("Failed to run `patchelf` command. Make sure you have it on your PATH or installed in your venv."); - } + runPatchelf("set SONAME", "--debug", "--set-soname", newId, tempfile.toString()); } @Override public void changeOrAddDependency(String oldName, String newName) throws IOException, InterruptedException { - var pb = newProcessBuilder(context); - pb.command(getPatchelf(), "--debug", "--remove-needed", oldName, tempfile.toString()); - var proc = pb.start(); - if (proc.waitFor() != 0) { - throw new IOException("Failed to run `patchelf` command. Make sure you have it on your PATH or installed in your venv."); - } - pb.command(getPatchelf(), "--debug", "--add-needed", newName, tempfile.toString()); - proc = pb.start(); - if (proc.waitFor() != 0) { - throw new IOException("Failed to run `patchelf` command. Make sure you have it on your PATH or installed in your venv."); - } + runPatchelf("remove dependency", "--debug", "--remove-needed", oldName, tempfile.toString()); + runPatchelf("add dependency", "--debug", "--add-needed", newName, tempfile.toString()); } @Override From d23feeaf5e1146b9480aa82a74253b26ba2ecdcc Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Thu, 14 May 2026 06:33:33 +0200 Subject: [PATCH 4/4] Refactor patchelf/delvewheel error handling --- .../integration/advanced/NativeExtTest.java | 4 +- .../modules/GraalPythonModuleBuiltins.java | 3 +- .../objects/cext/copying/ElfFile.java | 83 +++++++++++++++---- .../cext/copying/NativeLibraryLocator.java | 43 ++++------ .../copying/NativeLibraryToolException.java | 53 ++++++++++++ .../builtins/objects/cext/copying/PEFile.java | 63 ++++++++++---- .../objects/cext/copying/SharedObject.java | 23 +++-- 7 files changed, 202 insertions(+), 70 deletions(-) create mode 100644 graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/copying/NativeLibraryToolException.java diff --git a/graalpython/com.oracle.graal.python.test.integration/src/com/oracle/graal/python/test/integration/advanced/NativeExtTest.java b/graalpython/com.oracle.graal.python.test.integration/src/com/oracle/graal/python/test/integration/advanced/NativeExtTest.java index ed7213ef9f..1dc193ed0d 100644 --- a/graalpython/com.oracle.graal.python.test.integration/src/com/oracle/graal/python/test/integration/advanced/NativeExtTest.java +++ b/graalpython/com.oracle.graal.python.test.integration/src/com/oracle/graal/python/test/integration/advanced/NativeExtTest.java @@ -131,8 +131,7 @@ public void testMissingDelvewheelError() throws IOException { Value exception = ex.getGuestObject(); Assert.assertTrue(exception.isException()); Assert.assertEquals(ex.getMessage(), "SystemError", exception.getMetaObject().getMetaSimpleName()); - Assert.assertTrue(ex.getMessage(), ex.getMessage().contains("delvewheel` " + DELVEWHEEL_VERSION)); - Assert.assertTrue(ex.getMessage(), ex.getMessage().contains("installed")); + Assert.assertTrue(ex.getMessage(), ex.getMessage().contains("delvewheel==" + DELVEWHEEL_VERSION)); } } finally { Files.deleteIfExists(tempDir); @@ -156,7 +155,6 @@ public void testMissingPatchelfError() throws IOException { Assert.assertTrue(exception.isException()); Assert.assertEquals(ex.getMessage(), "SystemError", exception.getMetaObject().getMetaSimpleName()); Assert.assertTrue(ex.getMessage(), ex.getMessage().contains("patchelf`")); - Assert.assertTrue(ex.getMessage(), ex.getMessage().contains("installed")); } } finally { Files.deleteIfExists(tempDir); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java index 919f463a0a..8af0696915 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java @@ -110,6 +110,7 @@ import com.oracle.graal.python.builtins.objects.cext.capi.transitions.CApiTransitions.HandlePointerConverter; import com.oracle.graal.python.builtins.objects.cext.capi.transitions.CApiTransitions.PythonObjectReference; import com.oracle.graal.python.builtins.objects.cext.copying.NativeLibraryLocator; +import com.oracle.graal.python.builtins.objects.cext.copying.NativeLibraryToolException; import com.oracle.graal.python.builtins.objects.cext.structs.CFields; import com.oracle.graal.python.builtins.objects.cext.structs.CStructAccess; import com.oracle.graal.python.builtins.objects.code.CodeNodes; @@ -1408,7 +1409,7 @@ static Object replicate(TruffleString venvPath, int count, @Bind PythonContext context) { try { NativeLibraryLocator.replicate(context.getEnv().getPublicTruffleFile(venvPath.toJavaStringUncached()), context, count); - } catch (IOException | InterruptedException e) { + } catch (NativeLibraryToolException e) { throw PRaiseNode.raiseStatic(node, PythonBuiltinClassType.ValueError, e); } return PNone.NONE; diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/copying/ElfFile.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/copying/ElfFile.java index c51de60ed7..c813edf84a 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/copying/ElfFile.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/copying/ElfFile.java @@ -41,7 +41,9 @@ package com.oracle.graal.python.builtins.objects.cext.copying; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; @@ -55,57 +57,106 @@ final class ElfFile extends SharedObject { private final PythonContext context; private final TruffleFile tempfile; - private String getPatchelf() throws IOException { + private String getPatchelf() throws NativeLibraryToolException { TruffleFile patchelf = which(context, "patchelf"); if (!patchelf.exists()) { - throw new IOException("Could not find `patchelf` on PATH. " + PATCHELF_INSTALL_INSTRUCTION); + throw new NativeLibraryToolException("Could not find `patchelf`. " + PATCHELF_INSTALL_INSTRUCTION); } return patchelf.toString(); } - private void runPatchelf(String action, String... arguments) throws IOException, InterruptedException { + private void runPatchelf(String action, String... arguments) throws NativeLibraryToolException { var command = new String[arguments.length + 1]; command[0] = getPatchelf(); System.arraycopy(arguments, 0, command, 1, arguments.length); var pb = newProcessBuilder(context); + var stderr = new ByteArrayOutputStream(); + pb.redirectError(pb.createRedirectToStream(stderr)); pb.command(command); Process proc; try { proc = pb.start(); } catch (IOException e) { - throw new IOException("Failed to start `patchelf` to " + action + ": " + e.getMessage() + ". " + PATCHELF_INSTALL_INSTRUCTION, e); + throw new NativeLibraryToolException("Failed to start `patchelf` to " + action + ": " + e.getMessage() + ". " + PATCHELF_INSTALL_INSTRUCTION, e); } - if (proc.waitFor() != 0) { - throw new IOException("Failed to run `patchelf` to " + action + " (exit code " + proc.exitValue() + "). " + PATCHELF_INSTALL_INSTRUCTION); + try { + if (proc.waitFor() != 0) { + throw new NativeLibraryToolException("Failed to run `patchelf` to " + action + " (exit code " + proc.exitValue() + "). " + PATCHELF_INSTALL_INSTRUCTION + + " Stderr: " + getStderr(stderr)); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new NativeLibraryToolException("Interrupted while waiting for `patchelf` to " + action + ". " + PATCHELF_INSTALL_INSTRUCTION, e); } } - ElfFile(byte[] b, PythonContext context) throws IOException { - this.context = context; - this.tempfile = context.getEnv().createTempFile(null, null, ".so"); - try (var os = this.tempfile.newOutputStream(StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE)) { + private static String getStderr(ByteArrayOutputStream stderr) { + String output = stderr.toString(StandardCharsets.UTF_8).strip(); + return output.isEmpty() ? "" : output; + } + + private static void deleteTempfile(TruffleFile tempfile) throws NativeLibraryToolException { + try { + tempfile.delete(); + } catch (IOException e) { + throw new NativeLibraryToolException("Failed to delete temporary ELF library copy '" + tempfile + "': " + e.getMessage(), e); + } + } + + private static void deleteTempfileAfterFailedInit(TruffleFile tempfile, IOException failure) throws NativeLibraryToolException { + var exception = new NativeLibraryToolException("Failed to write temporary ELF library copy '" + tempfile + "' for IsolateNativeModules relocation: " + failure.getMessage(), failure); + try { + deleteTempfile(tempfile); + } catch (NativeLibraryToolException e) { + exception.addSuppressed(e); + } + throw exception; + } + + private static void writeTempfile(TruffleFile tempfile, byte[] b) throws NativeLibraryToolException { + try (var os = tempfile.newOutputStream(StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE)) { os.write(b); + } catch (IOException e) { + deleteTempfileAfterFailedInit(tempfile, e); + } + } + + private static TruffleFile createTempfile(PythonContext context) throws NativeLibraryToolException { + try { + return context.getEnv().createTempFile(null, null, ".so"); + } catch (IOException e) { + throw new NativeLibraryToolException("Failed to create temporary ELF library copy for IsolateNativeModules relocation: " + e.getMessage(), e); } } + ElfFile(byte[] b, PythonContext context) throws NativeLibraryToolException { + this.context = context; + this.tempfile = createTempfile(context); + writeTempfile(tempfile, b); + } + @Override - public void setId(String newId) throws IOException, InterruptedException { + public void setId(String newId) throws NativeLibraryToolException { runPatchelf("set SONAME", "--debug", "--set-soname", newId, tempfile.toString()); } @Override - public void changeOrAddDependency(String oldName, String newName) throws IOException, InterruptedException { + public void changeOrAddDependency(String oldName, String newName) throws NativeLibraryToolException { runPatchelf("remove dependency", "--debug", "--remove-needed", oldName, tempfile.toString()); runPatchelf("add dependency", "--debug", "--add-needed", newName, tempfile.toString()); } @Override - public void write(TruffleFile copy) throws IOException { - tempfile.copy(copy, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES); + public void write(TruffleFile copy) throws NativeLibraryToolException { + try { + tempfile.copy(copy, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES); + } catch (IOException e) { + throw new NativeLibraryToolException("Failed to write relocated ELF library copy '" + copy + "': " + e.getMessage(), e); + } } @Override - public void close() throws IOException { - tempfile.delete(); + public void close() throws NativeLibraryToolException { + deleteTempfile(tempfile); } } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/copying/NativeLibraryLocator.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/copying/NativeLibraryLocator.java index f0a401f112..db4aa1d54f 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/copying/NativeLibraryLocator.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/copying/NativeLibraryLocator.java @@ -40,11 +40,11 @@ */ package com.oracle.graal.python.builtins.objects.cext.copying; +import static com.oracle.graal.python.nodes.StringLiterals.J_MAX_CAPI_COPIES; import static com.oracle.graal.python.nodes.StringLiterals.J_NATIVE; import static com.oracle.graal.python.nodes.StringLiterals.T_BASE_PREFIX; -import static com.oracle.graal.python.nodes.StringLiterals.J_MAX_CAPI_COPIES; import static com.oracle.graal.python.nodes.StringLiterals.T_PREFIX; -import static com.oracle.graal.python.util.PythonUtils.toTruffleStringUncached; +import static com.oracle.graal.python.util.PythonUtils.TS_ENCODING; import java.io.IOException; import java.util.concurrent.atomic.AtomicInteger; @@ -60,6 +60,7 @@ import com.oracle.truffle.api.TruffleFile; import com.oracle.truffle.api.TruffleLanguage.Env; import com.oracle.truffle.api.TruffleLogger; +import com.oracle.truffle.api.strings.TruffleString; /** * Given a GraalPy virtual environment, this class helps prepare that environment so that multiple @@ -142,7 +143,7 @@ public void close() { * the same time. The minimum number of concurrent contexts to prepare for is given with {@code * count}. */ - public static void replicate(TruffleFile venvDirectory, PythonContext context, int count) throws IOException, InterruptedException { + public static void replicate(TruffleFile venvDirectory, PythonContext context, int count) throws NativeLibraryToolException { if (count > MAX_CEXT_COPIES) { LOGGER.warning(() -> String.format("The current limit for concurrent Python contexts accessing the Python C API is %d, " + "but we are preparing %d copies. The extra copies will only be used if a different value " + @@ -150,24 +151,13 @@ public static void replicate(TruffleFile venvDirectory, PythonContext context, i } String suffix = context.getSoAbi().toJavaStringUncached(); TruffleFile capiLibrary = context.getPublicTruffleFileRelaxed(context.getCAPIHome()).resolve(PythonContext.getSupportLibName("python-" + J_NATIVE)); - try { - for (int i = 0; i < count; i++) { - // Relocate the C API library - replicate(capiLibrary, venvDirectory.resolve(copyNameOf(capiLibrary.getName(), i)), context, i); - // Relocate the core C extensions - walk(context.getPublicTruffleFileRelaxed(context.getCoreHome()), suffix, capiLibrary.getName(), context, i, (o, n) -> venvDirectory.resolve(n)); - // Relocate C extensions in the venv - walk(venvDirectory, suffix, capiLibrary.getName(), context, i, (o, n) -> o.resolveSibling(n)); - } - } catch (RuntimeException e) { - var cause = e.getCause(); - if (cause instanceof IOException ioCause) { - throw ioCause; - } else if (cause instanceof InterruptedException intCause) { - throw intCause; - } else { - throw e; - } + for (int i = 0; i < count; i++) { + // Relocate the C API library + replicate(capiLibrary, venvDirectory.resolve(copyNameOf(capiLibrary.getName(), i)), context, i); + // Relocate the core C extensions + walk(context.getPublicTruffleFileRelaxed(context.getCoreHome()), suffix, capiLibrary.getName(), context, i, (o, n) -> venvDirectory.resolve(n)); + // Relocate C extensions in the venv + walk(venvDirectory, suffix, capiLibrary.getName(), context, i, (o, n) -> o.resolveSibling(n)); } } @@ -223,14 +213,15 @@ private static String resolve(PythonContext context, TruffleFile original, int c if (!copy.isReadable()) { try { replicate(original, copy, context, capiSlot, capiOrignalName); - } catch (IOException | InterruptedException e) { - throw new ApiInitException(e); + } catch (NativeLibraryToolException e) { + throw new ApiInitException(TruffleString.fromJavaStringUncached(e.getMessage(), TS_ENCODING)); } } return copy.getPath(); } - private static void replicate(TruffleFile original, TruffleFile copy, PythonContext context, int slot, String... dependenciesToUpdate) throws IOException, InterruptedException { + private static void replicate(TruffleFile original, TruffleFile copy, PythonContext context, int slot, String... dependenciesToUpdate) + throws NativeLibraryToolException { try (var o = SharedObject.open(original, context)) { for (var depToUpdate : dependenciesToUpdate) { if (depToUpdate != null) { @@ -244,7 +235,7 @@ private static void replicate(TruffleFile original, TruffleFile copy, PythonCont } private static void walk(TruffleFile dir, String suffix, String capiOriginalName, PythonContext context, int capiSlot, BiFunction f) - throws IOException, InterruptedException { + throws NativeLibraryToolException { try (var ds = dir.newDirectoryStream()) { for (var e : ds) { if (e.isDirectory()) { @@ -253,6 +244,8 @@ private static void walk(TruffleFile dir, String suffix, String capiOriginalName replicate(e, f.apply(e, copyNameOf(e.getName(), capiSlot)), context, capiSlot, capiOriginalName); } } + } catch (IOException e) { + throw new NativeLibraryToolException("Failed to scan native library directory '" + dir + "' for IsolateNativeModules relocation: " + e.getMessage(), e); } } } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/copying/NativeLibraryToolException.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/copying/NativeLibraryToolException.java new file mode 100644 index 0000000000..f3be9f34da --- /dev/null +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/copying/NativeLibraryToolException.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The Universal Permissive License (UPL), Version 1.0 + * + * Subject to the condition set forth below, permission is hereby granted to any + * person obtaining a copy of this software, associated documentation and/or + * data (collectively the "Software"), free of charge and under any and all + * copyright rights in the Software, and any and all patent rights owned or + * freely licensable by each licensor hereunder covering either (i) the + * unmodified Software as contributed to or provided by such licensor, or (ii) + * the Larger Works (as defined below), to deal in both + * + * (a) the Software, and + * + * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if + * one is included with the Software each a "Larger Work" to which the Software + * is contributed by such licensors), + * + * without restriction, including without limitation the rights to copy, create + * derivative works of, display, perform, and distribute the Software and make, + * use, sell, offer for sale, import, export, have made, and have sold the + * Software and the Larger Work(s), and to sublicense the foregoing rights on + * either these or other terms. + * + * This license is subject to the following condition: + * + * The above copyright notice and either this complete permission notice or at a + * minimum a reference to the UPL must be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package com.oracle.graal.python.builtins.objects.cext.copying; + +public final class NativeLibraryToolException extends Exception { + private static final long serialVersionUID = 5478160327398513738L; + + NativeLibraryToolException(String message) { + super(message); + } + + NativeLibraryToolException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/copying/PEFile.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/copying/PEFile.java index 57b475243e..6227705191 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/copying/PEFile.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/copying/PEFile.java @@ -41,7 +41,9 @@ package com.oracle.graal.python.builtins.objects.cext.copying; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; @@ -51,25 +53,33 @@ final class PEFile extends SharedObject { private static final String DELVEWHEEL_VERSION = "1.9.0"; private static final String DELVEWHEEL_INSTALL_INSTRUCTION = "IsolateNativeModules option needs `delvewheel` tool to copy libraries. Make sure you have `delvewheel==" + DELVEWHEEL_VERSION + - "` available on PATH."; + "` available in the virtualenv or on PATH (needs environment access)."; private final PythonContext context; private final TruffleFile tempfile; - PEFile(byte[] b, PythonContext context) throws IOException { + PEFile(byte[] b, PythonContext context) throws NativeLibraryToolException { this.context = context; - this.tempfile = context.getEnv().createTempFile(null, null, ".dll"); - try (var os = this.tempfile.newOutputStream(StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE)) { + TruffleFile temp; + try { + temp = context.getEnv().createTempFile(null, null, ".dll"); + } catch (IOException e) { + throw new NativeLibraryToolException("Failed to create temporary PE library copy for IsolateNativeModules relocation: " + e.getMessage(), e); + } + this.tempfile = temp; + try (var os = tempfile.newOutputStream(StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE)) { os.write(b); + } catch (IOException e) { + throw new NativeLibraryToolException("Failed to write temporary PE library copy '" + tempfile + "' for IsolateNativeModules relocation: " + e.getMessage(), e); } } @Override - public void setId(String newId) throws IOException { + public void setId(String newId) { // TODO } - private String getDelvewheelPython() throws IOException { + private String getDelvewheelPython() throws NativeLibraryToolException { TruffleFile delvewheel = which(context, "delvewheel.exe"); if (!delvewheel.exists()) { delvewheel = which(context, "delvewheel.bat"); @@ -78,7 +88,7 @@ private String getDelvewheelPython() throws IOException { delvewheel = which(context, "delvewheel.cmd"); } if (!delvewheel.exists()) { - throw new IOException("Could not find `delvewheel` on PATH. " + DELVEWHEEL_INSTALL_INSTRUCTION); + throw new NativeLibraryToolException("Could not find `delvewheel`. " + DELVEWHEEL_INSTALL_INSTRUCTION); } TruffleFile python = delvewheel.resolveSibling("python.exe"); if (!python.exists()) { @@ -97,14 +107,16 @@ private String getDelvewheelPython() throws IOException { python = delvewheel.getParent().resolveSibling("python.cmd"); } if (!python.exists()) { - throw new IOException("Could not find Python executable next to `delvewheel` at '" + delvewheel + "'. " + DELVEWHEEL_INSTALL_INSTRUCTION); + throw new NativeLibraryToolException("Could not find Python executable next to `delvewheel` at '" + delvewheel + "'. " + DELVEWHEEL_INSTALL_INSTRUCTION); } return python.toString(); } @Override - public void changeOrAddDependency(String oldName, String newName) throws IOException, InterruptedException { + public void changeOrAddDependency(String oldName, String newName) throws NativeLibraryToolException { var pb = newProcessBuilder(context); + var stderr = new ByteArrayOutputStream(); + pb.redirectError(pb.createRedirectToStream(stderr)); var tempfileWithForwardSlashes = tempfile.toString().replace('\\', '/'); String pythonExe = getDelvewheelPython(); pb.command(pythonExe, "-c", @@ -114,20 +126,39 @@ public void changeOrAddDependency(String oldName, String newName) throws IOExcep try { proc = pb.start(); } catch (IOException e) { - throw new IOException("Failed to start `delvewheel` to copy required DLL: " + e.getMessage() + ". " + DELVEWHEEL_INSTALL_INSTRUCTION, e); + throw new NativeLibraryToolException("Failed to start `delvewheel` to copy required DLL: " + e.getMessage() + ". " + DELVEWHEEL_INSTALL_INSTRUCTION, e); } - if (proc.waitFor() != 0) { - throw new IOException("Failed to run `delvewheel` to copy required DLL (exit code " + proc.exitValue() + "). " + DELVEWHEEL_INSTALL_INSTRUCTION); + try { + if (proc.waitFor() != 0) { + throw new NativeLibraryToolException("Failed to run `delvewheel` to copy required DLL (exit code " + proc.exitValue() + "). " + DELVEWHEEL_INSTALL_INSTRUCTION + + " Stderr: " + getStderr(stderr)); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new NativeLibraryToolException("Interrupted while waiting for `delvewheel` to copy required DLL. " + DELVEWHEEL_INSTALL_INSTRUCTION, e); } } + private static String getStderr(ByteArrayOutputStream stderr) { + String output = stderr.toString(StandardCharsets.UTF_8).strip(); + return output.isEmpty() ? "" : output; + } + @Override - public void write(TruffleFile copy) throws IOException { - tempfile.copy(copy, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES); + public void write(TruffleFile copy) throws NativeLibraryToolException { + try { + tempfile.copy(copy, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES); + } catch (IOException e) { + throw new NativeLibraryToolException("Failed to write relocated PE library copy '" + copy + "': " + e.getMessage(), e); + } } @Override - public void close() throws IOException { - tempfile.delete(); + public void close() throws NativeLibraryToolException { + try { + tempfile.delete(); + } catch (IOException e) { + throw new NativeLibraryToolException("Failed to delete temporary PE library copy '" + tempfile + "': " + e.getMessage(), e); + } } } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/copying/SharedObject.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/copying/SharedObject.java index 0fd42baffd..2876aad951 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/copying/SharedObject.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/copying/SharedObject.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -51,25 +51,30 @@ import com.oracle.truffle.api.io.TruffleProcessBuilder; abstract class SharedObject implements AutoCloseable { - abstract void setId(String newId) throws IOException, InterruptedException; + abstract void setId(String newId) throws NativeLibraryToolException; - abstract void changeOrAddDependency(String oldName, String newName) throws IOException, InterruptedException; + abstract void changeOrAddDependency(String oldName, String newName) throws NativeLibraryToolException; - abstract void write(TruffleFile copy) throws IOException, InterruptedException; + abstract void write(TruffleFile copy) throws NativeLibraryToolException; - public abstract void close() throws IOException; + public abstract void close() throws NativeLibraryToolException; - static SharedObject open(TruffleFile file, PythonContext context) throws IOException { - var f = file.readAllBytes(); + static SharedObject open(TruffleFile file, PythonContext context) throws NativeLibraryToolException { + byte[] f; + try { + f = file.readAllBytes(); + } catch (IOException e) { + throw new NativeLibraryToolException("Failed to read native library '" + file + "' for IsolateNativeModules relocation: " + e.getMessage(), e); + } switch (f[0]) { case 0x7f: return new ElfFile(f, context); case 0x4d, 0x5a: return new PEFile(f, context); case (byte) 0xca, (byte) 0xfe, (byte) 0xce, (byte) 0xcf: - throw new IOException("Modifying Mach-O files is not yet supported"); + throw new NativeLibraryToolException("Failed to relocate native library '" + file + "': modifying Mach-O files is not yet supported."); default: - throw new IOException("Unknown shared object format"); + throw new NativeLibraryToolException("Failed to relocate native library '" + file + "': unknown shared object format."); } }