Merge pull request #1134 from losttech/PR/ExceptionsImprovement

Improve Python <-> .NET exception integration

.github/workflows/main.yml

@@ -56,6 +56,9 @@ jobs:
         run: |
           python -m pythonnet.find_libpython --export | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
 
+      - name: Embedding tests
+        run: dotnet test --runtime any-${{ matrix.platform }} src/embed_tests/
+
       - name: Python Tests (Mono)
         if: ${{ matrix.os != 'windows' }}
         run: pytest --runtime mono
@@ -67,9 +70,6 @@ jobs:
         if: ${{ matrix.os == 'windows' }}
         run: pytest --runtime netfx
 
-      - name: Embedding tests
-        run: dotnet test --runtime any-${{ matrix.platform }} src/embed_tests/
-
       - name: Python tests run from .NET
         run: dotnet test --runtime any-${{ matrix.platform }} src/python_tests_runner/
 

CHANGELOG.md

@@ -14,6 +14,9 @@ This document follows the conventions laid out in [Keep a CHANGELOG][].
 -   Add GetPythonThreadID and Interrupt methods in PythonEngine
 -   Ability to implement delegates with `ref` and `out` parameters in Python, by returning the modified parameter values in a tuple. ([#1355][i1355])
 -   `PyType` - a wrapper for Python type objects, that also permits creating new heap types from `TypeSpec`
+-    Improved exception handling:
+  -  exceptions can now be converted with codecs
+  -  `InnerException` and `__cause__` are propagated properly
 
 ### Changed
 -   Drop support for Python 2, 3.4, and 3.5
@@ -44,7 +47,9 @@ One must now either use enum members (e.g. `MyEnum.Option`), or use enum constru
 -   Sign Runtime DLL with a strong name
 -   Implement loading through `clr_loader` instead of the included `ClrModule`, enables
     support for .NET Core
+-   .NET and Python exceptions are preserved when crossing Python/.NET boundary
 -   BREAKING: custom encoders are no longer called for instances of `System.Type`
+-   `PythonException.Restore` no longer clears `PythonException` instance.
 
 ### Fixed
 
@@ -70,6 +75,7 @@ One must now either use enum members (e.g. `MyEnum.Option`), or use enum constru
 ### Removed
 
 -   implicit assembly loading (you have to explicitly `clr.AddReference` before doing import)
+-   messages in `PythonException` no longer start with exception type
 -   support for .NET Framework 4.0-4.6; Mono before 5.4. Python.NET now requires .NET Standard 2.0
 (see [the matrix](https://docs.microsoft.com/en-us/dotnet/standard/net-standard#net-implementation-support))
 

src/embed_tests/Codecs.cs

@@ -322,6 +322,65 @@ class ListAsRawEncoder(RawProxyEncoder):
 
             PythonEngine.Exec(PyCode);
         }
+
+        const string TestExceptionMessage = "Hello World!";
+        [Test]
+        public void ExceptionEncoded()
+        {
+            PyObjectConversions.RegisterEncoder(new ValueErrorCodec());
+            void CallMe() => throw new ValueErrorWrapper(TestExceptionMessage);
+            var callMeAction = new Action(CallMe);
+            using var _ = Py.GIL();
+            using var scope = Py.CreateScope();
+            scope.Exec(@"
+def call(func):
+  try:
+    func()
+  except ValueError as e:
+    return str(e)
+");
+            var callFunc = scope.Get("call");
+            string message = callFunc.Invoke(callMeAction.ToPython()).As<string>();
+            Assert.AreEqual(TestExceptionMessage, message);
+        }
+
+        [Test]
+        public void ExceptionDecoded()
+        {
+            PyObjectConversions.RegisterDecoder(new ValueErrorCodec());
+            using var _ = Py.GIL();
+            using var scope = Py.CreateScope();
+            var error = Assert.Throws<ValueErrorWrapper>(()
+                => PythonEngine.Exec($"raise ValueError('{TestExceptionMessage}')"));
+            Assert.AreEqual(TestExceptionMessage, error.Message);
+        }
+
+        class ValueErrorWrapper : Exception
+        {
+            public ValueErrorWrapper(string message) : base(message) { }
+        }
+
+        class ValueErrorCodec : IPyObjectEncoder, IPyObjectDecoder
+        {
+            public bool CanDecode(PyObject objectType, Type targetType)
+                => this.CanEncode(targetType) && objectType.Equals(PythonEngine.Eval("ValueError"));
+
+            public bool CanEncode(Type type) => type == typeof(ValueErrorWrapper)
+                                                || typeof(ValueErrorWrapper).IsSubclassOf(type);
+
+            public bool TryDecode<T>(PyObject pyObj, out T value)
+            {
+                var message = pyObj.GetAttr("args")[0].As<string>();
+                value = (T)(object)new ValueErrorWrapper(message);
+                return true;
+            }
+
+            public PyObject TryEncode(object value)
+            {
+                var error = (ValueErrorWrapper)value;
+                return PythonEngine.Eval("ValueError").Invoke(error.Message.ToPython());
+            }
+        }
     }
 
     /// <summary>

src/embed_tests/TestCallbacks.cs

@@ -24,7 +24,7 @@ public class TestCallbacks {
             using (Py.GIL()) {
                 dynamic callWith42 = PythonEngine.Eval("lambda f: f([42])");
                 var error =  Assert.Throws<PythonException>(() => callWith42(aFunctionThatCallsIntoPython.ToPython()));
-                Assert.AreEqual("TypeError", error.PythonTypeName);
+                Assert.AreEqual("TypeError", error.Type.Name);
                 string expectedArgTypes = "(<class 'list'>)";
                 StringAssert.EndsWith(expectedArgTypes, error.Message);
             }

src/embed_tests/TestPyFloat.cs

@@ -95,7 +95,7 @@ public void StringBadCtor()
 
             var ex = Assert.Throws<PythonException>(() => a = new PyFloat(i));
 
-            StringAssert.StartsWith("ValueError : could not convert string to float", ex.Message);
+            StringAssert.StartsWith("could not convert string to float", ex.Message);
             Assert.IsNull(a);
         }
 
@@ -132,7 +132,7 @@ public void AsFloatBad()
             PyFloat a = null;
 
             var ex = Assert.Throws<PythonException>(() => a = PyFloat.AsFloat(s));
-            StringAssert.StartsWith("ValueError : could not convert string to float", ex.Message);
+            StringAssert.StartsWith("could not convert string to float", ex.Message);
             Assert.IsNull(a);
         }
     }

src/embed_tests/TestPyInt.cs

@@ -128,7 +128,7 @@ public void TestCtorBadString()
 
             var ex = Assert.Throws<PythonException>(() => a = new PyInt(i));
 
-            StringAssert.StartsWith("ValueError : invalid literal for int", ex.Message);
+            StringAssert.StartsWith("invalid literal for int", ex.Message);
             Assert.IsNull(a);
         }
 
@@ -161,7 +161,7 @@ public void TestAsIntBad()
             PyInt a = null;
 
             var ex = Assert.Throws<PythonException>(() => a = PyInt.AsInt(s));
-            StringAssert.StartsWith("ValueError : invalid literal for int", ex.Message);
+            StringAssert.StartsWith("invalid literal for int", ex.Message);
             Assert.IsNull(a);
         }
 

src/embed_tests/TestPyList.cs

@@ -41,7 +41,7 @@ public void TestStringAsListType()
 
             var ex = Assert.Throws<PythonException>(() => t = PyList.AsList(i));
 
-            Assert.AreEqual("TypeError : 'int' object is not iterable", ex.Message);
+            Assert.AreEqual("'int' object is not iterable", ex.Message);
             Assert.IsNull(t);
         }
 

src/embed_tests/TestPyLong.cs

@@ -144,7 +144,7 @@ public void TestCtorBadString()
 
             var ex = Assert.Throws<PythonException>(() => a = new PyLong(i));
 
-            StringAssert.StartsWith("ValueError : invalid literal", ex.Message);
+            StringAssert.StartsWith("invalid literal", ex.Message);
             Assert.IsNull(a);
         }
 
@@ -177,7 +177,7 @@ public void TestAsLongBad()
             PyLong a = null;
 
             var ex = Assert.Throws<PythonException>(() => a = PyLong.AsLong(s));
-            StringAssert.StartsWith("ValueError : invalid literal", ex.Message);
+            StringAssert.StartsWith("invalid literal", ex.Message);
             Assert.IsNull(a);
         }
 

src/embed_tests/TestPyTuple.cs

@@ -104,7 +104,7 @@ public void TestPyTupleInvalidAppend()
 
             var ex = Assert.Throws<PythonException>(() => t.Concat(s));
 
-            StringAssert.StartsWith("TypeError : can only concatenate tuple", ex.Message);
+            StringAssert.StartsWith("can only concatenate tuple", ex.Message);
             Assert.AreEqual(0, t.Length());
             Assert.IsEmpty(t);
         }
@@ -164,7 +164,7 @@ public void TestInvalidAsTuple()
 
             var ex = Assert.Throws<PythonException>(() => t = PyTuple.AsTuple(i));
 
-            Assert.AreEqual("TypeError : 'int' object is not iterable", ex.Message);
+            Assert.AreEqual("'int' object is not iterable", ex.Message);
             Assert.IsNull(t);
         }
     }

src/embed_tests/TestPyType.cs

@@ -40,6 +40,7 @@ public void CanCreateHeapType()
 
             using var type = new PyType(spec);
             Assert.AreEqual(name, type.GetAttr("__name__").As<string>());
+            Assert.AreEqual(name, type.Name);
             Assert.AreEqual(docStr, type.GetAttr("__doc__").As<string>());
         }
     }

src/embed_tests/TestPyWith.cs

@@ -51,7 +51,7 @@ class CmTest:
             catch (PythonException e)
             {
                 TestContext.Out.WriteLine(e.Message);
-                Assert.IsTrue(e.Message.Contains("ZeroDivisionError"));
+                Assert.IsTrue(e.Type.Name == "ZeroDivisionError");
             }
         }
 

src/embed_tests/TestPythonException.cs

@@ -30,31 +30,61 @@ public void TestMessage()
 
             var ex = Assert.Throws<PythonException>(() => foo = list[0]);
 
-            Assert.AreEqual("IndexError : list index out of range", ex.Message);
+            Assert.AreEqual("list index out of range", ex.Message);
+            Assert.IsNull(foo);
+        }
+
+        [Test]
+        public void TestType()
+        {
+            var list = new PyList();
+            PyObject foo = null;
+
+            var ex = Assert.Throws<PythonException>(() => foo = list[0]);
+
+            Assert.AreEqual("IndexError", ex.Type.Name);
             Assert.IsNull(foo);
         }
 
         [Test]
         public void TestNoError()
         {
-            var e = new PythonException(); // There is no PyErr to fetch
-            Assert.AreEqual("", e.Message);
+            // There is no PyErr to fetch
+            Assert.Throws<InvalidOperationException>(() => PythonException.FetchCurrentRaw());
+            var currentError = PythonException.FetchCurrentOrNullRaw();
+            Assert.IsNull(currentError);
         }
 
         [Test]
-        public void TestPythonErrorTypeName()
+        public void TestNestedExceptions()
         {
             try
             {
-                var module = PyModule.Import("really____unknown___module");
-                Assert.Fail("Unknown module should not be loaded");
+                PythonEngine.Exec(@"
+try:
+  raise Exception('inner')
+except Exception as ex:
+  raise Exception('outer') from ex
+");
             }
             catch (PythonException ex)
             {
-                Assert.That(ex.PythonTypeName, Is.EqualTo("ModuleNotFoundError").Or.EqualTo("ImportError"));
+                Assert.That(ex.InnerException, Is.InstanceOf<PythonException>());
+                Assert.That(ex.InnerException.Message, Is.EqualTo("inner"));
             }
         }
 
+        [Test]
+        public void InnerIsEmptyWithNoCause()
+        {
+            var list = new PyList();
+            PyObject foo = null;
+
+            var ex = Assert.Throws<PythonException>(() => foo = list[0]);
+
+            Assert.IsNull(ex.InnerException);
+        }
+
         [Test]
         public void TestPythonExceptionFormat()
         {
@@ -83,13 +113,6 @@ public void TestPythonExceptionFormat()
             }
         }
 
-        [Test]
-        public void TestPythonExceptionFormatNoError()
-        {
-            var ex = new PythonException();
-            Assert.AreEqual(ex.StackTrace, ex.Format());
-        }
-
         [Test]
         public void TestPythonExceptionFormatNoTraceback()
         {
@@ -132,30 +155,27 @@ class TestException(NameError):
                 Assert.IsTrue(scope.TryGet("TestException", out PyObject type));
 
                 PyObject str = "dummy string".ToPython();
-                IntPtr typePtr = type.Handle;
-                IntPtr strPtr = str.Handle;
-                IntPtr tbPtr = Runtime.Runtime.None.Handle;
-                Runtime.Runtime.XIncref(typePtr);
-                Runtime.Runtime.XIncref(strPtr);
-                Runtime.Runtime.XIncref(tbPtr);
+                var typePtr = new NewReference(type.Reference);
+                var strPtr = new NewReference(str.Reference);
+                var tbPtr = new NewReference(Runtime.Runtime.None.Reference);
                 Runtime.Runtime.PyErr_NormalizeException(ref typePtr, ref strPtr, ref tbPtr);
 
-                using (PyObject typeObj = new PyObject(typePtr), strObj = new PyObject(strPtr), tbObj = new PyObject(tbPtr))
-                {
-                    // the type returned from PyErr_NormalizeException should not be the same type since a new
-                    // exception was raised by initializing the exception
-                    Assert.AreNotEqual(type.Handle, typePtr);
-                    // the message should now be the string from the throw exception during normalization
-                    Assert.AreEqual("invalid literal for int() with base 10: 'dummy string'", strObj.ToString());
-                }
+                using var typeObj = typePtr.MoveToPyObject();
+                using var strObj = strPtr.MoveToPyObject();
+                using var tbObj = tbPtr.MoveToPyObject();
+                // the type returned from PyErr_NormalizeException should not be the same type since a new
+                // exception was raised by initializing the exception
+                Assert.AreNotEqual(type.Handle, typeObj.Handle);
+                // the message should now be the string from the throw exception during normalization
+                Assert.AreEqual("invalid literal for int() with base 10: 'dummy string'", strObj.ToString());
             }
         }
 
         [Test]
         public void TestPythonException_Normalize_ThrowsWhenErrorSet()
         {
             Exceptions.SetError(Exceptions.TypeError, "Error!");
-            var pythonException = new PythonException();
+            var pythonException = PythonException.FetchCurrentRaw();
             Exceptions.SetError(Exceptions.TypeError, "Another error");
             Assert.Throws<InvalidOperationException>(() => pythonException.Normalize());
         }

src/embed_tests/TestRuntime.cs

@@ -2,7 +2,6 @@
 using System.Collections.Generic;
 using NUnit.Framework;
 using Python.Runtime;
-using Python.Runtime.Platform;
 
 namespace Python.EmbeddingTest
 {
@@ -102,7 +101,7 @@ public static void PyCheck_Iter_PyObject_IsIterable_ThreadingLock_Test()
                 Exceptions.ErrorCheck(threadingDict);
                 var lockType = Runtime.Runtime.PyDict_GetItemString(threadingDict, "Lock");
                 if (lockType.IsNull)
-                    throw new PythonException();
+                    throw PythonException.ThrowLastAsClrException();
 
                 using var args = NewReference.DangerousFromPointer(Runtime.Runtime.PyTuple_New(0));
                 using var lockInstance = Runtime.Runtime.PyObject_CallObject(lockType, args);

src/embed_tests/pyimport.cs

@@ -102,8 +102,7 @@ import clr
 clr.AddReference('{path}')
 ";
 
-            var error = Assert.Throws<PythonException>(() => PythonEngine.Exec(code));
-            Assert.AreEqual(nameof(FileLoadException), error.PythonTypeName);
+            Assert.Throws<FileLoadException>(() => PythonEngine.Exec(code));
         }
     }
 }

src/embed_tests/pyinitialize.cs

@@ -158,9 +158,10 @@ public static void TestRunExitFuncs()
             catch (PythonException e)
             {
                 string msg = e.ToString();
+                bool isImportError = e.Is(Exceptions.ImportError);
                 Runtime.Runtime.Shutdown();
 
-                if (e.IsMatches(Exceptions.ImportError))
+                if (isImportError)
                 {
                     Assert.Ignore("no atexit module");
                 }

src/runtime/BorrowedReference.cs

@@ -16,6 +16,8 @@ public IntPtr DangerousGetAddress()
         /// <summary>Gets a raw pointer to the Python object</summary>
         public IntPtr DangerousGetAddressOrNull() => this.pointer;
 
+        public static BorrowedReference Null => new BorrowedReference();
+
         /// <summary>
         /// Creates new instance of <see cref="BorrowedReference"/> from raw pointer. Unsafe.
         /// </summary>