From 447cca7fcbf730603ca569c1d849a074e1b5731e Mon Sep 17 00:00:00 2001 From: Tim Maxwell Date: Thu, 5 Oct 2023 16:01:09 -0700 Subject: [PATCH 1/5] Add support for various Python 3.11 things --- src/tblib/pickling_support.py | 82 +++++++++++++++++++++++++++++++--- tests/test_pickle_exception.py | 40 +++++++++++++++-- 2 files changed, 112 insertions(+), 10 deletions(-) diff --git a/src/tblib/pickling_support.py b/src/tblib/pickling_support.py index d58799f..451374b 100644 --- a/src/tblib/pickling_support.py +++ b/src/tblib/pickling_support.py @@ -22,14 +22,20 @@ def pickle_traceback(tb): return unpickle_traceback, (Frame(tb.tb_frame), tb.tb_lineno, tb.tb_next and Traceback(tb.tb_next)) -def unpickle_exception(func, args, cause, tb): +# unpickle_exception_3_11() / pickle_exception_3_11() are used in Python 3.11 and newer + +def unpickle_exception_3_11(func, args, cause, context, notes, suppress_context, tb): inst = func(*args) inst.__cause__ = cause + inst.__context__ = context + if notes is not None: + inst.__notes__ = notes + inst.__suppress_context__ = suppress_context inst.__traceback__ = tb return inst -def pickle_exception(obj): +def pickle_exception_3_11(obj): # All exceptions, unlike generic Python objects, define __reduce_ex__ # __reduce_ex__(4) should be no different from __reduce_ex__(3). # __reduce_ex__(5) could bring benefits in the unlikely case the exception @@ -43,9 +49,49 @@ def pickle_exception(obj): assert isinstance(rv, tuple) assert len(rv) >= 2 + return ( + unpickle_exception_3_11, + rv[:2] + ( + obj.__cause__, + obj.__context__, + getattr(obj, "__notes__", None), + obj.__suppress_context__, + obj.__traceback__, + ), + ) + rv[2:] + + +# unpickle_exception() / pickle_exception() are used on Python 3.10 and older; or when deserializing +# old Pickle archives created by Python 3.10 and older. + +def unpickle_exception(func, args, cause, tb): + inst = func(*args) + inst.__cause__ = cause + inst.__traceback__ = tb + return inst + + +def pickle_exception(obj): + rv = obj.__reduce_ex__(3) + if isinstance(rv, str): + raise TypeError('str __reduce__ output is not supported') + assert isinstance(rv, tuple) + assert len(rv) >= 2 + + # NOTE: The __context__ and __suppress_context__ attributes actually existed prior to Python + # 3.11, so in theory we should support them here. But existing Pickle archives might refer to + # unpickle_exception(), so we need to keep it around anyway; and it's not worth the trouble + # introducing a third pair of pickling/unpickling functions. + return (unpickle_exception, rv[:2] + (obj.__cause__, obj.__traceback__)) + rv[2:] +if sys.version_info >= (3, 11): + pickle_exception_latest = pickle_exception_3_11 +else: + pickle_exception_latest = pickle_exception + + def _get_subclasses(cls): # Depth-first traversal of all direct and indirect subclasses of cls to_visit = [cls] @@ -68,18 +114,40 @@ def install(*exc_classes_or_instances): if not exc_classes_or_instances: for exception_cls in _get_subclasses(BaseException): - copyreg.pickle(exception_cls, pickle_exception) + copyreg.pickle(exception_cls, pickle_exception_latest) return for exc in exc_classes_or_instances: if isinstance(exc, BaseException): - while exc is not None: - copyreg.pickle(type(exc), pickle_exception) - exc = exc.__cause__ + _install_for_instance(exc, set()) elif isinstance(exc, type) and issubclass(exc, BaseException): - copyreg.pickle(exc, pickle_exception) + copyreg.pickle(exc, pickle_exception_latest) # Allow using @install as a decorator for Exception classes if len(exc_classes_or_instances) == 1: return exc else: raise TypeError('Expected subclasses or instances of BaseException, got %s' % (type(exc))) + +def _install_for_instance(exc, seen): + assert isinstance(exc, BaseException) + + # Prevent infinite recursion if we somehow get a self-referential exception. (Self-referential + # exceptions should never normally happen, but if it did somehow happen, we want to pickle the + # exception faithfully so the developer can troubleshoot why it happened.) + if id(exc) in seen: + return + seen.add(id(exc)) + + copyreg.pickle(type(exc), pickle_exception_latest) + + if exc.__cause__ is not None: + _install_for_instance(exc.__cause__, seen) + if exc.__context__ is not None: + _install_for_instance(exc.__context__, seen) + + # This case is meant to cover BaseExceptionGroup on Python 3.11 as well as backports like the + # exceptiongroup module + if hasattr(exc, "exceptions") and isinstance(exc.exceptions, (tuple, list)): + for subexc in exc.exceptions: + if isinstance(subexc, BaseException): + _install_for_instance(subexc, seen) \ No newline at end of file diff --git a/tests/test_pickle_exception.py b/tests/test_pickle_exception.py index aa10762..c997c75 100644 --- a/tests/test_pickle_exception.py +++ b/tests/test_pickle_exception.py @@ -12,6 +12,7 @@ import tblib.pickling_support has_python3 = sys.version_info.major >= 3 +has_python311 = sys.version_info >= (3, 11) @pytest.fixture @@ -33,17 +34,24 @@ def test_install(clear_dispatch_table, how, protocol): if how == 'global': tblib.pickling_support.install() elif how == 'class': - tblib.pickling_support.install(CustomError, ZeroDivisionError) + tblib.pickling_support.install(CustomError, ValueError, ZeroDivisionError) try: try: - 1 / 0 # noqa: B018 + try: + 1 / 0 # noqa: B018 + finally: + # In Python 3, the ValueError's __context__ will be the ZeroDivisionError + raise ValueError("blah") except Exception as e: # Python 3 only syntax # raise CustomError("foo") from e new_e = CustomError('foo') if has_python3: new_e.__cause__ = e + if has_python311: + new_e.add_note("note 1") + new_e.add_note("note 2") raise new_e except Exception as e: exc = e @@ -54,6 +62,7 @@ def test_install(clear_dispatch_table, how, protocol): exc.x = 1 if has_python3: exc.__cause__.x = 2 + exc.__cause__.__context__.x = 3 if how == 'instance': tblib.pickling_support.install(exc) @@ -65,10 +74,17 @@ def test_install(clear_dispatch_table, how, protocol): assert exc.x == 1 if has_python3: assert exc.__traceback__ is not None - assert isinstance(exc.__cause__, ZeroDivisionError) + assert isinstance(exc.__cause__, ValueError) assert exc.__cause__.__traceback__ is not None assert exc.__cause__.x == 2 assert exc.__cause__.__cause__ is None + if has_python311: + assert exc.__notes__ == ["note 1", "note 2"] + assert isinstance(exc.__cause__.__context__, ZeroDivisionError) + assert exc.__cause__.__context__.x == 3 + assert exc.__cause__.__context__.__cause__ is None + assert exc.__cause__.__context__.__context__ is None + @tblib.pickling_support.install @@ -90,6 +106,24 @@ def test_install_decorator(): assert exc.__traceback__ is not None +@pytest.mark.skipif(sys.version_info < (3,11), reason="no BaseExceptionGroup before Python 3.11") +def test_install_instance_recursively(clear_dispatch_table): + exc = BaseExceptionGroup( + "test", + [ + ValueError("foo"), + CustomError("bar") + ] + ) + exc.exceptions[0].__cause__ = ZeroDivisionError("baz") + exc.exceptions[0].__cause__.__context__ = AttributeError("quux") + + tblib.pickling_support.install(exc) + + installed = set(c for c in copyreg.dispatch_table if issubclass(c, BaseException)) + assert installed == {ExceptionGroup, ValueError, CustomError, ZeroDivisionError, AttributeError} + + @pytest.mark.skipif(sys.version_info[0] < 3, reason='No checks done in Python 2') def test_install_typeerror(): with pytest.raises(TypeError): From 045315244888ab55fade7de678db2129c4b69c62 Mon Sep 17 00:00:00 2001 From: Tim Maxwell Date: Thu, 5 Oct 2023 17:56:55 -0700 Subject: [PATCH 2/5] Simpler design that merges the pickle/unpickle functions --- src/tblib/pickling_support.py | 58 +++++++++-------------------------- 1 file changed, 14 insertions(+), 44 deletions(-) diff --git a/src/tblib/pickling_support.py b/src/tblib/pickling_support.py index 451374b..8ad250b 100644 --- a/src/tblib/pickling_support.py +++ b/src/tblib/pickling_support.py @@ -22,20 +22,20 @@ def pickle_traceback(tb): return unpickle_traceback, (Frame(tb.tb_frame), tb.tb_lineno, tb.tb_next and Traceback(tb.tb_next)) -# unpickle_exception_3_11() / pickle_exception_3_11() are used in Python 3.11 and newer - -def unpickle_exception_3_11(func, args, cause, context, notes, suppress_context, tb): +# Note: Older versions of tblib will generate pickle archives that call unpickle_exception() with +# fewer arguments. We assign default values to some of the arguments to support this. +def unpickle_exception(func, args, cause, tb, context=None, suppress_context=False, notes=None): inst = func(*args) inst.__cause__ = cause + inst.__traceback__ = tb inst.__context__ = context + inst.__suppress_context__ = suppress_context if notes is not None: inst.__notes__ = notes - inst.__suppress_context__ = suppress_context - inst.__traceback__ = tb return inst -def pickle_exception_3_11(obj): +def pickle_exception(obj): # All exceptions, unlike generic Python objects, define __reduce_ex__ # __reduce_ex__(4) should be no different from __reduce_ex__(3). # __reduce_ex__(5) could bring benefits in the unlikely case the exception @@ -50,48 +50,18 @@ def pickle_exception_3_11(obj): assert len(rv) >= 2 return ( - unpickle_exception_3_11, + unpickle_exception, rv[:2] + ( obj.__cause__, + obj.__traceback__, obj.__context__, - getattr(obj, "__notes__", None), obj.__suppress_context__, - obj.__traceback__, + # __notes__ doesn't exist prior to Python 3.11; and even on Python 3.11 it may be absent + getattr(obj, "__notes__", None), ), ) + rv[2:] -# unpickle_exception() / pickle_exception() are used on Python 3.10 and older; or when deserializing -# old Pickle archives created by Python 3.10 and older. - -def unpickle_exception(func, args, cause, tb): - inst = func(*args) - inst.__cause__ = cause - inst.__traceback__ = tb - return inst - - -def pickle_exception(obj): - rv = obj.__reduce_ex__(3) - if isinstance(rv, str): - raise TypeError('str __reduce__ output is not supported') - assert isinstance(rv, tuple) - assert len(rv) >= 2 - - # NOTE: The __context__ and __suppress_context__ attributes actually existed prior to Python - # 3.11, so in theory we should support them here. But existing Pickle archives might refer to - # unpickle_exception(), so we need to keep it around anyway; and it's not worth the trouble - # introducing a third pair of pickling/unpickling functions. - - return (unpickle_exception, rv[:2] + (obj.__cause__, obj.__traceback__)) + rv[2:] - - -if sys.version_info >= (3, 11): - pickle_exception_latest = pickle_exception_3_11 -else: - pickle_exception_latest = pickle_exception - - def _get_subclasses(cls): # Depth-first traversal of all direct and indirect subclasses of cls to_visit = [cls] @@ -114,14 +84,14 @@ def install(*exc_classes_or_instances): if not exc_classes_or_instances: for exception_cls in _get_subclasses(BaseException): - copyreg.pickle(exception_cls, pickle_exception_latest) + copyreg.pickle(exception_cls, pickle_exception) return for exc in exc_classes_or_instances: if isinstance(exc, BaseException): _install_for_instance(exc, set()) elif isinstance(exc, type) and issubclass(exc, BaseException): - copyreg.pickle(exc, pickle_exception_latest) + copyreg.pickle(exc, pickle_exception) # Allow using @install as a decorator for Exception classes if len(exc_classes_or_instances) == 1: return exc @@ -138,7 +108,7 @@ def _install_for_instance(exc, seen): return seen.add(id(exc)) - copyreg.pickle(type(exc), pickle_exception_latest) + copyreg.pickle(type(exc), pickle_exception) if exc.__cause__ is not None: _install_for_instance(exc.__cause__, seen) @@ -150,4 +120,4 @@ def _install_for_instance(exc, seen): if hasattr(exc, "exceptions") and isinstance(exc.exceptions, (tuple, list)): for subexc in exc.exceptions: if isinstance(subexc, BaseException): - _install_for_instance(subexc, seen) \ No newline at end of file + _install_for_instance(subexc, seen) From d53ab289802c38f82c1c618d8f0704c93c353bc7 Mon Sep 17 00:00:00 2001 From: Tim Maxwell Date: Thu, 5 Oct 2023 18:07:56 -0700 Subject: [PATCH 3/5] bump version; test more stuff in pre-3.11 --- setup.py | 2 +- tests/test_pickle_exception.py | 17 +++++++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index f47d250..6f4f8fc 100755 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ def read(*names, **kwargs): setup( name='tblib', - version='2.0.0', + version='2.1.0', license='BSD-2-Clause', description='Traceback serialization library.', long_description='{}\n{}'.format( diff --git a/tests/test_pickle_exception.py b/tests/test_pickle_exception.py index c997c75..dbf19da 100644 --- a/tests/test_pickle_exception.py +++ b/tests/test_pickle_exception.py @@ -41,9 +41,10 @@ def test_install(clear_dispatch_table, how, protocol): try: 1 / 0 # noqa: B018 finally: - # In Python 3, the ValueError's __context__ will be the ZeroDivisionError + # The ValueError's __context__ will be the ZeroDivisionError raise ValueError("blah") except Exception as e: + assert isinstance(e.__context__, ZeroDivisionError) # Python 3 only syntax # raise CustomError("foo") from e new_e = CustomError('foo') @@ -74,16 +75,20 @@ def test_install(clear_dispatch_table, how, protocol): assert exc.x == 1 if has_python3: assert exc.__traceback__ is not None + assert isinstance(exc.__cause__, ValueError) assert exc.__cause__.__traceback__ is not None assert exc.__cause__.x == 2 assert exc.__cause__.__cause__ is None + + assert isinstance(exc.__cause__.__context__, ZeroDivisionError) + assert exc.__cause__.__context__.x == 3 + assert exc.__cause__.__context__.__cause__ is None + assert exc.__cause__.__context__.__context__ is None + if has_python311: assert exc.__notes__ == ["note 1", "note 2"] - assert isinstance(exc.__cause__.__context__, ZeroDivisionError) - assert exc.__cause__.__context__.x == 3 - assert exc.__cause__.__context__.__cause__ is None - assert exc.__cause__.__context__.__context__ is None + @@ -106,7 +111,7 @@ def test_install_decorator(): assert exc.__traceback__ is not None -@pytest.mark.skipif(sys.version_info < (3,11), reason="no BaseExceptionGroup before Python 3.11") +@pytest.mark.skipif(not has_python311, reason="no BaseExceptionGroup before Python 3.11") def test_install_instance_recursively(clear_dispatch_table): exc = BaseExceptionGroup( "test", From a64b5309dfb1a146e252dc2623a3bed993bf2110 Mon Sep 17 00:00:00 2001 From: Tim Maxwell Date: Fri, 6 Oct 2023 12:27:24 -0700 Subject: [PATCH 4/5] Run linter --- src/tblib/pickling_support.py | 8 +++++--- tests/test_pickle_exception.py | 26 +++++++++----------------- 2 files changed, 14 insertions(+), 20 deletions(-) diff --git a/src/tblib/pickling_support.py b/src/tblib/pickling_support.py index 8ad250b..ecfdbac 100644 --- a/src/tblib/pickling_support.py +++ b/src/tblib/pickling_support.py @@ -51,13 +51,14 @@ def pickle_exception(obj): return ( unpickle_exception, - rv[:2] + ( + rv[:2] + + ( obj.__cause__, obj.__traceback__, obj.__context__, obj.__suppress_context__, # __notes__ doesn't exist prior to Python 3.11; and even on Python 3.11 it may be absent - getattr(obj, "__notes__", None), + getattr(obj, '__notes__', None), ), ) + rv[2:] @@ -98,6 +99,7 @@ def install(*exc_classes_or_instances): else: raise TypeError('Expected subclasses or instances of BaseException, got %s' % (type(exc))) + def _install_for_instance(exc, seen): assert isinstance(exc, BaseException) @@ -117,7 +119,7 @@ def _install_for_instance(exc, seen): # This case is meant to cover BaseExceptionGroup on Python 3.11 as well as backports like the # exceptiongroup module - if hasattr(exc, "exceptions") and isinstance(exc.exceptions, (tuple, list)): + if hasattr(exc, 'exceptions') and isinstance(exc.exceptions, (tuple, list)): for subexc in exc.exceptions: if isinstance(subexc, BaseException): _install_for_instance(subexc, seen) diff --git a/tests/test_pickle_exception.py b/tests/test_pickle_exception.py index dbf19da..12952fb 100644 --- a/tests/test_pickle_exception.py +++ b/tests/test_pickle_exception.py @@ -42,7 +42,7 @@ def test_install(clear_dispatch_table, how, protocol): 1 / 0 # noqa: B018 finally: # The ValueError's __context__ will be the ZeroDivisionError - raise ValueError("blah") + raise ValueError('blah') except Exception as e: assert isinstance(e.__context__, ZeroDivisionError) # Python 3 only syntax @@ -51,8 +51,8 @@ def test_install(clear_dispatch_table, how, protocol): if has_python3: new_e.__cause__ = e if has_python311: - new_e.add_note("note 1") - new_e.add_note("note 2") + new_e.add_note('note 1') + new_e.add_note('note 2') raise new_e except Exception as e: exc = e @@ -87,9 +87,7 @@ def test_install(clear_dispatch_table, how, protocol): assert exc.__cause__.__context__.__context__ is None if has_python311: - assert exc.__notes__ == ["note 1", "note 2"] - - + assert exc.__notes__ == ['note 1', 'note 2'] @tblib.pickling_support.install @@ -111,21 +109,15 @@ def test_install_decorator(): assert exc.__traceback__ is not None -@pytest.mark.skipif(not has_python311, reason="no BaseExceptionGroup before Python 3.11") +@pytest.mark.skipif(not has_python311, reason='no BaseExceptionGroup before Python 3.11') def test_install_instance_recursively(clear_dispatch_table): - exc = BaseExceptionGroup( - "test", - [ - ValueError("foo"), - CustomError("bar") - ] - ) - exc.exceptions[0].__cause__ = ZeroDivisionError("baz") - exc.exceptions[0].__cause__.__context__ = AttributeError("quux") + exc = BaseExceptionGroup('test', [ValueError('foo'), CustomError('bar')]) + exc.exceptions[0].__cause__ = ZeroDivisionError('baz') + exc.exceptions[0].__cause__.__context__ = AttributeError('quux') tblib.pickling_support.install(exc) - installed = set(c for c in copyreg.dispatch_table if issubclass(c, BaseException)) + installed = {c for c in copyreg.dispatch_table if issubclass(c, BaseException)} assert installed == {ExceptionGroup, ValueError, CustomError, ZeroDivisionError, AttributeError} From b6215ef0bbac76db92f8dcdca9e4ba9237e513c7 Mon Sep 17 00:00:00 2001 From: Tim Maxwell Date: Fri, 6 Oct 2023 12:28:53 -0700 Subject: [PATCH 5/5] Fix remaining error manually --- tests/test_pickle_exception.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_pickle_exception.py b/tests/test_pickle_exception.py index 12952fb..d28fb14 100644 --- a/tests/test_pickle_exception.py +++ b/tests/test_pickle_exception.py @@ -44,7 +44,6 @@ def test_install(clear_dispatch_table, how, protocol): # The ValueError's __context__ will be the ZeroDivisionError raise ValueError('blah') except Exception as e: - assert isinstance(e.__context__, ZeroDivisionError) # Python 3 only syntax # raise CustomError("foo") from e new_e = CustomError('foo')