Add a context manager that records errors, links them, and raises

CHANGES.txt

@@ -19,6 +19,9 @@ Deprecations:
 
 New Features:
 
+- Errors that GDAL handles internally within GDALDatasetRasterIO() and
+  GDALRasterIO() and WarpAndChunk() are chained together to be visible and
+  accessable from Python (#2526).
 - The new "rio create" command allows creation of new, empty datasets (#3023).
 - An optional range keyword argument (like that of numpy.histogram()) has been
   added to show_hist() (#2873, #3001).

rasterio/_env.pyx

@@ -69,7 +69,11 @@ cdef VSIFilesystemPluginCallbacksStruct* filepath_plugin = NULL
 cdef VSIFilesystemPluginCallbacksStruct* pyopener_plugin = NULL
 
 
-cdef void log_error(CPLErr err_class, int err_no, const char* msg) with gil:
+cdef void log_error(
+    CPLErr err_class,
+    int err_no,
+    const char* msg,
+) noexcept with gil:
     """Send CPL errors to Python's logger.
 
     Because this function is called by GDAL with no Python context, we
@@ -97,10 +101,18 @@ cdef void log_error(CPLErr err_class, int err_no, const char* msg) with gil:
 # Definition of GDAL callback functions, one for Windows and one for
 # other platforms. Each calls log_error().
 IF UNAME_SYSNAME == "Windows":
-    cdef void __stdcall logging_error_handler(CPLErr err_class, int err_no, const char* msg) with gil:
+    cdef void __stdcall logging_error_handler(
+        CPLErr err_class,
+        int err_no,
+        const char* msg,
+    ) noexcept with gil:
         log_error(err_class, err_no, msg)
 ELSE:
-    cdef void logging_error_handler(CPLErr err_class, int err_no, const char* msg) with gil:
+    cdef void logging_error_handler(
+        CPLErr err_class,
+        int err_no,
+        const char* msg,
+    ) noexcept with gil:
         log_error(err_class, err_no, msg)
 
 

rasterio/_err.pxd

@@ -2,9 +2,12 @@ include "gdal.pxi"
 
 from libc.stdio cimport *
 
-
 cdef int exc_wrap(int retval) except -1
 cdef int exc_wrap_int(int retval) except -1
 cdef OGRErr exc_wrap_ogrerr(OGRErr retval) except -1
 cdef void *exc_wrap_pointer(void *ptr) except NULL
 cdef VSILFILE *exc_wrap_vsilfile(VSILFILE *f) except NULL
+
+cdef class StackChecker:
+    cdef object error_stack
+    cdef int exc_wrap_int(self, int retval) except -1

rasterio/_err.pyx

@@ -1,14 +1,21 @@
+# cython: c_string_type=unicode, c_string_encoding=utf8
 """rasterio._err
 
 Exception-raising wrappers for GDAL API functions.
 
 """
 
+import contextlib
+from contextvars import ContextVar
 from enum import IntEnum
+from itertools import zip_longest
 import logging
 
 log = logging.getLogger(__name__)
 
+_ERROR_STACK = ContextVar("error_stack")
+_ERROR_STACK.set([])
+
 _GDAL_DEBUG_DOCS = (
     "https://rasterio.readthedocs.io/en/latest/topics/errors.html"
     "#debugging-internal-gdal-functions"
@@ -106,6 +113,14 @@ class CPLE_AWSError(CPLE_BaseError):
     pass
 
 
+cdef dict _LEVEL_MAP = {
+    0: 0,
+    1: logging.DEBUG,
+    2: logging.WARNING,
+    3: logging.ERROR,
+    4: logging.CRITICAL
+}
+
 # Map of GDAL error numbers to the Python exceptions.
 exception_map = {
     1: CPLE_AppDefinedError,
@@ -130,6 +145,26 @@ exception_map = {
     17: CPLE_AWSError
 }
 
+cdef dict _CODE_MAP = {
+    0: 'CPLE_None',
+    1: 'CPLE_AppDefined',
+    2: 'CPLE_OutOfMemory',
+    3: 'CPLE_FileIO',
+    4: 'CPLE_OpenFailed',
+    5: 'CPLE_IllegalArg',
+    6: 'CPLE_NotSupported',
+    7: 'CPLE_AssertionFailed',
+    8: 'CPLE_NoWriteAccess',
+    9: 'CPLE_UserInterrupt',
+    10: 'ObjectNull',
+    11: 'CPLE_HttpResponse',
+    12: 'CPLE_AWSBucketNotFound',
+    13: 'CPLE_AWSObjectNotFound',
+    14: 'CPLE_AWSAccessDenied',
+    15: 'CPLE_AWSInvalidCredentials',
+    16: 'CPLE_AWSSignatureDoesNotMatch',
+    17: 'CPLE_AWSError'
+}
 
 # CPL Error types as an enum.
 class GDALError(IntEnum):
@@ -148,26 +183,25 @@ cdef inline object exc_check():
     An Exception, SystemExit, or None
 
     """
-    cdef const char *msg_c = NULL
+    cdef const char *msg = NULL
 
     err_type = CPLGetLastErrorType()
     err_no = CPLGetLastErrorNo()
-    msg_c = CPLGetLastErrorMsg()
+    msg = CPLGetLastErrorMsg()
 
-    if msg_c == NULL:
-        msg = "No error message."
+    if msg == NULL:
+        message = "No error message."
     else:
-        msg_b = msg_c
-        msg = msg_b.decode('utf-8')
-        msg = msg.replace("`", "'")
-        msg = msg.replace("\n", " ")
+        message = msg
+        message = message.replace("`", "'")
+        message = message.replace("\n", " ")
 
     if err_type == 3:
-        exception = exception_map.get(err_no, CPLE_BaseError)(err_type, err_no, msg)
+        exception = exception_map.get(err_no, CPLE_BaseError)(err_type, err_no, message)
         CPLErrorReset()
         return exception
     elif err_type == 4:
-        exception = SystemExit("Fatal error: {0}".format((err_type, err_no, msg)))
+        exception = SystemExit("Fatal error: {0}".format((err_type, err_no, message)))
         CPLErrorReset()
         return exception
     else:
@@ -183,11 +217,71 @@ cdef int exc_wrap(int retval) except -1:
     return retval
 
 
+cdef void log_error(
+    CPLErr err_class,
+    int err_no,
+    const char* msg,
+) noexcept with gil:
+    """Send CPL errors to Python's logger.
+
+    Because this function is called by GDAL with no Python context, we
+    can't propagate exceptions that we might raise here. They'll be
+    ignored.
+
+    """
+    if err_no in _CODE_MAP:
+        # We've observed that some GDAL functions may emit multiple
+        # ERROR level messages and yet succeed. We want to see those
+        # messages in our log file, but not at the ERROR level. We
+        # turn the level down to INFO.
+        if err_class == 3:
+            log.info(
+                "GDAL signalled an error: err_no=%r, msg=%r",
+                err_no,
+                msg
+            )
+        elif err_no == 0:
+            log.log(_LEVEL_MAP[err_class], "%s", msg)
+        else:
+            log.log(_LEVEL_MAP[err_class], "%s:%s", _CODE_MAP[err_no], msg)
+    else:
+        log.info("Unknown error number %r", err_no)
+
+
+IF UNAME_SYSNAME == "Windows":
+    cdef void __stdcall chaining_error_handler(
+        CPLErr err_class,
+        int err_no,
+        const char* msg
+    ) noexcept with gil:
+        global _ERROR_STACK
+        log_error(err_class, err_no, msg)
+        if err_class == 3:
+            stack = _ERROR_STACK.get()
+            stack.append(
+                exception_map.get(err_no, CPLE_BaseError)(err_class, err_no, msg),
+            )
+            _ERROR_STACK.set(stack)
+ELSE:
+    cdef void chaining_error_handler(
+        CPLErr err_class,
+        int err_no,
+        const char* msg
+    ) noexcept with gil:
+        global _ERROR_STACK
+        log_error(err_class, err_no, msg)
+        if err_class == 3:
+            stack = _ERROR_STACK.get()
+            stack.append(
+                exception_map.get(err_no, CPLE_BaseError)(err_class, err_no, msg),
+            )
+            _ERROR_STACK.set(stack)
+
+
 cdef int exc_wrap_int(int err) except -1:
     """Wrap a GDAL/OGR function that returns CPLErr or OGRErr (int)
 
     Raises a Rasterio exception if a non-fatal error has be set.
-
     """
     if err:
         exc = exc_check()
@@ -209,11 +303,55 @@ cdef OGRErr exc_wrap_ogrerr(OGRErr err) except -1:
         raise CPLE_BaseError(3, err, "OGR Error code {}".format(err))
 
 
+cdef class StackChecker:
+
+    def __init__(self, error_stack=None):
+        self.error_stack = error_stack or {}
+
+    cdef int exc_wrap_int(self, int err) except -1:
+        """Wrap a GDAL/OGR function that returns CPLErr (int).
+
+        Raises a Rasterio exception if a non-fatal error has be set.
+        """
+        if err:
+            stack = self.error_stack.get()
+            for error, cause in zip_longest(stack[::-1], stack[::-1][1:]):
+                if error is not None and cause is not None:
+                    error.__cause__ = cause
+
+            if stack:
+                last = stack.pop()
+                if last is not None:
+                    raise last
+
+        return err
+
+
+@contextlib.contextmanager
+def stack_errors():
+    # TODO: better name?
+    # Note: this manager produces one chain of errors and thus assumes
+    # that no more than one GDAL function is called.
+    CPLErrorReset()
+    global _ERROR_STACK
+    _ERROR_STACK.set([])
+
+    # chaining_error_handler (better name a TODO) records GDAL errors
+    # in the order they occur and converts to exceptions.
+    CPLPushErrorHandlerEx(<CPLErrorHandler>chaining_error_handler, NULL)
+
+    # Run code in the `with` block.
+    yield StackChecker(_ERROR_STACK)
+
+    CPLPopErrorHandler()
+    _ERROR_STACK.set([])
+    CPLErrorReset()
+
+
 cdef void *exc_wrap_pointer(void *ptr) except NULL:
     """Wrap a GDAL/OGR function that returns GDALDatasetH etc (void *)
 
     Raises a Rasterio exception if a non-fatal error has be set.
-
     """
     if ptr == NULL:
         exc = exc_check()