GH-38626: [Python] Fix segfault when PyArrow is imported at shutdown (#…

…38637)

### Rationale for this change

Some C++ destructors may be called after the Python interpreter has ceased to exist.
If such a destructor tries to call back in the Python interpreter, for example by calling `Py_DECREF`, we get a crash.

### What changes are included in this PR?

Protect `OwnedRef` and `OwneRefNoGIL` destructors against decref'ing a Python object after Python finalization.

### Are these changes tested?

Yes.

### Are there any user-facing changes?

No.
* Closes: #38626

Authored-by: Antoine Pitrou <antoine@python.org>
Signed-off-by: Joris Van den Bossche <jorisvandenbossche@gmail.com>

python/pyarrow/src/arrow/python/common.h

@@ -188,7 +188,12 @@ class ARROW_PYTHON_EXPORT OwnedRef {
     return *this;
   }
 
-  ~OwnedRef() { reset(); }
+  ~OwnedRef() {
+    // GH-38626: destructor may be called after the Python interpreter is finalized.
+    if (Py_IsInitialized()) {
+      reset();
+    }
+  }
 
   void reset(PyObject* obj) {
     Py_XDECREF(obj_);
@@ -225,13 +230,11 @@ class ARROW_PYTHON_EXPORT OwnedRefNoGIL : public OwnedRef {
   explicit OwnedRefNoGIL(PyObject* obj) : OwnedRef(obj) {}
 
   ~OwnedRefNoGIL() {
-    // This destructor may be called after the Python interpreter is finalized.
-    // At least avoid spurious attempts to take the GIL when not necessary.
-    if (obj() == NULLPTR) {
-      return;
+    // GH-38626: destructor may be called after the Python interpreter is finalized.
+    if (Py_IsInitialized() && obj() != NULLPTR) {
+      PyAcquireGIL lock;
+      reset();
     }
-    PyAcquireGIL lock;
-    reset();
   }
 };
 

python/pyarrow/tests/test_misc.py

@@ -117,6 +117,19 @@ def test_runtime_info():
         subprocess.check_call([sys.executable, "-c", code], env=env)
 
 
+def test_import_at_shutdown():
+    # GH-38626: importing PyArrow at interpreter shutdown would crash
+    code = """if 1:
+        import atexit
+
+        def import_arrow():
+            import pyarrow
+
+        atexit.register(import_arrow)
+        """
+    subprocess.check_call([sys.executable, "-c", code])
+
+
 @pytest.mark.skipif(sys.platform == "win32",
                     reason="Path to timezone database is not configurable "
                            "on non-Windows platforms")