Merge pull request #14828 from meeseeksmachine/auto-backport-of-pr-14…

…810-on-v5.3.x

Backport PR #14810 on branch v5.3.x (FITS_rec._scale_back(): Incorrectly checks heapsize for TFORMn='Q')

astropy/io/fits/fitsrec.py

@@ -1130,7 +1130,7 @@ def _scale_back(self, update_heap_pointers=True):
                 # Even if this VLA has not been read or updated, we need to
                 # include the size of its constituent arrays in the heap size
                 # total
-                if heapsize >= 2**31:
+                if type(recformat) == _FormatP and heapsize >= 2**31:
                     raise ValueError(
                         "The heapsize limit for 'P' format has been reached. "
                         "Please consider using the 'Q' format for your file."

astropy/io/fits/tests/test_table.py

@@ -3230,6 +3230,29 @@ def test_heapsize_P_limit(self):
         ):
             t.writeto(self.temp("matrix.fits"))
 
+    @pytest.mark.skipif(sys.maxsize < 2**32, reason="requires 64-bit system")
+    @pytest.mark.skipif(sys.platform == "win32", reason="Cannot test on Windows")
+    @pytest.mark.hugemem
+    def test_heapsize_Q_limit(self):
+        """
+        Regression test for https://github.com/astropy/astropy/issues/14808
+
+        Check if the error is no longer raised when the heap size is bigger than what can be
+        indexed with a 32 bit signed int.
+        """
+
+        # a matrix with variable length array elements is created
+        nelem = 2**28
+        matrix = np.zeros(1, dtype=np.object_)
+        matrix[0] = np.arange(0.0, float(nelem + 1))
+
+        col = fits.Column(name="MATRIX", format=f"QD({nelem})", unit="", array=matrix)
+
+        t = fits.BinTableHDU.from_columns([col])
+        t.name = "MATRIX"
+
+        t.writeto(self.temp("matrix.fits"))
+
     def test_empty_vla_raw_data(self):
         """
         Regression test for https://github.com/astropy/astropy/issues/12881

docs/changes/io.fits/14810.bugfix.rst

@@ -0,0 +1,2 @@
+Fixes an issue where FITS_rec was incorrectly raising a ValueError exception when the heapsize was greater than 2**31
+when the Column type was 'Q' instead of 'P'.