py312: changes in PyLong internals

The layout for python integers changed in python 3.12.
We add a module `sage.cpython.pycore_long` which copies the new
(internal) api of PyLong from python 3.12. We also implement fallback
version of these functions suitable for python pre-3.12.

Note the files implementing the `pycore_long` module (`pycore_long.pxd`
and `pycore_long.h`) are shared with fpylll and cypari2.

src/sage/arith/long.pxd

@@ -19,6 +19,8 @@ from libc.limits cimport LONG_MIN, LONG_MAX
 from cpython.object cimport Py_SIZE
 from cpython.number cimport PyNumber_Index, PyIndex_Check
 from cpython.longintrepr cimport py_long, PyLong_SHIFT, digit
+from sage.cpython.pycore_long cimport (
+    ob_digit, _PyLong_IsNegative, _PyLong_DigitCount)
 
 from sage.libs.gmp.mpz cimport mpz_fits_slong_p, mpz_get_si
 from sage.rings.integer_fake cimport is_Integer, Integer_AS_MPZ
@@ -299,8 +301,11 @@ cdef inline bint integer_check_long_py(x, long* value, int* err):
         return 0
 
     # x is a Python "int" (aka PyLongObject or py_long in cython)
-    cdef const digit* D = (<py_long>x).ob_digit
-    cdef Py_ssize_t size = Py_SIZE(x)
+    cdef const digit* D = ob_digit(x)
+    cdef Py_ssize_t size = _PyLong_DigitCount(x)
+
+    if _PyLong_IsNegative(x):
+        size = -size
 
     # We assume PyLong_SHIFT <= BITS_IN_LONG <= 3 * PyLong_SHIFT.
     # This is true in all the default configurations:

src/sage/cpython/pycore_long.h

@@ -0,0 +1,98 @@
+#include "Python.h"
+#include <stdbool.h>
+
+#if PY_VERSION_HEX >= 0x030C00A5
+#define ob_digit(o)  (((PyLongObject*)o)->long_value.ob_digit)
+#else
+#define ob_digit(o)  (((PyLongObject*)o)->ob_digit)
+#endif
+
+#if PY_VERSION_HEX >= 0x030C00A7
+// taken from cpython:Include/internal/pycore_long.h @ 3.12
+
+/* Long value tag bits:
+ * 0-1: Sign bits value = (1-sign), ie. negative=2, positive=0, zero=1.
+ * 2: Reserved for immortality bit
+ * 3+ Unsigned digit count
+ */
+#define SIGN_MASK 3
+#define SIGN_ZERO 1
+#define SIGN_NEGATIVE 2
+#define NON_SIZE_BITS 3
+
+static inline bool
+_PyLong_IsZero(const PyLongObject *op)
+{
+    return (op->long_value.lv_tag & SIGN_MASK) == SIGN_ZERO;
+}
+
+static inline bool
+_PyLong_IsNegative(const PyLongObject *op)
+{
+    return (op->long_value.lv_tag & SIGN_MASK) == SIGN_NEGATIVE;
+}
+
+static inline bool
+_PyLong_IsPositive(const PyLongObject *op)
+{
+    return (op->long_value.lv_tag & SIGN_MASK) == 0;
+}
+
+static inline Py_ssize_t
+_PyLong_DigitCount(const PyLongObject *op)
+{
+    assert(PyLong_Check(op));
+    return op->long_value.lv_tag >> NON_SIZE_BITS;
+}
+
+#define TAG_FROM_SIGN_AND_SIZE(sign, size) ((1 - (sign)) | ((size) << NON_SIZE_BITS))
+
+static inline void
+_PyLong_SetSignAndDigitCount(PyLongObject *op, int sign, Py_ssize_t size)
+{
+    assert(size >= 0);
+    assert(-1 <= sign && sign <= 1);
+    assert(sign != 0 || size == 0);
+    op->long_value.lv_tag = TAG_FROM_SIGN_AND_SIZE(sign, (size_t)size);
+}
+
+#else
+// fallback for < 3.12
+
+static inline bool
+_PyLong_IsZero(const PyLongObject *op)
+{
+    return Py_SIZE(op) == 0;
+}
+
+static inline bool
+_PyLong_IsNegative(const PyLongObject *op)
+{
+    return Py_SIZE(op) < 0;
+}
+
+static inline bool
+_PyLong_IsPositive(const PyLongObject *op)
+{
+    return Py_SIZE(op) > 0;
+}
+
+static inline Py_ssize_t
+_PyLong_DigitCount(const PyLongObject *op)
+{
+    Py_ssize_t size = Py_SIZE(op);
+    return size < 0 ? -size : size;
+}
+
+static inline void
+_PyLong_SetSignAndDigitCount(PyLongObject *op, int sign, Py_ssize_t size)
+{
+#if (PY_MAJOR_VERSION == 3) && (PY_MINOR_VERSION < 9)
+// The function Py_SET_SIZE is defined starting with python 3.9.
+    Py_SIZE(o) = size;
+#else
+    Py_SET_SIZE(op, sign < 0 ? -size : size);
+#endif
+}
+
+#endif

src/sage/cpython/pycore_long.pxd

@@ -0,0 +1,9 @@
+from cpython.longintrepr cimport py_long, digit
+
+cdef extern from "pycore_long.h":
+    digit* ob_digit(py_long o)
+    bint _PyLong_IsZero(py_long o)
+    bint _PyLong_IsNegative(py_long o)
+    bint _PyLong_IsPositive(py_long o)
+    Py_ssize_t _PyLong_DigitCount(py_long o)
+    void _PyLong_SetSignAndDigitCount(py_long o, int sign, Py_ssize_t size)

src/sage/libs/gmp/pylong.pxd

@@ -2,9 +2,10 @@
 Various functions to deal with conversion mpz <-> Python int/long
 """
 
+from cpython.longintrepr cimport py_long
 from sage.libs.gmp.types cimport *
 
 cdef mpz_get_pylong(mpz_srcptr z)
 cdef mpz_get_pyintlong(mpz_srcptr z)
-cdef int mpz_set_pylong(mpz_ptr z, L) except -1
+cdef int mpz_set_pylong(mpz_ptr z, py_long L) except -1
 cdef Py_hash_t mpz_pythonhash(mpz_srcptr z)

src/sage/libs/gmp/pylong.pyx

@@ -28,6 +28,8 @@ AUTHORS:
 from cpython.object cimport Py_SIZE
 from cpython.long cimport PyLong_FromLong
 from cpython.longintrepr cimport _PyLong_New, py_long, digit, PyLong_SHIFT
+from sage.cpython.pycore_long cimport (ob_digit, _PyLong_IsNegative,
+    _PyLong_DigitCount, _PyLong_SetSignAndDigitCount)
 from .mpz cimport *
 
 cdef extern from *:
@@ -60,12 +62,9 @@ cdef mpz_get_pylong_large(mpz_srcptr z):
     """
     cdef size_t nbits = mpz_sizeinbase(z, 2)
     cdef size_t pylong_size = (nbits + PyLong_SHIFT - 1) // PyLong_SHIFT
-    L = _PyLong_New(pylong_size)
-    mpz_export(L.ob_digit, NULL,
-            -1, sizeof(digit), 0, PyLong_nails, z)
-    if mpz_sgn(z) < 0:
-        # Set correct size
-        Py_SET_SIZE(L, -pylong_size)
+    cdef py_long L = _PyLong_New(pylong_size)
+    mpz_export(ob_digit(L), NULL, -1, sizeof(digit), 0, PyLong_nails, z)
+    _PyLong_SetSignAndDigitCount(L, mpz_sgn(z), pylong_size)
     return L
 
 
@@ -88,16 +87,13 @@ cdef mpz_get_pyintlong(mpz_srcptr z):
     return mpz_get_pylong_large(z)
 
 
-cdef int mpz_set_pylong(mpz_ptr z, L) except -1:
+cdef int mpz_set_pylong(mpz_ptr z, py_long L) except -1:
     """
     Convert a Python ``long`` `L` to an ``mpz``.
     """
-    cdef Py_ssize_t pylong_size = Py_SIZE(L)
-    if pylong_size < 0:
-        pylong_size = -pylong_size
-    mpz_import(z, pylong_size, -1, sizeof(digit), 0, PyLong_nails,
-            (<py_long>L).ob_digit)
-    if Py_SIZE(L) < 0:
+    cdef Py_ssize_t pylong_size = _PyLong_DigitCount(L)
+    mpz_import(z, pylong_size, -1, sizeof(digit), 0, PyLong_nails, ob_digit(L))
+    if _PyLong_IsNegative(L):
         mpz_neg(z, z)
 
 

src/sage/symbolic/ginac/numeric.cpp

@@ -67,6 +67,7 @@
 #include "archive.h"
 #include "tostring.h"
 #include "utils.h"
+#include "../../cpython/pycore_long.h"
 
 #pragma clang diagnostic push
 #pragma clang diagnostic ignored "-Wdeprecated-register"
@@ -706,18 +707,12 @@ static long _mpq_pythonhash(mpq_t the_rat)
 // Initialize an mpz_t from a Python long integer
 static void _mpz_set_pylong(mpz_t z, PyLongObject* l)
 {
-    Py_ssize_t pylong_size = Py_SIZE(l);
-    int sign = 1;
-
-    if (pylong_size < 0) {
-        pylong_size = -pylong_size;
-        sign = -1;
-    }
+    Py_ssize_t pylong_size = _PyLong_DigitCount(l);
 
     mpz_import(z, pylong_size, -1, sizeof(digit), 0,
-               8*sizeof(digit) - PyLong_SHIFT, l->ob_digit);
+               8*sizeof(digit) - PyLong_SHIFT, ob_digit(l));
 
-    if (sign < 0)
+    if (_PyLong_IsNegative(l))
         mpz_neg(z, z);
 }