fix: intermittent segfault in PyPy (#3089)

* debug: add debugging flag

* test: debug

* fix: allocate memory for 2d arrays in tests

* fix: remove legacy parameter from api

* debug: exclude some tests that rely on 2d array parameters in cpu kernels

* debug: exclude cpu kernel unt tests with 2d arrays as parameters

* fix: typo

* test: remove coverage for pypy

* fix: restore tests

* fix: restore cpu kernel unit tests

* fix: remove debugger

* fix: restore tests

* fix: add more tests

---------

Co-authored-by: Jim Pivarski <jpivarski@users.noreply.github.com>

.github/workflows/test.yml

@@ -139,16 +139,21 @@ jobs:
       - name: Test CPU kernels with explicitly defined values
         run: python -m pytest -vv -rs awkward-cpp/tests-cpu-kernels-explicit
 
-      - name: Test non-kernels
+      - name: Test non-kernels (Python)
         run: >-
           python -m pytest -vv -rs tests --cov=awkward --cov-report=term
           --cov-report=xml
+        if: startsWith(matrix.python-version, '3.')
+
+      - name: Test non-kernels (PyPy)
+        run: >-
+          python -m pytest -vv -rs tests
+        if: startsWith(matrix.python-version, 'pypy')
 
       - name: Upload Codecov results
         if: (matrix.python-version == '3.9') && (matrix.runs-on == 'ubuntu-latest')
         uses: codecov/codecov-action@v4
 
-
   Linux-ROOT:
     runs-on: ubuntu-20.04
 

awkward-cpp/src/cpu-kernels/awkward_UnionArray_flatten_length.cpp

@@ -13,11 +13,11 @@ ERROR awkward_UnionArray_flatten_length(
   T** offsetsraws) {
   *total_length = 0;
   for (int64_t i = 0;  i < length;  i++) {
-    FROMTAGS tag = fromtags[i];
-    FROMINDEX idx = fromindex[i];
+    int64_t tag = (int64_t)fromtags[i];
+    int64_t idx = (int64_t)fromindex[i];
     T start = offsetsraws[tag][idx];
     T stop = offsetsraws[tag][idx + 1];
-    *total_length = *total_length + (stop - start);
+    *total_length = *total_length + (int64_t)(stop - start);
   }
   return success();
 }

awkward-cpp/src/cpu-kernels/awkward_unique_ranges.cpp

@@ -7,7 +7,6 @@
 template <typename T>
 ERROR awkward_unique_ranges(
   T* toptr,
-  int64_t /* length */,   // FIXME: this argument is not needed
   const int64_t* fromoffsets,
   int64_t offsetslength,
   int64_t* tooffsets) {
@@ -28,139 +27,119 @@ ERROR awkward_unique_ranges(
 
 ERROR awkward_unique_ranges_int8(
   int8_t* toptr,
-  int64_t length,
   const int64_t* fromoffsets,
   int64_t offsetslength,
   int64_t* tooffsets) {
     return awkward_unique_ranges<int8_t>(
       toptr,
-      length,
       fromoffsets,
       offsetslength,
       tooffsets);
 }
 
 ERROR awkward_unique_ranges_uint8(
   uint8_t* toptr,
-  int64_t length,
   const int64_t* fromoffsets,
   int64_t offsetslength,
   int64_t* tooffsets) {
     return awkward_unique_ranges<uint8_t>(
       toptr,
-      length,
       fromoffsets,
       offsetslength,
       tooffsets);
 }
 
 ERROR awkward_unique_ranges_int16(
   int16_t* toptr,
-  int64_t length,
   const int64_t* fromoffsets,
   int64_t offsetslength,
   int64_t* tooffsets) {
     return awkward_unique_ranges<int16_t>(
       toptr,
-      length,
       fromoffsets,
       offsetslength,
       tooffsets);
 }
 
 ERROR awkward_unique_ranges_uint16(
   uint16_t* toptr,
-  int64_t length,
   const int64_t* fromoffsets,
   int64_t offsetslength,
   int64_t* tooffsets) {
     return awkward_unique_ranges<uint16_t>(
       toptr,
-      length,
       fromoffsets,
       offsetslength,
       tooffsets);
 }
 
 ERROR awkward_unique_ranges_int32(
   int32_t* toptr,
-  int64_t length,
   const int64_t* fromoffsets,
   int64_t offsetslength,
   int64_t* tooffsets) {
     return awkward_unique_ranges<int32_t>(
       toptr,
-      length,
       fromoffsets,
       offsetslength,
       tooffsets);
 }
 
 ERROR awkward_unique_ranges_uint32(
   uint32_t* toptr,
-  int64_t length,
   const int64_t* fromoffsets,
   int64_t offsetslength,
   int64_t* tooffsets) {
     return awkward_unique_ranges<uint32_t>(
       toptr,
-      length,
       fromoffsets,
       offsetslength,
       tooffsets);
 }
 
 ERROR awkward_unique_ranges_int64(
   int64_t* toptr,
-  int64_t length,
   const int64_t* fromoffsets,
   int64_t offsetslength,
   int64_t* tooffsets) {
     return awkward_unique_ranges<int64_t>(
       toptr,
-      length,
       fromoffsets,
       offsetslength,
       tooffsets);
 }
 
 ERROR awkward_unique_ranges_uint64(
   uint64_t* toptr,
-  int64_t length,
   const int64_t* fromoffsets,
   int64_t offsetslength,
   int64_t* tooffsets) {
     return awkward_unique_ranges<uint64_t>(
       toptr,
-      length,
       fromoffsets,
       offsetslength,
       tooffsets);
 }
 
 ERROR awkward_unique_ranges_float32(
   float* toptr,
-  int64_t length,
   const int64_t* fromoffsets,
   int64_t offsetslength,
   int64_t* tooffsets) {
     return awkward_unique_ranges<float>(
       toptr,
-      length,
       fromoffsets,
       offsetslength,
       tooffsets);
 }
 
 ERROR awkward_unique_ranges_float64(
   double* toptr,
-  int64_t length,
   const int64_t* fromoffsets,
   int64_t offsetslength,
   int64_t* tooffsets) {
     return awkward_unique_ranges<double>(
       toptr,
-      length,
       fromoffsets,
       offsetslength,
       tooffsets);

awkward-cpp/src/cpu-kernels/awkward_unique_ranges_bool.cpp

@@ -6,7 +6,6 @@
 
 ERROR awkward_unique_ranges_bool(
   bool* toptr,
-  int64_t /* length */,   // FIXME: this argument is not needed
   const int64_t* fromoffsets,
   int64_t offsetslength,
   int64_t* tooffsets) {

awkward-cpp/src/cpu-kernels/kernel-utils.cpp

@@ -60,7 +60,7 @@ void awkward_ListArray_combinations_step(
     }
     if (j + 1 == n) {
       for (int64_t k = 0;  k < n;  k++) {
-        tocarry[k][toindex[k]] = fromindex[k];
+        tocarry[k][toindex[k]] = (T)fromindex[k];
         toindex[k]++;
       }
     }

dev/generate-tests.py

@@ -518,7 +518,10 @@ def gencpukerneltests(specdict):
             )
 
             f.write(
-                "import ctypes\nimport pytest\n\nfrom awkward_cpp.cpu_kernels import lib\n\n"
+                "import ctypes\n"
+                "import numpy as np\n"
+                "import pytest\n\n"
+                "from awkward_cpp.cpu_kernels import lib\n\n"
             )
             num = 1
             if spec.tests == []:
@@ -547,13 +550,29 @@ def gencpukerneltests(specdict):
                         elif count == 2:
                             f.write(
                                 " " * 4
-                                + f"{arg}_ptr = (ctypes.POINTER(ctypes.c_{typename}) * len({arg}))()\n"
+                                + f"{typename}Ptr = ctypes.POINTER(ctypes.c_{typename})\n"
                                 + " " * 4
-                                + f"for i in range(len({arg})):\n"
+                                + f"{typename}PtrPtr = ctypes.POINTER({typename}Ptr)\n"
+                                + " " * 4
+                                + f"dim0 = len({arg})\n"
+                                + " " * 4
+                                + f"dim1 = len({arg}[0])\n"
+                                + " " * 4
+                                + f"{arg}_np_arr_2d = np.empty([dim0, dim1], dtype=np.{typename})\n"
+                                + " " * 4
+                                + "for i in range(dim0):\n"
                                 + " " * 8
-                                + f"{arg}_ptr[i] = (ctypes.c_{typename} * len({arg}[i]))(*{arg}[i])\n"
+                                + "for j in range(dim1):\n"
+                                + " " * 12
+                                + f"{arg}_np_arr_2d[i][j] = {arg}[i][j]\n"
+                                + " " * 4
+                                + f"{arg}_ct_arr = np.ctypeslib.as_ctypes({arg}_np_arr_2d)\n"
+                                + " " * 4
+                                + f"{typename}PtrArr = {typename}Ptr * {arg}_ct_arr._length_\n"
                                 + " " * 4
-                                + f"{arg} = {arg}_ptr\n"
+                                + f"{arg}_ct_ptr = ctypes.cast({typename}PtrArr(*(ctypes.cast(row, {typename}Ptr) for row in {arg}_ct_arr)), {typename}PtrPtr)\n"
+                                + " " * 4
+                                + f"{arg} = {arg}_ct_ptr\n"
                             )
                 f.write(" " * 4 + "funcC = getattr(lib, '" + spec.name + "')\n")
                 args = ""
@@ -627,6 +646,7 @@ def gencpuunittests(specdict):
 
                 f.write(
                     "import ctypes\n"
+                    "import numpy as np\n"
                     "import pytest\n\n"
                     "from awkward_cpp.cpu_kernels import lib\n\n"
                 )
@@ -662,13 +682,29 @@ def gencpuunittests(specdict):
                                 elif count == 2:
                                     f.write(
                                         " " * 4
-                                        + f"{arg}_ptr = (ctypes.POINTER(ctypes.c_{typename}) * len({arg}))()\n"
+                                        + f"{typename}Ptr = ctypes.POINTER(ctypes.c_{typename})\n"
+                                        + " " * 4
+                                        + f"{typename}PtrPtr = ctypes.POINTER({typename}Ptr)\n"
+                                        + " " * 4
+                                        + f"dim0 = len({arg})\n"
+                                        + " " * 4
+                                        + f"dim1 = len({arg}[0])\n"
+                                        + " " * 4
+                                        + f"{arg}_np_arr_2d = np.empty([dim0, dim1], dtype=np.{typename})\n"
                                         + " " * 4
-                                        + f"for i in range(len({arg})):\n"
+                                        + "for i in range(dim0):\n"
                                         + " " * 8
-                                        + f"{arg}_ptr[i] = (ctypes.c_{typename} * len({arg}[i]))(*{arg}[i])\n"
+                                        + "for j in range(dim1):\n"
+                                        + " " * 12
+                                        + f"{arg}_np_arr_2d[i][j] = {arg}[i][j]\n"
                                         + " " * 4
-                                        + f"{arg} = {arg}_ptr\n"
+                                        + f"{arg}_ct_arr = np.ctypeslib.as_ctypes({arg}_np_arr_2d)\n"
+                                        + " " * 4
+                                        + f"{typename}PtrArr = {typename}Ptr * {arg}_ct_arr._length_\n"
+                                        + " " * 4
+                                        + f"{arg}_ct_ptr = ctypes.cast({typename}PtrArr(*(ctypes.cast(row, {typename}Ptr) for row in {arg}_ct_arr)), {typename}PtrPtr)\n"
+                                        + " " * 4
+                                        + f"{arg} = {arg}_ct_ptr\n"
                                     )
                         for arg, val in test["inputs"].items():
                             typename = gettype(arg, spec.args)
@@ -684,13 +720,29 @@ def gencpuunittests(specdict):
                                 elif count == 2:
                                     f.write(
                                         " " * 4
-                                        + f"{arg}_ptr = (ctypes.POINTER(ctypes.c_{typename}) * len({arg}))()\n"
+                                        + f"{typename}Ptr = ctypes.POINTER(ctypes.c_{typename})\n"
+                                        + " " * 4
+                                        + f"{typename}PtrPtr = ctypes.POINTER({typename}Ptr)\n"
+                                        + " " * 4
+                                        + f"dim0 = len({arg})\n"
                                         + " " * 4
-                                        + f"for i in range(len({arg})):\n"
+                                        + f"dim1 = len({arg}[0])\n"
+                                        + " " * 4
+                                        + f"{arg}_np_arr_2d = np.empty([dim0, dim1], dtype=np.{typename})\n"
+                                        + " " * 4
+                                        + "for i in range(dim0):\n"
                                         + " " * 8
-                                        + f"{arg}_ptr[i] = (ctypes.c_{typename} * len({arg}[i]))(*{arg}[i])\n"
+                                        + "for j in range(dim1):\n"
+                                        + " " * 12
+                                        + f"{arg}_np_arr_2d[i][j] = {arg}[i][j]\n"
+                                        + " " * 4
+                                        + f"{arg}_ct_arr = np.ctypeslib.as_ctypes({arg}_np_arr_2d)\n"
+                                        + " " * 4
+                                        + f"{typename}PtrArr = {typename}Ptr * {arg}_ct_arr._length_\n"
+                                        + " " * 4
+                                        + f"{arg}_ct_ptr = ctypes.cast({typename}PtrArr(*(ctypes.cast(row, {typename}Ptr) for row in {arg}_ct_arr)), {typename}PtrPtr)\n"
                                         + " " * 4
-                                        + f"{arg} = {arg}_ptr\n"
+                                        + f"{arg} = {arg}_ct_ptr\n"
                                     )
 
                         f.write(" " * 4 + "funcC = getattr(lib, '" + spec.name + "')\n")
@@ -1183,7 +1235,7 @@ def genunittests():
             os.path.join(CURRENT_DIR, "..", "awkward-cpp", "tests-spec-explicit", func),
             "w",
         ) as file:
-            file.write("import pytest\nimport numpy\nimport kernels\n\n")
+            file.write("import pytest\n" "import numpy\n" "import kernels\n\n")
             for test in function["tests"]:
                 num += 1
                 funcName = "def test_" + function["name"] + "_" + str(num) + "():\n"