BF-36

.gitignore

@@ -10,4 +10,5 @@ firefox-102.2.0/*
 firefox-*/*
 firefox-*/
 tests/__pycache__/*
-tests/python/__pycache__/*
+tests/python/__pycache__/*
+Testing/Temporary

include/IntType.hh

@@ -1,11 +1,11 @@
 /**
  * @file IntType.hh
- * @author Caleb Aikens (caleb@distributive.network) & Giovanni Tedesco (giovanni@distributive.network)
+ * @author Caleb Aikens (caleb@distributive.network) & Giovanni Tedesco (giovanni@distributive.network) & Tom Tang (xmader@distributive.network)
  * @brief Struct for representing python ints
- * @version 0.1
- * @date 2022-07-27
+ * @version 0.2
+ * @date 2023-03-16
  *
- * @copyright Copyright (c) 2022
+ * @copyright Copyright (c) 2023
  *
  */
 
@@ -15,19 +15,36 @@
 #include "PyType.hh"
 #include "TypeEnum.hh"
 
+#include <jsapi.h>
+
 #include <Python.h>
 
 #include <iostream>
 
 /**
- * @brief This struct represents the 'int' type in Python, which is represented as a 'long' in C++. It inherits from the PyType struct
+ * @brief This struct represents the 'int' type (arbitrary-precision) in Python. It inherits from the PyType struct
  */
 struct IntType : public PyType {
 public:
   IntType(PyObject *object);
   IntType(long n);
+
+  /**
+   * @brief Construct a new IntType object from a JS::BigInt.
+   *
+   * @param cx - javascript context pointer
+   * @param str - JS::BigInt pointer
+   */
+  IntType(JSContext *cx, JS::BigInt *bigint);
+
   const TYPE returnType = TYPE::INT;
-  long getValue() const;
+
+  /**
+   * @brief Convert the IntType object to a JS::BigInt
+   *
+   * @param cx - javascript context pointer
+   */
+  JS::BigInt *toJsBigInt(JSContext *cx);
 
 protected:
   virtual void print(std::ostream &os) const override;

include/modules/pythonmonkey/pythonmonkey.hh

@@ -19,7 +19,8 @@
 
 #include <Python.h>
 
-#define PythonMonkey_Null PyObject_GetAttrString(PyState_FindModule(&pythonmonkey), "null") /**< macro for python null object*/
+#define PythonMonkey_Null   PyObject_GetAttrString(PyState_FindModule(&pythonmonkey), "null")   /**< macro for pythonmonkey.null object*/
+#define PythonMonkey_BigInt PyObject_GetAttrString(PyState_FindModule(&pythonmonkey), "bigint") /**< macro for pythonmonkey.bigint class object */
 
 static JSContext *cx; /**< pointer to PythonMonkey's JSContext */
 static JS::Rooted<JSObject *> *global; /**< pointer to the global object of PythonMonkey's JSContext */

src/CMakeLists.txt

@@ -1,12 +1,7 @@
 include_directories(${CMAKE_CURRENT_LIST_DIR})
 
-#list(APPEND EXPLORE_SOURCE_FILES  ${SOURCE_FILES} "${CMAKE_SOURCE_DIR}/src/modules/explore/explore.cc")
 list(APPEND PYTHONMONKEY_SOURCE_FILES ${SOURCE_FILES} "${CMAKE_SOURCE_DIR}/src/modules/pythonmonkey/pythonmonkey.cc")
 
-# add_library(explore SHARED
-#     ${EXPLORE_SOURCE_FILES}
-# )
-
 add_library(pythonmonkey SHARED
     ${PYTHONMONKEY_SOURCE_FILES}
 )
@@ -16,18 +11,9 @@ execute_process(
     OUTPUT_VARIABLE pyloc
 )
 
-# target_include_directories(explore  PUBLIC ..)
 target_include_directories(pythonmonkey PUBLIC ..)
 
 if(WIN32)
-  # set_target_properties(
-  #   explore
-  #   PROPERTIES
-  #     PREFIX ""
-  #     SUFFIX ".pyd"
-  #     OUTPUT_NAME "explore"
-  #     CXX_STANDARD 17
-  # )
   set_target_properties(
     pythonmonkey
     PROPERTIES
@@ -37,23 +23,16 @@ if(WIN32)
       CXX_STANDARD 17
   )
 elseif(UNIX)
-  # set_target_properties(
-  #   explore
-  #   PROPERTIES 
-  #     PREFIX ""
-  #     SUFFIX ".so"
-  # )
   set_target_properties(
     pythonmonkey
     PROPERTIES
     PREFIX ""
     SUFFIX ".so"
   )
 endif()
-# target_link_libraries(explore ${PYTHON_LIBRARIES})
+
 target_link_libraries(pythonmonkey ${PYTHON_LIBRARIES})
 target_link_libraries(pythonmonkey ${SPIDERMONKEY_LIBRARIES})
 
-# target_include_directories(explore PRIVATE ${PYTHON_INCLUDE_DIR})
 target_include_directories(pythonmonkey PRIVATE ${PYTHON_INCLUDE_DIR})
 target_include_directories(pythonmonkey PRIVATE ${SPIDERMONKEY_INCLUDE_DIR})

src/IntType.cc

@@ -1,21 +1,147 @@
+#include "include/modules/pythonmonkey/pythonmonkey.hh"
+
 #include "include/IntType.hh"
 
 #include "include/PyType.hh"
 #include "include/TypeEnum.hh"
 
+#include <jsapi.h>
+#include <js/BigInt.h>
+
 #include <Python.h>
 
 #include <iostream>
+#include <bit>
+
+#define SIGN_BIT_MASK 0b1000 // https://hg.mozilla.org/releases/mozilla-esr102/file/tip/js/src/vm/BigIntType.h#l40
+#define CELL_HEADER_LENGTH 8 // https://hg.mozilla.org/releases/mozilla-esr102/file/tip/js/src/gc/Cell.h#l602
+
+#define JS_DIGIT_BIT JS_BITS_PER_WORD
+#define PY_DIGIT_BIT PYLONG_BITS_IN_DIGIT
+
+#define js_digit_t uintptr_t // https://hg.mozilla.org/releases/mozilla-esr102/file/tip/js/src/vm/BigIntType.h#l36
+#define JS_DIGIT_BYTE (sizeof(js_digit_t)/sizeof(uint8_t))
 
+#define JS_INLINE_DIGIT_MAX_LEN 1 // https://hg.mozilla.org/releases/mozilla-esr102/file/tip/js/src/vm/BigIntType.h#l43
+
+static const char HEX_CHAR_LOOKUP_TABLE[] = "0123456789ABCDEF";
 
 IntType::IntType(PyObject *object) : PyType(object) {}
 
 IntType::IntType(long n) : PyType(Py_BuildValue("i", n)) {}
 
-long IntType::getValue() const {
-  return PyLong_AS_LONG(pyObject);
+IntType::IntType(JSContext *cx, JS::BigInt *bigint) {
+  // Get the sign bit
+  bool isNegative = BigIntIsNegative(bigint);
+
+  // Read the digits count in this JS BigInt
+  //    see https://hg.mozilla.org/releases/mozilla-esr102/file/tip/js/src/vm/BigIntType.h#l48
+  //        https://hg.mozilla.org/releases/mozilla-esr102/file/tip/js/src/gc/Cell.h#l623
+  uint32_t jsDigitCount = ((uint32_t *)bigint)[1];
+
+  // Get all the 64-bit (assuming we compile on 64-bit OS) "digits" from JS BigInt
+  js_digit_t *jsDigits = (js_digit_t *)(((char *)bigint) + CELL_HEADER_LENGTH);
+  if (jsDigitCount > JS_INLINE_DIGIT_MAX_LEN) { // hasHeapDigits
+    // We actually have a pointer to the digit storage if the number cannot fit in one uint64_t
+    //    see https://hg.mozilla.org/releases/mozilla-esr102/file/tip/js/src/vm/BigIntType.h#l54
+    jsDigits = *((js_digit_t **)jsDigits);
+  }
+  //
+  // The digit storage starts with the least significant digit (little-endian digit order).
+  // Byte order within a digit is native-endian.
+
+  if constexpr (std::endian::native == std::endian::big) { // C++20
+    // @TODO (Tom Tang): use C++23 std::byteswap?
+    printf("big-endian cpu is not supported by PythonMonkey yet");
+    return;
+  }
+  // If the native endianness is also little-endian,
+  // we now have consecutive bytes of 8-bit "digits" in little-endian order
+  const uint8_t *bytes = const_cast<const uint8_t *>((uint8_t *)jsDigits);
+  if (jsDigitCount == 0) {
+    // Create a new object instead of reusing the object for int 0
+    //    see https://github.com/python/cpython/blob/3.9/Objects/longobject.c#L862
+    //        https://github.com/python/cpython/blob/3.9/Objects/longobject.c#L310
+    pyObject = (PyObject *)_PyLong_New(0);
+  } else {
+    pyObject = _PyLong_FromByteArray(bytes, jsDigitCount * JS_DIGIT_BYTE, true, false);
+  }
+
+  // Set the sign bit
+  //    see https://github.com/python/cpython/blob/3.9/Objects/longobject.c#L956
+  if (isNegative) {
+    ssize_t pyDigitCount = Py_SIZE(pyObject);
+    Py_SET_SIZE(pyObject, -pyDigitCount);
+  }
+
+  // Cast to a pythonmonkey.bigint to differentiate it from a normal Python int,
+  //  allowing Py<->JS two-way BigInt conversion
+  Py_SET_TYPE(pyObject, (PyTypeObject *)(PythonMonkey_BigInt));
+}
+
+JS::BigInt *IntType::toJsBigInt(JSContext *cx) {
+  // Figure out how many 64-bit "digits" we would have for JS BigInt
+  //    see https://github.com/python/cpython/blob/3.9/Modules/_randommodule.c#L306
+  size_t bitCount = _PyLong_NumBits(pyObject);
+  if (bitCount == (size_t)-1 && PyErr_Occurred())
+    return nullptr;
+  uint32_t jsDigitCount = bitCount == 0 ? 1 : (bitCount - 1) / JS_DIGIT_BIT + 1;
+  // Get the sign bit
+  //    see https://github.com/python/cpython/blob/3.9/Objects/longobject.c#L977
+  ssize_t pyDigitCount = Py_SIZE(pyObject); // negative on negative numbers
+  bool isNegative = pyDigitCount < 0;
+  // Force to make the number positive otherwise _PyLong_AsByteArray would complain
+  //    see https://github.com/python/cpython/blob/3.9/Objects/longobject.c#L980
+  if (isNegative) {
+    Py_SET_SIZE(pyObject, /*abs()*/ -pyDigitCount);
+  }
+
+  JS::BigInt *bigint = nullptr;
+  if (jsDigitCount <= 1) {
+    // Fast path for int fits in one js_digit_t (uint64 on 64-bit OS)
+    bigint = JS::detail::BigIntFromUint64(cx, PyLong_AsUnsignedLongLong(pyObject));
+  } else {
+    // Convert to bytes of 8-bit "digits" in **big-endian** order
+    size_t byteCount = (size_t)JS_DIGIT_BYTE * jsDigitCount;
+    uint8_t *bytes = (uint8_t *)PyMem_Malloc(byteCount);
+    _PyLong_AsByteArray((PyLongObject *)pyObject, bytes, byteCount, /*is_little_endian*/ false, false);
+
+    // Convert pm.bigint to JS::BigInt through hex strings (no public API to convert directly through bytes)
+    // TODO (Tom Tang): We could manually allocate the memory, https://hg.mozilla.org/releases/mozilla-esr102/file/tip/js/src/vm/BigIntType.cpp#l162, but still no public API
+    // TODO (Tom Tang): Could we fill in an object with similar memory alignment (maybe by NewArrayBufferWithContents), and coerce it to BigInt?
+
+    // Calculate the number of chars required to represent the bigint in hex string
+    size_t charCount = byteCount * 2;
+    // Convert bytes to hex string (big-endian)
+    std::vector<char> chars = std::vector<char>(charCount); // can't be null-terminated, otherwise SimpleStringToBigInt would read the extra \0 character and then segfault
+    for (size_t i = 0, j = 0; i < charCount; i += 2, j++) {
+      chars[i] = HEX_CHAR_LOOKUP_TABLE[(bytes[j] >> 4)&0xf]; // high nibble
+      chars[i+1] = HEX_CHAR_LOOKUP_TABLE[bytes[j]&0xf];      // low  nibble
+    }
+    PyMem_Free(bytes);
+
+    // Convert hex string to JS::BigInt
+    mozilla::Span<const char> strSpan = mozilla::Span<const char>(chars); // storing only a pointer to the underlying array and length
+    bigint = JS::SimpleStringToBigInt(cx, strSpan, 16);
+  }
+
+  if (isNegative) {
+    // Make negative number back negative
+    // TODO (Tom Tang): use _PyLong_Copy to create a new object. Not thread-safe here
+    Py_SET_SIZE(pyObject, pyDigitCount);
+
+    // Set the sign bit
+    // https://hg.mozilla.org/releases/mozilla-esr102/file/tip/js/src/vm/BigIntType.cpp#l1801
+    /* flagsField */ ((uint32_t *)bigint)[0] |= SIGN_BIT_MASK;
+  }
+
+  return bigint;
 }
 
 void IntType::print(std::ostream &os) const {
-  os << this->getValue();
+  // Making sure the value does not overflow even if the int has millions of bits of precision
+  PyObject *str = PyObject_Str(pyObject);
+  os << PyUnicode_AsUTF8(str);
+  // https://pythonextensionpatterns.readthedocs.io/en/latest/refcount.html#new-references
+  Py_DECREF(str); // free
 }

src/jsTypeFactory.cc

@@ -16,6 +16,7 @@
 #include "include/FuncType.hh"
 #include "include/pyTypeFactory.hh"
 #include "include/StrType.hh"
+#include "include/IntType.hh"
 
 #include <jsapi.h>
 #include <jsfriendapi.h>
@@ -65,7 +66,15 @@ JS::Value jsTypeFactory(JSContext *cx, PyObject *object) {
     returnType.setBoolean(PyLong_AsLong(object));
   }
   else if (PyLong_Check(object)) {
-    returnType.setNumber(PyLong_AsLong(object));
+    if (PyObject_IsInstance(object, PythonMonkey_BigInt)) { // pm.bigint is a subclass of the builtin int type
+      JS::BigInt *bigint = IntType(object).toJsBigInt(cx);
+      returnType.setBigInt(bigint);
+    } else if (_PyLong_NumBits(object) <= 53) { // num <= JS Number.MAX_SAFE_INTEGER, the mantissa of a float64 is 53 bits (with 52 explicitly stored and the highest bit always being 1)
+      uint64_t num = PyLong_AsLongLong(object);
+      returnType.setNumber(num);
+    } else {
+      PyErr_SetString(PyExc_OverflowError, "Absolute value of the integer exceeds JS Number.MAX_SAFE_INTEGER. Use pythonmonkey.bigint instead.");
+    }
   }
   else if (PyFloat_Check(object)) {
     returnType.setNumber(PyFloat_AsDouble(object));
@@ -127,7 +136,7 @@ JS::Value jsTypeFactory(JSContext *cx, PyObject *object) {
     returnType.setNull();
   }
   else {
-    PyErr_SetString(PyExc_TypeError, "Python types other than bool, int, float, str, None, and our custom Null type are not supported by pythonmonkey yet.");
+    PyErr_SetString(PyExc_TypeError, "Python types other than bool, int, pythonmonkey.bigint, float, str, None, and our custom Null type are not supported by pythonmonkey yet.");
   }
   return returnType;