Implement FastList type for Struct list fields

Signed-off-by: Petr Ogarok <petr.ogarok@point72.com>
Point72 · May 24, 2024 · bdc5688 · bdc5688
1 parent 3c365c1
commit bdc5688
Show file tree

Hide file tree

Showing 18 changed files with 1,908 additions and 442 deletions.
diff --git a/cpp/csp/engine/CspType.cpp b/cpp/csp/engine/CspType.cpp
@@ -27,16 +27,19 @@ INIT_CSP_ENUM( CspType::Type,
            "DIALECT_GENERIC" 
     );
 
-CspTypePtr & CspArrayType::create( const CspTypePtr & elemType )
+CspTypePtr & CspArrayType::create( const CspTypePtr & elemType, bool isPyStructFastList )
 {
-    using Cache = std::unordered_map<const CspType*,CspTypePtr>;
+    using Cache = std::unordered_map<const CspType *, CspTypePtr>;
     static std::mutex s_mutex;
     static Cache      s_cache;
+    static Cache      s_pyStructFastListCache;
+
+    auto & cache = isPyStructFastList ? s_pyStructFastListCache : s_cache;
 
     std::lock_guard<std::mutex> guard( s_mutex );
-    auto rv = s_cache.emplace( elemType.get(), nullptr );
+    auto rv = cache.emplace( elemType.get(), nullptr );
     if( rv.second )
-        rv.first -> second = std::make_shared<CspArrayType>( elemType );
+        rv.first -> second = std::make_shared<CspArrayType>( elemType, isPyStructFastList );
     return rv.first -> second;
 }
 

diff --git a/cpp/csp/engine/CspType.h b/cpp/csp/engine/CspType.h
@@ -170,19 +170,21 @@ class CspStructType : public CspType
 class CspArrayType : public CspType
 {
 public:
-    CspArrayType( CspTypePtr elemType ) : CspType( CspType::Type::ARRAY ),
-                                          m_elemType( elemType )
+    CspArrayType( CspTypePtr elemType, bool isPyStructFastList = false ) :
+        CspType( CspType::Type::ARRAY ), m_elemType( elemType ), m_isPyStructFastList( isPyStructFastList )
     {}
     ~CspArrayType() {}
 
     const CspTypePtr & elemType() const { return m_elemType; }
+    bool isPyStructFastList() const     { return m_isPyStructFastList; }
 
     //Used by BURST mode to avoid creating more instances of CspArrayTypes than needed
     //returns CspArrayType with the given elemType
-    static CspTypePtr & create( const CspTypePtr & elemType );
+    static CspTypePtr & create( const CspTypePtr & elemType, bool isPyStructFastList = false );
 
 private:
     CspTypePtr m_elemType;
+    bool       m_isPyStructFastList;
 };
 
 template<> struct CspType::TypeTraits::fromCType<bool>                     { static constexpr CspType::TypeTraits::_enum type = CspType::TypeTraits::BOOL;            };

diff --git a/cpp/csp/python/CMakeLists.txt b/cpp/csp/python/CMakeLists.txt
@@ -5,15 +5,20 @@ set(CSPTYPESIMPL_PUBLIC_HEADERS
         PyCspType.h
         PyStruct.h
         PyStructList.h
-        PyStructList_impl.h)
+        PyStructList_impl.h
+        PyStructFastList.h
+        PyStructFastList_impl.h)
 
 add_library(csptypesimpl
             csptypesimpl.cpp
             CspTypeFactory.cpp
             PyCspEnum.cpp
             PyCspType.cpp
             PyStruct.cpp
-            PyStructToJson.cpp)
+            PyStructToJson.cpp
+            PyStructList_impl.h
+            PyStructFastList_impl.h
+            VectorWrapper.h)
 set_target_properties(csptypesimpl PROPERTIES PUBLIC_HEADER "${CSPTYPESIMPL_PUBLIC_HEADERS}")
 target_compile_definitions(csptypesimpl PUBLIC RAPIDJSON_HAS_STDSTRING=1)
 target_link_libraries(csptypesimpl csp_core csp_types)
@@ -43,9 +48,7 @@ set(CSPIMPL_PUBLIC_HEADERS
         PyOutputAdapterWrapper.h
         PyOutputProxy.h
         PyConstants.h
-        PyStructToJson.h
-        PyStructList.h
-        PyStructList_impl.h)
+        PyStructToJson.h)
 
 add_library(cspimpl SHARED
         cspimpl.cpp

diff --git a/cpp/csp/python/Conversions.h b/cpp/csp/python/Conversions.h
@@ -14,6 +14,7 @@
 #include <csp/python/PyCspType.h>
 #include <csp/python/PyObjectPtr.h>
 #include <csp/python/PyStruct.h>
+#include <csp/python/PyStructFastList.h>
 #include <csp/python/PyStructList.h>
 #include <Python.h>
 #include <string>
@@ -81,7 +82,7 @@ inline T fromPython( PyObject * o )
     return T{};
 }
 
-//work around for inabilty to partially spcialize fromPython on vector<T>
+//work around for inabilty to partially specialize fromPython on vector<T>
 template<typename T>
 struct FromPython
 {
@@ -768,11 +769,11 @@ inline PyObject * toPython( const DictionaryPtr & value)
 }
 
 template<typename StorageT>
-inline PyObject * toPython( const std::vector<StorageT> & v, const CspType & type )
+inline PyObject * toPython( const std::vector<StorageT> & v, const CspType & arrayType )
 {
-    assert( type.type() == CspType::Type::ARRAY );
+    assert( arrayType.type() == CspType::Type::ARRAY );
 
-    const CspType & elemType = *static_cast<const CspArrayType &>( type ).elemType();
+    const CspType & elemType = *static_cast<const CspArrayType &>( arrayType ).elemType();
     size_t size = v.size();
     PyObjectPtr list = PyObjectPtr::check( PyList_New( size ) );
 
@@ -785,18 +786,27 @@ inline PyObject * toPython( const std::vector<StorageT> & v, const CspType & typ
 }
 
 template<typename StorageT>
-inline PyObject * toPython( const std::vector<StorageT> & v, const CspType & type, const PyStruct * pystruct )
+inline PyObject * toPython( const std::vector<StorageT> & v, const CspType & arrayType, const PyStruct * pystruct )
 {
-    assert( type.type() == CspType::Type::ARRAY );
+    assert( arrayType.type() == CspType::Type::ARRAY );
 
-    const CspTypePtr elemType = static_cast<const CspArrayType &>( type ).elemType();
+    const CspArrayType & cspArrayType = static_cast<const CspArrayType &>( arrayType );
+    const CspTypePtr elemType = cspArrayType.elemType();
     using ElemT = typename CspType::Type::toCArrayElemType<StorageT>::type;
     size_t sz = v.size();
 
+    // Create PyStructFastList when requested
+    if( cspArrayType.isPyStructFastList() )
+    {
+        PyObject * fl = PyStructFastList<StorageT>::PyType.tp_alloc( &PyStructFastList<StorageT>::PyType, 0 );
+        new ( fl ) PyStructFastList<StorageT>( const_cast<PyStruct *>( pystruct ), const_cast<std::vector<StorageT> &>( v ), cspArrayType );
+        return fl;
+    }
+    // Create PyStructList otherwise
     // TODO: Implement more efficient list allocation by pre-allocating the space and filling it using PyList_SET_ITEM.
     // As of now, the problem is that Python is not allowing to resize the list via API, and it cannot allocate the list at the base of PyStructList, it can only allocate it somewhere in memory not under control.
     PyObject * psl = PyStructList<StorageT>::PyType.tp_alloc( &PyStructList<StorageT>::PyType, 0 );
-    new ( psl ) PyStructList<StorageT>( const_cast<PyStruct *>( pystruct ), const_cast<std::vector<StorageT> &>( v ), *elemType );
+    new ( psl ) PyStructList<StorageT>( const_cast<PyStruct *>( pystruct ), const_cast<std::vector<StorageT> &>( v ), cspArrayType );
 
     for( size_t index = 0; index < sz; ++index )
     {
@@ -810,15 +820,16 @@ inline PyObject * toPython( const std::vector<StorageT> & v, const CspType & typ
 template<typename StorageT>
 struct FromPython<std::vector<StorageT>>
 {
-    static std::vector<StorageT> impl( PyObject * o, const CspType & type )
+    static std::vector<StorageT> impl( PyObject * o, const CspType & arrayType )
     {
-        assert( type.type() == CspType::Type::ARRAY );
-        const CspType & elemType = *static_cast<const CspArrayType &>( type ).elemType();
+        assert( arrayType.type() == CspType::Type::ARRAY );
 
         using ElemT = typename CspType::Type::toCArrayElemType<StorageT>::type;
 
+        const CspType & elemType = *static_cast<const CspArrayType &>( arrayType ).elemType();
+
         std::vector<StorageT> out;
-        //fast path list and tuples since we can size up front
+        //fast path for list and tuple since we can size up front
         if( PyList_Check( o ) )
         {
             size_t size = PyList_GET_SIZE( o );

diff --git a/cpp/csp/python/CspTypeFactory.cpp b/cpp/csp/python/CspTypeFactory.cpp
@@ -20,14 +20,24 @@ CspTypePtr & CspTypeFactory::typeFromPyType( PyObject * pyTypeObj )
     // List objects shouldn't be cached since they are temporary objects
     if( PyList_Check( pyTypeObj ) )
     {
-        if( PyList_GET_SIZE( ( PyObject * ) pyTypeObj ) != 1 )
-            CSP_THROW( TypeError, "Expected list types to be single element of sub-type" );
+        if( PyList_GET_SIZE( ( PyObject * ) pyTypeObj ) != 1 && PyList_GET_SIZE( ( PyObject * ) pyTypeObj ) != 2 )
+            CSP_THROW( TypeError, "Expected list types post-normalization to be one or two elements: sub-type and optional FastList flag" );
 
         PyObject *pySubType = PyList_GET_ITEM( pyTypeObj, 0 );
         if( !PyType_Check( pySubType ) )
             CSP_THROW( TypeError, "nested typed lists are not supported" );
+
+        bool useFastList = false;
+        if( PyList_GET_SIZE( ( PyObject * ) pyTypeObj ) == 2 )
+        {
+            PyObject *pyUseFastList = PyList_GET_ITEM( pyTypeObj, 1 );
+            if( !PyBool_Check( pyUseFastList ) || pyUseFastList != Py_True )
+                CSP_THROW( TypeError, "expected bool True as second list type argument" );
+            useFastList = true;
+        }
+
         CspTypePtr elemType = typeFromPyType( pySubType );
-        return CspArrayType::create( elemType );
+        return CspArrayType::create( elemType, useFastList );
     }
 
     PyTypeObject *pyType = (PyTypeObject*) pyTypeObj;

diff --git a/cpp/csp/python/PyObjectPtr.h b/cpp/csp/python/PyObjectPtr.h
@@ -38,19 +38,11 @@ class PyPtr
         return *this;
     }
 
-    bool operator==( const PyPtr & rhs ) const
-    {
-        if( m_obj == rhs.m_obj )
-            return true;
+    bool operator==( const PyPtr & rhs ) const { return generic_compare( rhs, Py_EQ ); }
 
-        if( !m_obj || !rhs.m_obj )
-            return false;
+    bool operator<( const PyPtr & rhs ) const { return generic_compare( rhs, Py_LT ); }
 
-        int rv = PyObject_RichCompareBool( m_obj, rhs.m_obj, Py_EQ );
-        if( rv == -1 )
-            CSP_THROW( PythonPassthrough, "" );
-        return rv == 1;
-    }
+    bool operator>( const PyPtr & rhs ) const { return generic_compare( rhs, Py_GT ); }
 
     operator bool() const { return m_obj != nullptr; }
 
@@ -101,6 +93,17 @@ class PyPtr
     PyPtr( PyObjectOwn * o ) { m_obj = ( PYOBJECT_T * ) o; }
 
     PYOBJECT_T * m_obj;
+
+    bool generic_compare( const PyPtr & rhs, int opid ) const
+    {
+        if( !m_obj || !rhs.m_obj )
+            CSP_THROW( PythonPassthrough, "" );
+
+        int rv = PyObject_RichCompareBool( m_obj, rhs.m_obj, opid );
+        if( rv == -1 )
+            CSP_THROW( PythonPassthrough, "" );
+        return rv;
+    }
 };
 
 using PyObjectPtr     = PyPtr<PyObject>;

diff --git a/cpp/csp/python/PyStruct.cpp b/cpp/csp/python/PyStruct.cpp
@@ -3,6 +3,7 @@
 #include <csp/python/InitHelper.h>
 #include <csp/python/PyObjectPtr.h>
 #include <csp/python/PyStruct.h>
+#include <csp/python/PyStructFastList_impl.h>
 #include <csp/python/PyStructList_impl.h>
 #include <csp/python/PyStructToJson.h>
 #include <unordered_set>
@@ -477,12 +478,9 @@ void PyStruct::setattr( Struct * s, PyObject * attr, PyObject * value )
 
 // Struct printing code
 
-// forward declarations
+// forward declaration
 void repr_struct( const Struct * struct_, std::string & tl_repr, bool show_unset );
 
-template<typename ElemT>
-void repr_array( const std::vector<ElemT> & val, const CspArrayType & arrayType, std::string & tl_repr, bool show_unset );
-
 // helper functions for formatting to Python standard
 void format_bool( const bool val, std::string & tl_repr ) {  tl_repr += ( ( val ? "True" : "False" ) ); }
 void format_double( const double val, std::string & tl_repr )
@@ -555,14 +553,14 @@ void repr_field( const Struct * struct_, const StructFieldPtr & field, std::stri
             auto const * arrayType = static_cast<const CspArrayType*>( field -> type().get() );
             const CspType * elemType = arrayType -> elemType().get();
 
-            switchCspType( elemType, [ field, struct_, &arrayType, &tl_repr, show_unset ]( auto tag )
+            switchCspType( elemType, [ field, struct_, &elemType, &tl_repr, show_unset ]( auto tag )
             {
                 //workaround for MS compiler bug, separate into two using lines... :/
                 using TagType = decltype( tag );
                 using CElemType = typename TagType::type;
                 using ArrayType = typename CspType::Type::toCArrayType<CElemType>::type;
                 const ArrayType & val = field -> value<ArrayType>( struct_ );
-                repr_array( val, *arrayType, tl_repr, show_unset );
+                repr_array( val, *elemType, tl_repr, show_unset );
             } );
 
             break;
@@ -585,7 +583,7 @@ void repr_field( const Struct * struct_, const StructFieldPtr & field, std::stri
 }
 
 template<typename StorageT>
-void repr_array( const std::vector<StorageT> & val, const CspArrayType & arrayType, std::string & tl_repr, bool show_unset )
+void repr_array( const std::vector<StorageT> & val, const CspType & elemType, std::string & tl_repr, bool show_unset )
 {
     using ElemT = typename CspType::Type::toCArrayElemType<StorageT>::type;
     tl_repr += "[";
@@ -611,11 +609,11 @@ void repr_array( const std::vector<StorageT> & val, const CspArrayType & arrayTy
         else if constexpr( std::is_integral<ElemT>::value )
             tl_repr += std::to_string( *it );
         else if constexpr( is_vector<ElemT>::value )
-            repr_array( *it, static_cast<const CspArrayType&>( arrayType.elemType() ), tl_repr, show_unset ); // recursive, allows for nested arrays!
+            repr_array( *it, elemType, tl_repr, show_unset ); // recursive, allows for nested arrays!
         else
         {
             // if the element is an enum, generic or datetime type, convert to python
-            PyObjectPtr attr = PyObjectPtr::own( toPython( *it, *( arrayType.elemType().get() ) ) );
+            PyObjectPtr attr = PyObjectPtr::own( toPython( *it, elemType ) );
             format_pyobject( attr, tl_repr );
         }
     }
@@ -1022,7 +1020,6 @@ REGISTER_TYPE_INIT( &PyStructMeta::PyType, "PyStructMeta" )
 REGISTER_TYPE_INIT( &PyStruct::PyType,     "PyStruct" )
 
 // Instantiate all templates for PyStructList class
-template struct PyStructList<bool>;
 template struct PyStructList<int8_t>;
 template struct PyStructList<uint8_t>;
 template struct PyStructList<int16_t>;
@@ -1041,4 +1038,23 @@ template struct PyStructList<DialectGenericType>;
 template struct PyStructList<StructPtr>;
 template struct PyStructList<CspEnum>;
 
+// Instantiate all templates for PyStructFastList class
+template struct PyStructFastList<int8_t>;
+template struct PyStructFastList<uint8_t>;
+template struct PyStructFastList<int16_t>;
+template struct PyStructFastList<uint16_t>;
+template struct PyStructFastList<int32_t>;
+template struct PyStructFastList<uint32_t>;
+template struct PyStructFastList<int64_t>;
+template struct PyStructFastList<uint64_t>;
+template struct PyStructFastList<double>;
+template struct PyStructFastList<DateTime>;
+template struct PyStructFastList<TimeDelta>;
+template struct PyStructFastList<Date>;
+template struct PyStructFastList<Time>;
+template struct PyStructFastList<std::string>;
+template struct PyStructFastList<DialectGenericType>;
+template struct PyStructFastList<StructPtr>;
+template struct PyStructFastList<CspEnum>;
+
 }
diff --git a/cpp/csp/python/PyStruct.h b/cpp/csp/python/PyStruct.h
@@ -78,6 +78,10 @@ struct CSPTYPESIMPL_EXPORT PyStruct : public PyObject
     static PyTypeObject PyType;
 };
 
+// Array struct field printing function
+template<typename ElemT>
+void repr_array( const std::vector<ElemT> & val, const CspType & elemType, std::string & tl_repr, bool show_unset );
+
 }
 
 #endif
diff --git a/cpp/csp/python/PyStructFastList.h b/cpp/csp/python/PyStructFastList.h
@@ -0,0 +1,43 @@
+#ifndef _IN_CSP_PYTHON_PYSTRUCTFASTLIST_H
+#define _IN_CSP_PYTHON_PYSTRUCTFASTLIST_H
+
+#include <csp/python/Conversions.h>
+#include <csp/python/InitHelper.h>
+#include <csp/python/PyIterator.h>
+#include <csp/python/PyStruct.h>
+#include <csp/python/VectorWrapper.h>
+#include <Python.h>
+#include <vector>
+
+namespace csp::python
+{
+
+template<typename StorageT>
+struct PyStructFastList : public PyObject
+{
+    using ElemT = typename CspType::Type::toCArrayElemType<StorageT>::type;
+
+    PyStructFastList( PyStruct * p, std::vector<StorageT> & v, const CspType & type ) : pystruct( p ), vector( VectorWrapper<StorageT>( v ) ), arrayType( type )
+    {
+        Py_INCREF( pystruct );
+    }
+
+    PyStruct * pystruct;              // Pointer to PyStruct for proper reference counting
+    VectorWrapper<StorageT> vector;   // Field value for modifying
+
+    const CspType & arrayType;        // We require the type information of any non-primitive type, i.e. Struct or Enum, since they contain a meta
+    static PyTypeObject PyType;
+    static bool s_typeRegister;
+
+    inline CspTypePtr elemType() const { return static_cast<const CspArrayType &>( arrayType ).elemType(); }
+
+    inline PyObject * toPythonValue( const StorageT & value ) const;
+    inline StorageT fromPythonValue( PyObject * value ) const;
+};
+
+template<typename StorageT> bool PyStructFastList<StorageT>::s_typeRegister = InitHelper::instance().registerCallback( 
+    InitHelper::typeInitCallback( &PyStructFastList<StorageT>::PyType, "" ) );
+
+}
+
+#endif