diff --git a/cpp/csp/engine/CspType.cpp b/cpp/csp/engine/CspType.cpp index c5d2599e..884fc17c 100644 --- a/cpp/csp/engine/CspType.cpp +++ b/cpp/csp/engine/CspType.cpp @@ -27,16 +27,19 @@ INIT_CSP_ENUM( CspType::Type, "DIALECT_GENERIC" ); -CspTypePtr & CspArrayType::create( const CspTypePtr & elemType ) +CspTypePtr & CspArrayType::create( const CspTypePtr & elemType, bool isPyStructFastList ) { - using Cache = std::unordered_map; + using Cache = std::unordered_map; static std::mutex s_mutex; static Cache s_cache; + static Cache s_pyStructFastListCache; + + auto & cache = isPyStructFastList ? s_pyStructFastListCache : s_cache; std::lock_guard guard( s_mutex ); - auto rv = s_cache.emplace( elemType.get(), nullptr ); + auto rv = cache.emplace( elemType.get(), nullptr ); if( rv.second ) - rv.first -> second = std::make_shared( elemType ); + rv.first -> second = std::make_shared( elemType, isPyStructFastList ); return rv.first -> second; } diff --git a/cpp/csp/engine/CspType.h b/cpp/csp/engine/CspType.h index 2ad7c0b9..b85ab414 100644 --- a/cpp/csp/engine/CspType.h +++ b/cpp/csp/engine/CspType.h @@ -170,19 +170,21 @@ class CspStructType : public CspType class CspArrayType : public CspType { public: - CspArrayType( CspTypePtr elemType ) : CspType( CspType::Type::ARRAY ), - m_elemType( elemType ) + CspArrayType( CspTypePtr elemType, bool isPyStructFastList = false ) : + CspType( CspType::Type::ARRAY ), m_elemType( elemType ), m_isPyStructFastList( isPyStructFastList ) {} ~CspArrayType() {} const CspTypePtr & elemType() const { return m_elemType; } + bool isPyStructFastList() const { return m_isPyStructFastList; } //Used by BURST mode to avoid creating more instances of CspArrayTypes than needed //returns CspArrayType with the given elemType - static CspTypePtr & create( const CspTypePtr & elemType ); + static CspTypePtr & create( const CspTypePtr & elemType, bool isPyStructFastList = false ); private: CspTypePtr m_elemType; + bool m_isPyStructFastList; }; template<> struct CspType::TypeTraits::fromCType { static constexpr CspType::TypeTraits::_enum type = CspType::TypeTraits::BOOL; }; diff --git a/cpp/csp/python/CMakeLists.txt b/cpp/csp/python/CMakeLists.txt index c4879410..5242a63f 100644 --- a/cpp/csp/python/CMakeLists.txt +++ b/cpp/csp/python/CMakeLists.txt @@ -5,7 +5,9 @@ set(CSPTYPESIMPL_PUBLIC_HEADERS PyCspType.h PyStruct.h PyStructList.h - PyStructList_impl.h) + PyStructList_impl.h + PyStructFastList.h + PyStructFastList_impl.h) add_library(csptypesimpl csptypesimpl.cpp @@ -13,7 +15,10 @@ add_library(csptypesimpl PyCspEnum.cpp PyCspType.cpp PyStruct.cpp - PyStructToJson.cpp) + PyStructToJson.cpp + PyStructList_impl.h + PyStructFastList_impl.h + VectorWrapper.h) set_target_properties(csptypesimpl PROPERTIES PUBLIC_HEADER "${CSPTYPESIMPL_PUBLIC_HEADERS}") target_compile_definitions(csptypesimpl PUBLIC RAPIDJSON_HAS_STDSTRING=1) target_link_libraries(csptypesimpl csp_core csp_types) @@ -43,9 +48,7 @@ set(CSPIMPL_PUBLIC_HEADERS PyOutputAdapterWrapper.h PyOutputProxy.h PyConstants.h - PyStructToJson.h - PyStructList.h - PyStructList_impl.h) + PyStructToJson.h) add_library(cspimpl SHARED cspimpl.cpp diff --git a/cpp/csp/python/Conversions.h b/cpp/csp/python/Conversions.h index 85f1df66..1fe10f56 100644 --- a/cpp/csp/python/Conversions.h +++ b/cpp/csp/python/Conversions.h @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -81,7 +82,7 @@ inline T fromPython( PyObject * o ) return T{}; } -//work around for inabilty to partially spcialize fromPython on vector +//work around for inabilty to partially specialize fromPython on vector template struct FromPython { @@ -768,11 +769,11 @@ inline PyObject * toPython( const DictionaryPtr & value) } template -inline PyObject * toPython( const std::vector & v, const CspType & type ) +inline PyObject * toPython( const std::vector & v, const CspType & arrayType ) { - assert( type.type() == CspType::Type::ARRAY ); + assert( arrayType.type() == CspType::Type::ARRAY ); - const CspType & elemType = *static_cast( type ).elemType(); + const CspType & elemType = *static_cast( arrayType ).elemType(); size_t size = v.size(); PyObjectPtr list = PyObjectPtr::check( PyList_New( size ) ); @@ -785,18 +786,27 @@ inline PyObject * toPython( const std::vector & v, const CspType & typ } template -inline PyObject * toPython( const std::vector & v, const CspType & type, const PyStruct * pystruct ) +inline PyObject * toPython( const std::vector & v, const CspType & arrayType, const PyStruct * pystruct ) { - assert( type.type() == CspType::Type::ARRAY ); + assert( arrayType.type() == CspType::Type::ARRAY ); - const CspTypePtr elemType = static_cast( type ).elemType(); + const CspArrayType & cspArrayType = static_cast( arrayType ); + const CspTypePtr elemType = cspArrayType.elemType(); using ElemT = typename CspType::Type::toCArrayElemType::type; size_t sz = v.size(); + // Create PyStructFastList when requested + if( cspArrayType.isPyStructFastList() ) + { + PyObject * fl = PyStructFastList::PyType.tp_alloc( &PyStructFastList::PyType, 0 ); + new ( fl ) PyStructFastList( const_cast( pystruct ), const_cast &>( v ), cspArrayType ); + return fl; + } + // Create PyStructList otherwise // TODO: Implement more efficient list allocation by pre-allocating the space and filling it using PyList_SET_ITEM. // As of now, the problem is that Python is not allowing to resize the list via API, and it cannot allocate the list at the base of PyStructList, it can only allocate it somewhere in memory not under control. PyObject * psl = PyStructList::PyType.tp_alloc( &PyStructList::PyType, 0 ); - new ( psl ) PyStructList( const_cast( pystruct ), const_cast &>( v ), *elemType ); + new ( psl ) PyStructList( const_cast( pystruct ), const_cast &>( v ), cspArrayType ); for( size_t index = 0; index < sz; ++index ) { @@ -810,15 +820,16 @@ inline PyObject * toPython( const std::vector & v, const CspType & typ template struct FromPython> { - static std::vector impl( PyObject * o, const CspType & type ) + static std::vector impl( PyObject * o, const CspType & arrayType ) { - assert( type.type() == CspType::Type::ARRAY ); - const CspType & elemType = *static_cast( type ).elemType(); + assert( arrayType.type() == CspType::Type::ARRAY ); using ElemT = typename CspType::Type::toCArrayElemType::type; + const CspType & elemType = *static_cast( arrayType ).elemType(); + std::vector out; - //fast path list and tuples since we can size up front + //fast path for list and tuple since we can size up front if( PyList_Check( o ) ) { size_t size = PyList_GET_SIZE( o ); diff --git a/cpp/csp/python/CspTypeFactory.cpp b/cpp/csp/python/CspTypeFactory.cpp index 8e150991..a1bfd027 100644 --- a/cpp/csp/python/CspTypeFactory.cpp +++ b/cpp/csp/python/CspTypeFactory.cpp @@ -20,14 +20,24 @@ CspTypePtr & CspTypeFactory::typeFromPyType( PyObject * pyTypeObj ) // List objects shouldn't be cached since they are temporary objects if( PyList_Check( pyTypeObj ) ) { - if( PyList_GET_SIZE( ( PyObject * ) pyTypeObj ) != 1 ) - CSP_THROW( TypeError, "Expected list types to be single element of sub-type" ); + if( PyList_GET_SIZE( ( PyObject * ) pyTypeObj ) != 1 && PyList_GET_SIZE( ( PyObject * ) pyTypeObj ) != 2 ) + CSP_THROW( TypeError, "Expected list types post-normalization to be one or two elements: sub-type and optional FastList flag" ); PyObject *pySubType = PyList_GET_ITEM( pyTypeObj, 0 ); if( !PyType_Check( pySubType ) ) CSP_THROW( TypeError, "nested typed lists are not supported" ); + + bool useFastList = false; + if( PyList_GET_SIZE( ( PyObject * ) pyTypeObj ) == 2 ) + { + PyObject *pyUseFastList = PyList_GET_ITEM( pyTypeObj, 1 ); + if( !PyBool_Check( pyUseFastList ) || pyUseFastList != Py_True ) + CSP_THROW( TypeError, "expected bool True as second list type argument" ); + useFastList = true; + } + CspTypePtr elemType = typeFromPyType( pySubType ); - return CspArrayType::create( elemType ); + return CspArrayType::create( elemType, useFastList ); } PyTypeObject *pyType = (PyTypeObject*) pyTypeObj; diff --git a/cpp/csp/python/PyObjectPtr.h b/cpp/csp/python/PyObjectPtr.h index 0fa66bff..448b10f3 100644 --- a/cpp/csp/python/PyObjectPtr.h +++ b/cpp/csp/python/PyObjectPtr.h @@ -38,19 +38,11 @@ class PyPtr return *this; } - bool operator==( const PyPtr & rhs ) const - { - if( m_obj == rhs.m_obj ) - return true; + bool operator==( const PyPtr & rhs ) const { return generic_compare( rhs, Py_EQ ); } - if( !m_obj || !rhs.m_obj ) - return false; + bool operator<( const PyPtr & rhs ) const { return generic_compare( rhs, Py_LT ); } - int rv = PyObject_RichCompareBool( m_obj, rhs.m_obj, Py_EQ ); - if( rv == -1 ) - CSP_THROW( PythonPassthrough, "" ); - return rv == 1; - } + bool operator>( const PyPtr & rhs ) const { return generic_compare( rhs, Py_GT ); } operator bool() const { return m_obj != nullptr; } @@ -101,6 +93,17 @@ class PyPtr PyPtr( PyObjectOwn * o ) { m_obj = ( PYOBJECT_T * ) o; } PYOBJECT_T * m_obj; + + bool generic_compare( const PyPtr & rhs, int opid ) const + { + if( !m_obj || !rhs.m_obj ) + CSP_THROW( PythonPassthrough, "" ); + + int rv = PyObject_RichCompareBool( m_obj, rhs.m_obj, opid ); + if( rv == -1 ) + CSP_THROW( PythonPassthrough, "" ); + return rv; + } }; using PyObjectPtr = PyPtr; diff --git a/cpp/csp/python/PyStruct.cpp b/cpp/csp/python/PyStruct.cpp index 1a9c9a15..fe30fedf 100644 --- a/cpp/csp/python/PyStruct.cpp +++ b/cpp/csp/python/PyStruct.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -477,12 +478,9 @@ void PyStruct::setattr( Struct * s, PyObject * attr, PyObject * value ) // Struct printing code -// forward declarations +// forward declaration void repr_struct( const Struct * struct_, std::string & tl_repr, bool show_unset ); -template -void repr_array( const std::vector & val, const CspArrayType & arrayType, std::string & tl_repr, bool show_unset ); - // helper functions for formatting to Python standard void format_bool( const bool val, std::string & tl_repr ) { tl_repr += ( ( val ? "True" : "False" ) ); } void format_double( const double val, std::string & tl_repr ) @@ -555,14 +553,14 @@ void repr_field( const Struct * struct_, const StructFieldPtr & field, std::stri auto const * arrayType = static_cast( field -> type().get() ); const CspType * elemType = arrayType -> elemType().get(); - switchCspType( elemType, [ field, struct_, &arrayType, &tl_repr, show_unset ]( auto tag ) + switchCspType( elemType, [ field, struct_, &elemType, &tl_repr, show_unset ]( auto tag ) { //workaround for MS compiler bug, separate into two using lines... :/ using TagType = decltype( tag ); using CElemType = typename TagType::type; using ArrayType = typename CspType::Type::toCArrayType::type; const ArrayType & val = field -> value( struct_ ); - repr_array( val, *arrayType, tl_repr, show_unset ); + repr_array( val, *elemType, tl_repr, show_unset ); } ); break; @@ -585,7 +583,7 @@ void repr_field( const Struct * struct_, const StructFieldPtr & field, std::stri } template -void repr_array( const std::vector & val, const CspArrayType & arrayType, std::string & tl_repr, bool show_unset ) +void repr_array( const std::vector & val, const CspType & elemType, std::string & tl_repr, bool show_unset ) { using ElemT = typename CspType::Type::toCArrayElemType::type; tl_repr += "["; @@ -611,11 +609,11 @@ void repr_array( const std::vector & val, const CspArrayType & arrayTy else if constexpr( std::is_integral::value ) tl_repr += std::to_string( *it ); else if constexpr( is_vector::value ) - repr_array( *it, static_cast( arrayType.elemType() ), tl_repr, show_unset ); // recursive, allows for nested arrays! + repr_array( *it, elemType, tl_repr, show_unset ); // recursive, allows for nested arrays! else { // if the element is an enum, generic or datetime type, convert to python - PyObjectPtr attr = PyObjectPtr::own( toPython( *it, *( arrayType.elemType().get() ) ) ); + PyObjectPtr attr = PyObjectPtr::own( toPython( *it, elemType ) ); format_pyobject( attr, tl_repr ); } } @@ -1022,7 +1020,6 @@ REGISTER_TYPE_INIT( &PyStructMeta::PyType, "PyStructMeta" ) REGISTER_TYPE_INIT( &PyStruct::PyType, "PyStruct" ) // Instantiate all templates for PyStructList class -template struct PyStructList; template struct PyStructList; template struct PyStructList; template struct PyStructList; @@ -1041,4 +1038,23 @@ template struct PyStructList; template struct PyStructList; template struct PyStructList; +// Instantiate all templates for PyStructFastList class +template struct PyStructFastList; +template struct PyStructFastList; +template struct PyStructFastList; +template struct PyStructFastList; +template struct PyStructFastList; +template struct PyStructFastList; +template struct PyStructFastList; +template struct PyStructFastList; +template struct PyStructFastList; +template struct PyStructFastList; +template struct PyStructFastList; +template struct PyStructFastList; +template struct PyStructFastList