Python 3.12 build support (Point72#221)

* resolve Point72#13 - Python 3.12 build support. 
   * python 3.12 - fix unbound local issue - changed how ts inputs to PyNode are reset to null due to new LOAD_FAST vs LOAD_FAST_CHECK opcodes in Python 3.12.  Inject DELETE opcodes into bytecode rather than setting directly to null in c++
   * cibuildwheel 2.11.2 -> 2.16.5

Signed-off-by: Rob Ambalu <robert.ambalu@point72.com>

.github/actions/setup-caches/action.yml

@@ -10,6 +10,7 @@ inputs:
       - 'cp39'
       - 'cp310'
       - 'cp311'
+      - 'cp312'
     default: 'cp39'
 
 runs:

.github/actions/setup-dependencies/action.yml

@@ -10,6 +10,7 @@ inputs:
       - 'cp39'
       - 'cp310'
       - 'cp311'
+      - 'cp312'
     default: 'cp39'
 
 runs:

.github/actions/setup-python/action.yml

@@ -10,6 +10,7 @@ inputs:
       - '3.9'
       - '3.10'
       - '3.11'
+      - '3.12'
     default: '3.9'
 
 runs:
@@ -33,4 +34,4 @@ runs:
 
     - name: Install cibuildwheel and twine
       shell: bash
-      run: pip install cibuildwheel==2.11.2 twine
+      run: pip install cibuildwheel==2.16.5 twine

.github/workflows/build.yml

@@ -179,11 +179,13 @@ jobs:
             - "3.9"
             - "3.10"
             - "3.11"
+            - "3.12"
           cibuildwheel:
             - "cp38"
             - "cp39"
             - "cp310"
             - "cp311"
+            - "cp312"
           is-full-run:
             - ${{ needs.initialize.outputs.FULL_RUN == 'true' }}
           exclude:
@@ -198,24 +200,41 @@ jobs:
               cibuildwheel: "cp310"
             - python-version: "3.8"
               cibuildwheel: "cp311"
+            - python-version: "3.8"
+              cibuildwheel: "cp312"
             - python-version: "3.9"
               cibuildwheel: "cp38"
             - python-version: "3.9"
               cibuildwheel: "cp310"
             - python-version: "3.9"
               cibuildwheel: "cp311"
+            - python-version: "3.9"
+              cibuildwheel: "cp312"
             - python-version: "3.10"
               cibuildwheel: "cp38"
             - python-version: "3.10"
               cibuildwheel: "cp39"
             - python-version: "3.10"
               cibuildwheel: "cp311"
+            - python-version: "3.10"
+              cibuildwheel: "cp312"
             - python-version: "3.11"
               cibuildwheel: "cp38"
             - python-version: "3.11"
               cibuildwheel: "cp39"
             - python-version: "3.11"
               cibuildwheel: "cp310"
+            - python-version: "3.11"
+              cibuildwheel: "cp312"
+            - python-version: "3.12"
+              cibuildwheel: "cp38"
+            - python-version: "3.12"
+              cibuildwheel: "cp39"
+            - python-version: "3.12"
+              cibuildwheel: "cp310"
+            - python-version: "3.12"
+              cibuildwheel: "cp311"
+
 
             ##############################################
             # Things to exclude if not a full matrix run #
@@ -402,6 +421,7 @@ jobs:
             - 3.9
             - "3.10"
             - 3.11
+            - 3.12
           is-full-run:
             - ${{ needs.initialize.outputs.FULL_RUN == 'true' }}
           exclude:

.github/workflows/conda.yml

@@ -52,7 +52,7 @@ jobs:
         - name: Set up Caches
           uses: ./.github/actions/setup-caches
           with:
-            cibuildwheel: 'cp311'
+            cibuildwheel: 'cp312'
 
         - name: Python Lint Steps
           run: make lint

CMakeLists.txt

@@ -98,7 +98,7 @@ if(CSP_USE_CCACHE)
 endif()
 
 if(NOT DEFINED CSP_PYTHON_VERSION)
-    set(CSP_PYTHON_VERSION 3.11)
+    set(CSP_PYTHON_VERSION 3.12)
 endif()
 
 # Path to python folder for autogen

conda/dev-environment-unix.yml

@@ -38,7 +38,7 @@ dependencies:
  - pytest-asyncio
  - pytest-cov
  - pytest-sugar
- - python<3.12
+ - python<3.13
  - python-rapidjson
  - rapidjson
  - requests

cpp/csp/python/PyNode.cpp

@@ -61,8 +61,8 @@ void PyNode::init( PyObjectPtr inputs, PyObjectPtr outputs )
     m_localVars = ( PyObject *** ) calloc( numInputs(), sizeof( PyObject ** ) );
 
     //printf( "Starting %s slots: %ld rank: %d\n", name(), slots, rank() );
-    PyCodeObject * code = ( PyCodeObject * ) pygen -> gi_code;
 #if IS_PRE_PYTHON_3_11
+    PyCodeObject * code = ( PyCodeObject * ) pygen -> gi_code;
     Py_ssize_t numCells = PyTuple_GET_SIZE( code -> co_cellvars );
     size_t cell2argIdx = 0;
     for( int stackloc = code -> co_argcount; stackloc < code -> co_nlocals + numCells; ++stackloc )
@@ -82,12 +82,13 @@ void PyNode::init( PyObjectPtr inputs, PyObjectPtr outputs )
                 continue;
             var = &( ( ( PyCellObject * ) *var ) -> ob_ref );
         }
-//PY311 changes
+//PY311+ changes
 #else
+    _PyInterpreterFrame * frame = ( _PyInterpreterFrame * ) pygen -> gi_iframe;
+    PyCodeObject * code = frame -> f_code;
     int localPlusIndex = 0;
     for( int stackloc = code -> co_argcount; stackloc < code -> co_nlocalsplus; ++stackloc, ++localPlusIndex )
     {
-        _PyInterpreterFrame * frame = ( _PyInterpreterFrame * ) pygen -> gi_iframe;
         PyObject **var = &frame -> localsplus[stackloc];
 
         auto kind = _PyLocals_GetKind(code -> co_localspluskinds, localPlusIndex );
@@ -113,19 +114,18 @@ void PyNode::init( PyObjectPtr inputs, PyObjectPtr outputs )
         std::string vartype = PyUnicode_AsUTF8( PyTuple_GET_ITEM( *var, 0 ) );
         int index           = fromPython<int64_t>( PyTuple_GET_ITEM( *var, 1 ) );
 
-        //decref tuple at this point its no longer needed and will be replaced
-        Py_DECREF( *var );
-
         if( vartype == INPUT_VAR_VAR )
         {
             CSP_ASSERT( !isInputBasket( index ) );
 
             m_localVars[ index ] = var;
-            //assign null to location so users get reference before assignment errors
-            *var = nullptr;
+            //These vars will be "deleted" from the python stack after start 
             continue;
         }
 
+        //decref tuple at this point its no longer needed and will be replaced
+        Py_DECREF( *var );
+
         PyObject * newvalue = nullptr;
         if( vartype == NODEREF_VAR )
             newvalue = toPython( reinterpret_cast<uint64_t>( static_cast<csp::Node*>(this) ) );

csp/impl/wiring/node_parser.py

@@ -843,10 +843,14 @@ def _parse_impl(self):
         else:
             start_and_body = startblock + body
 
+        # delete ts_var variables *after* start so that they raise Unbound local exceptions if they get accessed before first tick
+        del_vars = []
+        for v in ts_vars:
+            del_vars.append(ast.Delete(targets=[ast.Name(id=v.targets[0].id, ctx=ast.Del())]))
         # Yield before start block so we can setup stack frame before executing
         # However, this initial yield shouldn't be within the try-finally block, since if a node does not start, it's stop() logic should not be invoked
         # This avoids an issue where one node raises an exception upon start(), and then other nodes execute their stop() without having ever started
-        start_and_body = [ast.Expr(value=ast.Yield(value=None))] + start_and_body
+        start_and_body = [ast.Expr(value=ast.Yield(value=None))] + del_vars + start_and_body
         newbody = init_block + start_and_body
 
         newfuncdef = ast.FunctionDef(name=self._name, body=newbody, returns=None)

csp/impl/wiring/runtime.py

@@ -18,7 +18,7 @@
 def _normalize_run_times(starttime, endtime, realtime):
     if starttime is None:
         if realtime:
-            starttime = datetime.utcnow()
+            starttime = datetime.now(pytz.UTC).replace(tzinfo=None)
         else:
             raise RuntimeError("starttime argument is required")
     if endtime is None:
@@ -199,8 +199,8 @@ def run(
             mem_cache.clear(clear_user_objects=False)
 
             # Ensure we dont start running realtime engines before starttime if its in the future
-            if starttime > datetime.utcnow() and realtime:
-                time.sleep((starttime - datetime.utcnow()).total_seconds())
+            if starttime > datetime.now(pytz.UTC).replace(tzinfo=None) and realtime:
+                time.sleep((starttime - datetime.now(pytz.UTC)).total_seconds())
 
             with mem_cache:
                 return engine.run(starttime, endtime)

csp/tests/test_random.py

@@ -116,7 +116,7 @@ def test_brownian_motion(self):
             endtime=timedelta(seconds=100),
         )[0]
         err = bm_out[-1][1] - data.sum(axis=0)
-        self.assertAlmostEquals(np.abs(err).max(), 0.0)
+        self.assertAlmostEqual(np.abs(err).max(), 0.0)
 
     def test_brownian_motion_1d(self):
         mean = 10.0

pyproject.toml

@@ -38,6 +38,7 @@ classifiers = [
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
 ]
 
 
@@ -113,7 +114,7 @@ slack = [
 ignore = []
 
 [tool.cibuildwheel]
-build = "cp38-* cp39-* cp310-* cp311-*"
+build = "cp38-* cp39-* cp310-* cp311-* cp312-*"
 test-command = "echo 'TODO'"
 test-requires = [
     "pytest",