Merge pull request Point72#174 from Point72/tkp/slacktut

Start organizing examples, increase test coverage of examples

Makefile

@@ -44,8 +44,8 @@ lint-cpp:
 	echo "C++ linting disabled for now"
 
 lint-docs:
-	python -m mdformat --check docs/wiki/ README.md examples/README.md
-	python -m codespell_lib docs/wiki/ README.md examples/README.md
+	python -m mdformat --check docs/wiki/ README.md examples/
+	python -m codespell_lib docs/wiki/ README.md examples/ --skip "*.cpp,*.h"
 
 # lint: lint-py lint-cpp  ## run lints
 lint: lint-py lint-docs ## run lints
@@ -62,8 +62,8 @@ fix-cpp:
 	echo "C++ autoformatting disabled for now"
 
 fix-docs:
-	python -m mdformat docs/wiki/ README.md examples/README.md
-	python -m codespell_lib --write docs/wiki/ README.md examples/README.md
+	python -m mdformat docs/wiki/ README.md examples/
+	python -m codespell_lib --write docs/wiki/ README.md examples/ --skip "*.cpp,*.h"
 
 fix: fix-py fix-cpp fix-docs ## run autofixers
 

conda/dev-environment-unix.yml

@@ -5,17 +5,19 @@ channels:
 dependencies:
  - bison
  - brotli
+ - build
  - bump2version>=1
  - cmake
  - codespell
  - compilers
  - cyrus-sasl
  - exprtk
  - flex
+ - graphviz
  - python-graphviz
  - gtest
- - httpx
- - isort
+ - httpx>=0.20,<1
+ - isort>=5,<6
  - libarrow=15
  - librdkafka
  - libboost-headers
@@ -24,16 +26,18 @@ dependencies:
  - mdformat
  - ninja
  - numpy
+ - pillow
  - psutil
  - pyarrow=15
  - pandas
  - pillow
  - polars
+ - psutil
  - pytz
  - pytest
+ - pytest-asyncio
  - pytest-cov
  - pytest-sugar
- - pytest-asyncio
  - python<3.12
  - python-rapidjson
  - rapidjson
@@ -44,6 +48,7 @@ dependencies:
  - slack-sdk
  - sqlalchemy
  - tar
+ - threadpoolctl
  - tornado
  - twine
  - unzip

csp/tests/test_examples.py

@@ -17,10 +17,11 @@ def _get_module(folder, filename):
         return None
 
 
-def _get_modules_to_test(folder):
+def _get_modules_to_test(*folders):
+    folder = ".".join(folders) if len(folders) > 0 else folders[0]
     return [
         (file, _get_module(folder, file))
-        for file in os.listdir(os.path.join(EXAMPLES_ROOT, folder))
+        for file in os.listdir(os.path.join(EXAMPLES_ROOT, *folders))
         if file.endswith(".py")
     ]
 
@@ -31,12 +32,37 @@ def _no_examples_folder_or_running_sdist_tests():
 
 @pytest.mark.skipif(_no_examples_folder_or_running_sdist_tests(), reason="no examples present or manually skipping")
 class TestExamples:
-    @pytest.mark.parametrize("filename,module", _get_modules_to_test("1_basics"))
+    @pytest.mark.parametrize("filename,module", _get_modules_to_test("01_basics"))
     def test_1_basics(self, filename, module):
         assert module.main
         module.main()
 
-    @pytest.mark.parametrize("filename,module", _get_modules_to_test("2_intermediate"))
+    @pytest.mark.parametrize("filename,module", _get_modules_to_test("02_intermediate"))
     def test_2_intermediate(self, filename, module):
         assert module.main
         module.main()
+
+    @pytest.mark.parametrize("filename,module", _get_modules_to_test("03_using_adapters", "parquet"))
+    def test_3_adapters_parquet(self, filename, module):
+        assert module.main
+        module.main()
+
+    @pytest.mark.parametrize("filename,module", _get_modules_to_test("04_writing_adapters"))
+    def test_4_writing_adapters(self, filename, module):
+        assert module.main
+        module.main()
+
+    @pytest.mark.parametrize("filename,module", _get_modules_to_test("06_advanced"))
+    def test_6_advanced(self, filename, module):
+        assert module.main
+        module.main()
+
+    @pytest.mark.parametrize("filename,module", _get_modules_to_test("98_just_for_fun"))
+    def test_98_just_for_fun(self, filename, module):
+        assert module.main
+        module.main()
+
+    @pytest.mark.parametrize("filename,module", _get_modules_to_test("99_developer_tools"))
+    def test_99_developer_tools(self, filename, module):
+        assert module.main
+        module.main()

examples/01_basics/README.md

@@ -0,0 +1,6 @@
+# Basics
+
+- [Simplest Possible Graph](./e1_basic.py)
+- [Ticking Graphs](./e2_ticking.py)
+- [Complete Example (Trading)](./e3_trade_pnl.py)
+- [Visualizing a Graph](./e4_show_graph.py)

examples/01_basics/e1_basic.py

@@ -0,0 +1,29 @@
+from datetime import datetime
+
+import csp
+from csp import ts
+
+
+@csp.node
+def add(x: ts[int], y: ts[int]) -> ts[int]:
+    return x + y
+
+
+@csp.graph
+def my_graph():
+    x = csp.const(1)
+    y = csp.const(2)
+
+    sum = add(x, y)
+
+    csp.print("x", x)
+    csp.print("y", y)
+    csp.print("sum", sum)
+
+
+def main():
+    csp.run(my_graph, starttime=datetime.now())
+
+
+if __name__ == "__main__":
+    main()

examples/01_basics/e2_ticking.py

@@ -0,0 +1,49 @@
+from datetime import datetime, timedelta
+
+import csp
+from csp import ts
+
+
+@csp.node
+def add(x: ts[int], y: ts[int]) -> ts[int]:
+    return x + y
+
+
+@csp.node
+def accum(val: ts[int]) -> ts[int]:
+    with csp.state():
+        s_sum = 0
+    if csp.ticked(val):
+        s_sum += val
+        return val
+
+
+@csp.graph
+def my_graph():
+    st = datetime(2020, 1, 1)
+
+    # Dummy x values
+    x = csp.curve(int, [(st + timedelta(1), 1), (st + timedelta(2), 2), (st + timedelta(3), 3)])
+
+    # Dummy y values
+    y = csp.curve(int, [(st + timedelta(1), -1), (st + timedelta(3), -1), (st + timedelta(4), -1)])
+
+    # Add the time series
+    sum = add(x, y)
+
+    # Accumulate the result
+    acc = accum(sum)
+
+    csp.print("x", x)
+    csp.print("y", y)
+    csp.print("sum", sum)
+    csp.print("accum", acc)
+
+
+def main():
+    start = datetime(2020, 1, 1)
+    csp.run(my_graph, starttime=start)
+
+
+if __name__ == "__main__":
+    main()

examples/02_intermediate/README.md

@@ -0,0 +1,6 @@
+# Intermediate
+
+- [Graph Loops (`csp.feedback`)](./e1_feedback.py)
+- [Statistics Nodes](./e2_stats.py)
+- [Statistics Nodes with Numpy](./e3_numpy_stats.py)
+- [Expression Nodes with `exprtk`](./e4_exprtk.py)

examples/2_intermediate/e_13_feedback.py → examples/02_intermediate/e1_feedback.py

@@ -80,7 +80,7 @@ def main():
     if show_graph:
         csp.showgraph.show_graph(my_graph)
     else:
-        csp.run(my_graph, starttime=datetime.utcnow(), endtime=timedelta(seconds=60), realtime=True)
+        csp.run(my_graph, starttime=datetime.utcnow(), endtime=timedelta(seconds=5), realtime=False)
 
 
 if __name__ == "__main__":

examples/2_intermediate/e_21_numpy_stats.py → examples/02_intermediate/e3_numpy_stats.py

@@ -56,7 +56,7 @@ def numpy_stats_graph():
 
 
 def main():
-    results = csp.run(numpy_stats_graph, starttime=st, endtime=st + timedelta(minutes=10))
+    results = csp.run(numpy_stats_graph, starttime=st, endtime=st + timedelta(minutes=10), realtime=False)
 
     print("Price Averages\n")
     for i in range(10):

examples/03_using_adapters/kafka/README.md

@@ -0,0 +1,3 @@
+# Kafka Adapter
+
+- [Kafka Example](./e1_kafka.py)

examples/03_using_adapters/parquet/README.md

@@ -0,0 +1,3 @@
+# Parquet Adapter
+
+- [Parquet Writer](./e1_parquet_writer.py)

examples/3_using_adapters/e_06_parquet_writer_example.py → examples/03_using_adapters/parquet/e1_parquet_writer.py

@@ -48,12 +48,12 @@ def my_graph(struct_file_name: str, series_file_name: str):
     write_series(series_file_name)
 
 
-if __name__ == "__main__":
+def main():
     with tempfile.NamedTemporaryFile(suffix=".parquet") as struct_file:
         struct_file.file.close()
         with tempfile.NamedTemporaryFile(suffix=".parquet") as series_file:
             series_file.file.close()
-            g = csp.run(
+            csp.run(
                 my_graph,
                 struct_file.name,
                 series_file.name,
@@ -72,3 +72,7 @@ def my_graph(struct_file_name: str, series_file_name: str):
                 print(f"Struct data:\n{struct_df}")
                 series_df = pandas.read_parquet(series_file.name)
                 print(f"Series data:\n{series_df}")
+
+
+if __name__ == "__main__":
+    main()

examples/03_using_adapters/slack/README.me

@@ -0,0 +1 @@
+# Slack Adapter

examples/03_using_adapters/websocket/README.md

@@ -0,0 +1,3 @@
+# Websocket Adapter
+
+- [Websocket Output](./e1_websocket_output.py)

examples/3_using_adapters/e_11_websocket_output.py → examples/03_using_adapters/websocket/e1_websocket_output.py

@@ -29,7 +29,7 @@ def times(timer: ts[bool]) -> ts[datetime]:
 
 
 @csp.graph
-def main(port: int, num_keys: int):
+def my_graph(port: int, num_keys: int):
     snap = csp.timer(timedelta(seconds=0.25))
     angle = csp.count(snap)
 
@@ -57,7 +57,10 @@ def main(port: int, num_keys: int):
 port = 7677
 num_keys = 10
 
-csp.run(main, port, num_keys, starttime=datetime.utcnow(), endtime=timedelta(seconds=360), realtime=True)
+
+def main():
+    csp.run(my_graph, port, num_keys, starttime=datetime.utcnow(), endtime=timedelta(seconds=360), realtime=True)
+
 
 """ Sample html to view the data.  Note to put your machine name on the websocket line below
 <html>
@@ -84,3 +87,6 @@ def main(port: int, num_keys: int):
 </body>
 </html>
 """
+
+if __name__ == "__main__":
+    main()

examples/04_writing_adapters/README.md

@@ -0,0 +1,9 @@
+# Writing Adapters
+
+- [Generic Push Adapter](./e1_generic_push_adapter.py)
+- [Pull Input Adapter](./e2_pullinput.py)
+- [Pull Input Adapter with Adapter Manager](./e3_adaptermanager_pullinput.py)
+- [Push Input Adapter](./e4_pushinput.py)
+- [Push Input Adapter with Adapter Manager](./e5_adaptermanager_pushinput.py)
+- [Output Adapter](./e6_outputadapter.py)
+- [Complete Input/Output Adapter with Adapter Manager](./e7_adaptermanager_inputoutput.py)

examples/4_writing_adapters/e_14_generic_push_adapter.py → examples/04_writing_adapters/e1_generic_push_adapter.py

@@ -32,7 +32,7 @@ def _run(self):
         counter = 0
         # Optionally, we can wait for the adapter to start before proceeding
         # Alternatively we can start pushing data, but push_tick may fail and return False if
-        # the csp engine isnt ready yet
+        # the csp engine isn't ready yet
         self._adapter.wait_for_start()
 
         while self._active and not self._adapter.stopped():
@@ -52,4 +52,9 @@ def my_graph():
     csp.print("data", adapter.out())
 
 
-csp.run(my_graph, realtime=True, starttime=datetime.utcnow(), endtime=timedelta(seconds=10))
+def main():
+    csp.run(my_graph, realtime=True, starttime=datetime.utcnow(), endtime=timedelta(seconds=2))
+
+
+if __name__ == "__main__":
+    main()

examples/4_writing_adapters/e_14_user_adapters_01_pullinput.py → examples/04_writing_adapters/e2_pullinput.py

@@ -1,9 +1,9 @@
 """
-PullInputAdapter is the simplest form of an input adapter for historical data.  One instance is created
-to provide data on a single timeseries.  There are use cases for this construct, though they are limited.
-This is useful when feeding a single source of historical data into a single timeseries.  In most cases however,
-you will likely have a single source that is processed and used to provide data to multiple inputs.  For that construct
-see e_14_user_adapters_02_adaptermanager_siminput.py
+PullInputAdapter is the simplest form of an input adapter for historical data. One instance is created
+to provide data on a single timeseries. There are use cases for this construct, though they are limited.
+This is useful when feeding a single source of historical data into a single timeseries. In most cases however,
+you will likely have a single source that is processed and used to provide data to multiple inputs. For that construct
+see e3_adaptermanager_pullinput.py
 """
 
 from datetime import datetime, timedelta
@@ -45,7 +45,7 @@ def next(self):
         return None
 
 
-# MyPullAdapter is the graph-building time construct.  This is simply a representation of what the
+# MyPullAdapter is the graph-building time construct. This is simply a representation of what the
 # input adapter is and how to create it, including the Impl to use and arguments to pass into it upon construction
 MyPullAdapter = py_pull_adapter_def("MyPullAdapter", MyPullAdapterImpl, ts[int], interval=timedelta, num_ticks=int)
 
@@ -58,4 +58,9 @@ def my_graph():
     print("End of graph building")
 
 
-csp.run(my_graph, starttime=datetime(2020, 12, 28))
+def main():
+    csp.run(my_graph, starttime=datetime(2020, 12, 28))
+
+
+if __name__ == "__main__":
+    main()