feature(plugin): dataframes plugin (#177)

* Plugin structure Working @dataframe
hopeit-git · Mar 20, 2024 · bed7ba3 · bed7ba3
1 parent 8899de0
commit bed7ba3
Show file tree

Hide file tree

Showing 49 changed files with 3,394 additions and 177 deletions.
diff --git a/.github/workflows/hopeit-engine-pypi-publishing.yml b/.github/workflows/hopeit-engine-pypi-publishing.yml
@@ -50,6 +50,9 @@ jobs:
     - name: make plugin apps-client
       run: |
         make PLUGINFOLDER=plugins/clients/apps-client/ dist-plugin
+    - name: make plugin dataframes
+      run: |
+        make PLUGINFOLDER=plugins/data/dataframes/ dist-plugin
 
     - name: Publish hopeit.engine on PyPI
       env: 
@@ -91,3 +94,8 @@ jobs:
         PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
       run: |
         make PLUGINFOLDER=plugins/clients/apps-client pypi-plugin
+    - name: Publish plugin dataframes on PyPI
+      env: 
+        PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
+      run: |
+        make PLUGINFOLDER=plugins/data/dataframes pypi-plugin
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -45,6 +45,22 @@
             "cwd": "${workspaceFolder}",
             "console": "integratedTerminal"
         },
+        {
+            "name": "dataframes-example",
+            "type": "python",
+            "request": "launch",
+            "module": "hopeit.server.web",
+            "env": {
+            },
+            "args": [
+                "--port=8040",
+                "--config-files=engine/config/dev-local.json,plugins/auth/basic-auth/config/plugin-config.json,plugins/ops/config-manager/config/plugin-config.json,plugins/data/dataframes/config/plugin-config.json,apps/examples/dataframes-example/config/app-config.json",
+                "--api-file=apps/examples/dataframes-example/api/openapi.json",
+                "--start-streams",
+            ],
+            "cwd": "${workspaceFolder}",
+            "console": "integratedTerminal"
+        },
         {
             "name": "apps-visualizer",
             "type": "python",

diff --git a/Makefile b/Makefile
@@ -60,6 +60,10 @@ install-plugin:
 	cd $(PLUGINFOLDER) && \
 	pip install -U -e .
 
+install-plugin-extras:
+	cd $(PLUGINFOLDER) && \
+	pip install -U -e .[$(PLUGINEXTRAS)]
+
 qa: test check
 	echo "DONE."
 
@@ -104,6 +108,7 @@ pypi-test-plugin:
 update-examples-api: install-examples
 	bash apps/examples/simple-example/api/create_openapi_file.sh && \
 	bash apps/examples/client-example/api/create_openapi_file.sh && \
+	bash apps/examples/dataframes-example/api/create_openapi_file.sh && \
 	bash plugins/ops/apps-visualizer/api/create_openapi_file.sh
 
 install-examples:
@@ -116,8 +121,10 @@ install-examples:
 	make PLUGINFOLDER=plugins/ops/apps-visualizer install-plugin && \
 	make PLUGINFOLDER=plugins/auth/basic-auth install-plugin && \
 	make PLUGINFOLDER=plugins/clients/apps-client install-plugin && \
+	make PLUGINFOLDER=plugins/data/dataframes PLUGINEXTRAS=pyarrow install-plugin-extras && \
 	make APPFOLDER=apps/examples/simple-example install-app && \
-	make APPFOLDER=apps/examples/client-example install-app
+	make APPFOLDER=apps/examples/client-example install-app && \
+	make APPFOLDER=apps/examples/dataframes-example install-app
 
 run-simple-example:
 	export PYTHONPATH=apps/examples/simple-example/src && \

diff --git a/README.md b/README.md
@@ -96,6 +96,16 @@ Python library that exposes helpers to persist data objects to Redis.
 pip install hopeit.engine[redis-storage]
 ```
 
+**hopeit.dataframes**
+
+Plugin to support working with pandas dataframes as they were objects,
+supporting them as web request and respose payloads and transferring them
+through streams.
+
+```
+pip install hopeit.engine[dataframes]
+```
+
 
 ### Motivation
 

diff --git a/apps/build/ci-static-apps.sh b/apps/build/ci-static-apps.sh
@@ -27,6 +27,14 @@ code+=$?
 python3 -m pylint apps/examples/client-example/src/client_example/
 code+=$?
 
+echo "apps/dataframes-example"
+export MYPYPATH=engine/src/:plugins/storage/fs/src/:plugins/data/dataframes/src/:apps/examples/simple-example/src/:apps/examples/dataframes-example/src/ && python3 -m mypy --namespace-packages -p dataframes_example
+code+=$?
+python3 -m flake8 --max-line-length=120 apps/examples/dataframes-example/src/
+code+=$?
+python3 -m pylint apps/examples/dataframes-example/src/dataframes_example/
+code+=$?
+
 if [ $code -gt 0 ]
 then
   echo "[FAILED] CI STATIC ANALYSIS"

diff --git a/apps/examples/client-example/api/openapi.json b/apps/examples/client-example/api/openapi.json
@@ -1,12 +1,12 @@
 {
   "openapi": "3.0.3",
   "info": {
-    "version": "0.23",
+    "version": "0.24",
     "title": "Client Example",
     "description": "Client Example"
   },
   "paths": {
-    "/api/config-manager/0x23/runtime-apps-config": {
+    "/api/config-manager/0x24/runtime-apps-config": {
       "get": {
         "summary": "Config Manager: Runtime Apps Config",
         "description": "Returns the runtime config for the Apps running on this server",
@@ -62,11 +62,11 @@
           }
         },
         "tags": [
-          "config_manager.0x23"
+          "config_manager.0x24"
         ]
       }
     },
-    "/api/config-manager/0x23/cluster-apps-config": {
+    "/api/config-manager/0x24/cluster-apps-config": {
       "get": {
         "summary": "Config Manager: Cluster Apps Config",
         "description": "Handle remote access to runtime configuration for a group of hosts",
@@ -122,11 +122,11 @@
           }
         },
         "tags": [
-          "config_manager.0x23"
+          "config_manager.0x24"
         ]
       }
     },
-    "/api/client-example/0x23/call-unsecured": {
+    "/api/client-example/0x24/call-unsecured": {
       "get": {
         "summary": "Client Example: Call Unsecured",
         "description": "List all available Something objects connecting to simple-example app",
@@ -196,11 +196,11 @@
           }
         },
         "tags": [
-          "client_example.0x23"
+          "client_example.0x24"
         ]
       }
     },
-    "/api/client-example/0x23/count-and-save": {
+    "/api/client-example/0x24/count-and-save": {
       "get": {
         "summary": "Client Example: Count Objects and Save new one",
         "description": "Count all available Something objects connecting to simple-example app",
@@ -267,7 +267,7 @@
           }
         },
         "tags": [
-          "client_example.0x23"
+          "client_example.0x24"
         ],
         "security": [
           {
@@ -276,7 +276,7 @@
         ]
       }
     },
-    "/api/client-example/0x23/handle-responses": {
+    "/api/client-example/0x24/handle-responses": {
       "get": {
         "summary": "Client Example: Handle Responses",
         "description": "Non default responses and UnhandledResponse exception\n\nTo manage different types of responses from the same endpoint we can use the `responses` parameter where we list the\nhttp response status codes expected and the corresponding data type for each one. In this example `app_call` expect\nand handle, 200 and 404 responses.\n\nAlso in the code you can see how to handle an expection of type `UnhandledResponse` and log as warining.",
@@ -361,7 +361,7 @@
           }
         },
         "tags": [
-          "client_example.0x23"
+          "client_example.0x24"
         ],
         "security": [
           {
@@ -835,7 +835,7 @@
           },
           "engine_version": {
             "type": "string",
-            "default": "0.23.0"
+            "default": "0.24.0"
           }
         },
         "x-module-name": "hopeit.server.config",

diff --git a/apps/examples/dataframes-example/api/create_openapi_file.sh b/apps/examples/dataframes-example/api/create_openapi_file.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+export PYTHONPATH=./apps/examples/dataframes-example/src && \
+hopeit_openapi create \
+--title="Dataframes Example" \
+--description="Dataframes Example" \
+--api-version="$(python -m hopeit.server.version APPS_API_VERSION)" \
+--config-files=engine/config/dev-local.json,plugins/auth/basic-auth/config/plugin-config.json,plugins/ops/config-manager/config/plugin-config.json,plugins/data/dataframes/config/plugin-config.json,apps/examples/dataframes-example/config/app-config.json \
+--output-file=apps/examples/dataframes-example/api/openapi.json