Version 0.2.0

.gitignore

@@ -4,14 +4,15 @@ build
 dist
 *.egg-info
 
+# test
 *.test_results
 *-test-results.xml
+.coverage
 
 .tools*
 .venv*
 
 # vscode
-*.code-workspace
 .vscode
 
 # python
@@ -21,6 +22,3 @@ __pycache__
 *.py[co]
 
 FUTURE.md
-
-# temp
-examples/pipelayer_microservice*

HISTORY.md

@@ -1,2 +1,10 @@
-## 0.1.0
-Initial release.
+# Version History
+
+## 0.2.0 - 1/22/21
+* Adds support for static/module/lamba functions as well as `pipelayer.Filter` types as Pipeline Filters
+* Basic signature validation for filter functions
+* context property in `pipelayer.Filter` is typed as `Union[Context, Any]`
+* Handles all exceptions raises by filters and raises a `PipelineException` with the original exception assigned to the `inner_exception` property
+
+## 0.1.0 - 1/16/2021
+* Pipeline composed of single-method classes that inherit from `pipelayer.Filter`

README.md

@@ -1,34 +1,14 @@
 # PipeLayer
-PipeLayer is a lightweight pipeline framework. Define a series of filters, and chain them together to create modular applications.
-
-```python
-from app_context import AppContext
-from app_settings import AppSettings
-from pipelayer import Pipeline
-from hello_filter import HelloFilter
-from world_filter import WorldFilter
-from logging import Logger
-
-app_settings = AppSettings()
-app_context = AppContext(app_settings, Logger("Logger"))
-pipeline = Pipeline.create(app_context, "Hello World Pipeline")
-
-output = pipeline.run([
-    HelloFilter(),
-    WorldFilter()
-])
-
-print(f"Pipeline Output: {output}")
-print(pipeline.manifest.__dict__)
-```
+PipeLayer is a lightweight Python pipeline framework. Define a series of filters, and chain them together to create modular applications.
 <br>
 
 ### Table of Contents
 
 * [Installation](#installation)
-* [Quick Start](#quick-start)
+* [Getting Started](#getting-started)
 * [The Framework](#the-framework)
-* [Examples](#examples)
+* _[VERSION HISTORY](HISTORY.md)_
+* _[LICENSE](LICENSE.txt)_
 <br><br>
 
 ## Installation
@@ -38,10 +18,10 @@ From the command line:
 pip install pipelayer
 ```
 
-## Quick Start
+## Getting Started
 
 ### Step 1: Application Settings
-Create a class called AppSettings that inherits from the [`pipelayer.Settings`](Settings) class:
+Create a class called AppSettings that inherits from [`pipelayer.Settings`](Settings):
 
 `app_settings.py`
 ```python
@@ -84,49 +64,58 @@ class AppContext(Context):
 ```
 
 ### Step 3: Create Pipeline Filters
-Create classes that inherits from `pipelayer.Filter`:
+Filters can be defined in the following ways:
+* Classes that inherits from `pipelayer.Filter` and implement the `run` method
+* Functions (instance/class/static/module) that have the following signature `func(context: Any, data: Any)`
+* Anonymous functions (lambda) with two arguments that follow the same pattern for regular functions: `my_func = lambda context, data: data`
 
-`hello_filter.py`
+`hello_world_filters.py`
 ```python
 from pipelayer import Filter
 
 
 class HelloFilter(Filter):
     def run(self, context) -> str:
         return "Hello"
-```
-
-`world_filter.py`
-```python
-from pipelayer import Filter
 
 
 class WorldFilter(Filter):
     def run(self, context, data) -> str:
-        return data + " World!"
+        return f"{data},  World!"
+```
+
+`functions.py`
+```python
+def create_message_dict(context, data: str) -> dict:
+    return {"message": data}
 ```
 
 ### Step 4: Create a Pipeline
 Create a module to run the pipeline:
 
 `app.py`
 ```python
+from logging import Logger
+from pipelayer import Pipeline
+
 from app_context import AppContext
 from app_settings import AppSettings
-from pipelayer import Pipeline
-from hello_filter import HelloFilter
-from world_filter import WorldFilter
-from logging import Logger
+from functions import create_message
+from hello_world_filters import HelloFilter, WorldFilter
 
 app_settings = AppSettings()
 app_context = AppContext(app_settings, Logger("Logger"))
 hello_world_pipeline = Pipeline.create(app_context, "Hello World Pipeline")
 
 output = hello_world_pipeline.run([
-    HelloFilter(),
-    WorldFilter()
+    HelloFilter,                           # pipeline.Filter type
+    WorldFilter(),                         # pipeline.Filter instance
+    create_message_dict                    # function type
+    lambda context, data: json.dumps(data) # anonymous function
 ])
 
+# output = '{"message": "Hello, World!"}'
+
 print(f"Pipeline Output: {output}")
 print(hello_world_pipeline.manifest.__dict__)
 
@@ -145,6 +134,7 @@ run app.py
 * [Context](#pipelayer.context)
 * [Settings](#pipelayer.settings)
 * [Manifest](#pipelayer.manifest)
+* [Utilities](#utilities)
 <br><br>
 
 
@@ -163,7 +153,7 @@ An instance of `pipelayer.Manifest` that is created at runtime.
 
 ***Methods***
 
-__`create(context: Context, name: str = "") -> Pipeline`__<br>
+__`create(context: Union[Context, Any], name: str = "") -> Pipeline`__<br>
 The factory method to create a pipeline
 
 __`run(filters: List[Filter], data: Any = None) -> Any`__<br>
@@ -187,7 +177,7 @@ Optional.
 
 ***Methods***
 
-__`run(context: Context, data: Any) -> Any`__<br>
+__`run(context: Union[Context, Any], data: Any) -> Any`__<br>
 The type of the `data` argument in the abstract class is `Any`, but you can use the correct type for the data when implementing method. The same is true for the return type.
 <br><br>
 
@@ -216,3 +206,9 @@ The settings class is an optional base class for applications settings that inhe
 ### __`pipelayer.Manifest`__
 The Manifest keeps a record of [`Pipeline`](#pipeline) and [`Filter`](#filter) activity.
 
+
+### Utilities
+
+### __`pipelayer.util.render_manifest(manifest: Manifest, indent: int = 2) -> str`__
+Static function that renders formatted JSON data
+

examples/simple_pipelayer/.gitignore

@@ -11,7 +11,6 @@ dist
 .venv*
 
 # vscode
-*.code-workspace
 .vscode
 
 # python
@@ -21,6 +20,3 @@ __pycache__
 *.py[co]
 
 FUTURE.md
-
-# temp
-examples/pipelayer_microservice*

examples/simple_pipelayer/gt.simple_pipelayer.code-workspace

@@ -0,0 +1,38 @@
+{
+    "folders": [
+        {
+            "path": "."
+        }
+    ],
+    "settings": {
+        "python.pythonPath": ".venv\\Scripts\\python.exe",
+        "python.analysis.extraPaths": [
+            "./src",
+            "./test"
+        ],
+        "python.formatting.provider": "autopep8",
+        "editor.formatOnSave": true,
+        "python.formatting.autopep8Args": [
+            "--max_line_length",
+            "120"
+        ],
+        "python.linting.enabled": true,
+        "python.linting.flake8Enabled": true,
+        "python.linting.lintOnSave": true,
+        "python.linting.flake8Args": [
+            "--max-line-length",
+            "120",
+            "--inline-quotes",
+            "double",
+            "--exclude",
+            ".venv"
+        ],
+        "editor.semanticHighlighting.enabled": true,
+        "python.testing.pytestArgs": [
+            "test"
+        ],
+        "python.testing.unittestEnabled": false,
+        "python.testing.nosetestsEnabled": false,
+        "python.testing.pytestEnabled": true
+    }
+}

examples/simple_pipelayer/pyproject.toml

@@ -19,3 +19,7 @@ markers = [
     "happy",
     "sad"
 ]
+
+[tool.coverage.report]
+fail_under = 100
+show_missing = true

examples/simple_pipelayer/scripts/run.ps1

@@ -0,0 +1,3 @@
+. .\scripts\_lib.ps1
+
+Run_App

examples/simple_pipelayer/src/app/_path_patch.py

@@ -3,6 +3,6 @@
 import sys
 from pathlib import Path
 
-project_root = Path(os.path.abspath(__file__)).parents[3]
+project_root = Path(os.path.abspath(__file__)).parents[4]
+print(project_root)
 sys.path.insert(0, f"{project_root}/src")
-sys.path.insert(0, f"{project_root}/examples/simple_pipelayer/src")

examples/simple_pipelayer/src/app/app.py

@@ -1,31 +1,33 @@
+# region Imports
+# ==========================================================================
+# Uncomment this patch if running the example within of the pipelayer project
+# import app._path_patch  # NOQA F401
+
+import json
 from logging import Logger
 from typing import Any, Tuple
 
 from pipelayer import Pipeline
 
-# =========================================================================
-# Remove this patch if running the example outside of the pipelayer project
-import app._path_patch  # NOQA F401
 from app.app_context import AppContext
 from app.app_settings import AppSettings
+from app.filter.create_response import create_response
 from app.filter.hello_filter import HelloFilter
 from app.filter.world_filter import WorldFilter
-from app.render import print_manifest
+
+# endregion
 
 
 def main() -> Tuple[Pipeline, Any]:
     app_settings = AppSettings()
 
     context = AppContext(app_settings, Logger("Logger"))
     pipeline = Pipeline.create(context, "Hello World Pipeline")
-    filters = [HelloFilter(), WorldFilter()]
+    filters = [
+        HelloFilter,
+        WorldFilter("Append ', World.'", post_process=create_response),
+        lambda context, data: json.dumps(data)
+    ]
     output = pipeline.run(filters, None)
 
-    print_manifest(pipeline.manifest)
-
     return pipeline, output
-
-
-if __name__ == "__main__":
-    from app.render import print_output
-    print_output(main())

examples/simple_pipelayer/src/app/filter/create_response.py

@@ -0,0 +1,5 @@
+from app.app_context import AppContext
+
+
+def create_response(context: AppContext, data: str) -> dict:
+    return {"message": data}

examples/simple_pipelayer/src/app/filter/hello_filter.py

@@ -4,6 +4,5 @@
 
 
 class HelloFilter(Filter):
-
     def run(self, context: AppContext, data: str = None) -> str:
         return "Hello"

examples/simple_pipelayer/src/app/filter/world_filter.py

@@ -4,6 +4,5 @@
 
 
 class WorldFilter(Filter):
-
     def run(self, context: AppContext, data: str) -> str:
-        return f"{data} World"
+        return f"{data}, World."

examples/simple_pipelayer/src/app/render.py

@@ -17,16 +17,14 @@ class Color:
 
 
 def print_manifest(manifest: Manifest) -> None:
-    print(f"{Color.OKCYAN}=================={Color.ENDC}")
-    print(f"{Color.BOLD}Pipeline Manifest:{Color.ENDC}")
-    print(f"{Color.OKCYAN}------------------{Color.ENDC}")
-    print(render_manifest(manifest))
-    print("")
+    _render("MANIFEST", render_manifest(manifest))
 
 
 def print_output(output: Any) -> None:
-    print(f"{Color.OKCYAN}=================={Color.ENDC}")
-    print(f"{Color.BOLD}Pipeline Output:{Color.ENDC}")
-    print(f"{Color.OKCYAN}------------------{Color.ENDC}")
-    print(output)
-    print("")
+    _render("OUTPUT", output)
+
+
+def _render(message: str, output: Any) -> None:
+    print(f"{Color.OKCYAN}--------------------------------------------------------------------------------{Color.ENDC}")
+    print(f"{Color.OKGREEN}{message}:{Color.ENDC}\n")
+    print(output + "\n")

examples/simple_pipelayer/src/requirements.txt

@@ -1 +1 @@
-pipelayer~=0.1.0
+pipelayer~=0.2.0

examples/simple_pipelayer/src/run.py

@@ -0,0 +1,7 @@
+from app.app import main
+from app.render import print_manifest, print_output
+
+pipeline, output = main()
+
+print_manifest(pipeline.manifest)
+print_output(output)

examples/simple_pipelayer/test/__init__.py

@@ -3,5 +3,4 @@
 
 project_root = Path(__file__).parents[1]
 sys.path.insert(0, f"{project_root}/src")
-sys.path.insert(1, f"{project_root}/examples")
 sys.path.insert(2, f"{project_root}/test")