Ml management structure (#87)

* dataset library classes * executor stubs * orm stub * more stubs
eyadgaran · Oct 20, 2021 · 9e3237c · 9e3237c
1 parent e85f064
commit 9e3237c
Show file tree

Hide file tree

Showing 15 changed files with 94 additions and 0 deletions.
diff --git a/simpleml/adapters/__init__.py b/simpleml/adapters/__init__.py
@@ -0,0 +1,10 @@
+'''
+Adapters for compatibility between persistables and executors. Adapters are the
+pluggable wrappers for persistables that ensure interoperability between
+steps in processing.
+
+Adapters must not contain any logic central to the processing. They can only be
+interfaces to align formats, signatures, etc for consistent behavior.
+'''
+
+__author__ = 'Elisha Yadgaran'
diff --git a/simpleml/datasets/dask/__init__.py b/simpleml/datasets/dask/__init__.py
@@ -0,0 +1,5 @@
+'''
+Dataset Library support for Dask
+'''
+
+__author__ = 'Elisha Yadgaran'
diff --git a/simpleml/datasets/numpy/__init__.py b/simpleml/datasets/numpy/__init__.py
@@ -0,0 +1,5 @@
+'''
+Dataset Library support for Numpy
+'''
+
+__author__ = 'Elisha Yadgaran'
diff --git a/simpleml/datasets/pandas/__init__.py b/simpleml/datasets/pandas/__init__.py
@@ -0,0 +1,5 @@
+'''
+Dataset Library support for Pandas
+'''
+
+__author__ = 'Elisha Yadgaran'
diff --git a/simpleml/datasets/ray/__init__.py b/simpleml/datasets/ray/__init__.py
@@ -0,0 +1,5 @@
+'''
+Dataset Library support for Ray
+'''
+
+__author__ = 'Elisha Yadgaran'
diff --git a/simpleml/datasets/tensorflow/__init__.py b/simpleml/datasets/tensorflow/__init__.py
@@ -0,0 +1,5 @@
+'''
+Dataset Library support for Tensorflow
+'''
+
+__author__ = 'Elisha Yadgaran'
diff --git a/simpleml/executors/__init__.py b/simpleml/executors/__init__.py
@@ -0,0 +1,12 @@
+'''
+Executor frameworks
+
+An executor is the backend that operates over persistables to use and create
+artifacts. A central characteristic of executors is that they are interchangeable;
+outputs and artifacts will be the same. There is NO GUARANTEE that all persistable
+types will be compatible with all executor frameworks. There is also no guarantee
+that executors support native operations for the persistable types. Outputs will
+be the same, but using different executors may result in more efficient processing
+'''
+
+__author__ = 'Elisha Yadgaran'
diff --git a/simpleml/executors/airflow/__init__.py b/simpleml/executors/airflow/__init__.py
@@ -0,0 +1,5 @@
+'''
+Executor support for airflow backend
+'''
+
+__author__ = 'Elisha Yadgaran'
diff --git a/simpleml/executors/beam/__init__.py b/simpleml/executors/beam/__init__.py
@@ -0,0 +1,5 @@
+'''
+Executor support for beam backend
+'''
+
+__author__ = 'Elisha Yadgaran'
diff --git a/simpleml/executors/dask/__init__.py b/simpleml/executors/dask/__init__.py
@@ -0,0 +1,5 @@
+'''
+Executor support for dask backend
+'''
+
+__author__ = 'Elisha Yadgaran'
diff --git a/simpleml/executors/kubernetes/__init__.py b/simpleml/executors/kubernetes/__init__.py
@@ -0,0 +1,5 @@
+'''
+Executor support for kubernetes backend
+'''
+
+__author__ = 'Elisha Yadgaran'
diff --git a/simpleml/executors/multiprocessing/__init__.py b/simpleml/executors/multiprocessing/__init__.py
@@ -0,0 +1,5 @@
+'''
+Executor support for multiprocessing backend
+'''
+
+__author__ = 'Elisha Yadgaran'
diff --git a/simpleml/executors/ray/__init__.py b/simpleml/executors/ray/__init__.py
@@ -0,0 +1,5 @@
+'''
+Executor support for ray backend
+'''
+
+__author__ = 'Elisha Yadgaran'
diff --git a/simpleml/executors/threading/__init__.py b/simpleml/executors/threading/__init__.py
@@ -0,0 +1,5 @@
+'''
+Executor support for threading backend
+'''
+
+__author__ = 'Elisha Yadgaran'
diff --git a/simpleml/orm/__init__.py b/simpleml/orm/__init__.py
@@ -0,0 +1,12 @@
+'''
+ORM package
+
+Focuses on all database related interaction, intentionally separated to allow
+parallel Persistable objects to only deal with the glue interactions across
+python types and libraries
+
+Each mapped Persistable table model has a 1:1 parallel class
+defined as a native python object
+'''
+
+__author__ = 'Elisha Yadgaran'