From 9e9d95d4ed2a206982311aea6bb41bc44bf1af4f Mon Sep 17 00:00:00 2001 From: Bastin Robins J Date: Tue, 11 Jun 2019 10:11:42 +0530 Subject: [PATCH 1/3] Modified: Basic ML skeleton created with endpoints working --- .gitignore | 4 ++ controllers/__init__.py | 2 +- requirements.txt | 4 +- routes.yml | 4 +- sparx.py | 92 +++++++++++++++++++++++++++++++++++++---- 5 files changed, 93 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index 70d910f..6991272 100644 --- a/.gitignore +++ b/.gitignore @@ -105,3 +105,7 @@ venv.bak/ # mypy .mypy_cache/ +resources/models/ +resources/temp/ +controllers/* +!controllers/IndexController.py diff --git a/controllers/__init__.py b/controllers/__init__.py index 76a251d..a78cde9 100644 --- a/controllers/__init__.py +++ b/controllers/__init__.py @@ -1,5 +1,5 @@ +from iris_controller import * from ApiController import * -from IndexController import * from os.path import dirname, basename, isfile, join import glob diff --git a/requirements.txt b/requirements.txt index 0b9f86f..b597fb1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,6 @@ tornado==3.1.1 pandas==0.19.2 singledispatch -backports_abc \ No newline at end of file +backports_abc + +# If ML skeleton install pandas, numpy, scipy, sklearn ..etc \ No newline at end of file diff --git a/routes.yml b/routes.yml index 0423cf1..311c859 100644 --- a/routes.yml +++ b/routes.yml @@ -1,6 +1,6 @@ controllers: -- IndexController - ApiController +- iris_controller urls: -- / - /api +- /api/predict/iris diff --git a/sparx.py b/sparx.py index 72a4305..95d4ede 100644 --- a/sparx.py +++ b/sparx.py @@ -12,6 +12,22 @@ from version import version +__UPLOADS__ = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'resources/') +__MODELS__ = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'resources/models/') +# __STORAGE__ = + + +# Argument parser +parser = argparse.ArgumentParser(description="Sparx Application Command Line") +parser.add_argument('service', help="Type of service command") +parser.add_argument('-t', '--type', type=str, metavar='', help='Type of command add or rm') +parser.add_argument('-u', '--url', type=str, metavar='', help='API endpoints url') +parser.add_argument('-c', '--controller', type=str, metavar='', help='Endpoint controller') +parser.add_argument('-m', '--model', type=str, metavar='', help='Pickled predictive model `*.pkl` ') +args = parser.parse_args() + + + def line_prepender(filename, line): ''' Add new line to give file first line ''' with open(filename, 'r+') as f: @@ -68,6 +84,15 @@ def clone_script(origin_script, controller_name): f.write(new_content) + +def clone_model(src, controller_name, destination="resources/models"): + ''' Copy trained model from local to model repository ''' + destination = os.path.join(destination, controller_name + '.pkl') + copyfile(src, destination) + print "Model imported successfully..." + + + def remove_declaration(controller_name): ''' `TODO` Scan and remove the controller declaration and save the file ''' @@ -89,6 +114,17 @@ def add_service(url, controller_name): print "new service generated ..." +def add_ml_service(url, controller_name, model_file): + ''' Add ML/DL pickled model service ''' + add_to_yaml('routes.yml', url, controller_name) + clone_script('core/skeleton/model_controller.py', controller_name) + clone_model(model_file, controller_name) + line_prepender('controllers/__init__.py', 'from ' + str(controller_name) + ' import *') + print "new machine learning service generated ..." + + + + def remove_service_files(controller_name): ''' Remove controller.py and .py* files ''' @@ -98,10 +134,16 @@ def remove_service_files(controller_name): if os.path.exists('controllers/' + controller_name + '.pyc'): os.remove('controllers/'+ controller_name + '.pyc') + if os.path.exists('resources/models/' + controller_name + '.pkl'): + os.remove('resources/models/' + controller_name + '.pkl') + def remove_service(controller_name): ''' Remove the service added to the application ''' + if controller_name == 'IndexController': + sys.exit('Error: IndexController cannot be removed!') + # Modify yaml remove_from_yaml('routes.yml', controller_name) @@ -114,7 +156,8 @@ def remove_service(controller_name): print "Removed {}".format(controller_name) -def list_all_routes(): + +def list_all_endpoints(): ''' List all apps created inside give directory ''' @@ -142,16 +185,47 @@ def start_app(): if __name__ == "__main__": - if sys.argv[1] == 'add': - add_service(sys.argv[2], sys.argv[3]) - - elif sys.argv[1] == 'rm': - remove_service(sys.argv[2]) + # Check if `add` service command + if args.service == 'add': + if args.url != None and args.controller != None: + + # check if model flag is defined + if args.model == None: + + # create a normal service if model is `None` + add_service(args.url, args.controller) + + else: + # check if give model file exists + if os.path.exists(args.model): + + # Add ml service if model is defined + add_ml_service(args.url, args.controller, args.model) + + else: + # throw exception + print 'Model path not defined' + + else: + + # Throw -u and -c required exception + print "`-u` URL required" + print "`-c` Controller required" + + + # Check if `rm` (remove) command + elif args.service == 'rm': + if args.controller != None: + remove_service(args.controller) + else: + print "controller name is required `-c`" - elif sys.argv[1] == 'list': - list_all_routes() + # List all service from routes.yml + elif args.service == 'endpoints': + list_all_endpoints() - elif sys.argv[1] == 'start': + # Start the sparx-core server + elif args.service == 'start': start_app() else: From d6184be243e6d2b61ab5ebb166e6af81b113dd2f Mon Sep 17 00:00:00 2001 From: Bastin Robins J Date: Tue, 11 Jun 2019 10:12:03 +0530 Subject: [PATCH 2/3] Added: Model controller skeleton added --- core/skeleton/model_controller.py | 79 +++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 core/skeleton/model_controller.py diff --git a/core/skeleton/model_controller.py b/core/skeleton/model_controller.py new file mode 100644 index 0000000..f633591 --- /dev/null +++ b/core/skeleton/model_controller.py @@ -0,0 +1,79 @@ +import os +import json +import uuid +import numpy as np +import pandas as pd +import tornado.web +from base_handler import base_handler +from sklearn.externals import joblib + + + +def convert_req_to_csv(fileinfo): + ''' Get the batch file from request payload and convert to pandas + dataframes + ''' + fname = fileinfo['filename'] + + # Actual filename + extn = os.path.splitext(fname)[1] + + # # New filename + cname = str(uuid.uuid4()) + extn + fh = open('temp.csv', 'w') + fh.write(fileinfo['body']) + + + + + +class controller_name(base_handler): + + def get(self): + ''' start the basic API seletion ''' + data = dict( + source='controllers/{}'.format('controller_name.py'), + response=[] + ) + self.write(json.dumps(data)) + self.finish() + + + def post(self): + ''' Create resources ''' + + # Load the pickled model + clf = joblib.load(os.path.join(os.path.abspath(\ + os.path.join(os.path.dirname(os.path.abspath(__file__)),"..", 'resources')), 'models', 'controller_name.pkl')) + + fileinfo = self.request.files['batch'][0] + + # Actual filename + extn = os.path.splitext(fileinfo['filename'])[1] + + random_filename = str(uuid.uuid4()) + extn + + # Generate file + temp_path = os.path.join(os.path.abspath(\ + os.path.join(os.path.dirname(os.path.abspath(__file__)),"..", 'resources')), 'temp', random_filename) + + # Save batch test file + with open(temp_path, 'w') as f: + f.write(fileinfo['body']) + + # Read saved file using pandas dataframe + x_test = pd.read_csv(temp_path) + + # Batch prediction + predicted = clf.predict(x_test.values) + + # Add predicted columns to given dataframe + x_test.insert(loc=len(x_test.columns), column='predicted', value=predicted) + + # Response the given dataframe as JSON + self.set_header('Content-Type', 'application/json') + + + self.write(x_test.reset_index().to_json(orient='records')) + self.finish() + From 1d3948e92051d6232ad192d257bd4be19f29c7af Mon Sep 17 00:00:00 2001 From: Bastin Robins J Date: Tue, 11 Jun 2019 10:27:51 +0530 Subject: [PATCH 3/3] Added: docs directory added --- docs/Makefile | 20 ++++ docs/make.bat | 36 +++++++ docs/source/authors.rst | 16 ++++ docs/source/conf.py | 177 +++++++++++++++++++++++++++++++++++ docs/source/contributing.rst | 5 + docs/source/help.rst | 8 ++ docs/source/history.rst | 8 ++ docs/source/index.rst | 27 ++++++ docs/source/installation.rst | 12 +++ docs/source/usage.rst | 15 +++ 10 files changed, 324 insertions(+) create mode 100644 docs/Makefile create mode 100644 docs/make.bat create mode 100644 docs/source/authors.rst create mode 100644 docs/source/conf.py create mode 100644 docs/source/contributing.rst create mode 100644 docs/source/help.rst create mode 100644 docs/source/history.rst create mode 100644 docs/source/index.rst create mode 100644 docs/source/installation.rst create mode 100644 docs/source/usage.rst diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..4794613 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = python -msphinx +SPHINXPROJ = sparx-core +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..a28b68c --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=python -msphinx +) +set SOURCEDIR=source +set BUILDDIR=build +set SPHINXPROJ=sparx-core + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The Sphinx module was not found. Make sure you have Sphinx installed, + echo.then set the SPHINXBUILD environment variable to point to the full + echo.path of the 'sphinx-build' executable. Alternatively you may add the + echo.Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/docs/source/authors.rst b/docs/source/authors.rst new file mode 100644 index 0000000..8bb2de5 --- /dev/null +++ b/docs/source/authors.rst @@ -0,0 +1,16 @@ +:orphan: + +======= +Authors +======= + +Development Lead +---------------- + +* `Bastin Robins `__ +* `Vandana Bhagat `__ + +Project Support +--------------- + +* `Kothandaraman Sridharan `__ diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..3fcd153 --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,177 @@ +# -*- coding: utf-8 -*- +# +# sparx-core documentation build configuration file, created by +# sphinx-quickstart on Tue Jun 11 10:16:17 2019. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['sphinx.ext.autodoc'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['ntemplates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'sparx-core' +copyright = u'2019, Bastin Robins .J' +author = u'Bastin Robins .J' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = u'0.0.1' +# The full version, including alpha/beta/rc tags. +release = u'0.0.1' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = [] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'sphinx_rtd_theme' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['nstatic'] + + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = 'sparx-coredoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'sparx-core.tex', u'sparx-core Documentation', + u'Bastin Robins .J', 'manual'), +] + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'sparx-core', u'sparx-core Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'sparx-core', u'sparx-core Documentation', + author, 'sparx-core', 'One line description of project.', + 'Miscellaneous'), +] + + + +# -- Options for Epub output ---------------------------------------------- + +# Bibliographic Dublin Core info. +epub_title = project +epub_author = author +epub_publisher = author +epub_copyright = copyright + +# The unique identifier of the text. This can be a ISBN number +# or the project homepage. +# +# epub_identifier = '' + +# A unique identification for the text. +# +# epub_uid = '' + +# A list of files that should not be packed into the epub file. +epub_exclude_files = ['search.html'] + + diff --git a/docs/source/contributing.rst b/docs/source/contributing.rst new file mode 100644 index 0000000..dacb831 --- /dev/null +++ b/docs/source/contributing.rst @@ -0,0 +1,5 @@ +:orphan: + +============= +Contributing +============= diff --git a/docs/source/help.rst b/docs/source/help.rst new file mode 100644 index 0000000..92eb9ac --- /dev/null +++ b/docs/source/help.rst @@ -0,0 +1,8 @@ +:orphan: + + + +Help +================= + +TODO: write content \ No newline at end of file diff --git a/docs/source/history.rst b/docs/source/history.rst new file mode 100644 index 0000000..c69ef4e --- /dev/null +++ b/docs/source/history.rst @@ -0,0 +1,8 @@ +======= +History +======= + +0.0.1 (pre-release on 22 July 2017) +------------------------------------ + +* sparx diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..14543fd --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,27 @@ +.. sparx-core documentation master file, created by + sphinx-quickstart on Tue Jun 11 10:16:17 2019. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Sparx Core Documentation +====================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + help + installation + usage + authors + contributing + history + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/source/installation.rst b/docs/source/installation.rst new file mode 100644 index 0000000..9a12e0f --- /dev/null +++ b/docs/source/installation.rst @@ -0,0 +1,12 @@ +:orphan: + + + +Installation +------------ + +Install the extension with using pip. + +.. code:: bash + + $ pip install -U sparx-cli \ No newline at end of file diff --git a/docs/source/usage.rst b/docs/source/usage.rst new file mode 100644 index 0000000..6f40d39 --- /dev/null +++ b/docs/source/usage.rst @@ -0,0 +1,15 @@ +:orphan: + +Quickstart +=========== + +⚡️Sparx Core is the new API server code which has been build for faster microservices + +Simple Usage +~~~~~~~~~~~~ + +.. code-block:: python + + >>> sparx + +