project templating

.pre-commit-config.yaml

@@ -79,55 +79,43 @@ repos:
   rev: v1.3.1
   hooks:
   - id: forbid-tabs
-  - id: insert-license
-    name: Add license for all python files
-    exclude: ^\.github/.*$
-    types: [python]
-    args:
-    - --comment-style
-    - "|#|"
-    - --license-filepath
-    - licenses/templates/LICENSE.txt
-    - --fuzzy-match-generates-todo
   - id: insert-license
     name: Add license for all rst files
+    types: [rst]
     exclude: ^\.github/.*$
     args:
     - --comment-style
     - "||"
     - --license-filepath
     - licenses/templates/LICENSE.rst
     - --fuzzy-match-generates-todo
-    files: \.rst$
   - id: insert-license
-    name: Add license for all yaml files
+    name: Add license for all md files
+    types: [markdown]
     exclude: ^\.github/.*$
-    types: [yaml]
-    files: \.yml$|\.yaml$
     args:
     - --comment-style
-    - "|#|"
+    - "<!--|| -->"
     - --license-filepath
     - licenses/templates/LICENSE.txt
     - --fuzzy-match-generates-todo
   - id: insert-license
-    name: Add license for all md files
-    files: \.md$
+    name: Add license for all jinja files
+    types: [jinja]
     exclude: ^\.github/.*$
     args:
     - --comment-style
-    - "<!--|| -->"
+    - "{#||-#}"
     - --license-filepath
     - licenses/templates/LICENSE.txt
     - --fuzzy-match-generates-todo
   - id: insert-license
     name: Add license for all other files
+    types_or: [toml, ini, yaml, python]
     exclude: ^\.github/.*$
     args:
     - --comment-style
     - "|#|"
     - --license-filepath
     - licenses/templates/LICENSE.txt
     - --fuzzy-match-generates-todo
-    files: >
-      \.toml$|\.ini$|\.readthedocs$

constraints.txt

@@ -10,7 +10,9 @@ alembic==1.8.1
     # via mlflow
 anyio==3.6.1
     # via starlette
-astroid==2.12.4
+appdirs==1.4.4
+    # via sphinx-immaterial
+astroid==2.12.5
     # via pylint
 attrs==22.1.0
     # via
@@ -89,7 +91,7 @@ flask==2.2.2
     # via
     #   mlflow
     #   prometheus-flask-exporter
-fsspec==2022.7.1
+fsspec==2022.8.0
     # via dask
 future==0.18.2
     # via pyhive
@@ -126,12 +128,13 @@ itsdangerous==2.1.2
 jinja2==3.1.2
     # via
     #   flask
+    #   forml (setup.py)
     #   nbconvert
     #   nbsphinx
     #   sphinx
 joblib==1.1.0
     # via scikit-learn
-jsonschema==4.14.0
+jsonschema==4.15.0
     # via nbformat
 jupyter-client==7.3.5
     # via nbclient
@@ -150,7 +153,7 @@ locket==1.0.0
     # via partd
 lxml==4.9.1
     # via nbconvert
-mako==1.2.1
+mako==1.2.2
     # via alembic
 markupsafe==2.1.1
     # via
@@ -206,7 +209,7 @@ packaging==21.3
     #   nbconvert
     #   pytest
     #   sphinx
-pandas==1.4.3
+pandas==1.4.4
     # via
     #   forml (setup.py)
     #   mlflow
@@ -245,7 +248,7 @@ pycln==2.1.1
     # via forml (setup.py)
 pycodestyle==2.9.1
     # via flake8
-pydantic==1.9.2
+pydantic==1.10.1
     # via sphinx-immaterial
 pyflakes==2.5.0
     # via flake8
@@ -298,6 +301,7 @@ requests==2.28.1
     #   docker
     #   mlflow
     #   sphinx
+    #   sphinx-immaterial
 scikit-learn==1.1.2
     # via forml (setup.py)
 scipy==1.9.1
@@ -329,7 +333,7 @@ sphinx==5.1.1
     #   sphinxcontrib-details-directive
 sphinx-copybutton==0.5.0
     # via forml (setup.py)
-sphinx-immaterial==0.8.1
+sphinx-immaterial==0.9.0
     # via forml (setup.py)
 sphinxcontrib-applehelp==1.0.2
     # via sphinx
@@ -405,7 +409,7 @@ urllib3==1.26.12
     # via requests
 uvicorn==0.18.3
     # via forml (setup.py)
-virtualenv==20.16.3
+virtualenv==20.16.4
     # via pre-commit
 webencodings==0.5.1
     # via

docs/application.rst

@@ -232,7 +232,7 @@ depending on uniqueness of each application :meth:`name <forml.application.Descr
 
     $ forml application put titanic.py
     $ forml application list
-    forml-example-titanic
+    forml-tutorial-titanic
 
 
 .. note::

docs/lifecycle.rst

@@ -200,7 +200,7 @@ Example:
 
 .. code-block:: console
 
-    $ forml model train forml-example-titanic
+    $ forml model train forml-tutorial-titanic
 
 Tune
 """"
@@ -212,7 +212,7 @@ Example:
 
 .. code-block:: console
 
-    $ forml model tune forml-example-titanic
+    $ forml model tune forml-tutorial-titanic
 
 Apply
 """""
@@ -225,7 +225,7 @@ Example:
 
 .. code-block:: console
 
-    $ forml model apply forml-example-titanic
+    $ forml model apply forml-tutorial-titanic
 
 .. seealso::
    In addition to this commandline based batch mechanism, the :ref:`serving engine <serving>`
@@ -242,4 +242,4 @@ Example:
 
 .. code-block:: console
 
-    $ forml model eval forml-example-titanic
+    $ forml model eval forml-tutorial-titanic

docs/project.rst

@@ -110,7 +110,7 @@ as the custom setuptools ``disctlass`` . The rest is the usual ``setup.py`` cont
     import setuptools
     from forml import project
 
-    setuptools.setup(name='forml-example-titanic',
+    setuptools.setup(name='forml-tutorial-titanic',
                      version='0.1.dev0',
                      packages=setuptools.find_packages(include=['titanic*'], where=os.path.dirname(__file__)),
                      setup_requires=['forml'],

docs/tutorials/titanic/lifecycle.rst

@@ -94,10 +94,10 @@ free to change the directory to another location before executing the commands.
    .. code-block:: console
 
        $ forml model list
-       forml-example-titanic
-       $ forml model list forml-example-titanic
+       forml-tutorial-titanic
+       $ forml model list forml-tutorial-titanic
        0.1.dev0
-       $ forml model list forml-example-titanic 0.1.dev0
+       $ forml model list forml-tutorial-titanic 0.1.dev0
 
    The output shows the project artifact is available in the registry as a release ``0.1.dev0``
    not having any generation yet (the last command not producing any output).
@@ -108,8 +108,8 @@ free to change the directory to another location before executing the commands.
 
    .. code-block:: console
 
-       $ forml model train forml-example-titanic
-       $ forml model list forml-example-titanic 0.1.dev0
+       $ forml model train forml-tutorial-titanic
+       $ forml model list forml-tutorial-titanic 0.1.dev0
        1
 
    Now we have our first :ref:`generation <registry-assets>` of the titanic models available in the
@@ -119,7 +119,7 @@ free to change the directory to another location before executing the commands.
 
    .. code-block:: console
 
-       $ forml model apply forml-example-titanic
+       $ forml model apply forml-tutorial-titanic
        [0.38717846 0.37779938 0.38008973 0.37771585 0.3873835  0.38832168
        0.38671783 0.38736506 0.38115396 0.37622997 0.37642134 0.37965842
        ...
@@ -131,7 +131,7 @@ free to change the directory to another location before executing the commands.
 
    .. code-block:: console
 
-       $ forml model -R visual apply forml-example-titanic
+       $ forml model -R visual apply forml-tutorial-titanic
 
    .. image:: ../../_static/images/titanic-apply.png
       :target: ../../_static/images/titanic-apply.png

docs/tutorials/titanic/serving.rst

@@ -63,7 +63,7 @@ the means of publishing into a :ref:`platform-configured <platform-config>` appl
 
     $ forml application put application.py
     $ forml application list
-    forml-example-titanic
+    forml-tutorial-titanic
 
 
 Serving
@@ -100,14 +100,14 @@ Let's explore the capabilities using manual ``curl`` queries:
 
    .. code-block:: console
 
-       $ curl -X POST -H 'Content-Type: application/json' -d '[{"Pclass":1, "Name":"Foo", "Sex": "male", "Age": 34, "SibSp": 3, "Parch": 2, "Ticket": "13", "Fare": 10.1, "Cabin": "123", "Embarked": "S"}]' http://127.0.0.1:8080/forml-example-titanic
+       $ curl -X POST -H 'Content-Type: application/json' -d '[{"Pclass":1, "Name":"Foo", "Sex": "male", "Age": 34, "SibSp": 3, "Parch": 2, "Ticket": "13", "Fare": 10.1, "Cabin": "123", "Embarked": "S"}]' http://127.0.0.1:8080/forml-tutorial-titanic
        [{"c0":0.3459976655}]
 
 #. Making the same query but requesting the result to be encoded as CSV:
 
    .. code-block:: console
 
-       $ curl -X POST -H 'Content-Type: application/json' -H 'Accept: text/csv' -d '[{"Pclass":1,"Name":"Foo", "Sex": "male", "Age": 34, "SibSp": 3, "Parch": 2, "Ticket": "13", "Fare": 10.1, "Cabin": "123", "Embarked": "S"}]' http://127.0.0.1:8080/forml-example-titanic
+       $ curl -X POST -H 'Content-Type: application/json' -H 'Accept: text/csv' -d '[{"Pclass":1,"Name":"Foo", "Sex": "male", "Age": 34, "SibSp": 3, "Parch": 2, "Ticket": "13", "Fare": 10.1, "Cabin": "123", "Embarked": "S"}]' http://127.0.0.1:8080/forml-tutorial-titanic
        c0
        0.34599766550668526
 
@@ -116,7 +116,7 @@ Let's explore the capabilities using manual ``curl`` queries:
 
    .. code-block:: console
 
-       $ curl -X POST -H 'Content-Type: application/json; format=pandas-split' -H 'Accept: application/json; format=pandas-values' -d '{"columns": ["Pclass", "Name", "Sex", "Age", "SibSp", "Parch", "Ticket", "Fare", "Cabin", "Embarked"], "data": [[1, "Foo", "male", 34, 3, 2, 13, 10.1, "123", "S"]]}' http://127.0.0.1:8080/forml-example-titanic
+       $ curl -X POST -H 'Content-Type: application/json; format=pandas-split' -H 'Accept: application/json; format=pandas-values' -d '{"columns": ["Pclass", "Name", "Sex", "Age", "SibSp", "Parch", "Ticket", "Fare", "Cabin", "Embarked"], "data": [[1, "Foo", "male", 34, 3, 2, 13, 10.1, "123", "S"]]}' http://127.0.0.1:8080/forml-tutorial-titanic
        [[0.3459976655]]
 
 That concludes this Titanic Challenge tutorial, from here you can continue to the other

docs/tutorials/titanic/setup.rst

@@ -28,7 +28,7 @@ Run the following shell command to create the initial :ref:`project structure <p
 
 .. code-block:: console
 
-    $ forml project init --requirements=openschema,pandas,scikit-learn,numpy forml-tutorial-titanic
+    $ forml project init --requirements=openschema,pandas,scikit-learn,numpy --package titanic forml-tutorial-titanic
 
 You should see a directory structure like this:
 

forml/setup/_conf.py

@@ -266,6 +266,7 @@ def _lookup(cls, reference: typing.Iterable[str]) -> typing.Sequence[Section]:
 
 
 SECTION_LOGGING = 'LOGGING'
+SECTION_TEMPLATING = 'TEMPLATING'
 SECTION_PLATFORM = 'PLATFORM'
 SECTION_REGISTRY = 'REGISTRY'
 SECTION_FEED = 'FEED'
@@ -291,17 +292,20 @@ def _lookup(cls, reference: typing.Iterable[str]) -> typing.Sequence[Section]:
 OPT_EVAL = 'eval'
 
 APPNAME = 'forml'
+PRJNAME = re.sub(r'\.[^.]*$', '', pathlib.Path(sys.argv[0]).name)
+#: System-level setup directory
 SYSDIR = pathlib.Path('/etc') / APPNAME
+#: User-level setup directory
 USRDIR = pathlib.Path(os.getenv(f'{APPNAME.upper()}_HOME', pathlib.Path.home() / f'.{APPNAME}'))
+#: Sequence of setup directories in ascending priority order
 PATH = pathlib.Path(__file__).parent, SYSDIR, USRDIR
+#: Main config file name
 APPCFG = 'config.toml'
-PRJNAME = re.sub(r'\.[^.]*$', '', pathlib.Path(sys.argv[0]).name)
 
 DEFAULTS = {
     # all static defaults should go rather to the ./config.toml (in this package)
     OPT_TMPDIR: tempfile.gettempdir(),
     SECTION_LOGGING: {
-        OPT_CONFIG: 'logging.ini',
         OPT_FACILITY: handlers.SysLogHandler.LOG_USER,
         OPT_PATH: f'./{PRJNAME}.log',
     },

forml/setup/_run/__init__.py

@@ -70,7 +70,7 @@ def print(listing: typing.Iterable[typing.Any]) -> None:
 def group(
     context: core.Context,
     config: typing.Optional[str],
-    loglevel: typing.Optional[str],  # pylint: disable=unused-argument
+    loglevel: typing.Optional[str],
     logfile: typing.Optional[str],
 ):
     """Lifecycle Management for Datascience Projects."""

forml/setup/_run/project.py

@@ -30,6 +30,8 @@
 import forml
 from forml.io import dsl
 
+from .. import _templating
+
 if typing.TYPE_CHECKING:
     from .. import _run
 
@@ -50,7 +52,7 @@ def _setup_path(self) -> pathlib.Path:
         """Get the absolute setup.py path."""
         return self.path / self.SETUP_NAME
 
-    def run_setup(self, *argv: str, **options):
+    def run_setup(self, *argv: str, **options) -> None:
         """Interim hack to call the setup.py"""
         sys.argv[:] = [str(self._setup_path), *argv, *(a for k, v in options.items() if v for a in (f'--{k}', v))]
         try:
@@ -60,9 +62,20 @@ def run_setup(self, *argv: str, **options):
         except FileNotFoundError as err:
             raise forml.MissingError(f'Invalid ForML project: {err}') from err
 
+    def create_project(
+        self,
+        name: str,
+        template: typing.Optional[str],
+        package: typing.Optional[str],
+        version: typing.Optional[str],
+        requirements: typing.Sequence[str],
+    ) -> None:
+        """Helper for creating a new project structure."""
+        _templating.project(name, self.path, template, package, version, requirements)
+
 
 @click.group(name='project')
-@click.option('--path', type=click.Path(exists=True, dir_okay=True), help='Project root directory.')
+@click.option('--path', type=click.Path(exists=False, dir_okay=True, file_okay=False), help='Project root directory.')
 @click.pass_context
 def group(context: core.Context, path: typing.Optional[str]):
     """Project command group (development lifecycle)."""
@@ -71,14 +84,21 @@ def group(context: core.Context, path: typing.Optional[str]):
 
 @group.command()
 @click.argument('name', required=True)
+@click.option('--template', type=str, help='Name of existing project template.')
 @click.option('--package', type=str, help='Full python package path to be used.')
+@click.option('--version', type=str, help='Initial project version.')
 @click.option('-r', '--requirements', multiple=True, type=str, help='List of install requirements.')
 @click.pass_obj
 def init(
-    scope: Scope, name: str, package: typing.Optional[str], requirements: typing.Optional[typing.Sequence[str]]
+    scope: Scope,
+    name: str,
+    template: typing.Optional[str],
+    package: typing.Optional[str],
+    version: typing.Optional[str],
+    requirements: typing.Sequence[str],
 ) -> None:
     """Create skeleton for a new project."""
-    raise forml.MissingError(f'Creating project {name}... not implemented')
+    scope.create_project(name, template, package, version, [r.strip() for t in requirements for r in t.split(',')])
 
 
 @group.command()