From ef059f8245d24339bd7d098ad9f57ae68f9fb773 Mon Sep 17 00:00:00 2001 From: Antony Mayi Date: Wed, 31 Aug 2022 13:51:30 +0000 Subject: [PATCH] project templating --- .pre-commit-config.yaml | 28 ++-- constraints.txt | 20 +-- docs/application.rst | 2 +- docs/lifecycle.rst | 8 +- docs/project.rst | 2 +- docs/tutorials/titanic/lifecycle.rst | 14 +- docs/tutorials/titanic/serving.rst | 8 +- docs/tutorials/titanic/setup.rst | 2 +- forml/setup/_conf.py | 8 +- forml/setup/_run/__init__.py | 2 +- forml/setup/_run/project.py | 28 +++- forml/setup/_templating.py | 135 ++++++++++++++++++ forml/setup/config.toml | 9 ++ forml/setup/templates/default/setup.py.jinja | 38 +++++ .../{{ project.package }}/__init__.py.jinja | 18 +++ .../{{ project.package }}/evaluation.py.jinja | 40 ++++++ .../{{ project.package }}/pipeline.py.jinja | 32 +++++ .../{{ project.package }}/source.py.jinja | 37 +++++ setup.py | 3 +- tests/setup/_conf/test_conf.py | 74 ---------- tests/setup/template/module.py.jinja | 24 ++++ .../{{ project.package }}/__init__.py.jinja | 18 +++ .../{{ project.package }}/module.py.jinja | 24 ++++ .../{_conf/test_parsed.py => test_conf.py} | 58 +++++++- tests/setup/test_templating.py | 93 ++++++++++++ tutorials/titanic/setup.py | 2 +- 26 files changed, 597 insertions(+), 130 deletions(-) create mode 100644 forml/setup/_templating.py create mode 100644 forml/setup/templates/default/setup.py.jinja create mode 100644 forml/setup/templates/default/{{ project.package }}/__init__.py.jinja create mode 100644 forml/setup/templates/default/{{ project.package }}/evaluation.py.jinja create mode 100644 forml/setup/templates/default/{{ project.package }}/pipeline.py.jinja create mode 100644 forml/setup/templates/default/{{ project.package }}/source.py.jinja delete mode 100644 tests/setup/_conf/test_conf.py create mode 100644 tests/setup/template/module.py.jinja create mode 100644 tests/setup/template/{{ project.package }}/__init__.py.jinja create mode 100644 tests/setup/template/{{ project.package }}/module.py.jinja rename tests/setup/{_conf/test_parsed.py => test_conf.py} (66%) create mode 100644 tests/setup/test_templating.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e7b972b7..76f2481e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -79,18 +79,9 @@ repos: rev: v1.3.1 hooks: - id: forbid-tabs - - id: insert-license - name: Add license for all python files - exclude: ^\.github/.*$ - types: [python] - args: - - --comment-style - - "|#|" - - --license-filepath - - licenses/templates/LICENSE.txt - - --fuzzy-match-generates-todo - id: insert-license name: Add license for all rst files + types: [rst] exclude: ^\.github/.*$ args: - --comment-style @@ -98,30 +89,29 @@ repos: - --license-filepath - licenses/templates/LICENSE.rst - --fuzzy-match-generates-todo - files: \.rst$ - id: insert-license - name: Add license for all yaml files + name: Add license for all md files + types: [markdown] exclude: ^\.github/.*$ - types: [yaml] - files: \.yml$|\.yaml$ args: - --comment-style - - "|#|" + - "" - --license-filepath - licenses/templates/LICENSE.txt - --fuzzy-match-generates-todo - id: insert-license - name: Add license for all md files - files: \.md$ + name: Add license for all jinja files + types: [jinja] exclude: ^\.github/.*$ args: - --comment-style - - "" + - "{#||-#}" - --license-filepath - licenses/templates/LICENSE.txt - --fuzzy-match-generates-todo - id: insert-license name: Add license for all other files + types_or: [toml, ini, yaml, python] exclude: ^\.github/.*$ args: - --comment-style @@ -129,5 +119,3 @@ repos: - --license-filepath - licenses/templates/LICENSE.txt - --fuzzy-match-generates-todo - files: > - \.toml$|\.ini$|\.readthedocs$ diff --git a/constraints.txt b/constraints.txt index 606ebd82..8ab9a9fb 100644 --- a/constraints.txt +++ b/constraints.txt @@ -10,7 +10,9 @@ alembic==1.8.1 # via mlflow anyio==3.6.1 # via starlette -astroid==2.12.4 +appdirs==1.4.4 + # via sphinx-immaterial +astroid==2.12.5 # via pylint attrs==22.1.0 # via @@ -89,7 +91,7 @@ flask==2.2.2 # via # mlflow # prometheus-flask-exporter -fsspec==2022.7.1 +fsspec==2022.8.0 # via dask future==0.18.2 # via pyhive @@ -126,12 +128,13 @@ itsdangerous==2.1.2 jinja2==3.1.2 # via # flask + # forml (setup.py) # nbconvert # nbsphinx # sphinx joblib==1.1.0 # via scikit-learn -jsonschema==4.14.0 +jsonschema==4.15.0 # via nbformat jupyter-client==7.3.5 # via nbclient @@ -150,7 +153,7 @@ locket==1.0.0 # via partd lxml==4.9.1 # via nbconvert -mako==1.2.1 +mako==1.2.2 # via alembic markupsafe==2.1.1 # via @@ -206,7 +209,7 @@ packaging==21.3 # nbconvert # pytest # sphinx -pandas==1.4.3 +pandas==1.4.4 # via # forml (setup.py) # mlflow @@ -245,7 +248,7 @@ pycln==2.1.1 # via forml (setup.py) pycodestyle==2.9.1 # via flake8 -pydantic==1.9.2 +pydantic==1.10.1 # via sphinx-immaterial pyflakes==2.5.0 # via flake8 @@ -298,6 +301,7 @@ requests==2.28.1 # docker # mlflow # sphinx + # sphinx-immaterial scikit-learn==1.1.2 # via forml (setup.py) scipy==1.9.1 @@ -329,7 +333,7 @@ sphinx==5.1.1 # sphinxcontrib-details-directive sphinx-copybutton==0.5.0 # via forml (setup.py) -sphinx-immaterial==0.8.1 +sphinx-immaterial==0.9.0 # via forml (setup.py) sphinxcontrib-applehelp==1.0.2 # via sphinx @@ -405,7 +409,7 @@ urllib3==1.26.12 # via requests uvicorn==0.18.3 # via forml (setup.py) -virtualenv==20.16.3 +virtualenv==20.16.4 # via pre-commit webencodings==0.5.1 # via diff --git a/docs/application.rst b/docs/application.rst index 3c9ca187..08b7433c 100644 --- a/docs/application.rst +++ b/docs/application.rst @@ -232,7 +232,7 @@ depending on uniqueness of each application :meth:`name ` @@ -242,4 +242,4 @@ Example: .. code-block:: console - $ forml model eval forml-example-titanic + $ forml model eval forml-tutorial-titanic diff --git a/docs/project.rst b/docs/project.rst index 3de0f075..fda1754b 100644 --- a/docs/project.rst +++ b/docs/project.rst @@ -110,7 +110,7 @@ as the custom setuptools ``disctlass`` . The rest is the usual ``setup.py`` cont import setuptools from forml import project - setuptools.setup(name='forml-example-titanic', + setuptools.setup(name='forml-tutorial-titanic', version='0.1.dev0', packages=setuptools.find_packages(include=['titanic*'], where=os.path.dirname(__file__)), setup_requires=['forml'], diff --git a/docs/tutorials/titanic/lifecycle.rst b/docs/tutorials/titanic/lifecycle.rst index 8a1d59b7..1656f542 100644 --- a/docs/tutorials/titanic/lifecycle.rst +++ b/docs/tutorials/titanic/lifecycle.rst @@ -94,10 +94,10 @@ free to change the directory to another location before executing the commands. .. code-block:: console $ forml model list - forml-example-titanic - $ forml model list forml-example-titanic + forml-tutorial-titanic + $ forml model list forml-tutorial-titanic 0.1.dev0 - $ forml model list forml-example-titanic 0.1.dev0 + $ forml model list forml-tutorial-titanic 0.1.dev0 The output shows the project artifact is available in the registry as a release ``0.1.dev0`` not having any generation yet (the last command not producing any output). @@ -108,8 +108,8 @@ free to change the directory to another location before executing the commands. .. code-block:: console - $ forml model train forml-example-titanic - $ forml model list forml-example-titanic 0.1.dev0 + $ forml model train forml-tutorial-titanic + $ forml model list forml-tutorial-titanic 0.1.dev0 1 Now we have our first :ref:`generation ` of the titanic models available in the @@ -119,7 +119,7 @@ free to change the directory to another location before executing the commands. .. code-block:: console - $ forml model apply forml-example-titanic + $ forml model apply forml-tutorial-titanic [0.38717846 0.37779938 0.38008973 0.37771585 0.3873835 0.38832168 0.38671783 0.38736506 0.38115396 0.37622997 0.37642134 0.37965842 ... @@ -131,7 +131,7 @@ free to change the directory to another location before executing the commands. .. code-block:: console - $ forml model -R visual apply forml-example-titanic + $ forml model -R visual apply forml-tutorial-titanic .. image:: ../../_static/images/titanic-apply.png :target: ../../_static/images/titanic-apply.png diff --git a/docs/tutorials/titanic/serving.rst b/docs/tutorials/titanic/serving.rst index f05c4637..b4c0bf35 100644 --- a/docs/tutorials/titanic/serving.rst +++ b/docs/tutorials/titanic/serving.rst @@ -63,7 +63,7 @@ the means of publishing into a :ref:`platform-configured ` appl $ forml application put application.py $ forml application list - forml-example-titanic + forml-tutorial-titanic Serving @@ -100,14 +100,14 @@ Let's explore the capabilities using manual ``curl`` queries: .. code-block:: console - $ curl -X POST -H 'Content-Type: application/json' -d '[{"Pclass":1, "Name":"Foo", "Sex": "male", "Age": 34, "SibSp": 3, "Parch": 2, "Ticket": "13", "Fare": 10.1, "Cabin": "123", "Embarked": "S"}]' http://127.0.0.1:8080/forml-example-titanic + $ curl -X POST -H 'Content-Type: application/json' -d '[{"Pclass":1, "Name":"Foo", "Sex": "male", "Age": 34, "SibSp": 3, "Parch": 2, "Ticket": "13", "Fare": 10.1, "Cabin": "123", "Embarked": "S"}]' http://127.0.0.1:8080/forml-tutorial-titanic [{"c0":0.3459976655}] #. Making the same query but requesting the result to be encoded as CSV: .. code-block:: console - $ curl -X POST -H 'Content-Type: application/json' -H 'Accept: text/csv' -d '[{"Pclass":1,"Name":"Foo", "Sex": "male", "Age": 34, "SibSp": 3, "Parch": 2, "Ticket": "13", "Fare": 10.1, "Cabin": "123", "Embarked": "S"}]' http://127.0.0.1:8080/forml-example-titanic + $ curl -X POST -H 'Content-Type: application/json' -H 'Accept: text/csv' -d '[{"Pclass":1,"Name":"Foo", "Sex": "male", "Age": 34, "SibSp": 3, "Parch": 2, "Ticket": "13", "Fare": 10.1, "Cabin": "123", "Embarked": "S"}]' http://127.0.0.1:8080/forml-tutorial-titanic c0 0.34599766550668526 @@ -116,7 +116,7 @@ Let's explore the capabilities using manual ``curl`` queries: .. code-block:: console - $ curl -X POST -H 'Content-Type: application/json; format=pandas-split' -H 'Accept: application/json; format=pandas-values' -d '{"columns": ["Pclass", "Name", "Sex", "Age", "SibSp", "Parch", "Ticket", "Fare", "Cabin", "Embarked"], "data": [[1, "Foo", "male", 34, 3, 2, 13, 10.1, "123", "S"]]}' http://127.0.0.1:8080/forml-example-titanic + $ curl -X POST -H 'Content-Type: application/json; format=pandas-split' -H 'Accept: application/json; format=pandas-values' -d '{"columns": ["Pclass", "Name", "Sex", "Age", "SibSp", "Parch", "Ticket", "Fare", "Cabin", "Embarked"], "data": [[1, "Foo", "male", 34, 3, 2, 13, 10.1, "123", "S"]]}' http://127.0.0.1:8080/forml-tutorial-titanic [[0.3459976655]] That concludes this Titanic Challenge tutorial, from here you can continue to the other diff --git a/docs/tutorials/titanic/setup.rst b/docs/tutorials/titanic/setup.rst index 49956559..00f038af 100644 --- a/docs/tutorials/titanic/setup.rst +++ b/docs/tutorials/titanic/setup.rst @@ -28,7 +28,7 @@ Run the following shell command to create the initial :ref:`project structure

typing.Sequence[Section]: SECTION_LOGGING = 'LOGGING' +SECTION_TEMPLATING = 'TEMPLATING' SECTION_PLATFORM = 'PLATFORM' SECTION_REGISTRY = 'REGISTRY' SECTION_FEED = 'FEED' @@ -291,17 +292,20 @@ def _lookup(cls, reference: typing.Iterable[str]) -> typing.Sequence[Section]: OPT_EVAL = 'eval' APPNAME = 'forml' +PRJNAME = re.sub(r'\.[^.]*$', '', pathlib.Path(sys.argv[0]).name) +#: System-level setup directory SYSDIR = pathlib.Path('/etc') / APPNAME +#: User-level setup directory USRDIR = pathlib.Path(os.getenv(f'{APPNAME.upper()}_HOME', pathlib.Path.home() / f'.{APPNAME}')) +#: Sequence of setup directories in ascending priority order PATH = pathlib.Path(__file__).parent, SYSDIR, USRDIR +#: Main config file name APPCFG = 'config.toml' -PRJNAME = re.sub(r'\.[^.]*$', '', pathlib.Path(sys.argv[0]).name) DEFAULTS = { # all static defaults should go rather to the ./config.toml (in this package) OPT_TMPDIR: tempfile.gettempdir(), SECTION_LOGGING: { - OPT_CONFIG: 'logging.ini', OPT_FACILITY: handlers.SysLogHandler.LOG_USER, OPT_PATH: f'./{PRJNAME}.log', }, diff --git a/forml/setup/_run/__init__.py b/forml/setup/_run/__init__.py index 2d4520ad..458674bc 100644 --- a/forml/setup/_run/__init__.py +++ b/forml/setup/_run/__init__.py @@ -70,7 +70,7 @@ def print(listing: typing.Iterable[typing.Any]) -> None: def group( context: core.Context, config: typing.Optional[str], - loglevel: typing.Optional[str], # pylint: disable=unused-argument + loglevel: typing.Optional[str], logfile: typing.Optional[str], ): """Lifecycle Management for Datascience Projects.""" diff --git a/forml/setup/_run/project.py b/forml/setup/_run/project.py index b37b6cfc..6d808c97 100644 --- a/forml/setup/_run/project.py +++ b/forml/setup/_run/project.py @@ -30,6 +30,8 @@ import forml from forml.io import dsl +from .. import _templating + if typing.TYPE_CHECKING: from .. import _run @@ -50,7 +52,7 @@ def _setup_path(self) -> pathlib.Path: """Get the absolute setup.py path.""" return self.path / self.SETUP_NAME - def run_setup(self, *argv: str, **options): + def run_setup(self, *argv: str, **options) -> None: """Interim hack to call the setup.py""" sys.argv[:] = [str(self._setup_path), *argv, *(a for k, v in options.items() if v for a in (f'--{k}', v))] try: @@ -60,9 +62,20 @@ def run_setup(self, *argv: str, **options): except FileNotFoundError as err: raise forml.MissingError(f'Invalid ForML project: {err}') from err + def create_project( + self, + name: str, + template: typing.Optional[str], + package: typing.Optional[str], + version: typing.Optional[str], + requirements: typing.Sequence[str], + ) -> None: + """Helper for creating a new project structure.""" + _templating.project(name, self.path, template, package, version, requirements) + @click.group(name='project') -@click.option('--path', type=click.Path(exists=True, dir_okay=True), help='Project root directory.') +@click.option('--path', type=click.Path(exists=False, dir_okay=True, file_okay=False), help='Project root directory.') @click.pass_context def group(context: core.Context, path: typing.Optional[str]): """Project command group (development lifecycle).""" @@ -71,14 +84,21 @@ def group(context: core.Context, path: typing.Optional[str]): @group.command() @click.argument('name', required=True) +@click.option('--template', type=str, help='Name of existing project template.') @click.option('--package', type=str, help='Full python package path to be used.') +@click.option('--version', type=str, help='Initial project version.') @click.option('-r', '--requirements', multiple=True, type=str, help='List of install requirements.') @click.pass_obj def init( - scope: Scope, name: str, package: typing.Optional[str], requirements: typing.Optional[typing.Sequence[str]] + scope: Scope, + name: str, + template: typing.Optional[str], + package: typing.Optional[str], + version: typing.Optional[str], + requirements: typing.Sequence[str], ) -> None: """Create skeleton for a new project.""" - raise forml.MissingError(f'Creating project {name}... not implemented') + scope.create_project(name, template, package, version, [r.strip() for t in requirements for r in t.split(',')]) @group.command() diff --git a/forml/setup/_templating.py b/forml/setup/_templating.py new file mode 100644 index 00000000..22f41f88 --- /dev/null +++ b/forml/setup/_templating.py @@ -0,0 +1,135 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +ForML projects templating. +""" +import datetime +import getpass +import logging +import pathlib +import typing + +import jinja2 + +import forml + +from . import _conf + +LOGGER = logging.getLogger(__name__) + + +def find(name: str) -> pathlib.Path: + """Find the project template with the given name. + + The templates are searched for in the config locations (:data:`setup.USRDIR + ` and :data:`setup.SYSDIR `. + + Args: + name: Name of the template to be found (directory name). + + Returns: + Path to the given template. + + Raises: + forml.MissingError: If there is no template with that name. + + """ + for root in reversed(_conf.PATH): # top priority first + path = root / _conf.CONFIG[_conf.SECTION_TEMPLATING][_conf.OPT_PATH] / name + if path.is_dir(): + return path + raise forml.MissingError(f'Template {name} not found') + + +def generate(target: pathlib.Path, template: pathlib.Path, context: typing.Mapping[str, typing.Any]) -> None: + """Generate a directory structure based on the given template. + + Args: + target: Root directory of the generated structure. + template: Source template directory. + context: Jinja context to be used for the template interpolation. + + Raises: + forml.InvalidError: If any of the target elements already exist. + """ + + def gendir(srcdir: pathlib.Path, dstdir: pathlib.Path) -> None: + """Recursive generator of individual directory levels.""" + assert dstdir.is_dir(), f'Destination {dstdir} not a directory.' + for src in srcdir.iterdir(): + srcname = env.from_string(src.name).render(context) + *subdirs, dstname = srcname.split('.', srcname[: srcname.rfind('.py')].count('.')) + dst = dstdir + for sub in subdirs: + dst /= sub + dst.mkdir(parents=False, exist_ok=True) + dst /= dstname + if dst.exists(): + raise forml.InvalidError(f'{dst} already exists.') + dstmode = src.stat().st_mode + if src.is_dir(): + dst.mkdir(parents=False, exist_ok=False, mode=dstmode) + gendir(src, dst) + continue + assert src.is_file(), f'Source {src} not a file.' + LOGGER.debug('Rendering %s as %s', src, dst) + content = env.get_template(str(src.relative_to(template))).render(context) + if dst.suffix == '.jinja': + dst = dst.with_suffix('') + dst.touch(mode=dstmode, exist_ok=False) + dst.write_text(content) + + target.mkdir(parents=True, exist_ok=True) + env = jinja2.Environment(loader=jinja2.FileSystemLoader(template)) + gendir(template, target) + + +def project( + name: str, + path: pathlib.Path, + template: typing.Optional[str], + package: typing.Optional[str], + version: typing.Optional[str], + requirements: typing.Sequence[str], +) -> None: + """Generate a new project structure based on the requested template. + + Args: + name: New project name + path: Project root directory. + template: Name of a project template to use. + package: Python package path to be used. + version: Initial project version. + requirements: List of project install dependencies. + """ + context = { + 'forml': {'version': forml.__version__}, + 'project': { + 'name': name, + 'package': package or name.replace('-', '_'), + 'version': version, + 'requirements': requirements, + }, + 'system': {'date': datetime.datetime.utcnow(), 'user': getpass.getuser()}, + } + if path.exists(): + if not path.is_dir(): + raise forml.InvalidError(f'Target {path} is not a directory.') + path /= name + src = find(template or _conf.CONFIG[_conf.SECTION_TEMPLATING][_conf.OPT_DEFAULT]) + generate(path, src, context) diff --git a/forml/setup/config.toml b/forml/setup/config.toml index 7c229768..a8d3f55b 100644 --- a/forml/setup/config.toml +++ b/forml/setup/config.toml @@ -16,8 +16,17 @@ # under the License. [LOGGING] +# name (will be searched within the config locations) or absolute path +# of the logger config file config = "logging.ini" +[TEMPLATING] +# name (will be searched within the config locations) or absolute path +# of the project templates directory +path = "templates" +# name of the default template +default = "default" + [RUNNER] default = "dask" diff --git a/forml/setup/templates/default/setup.py.jinja b/forml/setup/templates/default/setup.py.jinja new file mode 100644 index 00000000..a46cb8c7 --- /dev/null +++ b/forml/setup/templates/default/setup.py.jinja @@ -0,0 +1,38 @@ +{# + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +-#} + +""" +{{ project.name|title }} project. + +Generated on {{ system.date }} by {{ system.user }} using ForML {{ forml.version }}. +""" +import os + +import setuptools + +from forml import project + +setuptools.setup( + name='{{ project.name }}', + version='{{ project.version|default('0.1.dev1', True) }}', + packages=setuptools.find_namespace_packages(include=['{{ project.package }}*'], where=os.path.dirname(__file__)), + setup_requires=['forml'], + install_requires={{ project.requirements|list|safe }}, + distclass=project.Distribution, +) diff --git a/forml/setup/templates/default/{{ project.package }}/__init__.py.jinja b/forml/setup/templates/default/{{ project.package }}/__init__.py.jinja new file mode 100644 index 00000000..cb375c3b --- /dev/null +++ b/forml/setup/templates/default/{{ project.package }}/__init__.py.jinja @@ -0,0 +1,18 @@ +{# + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +-#} diff --git a/forml/setup/templates/default/{{ project.package }}/evaluation.py.jinja b/forml/setup/templates/default/{{ project.package }}/evaluation.py.jinja new file mode 100644 index 00000000..d37aad63 --- /dev/null +++ b/forml/setup/templates/default/{{ project.package }}/evaluation.py.jinja @@ -0,0 +1,40 @@ +{# + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +-#} + +""" +{{ project.name|title }} project evaluation. + +Generated on {{ system.date }} by {{ system.user }} using ForML {{ forml.version }}. +""" + +import numpy +from sklearn import metrics + +from forml import evaluation, project + +# Using accuracy on a 20% holdout dataset: +EVALUATION = project.Evaluation( + evaluation.Function( + lambda t, p: metrics.accuracy_score(t, numpy.round(p)) + ), + evaluation.HoldOut(test_size=0.2, stratify=True, random_state=42), +) + +# Registering the descriptor +project.setup(EVALUATION) diff --git a/forml/setup/templates/default/{{ project.package }}/pipeline.py.jinja b/forml/setup/templates/default/{{ project.package }}/pipeline.py.jinja new file mode 100644 index 00000000..d8d4f863 --- /dev/null +++ b/forml/setup/templates/default/{{ project.package }}/pipeline.py.jinja @@ -0,0 +1,32 @@ +{# + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +-#} + +""" +{{ project.name|title }} project pipeline. + +Generated on {{ system.date }} by {{ system.user }} using ForML {{ forml.version }}. +""" + +from forml import project + +# The actual pipeline composition using our preprocessing and model operators: +PIPELINE = ... + +# Registering the pipeline +project.setup(PIPELINE) diff --git a/forml/setup/templates/default/{{ project.package }}/source.py.jinja b/forml/setup/templates/default/{{ project.package }}/source.py.jinja new file mode 100644 index 00000000..d1dd7e97 --- /dev/null +++ b/forml/setup/templates/default/{{ project.package }}/source.py.jinja @@ -0,0 +1,37 @@ +{# + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +-#} + +""" +{{ project.name|title }} project source. + +Generated on {{ system.date }} by {{ system.user }} using ForML {{ forml.version }}. +""" + +from forml import project +from forml.pipeline import payload + +# Using the ForML DSL to specify the data source: +FEATURES = ... +OUTCOMES = ... + +# Setting up the source descriptor: +SOURCE = project.Source.query(FEATURES, OUTCOMES) + +# Registering the descriptor +project.setup(SOURCE) diff --git a/setup.py b/setup.py index 7ca65327..97101a08 100644 --- a/setup.py +++ b/setup.py @@ -68,11 +68,12 @@ maintainer_email='forml-dev@googlegroups.com', license='Apache License 2.0', packages=setuptools.find_packages(include=['forml*'], where=os.path.dirname(__file__)), - package_data={'forml.setup': ['config.toml', 'logging.ini']}, + package_data={'forml.setup': ['config.toml', 'logging.ini', 'templates/**']}, setup_requires=['setuptools', 'wheel', 'tomli'], install_requires=[ 'click', 'cloudpickle', + 'jinja2', 'numpy', 'packaging>=20.0', 'pandas', diff --git a/tests/setup/_conf/test_conf.py b/tests/setup/_conf/test_conf.py deleted file mode 100644 index 644f743c..00000000 --- a/tests/setup/_conf/test_conf.py +++ /dev/null @@ -1,74 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -""" -ForML config unit tests. -""" -# pylint: disable=protected-access -import pathlib -import types -import typing - -import pytest - -from forml.setup import _conf - - -def test_exists(cfg_file: pathlib.Path): - """Test the config file exists.""" - assert cfg_file.is_file() - - -def test_src(cfg_file: pathlib.Path): - """Test the registry config field.""" - assert cfg_file in _conf.CONFIG.sources - - -def test_get(): - """Test the get value matches the test config.toml""" - assert _conf.CONFIG['foobar'] == 'baz' - - -class TestConfig: - """Parser unit tests.""" - - @staticmethod - @pytest.fixture(scope='session') - def defaults() -> typing.Mapping[str, typing.Any]: - """Default values fixtures.""" - return types.MappingProxyType({'foo': 'bar', 'baz': {'scalar': 1, 'seq': [10, 3, 'asd']}}) - - @staticmethod - @pytest.fixture(scope='function') - def parser(defaults: typing.Mapping[str, typing.Any]) -> _conf.Config: - """Parser fixture.""" - return _conf.Config(defaults) - - def test_update(self, parser: _conf.Config): - """Parser update tests.""" - parser.update(baz={'another': 2}) - assert parser['baz']['scalar'] == 1 - parser.update({'baz': {'scalar': 3}}) - assert parser['baz']['scalar'] == 3 - assert parser['baz']['another'] == 2 - parser.update({'baz': {'seq': [3, 'qwe', 'asd']}}) - assert parser['baz']['seq'] == (3, 'qwe', 'asd', 10) - - def test_read(self, parser: _conf.Config, cfg_file: pathlib.Path): - """Test parser file reading.""" - parser.read(cfg_file) - assert cfg_file in parser.sources diff --git a/tests/setup/template/module.py.jinja b/tests/setup/template/module.py.jinja new file mode 100644 index 00000000..e6719424 --- /dev/null +++ b/tests/setup/template/module.py.jinja @@ -0,0 +1,24 @@ +{# + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +-#} + +FORML_VERSION = '{{ forml.version }}' +PROJECT_NAME = '{{ project.name }}' +PROJECT_VERSION = '{{ project.version|default('0.1.dev1', True) }}' +PROJECT_PACKAGE = '{{ project.package }}' +PROJECT_REQUIREMENTS = {{ project.requirements|list|safe }} diff --git a/tests/setup/template/{{ project.package }}/__init__.py.jinja b/tests/setup/template/{{ project.package }}/__init__.py.jinja new file mode 100644 index 00000000..cb375c3b --- /dev/null +++ b/tests/setup/template/{{ project.package }}/__init__.py.jinja @@ -0,0 +1,18 @@ +{# + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +-#} diff --git a/tests/setup/template/{{ project.package }}/module.py.jinja b/tests/setup/template/{{ project.package }}/module.py.jinja new file mode 100644 index 00000000..e6719424 --- /dev/null +++ b/tests/setup/template/{{ project.package }}/module.py.jinja @@ -0,0 +1,24 @@ +{# + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +-#} + +FORML_VERSION = '{{ forml.version }}' +PROJECT_NAME = '{{ project.name }}' +PROJECT_VERSION = '{{ project.version|default('0.1.dev1', True) }}' +PROJECT_PACKAGE = '{{ project.package }}' +PROJECT_REQUIREMENTS = {{ project.requirements|list|safe }} diff --git a/tests/setup/_conf/test_parsed.py b/tests/setup/test_conf.py similarity index 66% rename from tests/setup/_conf/test_parsed.py rename to tests/setup/test_conf.py index 90a04f4e..017e1778 100644 --- a/tests/setup/_conf/test_parsed.py +++ b/tests/setup/test_conf.py @@ -16,9 +16,11 @@ # under the License. """ -ForML section config unit tests. +ForML config unit tests. """ import abc +import pathlib +import types import typing import pytest @@ -27,6 +29,60 @@ from forml.setup import _conf +def test_exists(cfg_file: pathlib.Path): + """Test the config file exists.""" + assert cfg_file.is_file() + + +def test_src(cfg_file: pathlib.Path): + """Test the registry config field.""" + assert cfg_file in _conf.CONFIG.sources + + +def test_get(): + """Test the get value matches the test config.toml""" + assert _conf.CONFIG['foobar'] == 'baz' + + +def test_defaults(): + """Test the static defaults.""" + assert _conf.CONFIG[_conf.SECTION_LOGGING][_conf.OPT_CONFIG] == 'logging.ini' + assert _conf.CONFIG[_conf.SECTION_TEMPLATING][_conf.OPT_PATH] == 'templates' + assert _conf.CONFIG[_conf.SECTION_TEMPLATING][_conf.OPT_DEFAULT] == 'default' + assert _conf.CONFIG[_conf.OPT_TMPDIR] + + +class TestConfig: + """Parser unit tests.""" + + @staticmethod + @pytest.fixture(scope='session') + def defaults() -> typing.Mapping[str, typing.Any]: + """Default values fixtures.""" + return types.MappingProxyType({'foo': 'bar', 'baz': {'scalar': 1, 'seq': [10, 3, 'asd']}}) + + @staticmethod + @pytest.fixture(scope='function') + def parser(defaults: typing.Mapping[str, typing.Any]) -> _conf.Config: + """Parser fixture.""" + return _conf.Config(defaults) + + def test_update(self, parser: _conf.Config): + """Parser update tests.""" + parser.update(baz={'another': 2}) + assert parser['baz']['scalar'] == 1 + parser.update({'baz': {'scalar': 3}}) + assert parser['baz']['scalar'] == 3 + assert parser['baz']['another'] == 2 + parser.update({'baz': {'seq': [3, 'qwe', 'asd']}}) + assert parser['baz']['seq'] == (3, 'qwe', 'asd', 10) + + def test_read(self, parser: _conf.Config, cfg_file: pathlib.Path): + """Test parser file reading.""" + parser.read(cfg_file) + assert cfg_file in parser.sources + + class Resolved(metaclass=abc.ABCMeta): """Base class for parsed section tests using the test config from the config.toml.""" diff --git a/tests/setup/test_templating.py b/tests/setup/test_templating.py new file mode 100644 index 00000000..a0b7001e --- /dev/null +++ b/tests/setup/test_templating.py @@ -0,0 +1,93 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Project templating tests. +""" +import pathlib +import types +import typing + +import pytest + +import forml +from forml import project, setup +from forml.io import asset +from forml.setup import _conf, _templating + + +def test_find(): + """Template finder tests.""" + assert _templating.find(_conf.CONFIG[_conf.SECTION_TEMPLATING][_conf.OPT_DEFAULT]).exists() + with pytest.raises(forml.MissingError, match='not found'): + _templating.find('foobar123') + + +@pytest.fixture(scope='session') +def project_template() -> pathlib.Path: + """Fixture for the test project template.""" + return pathlib.Path(__file__).parent / 'template' + + +@pytest.fixture(scope='session') +def project_requirements() -> typing.Sequence[str]: + """Fixture for the test project requirements.""" + return 'foo', 'bar', 'baz' + + +@pytest.fixture(scope='session') +def templating_context( + project_name: asset.Project.Key, + project_release: asset.Release.Key, + project_manifest: project.Manifest, + project_requirements: typing.Sequence[str], +) -> typing.Mapping[str, typing.Any]: + """Fixture for the templating context.""" + return types.MappingProxyType( + { + 'forml': {'version': forml.__version__}, + 'project': { + 'name': project_name, + 'package': project_manifest.package, + 'version': project_release, + 'requirements': project_requirements, + }, + } + ) + + +def test_generate( + project_template: pathlib.Path, + project_name: asset.Project.Key, + project_release: asset.Release.Key, + project_manifest: project.Manifest, + project_requirements: typing.Sequence[str], + templating_context: typing.Mapping[str, typing.Any], + tmp_path: pathlib.Path, +): + """Templating generator test.""" + _templating.generate(tmp_path, project_template, templating_context) + for modname in ('module', f'{project_manifest.package}.module'): + module = setup.isolated(modname, tmp_path) + assert module.FORML_VERSION == forml.__version__ + assert module.PROJECT_NAME == project_name + assert module.PROJECT_PACKAGE == project_manifest.package + assert module.PROJECT_VERSION == str(project_release) + assert module.PROJECT_REQUIREMENTS == list(project_requirements) + + with pytest.raises(forml.InvalidError, match='already exists'): + _templating.generate(tmp_path, project_template, templating_context) diff --git a/tutorials/titanic/setup.py b/tutorials/titanic/setup.py index 45a318af..9ee9a095 100644 --- a/tutorials/titanic/setup.py +++ b/tutorials/titanic/setup.py @@ -25,7 +25,7 @@ from forml import project setuptools.setup( - name='forml-example-titanic', + name='forml-tutorial-titanic', version='0.1.dev0', packages=setuptools.find_packages(include=['titanic*'], where=os.path.dirname(__file__)), setup_requires=['forml'],