From 0dfb24fd4a2dd3d0c6abe63aaba63c77fe98131c Mon Sep 17 00:00:00 2001 From: Alan Chin Date: Wed, 27 Oct 2021 13:29:34 -0700 Subject: [PATCH 1/4] Initial changes to support runtime sub package installs --- .gitignore | 1 + pyproject.toml | 3 ++ setup.cfg | 143 ++++++++++++++++++++++++++++++++++++++++++++++++- setup.py | 135 ++-------------------------------------------- 4 files changed, 150 insertions(+), 132 deletions(-) diff --git a/.gitignore b/.gitignore index e2e6139af..4d36c9c88 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,7 @@ dist/ downloads/ eggs/ .eggs/ +examples lib/ lib64/ parts/ diff --git a/pyproject.toml b/pyproject.toml index 45f753b42..8a4b30104 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,6 @@ +[build-system] +requires=["jupyter_packaging>=0.9,<2"] + [tool.black] # max line length; same as 'max-line-length' in setup.cfg line-length = 120 diff --git a/setup.cfg b/setup.cfg index eb0fa1856..69e2bd49e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,12 +13,151 @@ # See the License for the specific language governing permissions and # limitations under the License. # +[metadata] +name=elyra +version=attr: elyra._version.__version__ +url=https://github.com/elyra-ai/elyra +description=Elyra provides AI Centric extensions to JupyterLab +long_description=Elyra is a set of AI centric extensions to JupyterLab. It aims to help data scientists, + machine learning engineers and AI developer’s through the model development life cycle complexities. +author=Elyra Maintainers +license=Apache License Version 2.0 +platforms = Linux, Mac OS X, Windows +classifiers= + License :: OSI Approved :: Apache Software License" + Operating System :: OS Independent" + Topic :: Scientific/Engineering" + Topic :: Scientific/Engineering :: Artificial Intelligence" + Topic :: Software Development" + Topic :: Software Development :: Libraries" + Topic :: Software Development :: Libraries :: Python Modules" + Programming Language :: Python :: 3.7" + Programming Language :: Python :: 3.8" + Programming Language :: Python :: 3.9" + Programming Language :: Python :: 3.10" [bdist_wheel] universal=0 -[metadata] -description_file=README.md +[options] +zip_safe = False +include_package_data = True +packages = find: +python_requires = >=3.7 +install_requires = + autopep8>=1.5.0 + black<=21.12b0 # Cap due to psf/black#2846 + click>=8 # elyra-ai/elyra#2579 + colorama + deprecation + entrypoints>=0.3 + jinja2>=3 + jsonschema>=3.2.0 + jupyter_core>=4.6.0 + jupyter_client>=6.1.7 + jupyter-packaging>=0.10 + jupyter_server>=1.7.0 + jupyterlab>=3.3.0 + jupyterlab-git~=0.32 # Avoid breaking 1.x changes + jupyterlab-lsp>=3.8.0 + jupyter-resource-usage>=0.5.1 + MarkupSafe>=2.1 + minio>=7.0.0 + nbclient>=0.5.1 + nbconvert>=6.4.5 + nbdime~=3.1 # Cap from jupyterlab-git + nbformat>=5.1.2 + networkx>=2.5.1 + papermill>=2.3.4 + python-lsp-server[all]>=1.1.0 + pyyaml>=5.3.1 + requests>=2.25.1 + rfc3986-validator>=0.1.1 + tornado>=6.1.0 + typing-extensions>=3.10 + traitlets>=4.3.2 + urllib3>=1.26.5 + watchdog>=2.1.3 + websocket-client + yaspin + +[options.extras_require] +airflow = + pygithub +airflow-examples = + elyra-examples-airflow-catalog +gitlab = + python-gitlab +kfp = + jsonschema>=3.2.0,<4.0 + kfp>=1.7.0,<2.0,!=1.7.2 + pyyaml>=5.3.1,<6.0 + typing-extensions>=3.10,<4 +kfp-examples = + elyra-examples-kfp-catalog +kfp-tekton = + kfp-tekton ~=1.2.0 # See elyra-ai/elyra/pull/2034 for fix pack pinning +test = + elyra-examples-airflow-catalog + elyra-examples-kfp-catalog + flake8>=3.5.0 + flake8-import-order>=0.18.1 + git-python + hacking + importlib-resources + pytest>=5.4.1 + pytest-console-scripts + pytest-tornasync + pytest_virtualenv + requests-mock + requests-unixsocket +all = + %(airflow)s + %(airflow-examples)s + %(gitlab)s + %(kfp)s + %(kfp-examples)s + %(kfp-tekton)s + %(test)s + +[options.entry_points] +console_scripts = + elyra-metadata = elyra.metadata.metadata_app:MetadataApp.main + elyra-pipeline = elyra.cli.pipeline_app:pipeline + jupyter-elyra = elyra.elyra_app:launch_instance +metadata.schemaspaces = + runtimes = elyra.metadata.schemaspaces:Runtimes + runtimes-images = elyra.metadata.schemaspaces:RuntimeImages + code-snippets = elyra.metadata.schemaspaces:CodeSnippets + component-catalogs = elyra.metadata.schemaspaces:ComponentCatalogs + metadata-tests = elyra.tests.metadata.test_utils:MetadataTestSchemaspace +metadata.schemas_providers = + runtimes = elyra.metadata.schemasproviders:RuntimesSchemas + runtimes-images = elyra.metadata.schemasproviders:RuntimeImagesSchemas + code-snippets = elyra.metadata.schemasproviders:CodeSnippetsSchemas + component-catalogs = elyra.metadata.schemasproviders:ComponentCatalogsSchemas + airflow-provider-package-catalog-schema = elyra.pipeline.airflow.provider_package_catalog_connector.airflow_provider_package_schema_provider:AirflowProviderPackageSchemasProvider [airflow] + airflow-package-catalog-schema = elyra.pipeline.airflow.package_catalog_connector.airflow_package_schema_provider:AirflowPackageSchemasProvider [airflow] + metadata-tests = elyra.tests.metadata.test_utils:MetadataTestSchemasProvider +elyra.pipeline.processors = + local = elyra.pipeline.local.processor_local:LocalPipelineProcessor + airflow = elyra.pipeline.airflow.processor_airflow:AirflowPipelineProcessor [airflow] + kfp = elyra.pipeline.kfp.processor_kfp:KfpPipelineProcessor [kfp, kfp_tekton] +elyra.component.catalog_types = + url-catalog = elyra.pipeline.catalog_connector:UrlComponentCatalogConnector + local-file-catalog = elyra.pipeline.catalog_connector:FilesystemComponentCatalogConnector + local-directory-catalog = elyra.pipeline.catalog_connector:DirectoryComponentCatalogConnector + airflow-provider-package-catalog = elyra.pipeline.airflow.provider_package_catalog_connector.airflow_provider_package_catalog_connector:AirflowProviderPackageCatalogConnector [airflow] + airflow-package-catalog = elyra.pipeline.airflow.package_catalog_connector.airflow_package_catalog_connector:AirflowPackageCatalogConnector [airflow] +papermill.engine = + ElyraEngine = elyra.pipeline.elyra_engine:ElyraEngine + +[options.packages.find] +exclude = + docs* + tests + tests.* + elyra.tests.* [flake8] application-import-names = elyra, test diff --git a/setup.py b/setup.py index d04bab6bb..f0f774c6e 100644 --- a/setup.py +++ b/setup.py @@ -16,19 +16,13 @@ import os from jupyter_packaging import get_data_files -from setuptools import setup, find_packages - -long_desc = """ - Elyra is a set of AI centric extensions to JupyterLab. It aims to help data scientists, - machine learning engineers and AI developer’s through the model development life cycle complexities. - """ +from setuptools import setup here = os.path.abspath(os.path.dirname(__file__)) -version_ns = {} -with open(os.path.join(here, "elyra", "_version.py")) as f: - exec(f.read(), {}, version_ns) - +##################################################################################################### +# Additional files and archives to be packaged with distribution +##################################################################################################### data_files_spec = [ ("etc/jupyter/jupyter_notebook_config.d", "etc/config/jupyter_notebook_config.d", "*.json"), ("etc/jupyter/jupyter_server_config.d", "etc/config/jupyter_server_config.d", "*.json"), @@ -41,126 +35,7 @@ ("share/jupyter/labextensions", "dist/labextensions", "**"), ] -runtime_extras = { - "kfp-tekton": [ - "kfp-tekton~=1.2.0", - ], # See elyra-ai/elyra/pull/2034 for fix pack pinning - # Kubeflow Pipelines example components - # (https://github.com/elyra-ai/examples/tree/master/component-catalog-connectors/kfp-example-components-connector) - "kfp-examples": ["elyra-examples-kfp-catalog"], - # Use gitlab as Airflow DAG repository - "gitlab": ["python-gitlab"], -} -runtime_extras["all"] = list(set(sum(runtime_extras.values(), []))) - -setup_args = dict( - name="elyra", - version=version_ns["__version__"], - url="https://github.com/elyra-ai/elyra", - description="Elyra provides AI Centric extensions to JupyterLab", - long_description=long_desc, - author="Elyra Maintainers", - license="Apache License Version 2.0", - data_files=get_data_files(data_files_spec), - packages=find_packages(), - install_requires=[ - "autopep8>=1.5.0", - "click>=8", # elyra-ai/elyra#2579 - "colorama", - "deprecation", - "entrypoints>=0.3", - "jinja2>=3", - "jsonschema>=3.2.0,<4.0", # Cap from kfp - "jupyter_core>=4.6.0", - "jupyter_client>=6.1.7", - "jupyter-packaging>=0.10", - "jupyter_server>=1.7.0", - "jupyterlab>=3.3.0", - "jupyterlab-git~=0.32", # Avoid breaking 1.x changes - "jupyterlab-lsp>=3.8.0", - "jupyter-resource-usage>=0.5.1", - "MarkupSafe>=2.1", - "minio>=7.0.0", - "nbclient>=0.5.1", - "nbconvert>=6.4.5", - "nbdime~=3.1", # Cap from jupyterlab-git - "nbformat>=5.1.2", - "networkx>=2.5.1", - "papermill>=2.3.4", - "python-lsp-server[all]>=1.1.0", - "pyyaml>=5.3.1,<6.0", # Cap from kfp - "requests>=2.25.1", - "rfc3986-validator>=0.1.1", - "tornado>=6.1.0", - "typing-extensions>=3.10,<4", # Cap from kfp - "traitlets>=4.3.2", - "urllib3>=1.26.5", - "watchdog>=2.1.3", - "websocket-client", - "yaspin", - # KFP runtime dependencies - "kfp>=1.7.0,<2.0,!=1.7.2", # We cap the SDK to <2.0 due to possible breaking changes - # Airflow runtime dependencies - "pygithub", - "black<=21.12b0", # Cap due to psf/black#2846 - ], - extras_require={ - "test": ["elyra-examples-kfp-catalog", "pytest", "pytest-tornasync"], - **runtime_extras, - }, - include_package_data=True, - classifiers=[ - "License :: OSI Approved :: Apache Software License", - "Operating System :: OS Independent", - "Topic :: Scientific/Engineering", - "Topic :: Scientific/Engineering :: Artificial Intelligence", - "Topic :: Software Development", - "Topic :: Software Development :: Libraries", - "Topic :: Software Development :: Libraries :: Python Modules", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - ], - entry_points={ - "console_scripts": [ - "elyra-metadata = elyra.metadata.metadata_app:MetadataApp.main", - "elyra-pipeline = elyra.cli.pipeline_app:pipeline", - "jupyter-elyra = elyra.elyra_app:launch_instance", - ], - "metadata.schemaspaces": [ - "runtimes = elyra.metadata.schemaspaces:Runtimes", - "runtimes-images = elyra.metadata.schemaspaces:RuntimeImages", - "code-snippets = elyra.metadata.schemaspaces:CodeSnippets", - "component-catalogs = elyra.metadata.schemaspaces:ComponentCatalogs", - "metadata-tests = elyra.tests.metadata.test_utils:MetadataTestSchemaspace", - ], - "metadata.schemas_providers": [ - "runtimes = elyra.metadata.schemasproviders:RuntimesSchemas", - "runtimes-images = elyra.metadata.schemasproviders:RuntimeImagesSchemas", - "code-snippets = elyra.metadata.schemasproviders:CodeSnippetsSchemas", - "component-catalogs = elyra.metadata.schemasproviders:ComponentCatalogsSchemas", - "airflow-provider-package-catalog-schema = elyra.pipeline.airflow.provider_package_catalog_connector.airflow_provider_package_schema_provider:AirflowProviderPackageSchemasProvider", # noqa: E501 - "airflow-package-catalog-schema = elyra.pipeline.airflow.package_catalog_connector.airflow_package_schema_provider:AirflowPackageSchemasProvider", # noqa: E501 - "metadata-tests = elyra.tests.metadata.test_utils:MetadataTestSchemasProvider", - ], - "elyra.pipeline.processors": [ - "local = elyra.pipeline.local.processor_local:LocalPipelineProcessor", - "airflow = elyra.pipeline.airflow.processor_airflow:AirflowPipelineProcessor", - "kfp = elyra.pipeline.kfp.processor_kfp:KfpPipelineProcessor", - ], - "elyra.component.catalog_types": [ - "url-catalog = elyra.pipeline.catalog_connector:UrlComponentCatalogConnector", - "local-file-catalog = elyra.pipeline.catalog_connector:FilesystemComponentCatalogConnector", - "local-directory-catalog = elyra.pipeline.catalog_connector:DirectoryComponentCatalogConnector", - "airflow-provider-package-catalog = elyra.pipeline.airflow.provider_package_catalog_connector.airflow_provider_package_catalog_connector:AirflowProviderPackageCatalogConnector", # noqa: E501 - "airflow-package-catalog = elyra.pipeline.airflow.package_catalog_connector.airflow_package_catalog_connector:AirflowPackageCatalogConnector", # noqa: E501 - ], - "papermill.engine": [ - "ElyraEngine = elyra.pipeline.elyra_engine:ElyraEngine", - ], - }, -) +setup_args = dict(data_files=get_data_files(data_files_spec)) if __name__ == "__main__": setup(**setup_args) From b2c7e22a65811c504932890b2c5ab0fbda641d65 Mon Sep 17 00:00:00 2001 From: Alan Chin Date: Tue, 5 Apr 2022 17:03:44 -0700 Subject: [PATCH 2/4] Fix typos in metadata --- setup.cfg | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/setup.cfg b/setup.cfg index 69e2bd49e..0b1dc147b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,17 +24,17 @@ author=Elyra Maintainers license=Apache License Version 2.0 platforms = Linux, Mac OS X, Windows classifiers= - License :: OSI Approved :: Apache Software License" - Operating System :: OS Independent" - Topic :: Scientific/Engineering" - Topic :: Scientific/Engineering :: Artificial Intelligence" - Topic :: Software Development" - Topic :: Software Development :: Libraries" - Topic :: Software Development :: Libraries :: Python Modules" - Programming Language :: Python :: 3.7" - Programming Language :: Python :: 3.8" - Programming Language :: Python :: 3.9" - Programming Language :: Python :: 3.10" + License :: OSI Approved :: Apache Software License + Operating System :: OS Independent + Topic :: Scientific/Engineering + Topic :: Scientific/Engineering :: Artificial Intelligence + Topic :: Software Development + Topic :: Software Development :: Libraries + Topic :: Software Development :: Libraries :: Python Modules + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 [bdist_wheel] universal=0 From 0531265a94300454ee31c7d7f8b23f6739fc0302 Mon Sep 17 00:00:00 2001 From: Alan Chin Date: Mon, 11 Apr 2022 08:26:47 -0700 Subject: [PATCH 3/4] Load processor only if library is found --- elyra/metadata/schemasproviders.py | 24 +++++++++++++++++----- elyra/pipeline/processor.py | 32 ++++++++++++++++++------------ setup.cfg | 4 +--- 3 files changed, 39 insertions(+), 21 deletions(-) diff --git a/elyra/metadata/schemasproviders.py b/elyra/metadata/schemasproviders.py index 00caa0053..86cf7ab4e 100644 --- a/elyra/metadata/schemasproviders.py +++ b/elyra/metadata/schemasproviders.py @@ -29,6 +29,20 @@ # We may not have kfp-tekton available and that's okay! TektonClient = None +try: + from kfp import compiler # noqa +except ImportError: + # KFP python package is not installed + compiler = None + +try: + # Check to see if Airflow package is installed, since we do not have any dependencies + # on any Airflow package, we use GitHub as our canary + from github import Github # noqa +except ImportError: + # Github package is not installed, we use GitHub as our default DAG repository + Github = None + from elyra.metadata.schema import SchemasProvider from elyra.metadata.schemaspaces import CodeSnippets from elyra.metadata.schemaspaces import ComponentCatalogs @@ -51,7 +65,7 @@ class ElyraSchemasProvider(SchemasProvider, metaclass=ABCMeta): schema_json = json.load(f) local_schemas.append(schema_json) - def __init__(self): + def __init__(self, *args, **kwargs): self.log = log.get_logger() # get set of registered runtimes self._runtime_processor_names = set() @@ -82,17 +96,17 @@ def get_schemas(self) -> List[Dict]: schemas = self.get_local_schemas_by_schemaspace(Runtimes.RUNTIMES_SCHEMASPACE_ID) for schema in schemas: if schema["name"] in self._runtime_processor_names: - runtime_schemas.append(schema) - if schema["name"] == "kfp": + if schema["name"] == "kfp" and compiler: kfp_schema_present = True - elif schema["name"] == "airflow": + runtime_schemas.append(schema) + elif schema["name"] == "airflow" and Github: airflow_schema_present = True + runtime_schemas.append(schema) else: self.log.error( f"No entrypoint with name '{schema['name']}' was found in group " f"'elyra.pipeline.processor' to match the schema with the same name. Skipping..." ) - if kfp_schema_present: # Update the kfp engine enum to reflect current packages... # If TektonClient package is missing, navigate to the engine property # and remove 'tekton' entry if present and return updated result. diff --git a/elyra/pipeline/processor.py b/elyra/pipeline/processor.py index cdeee8736..de9c54642 100644 --- a/elyra/pipeline/processor.py +++ b/elyra/pipeline/processor.py @@ -35,6 +35,8 @@ from urllib3.exceptions import MaxRetryError from elyra.metadata.manager import MetadataManager +from elyra.metadata.schema import SchemaManager +from elyra.metadata.schemaspaces import Runtimes from elyra.pipeline.component import Component from elyra.pipeline.component_catalog import ComponentCache from elyra.pipeline.pipeline import GenericOperation @@ -57,19 +59,23 @@ def __init__(self, **kwargs): self.root_dir = get_expanded_path(kwargs.get("root_dir")) # Register all known processors based on entrypoint configuration for processor in entrypoints.get_group_all("elyra.pipeline.processors"): - try: - # instantiate an actual instance of the processor - processor_instance = processor.load()(self.root_dir, parent=kwargs.get("parent")) # Load an instance - self.log.info( - f"Registering {processor.name} processor " f'"{processor.module_name}.{processor.object_name}"...' - ) - self.add_processor(processor_instance) - except Exception as err: - # log and ignore initialization errors - self.log.error( - f"Error registering {processor.name} processor " - f'"{processor.module_name}.{processor.object_name}" - {err}' - ) + if processor.name in SchemaManager.instance().get_schemasproviders(Runtimes.RUNTIMES_SCHEMASPACE_ID): + try: + # instantiate an actual instance of the processor + processor_instance = processor.load()( + self.root_dir, parent=kwargs.get("parent") + ) # Load an instance + self.log.info( + f"Registering {processor.name} processor " + f'"{processor.module_name}.{processor.object_name}"...' + ) + self.add_processor(processor_instance) + except Exception as err: + # log and ignore initialization errors + self.log.error( + f"Error registering {processor.name} processor " + f'"{processor.module_name}.{processor.object_name}" - {err}' + ) def add_processor(self, processor): self.log.debug(f"Registering {processor.type.value} runtime processor '{processor.name}'") diff --git a/setup.cfg b/setup.cfg index 0b1dc147b..04f55eeb7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -155,9 +155,7 @@ papermill.engine = [options.packages.find] exclude = docs* - tests - tests.* - elyra.tests.* + tests* [flake8] application-import-names = elyra, test From 3a30545230d6cce5b270f137aeb496424ad21944 Mon Sep 17 00:00:00 2001 From: Alan Chin Date: Wed, 20 Apr 2022 13:59:23 -0700 Subject: [PATCH 4/4] Remove test deps no longer needed --- setup.cfg | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 04f55eeb7..95d1751f9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -85,6 +85,7 @@ install_requires = airflow = pygithub airflow-examples = + %(airflow)s elyra-examples-airflow-catalog gitlab = python-gitlab @@ -94,16 +95,14 @@ kfp = pyyaml>=5.3.1,<6.0 typing-extensions>=3.10,<4 kfp-examples = + %(kfp)s elyra-examples-kfp-catalog kfp-tekton = kfp-tekton ~=1.2.0 # See elyra-ai/elyra/pull/2034 for fix pack pinning test = elyra-examples-airflow-catalog elyra-examples-kfp-catalog - flake8>=3.5.0 - flake8-import-order>=0.18.1 git-python - hacking importlib-resources pytest>=5.4.1 pytest-console-scripts