Add airflow package catalog connector

.github/workflows/build.yml

@@ -73,6 +73,15 @@ jobs:
       - name: Check format
         run: make prettier-check-ui
 
+  lint-connectors:
+    name: Lint Connectors
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Lint
+        run: make lint-connectors
+
+
   test-server:
     name: Test Server
     runs-on: ubuntu-latest

Makefile

@@ -14,7 +14,7 @@
 # limitations under the License.
 #
 
-.PHONY: help purge install-all install-examples install uninstall clean test-dependencies lint-server lint-ui lint yarn-install eslint-ui eslint-check-ui prettier-ui prettier-check-ui flake lint-server-dependencies dev-link dev-unlink
+.PHONY: help purge install-all install-examples install uninstall clean test-dependencies lint-server lint-ui lint-connectors lint yarn-install eslint-ui eslint-check-ui prettier-ui prettier-check-ui flake lint-server-dependencies dev-link dev-unlink
 .PHONY: build-ui build-server install-server watch install-extensions build-jupyterlab install-server-package check-install only-install-server
 .PHONY: test-server test-ui test-integration test-integration-debug test docs-dependencies docs dist-ui release pytest
 .PHONY: validate-runtime-images elyra-image publish-elyra-image kf-notebook-image
@@ -90,6 +90,9 @@ build-dependencies:
 lint-server: build-dependencies
 	flake8 elyra
 
+lint-connectors: build-dependencies
+	flake8 catalog-connectors
+
 prettier-check-ui:
 	yarn prettier:check
 
@@ -104,7 +107,7 @@ eslint-ui:
 
 lint-ui: prettier-ui eslint-ui
 
-lint: lint-ui prettier-ui lint-server ## Run linters
+lint: lint-ui prettier-ui lint-server lint-connectors ## Run linters
 
 dev-link:
 	yarn link @elyra/pipeline-services

catalog-connectors/airflow/README.md

@@ -0,0 +1,3 @@
+### Apache Airflow catalog connectors
+
+This directory includes community maintained catalog connectors for Apache Airflow.

catalog-connectors/airflow/airflow-package-catalog-connector/MANIFEST.in

@@ -0,0 +1,38 @@
+#
+# Copyright 2018-2022 Elyra Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# exclude from ANY directory
+global-exclude *.ipynb
+global-exclude *.py[cod]
+global-exclude __pycache__
+global-exclude .git
+global-exclude .ipynb_checkpoints
+global-exclude .DS_Store
+global-exclude *.sh
+global-exclude docs
+global-exclude tests
+
+# explicit includes
+include CONTRIBUTING.md
+include README.md
+include LICENSE
+include dist/*.tgz
+
+recursive-exclude * __pycache__
+recursive-exclude * *.py[co]
+
+# include the connector schema file
+include airflow_package_catalog_connector/airflow-package-catalog.json

catalog-connectors/airflow/airflow-package-catalog-connector/Makefile

@@ -0,0 +1,54 @@
+#
+# Copyright 2021-2022 Elyra Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+.PHONY: help clean lint test-dependencies source-install install dist
+.PHONY: publish test-publish
+
+SHELL:=/bin/bash
+
+PACKAGE_NAME=airflow-package-catalog-connector
+PACKAGE_PATH=airflow_package_catalog_connector 
+
+help:
+# http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
+	@grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+
+clean:
+	rm -rf dist/
+	rm -rf build/
+	rm -rf *.egg-info
+	- pip uninstall -y $(PACKAGE_NAME)
+
+test-dependencies:
+	@pip install -q -r test_requirements.txt
+
+lint: test-dependencies
+	flake8 $(PACKAGE_PATH)
+
+dist: clean lint  ## Build distribution
+	python setup.py bdist_wheel sdist	
+
+source-install: dist  ## Install component connector package from source 
+	pip install .
+
+install:  ## Install component connector package from PyPI
+	pip install $(PACKAGE_NAME)
+
+test-publish:  dist ## Upload package to test PyPI
+	twine upload --repository testpypi dist/*
+
+publish:  dist ## Upload package to PyPI
+	twine upload dist/*

catalog-connectors/airflow/airflow-package-catalog-connector/README.md

@@ -0,0 +1,57 @@
+## Airflow package catalog connector
+
+This catalog connector enables Elyra to load pipeline components from [Apache Airflow packages](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/operators/index.html).
+
+### Install the connector
+
+You can install this catalog connector from PyPI or source code. Note that a **rebuild of JupyterLab is not required**.
+
+**Prerequisites**
+
+- [Elyra](https://elyra.readthedocs.io/en/stable/getting_started/installation.html) (version 3.3 and above).
+- Apache Airflow 2.x
+
+**Install from PyPI**
+
+  ```
+  $ pip install airflow-package-catalog-connector
+  ```
+
+**Install from source code**
+
+   ```
+   $ git clone https://github.com/elyra-ai/elyra.git
+   $ cd catalog-connectors/airflow/airflow-package-catalog-connector/
+   $ make source-install
+   ```
+
+### Use the connector
+
+1. Launch JupyterLab.
+1. [Open the '`Manage Components`' panel](
+https://elyra.readthedocs.io/en/stable/user_guide/pipeline-components.html#managing-custom-components-using-the-jupyterlab-ui).
+1. Add a new Airflow package component catalog ('`+`' > '`New Apache Airflow package operator catalog`').
+1. Specify a catalog name, e.g. '`Airflow 1.10.15 package`'.
+1. (Optional) Specify a category under which the catalog's components will be organized in the palette.
+1. Configure the `Airflow package download URL`. The URL must reference a location that Elyra can access using an HTTP GET request, without the need to authenticate. If the Airflow package is stored on PyPI:
+   1. Search for the desired package version on PyPI.
+   1. In the package's `Navigation` section open the `Download files` link.
+   1. Copy the download link for the package's wheel.
+   ![Example download link for Airflow 1.10](doc/images/airflow_110_download_link_example.png)
+1. Save the catalog entry.
+1. Open the Visual Pipeline Editor and expand the palette. The catalog components are displayed.
+
+### Uninstall the connector
+
+1. Remove all 'Apache Airflow package component catalog' catalog entries from the '`Manage Components`' panel.
+1. Stop JupyterLab.
+1. Uninstall the `airflow-package-catalog-connector` package.
+   ```
+   $ pip uninstall -y airflow-package-catalog-connector
+   ```
+
+### Troubleshooting
+
+**Problem: The palette does not display any components from the configured catalog.**
+
+**Solution:** If the the Elyra GUI does not display any error message indicating that a problem was encountered, inspect the JupyterLab log file.

catalog-connectors/airflow/airflow-package-catalog-connector/airflow_package_catalog_connector/__init__.py

@@ -0,0 +1,15 @@
+#
+# Copyright 2018-2022 Elyra Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#

catalog-connectors/airflow/airflow-package-catalog-connector/airflow_package_catalog_connector/_version.py

@@ -0,0 +1,16 @@
+#
+# Copyright 2018-2022 Elyra Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+__version__ = '0.0.1'

catalog-connectors/airflow/airflow-package-catalog-connector/airflow_package_catalog_connector/airflow-package-catalog.json

@@ -0,0 +1,74 @@
+{
+  "$schema": "https://raw.githubusercontent.com/elyra-ai/elyra/master/elyra/metadata/schemas/meta-schema.json",
+  "$id": "https://raw.githubusercontent.com/elyra-ai/elyra/master/catalog-connectors/airflow/airflow-package-catalog-connector/airflow-package-catalog.json",
+  "title": "Apache Airflow package operator catalog",
+  "name": "airflow-package-catalog",
+  "schemaspace": "component-catalogs",
+  "schemaspace_id": "8dc89ca3-4b90-41fd-adb9-9510ad346620",
+  "metadata_class_name": "elyra.pipeline.component_metadata.ComponentCatalogMetadata",
+  "uihints": {
+    "title": "Apache Airflow core operator catalog",
+    "icon": "",
+    "reference_url": "https://github.com/elyra-ai/elyra/tree/master/catalog-connectors/airflow/airflow-package-catalog-connector"
+  },
+  "properties": {
+    "schema_name": {
+      "title": "Schema Name",
+      "description": "The schema associated with this instance",
+      "type": "string",
+      "const": "airflow-package-catalog"
+    },
+    "display_name": {
+      "title": "Display Name",
+      "description": "Display name of this Component Catalog",
+      "type": "string",
+      "minLength": 1
+    },
+    "metadata": {
+      "description": "Additional data specific to this metadata",
+      "type": "object",
+      "properties": {
+        "description": {
+          "title": "Description",
+          "description": "Description of this Component Catalog",
+          "type": "string",
+          "default": "Airflow package operator catalog"
+        },
+        "runtime_type": {
+          "title": "Runtime",
+          "description": "List of runtime types this catalog supports",
+          "type": "string",
+          "enum": ["APACHE_AIRFLOW"],
+          "default": "APACHE_AIRFLOW",
+          "uihints": {
+            "field_type": "dropdown"
+          }
+        },
+        "categories": {
+          "title": "Category Names",
+          "description": "Assign the operators in the catalog to one or more categories, to group them in the visual pipeline editor palette.",
+          "type": "array",
+          "items": {
+            "type": "string",
+            "maxLength": 18
+          },
+          "default": ["Core packages"],
+          "uihints": {
+            "field_type": "array",
+            "category": "Component Categories"
+          }
+        },
+        "airflow_package_download_url": {
+          "title": "Airflow package download URL",
+          "description": "URL where the Apache Airflow package wheel can be downloaded.",
+          "type": "string",
+          "uihints": {
+            "category": "Source"
+          }
+        }
+      },
+      "required": ["runtime_type", "airflow_package_download_url"]
+    }
+  },
+  "required": ["schema_name", "display_name", "metadata"]
+}