feat(connector,operator,component): adopt jsonscema validation and dataclass type hint

.pylint.ini

@@ -7,7 +7,7 @@ extension-pkg-whitelist=
 
 # Add files or directories to the blacklist. They should be base names, not
 # paths.
-ignore=CVS,protogen,protobufs
+ignore=CVS,protogen,protobufs,schema
 
 # Add files or directories matching the regex patterns to the blacklist. The
 # regex matches against base names, not paths.
@@ -68,6 +68,7 @@ disable=
     too-many-ancestors,
     too-many-instance-attributes,
     too-many-statements,
+    too-many-lines,
     attribute-defined-outside-init,
     unsupported-assignment-operation,
     unsupported-delete-operation,

Makefile

@@ -49,6 +49,10 @@ get-proto:
 update-proto:
 	@ git submodule update --remote --merge
 
+.PHONY: update-specs
+update-specs:
+	@ bash bin/specs
+
 # TEST ########################################################################
 
 RANDOM_SEED ?= $(shell date +%s)

bin/specs

@@ -0,0 +1,74 @@
+#!/bin bash
+
+connector_schemas=("airbyte" "bigquery" "googlecloudstorage" "stabilityai" "googlesearch" "airbyte" "huggingface" "instill" "numbers" "openai" "pinecone" "redis" "restapi" "website")
+
+for connector in ${connector_schemas[@]}; do
+    echo "=====================@@@ Fetching and processing $connector @@@====================="
+    if [[ "$connector" == 'stabilityai' ]] || [[ "$connector" == 'openai' ]]; then
+        echo "Downloading ${connector}.json..."
+        curl https://raw.githubusercontent.com/instill-ai/connector/main/pkg/${connector}/config/${connector}.json -L -s -o ./instill/resources/schema/jsons/${connector}.json
+    fi
+    curl https://raw.githubusercontent.com/instill-ai/connector/main/pkg/${connector}/config/definitions.json -L -s -o ./instill/resources/schema/jsons/${connector}_definitions.json
+    echo "Downloading ${connector}_definitions.json..."
+    cat <<<$(jq '.[0].spec.resource_specification' ./instill/resources/schema/jsons/${connector}_definitions.json) >./instill/resources/schema/jsons/${connector}_definitions.json
+    o_path=./instill/resources/schema/${connector}.py
+    if [ "$connector" == "airbyte" ]; then
+        o_path=./instill/resources/schema/${connector}/
+    fi
+    echo "Generating ${o_path}..."
+    datamodel-codegen --strip-default-none --disable-timestamp --use-schema-description --use-title-as-name --input ./instill/resources/schema/jsons/${connector}_definitions.json --input-file-type jsonschema --output ${o_path} --output-model-type dataclasses.dataclass
+    # tasks
+    echo "Downloading ${connector}_tasks.json..."
+    curl https://raw.githubusercontent.com/instill-ai/connector/main/pkg/${connector}/config/tasks.json -L -s -o ./instill/resources/schema/jsons/${connector}_tasks.json
+
+    common=$(cat ./instill/resources/schema/jsons/${connector}_tasks.json | jq -rc '."$defs" // empty')
+    cat ./instill/resources/schema/jsons/${connector}_tasks.json | jq -rc 'to_entries | .[]' | while read line; do
+        task=$(echo $line | jq -rc '.key')
+        if [[ "$task" == '$defs' ]]; then
+            continue
+        fi
+        echo $line | jq -rc '.value | to_entries | .[]' | while read inner_line; do
+            schema=$(echo $inner_line | jq -rc '.value')
+            io=$(echo $inner_line | jq -rc '.key')
+            filename=$(echo "${connector}_${task}_${io}" | tr "[:upper:]" "[:lower:]")
+            if [ ! -z "$common" ]; then
+                schema=${schema::${#schema}-1}
+                schema="${schema},\"\$defs\":${common}}"
+            fi
+            echo $schema >./instill/resources/schema/jsons/$filename.json
+            echo "Generating $filename.py..."
+            datamodel-codegen --strip-default-none --disable-timestamp --use-schema-description --use-title-as-name --input ./instill/resources/schema/jsons/$filename.json --input-file-type jsonschema --output ./instill/resources/schema/$filename.py --output-model-type dataclasses.dataclass
+        done
+    done
+done
+
+operator_schemas=("base64" "end" "image" "json" "start" "text")
+
+for operator in ${operator_schemas[@]}; do
+    echo "=====================@@@ Fetching and processing $operator @@@====================="
+    echo "Downloading ${operator}_definitions.json..."
+    curl https://raw.githubusercontent.com/instill-ai/operator/main/pkg/${operator}/config/definitions.json -L -s -o ./instill/resources/schema/jsons/${operator}_definitions.json
+    # tasks
+    echo "Downloading ${operator}_tasks.json..."
+    curl https://raw.githubusercontent.com/instill-ai/operator/main/pkg/${operator}/config/tasks.json -L -s -o ./instill/resources/schema/jsons/${operator}_tasks.json
+
+    common=$(cat ./instill/resources/schema/jsons/${operator}_tasks.json | jq -rc '."$defs" // empty')
+    cat ./instill/resources/schema/jsons/${operator}_tasks.json | jq -rc 'to_entries | .[]' | while read line; do
+        task=$(echo $line | jq -rc '.key')
+        if [[ "$task" == '$defs' ]]; then
+            continue
+        fi
+        echo $line | jq -rc '.value | to_entries | .[]' | while read inner_line; do
+            schema=$(echo $inner_line | jq -rc '.value')
+            io=$(echo $inner_line | jq -rc '.key')
+            filename=$(echo "${operator}_${task}_${io}" | tr "[:upper:]" "[:lower:]")
+            if [ ! -z "$common" ]; then
+                schema=${schema::${#schema}-1}
+                schema="${schema},\"\$defs\":${common}}"
+            fi
+            echo $schema >./instill/resources/schema/jsons/$filename.json
+            echo "Generating $filename.py..."
+            datamodel-codegen --strip-default-none --disable-timestamp --use-schema-description --use-title-as-name --input ./instill/resources/schema/jsons/$filename.json --input-file-type jsonschema --output ./instill/resources/schema/$filename.py --output-model-type dataclasses.dataclass
+        done
+    done
+done

instill/resources/connector.py

@@ -36,9 +36,7 @@ def __call__(self, task_inputs: list, mode="execute"):
                 self.resource.id, task_inputs
             )
             return resp.outputs
-        return self.client.pipeline_service.test_connector(
-            self.resource.id, task_inputs
-        ).state
+        return self.test()
 
     @property
     def client(self):
@@ -56,7 +54,9 @@ def resource(self):
     def resource(self, resource: connector_interface.Connector):
         self._resource = resource
 
-    def create_component(self, name: str, config: dict) -> pipeline_interface.Component:
+    def _create_component(
+        self, name: str, config: dict
+    ) -> pipeline_interface.Component:
         component = pipeline_interface.Component()
         component.id = name
         component.definition_name = self.get_definition().name

instill/resources/connector_ai.py

@@ -1,46 +1,179 @@
-# pylint: disable=no-member,wrong-import-position,no-name-in-module
+# pylint: disable=no-member,wrong-import-position,no-name-in-module,arguments-renamed
+import json
+from typing import Union
+
+import jsonschema
+
 from instill.clients import InstillClient
+from instill.protogen.vdp.pipeline.v1beta.pipeline_pb2 import Component
+from instill.resources import const
 from instill.resources.connector import Connector
+from instill.resources.schema import (
+    instill_task_classification_input,
+    instill_task_detection_input,
+    instill_task_image_to_image_input,
+    instill_task_instance_segmentation_input,
+    instill_task_keypoint_input,
+    instill_task_ocr_input,
+    instill_task_semantic_segmentation_input,
+    instill_task_text_generation_input,
+    instill_task_text_to_image_input,
+    instill_task_visual_question_answering_input,
+)
+from instill.resources.schema.huggingface import HuggingFaceConnectorSpec
+from instill.resources.schema.instill import (
+    InstillModelConnector as InstillModelConnectorConfig,
+)
+from instill.resources.schema.openai import OpenAIConnectorResource
+from instill.resources.schema.stabilityai import StabilityAIConnectorResource
 
 
-class InstillModelConnector(Connector):
+class HuggingfaceConnector(Connector):
+    """Huggingface Connector"""
+
+    with open(
+        f"{const.SPEC_PATH}/huggingface_definitions.json", "r", encoding="utf8"
+    ) as f:
+        definitions_jsonschema = json.loads(f.read())
+
     def __init__(
         self,
         client: InstillClient,
         name: str,
-        server_url: str,
+        config: HuggingFaceConnectorSpec,
+    ) -> None:
+        definition = "connector-definitions/hugging-face"
+
+        jsonschema.validate(vars(config), StabilityAIConnector.definitions_jsonschema)
+        super().__init__(client, name, definition, vars(config))
+
+
+class InstillModelConnector(Connector):
+    """Instill Model Connector"""
+
+    with open(f"{const.SPEC_PATH}/instill_definitions.json", "r", encoding="utf8") as f:
+        definitions_jsonschema = json.loads(f.read())
+
+    def __init__(
+        self,
+        client: InstillClient,
+        config: InstillModelConnectorConfig,
+        name: str = "model-connector",
     ) -> None:
         definition = "connector-definitions/instill-model"
-        configuration = {
-            "api_token": client.pipeline_service.hosts[
-                client.pipeline_service.instance
-            ].token,
-            "server_url": server_url,
-        }
-        super().__init__(client, name, definition, configuration)
+
+        if config.api_token == "":  # type: ignore
+            config.api_token = client.model_service.hosts[  # type: ignore
+                client.model_service.instance
+            ].token
+        if config.server_url == "":  # type: ignore
+            config.server_url = "http://api-gateway:8080"  # type: ignore
+
+        jsonschema.validate(vars(config), InstillModelConnector.definitions_jsonschema)
+        super().__init__(client, name, definition, vars(config))
+
+    def create_component(
+        self,
+        name: str,
+        inp: Union[
+            instill_task_classification_input.Input,
+            instill_task_detection_input.Input,
+            instill_task_instance_segmentation_input.Input,
+            instill_task_semantic_segmentation_input.Input,
+            instill_task_keypoint_input.Input,
+            instill_task_ocr_input.Input,
+            instill_task_image_to_image_input.Input,
+            instill_task_text_generation_input.Input,
+            instill_task_text_to_image_input.Input,
+            instill_task_visual_question_answering_input.Input,
+        ],
+    ) -> Component:
+        if isinstance(inp, instill_task_classification_input.Input):
+            config = {
+                "input": vars(inp),
+                "task": "TASK_CLASSIFICATION",
+            }
+        if isinstance(inp, instill_task_detection_input.Input):
+            config = {
+                "input": vars(inp),
+                "task": "TASK_DETECTION",
+            }
+        if isinstance(inp, instill_task_instance_segmentation_input.Input):
+            config = {
+                "input": vars(inp),
+                "task": "TASK_INSTANCE_SEGMENTATION",
+            }
+        if isinstance(inp, instill_task_semantic_segmentation_input.Input):
+            config = {
+                "input": vars(inp),
+                "task": "TASK_SEMANTIC_SEGMENTATION",
+            }
+        if isinstance(inp, instill_task_keypoint_input.Input):
+            config = {
+                "input": vars(inp),
+                "task": "TASK_KEYPOINT",
+            }
+        if isinstance(inp, instill_task_ocr_input.Input):
+            config = {
+                "input": vars(inp),
+                "task": "TASK_OCR",
+            }
+        if isinstance(inp, instill_task_image_to_image_input.Input):
+            config = {
+                "input": vars(inp),
+                "task": "TASK_IMAGE_TO_IMAGE",
+            }
+        if isinstance(inp, instill_task_text_generation_input.Input):
+            config = {
+                "input": vars(inp),
+                "task": "TASK_TEXT_GENERATION",
+            }
+        if isinstance(inp, instill_task_text_to_image_input.Input):
+            config = {
+                "input": vars(inp),
+                "task": "TASK_TEXT_TO_IMAGE",
+            }
+        if isinstance(inp, instill_task_visual_question_answering_input.Input):
+            config = {
+                "input": vars(inp),
+                "task": "TASK_VISUAL_QUESTION_ANSWERING",
+            }
+        return super()._create_component(name, config)
 
 
 class StabilityAIConnector(Connector):
+    """Stability AI Connector"""
+
+    with open(
+        f"{const.SPEC_PATH}/stabilityai_definitions.json", "r", encoding="utf8"
+    ) as f:
+        definitions_jsonschema = json.loads(f.read())
+
     def __init__(
         self,
         client: InstillClient,
         name: str,
-        api_key: str,
+        config: StabilityAIConnectorResource,
     ) -> None:
         definition = "connector-definitions/stability-ai"
-        configuration = {"api_key": api_key}
-        super().__init__(client, name, definition, configuration)
+
+        jsonschema.validate(vars(config), StabilityAIConnector.definitions_jsonschema)
+        super().__init__(client, name, definition, vars(config))
 
 
 class OpenAIConnector(Connector):
+    """OpenAI Connector"""
+
+    with open(f"{const.SPEC_PATH}/openai_definitions.json", "r", encoding="utf8") as f:
+        definitions_jsonschema = json.loads(f.read())
+
     def __init__(
         self,
         client: InstillClient,
         name: str,
-        api_key: str,
+        config: OpenAIConnectorResource,
     ) -> None:
         definition = "connector-definitions/openai"
-        configuration = {
-            "api_key": api_key,
-        }
-        super().__init__(client, name, definition, configuration)
+
+        jsonschema.validate(vars(config), OpenAIConnector.definitions_jsonschema)
+        super().__init__(client, name, definition, vars(config))