Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: simplify openapi_specifications to data_specifications #64

Merged
merged 2 commits into from
Mar 7, 2024
Merged

feat: simplify openapi_specifications to data_specifications #64

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a9d356a
feat: simplify openapi_specifications to data_specifications
donch1989 Mar 6, 2024
a865862
update testcases
donch1989 Mar 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ require (
github.com/frankban/quicktest v1.14.6
github.com/gabriel-vasile/mimetype v1.4.3
github.com/gofrs/uuid v4.4.0+incompatible
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240304063945-0080cc53de5e
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240306151355-4398dad0ba73
github.com/lestrrat-go/jspointer v0.0.0-20181205001929-82fadba7561c
github.com/lestrrat-go/jsref v0.0.0-20211028120858-c0bcbb5abf20
github.com/lestrrat-go/option v1.0.0
Expand Down
6 changes: 2 additions & 4 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,8 @@ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.2 h1:I/pwhnUln5wbMnTyRbzswA0/JxpK8sZj0aUfI3TV1So=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.2/go.mod h1:lsuH8kb4GlMdSlI4alNIBBSAt5CHJtg3i+0WuN9J5YM=
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240226101151-03ed57d8f5f7 h1:h8u5tmkmoHaJy6CpRNZtFLac5x6AGa9PWt2cfK7kH2Y=
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240226101151-03ed57d8f5f7/go.mod h1:jhEL0SauySMoPLVvx105DWyThju9sYTbsXIySVCArmM=
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240304063945-0080cc53de5e h1:fHbGDWVbaFFSrqRmy5cKDlBT/8nlsfS5m5LIo3P7Q70=
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240304063945-0080cc53de5e/go.mod h1:jhEL0SauySMoPLVvx105DWyThju9sYTbsXIySVCArmM=
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240306151355-4398dad0ba73 h1:3YT3WV9F1eltn5x3AhtOuRDO2zY3Aud6nk/som9YQvs=
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240306151355-4398dad0ba73/go.mod h1:jhEL0SauySMoPLVvx105DWyThju9sYTbsXIySVCArmM=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
Expand Down
55 changes: 17 additions & 38 deletions pkg/base/component.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ type IComponent interface {
// Generate Component Specification
generateComponentSpec(title string, availableTasks []*pipelinePB.ComponentTask) (*structpb.Struct, error)
// Generate OpenAPI Specifications
generateOpenAPISpecs(title string, availableTasks []string) (*structpb.Struct, error)
generateDataSpecs(title string, availableTasks []string) (map[string]*pipelinePB.DataSpecification, error)

// Load tasks
loadTasks(tasksJSON []byte) error
Expand Down Expand Up @@ -313,7 +313,7 @@ func (comp *Component) generateComponentSpec(title string, availableTasks []*pip

}

func convertDataSpecToOpenAPISpec(dataSpec *structpb.Struct) (*structpb.Struct, error) {
func formatDataSpec(dataSpec *structpb.Struct) (*structpb.Struct, error) {
// var err error
compSpec := proto.Clone(dataSpec).(*structpb.Struct)
if _, ok := compSpec.Fields["const"]; ok {
Expand All @@ -330,7 +330,7 @@ func convertDataSpecToOpenAPISpec(dataSpec *structpb.Struct) (*structpb.Struct,
compSpec.Fields["instillUIOrder"] = structpb.NewNumberValue(0)
}

converted, err := convertDataSpecToOpenAPISpec(compSpec.Fields["items"].GetStructValue())
converted, err := formatDataSpec(compSpec.Fields["items"].GetStructValue())
if err != nil {
return nil, err
}
Expand All @@ -353,7 +353,7 @@ func convertDataSpecToOpenAPISpec(dataSpec *structpb.Struct) (*structpb.Struct,
if err != nil {
return nil, err
}
converted, err := convertDataSpecToOpenAPISpec(s)
converted, err := formatDataSpec(s)
if err != nil {
return nil, err
}
Expand All @@ -367,7 +367,7 @@ func convertDataSpecToOpenAPISpec(dataSpec *structpb.Struct) (*structpb.Struct,
if err != nil {
return nil, err
}
converted, err := convertDataSpecToOpenAPISpec(s)
converted, err := formatDataSpec(s)
if err != nil {
return nil, err
}
Expand All @@ -382,7 +382,7 @@ func convertDataSpecToOpenAPISpec(dataSpec *structpb.Struct) (*structpb.Struct,
if err != nil {
return nil, err
}
converted, err := convertDataSpecToOpenAPISpec(s)
converted, err := formatDataSpec(s)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -417,46 +417,25 @@ func convertDataSpecToOpenAPISpec(dataSpec *structpb.Struct) (*structpb.Struct,
return compSpec, nil
}

func (comp *Component) generateOpenAPISpecs(title string, availableTasks []string) (*structpb.Struct, error) {
func (comp *Component) generateDataSpecs(title string, availableTasks []string) (map[string]*pipelinePB.DataSpecification, error) {

openAPITemplates := &structpb.Struct{Fields: map[string]*structpb.Value{}}
for _, availableTask := range availableTasks {
openAPITemplate := &structpb.Struct{}

err := protojson.Unmarshal([]byte(OpenAPITemplate), openAPITemplate)
specs := map[string]*pipelinePB.DataSpecification{}
for _, task := range availableTasks {
spec := &pipelinePB.DataSpecification{}
var err error
taskJSONStruct := proto.Clone(comp.tasks[task]).(*structpb.Struct)
spec.Input, err = formatDataSpec(taskJSONStruct.Fields["input"].GetStructValue())
if err != nil {
return nil, err
}
openAPITemplate.Fields["info"].GetStructValue().Fields["title"] = structpb.NewStringValue(fmt.Sprintf("%s Component - %s", title, availableTask))

walk := openAPITemplate
for _, key := range []string{"paths", "/execute", "post", "requestBody", "content", "application/json", "schema", "properties", "inputs"} {
walk = walk.Fields[key].GetStructValue()
}

taskJSONStruct := proto.Clone(comp.tasks[availableTask]).(*structpb.Struct)

inputStruct, err := convertDataSpecToOpenAPISpec(taskJSONStruct.Fields["input"].GetStructValue())
spec.Output, err = formatDataSpec(taskJSONStruct.Fields["output"].GetStructValue())
if err != nil {
return nil, fmt.Errorf("task %s: %s error: %+v", title, availableTask, err)
}
walk.Fields["items"] = structpb.NewStructValue(inputStruct)

walk = openAPITemplate
for _, key := range []string{"paths", "/execute", "post", "responses", "200", "content", "application/json", "schema", "properties", "outputs"} {
walk = walk.Fields[key].GetStructValue()
}

outputStruct, err := convertDataSpecToOpenAPISpec(taskJSONStruct.Fields["output"].GetStructValue())
if err != nil {
return nil, fmt.Errorf("task %s: %s error: %+v", title, availableTask, err)
return nil, err
}
walk.Fields["items"] = structpb.NewStructValue(outputStruct)

openAPITemplates.Fields[availableTask] = structpb.NewStructValue(openAPITemplate)
specs[task] = spec
}

return openAPITemplates, nil
return specs, nil
}

func (comp *Component) loadTasks(tasksJSONBytes []byte) error {
Expand Down
10 changes: 5 additions & 5 deletions pkg/base/connector.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,9 @@ type IConnector interface {
// Add definition
AddConnectorDefinition(def *pipelinePB.ConnectorDefinition) error
// Get the connector definition by definition uid
GetConnectorDefinitionByUID(defUID uuid.UUID, resourceConfig *structpb.Struct, componentConfig *structpb.Struct) (*pipelinePB.ConnectorDefinition, error)
GetConnectorDefinitionByUID(defUID uuid.UUID, resourceConfig *structpb.Struct, component *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error)
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note:
The reason we want to pass component data into the definition is that some fields are dynamically generated. For example, in the Instill Model Connector, the enum inside the definition data JSON schema is generated by the connector setting on the fly. So we need to pass component data into it and generate a correct JSON schema.

The current implementation design looks a little bit weird, but it works. I think we can improve the structure in the future.

Copy link
Collaborator

@jvallesm jvallesm Mar 7, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍 Thanks for the clarification, I wondered why we had these unused fields but with this info I realised that each connector implementation (e.g. restapi.Init() must implement this interface and there we make use of these fields.

We'll need to update now the connector and operator repos with the new interface, right? Nevermind, just saw instill-ai/connector#135

// Get the connector definition by definition id
GetConnectorDefinitionByID(defID string, resourceConfig *structpb.Struct, componentConfig *structpb.Struct) (*pipelinePB.ConnectorDefinition, error)
GetConnectorDefinitionByID(defID string, resourceConfig *structpb.Struct, component *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error)
// Get the list of connector definitions under this connector
ListConnectorDefinitions() []*pipelinePB.ConnectorDefinition

Expand Down Expand Up @@ -93,7 +93,7 @@ func (c *Connector) LoadConnectorDefinitions(definitionsJSONBytes []byte, tasksJ
return err
}

def.Spec.OpenapiSpecifications, err = c.generateOpenAPISpecs(def.Title, availableTasks)
def.Spec.DataSpecifications, err = c.generateDataSpecs(def.Title, availableTasks)
if err != nil {
return err
}
Expand Down Expand Up @@ -188,7 +188,7 @@ func (c *Connector) ListConnectorDefinitions() []*pipelinePB.ConnectorDefinition
}

// GetConnectorDefinitionByUID gets the connector definition by definition uid
func (c *Connector) GetConnectorDefinitionByUID(defUID uuid.UUID, _ /*resourceConfig*/ *structpb.Struct, _ /* componentConfig */ *structpb.Struct) (*pipelinePB.ConnectorDefinition, error) {
func (c *Connector) GetConnectorDefinitionByUID(defUID uuid.UUID, _ /*resourceConfig */ *structpb.Struct, _ /*component*/ *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error) {
def, err := c.Component.getDefinitionByUID(defUID)
if err != nil {
return nil, err
Expand All @@ -203,7 +203,7 @@ func (c *Connector) GetConnectorDefinitionByUID(defUID uuid.UUID, _ /*resourceCo
}

// GetConnectorDefinitionByID gets the connector definition by definition id
func (c *Connector) GetConnectorDefinitionByID(defID string, resourceConfig *structpb.Struct, componentConfig *structpb.Struct) (*pipelinePB.ConnectorDefinition, error) {
func (c *Connector) GetConnectorDefinitionByID(defID string, _ /*resourceConfig*/ *structpb.Struct, _ /*component*/ *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error) {
def, err := c.Component.getDefinitionByID(defID)
if err != nil {
return nil, err
Expand Down
11 changes: 5 additions & 6 deletions pkg/base/operator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import (

"github.com/gofrs/uuid"
"google.golang.org/protobuf/encoding/protojson"
"google.golang.org/protobuf/types/known/structpb"

pipelinePB "github.com/instill-ai/protogen-go/vdp/pipeline/v1beta"
)
Expand All @@ -21,9 +20,9 @@ type IOperator interface {
// Add definition
AddOperatorDefinition(def *pipelinePB.OperatorDefinition) error
// Get the operator definition by definition uid
GetOperatorDefinitionByUID(defUID uuid.UUID, componentConfig *structpb.Struct) (*pipelinePB.OperatorDefinition, error)
GetOperatorDefinitionByUID(defUID uuid.UUID, component *pipelinePB.OperatorComponent) (*pipelinePB.OperatorDefinition, error)
// Get the operator definition by definition id
GetOperatorDefinitionByID(defID string, componentConfig *structpb.Struct) (*pipelinePB.OperatorDefinition, error)
GetOperatorDefinitionByID(defID string, component *pipelinePB.OperatorComponent) (*pipelinePB.OperatorDefinition, error)
// Get the list of operator definitions under this operator
ListOperatorDefinitions() []*pipelinePB.OperatorDefinition
}
Expand Down Expand Up @@ -74,7 +73,7 @@ func (o *Operator) LoadOperatorDefinitions(definitionsJSONBytes []byte, tasksJSO
return err
}

def.Spec.OpenapiSpecifications, err = o.generateOpenAPISpecs(def.Title, availableTasks)
def.Spec.DataSpecifications, err = o.generateDataSpecs(def.Title, availableTasks)
if err != nil {
return err
}
Expand Down Expand Up @@ -113,7 +112,7 @@ func (o *Operator) ListOperatorDefinitions() []*pipelinePB.OperatorDefinition {
}

// GetOperatorDefinitionByUID returns the operator definition by definition uid
func (o *Operator) GetOperatorDefinitionByUID(defUID uuid.UUID, _ /*componentConfig*/ *structpb.Struct) (*pipelinePB.OperatorDefinition, error) {
func (o *Operator) GetOperatorDefinitionByUID(defUID uuid.UUID, _ /*component*/ *pipelinePB.OperatorComponent) (*pipelinePB.OperatorDefinition, error) {
def, err := o.Component.getDefinitionByUID(defUID)
if err != nil {
return nil, err
Expand All @@ -128,7 +127,7 @@ func (o *Operator) GetOperatorDefinitionByUID(defUID uuid.UUID, _ /*componentCon
}

// GetOperatorDefinitionByID returns the operator definition by definition id
func (o *Operator) GetOperatorDefinitionByID(defID string, componentConfig *structpb.Struct) (*pipelinePB.OperatorDefinition, error) {
func (o *Operator) GetOperatorDefinitionByID(defID string, _ /*component*/ *pipelinePB.OperatorComponent) (*pipelinePB.OperatorDefinition, error) {
def, err := o.Component.getDefinitionByID(defID)
if err != nil {
return nil, err
Expand Down
147 changes: 52 additions & 95 deletions pkg/base/testdata/wantConnectorDefinition.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,105 +132,62 @@
"title": "OpenAI Component",
"type": "object"
},
"openapi_specifications": {
"data_specifications": {
"TASK_TEXT_EMBEDDINGS": {
"info": {
"title": "OpenAI Component - TASK_TEXT_EMBEDDINGS",
"version": "1.0.0"
"input": {
"instillEditOnNodeFields": [
"text",
"model"
],
"instillUIOrder": 0,
"properties": {
"model": {
"description": "ID of the model to use",
"instillShortDescription": "ID of the model to use",
"instillUIOrder": 0,
"title": "Model",
"type": "string"
},
"text": {
"description": "The text",
"instillShortDescription": "The text",
"instillUIOrder": 1,
"title": "Text",
"type": "string"
}
},
"required": [
"text",
"model"
],
"title": "Input",
"type": "object"
},
"openapi": "3.0.0",
"paths": {
"/execute": {
"post": {
"requestBody": {
"content": {
"application/json": {
"schema": {
"properties": {
"inputs": {
"items": {
"instillEditOnNodeFields": [
"text",
"model"
],
"instillUIOrder": 0,
"properties": {
"model": {
"description": "ID of the model to use",
"instillShortDescription": "ID of the model to use",
"instillUIOrder": 0,
"title": "Model",
"type": "string"
},
"text": {
"description": "The text",
"instillShortDescription": "The text",
"instillUIOrder": 1,
"title": "Text",
"type": "string"
}
},
"required": [
"text",
"model"
],
"title": "Input",
"type": "object"
},
"type": "array"
}
},
"type": "object"
}
}
},
"required": true
"output": {
"instillEditOnNodeFields": [
"embedding"
],
"instillUIOrder": 0,
"properties": {
"embedding": {
"instillUIOrder": 0,
"items": {
"description": "",
"instillFormat": "number",
"instillShortDescription": "",
"instillUIOrder": 0,
"title": "Embedding",
"type": "number"
},
"responses": {
"200": {
"content": {
"application/json": {
"schema": {
"properties": {
"outputs": {
"items": {
"instillEditOnNodeFields": [
"embedding"
],
"instillUIOrder": 0,
"properties": {
"embedding": {
"instillUIOrder": 0,
"items": {
"description": "",
"instillFormat": "number",
"instillShortDescription": "",
"instillUIOrder": 0,
"title": "Embedding",
"type": "number"
},
"title": "Embedding",
"type": "array"
}
},
"required": [
"embedding"
],
"title": "Output",
"type": "object"
},
"type": "array"
}
},
"type": "object"
}
}
},
"description": ""
}
}
"title": "Embedding",
"type": "array"
}
}
},
"required": [
"embedding"
],
"title": "Output",
"type": "object"
}
}
}
Expand Down
Loading
Loading