Skip to content

Commit

Permalink
feat: simplify openapi_specifications to data_specifications (#64)
Browse files Browse the repository at this point in the history 
Because

- Related to this instill-ai/protobufs#282,
currently, we have an `openapi_specification` field in the
connector/operator definition. The original purpose was that we could
generate an OpenAPI schema and have some UI to let the user debug on an
individual component. However, it turns out that we don't really need
that OpenAPI schema, and it makes our response very large and hard to
read.

This commit

- Simplifies `openapi_specification` into `data_specification` by
preserving only the input and output JSON schema and removing the nested
structure of the OpenAPI spec.
  • Loading branch information
@donch1989
donch1989 committed Mar 7, 2024
1 parent 81e34c4 commit 7c27d15
Show file tree
Hide file tree
Showing 7 changed files with 132 additions and 242 deletions.
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ require (
github.com/frankban/quicktest v1.14.6
github.com/gabriel-vasile/mimetype v1.4.3
github.com/gofrs/uuid v4.4.0+incompatible
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240304063945-0080cc53de5e
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240306151355-4398dad0ba73
github.com/lestrrat-go/jspointer v0.0.0-20181205001929-82fadba7561c
github.com/lestrrat-go/jsref v0.0.0-20211028120858-c0bcbb5abf20
github.com/lestrrat-go/option v1.0.0
Expand Down
6 changes: 2 additions & 4 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,8 @@ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.2 h1:I/pwhnUln5wbMnTyRbzswA0/JxpK8sZj0aUfI3TV1So=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.2/go.mod h1:lsuH8kb4GlMdSlI4alNIBBSAt5CHJtg3i+0WuN9J5YM=
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240226101151-03ed57d8f5f7 h1:h8u5tmkmoHaJy6CpRNZtFLac5x6AGa9PWt2cfK7kH2Y=
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240226101151-03ed57d8f5f7/go.mod h1:jhEL0SauySMoPLVvx105DWyThju9sYTbsXIySVCArmM=
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240304063945-0080cc53de5e h1:fHbGDWVbaFFSrqRmy5cKDlBT/8nlsfS5m5LIo3P7Q70=
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240304063945-0080cc53de5e/go.mod h1:jhEL0SauySMoPLVvx105DWyThju9sYTbsXIySVCArmM=
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240306151355-4398dad0ba73 h1:3YT3WV9F1eltn5x3AhtOuRDO2zY3Aud6nk/som9YQvs=
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240306151355-4398dad0ba73/go.mod h1:jhEL0SauySMoPLVvx105DWyThju9sYTbsXIySVCArmM=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
Expand Down
55 changes: 17 additions & 38 deletions pkg/base/component.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ type IComponent interface {
// Generate Component Specification
generateComponentSpec(title string, availableTasks []*pipelinePB.ComponentTask) (*structpb.Struct, error)
// Generate OpenAPI Specifications
generateOpenAPISpecs(title string, availableTasks []string) (*structpb.Struct, error)
generateDataSpecs(title string, availableTasks []string) (map[string]*pipelinePB.DataSpecification, error)

// Load tasks
loadTasks(tasksJSON []byte) error
Expand Down Expand Up @@ -313,7 +313,7 @@ func (comp *Component) generateComponentSpec(title string, availableTasks []*pip

}

func convertDataSpecToOpenAPISpec(dataSpec *structpb.Struct) (*structpb.Struct, error) {
func formatDataSpec(dataSpec *structpb.Struct) (*structpb.Struct, error) {
// var err error
compSpec := proto.Clone(dataSpec).(*structpb.Struct)
if _, ok := compSpec.Fields["const"]; ok {
Expand All @@ -330,7 +330,7 @@ func convertDataSpecToOpenAPISpec(dataSpec *structpb.Struct) (*structpb.Struct,
compSpec.Fields["instillUIOrder"] = structpb.NewNumberValue(0)
}

converted, err := convertDataSpecToOpenAPISpec(compSpec.Fields["items"].GetStructValue())
converted, err := formatDataSpec(compSpec.Fields["items"].GetStructValue())
if err != nil {
return nil, err
}
Expand All @@ -353,7 +353,7 @@ func convertDataSpecToOpenAPISpec(dataSpec *structpb.Struct) (*structpb.Struct,
if err != nil {
return nil, err
}
converted, err := convertDataSpecToOpenAPISpec(s)
converted, err := formatDataSpec(s)
if err != nil {
return nil, err
}
Expand All @@ -367,7 +367,7 @@ func convertDataSpecToOpenAPISpec(dataSpec *structpb.Struct) (*structpb.Struct,
if err != nil {
return nil, err
}
converted, err := convertDataSpecToOpenAPISpec(s)
converted, err := formatDataSpec(s)
if err != nil {
return nil, err
}
Expand All @@ -382,7 +382,7 @@ func convertDataSpecToOpenAPISpec(dataSpec *structpb.Struct) (*structpb.Struct,
if err != nil {
return nil, err
}
converted, err := convertDataSpecToOpenAPISpec(s)
converted, err := formatDataSpec(s)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -417,46 +417,25 @@ func convertDataSpecToOpenAPISpec(dataSpec *structpb.Struct) (*structpb.Struct,
return compSpec, nil
}

func (comp *Component) generateOpenAPISpecs(title string, availableTasks []string) (*structpb.Struct, error) {
func (comp *Component) generateDataSpecs(title string, availableTasks []string) (map[string]*pipelinePB.DataSpecification, error) {

openAPITemplates := &structpb.Struct{Fields: map[string]*structpb.Value{}}
for _, availableTask := range availableTasks {
openAPITemplate := &structpb.Struct{}

err := protojson.Unmarshal([]byte(OpenAPITemplate), openAPITemplate)
specs := map[string]*pipelinePB.DataSpecification{}
for _, task := range availableTasks {
spec := &pipelinePB.DataSpecification{}
var err error
taskJSONStruct := proto.Clone(comp.tasks[task]).(*structpb.Struct)
spec.Input, err = formatDataSpec(taskJSONStruct.Fields["input"].GetStructValue())
if err != nil {
return nil, err
}
openAPITemplate.Fields["info"].GetStructValue().Fields["title"] = structpb.NewStringValue(fmt.Sprintf("%s Component - %s", title, availableTask))

walk := openAPITemplate
for _, key := range []string{"paths", "/execute", "post", "requestBody", "content", "application/json", "schema", "properties", "inputs"} {
walk = walk.Fields[key].GetStructValue()
}

taskJSONStruct := proto.Clone(comp.tasks[availableTask]).(*structpb.Struct)

inputStruct, err := convertDataSpecToOpenAPISpec(taskJSONStruct.Fields["input"].GetStructValue())
spec.Output, err = formatDataSpec(taskJSONStruct.Fields["output"].GetStructValue())
if err != nil {
return nil, fmt.Errorf("task %s: %s error: %+v", title, availableTask, err)
}
walk.Fields["items"] = structpb.NewStructValue(inputStruct)

walk = openAPITemplate
for _, key := range []string{"paths", "/execute", "post", "responses", "200", "content", "application/json", "schema", "properties", "outputs"} {
walk = walk.Fields[key].GetStructValue()
}

outputStruct, err := convertDataSpecToOpenAPISpec(taskJSONStruct.Fields["output"].GetStructValue())
if err != nil {
return nil, fmt.Errorf("task %s: %s error: %+v", title, availableTask, err)
return nil, err
}
walk.Fields["items"] = structpb.NewStructValue(outputStruct)

openAPITemplates.Fields[availableTask] = structpb.NewStructValue(openAPITemplate)
specs[task] = spec
}

return openAPITemplates, nil
return specs, nil
}

func (comp *Component) loadTasks(tasksJSONBytes []byte) error {
Expand Down
10 changes: 5 additions & 5 deletions pkg/base/connector.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,9 @@ type IConnector interface {
// Add definition
AddConnectorDefinition(def *pipelinePB.ConnectorDefinition) error
// Get the connector definition by definition uid
GetConnectorDefinitionByUID(defUID uuid.UUID, resourceConfig *structpb.Struct, componentConfig *structpb.Struct) (*pipelinePB.ConnectorDefinition, error)
GetConnectorDefinitionByUID(defUID uuid.UUID, resourceConfig *structpb.Struct, component *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error)
// Get the connector definition by definition id
GetConnectorDefinitionByID(defID string, resourceConfig *structpb.Struct, componentConfig *structpb.Struct) (*pipelinePB.ConnectorDefinition, error)
GetConnectorDefinitionByID(defID string, resourceConfig *structpb.Struct, component *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error)
// Get the list of connector definitions under this connector
ListConnectorDefinitions() []*pipelinePB.ConnectorDefinition

Expand Down Expand Up @@ -93,7 +93,7 @@ func (c *Connector) LoadConnectorDefinitions(definitionsJSONBytes []byte, tasksJ
return err
}

def.Spec.OpenapiSpecifications, err = c.generateOpenAPISpecs(def.Title, availableTasks)
def.Spec.DataSpecifications, err = c.generateDataSpecs(def.Title, availableTasks)
if err != nil {
return err
}
Expand Down Expand Up @@ -188,7 +188,7 @@ func (c *Connector) ListConnectorDefinitions() []*pipelinePB.ConnectorDefinition
}

// GetConnectorDefinitionByUID gets the connector definition by definition uid
func (c *Connector) GetConnectorDefinitionByUID(defUID uuid.UUID, _ /*resourceConfig*/ *structpb.Struct, _ /* componentConfig */ *structpb.Struct) (*pipelinePB.ConnectorDefinition, error) {
func (c *Connector) GetConnectorDefinitionByUID(defUID uuid.UUID, _ /*resourceConfig */ *structpb.Struct, _ /*component*/ *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error) {
def, err := c.Component.getDefinitionByUID(defUID)
if err != nil {
return nil, err
Expand All @@ -203,7 +203,7 @@ func (c *Connector) GetConnectorDefinitionByUID(defUID uuid.UUID, _ /*resourceCo
}

// GetConnectorDefinitionByID gets the connector definition by definition id
func (c *Connector) GetConnectorDefinitionByID(defID string, resourceConfig *structpb.Struct, componentConfig *structpb.Struct) (*pipelinePB.ConnectorDefinition, error) {
func (c *Connector) GetConnectorDefinitionByID(defID string, _ /*resourceConfig*/ *structpb.Struct, _ /*component*/ *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error) {
def, err := c.Component.getDefinitionByID(defID)
if err != nil {
return nil, err
Expand Down
11 changes: 5 additions & 6 deletions pkg/base/operator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import (

"github.com/gofrs/uuid"
"google.golang.org/protobuf/encoding/protojson"
"google.golang.org/protobuf/types/known/structpb"

pipelinePB "github.com/instill-ai/protogen-go/vdp/pipeline/v1beta"
)
Expand All @@ -21,9 +20,9 @@ type IOperator interface {
// Add definition
AddOperatorDefinition(def *pipelinePB.OperatorDefinition) error
// Get the operator definition by definition uid
GetOperatorDefinitionByUID(defUID uuid.UUID, componentConfig *structpb.Struct) (*pipelinePB.OperatorDefinition, error)
GetOperatorDefinitionByUID(defUID uuid.UUID, component *pipelinePB.OperatorComponent) (*pipelinePB.OperatorDefinition, error)
// Get the operator definition by definition id
GetOperatorDefinitionByID(defID string, componentConfig *structpb.Struct) (*pipelinePB.OperatorDefinition, error)
GetOperatorDefinitionByID(defID string, component *pipelinePB.OperatorComponent) (*pipelinePB.OperatorDefinition, error)
// Get the list of operator definitions under this operator
ListOperatorDefinitions() []*pipelinePB.OperatorDefinition
}
Expand Down Expand Up @@ -74,7 +73,7 @@ func (o *Operator) LoadOperatorDefinitions(definitionsJSONBytes []byte, tasksJSO
return err
}

def.Spec.OpenapiSpecifications, err = o.generateOpenAPISpecs(def.Title, availableTasks)
def.Spec.DataSpecifications, err = o.generateDataSpecs(def.Title, availableTasks)
if err != nil {
return err
}
Expand Down Expand Up @@ -113,7 +112,7 @@ func (o *Operator) ListOperatorDefinitions() []*pipelinePB.OperatorDefinition {
}

// GetOperatorDefinitionByUID returns the operator definition by definition uid
func (o *Operator) GetOperatorDefinitionByUID(defUID uuid.UUID, _ /*componentConfig*/ *structpb.Struct) (*pipelinePB.OperatorDefinition, error) {
func (o *Operator) GetOperatorDefinitionByUID(defUID uuid.UUID, _ /*component*/ *pipelinePB.OperatorComponent) (*pipelinePB.OperatorDefinition, error) {
def, err := o.Component.getDefinitionByUID(defUID)
if err != nil {
return nil, err
Expand All @@ -128,7 +127,7 @@ func (o *Operator) GetOperatorDefinitionByUID(defUID uuid.UUID, _ /*componentCon
}

// GetOperatorDefinitionByID returns the operator definition by definition id
func (o *Operator) GetOperatorDefinitionByID(defID string, componentConfig *structpb.Struct) (*pipelinePB.OperatorDefinition, error) {
func (o *Operator) GetOperatorDefinitionByID(defID string, _ /*component*/ *pipelinePB.OperatorComponent) (*pipelinePB.OperatorDefinition, error) {
def, err := o.Component.getDefinitionByID(defID)
if err != nil {
return nil, err
Expand Down
147 changes: 52 additions & 95 deletions pkg/base/testdata/wantConnectorDefinition.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,105 +132,62 @@
"title": "OpenAI Component",
"type": "object"
},
"openapi_specifications": {
"data_specifications": {
"TASK_TEXT_EMBEDDINGS": {
"info": {
"title": "OpenAI Component - TASK_TEXT_EMBEDDINGS",
"version": "1.0.0"
"input": {
"instillEditOnNodeFields": [
"text",
"model"
],
"instillUIOrder": 0,
"properties": {
"model": {
"description": "ID of the model to use",
"instillShortDescription": "ID of the model to use",
"instillUIOrder": 0,
"title": "Model",
"type": "string"
},
"text": {
"description": "The text",
"instillShortDescription": "The text",
"instillUIOrder": 1,
"title": "Text",
"type": "string"
}
},
"required": [
"text",
"model"
],
"title": "Input",
"type": "object"
},
"openapi": "3.0.0",
"paths": {
"/execute": {
"post": {
"requestBody": {
"content": {
"application/json": {
"schema": {
"properties": {
"inputs": {
"items": {
"instillEditOnNodeFields": [
"text",
"model"
],
"instillUIOrder": 0,
"properties": {
"model": {
"description": "ID of the model to use",
"instillShortDescription": "ID of the model to use",
"instillUIOrder": 0,
"title": "Model",
"type": "string"
},
"text": {
"description": "The text",
"instillShortDescription": "The text",
"instillUIOrder": 1,
"title": "Text",
"type": "string"
}
},
"required": [
"text",
"model"
],
"title": "Input",
"type": "object"
},
"type": "array"
}
},
"type": "object"
}
}
},
"required": true
"output": {
"instillEditOnNodeFields": [
"embedding"
],
"instillUIOrder": 0,
"properties": {
"embedding": {
"instillUIOrder": 0,
"items": {
"description": "",
"instillFormat": "number",
"instillShortDescription": "",
"instillUIOrder": 0,
"title": "Embedding",
"type": "number"
},
"responses": {
"200": {
"content": {
"application/json": {
"schema": {
"properties": {
"outputs": {
"items": {
"instillEditOnNodeFields": [
"embedding"
],
"instillUIOrder": 0,
"properties": {
"embedding": {
"instillUIOrder": 0,
"items": {
"description": "",
"instillFormat": "number",
"instillShortDescription": "",
"instillUIOrder": 0,
"title": "Embedding",
"type": "number"
},
"title": "Embedding",
"type": "array"
}
},
"required": [
"embedding"
],
"title": "Output",
"type": "object"
},
"type": "array"
}
},
"type": "object"
}
}
},
"description": ""
}
}
"title": "Embedding",
"type": "array"
}
}
},
"required": [
"embedding"
],
"title": "Output",
"type": "object"
}
}
}
Expand Down
Loading

0 comments on commit 7c27d15

Please sign in to comment.