diff --git a/go.mod b/go.mod index fbaf9474..2175b519 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/gocolly/colly/v2 v2.1.0 github.com/gofrs/uuid v4.4.0+incompatible github.com/h2non/filetype v1.1.3 - github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240415024335-3b7581a069fd + github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240415145600-50085798159d github.com/instill-ai/x v0.4.0-alpha github.com/itchyny/gojq v0.12.14 github.com/lestrrat-go/jspointer v0.0.0-20181205001929-82fadba7561c diff --git a/go.sum b/go.sum index 241fef51..fc660b80 100644 --- a/go.sum +++ b/go.sum @@ -179,8 +179,8 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 h1:Wqo399gCIufwto+VfwCSvsnfGpF github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0/go.mod h1:qmOFXW2epJhM0qSnUUYpldc7gVz2KMQwJ/QYCDIa7XU= github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= -github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240415024335-3b7581a069fd h1:Q7Zm2Rc3Fg+r/6WQtwLY0nyoCCiSy6kJ4W8SQT/WQeQ= -github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240415024335-3b7581a069fd/go.mod h1:jhEL0SauySMoPLVvx105DWyThju9sYTbsXIySVCArmM= +github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240415145600-50085798159d h1:o8vynHQD3PG0b9VU6sw/9RUfOotxEcFnl9AjIhnPTWA= +github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240415145600-50085798159d/go.mod h1:jhEL0SauySMoPLVvx105DWyThju9sYTbsXIySVCArmM= github.com/instill-ai/x v0.4.0-alpha h1:zQV2VLbSHjMv6gyBN/2mwwrvWk0/mJM6ZKS12AzjfQg= github.com/instill-ai/x v0.4.0-alpha/go.mod h1:L6jmDPrUou6XskaLXZuK/gDeitdoPa9yE8ONKt1ZwCw= github.com/itchyny/gojq v0.12.14 h1:6k8vVtsrhQSYgSGg827AD+PVVaB1NLXEdX+dda2oZCc= diff --git a/pkg/base/connector.go b/pkg/base/connector.go index f68f67d7..4d53db97 100644 --- a/pkg/base/connector.go +++ b/pkg/base/connector.go @@ -27,9 +27,9 @@ type IConnector interface { // Add definition AddConnectorDefinition(def *pipelinePB.ConnectorDefinition) error // Get the connector definition by definition uid - GetConnectorDefinitionByUID(defUID uuid.UUID, resourceConfig *structpb.Struct, component *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error) + GetConnectorDefinitionByUID(defUID uuid.UUID, component *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error) // Get the connector definition by definition id - GetConnectorDefinitionByID(defID string, resourceConfig *structpb.Struct, component *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error) + GetConnectorDefinitionByID(defID string, component *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error) // Get the list of connector definitions under this connector ListConnectorDefinitions() []*pipelinePB.ConnectorDefinition @@ -182,7 +182,7 @@ func (c *Connector) ListConnectorDefinitions() []*pipelinePB.ConnectorDefinition } // GetConnectorDefinitionByUID gets the connector definition by definition uid -func (c *Connector) GetConnectorDefinitionByUID(defUID uuid.UUID, _ /*resourceConfig */ *structpb.Struct, _ /*component*/ *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error) { +func (c *Connector) GetConnectorDefinitionByUID(defUID uuid.UUID, _ /*component*/ *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error) { def, err := c.Component.getDefinitionByUID(defUID) if err != nil { return nil, err @@ -197,7 +197,7 @@ func (c *Connector) GetConnectorDefinitionByUID(defUID uuid.UUID, _ /*resourceCo } // GetConnectorDefinitionByID gets the connector definition by definition id -func (c *Connector) GetConnectorDefinitionByID(defID string, _ /*resourceConfig*/ *structpb.Struct, _ /*component*/ *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error) { +func (c *Connector) GetConnectorDefinitionByID(defID string, _ /*component*/ *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error) { def, err := c.Component.getDefinitionByID(defID) if err != nil { return nil, err diff --git a/pkg/connector/airbyte/v0/main.go b/pkg/connector/airbyte/v0/main.go index 398ac676..b1242f4d 100644 --- a/pkg/connector/airbyte/v0/main.go +++ b/pkg/connector/airbyte/v0/main.go @@ -87,13 +87,13 @@ func Init(logger *zap.Logger, options ConnectorOptions) base.IConnector { } if options.ExcludeLocalConnector { - def, _ := connector.GetConnectorDefinitionByID("airbyte-destination-local-json", nil, nil) + def, _ := connector.GetConnectorDefinitionByID("airbyte-destination-local-json", nil) (*def).Tombstone = true - def, _ = connector.GetConnectorDefinitionByID("airbyte-destination-csv", nil, nil) + def, _ = connector.GetConnectorDefinitionByID("airbyte-destination-csv", nil) (*def).Tombstone = true - def, _ = connector.GetConnectorDefinitionByID("airbyte-destination-sqlite", nil, nil) + def, _ = connector.GetConnectorDefinitionByID("airbyte-destination-sqlite", nil) (*def).Tombstone = true - def, _ = connector.GetConnectorDefinitionByID("airbyte-destination-duckdb", nil, nil) + def, _ = connector.GetConnectorDefinitionByID("airbyte-destination-duckdb", nil) (*def).Tombstone = true } @@ -157,7 +157,7 @@ func (e *Execution) Execute(inputs []*structpb.Struct) ([]*structpb.Struct, erro // Remove the last "\n" byteAbMsgs = byteAbMsgs[:len(byteAbMsgs)-1] - connDef, err := e.connector.GetConnectorDefinitionByUID(e.UID, nil, nil) + connDef, err := e.connector.GetConnectorDefinitionByUID(e.UID, nil) if err != nil { return nil, err } @@ -328,7 +328,7 @@ func (e *Execution) Execute(inputs []*structpb.Struct) ([]*structpb.Struct, erro func (c *Connector) Test(defUID uuid.UUID, config *structpb.Struct, logger *zap.Logger) error { - def, err := c.GetConnectorDefinitionByUID(defUID, nil, nil) + def, err := c.GetConnectorDefinitionByUID(defUID, nil) if err != nil { return err } diff --git a/pkg/connector/instill/v0/main.go b/pkg/connector/instill/v0/main.go index ea0975c7..22171762 100644 --- a/pkg/connector/instill/v0/main.go +++ b/pkg/connector/instill/v0/main.go @@ -199,13 +199,13 @@ func (c *Connector) Test(_ uuid.UUID, config *structpb.Struct, logger *zap.Logge return nil } -func (c *Connector) GetConnectorDefinitionByID(defID string, resourceConfig *structpb.Struct, component *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error) { - def, err := c.Connector.GetConnectorDefinitionByID(defID, resourceConfig, component) +func (c *Connector) GetConnectorDefinitionByID(defID string, component *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error) { + def, err := c.Connector.GetConnectorDefinitionByID(defID, component) if err != nil { return nil, err } - return c.GetConnectorDefinitionByUID(uuid.FromStringOrNil(def.Uid), resourceConfig, component) + return c.GetConnectorDefinitionByUID(uuid.FromStringOrNil(def.Uid), component) } type ModelsResp struct { @@ -216,23 +216,23 @@ type ModelsResp struct { } // Generate the model_name enum based on the task -func (c *Connector) GetConnectorDefinitionByUID(defUID uuid.UUID, resourceConfig *structpb.Struct, component *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error) { - oriDef, err := c.Connector.GetConnectorDefinitionByUID(defUID, resourceConfig, component) +func (c *Connector) GetConnectorDefinitionByUID(defUID uuid.UUID, component *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error) { + oriDef, err := c.Connector.GetConnectorDefinitionByUID(defUID, component) if err != nil { return nil, err } def := proto.Clone(oriDef).(*pipelinePB.ConnectorDefinition) - if resourceConfig != nil { - if getModelServerURL(resourceConfig) == "" { + if component.Connection != nil { + if getModelServerURL(component.Connection) == "" { return def, nil } - gRPCCLient, gRPCCLientConn := initModelPublicServiceClient(getModelServerURL(resourceConfig)) + gRPCCLient, gRPCCLientConn := initModelPublicServiceClient(getModelServerURL(component.Connection)) if gRPCCLientConn != nil { defer gRPCCLientConn.Close() } - ctx := metadata.NewOutgoingContext(context.Background(), getRequestMetadata(resourceConfig)) + ctx := metadata.NewOutgoingContext(context.Background(), getRequestMetadata(component.Connection)) // We should query by pages and accumulate them in the future pageToken := "" diff --git a/pkg/connector/integration_test.go b/pkg/connector/integration_test.go index 9203588a..7f2d72e9 100644 --- a/pkg/connector/integration_test.go +++ b/pkg/connector/integration_test.go @@ -42,7 +42,7 @@ func TestOpenAITextGeneration(t *testing.T) { logger := zap.NewNop() conn := Init(logger, emptyOptions) - def, err := conn.GetConnectorDefinitionByID("openai", nil, nil) + def, err := conn.GetConnectorDefinitionByID("openai", nil) c.Assert(err, qt.IsNil) uid, err := uuid.FromString(def.GetUid()) diff --git a/pkg/connector/main.go b/pkg/connector/main.go index e8058212..bd81328b 100644 --- a/pkg/connector/main.go +++ b/pkg/connector/main.go @@ -85,10 +85,10 @@ func (c *Connector) Test(defUID uuid.UUID, config *structpb.Struct, logger *zap. return c.connectorUIDMap[defUID].Test(defUID, config, logger) } -func (c *Connector) GetConnectorDefinitionByID(defID string, resourceConfig *structpb.Struct, component *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error) { - return c.connectorIDMap[defID].GetConnectorDefinitionByID(defID, resourceConfig, component) +func (c *Connector) GetConnectorDefinitionByID(defID string, component *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error) { + return c.connectorIDMap[defID].GetConnectorDefinitionByID(defID, component) } -func (c *Connector) GetConnectorDefinitionByUID(defUID uuid.UUID, resourceConfig *structpb.Struct, component *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error) { - return c.connectorUIDMap[defUID].GetConnectorDefinitionByUID(defUID, resourceConfig, component) +func (c *Connector) GetConnectorDefinitionByUID(defUID uuid.UUID, component *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error) { + return c.connectorUIDMap[defUID].GetConnectorDefinitionByUID(defUID, component) } diff --git a/pkg/connector/restapi/v0/main.go b/pkg/connector/restapi/v0/main.go index 83ae3e6d..4bfc5bde 100644 --- a/pkg/connector/restapi/v0/main.go +++ b/pkg/connector/restapi/v0/main.go @@ -170,18 +170,18 @@ func (c *Connector) Test(defUID uuid.UUID, config *structpb.Struct, logger *zap. return nil } -func (c *Connector) GetConnectorDefinitionByID(defID string, resourceConfig *structpb.Struct, component *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error) { - def, err := c.Connector.GetConnectorDefinitionByID(defID, resourceConfig, component) +func (c *Connector) GetConnectorDefinitionByID(defID string, component *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error) { + def, err := c.Connector.GetConnectorDefinitionByID(defID, component) if err != nil { return nil, err } - return c.GetConnectorDefinitionByUID(uuid.FromStringOrNil(def.Uid), resourceConfig, component) + return c.GetConnectorDefinitionByUID(uuid.FromStringOrNil(def.Uid), component) } // Generate the model_name enum based on the task -func (c *Connector) GetConnectorDefinitionByUID(defUID uuid.UUID, resourceConfig *structpb.Struct, component *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error) { - oriDef, err := c.Connector.GetConnectorDefinitionByUID(defUID, resourceConfig, component) +func (c *Connector) GetConnectorDefinitionByUID(defUID uuid.UUID, component *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error) { + oriDef, err := c.Connector.GetConnectorDefinitionByUID(defUID, component) if err != nil { return nil, err }