Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: adjust IConnector interface #83

Merged
merged 1 commit into from
Apr 15, 2024
Merged

feat: adjust IConnector interface #83

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ require (
github.com/gocolly/colly/v2 v2.1.0
github.com/gofrs/uuid v4.4.0+incompatible
github.com/h2non/filetype v1.1.3
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240415024335-3b7581a069fd
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240415145600-50085798159d
github.com/instill-ai/x v0.4.0-alpha
github.com/itchyny/gojq v0.12.14
github.com/lestrrat-go/jspointer v0.0.0-20181205001929-82fadba7561c
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -179,8 +179,8 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 h1:Wqo399gCIufwto+VfwCSvsnfGpF
github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0/go.mod h1:qmOFXW2epJhM0qSnUUYpldc7gVz2KMQwJ/QYCDIa7XU=
github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=
github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240415024335-3b7581a069fd h1:Q7Zm2Rc3Fg+r/6WQtwLY0nyoCCiSy6kJ4W8SQT/WQeQ=
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240415024335-3b7581a069fd/go.mod h1:jhEL0SauySMoPLVvx105DWyThju9sYTbsXIySVCArmM=
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240415145600-50085798159d h1:o8vynHQD3PG0b9VU6sw/9RUfOotxEcFnl9AjIhnPTWA=
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240415145600-50085798159d/go.mod h1:jhEL0SauySMoPLVvx105DWyThju9sYTbsXIySVCArmM=
github.com/instill-ai/x v0.4.0-alpha h1:zQV2VLbSHjMv6gyBN/2mwwrvWk0/mJM6ZKS12AzjfQg=
github.com/instill-ai/x v0.4.0-alpha/go.mod h1:L6jmDPrUou6XskaLXZuK/gDeitdoPa9yE8ONKt1ZwCw=
github.com/itchyny/gojq v0.12.14 h1:6k8vVtsrhQSYgSGg827AD+PVVaB1NLXEdX+dda2oZCc=
Expand Down
8 changes: 4 additions & 4 deletions pkg/base/connector.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,9 @@ type IConnector interface {
// Add definition
AddConnectorDefinition(def *pipelinePB.ConnectorDefinition) error
// Get the connector definition by definition uid
GetConnectorDefinitionByUID(defUID uuid.UUID, resourceConfig *structpb.Struct, component *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error)
GetConnectorDefinitionByUID(defUID uuid.UUID, component *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error)
// Get the connector definition by definition id
GetConnectorDefinitionByID(defID string, resourceConfig *structpb.Struct, component *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error)
GetConnectorDefinitionByID(defID string, component *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error)
// Get the list of connector definitions under this connector
ListConnectorDefinitions() []*pipelinePB.ConnectorDefinition

Expand Down Expand Up @@ -182,7 +182,7 @@ func (c *Connector) ListConnectorDefinitions() []*pipelinePB.ConnectorDefinition
}

// GetConnectorDefinitionByUID gets the connector definition by definition uid
func (c *Connector) GetConnectorDefinitionByUID(defUID uuid.UUID, _ /*resourceConfig */ *structpb.Struct, _ /*component*/ *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error) {
func (c *Connector) GetConnectorDefinitionByUID(defUID uuid.UUID, _ /*component*/ *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error) {
def, err := c.Component.getDefinitionByUID(defUID)
if err != nil {
return nil, err
Expand All @@ -197,7 +197,7 @@ func (c *Connector) GetConnectorDefinitionByUID(defUID uuid.UUID, _ /*resourceCo
}

// GetConnectorDefinitionByID gets the connector definition by definition id
func (c *Connector) GetConnectorDefinitionByID(defID string, _ /*resourceConfig*/ *structpb.Struct, _ /*component*/ *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error) {
func (c *Connector) GetConnectorDefinitionByID(defID string, _ /*component*/ *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error) {
def, err := c.Component.getDefinitionByID(defID)
if err != nil {
return nil, err
Expand Down
12 changes: 6 additions & 6 deletions pkg/connector/airbyte/v0/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,13 +87,13 @@ func Init(logger *zap.Logger, options ConnectorOptions) base.IConnector {
}

if options.ExcludeLocalConnector {
def, _ := connector.GetConnectorDefinitionByID("airbyte-destination-local-json", nil, nil)
def, _ := connector.GetConnectorDefinitionByID("airbyte-destination-local-json", nil)
(*def).Tombstone = true
def, _ = connector.GetConnectorDefinitionByID("airbyte-destination-csv", nil, nil)
def, _ = connector.GetConnectorDefinitionByID("airbyte-destination-csv", nil)
(*def).Tombstone = true
def, _ = connector.GetConnectorDefinitionByID("airbyte-destination-sqlite", nil, nil)
def, _ = connector.GetConnectorDefinitionByID("airbyte-destination-sqlite", nil)
(*def).Tombstone = true
def, _ = connector.GetConnectorDefinitionByID("airbyte-destination-duckdb", nil, nil)
def, _ = connector.GetConnectorDefinitionByID("airbyte-destination-duckdb", nil)
(*def).Tombstone = true
}

Expand Down Expand Up @@ -157,7 +157,7 @@ func (e *Execution) Execute(inputs []*structpb.Struct) ([]*structpb.Struct, erro
// Remove the last "\n"
byteAbMsgs = byteAbMsgs[:len(byteAbMsgs)-1]

connDef, err := e.connector.GetConnectorDefinitionByUID(e.UID, nil, nil)
connDef, err := e.connector.GetConnectorDefinitionByUID(e.UID, nil)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -328,7 +328,7 @@ func (e *Execution) Execute(inputs []*structpb.Struct) ([]*structpb.Struct, erro

func (c *Connector) Test(defUID uuid.UUID, config *structpb.Struct, logger *zap.Logger) error {

def, err := c.GetConnectorDefinitionByUID(defUID, nil, nil)
def, err := c.GetConnectorDefinitionByUID(defUID, nil)
if err != nil {
return err
}
Expand Down
18 changes: 9 additions & 9 deletions pkg/connector/instill/v0/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -199,13 +199,13 @@ func (c *Connector) Test(_ uuid.UUID, config *structpb.Struct, logger *zap.Logge
return nil
}

func (c *Connector) GetConnectorDefinitionByID(defID string, resourceConfig *structpb.Struct, component *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error) {
def, err := c.Connector.GetConnectorDefinitionByID(defID, resourceConfig, component)
func (c *Connector) GetConnectorDefinitionByID(defID string, component *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error) {
def, err := c.Connector.GetConnectorDefinitionByID(defID, component)
if err != nil {
return nil, err
}

return c.GetConnectorDefinitionByUID(uuid.FromStringOrNil(def.Uid), resourceConfig, component)
return c.GetConnectorDefinitionByUID(uuid.FromStringOrNil(def.Uid), component)
}

type ModelsResp struct {
Expand All @@ -216,23 +216,23 @@ type ModelsResp struct {
}

// Generate the model_name enum based on the task
func (c *Connector) GetConnectorDefinitionByUID(defUID uuid.UUID, resourceConfig *structpb.Struct, component *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error) {
oriDef, err := c.Connector.GetConnectorDefinitionByUID(defUID, resourceConfig, component)
func (c *Connector) GetConnectorDefinitionByUID(defUID uuid.UUID, component *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error) {
oriDef, err := c.Connector.GetConnectorDefinitionByUID(defUID, component)
if err != nil {
return nil, err
}
def := proto.Clone(oriDef).(*pipelinePB.ConnectorDefinition)

if resourceConfig != nil {
if getModelServerURL(resourceConfig) == "" {
if component.Connection != nil {
if getModelServerURL(component.Connection) == "" {
return def, nil
}

gRPCCLient, gRPCCLientConn := initModelPublicServiceClient(getModelServerURL(resourceConfig))
gRPCCLient, gRPCCLientConn := initModelPublicServiceClient(getModelServerURL(component.Connection))
if gRPCCLientConn != nil {
defer gRPCCLientConn.Close()
}
ctx := metadata.NewOutgoingContext(context.Background(), getRequestMetadata(resourceConfig))
ctx := metadata.NewOutgoingContext(context.Background(), getRequestMetadata(component.Connection))
// We should query by pages and accumulate them in the future

pageToken := ""
Expand Down
2 changes: 1 addition & 1 deletion pkg/connector/integration_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ func TestOpenAITextGeneration(t *testing.T) {
logger := zap.NewNop()
conn := Init(logger, emptyOptions)

def, err := conn.GetConnectorDefinitionByID("openai", nil, nil)
def, err := conn.GetConnectorDefinitionByID("openai", nil)
c.Assert(err, qt.IsNil)

uid, err := uuid.FromString(def.GetUid())
Expand Down
8 changes: 4 additions & 4 deletions pkg/connector/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,10 +85,10 @@ func (c *Connector) Test(defUID uuid.UUID, config *structpb.Struct, logger *zap.
return c.connectorUIDMap[defUID].Test(defUID, config, logger)
}

func (c *Connector) GetConnectorDefinitionByID(defID string, resourceConfig *structpb.Struct, component *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error) {
return c.connectorIDMap[defID].GetConnectorDefinitionByID(defID, resourceConfig, component)
func (c *Connector) GetConnectorDefinitionByID(defID string, component *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error) {
return c.connectorIDMap[defID].GetConnectorDefinitionByID(defID, component)
}

func (c *Connector) GetConnectorDefinitionByUID(defUID uuid.UUID, resourceConfig *structpb.Struct, component *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error) {
return c.connectorUIDMap[defUID].GetConnectorDefinitionByUID(defUID, resourceConfig, component)
func (c *Connector) GetConnectorDefinitionByUID(defUID uuid.UUID, component *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error) {
return c.connectorUIDMap[defUID].GetConnectorDefinitionByUID(defUID, component)
}
10 changes: 5 additions & 5 deletions pkg/connector/restapi/v0/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,18 +170,18 @@ func (c *Connector) Test(defUID uuid.UUID, config *structpb.Struct, logger *zap.
return nil
}

func (c *Connector) GetConnectorDefinitionByID(defID string, resourceConfig *structpb.Struct, component *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error) {
def, err := c.Connector.GetConnectorDefinitionByID(defID, resourceConfig, component)
func (c *Connector) GetConnectorDefinitionByID(defID string, component *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error) {
def, err := c.Connector.GetConnectorDefinitionByID(defID, component)
if err != nil {
return nil, err
}

return c.GetConnectorDefinitionByUID(uuid.FromStringOrNil(def.Uid), resourceConfig, component)
return c.GetConnectorDefinitionByUID(uuid.FromStringOrNil(def.Uid), component)
}

// Generate the model_name enum based on the task
func (c *Connector) GetConnectorDefinitionByUID(defUID uuid.UUID, resourceConfig *structpb.Struct, component *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error) {
oriDef, err := c.Connector.GetConnectorDefinitionByUID(defUID, resourceConfig, component)
func (c *Connector) GetConnectorDefinitionByUID(defUID uuid.UUID, component *pipelinePB.ConnectorComponent) (*pipelinePB.ConnectorDefinition, error) {
oriDef, err := c.Connector.GetConnectorDefinitionByUID(defUID, component)
if err != nil {
return nil, err
}
Expand Down
Loading