feat: add additional attribute in JSON schema for Instill Credit (#118)

Because

- We need to introduce new attributes in the JSON schema to help Console
generate the corresponding Instill-Credit-supported models list.

This commit

- refactor: rename `instillCredentialField` to `instillSecret`
- feat(openai): add additional attribute in JSON schema to indicate
which models support `instillSecret`
- feat: update `instillEditOnNodeFields` for all connectors

pkg/base/connector.go

@@ -26,7 +26,7 @@ type IConnector interface {
 	CreateExecution(sysVars map[string]any, connection *structpb.Struct, task string) (*ExecutionWrapper, error)
 	Test(sysVars map[string]any, connection *structpb.Struct) error
 
-	IsCredentialField(target string) bool
+	IsSecretField(target string) bool
 }
 
 // Connector implements the common connector methods.
@@ -36,8 +36,8 @@ type Connector struct {
 	taskInputSchemas  map[string]string
 	taskOutputSchemas map[string]string
 
-	definition       *pipelinePB.ConnectorDefinition
-	credentialFields []string
+	definition   *pipelinePB.ConnectorDefinition
+	secretFields []string
 }
 
 func (c *Connector) GetID() string {
@@ -67,7 +67,7 @@ func (c *Connector) LoadConnectorDefinition(definitionJSONBytes []byte, tasksJSO
 	var err error
 	var definitionJSON any
 
-	c.credentialFields = []string{}
+	c.secretFields = []string{}
 
 	err = json.Unmarshal(definitionJSONBytes, &definitionJSON)
 	if err != nil {
@@ -144,7 +144,7 @@ func (c *Connector) LoadConnectorDefinition(definitionJSONBytes []byte, tasksJSO
 		return err
 	}
 
-	c.initCredentialField(c.definition)
+	c.initSecretField(c.definition)
 
 	return nil
 
@@ -204,57 +204,57 @@ func (c *Connector) refineResourceSpec(resourceSpec *structpb.Struct) (*structpb
 	return spec, nil
 }
 
-// IsCredentialField checks if the target field is credential field
-func (c *Connector) IsCredentialField(target string) bool {
-	for _, field := range c.credentialFields {
+// IsSecretField checks if the target field is secret field
+func (c *Connector) IsSecretField(target string) bool {
+	for _, field := range c.secretFields {
 		if target == field {
 			return true
 		}
 	}
 	return false
 }
 
-// ListCredentialField lists the credential fields by definition id
-func (c *Connector) ListCredentialField() ([]string, error) {
-	return c.credentialFields, nil
+// ListSecretFields lists the secret fields by definition id
+func (c *Connector) ListSecretFields() ([]string, error) {
+	return c.secretFields, nil
 }
 
-func (c *Connector) initCredentialField(def *pipelinePB.ConnectorDefinition) {
-	if c.credentialFields == nil {
-		c.credentialFields = []string{}
+func (c *Connector) initSecretField(def *pipelinePB.ConnectorDefinition) {
+	if c.secretFields == nil {
+		c.secretFields = []string{}
 	}
-	credentialFields := []string{}
+	secretFields := []string{}
 	connection := def.Spec.GetComponentSpecification().GetFields()["properties"].GetStructValue().GetFields()["connection"].GetStructValue()
-	credentialFields = c.traverseCredentialField(connection.GetFields()["properties"], "", credentialFields)
+	secretFields = c.traverseSecretField(connection.GetFields()["properties"], "", secretFields)
 	if l, ok := connection.GetFields()["oneOf"]; ok {
 		for _, v := range l.GetListValue().Values {
-			credentialFields = c.traverseCredentialField(v.GetStructValue().GetFields()["properties"], "", credentialFields)
+			secretFields = c.traverseSecretField(v.GetStructValue().GetFields()["properties"], "", secretFields)
 		}
 	}
-	c.credentialFields = credentialFields
+	c.secretFields = secretFields
 }
 
-func (c *Connector) traverseCredentialField(input *structpb.Value, prefix string, credentialFields []string) []string {
+func (c *Connector) traverseSecretField(input *structpb.Value, prefix string, secretFields []string) []string {
 	for key, v := range input.GetStructValue().GetFields() {
-		if isCredential, ok := v.GetStructValue().GetFields()["instillCredentialField"]; ok {
-			if isCredential.GetBoolValue() || isCredential.GetStringValue() == "true" {
-				credentialFields = append(credentialFields, fmt.Sprintf("%s%s", prefix, key))
+		if isSecret, ok := v.GetStructValue().GetFields()["instillSecret"]; ok {
+			if isSecret.GetBoolValue() || isSecret.GetStringValue() == "true" {
+				secretFields = append(secretFields, fmt.Sprintf("%s%s", prefix, key))
 			}
 		}
 		if tp, ok := v.GetStructValue().GetFields()["type"]; ok {
 			if tp.GetStringValue() == "object" {
 				if l, ok := v.GetStructValue().GetFields()["oneOf"]; ok {
 					for _, v := range l.GetListValue().Values {
-						credentialFields = c.traverseCredentialField(v.GetStructValue().GetFields()["properties"], fmt.Sprintf("%s%s.", prefix, key), credentialFields)
+						secretFields = c.traverseSecretField(v.GetStructValue().GetFields()["properties"], fmt.Sprintf("%s%s.", prefix, key), secretFields)
 					}
 				}
-				credentialFields = c.traverseCredentialField(v.GetStructValue().GetFields()["properties"], fmt.Sprintf("%s%s.", prefix, key), credentialFields)
+				secretFields = c.traverseSecretField(v.GetStructValue().GetFields()["properties"], fmt.Sprintf("%s%s.", prefix, key), secretFields)
 			}
 
 		}
 	}
 
-	return credentialFields
+	return secretFields
 }
 
 // UsageHandlerCreator returns a function to initialize a UsageHandler.

pkg/base/testdata/connectorDef.json

@@ -16,7 +16,7 @@
       "properties": {
         "api_key": {
           "description": "Fill your OpenAI API key. To find your keys, visit your OpenAI's API Keys page.",
-          "instillCredentialField": true,
+          "instillSecret": true,
           "instillUIOrder": 0,
           "title": "API Key",
           "type": "string"

pkg/base/testdata/wantConnectorDefinition.json

@@ -117,7 +117,7 @@
           "properties": {
             "api_key": {
               "description": "Fill your OpenAI API key. To find your keys, visit your OpenAI's API Keys page.",
-              "instillCredentialField": true,
+              "instillSecret": true,
               "instillShortDescription": "Fill your OpenAI API key. To find your keys, visit your OpenAI's API Keys page.",
               "instillUIOrder": 0,
               "title": "API Key",

pkg/connector/archetypeai/v0/config/definition.json

@@ -23,7 +23,7 @@
           "instillAcceptFormats": [
             "string"
           ],
-          "instillCredentialField": true,
+          "instillSecret": true,
           "instillUIOrder": 0,
           "title": "API Key",
           "type": "string"
@@ -32,6 +32,9 @@
       "required": [
         "api_key"
       ],
+      "instillEditOnNodeFields": [
+        "api_key"
+      ],
       "title": "Archetype AI Connection",
       "type": "object"
     }

pkg/connector/bigquery/v0/config/definition.json

@@ -33,7 +33,7 @@
           "instillAcceptFormats": [
             "string"
           ],
-          "instillCredentialField": true,
+          "instillSecret": true,
           "instillUIOrder": 0,
           "instillUIMultiline": true,
           "title": "JSON Key File contents",
@@ -70,6 +70,12 @@
         "dataset_id",
         "table_name"
       ],
+      "instillEditOnNodeFields": [
+        "json_key",
+        "project_id",
+        "dataset_id",
+        "table_name"
+      ],
       "title": "BigQuery Connection",
       "type": "object"
     }

pkg/connector/googlecloudstorage/v0/config/definition.json

@@ -21,7 +21,7 @@
           "instillAcceptFormats": [
             "string"
           ],
-          "instillCredentialField": false,
+          "instillSecret": false,
           "instillUIOrder": 0,
           "title": "Bucket Name",
           "type": "string"
@@ -34,7 +34,7 @@
           "instillAcceptFormats": [
             "string"
           ],
-          "instillCredentialField": true,
+          "instillSecret": true,
           "instillUIOrder": 1,
           "instillUIMultiline": true,
           "title": "JSON Key File contents",
@@ -45,6 +45,10 @@
         "json_key",
         "bucket_name"
       ],
+      "instillEditOnNodeFields": [
+        "json_key",
+        "bucket_name"
+      ],
       "title": "Google Cloud Storage Connection",
       "type": "object"
     }

pkg/connector/googlesearch/v0/config/definition.json

@@ -18,7 +18,7 @@
           "instillUpstreamTypes": [
             "reference"
           ],
-          "instillCredentialField": true,
+          "instillSecret": true,
           "instillUIOrder": 0,
           "title": "API Key",
           "type": "string"
@@ -28,7 +28,7 @@
           "instillUpstreamTypes": [
             "value"
           ],
-          "instillCredentialField": false,
+          "instillSecret": false,
           "instillUIOrder": 1,
           "title": "Search Engine ID",
           "type": "string"
@@ -38,6 +38,10 @@
         "api_key",
         "cse_id"
       ],
+      "instillEditOnNodeFields": [
+        "api_key",
+        "cse_id"
+      ],
       "title": "Google Search Connection",
       "type": "object"
     }

pkg/connector/huggingface/v0/config/definition.json

@@ -37,7 +37,7 @@
           "instillAcceptFormats": [
             "string"
           ],
-          "instillCredentialField": true,
+          "instillSecret": true,
           "instillUIOrder": 0,
           "title": "API Key",
           "type": "string"
@@ -51,7 +51,7 @@
           "instillAcceptFormats": [
             "string"
           ],
-          "instillCredentialField": false,
+          "instillSecret": false,
           "instillUIOrder": 1,
           "title": "Base URL",
           "type": "string"
@@ -65,7 +65,7 @@
           "instillAcceptFormats": [
             "boolean"
           ],
-          "instillCredentialField": false,
+          "instillSecret": false,
           "instillUIOrder": 2,
           "title": "Is Custom Endpoint",
           "type": "boolean"
@@ -76,6 +76,11 @@
         "base_url",
         "is_custom_endpoint"
       ],
+      "instillEditOnNodeFields": [
+        "api_key",
+        "base_url",
+        "is_custom_endpoint"
+      ],
       "title": "Hugging Face Connection",
       "type": "object"
     }

pkg/connector/main.go

@@ -140,11 +140,9 @@ func (cs *Store) ListConnectorDefinitions(sysVars map[string]any, returnTombston
 	return defs
 }
 
-// IsCredentialField returns whether a given field in a connector contains
-// credentials.
-func (cs *Store) IsCredentialField(defUID uuid.UUID, target string) (bool, error) {
+func (cs *Store) IsSecretField(defUID uuid.UUID, target string) (bool, error) {
 	if con, ok := cs.connectorUIDMap[defUID]; ok {
-		return con.con.IsCredentialField(target), nil
+		return con.con.IsSecretField(target), nil
 	}
 	return false, fmt.Errorf("connector definition not found")
 }

pkg/connector/numbers/v0/config/definition.json

@@ -21,7 +21,7 @@
           "instillAcceptFormats": [
             "string"
           ],
-          "instillCredentialField": true,
+          "instillSecret": true,
           "instillUIOrder": 0,
           "title": "Capture token",
           "type": "string"
@@ -30,6 +30,9 @@
       "required": [
         "capture_token"
       ],
+      "instillEditOnNodeFields": [
+        "capture_token"
+      ],
       "title": "Numbers Protocol Connection",
       "type": "object"
     }

pkg/connector/openai/v0/config/definition.json

@@ -25,7 +25,8 @@
           "instillAcceptFormats": [
             "string"
           ],
-          "instillCredentialField": true,
+          "instillSecret": true,
+          "instillCredential": true,
           "instillUIOrder": 0,
           "title": "API Key",
           "type": "string"
@@ -46,6 +47,9 @@
       "required": [
         "api_key"
       ],
+      "instillEditOnNodeFields": [
+        "api_key"
+      ],
       "title": "OpenAI Connection",
       "type": "object"
     }

pkg/connector/openai/v0/config/tasks.json

@@ -260,6 +260,24 @@
             "reference",
             "template"
           ],
+          "instillCredentialMap": {
+            "values": [
+              "gpt-4-0125-preview",
+              "gpt-4-turbo-preview",
+              "gpt-4-1106-preview",
+              "gpt-4-vision-preview",
+              "gpt-4",
+              "gpt-4-0314",
+              "gpt-4-0613",
+              "gpt-4-32k",
+              "gpt-4-32k-0314",
+              "gpt-4-32k-0613",
+              "gpt-3.5-turbo"
+            ],
+            "targets": [
+              "connection.api_key"
+            ]
+          },
           "title": "Model"
         },
         "n": {

pkg/connector/pinecone/v0/config/definition.json

@@ -22,7 +22,7 @@
           "instillAcceptFormats": [
             "string"
           ],
-          "instillCredentialField": true,
+          "instillSecret": true,
           "instillUIOrder": 0,
           "title": "API Key",
           "type": "string"
@@ -35,7 +35,7 @@
           "instillAcceptFormats": [
             "string"
           ],
-          "instillCredentialField": false,
+          "instillSecret": false,
           "instillUIOrder": 1,
           "title": "Pinecone Base URL",
           "type": "string"
@@ -45,6 +45,10 @@
         "api_key",
         "url"
       ],
+      "instillEditOnNodeFields": [
+        "api_key",
+        "url"
+      ],
       "title": "Pinecone Connection",
       "type": "object"
     }