Skip to content
This repository has been archived by the owner on Nov 23, 2023. It is now read-only.

Commit

Permalink
feat: add json schema validation
Browse files Browse the repository at this point in the history
  • Loading branch information
@pinglin
pinglin committed May 19, 2022
1 parent 87b9fd3 commit bc609e2
Show file tree
Hide file tree
Showing 13 changed files with 291 additions and 117 deletions.
12 changes: 6 additions & 6 deletions cmd/init/convert.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,17 +39,17 @@ func unmarshalConnectorPB(jsonSliceMap interface{}, pb interface{}) error {
}
*pb = append(*pb, &def)
case *[]*connectorPB.SourceConnectorDefinition:
srcDef := connectorPB.SourceConnectorDefinition{}
if err := pj.Unmarshal(b, &srcDef); err != nil {
srcConnDef := connectorPB.SourceConnectorDefinition{}
if err := pj.Unmarshal(b, &srcConnDef); err != nil {
return err
}
*pb = append(*pb, &srcDef)
*pb = append(*pb, &srcConnDef)
case *[]*connectorPB.DestinationConnectorDefinition:
dstDef := connectorPB.DestinationConnectorDefinition{}
if err := pj.Unmarshal(b, &dstDef); err != nil {
dstConnDef := connectorPB.DestinationConnectorDefinition{}
if err := pj.Unmarshal(b, &dstConnDef); err != nil {
return err
}
*pb = append(*pb, &dstDef)
*pb = append(*pb, &dstConnDef)
case *[]*connectorPB.DockerImageSpec:
dockerImgSpec := connectorPB.DockerImageSpec{}
if err := pj.Unmarshal(b, &dockerImgSpec); err != nil {
Expand Down
62 changes: 40 additions & 22 deletions cmd/init/destination.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,13 @@ import (
"strings"
"time"

"github.com/gofrs/uuid"
"google.golang.org/protobuf/encoding/protojson"
"google.golang.org/protobuf/types/known/structpb"
"google.golang.org/protobuf/types/known/timestamppb"
"gorm.io/datatypes"
"gorm.io/gorm"

"github.com/gofrs/uuid"

"github.com/instill-ai/connector-backend/internal/logger"
"github.com/instill-ai/connector-backend/pkg/datamodel"

Expand All @@ -23,17 +25,33 @@ func createDestinationConnectorDefinition(db *gorm.DB, dstConnDef *connectorPB.D
return err
}

id := dstConnDef.GetId()
if id == "" {
id = connDef.GetDockerRepository()[strings.LastIndex(connDef.GetDockerRepository(), "/")+1:]
if id == "" {
// Only directness connector ends up this
id = strings.ToLower(connDef.GetTitle())
}
if dstConnDef.GetId() == "" {
dstConnDef.Id = connDef.GetDockerRepository()[strings.LastIndex(connDef.GetDockerRepository(), "/")+1:]
}

dstConnDef.ConnectorDefinition = connDef
dstConnDef.GetConnectorDefinition().CreateTime = &timestamppb.Timestamp{}
dstConnDef.GetConnectorDefinition().UpdateTime = &timestamppb.Timestamp{}
dstConnDef.GetConnectorDefinition().Tombstone = false
dstConnDef.GetConnectorDefinition().Public = true
dstConnDef.GetConnectorDefinition().Custom = false

dstConnDef.GetConnectorDefinition().Spec = &connectorPB.Spec{}
if err := protojson.Unmarshal(spec, dstConnDef.GetConnectorDefinition().Spec); err != nil {
logger.Fatal(err.Error())
}

if dstConnDef.GetConnectorDefinition().GetResourceRequirements() == nil {
dstConnDef.GetConnectorDefinition().ResourceRequirements = &structpb.Struct{}
}

// Validate JSON Schema before inserting into db
if err := datamodel.ValidateJSONSchema(datamodel.DstConnDefJSONSchema, dstConnDef, true); err != nil {
return err
}

releaseDate := func() *time.Time {
releaseDate := connDef.GetReleaseDate()
releaseDate := dstConnDef.GetConnectorDefinition().GetReleaseDate()
if releaseDate != nil {
t := time.Date(int(releaseDate.Year), time.Month(releaseDate.Month), int(releaseDate.Day), 0, 0, 0, 0, time.UTC)
return &t
Expand All @@ -42,7 +60,7 @@ func createDestinationConnectorDefinition(db *gorm.DB, dstConnDef *connectorPB.D
}()

resourceRequirements := func() datatypes.JSON {
s := connDef.GetResourceRequirements()
s := dstConnDef.GetConnectorDefinition().GetResourceRequirements()
if s != nil {
if b, err := s.MarshalJSON(); err != nil {
logger.Fatal(err.Error())
Expand All @@ -56,21 +74,21 @@ func createDestinationConnectorDefinition(db *gorm.DB, dstConnDef *connectorPB.D
if err := createConnectorDefinitionRecord(
db,
uid,
id,
connDef.GetTitle(),
connDef.GetDockerRepository(),
connDef.GetDockerImageTag(),
connDef.GetDocumentationUrl(),
connDef.GetIcon(),
connDef.GetTombstone(),
true, //dstDef.GetPublic(),
connDef.GetCustom(),
dstConnDef.GetId(),
dstConnDef.GetConnectorDefinition().GetTitle(),
dstConnDef.GetConnectorDefinition().GetDockerRepository(),
dstConnDef.GetConnectorDefinition().GetDockerImageTag(),
dstConnDef.GetConnectorDefinition().GetDocumentationUrl(),
dstConnDef.GetConnectorDefinition().GetIcon(),
dstConnDef.GetConnectorDefinition().GetTombstone(),
dstConnDef.GetConnectorDefinition().GetPublic(),
dstConnDef.GetConnectorDefinition().GetCustom(),
releaseDate,
spec,
resourceRequirements,
datamodel.ConnectorType(connectorPB.ConnectorType_CONNECTOR_TYPE_DESTINATION),
datamodel.ConnectionType(connDef.GetConnectionType()),
datamodel.ReleaseStage(connDef.GetReleaseStage()),
datamodel.ConnectionType(dstConnDef.GetConnectorDefinition().GetConnectionType()),
datamodel.ReleaseStage(dstConnDef.GetConnectorDefinition().GetReleaseStage()),
); err != nil {
return err
}
Expand Down
3 changes: 3 additions & 0 deletions cmd/init/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (

"github.com/instill-ai/connector-backend/configs"
"github.com/instill-ai/connector-backend/internal/logger"
"github.com/instill-ai/connector-backend/pkg/datamodel"

database "github.com/instill-ai/connector-backend/internal/db"
connectorPB "github.com/instill-ai/protogen-go/connector/v1alpha"
Expand Down Expand Up @@ -33,6 +34,8 @@ func main() {
db := database.GetConnection()
defer database.Close(db)

datamodel.InitJSONSchema()

srcConnDefs := []*connectorPB.SourceConnectorDefinition{}
srcDefs := []*connectorPB.ConnectorDefinition{}
dstConnDefs := []*connectorPB.DestinationConnectorDefinition{}
Expand Down
51 changes: 35 additions & 16 deletions cmd/init/source.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@ import (
"strings"
"time"

"google.golang.org/protobuf/encoding/protojson"
"google.golang.org/protobuf/types/known/structpb"
"google.golang.org/protobuf/types/known/timestamppb"
"gorm.io/datatypes"
"gorm.io/gorm"

Expand All @@ -23,13 +26,29 @@ func createSourceConnectorDefinition(db *gorm.DB, srcConnDef *connectorPB.Source
return err
}

id := srcConnDef.GetId()
if id == "" {
id = connDef.GetDockerRepository()[strings.LastIndex(connDef.GetDockerRepository(), "/")+1:]
if id == "" {
// Only directness connector ends up this
id = strings.ToLower(connDef.GetTitle())
}
if srcConnDef.GetId() == "" {
srcConnDef.Id = connDef.GetDockerRepository()[strings.LastIndex(connDef.GetDockerRepository(), "/")+1:]
}

srcConnDef.ConnectorDefinition = connDef
srcConnDef.GetConnectorDefinition().CreateTime = &timestamppb.Timestamp{}
srcConnDef.GetConnectorDefinition().UpdateTime = &timestamppb.Timestamp{}
srcConnDef.GetConnectorDefinition().Tombstone = false
srcConnDef.GetConnectorDefinition().Public = true
srcConnDef.GetConnectorDefinition().Custom = false

srcConnDef.GetConnectorDefinition().Spec = &connectorPB.Spec{}
if err := protojson.Unmarshal(spec, srcConnDef.GetConnectorDefinition().Spec); err != nil {
logger.Fatal(err.Error())
}

if srcConnDef.GetConnectorDefinition().GetResourceRequirements() == nil {
srcConnDef.GetConnectorDefinition().ResourceRequirements = &structpb.Struct{}
}

// Validate JSON Schema before inserting into db
if err := datamodel.ValidateJSONSchema(datamodel.SrcConnDefJSONSchema, srcConnDef, true); err != nil {
return err
}

releaseDate := func() *time.Time {
Expand All @@ -56,15 +75,15 @@ func createSourceConnectorDefinition(db *gorm.DB, srcConnDef *connectorPB.Source
if err := createConnectorDefinitionRecord(
db,
uid,
id,
connDef.GetTitle(),
connDef.GetDockerRepository(),
connDef.GetDockerImageTag(),
connDef.GetDocumentationUrl(),
connDef.GetIcon(),
connDef.GetTombstone(),
true, //srcDef.GetPublic(),
connDef.GetCustom(),
srcConnDef.GetId(),
srcConnDef.GetConnectorDefinition().GetTitle(),
srcConnDef.GetConnectorDefinition().GetDockerRepository(),
srcConnDef.GetConnectorDefinition().GetDockerImageTag(),
srcConnDef.GetConnectorDefinition().GetDocumentationUrl(),
srcConnDef.GetConnectorDefinition().GetIcon(),
srcConnDef.GetConnectorDefinition().GetTombstone(),
srcConnDef.GetConnectorDefinition().GetPublic(),
srcConnDef.GetConnectorDefinition().GetCustom(),
releaseDate,
spec,
resourceRequirements,
Expand Down
2 changes: 1 addition & 1 deletion configs/models/connector.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@
"type": "string",
"format": "date-time",
"title": "Create time",
"description": "The date when the connector definiton is created in the format YYYY-MM-DDTHH:MM::SS",
"description": "The date when the connector definition is created in the format YYYY-MM-DDTHH:MM::SS",
"examples": ["2022-04-29T00:20:06.703547Z"],
"readOnly": true,
"ui_order": 4,
Expand Down
31 changes: 8 additions & 23 deletions configs/models/connector_definition.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
"required": [
"title",
"documentation_url",
"icon",
"connection_type",
"spec",
"tombstone",
Expand Down Expand Up @@ -66,7 +65,7 @@
"format": "uri-reference",
"title": "Connector definition icon",
"description": "Icon for this connector definition",
"minLength": 1,
"minLength": 0,
"maxLength": 256,
"readOnly": true,
"ui_order": 4,
Expand Down Expand Up @@ -167,7 +166,7 @@
"type": "object",
"title": "Resource requirements",
"description": "Resource requirements blob. Must be a valid JSON that includes what resources are needed for the connector",
"additionalProperties": false,
"additionalProperties": true,
"ui_order": 12,
"ui_hidden": true,
"ui_disabled": true
Expand Down Expand Up @@ -218,19 +217,6 @@
"ui_disabled": true,
"ui_component": "text"
},
"changelog_url": {
"type": "string",
"format": "uri",
"title": "Changelog URL",
"description": "The Changelog URL of the connector",
"minLength": 0,
"maxLength": 1024,
"readOnly": true,
"ui_order": 1,
"ui_hidden": true,
"ui_disabled": true,
"ui_component": "text"
},
"connection_specification": {
"type": "object",
"title": "Connection specification",
Expand Down Expand Up @@ -281,6 +267,9 @@
"ui_order": 6
},
"advanced_auth": {
"type": "object",
"title": "Advanced auth flow",
"description": "Additional and optional specification object to describe what an 'advanced' Auth flow would need to function.",
"$ref": "#/definitions/AdvancedAuth",
"ui_order": 7,
"ui_disabled": true
Expand All @@ -305,10 +294,6 @@
"ui_enum": ["", "Append", "Overwrite", "Append dedup"]
},
"AdvancedAuth": {
"type": "object",
"title": "Advanced auth flow",
"description": "Additional and optional specification object to describe what an 'advanced' Auth flow would need to function.",
"required": [],
"properties": {
"auth_flow_type": {
"type": "string",
Expand Down Expand Up @@ -348,6 +333,9 @@
"ui_disabled": true
},
"oauth_config_specification": {
"type": "object",
"title": "OAuth configuration specification",
"description": "The supported oauth configuration specification",
"$ref": "#/definitions/OAuthConfigSpecification",
"ui_order": 3,
"ui_hidden": true,
Expand All @@ -356,9 +344,6 @@
}
},
"OAuthConfigSpecification": {
"type": "object",
"title": "OAuth configuration specification",
"required": [],
"properties": {
"oauth_user_input_from_connector_config_specification": {
"type": "object",
Expand Down
3 changes: 2 additions & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ require (
)

require (
github.com/instill-ai/protogen-go v0.1.5-alpha.0.20220517005010-6963c56ac4fc
github.com/instill-ai/protogen-go v0.1.5-alpha.0.20220519185924-775d2feab526
github.com/instill-ai/x v0.1.0-alpha.0.20220517204940-5a70916ce425
github.com/mennanov/fieldmask-utils v0.5.0
)
Expand Down Expand Up @@ -51,6 +51,7 @@ require (
github.com/mitchellh/mapstructure v1.4.3 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/santhosh-tekuri/jsonschema/v5 v5.0.0
go.uber.org/atomic v1.9.0 // indirect
go.uber.org/multierr v1.8.0 // indirect
golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect
Expand Down
6 changes: 4 additions & 2 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -603,8 +603,8 @@ github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH
github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/instill-ai/protogen-go v0.1.5-alpha.0.20220517005010-6963c56ac4fc h1:4rEgqtNcQB5vDcphGzDcf1uvqZW7JFILT4I8y5qPF3k=
github.com/instill-ai/protogen-go v0.1.5-alpha.0.20220517005010-6963c56ac4fc/go.mod h1:q2Pq4P0AY/59RGibT4nSDnOsA4wD4XhLueFRoGYNBjk=
github.com/instill-ai/protogen-go v0.1.5-alpha.0.20220519185924-775d2feab526 h1:ol2S5NubJ+IHsFA7WsPq0oWkyWZlkaCWZNvajG+Rpts=
github.com/instill-ai/protogen-go v0.1.5-alpha.0.20220519185924-775d2feab526/go.mod h1:q2Pq4P0AY/59RGibT4nSDnOsA4wD4XhLueFRoGYNBjk=
github.com/instill-ai/x v0.1.0-alpha.0.20220517204940-5a70916ce425 h1:XR5kKLN00JNR09G5oB176U39sFGyF27HHoVGA96zFrg=
github.com/instill-ai/x v0.1.0-alpha.0.20220517204940-5a70916ce425/go.mod h1:Chc+UChNgV4ejOlji4Ca2l+83fPdDbOML5ToQxEnJ4Y=
github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA=
Expand Down Expand Up @@ -934,6 +934,8 @@ github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfF
github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4=
github.com/santhosh-tekuri/jsonschema/v5 v5.0.0 h1:TToq11gyfNlrMFZiYujSekIsPd9AmsA2Bj/iv+s4JHE=
github.com/santhosh-tekuri/jsonschema/v5 v5.0.0/go.mod h1:FKdcjfQW6rpZSnxxUvEA5H/cDPdvJ/SZJQLWWXWGrZ0=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
Expand Down
4 changes: 0 additions & 4 deletions integration-test/const.js
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,5 @@
export const csvDstConfig = {
"supports_incremental": true,
"connection_specification": {
"destination_path": "/local"
},
"supported_destination_sync_modes": [2, 1]
};

export const csvDstDefinitionRscName = "destination-connector-definitions/destination-csv"
Expand Down
Loading

0 comments on commit bc609e2

Please sign in to comment.