From 2267334518dac7b678f6309392f023f5040515fe Mon Sep 17 00:00:00 2001 From: Yevgeny Pats <16490766+yevgenypats@users.noreply.github.com> Date: Sat, 16 Sep 2023 13:37:44 +0300 Subject: [PATCH 1/3] feat: Add support for jsonschema --- go.mod | 1 + go.sum | 2 ++ plugin/options.go | 6 ++++++ plugin/plugin.go | 31 +++++++++++++++++++++++++++++++ 4 files changed, 40 insertions(+) diff --git a/go.mod b/go.mod index 8248a090ff..b069d099f4 100644 --- a/go.mod +++ b/go.mod @@ -48,6 +48,7 @@ require ( github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rogpeppe/go-internal v1.10.0 // indirect + github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect diff --git a/go.sum b/go.sum index 9d0bad2b92..d0b7b1950f 100644 --- a/go.sum +++ b/go.sum @@ -202,6 +202,8 @@ github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc= github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= +github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= diff --git a/plugin/options.go b/plugin/options.go index 12e1500558..a96e64887f 100644 --- a/plugin/options.go +++ b/plugin/options.go @@ -23,6 +23,12 @@ func WithBuildTargets(targets []BuildTarget) Option { } } +func WithJSONSchema(schema string) Option { + return func(p *Plugin) { + p.schema = schema + } +} + type TableOptions struct { Tables []string SkipTables []string diff --git a/plugin/plugin.go b/plugin/plugin.go index 6f3374dad3..20427bfff5 100644 --- a/plugin/plugin.go +++ b/plugin/plugin.go @@ -2,13 +2,16 @@ package plugin import ( "context" + "encoding/json" "fmt" + "strings" "sync" "github.com/apache/arrow/go/v14/arrow" "github.com/cloudquery/plugin-sdk/v4/message" "github.com/cloudquery/plugin-sdk/v4/schema" "github.com/rs/zerolog" + "github.com/santhosh-tekuri/jsonschema/v5" ) var ErrNotImplemented = fmt.Errorf("not implemented") @@ -66,6 +69,10 @@ type Plugin struct { // NoInternalColumns if set to true will not add internal columns to tables such as _cq_id and _cq_parent_id // useful for sources such as PostgreSQL and other databases internalColumns bool + // schema is the JSONSchema of the plugin spec + schema string + // validator object to validate specs + schemaValidator *jsonschema.Schema } // NewPlugin returns a new CloudQuery Plugin with the given name, version and implementation. @@ -81,6 +88,19 @@ func NewPlugin(name string, version string, newClient NewClientFunc, options ... for _, opt := range options { opt(&p) } + if p.schema != "" { + c := jsonschema.NewCompiler() + c.Draft = jsonschema.Draft2020 + if err := c.AddResource("schema.json", strings.NewReader(p.schema)); err != nil { + panic(fmt.Errorf("failed add plugin JSONSchema: %w", err)) + } + schemaValidator, err := c.Compile("schema.json") + if err != nil { + panic(fmt.Errorf("failed to compile plugin JSONSchema: %w", err)) + } + p.schemaValidator = schemaValidator + } + return &p } @@ -120,6 +140,17 @@ func (p *Plugin) Init(ctx context.Context, spec []byte, options NewClientOptions } defer p.mu.Unlock() var err error + + if p.schemaValidator != nil { + var v any + if err := json.Unmarshal(spec, &v); err != nil { + return fmt.Errorf("failed to unmarshal plugin spec: %w", err) + } + if err := p.schemaValidator.Validate(v); err != nil { + return fmt.Errorf("plugin spec is invalid: %w", err) + } + } + p.client, err = p.newClient(ctx, p.logger, spec, options) if err != nil { return fmt.Errorf("failed to initialize client: %w", err) From 5e726296938a3354507a87a6a663bc8dbfb95d63 Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Wed, 20 Sep 2023 12:28:39 +0300 Subject: [PATCH 2/3] tidy --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index b069d099f4..5db21be692 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/google/uuid v1.3.0 github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0 github.com/rs/zerolog v1.29.1 + github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/spf13/cobra v1.6.1 github.com/stretchr/testify v1.8.4 github.com/thoas/go-funk v0.9.3 @@ -48,7 +49,6 @@ require ( github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rogpeppe/go-internal v1.10.0 // indirect - github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect From b6cd5c576e6bfef0e9c061cd427f8c8fdd754256 Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Wed, 20 Sep 2023 12:32:53 +0300 Subject: [PATCH 3/3] tidy x2 --- examples/simple_plugin/go.mod | 1 + examples/simple_plugin/go.sum | 2 ++ 2 files changed, 3 insertions(+) diff --git a/examples/simple_plugin/go.mod b/examples/simple_plugin/go.mod index c3284f409c..0bb0a8adaa 100644 --- a/examples/simple_plugin/go.mod +++ b/examples/simple_plugin/go.mod @@ -35,6 +35,7 @@ require ( github.com/mattn/go-isatty v0.0.19 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect github.com/spf13/cobra v1.6.1 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/testify v1.8.4 // indirect diff --git a/examples/simple_plugin/go.sum b/examples/simple_plugin/go.sum index eca4a339e2..c2603f6250 100644 --- a/examples/simple_plugin/go.sum +++ b/examples/simple_plugin/go.sum @@ -193,6 +193,8 @@ github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c= github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= +github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY=