diff --git a/.gitignore b/.gitignore index da59a40e6f..66f4ce53f9 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ config.hcl .vscode vendor +cover.out \ No newline at end of file diff --git a/Makefile b/Makefile index 7c150743e4..474bf5f2cd 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,6 @@ test: lint: golangci-lint run -.PHONY: generate-protobuf -generate-protobuf: +.PHONY: gen-proto +gen-proto: protoc --proto_path=. --go_out . --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative internal/pb/base.proto internal/pb/source.proto internal/pb/destination.proto \ No newline at end of file diff --git a/clients/destination.go b/clients/destination.go index 31bc46942a..b4df62f6d0 100644 --- a/clients/destination.go +++ b/clients/destination.go @@ -2,15 +2,14 @@ package clients import ( "context" + "encoding/json" "fmt" "github.com/cloudquery/plugin-sdk/internal/pb" "github.com/cloudquery/plugin-sdk/plugins" "github.com/cloudquery/plugin-sdk/schema" "github.com/cloudquery/plugin-sdk/specs" - "github.com/vmihailenco/msgpack/v5" "google.golang.org/grpc" - "gopkg.in/yaml.v3" ) type DestinationClient struct { @@ -31,23 +30,9 @@ func NewLocalDestinationClient(p plugins.DestinationPlugin) *DestinationClient { } } -func (c *DestinationClient) Configure(ctx context.Context, s specs.DestinationSpec) error { - if c.localClient != nil { - return c.localClient.Configure(ctx, s) - } - b, err := yaml.Marshal(s) - if err != nil { - return fmt.Errorf("failed to marshal spec: %w", err) - } - if _, err := c.pbClient.Configure(ctx, &pb.Configure_Request{Config: b}); err != nil { - return err - } - return nil -} - func (c *DestinationClient) GetExampleConfig(ctx context.Context) (string, error) { if c.localClient != nil { - return c.localClient.GetExampleConfig(ctx), nil + return c.localClient.ExampleConfig(), nil } res, err := c.pbClient.GetExampleConfig(ctx, &pb.GetExampleConfig_Request{}) if err != nil { @@ -56,42 +41,52 @@ func (c *DestinationClient) GetExampleConfig(ctx context.Context) (string, error return res.Config, nil } -func (c *DestinationClient) Save(ctx context.Context, msg *FetchResultMessage) error { - var saveClient pb.Destination_SaveClient - var err error - if c.pbClient != nil { - saveClient, err = c.pbClient.Save(ctx) - if err != nil { - return fmt.Errorf("failed to create save client: %w", err) - } - } +func (c *DestinationClient) Initialize(ctx context.Context, spec specs.Destination) error { if c.localClient != nil { - var resource schema.Resource - if err := msgpack.Unmarshal(msg.Resource, &resource); err != nil { - return fmt.Errorf("failed to unmarshal resources: %w", err) - } - if err := c.localClient.Save(ctx, []*schema.Resource{&resource}); err != nil { - return fmt.Errorf("failed to save resources: %w", err) - } - } else { - if err := saveClient.Send(&pb.Save_Request{Resources: msg.Resource}); err != nil { - return err - } + return c.localClient.Initialize(ctx, spec) + } + b, err := json.Marshal(spec) + if err != nil { + return fmt.Errorf("destination configure: failed to marshal spec: %w", err) + } + _, err = c.pbClient.Configure(ctx, &pb.Configure_Request{ + Config: b, + }) + if err != nil { + return fmt.Errorf("destination configure: failed to configure: %w", err) } - return nil } -func (c *DestinationClient) CreateTables(ctx context.Context, tables []*schema.Table) error { +func (c *DestinationClient) Migrate(ctx context.Context, tables []*schema.Table) error { if c.localClient != nil { - return c.localClient.CreateTables(ctx, tables) + return c.localClient.Migrate(ctx, tables) + } + b, err := json.Marshal(tables) + if err != nil { + return fmt.Errorf("destination migrate: failed to marshal plugin: %w", err) } - b, err := yaml.Marshal(tables) + _, err = c.pbClient.Migrate(ctx, &pb.Migrate_Request{Tables: b}) if err != nil { - return fmt.Errorf("failed to marshal tables: %w", err) + return fmt.Errorf("destination migrate: failed to migrate: %w", err) } - if _, err := c.pbClient.CreateTables(ctx, &pb.CreateTables_Request{Tables: b}); err != nil { - return err + return nil +} + +func (c *DestinationClient) Write(ctx context.Context, table string, data map[string]interface{}) error { + // var saveClient pb.Destination_SaveClient + // var err error + // if c.pbClient != nil { + // saveClient, err = c.pbClient.Write(ctx) + // if err != nil { + // return fmt.Errorf("failed to create save client: %w", err) + // } + // } + if c.localClient != nil { + if err := c.localClient.Write(ctx, table, data); err != nil { + return fmt.Errorf("failed to save resources: %w", err) + } } + return nil } diff --git a/clients/source.go b/clients/source.go index 10ccbc1f55..99f84a09fc 100644 --- a/clients/source.go +++ b/clients/source.go @@ -3,20 +3,15 @@ package clients import ( - "bytes" "context" + "encoding/json" "fmt" "io" - "text/template" "github.com/cloudquery/plugin-sdk/internal/pb" "github.com/cloudquery/plugin-sdk/schema" "github.com/cloudquery/plugin-sdk/specs" - "github.com/pkg/errors" - "github.com/vmihailenco/msgpack/v5" - "github.com/xeipuuv/gojsonschema" "google.golang.org/grpc" - "gopkg.in/yaml.v3" ) type SourceClient struct { @@ -27,14 +22,6 @@ type FetchResultMessage struct { Resource []byte } -const sourcePluginExampleConfigTemplate = `kind: source -spec: - name: {{.Name}} - version: {{.Version}} - configuration: - {{.PluginExampleConfig | indent 4}} -` - func NewSourceClient(cc grpc.ClientConnInterface) *SourceClient { return &SourceClient{ pbClient: pb.NewSourceClient(cc), @@ -47,52 +34,30 @@ func (c *SourceClient) GetTables(ctx context.Context) ([]*schema.Table, error) { return nil, err } var tables []*schema.Table - if err := msgpack.Unmarshal(res.Tables, &tables); err != nil { + if err := json.Unmarshal(res.Tables, &tables); err != nil { return nil, err } return tables, nil } -func (c *SourceClient) Configure(ctx context.Context, spec specs.SourceSpec) (*gojsonschema.Result, error) { - b, err := yaml.Marshal(spec) - if err != nil { - return nil, errors.Wrap(err, "failed to marshal source spec") - } - res, err := c.pbClient.Configure(ctx, &pb.Configure_Request{Config: b}) - if err != nil { - return nil, errors.Wrap(err, "failed to configure source") - } - var validationResult gojsonschema.Result - if err := msgpack.Unmarshal(res.JsonschemaResult, &validationResult); err != nil { - return nil, errors.Wrap(err, "failed to unmarshal validation result") - } - return &validationResult, nil -} - -func (c *SourceClient) GetExampleConfig(ctx context.Context) (string, error) { +func (c *SourceClient) ExampleConfig(ctx context.Context) (string, error) { res, err := c.pbClient.GetExampleConfig(ctx, &pb.GetExampleConfig_Request{}) if err != nil { return "", fmt.Errorf("failed to get example config: %w", err) } - t, err := template.New("source_plugin").Funcs(templateFuncMap()).Parse(sourcePluginExampleConfigTemplate) - if err != nil { - return "", fmt.Errorf("failed to parse template: %w", err) - } - var tpl bytes.Buffer - if err := t.Execute(&tpl, map[string]interface{}{ - "Name": res.Name, - "Version": res.Version, - "PluginExampleConfig": res.Config, - }); err != nil { - return "", fmt.Errorf("failed to generate example config: %w", err) - } - return tpl.String(), nil + return res.Config, nil } -func (c *SourceClient) Fetch(ctx context.Context, spec specs.SourceSpec, res chan<- *FetchResultMessage) error { - stream, err := c.pbClient.Fetch(ctx, &pb.Fetch_Request{}) +func (c *SourceClient) Sync(ctx context.Context, spec specs.Source, res chan<- *schema.Resource) error { + b, err := json.Marshal(spec) if err != nil { - return fmt.Errorf("failed to fetch resources: %w", err) + return fmt.Errorf("failed to marshal source spec: %w", err) + } + stream, err := c.pbClient.Sync(ctx, &pb.Sync_Request{ + Spec: b, + }) + if err != nil { + return fmt.Errorf("failed to sync resources: %w", err) } for { r, err := stream.Recv() @@ -102,8 +67,12 @@ func (c *SourceClient) Fetch(ctx context.Context, spec specs.SourceSpec, res cha } return fmt.Errorf("failed to fetch resources from stream: %w", err) } - res <- &FetchResultMessage{ - Resource: r.Resource, + var resource schema.Resource + err = json.Unmarshal(r.Resource, &resource) + if err != nil { + return fmt.Errorf("failed to unmarshal resource: %w", err) } + + res <- &resource } } diff --git a/clients/template.go b/clients/template.go deleted file mode 100644 index 3a4be441f4..0000000000 --- a/clients/template.go +++ /dev/null @@ -1,17 +0,0 @@ -package clients - -import ( - "strings" - "text/template" -) - -func templateFuncMap() template.FuncMap { - return template.FuncMap{ - "indent": indent, - } -} - -func indent(spaces int, v string) string { - pad := strings.Repeat(" ", spaces) - return pad + strings.ReplaceAll(v, "\n", "\n"+pad) -} diff --git a/codegen/doc.go b/codegen/doc.go new file mode 100644 index 0000000000..beb54c81b4 --- /dev/null +++ b/codegen/doc.go @@ -0,0 +1,2 @@ +//codgen helps autogenerate cloudquery plugins configured by definition +package codegen diff --git a/codegen/golang.go b/codegen/golang.go new file mode 100644 index 0000000000..1432fd7243 --- /dev/null +++ b/codegen/golang.go @@ -0,0 +1,201 @@ +package codegen + +import ( + "embed" + "fmt" + "go/ast" + "io" + "reflect" + "strings" + "text/template" + + "github.com/cloudquery/plugin-sdk/schema" + "github.com/iancoleman/strcase" + "golang.org/x/tools/go/packages" +) + +type TableOptions func(*TableDefinition) + +//go:embed templates/*.go.tpl +var TemplatesFS embed.FS + +func valueToSchemaType(v reflect.Type) (schema.ValueType, error) { + k := v.Kind() + switch k { + case reflect.String: + return schema.TypeString, nil + case reflect.Bool: + return schema.TypeBool, nil + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return schema.TypeInt, nil + case reflect.Float32, reflect.Float64: + return schema.TypeFloat, nil + case reflect.Map: + return schema.TypeJSON, nil + case reflect.Struct: + t := v.PkgPath() + "." + v.Name() + if t == "time.Time" { + return schema.TypeTimestamp, nil + } + return schema.TypeJSON, nil + case reflect.Pointer: + return valueToSchemaType(v.Elem()) + case reflect.Slice: + switch v.Elem().Kind() { + case reflect.String: + return schema.TypeStringArray, nil + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return schema.TypeIntArray, nil + default: + return schema.TypeJSON, nil + } + default: + return schema.TypeInvalid, fmt.Errorf("unsupported type: %s", k) + } +} + +func WithNameTransformer(transformer func(string) string) TableOptions { + return func(t *TableDefinition) { + t.Name = transformer(t.Name) + } +} + +func WithSkipFields(fields []string) TableOptions { + return func(t *TableDefinition) { + t.skipFields = fields + } +} + +func WithOverrideColumns(columns []ColumnDefinition) TableOptions { + return func(t *TableDefinition) { + t.overrideColumns = columns + } +} + +func WithExtraColumns(columns []ColumnDefinition) TableOptions { + return func(t *TableDefinition) { + t.extraColumns = columns + } +} + +func WithDescriptionsEnabled() TableOptions { + return func(t *TableDefinition) { + t.descriptionsEnabled = true + } +} + +func defaultTransformer(name string) string { + return strcase.ToSnake(name) +} + +func sliceContains(arr []string, s string) bool { + for _, v := range arr { + if v == s { + return true + } + } + return false +} + +func NewTableFromStruct(name string, obj interface{}, opts ...TableOptions) (*TableDefinition, error) { + t := TableDefinition{ + Name: name, + nameTransformer: defaultTransformer, + } + for _, opt := range opts { + opt(&t) + } + + e := reflect.ValueOf(obj) + if e.Kind() == reflect.Pointer { + e = e.Elem() + } + if e.Kind() != reflect.Struct { + return nil, fmt.Errorf("expected struct, got %s", e.Kind()) + } + + comments := make(map[string]string) + if t.descriptionsEnabled { + comments = readStructComments(e.Type().PkgPath(), e.Type().Name()) + } + + for _, c := range t.extraColumns { + if t.overrideColumns != nil { + if col := t.overrideColumns.GetByName(c.Name); col != nil { + t.Columns = append(t.Columns, *col) + continue + } + } + t.Columns = append(t.Columns, c) + } + + for i := 0; i < e.NumField(); i++ { + field := e.Type().Field(i) + if sliceContains(t.skipFields, field.Name) { + continue + } + + if t.overrideColumns != nil { + if col := t.overrideColumns.GetByName(t.nameTransformer(field.Name)); col != nil { + t.Columns = append(t.Columns, *col) + continue + } + } + + columnType, err := valueToSchemaType(field.Type) + if err != nil { + return nil, err + } + + // generate a PathResolver to use by default + pathResolver := fmt.Sprintf("schema.PathResolver(%q)", field.Name) + column := ColumnDefinition{ + Name: t.nameTransformer(field.Name), + Type: columnType, + Resolver: pathResolver, + Description: strings.ReplaceAll(comments[field.Name], "`", "'"), + } + t.Columns = append(t.Columns, column) + } + return &t, nil +} + +func (t *TableDefinition) GenerateTemplate(wr io.Writer) error { + tpl, err := template.New("table.go.tpl").ParseFS(TemplatesFS, "templates/*") + if err != nil { + return fmt.Errorf("failed to parse template: %w", err) + } + + if err := tpl.Execute(wr, t); err != nil { + return fmt.Errorf("failed to execute template: %w", err) + } + return nil +} + +func readStructComments(pkgPath string, structName string) map[string]string { + cfg := &packages.Config{Mode: packages.NeedFiles | packages.NeedSyntax} + pkgs, err := packages.Load(cfg, pkgPath) + if err != nil { + panic(err) + } + comments := make(map[string]string, 0) + for _, p := range pkgs { + for _, f := range p.Syntax { + ast.Inspect(f, func(n ast.Node) bool { + if st, ok := n.(*ast.TypeSpec); ok { + if st.Name.Name == structName { + for _, field := range st.Type.(*ast.StructType).Fields.List { + if len(field.Names) > 0 { + comments[field.Names[0].Name] = field.Doc.Text() + } + } + } + } + return true + }) + } + } + return comments +} diff --git a/codegen/golang_test.go b/codegen/golang_test.go new file mode 100644 index 0000000000..4a728eb7cb --- /dev/null +++ b/codegen/golang_test.go @@ -0,0 +1,100 @@ +package codegen + +import ( + "bytes" + "fmt" + "testing" + "time" + + "github.com/cloudquery/plugin-sdk/schema" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" +) + +type testStruct struct { + // IntCol this is an example documentation comment + IntCol int `json:"int_col,omitempty"` + StringCol string `json:"string_col,omitempty"` + FloatCol float64 `json:"float_col,omitempty"` + BoolCol bool `json:"bool_col,omitempty"` + JSONCol struct { + IntCol int `json:"int_col,omitempty"` + StringCol string `json:"string_col,omitempty"` + } + IntArrayCol []int `json:"int_array_col,omitempty"` + StringArrayCol []string `json:"string_array_col,omitempty"` + TimeCol time.Time `json:"time_col,omitempty"` + TimePointerCol *time.Time `json:"time_pointer_col,omitempty"` +} + +var expectedTestTable = TableDefinition{ + Name: "test_struct", + Columns: []ColumnDefinition{ + { + Name: "int_col", + Type: schema.TypeInt, + Resolver: `schema.PathResolver("IntCol")`, + }, + { + Name: "string_col", + Type: schema.TypeString, + Resolver: `schema.PathResolver("StringCol")`, + }, + { + Name: "float_col", + Type: schema.TypeFloat, + Resolver: `schema.PathResolver("FloatCol")`, + }, + { + Name: "bool_col", + Type: schema.TypeBool, + Resolver: `schema.PathResolver("BoolCol")`, + }, + { + Name: "json_col", + Type: schema.TypeJSON, + Resolver: `schema.PathResolver("JSONCol")`, + }, + { + Name: "int_array_col", + Type: schema.TypeIntArray, + Resolver: `schema.PathResolver("IntArrayCol")`, + }, + { + Name: "string_array_col", + Type: schema.TypeStringArray, + Resolver: `schema.PathResolver("StringArrayCol")`, + }, + { + Name: "time_col", + Type: schema.TypeTimestamp, + Resolver: `schema.PathResolver("TimeCol")`, + }, + { + Name: "time_pointer_col", + Type: schema.TypeTimestamp, + Resolver: `schema.PathResolver("TimePointerCol")`, + }, + }, + nameTransformer: defaultTransformer, +} + +func TestTableFromGoStruct(t *testing.T) { + table, err := NewTableFromStruct("test_struct", testStruct{}) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(table, &expectedTestTable, + cmpopts.IgnoreUnexported(TableDefinition{})); diff != "" { + t.Fatalf("table does not match expected. diff (-got, +want): %v", diff) + } + buf := bytes.NewBufferString("") + if err := table.GenerateTemplate(buf); err != nil { + t.Fatal(err) + } + fmt.Println(buf.String()) +} + +// func TestReadComments(t *testing.T) { +// readComments("github.com/google/go-cmp/cmp") +// } diff --git a/codegen/table.go b/codegen/table.go new file mode 100644 index 0000000000..9bf185d782 --- /dev/null +++ b/codegen/table.go @@ -0,0 +1,49 @@ +package codegen + +import ( + "github.com/cloudquery/plugin-sdk/schema" +) + +type ResourceDefinition struct { + Name string + Table *TableDefinition +} + +type TableDefinition struct { + Name string + Description string + Columns []ColumnDefinition + Relations []*TableDefinition + + Resolver string + IgnoreError string + Multiplex string + PostResourceResolver string + Options schema.TableCreationOptions + nameTransformer func(string) string + skipFields []string + overrideColumns ColumnDefinitions + extraColumns ColumnDefinitions + descriptionsEnabled bool +} + +type ColumnDefinitions []ColumnDefinition + +type ColumnDefinition struct { + // Name name of the column + Name string + Type schema.ValueType + Resolver string + Description string + IgnoreInTests bool + Options schema.ColumnCreationOptions +} + +func (c ColumnDefinitions) GetByName(name string) *ColumnDefinition { + for _, col := range c { + if col.Name == name { + return &col + } + } + return nil +} diff --git a/codegen/templates/column.go.tpl b/codegen/templates/column.go.tpl new file mode 100644 index 0000000000..5f936eaaaf --- /dev/null +++ b/codegen/templates/column.go.tpl @@ -0,0 +1,17 @@ +{ + Name: "{{.Name}}", + Type: schema.{{.Type}}, + {{- if .Resolver}} + Resolver: {{.Resolver}}, + {{- end}} + {{- if .Description}} + Description: `{{.Description}}`, + {{- end}} + {{- if .Options.PrimaryKey}} + CreationOptions: schema.ColumnCreationOptions{ + {{- if .Options.PrimaryKey}} + PrimaryKey: true, + {{- end }} + }, + {{- end}} +}, diff --git a/codegen/templates/table.go.tpl b/codegen/templates/table.go.tpl new file mode 100644 index 0000000000..48312412a2 --- /dev/null +++ b/codegen/templates/table.go.tpl @@ -0,0 +1,23 @@ +{ + Name: "{{.Name}}", + {{- if .Description}} + Description: "{{.Description}}", + {{- end}} + {{- if .Resolver}} + Resolver: {{.Resolver}}, + {{- end}} + {{- if .Multiplex}} + Multiplex: {{.Multiplex}}, + {{- end}} + {{- if .IgnoreError}} + IgnoreError: {{.IgnoreError}}, + {{- end}} + Columns: []schema.Column{ +{{range .Columns}}{{template "column.go.tpl" .}}{{end}} + }, +{{with .Relations}} + Relations: []*schema.Table{ +{{range .}}{{template "table.go.tpl" .}}{{end}} + }, +{{end}} +} \ No newline at end of file diff --git a/go.mod b/go.mod index 73115be458..02c6e7e0a7 100644 --- a/go.mod +++ b/go.mod @@ -30,8 +30,10 @@ require ( require ( github.com/BurntSushi/toml v1.1.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/ghodss/yaml v1.0.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.2 // indirect + github.com/google/go-cmp v0.5.8 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect github.com/jackc/pgconn v1.12.1 // indirect @@ -46,10 +48,12 @@ require ( github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/sergi/go-diff v1.2.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect + gitlab.com/bosi/decorder v0.2.3 // indirect golang.org/x/crypto v0.0.0-20220517005047-85d78b3ac167 // indirect golang.org/x/exp/typeparams v0.0.0-20220613132600-b0d781184e0d // indirect golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect @@ -60,5 +64,6 @@ require ( golang.org/x/tools v0.1.11 // indirect google.golang.org/genproto v0.0.0-20220314164441-57ef72a4c106 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect honnef.co/go/tools v0.3.2 // indirect ) diff --git a/go.sum b/go.sum index effc353d40..3fcd7b7bac 100644 --- a/go.sum +++ b/go.sum @@ -42,6 +42,7 @@ github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3 github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/georgysavva/scany v1.0.0 h1:9ar4458sgkWehk8bRsEe128FQV3pVKxdN4ytmCK6BEY= github.com/georgysavva/scany v1.0.0/go.mod h1:q8QyrfXjmBk9iJD00igd4lbkAKEXAH/zIYoZ0z/Wan4= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= @@ -226,6 +227,8 @@ github.com/rs/zerolog v1.27.0 h1:1T7qCieN22GVc8S4Q2yuexzBb1EqjbgjSH9RohbMjKs= github.com/rs/zerolog v1.27.0/go.mod h1:7frBqO0oezxmnO7GF86FY++uy8I0Tk/If5ni1G9Qc0U= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= +github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= @@ -268,6 +271,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= +gitlab.com/bosi/decorder v0.2.3 h1:gX4/RgK16ijY8V+BRQHAySfQAb354T7/xQpDB2n10P0= +gitlab.com/bosi/decorder v0.2.3/go.mod h1:9K1RB5+VPNQYtXtTDAzd2OEftsZb1oV0IrJrzChSdGE= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -438,12 +443,15 @@ google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscL google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/helpers/limit/limits.go b/helpers/limit/limits.go deleted file mode 100644 index 61126e3673..0000000000 --- a/helpers/limit/limits.go +++ /dev/null @@ -1,82 +0,0 @@ -package limit - -import ( - "fmt" - "math" - - "github.com/pbnjay/memory" -) - -type Rlimit struct { - Cur uint64 - Max uint64 -} - -const ( - gbInBytes int = 1024 * 1024 * 1024 - goroutinesPerGB float64 = 250000 - minimalGoRoutines float64 = 100 - goroutineReducer = 0.8 - // only use 75% of the available file descriptors, so to leave for other processes file descriptors - mfoReducer = 0.75 -) - -func GetMaxGoRoutines() uint64 { - limit := calculateGoRoutines(getMemory()) - ulimit, err := GetUlimit() - if err != nil || ulimit.Cur == 0 { - return limit - } - if ulimit.Cur > limit { - return limit - } - return ulimit.Cur -} - -// DiagnoseLimits verifies if user should increase ulimit or max file descriptors to improve number of expected -// goroutines in CQ to improve performance -func DiagnoseLimits() error { - // the amount of goroutines we want based on machine memory - want := calculateGoRoutines(getMemory()) - // calculate file descriptor limit - fds, err := calculateFileLimit() - if err != nil { - return err - } - if fds < want { - fmt.Printf("available descriptor capacity is %d want %d to run optimally, consider increasing max file descriptors on machine.", fds, want) - } - ulimit, err := GetUlimit() - if err != nil { - return err - } - if ulimit.Cur < want { - fmt.Printf("set ulimit capacity is %d want %d to run optimally, consider increasing ulimit on this machine.", ulimit.Cur, want) - } - return err -} - -func getMemory() uint64 { - return memory.TotalMemory() -} - -func calculateMemoryGoRoutines(totalMemory uint64) uint64 { - if totalMemory == 0 { - // assume we have 2 GB RAM - return uint64(math.Max(minimalGoRoutines, goroutinesPerGB*2*goroutineReducer)) - } - return uint64(math.Max(minimalGoRoutines, (goroutinesPerGB*float64(totalMemory)/float64(gbInBytes))*goroutineReducer)) -} - -func calculateGoRoutines(totalMemory uint64) uint64 { - total := calculateMemoryGoRoutines(totalMemory) - mfo, err := calculateFileLimit() - if err != nil { - return total - } - - if mfo < total { - return uint64(float64(mfo) * mfoReducer) - } - return total -} diff --git a/helpers/limit/limits_test.go b/helpers/limit/limits_test.go deleted file mode 100644 index 1ef11134b3..0000000000 --- a/helpers/limit/limits_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package limit - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestCalculateGoRoutines(t *testing.T) { - cases := []struct { - Name string - Memory uint64 - GoRoutines uint64 - }{ - {Name: "Zero", Memory: uint64(0), GoRoutines: uint64(400000)}, - {Name: "Below 1073741824", Memory: uint64(990498816), GoRoutines: uint64(184494)}, - {Name: "At 1073741824", Memory: uint64(1073741824), GoRoutines: uint64(200000)}, - {Name: "Above 1073741824", Memory: uint64(1573741824), GoRoutines: uint64(293132)}, - } - - for _, tc := range cases { - t.Run(tc.Name, func(t *testing.T) { - assert.Equal(t, int(tc.GoRoutines), int(calculateGoRoutines(tc.Memory))) - }) - } -} diff --git a/helpers/limit/sysctl_darwin.go b/helpers/limit/sysctl_darwin.go deleted file mode 100644 index 7771571f0d..0000000000 --- a/helpers/limit/sysctl_darwin.go +++ /dev/null @@ -1,27 +0,0 @@ -//go:build darwin - -package limit - -import ( - "github.com/lorenzosaino/go-sysctl" - "github.com/spf13/cast" -) - -func calculateFileLimit() (uint64, error) { - maxFileOpen, err := sysctl.Get("kern.maxfilesperproc") - if err != nil { - return 0, err - } - mfo, err := cast.ToUint64E(maxFileOpen) - if err != nil { - return 0, err - } - - fileNr, err := sysctl.Get("kern.num_files") - if err != nil { - return 0, err - } - fnr := cast.ToUint64(fileNr) - - return uint64(float64(mfo-fnr) * goroutineReducer), nil -} diff --git a/helpers/limit/sysctl_freebsd.go b/helpers/limit/sysctl_freebsd.go deleted file mode 100644 index 00dc5269ff..0000000000 --- a/helpers/limit/sysctl_freebsd.go +++ /dev/null @@ -1,9 +0,0 @@ -//go:build freebsd - -package limit - -import "errors" - -func calculateFileLimit() (uint64, error) { - return 0, errors.New("file descriptors limiter not supported on this platform") -} diff --git a/helpers/limit/sysctl_linux.go b/helpers/limit/sysctl_linux.go deleted file mode 100644 index 7917e608a8..0000000000 --- a/helpers/limit/sysctl_linux.go +++ /dev/null @@ -1,27 +0,0 @@ -//go:build linux - -package limit - -import ( - "github.com/lorenzosaino/go-sysctl" - "github.com/spf13/cast" -) - -func calculateFileLimit() (uint64, error) { - maxFileOpen, err := sysctl.Get("fs.file-max") - if err != nil { - return 0, err - } - mfo, err := cast.ToUint64E(maxFileOpen) - if err != nil { - return 0, err - } - - fileNr, err := sysctl.Get("fs.file-nr") - if err != nil { - return 0, err - } - fnr := cast.ToUint64(fileNr) - - return uint64(float64(mfo-fnr) * goroutineReducer), nil -} diff --git a/helpers/limit/sysctl_windows.go b/helpers/limit/sysctl_windows.go deleted file mode 100644 index 9924849243..0000000000 --- a/helpers/limit/sysctl_windows.go +++ /dev/null @@ -1,9 +0,0 @@ -//go:build windows - -package limit - -import "errors" - -func calculateFileLimit() (uint64, error) { - return 0, errors.New("file descriptors limiter not supported in windows") -} diff --git a/helpers/limit/ulimit_freebsd.go b/helpers/limit/ulimit_freebsd.go deleted file mode 100644 index 999b5032fc..0000000000 --- a/helpers/limit/ulimit_freebsd.go +++ /dev/null @@ -1,13 +0,0 @@ -//go:build freebsd - -package limit - -import ( - "syscall" -) - -func GetUlimit() (Rlimit, error) { - var rLimit syscall.Rlimit - err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit) - return Rlimit{uint64(rLimit.Cur), uint64(rLimit.Max)}, err -} diff --git a/helpers/limit/ulimit_unix.go b/helpers/limit/ulimit_unix.go deleted file mode 100644 index a65486569b..0000000000 --- a/helpers/limit/ulimit_unix.go +++ /dev/null @@ -1,13 +0,0 @@ -//go:build darwin || linux - -package limit - -import ( - "syscall" -) - -func GetUlimit() (Rlimit, error) { - var rLimit syscall.Rlimit - err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit) - return Rlimit{rLimit.Cur, rLimit.Max}, err -} diff --git a/helpers/limit/ulimit_win.go b/helpers/limit/ulimit_win.go deleted file mode 100644 index 64f31e291b..0000000000 --- a/helpers/limit/ulimit_win.go +++ /dev/null @@ -1,9 +0,0 @@ -//go:build windows - -package limit - -import "errors" - -func GetUlimit() (Rlimit, error) { - return Rlimit{0, 0}, errors.New("ulimit not supported in windows") -} diff --git a/helpers/sync.go b/helpers/sync.go index b5aa6c5375..acd435f7dd 100644 --- a/helpers/sync.go +++ b/helpers/sync.go @@ -1,18 +1,28 @@ package helpers import ( + "context" + "golang.org/x/sync/semaphore" ) // SemaphoreAcauireMax is calling TryAcquire with 1 until it fails and return n-TryAcquireSucess -func TryAcquireMax(sem *semaphore.Weighted, n int64) int64 { +func TryAcquireMax(ctx context.Context, sem *semaphore.Weighted, n int64) (int64, error) { newN := n for { if newN <= 0 { - return newN + return newN, nil + } + // first try will be blocking + if newN == n { + if err := sem.Acquire(ctx, 1); err != nil { + return newN, err + } + newN-- + continue } if !sem.TryAcquire(1) { - return newN + return newN, nil } newN-- } diff --git a/internal/pb/base.pb.go b/internal/pb/base.pb.go index a0aa659dd2..f3f0f253f7 100644 --- a/internal/pb/base.pb.go +++ b/internal/pb/base.pb.go @@ -7,10 +7,11 @@ package pb import ( - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" sync "sync" + + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" ) const ( diff --git a/internal/pb/destination.pb.go b/internal/pb/destination.pb.go index 3ab55efca8..2e004a4744 100644 --- a/internal/pb/destination.pb.go +++ b/internal/pb/destination.pb.go @@ -7,10 +7,11 @@ package pb import ( - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" sync "sync" + + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" ) const ( @@ -20,14 +21,14 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -type CreateTables struct { +type Migrate struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields } -func (x *CreateTables) Reset() { - *x = CreateTables{} +func (x *Migrate) Reset() { + *x = Migrate{} if protoimpl.UnsafeEnabled { mi := &file_internal_pb_destination_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -35,13 +36,13 @@ func (x *CreateTables) Reset() { } } -func (x *CreateTables) String() string { +func (x *Migrate) String() string { return protoimpl.X.MessageStringOf(x) } -func (*CreateTables) ProtoMessage() {} +func (*Migrate) ProtoMessage() {} -func (x *CreateTables) ProtoReflect() protoreflect.Message { +func (x *Migrate) ProtoReflect() protoreflect.Message { mi := &file_internal_pb_destination_proto_msgTypes[0] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -53,19 +54,19 @@ func (x *CreateTables) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use CreateTables.ProtoReflect.Descriptor instead. -func (*CreateTables) Descriptor() ([]byte, []int) { +// Deprecated: Use Migrate.ProtoReflect.Descriptor instead. +func (*Migrate) Descriptor() ([]byte, []int) { return file_internal_pb_destination_proto_rawDescGZIP(), []int{0} } -type Save struct { +type Write struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields } -func (x *Save) Reset() { - *x = Save{} +func (x *Write) Reset() { + *x = Write{} if protoimpl.UnsafeEnabled { mi := &file_internal_pb_destination_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -73,13 +74,13 @@ func (x *Save) Reset() { } } -func (x *Save) String() string { +func (x *Write) String() string { return protoimpl.X.MessageStringOf(x) } -func (*Save) ProtoMessage() {} +func (*Write) ProtoMessage() {} -func (x *Save) ProtoReflect() protoreflect.Message { +func (x *Write) ProtoReflect() protoreflect.Message { mi := &file_internal_pb_destination_proto_msgTypes[1] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -91,22 +92,23 @@ func (x *Save) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use Save.ProtoReflect.Descriptor instead. -func (*Save) Descriptor() ([]byte, []int) { +// Deprecated: Use Write.ProtoReflect.Descriptor instead. +func (*Write) Descriptor() ([]byte, []int) { return file_internal_pb_destination_proto_rawDescGZIP(), []int{1} } -type CreateTables_Request struct { +type Migrate_Request struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // marshalled []*schema.Table - Tables []byte `protobuf:"bytes,1,opt,name=tables,proto3" json:"tables,omitempty"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Version string `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` + Tables []byte `protobuf:"bytes,3,opt,name=tables,proto3" json:"tables,omitempty"` } -func (x *CreateTables_Request) Reset() { - *x = CreateTables_Request{} +func (x *Migrate_Request) Reset() { + *x = Migrate_Request{} if protoimpl.UnsafeEnabled { mi := &file_internal_pb_destination_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -114,13 +116,13 @@ func (x *CreateTables_Request) Reset() { } } -func (x *CreateTables_Request) String() string { +func (x *Migrate_Request) String() string { return protoimpl.X.MessageStringOf(x) } -func (*CreateTables_Request) ProtoMessage() {} +func (*Migrate_Request) ProtoMessage() {} -func (x *CreateTables_Request) ProtoReflect() protoreflect.Message { +func (x *Migrate_Request) ProtoReflect() protoreflect.Message { mi := &file_internal_pb_destination_proto_msgTypes[2] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -132,26 +134,40 @@ func (x *CreateTables_Request) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use CreateTables_Request.ProtoReflect.Descriptor instead. -func (*CreateTables_Request) Descriptor() ([]byte, []int) { +// Deprecated: Use Migrate_Request.ProtoReflect.Descriptor instead. +func (*Migrate_Request) Descriptor() ([]byte, []int) { return file_internal_pb_destination_proto_rawDescGZIP(), []int{0, 0} } -func (x *CreateTables_Request) GetTables() []byte { +func (x *Migrate_Request) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *Migrate_Request) GetVersion() string { + if x != nil { + return x.Version + } + return "" +} + +func (x *Migrate_Request) GetTables() []byte { if x != nil { return x.Tables } return nil } -type CreateTables_Response struct { +type Migrate_Response struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields } -func (x *CreateTables_Response) Reset() { - *x = CreateTables_Response{} +func (x *Migrate_Response) Reset() { + *x = Migrate_Response{} if protoimpl.UnsafeEnabled { mi := &file_internal_pb_destination_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -159,13 +175,13 @@ func (x *CreateTables_Response) Reset() { } } -func (x *CreateTables_Response) String() string { +func (x *Migrate_Response) String() string { return protoimpl.X.MessageStringOf(x) } -func (*CreateTables_Response) ProtoMessage() {} +func (*Migrate_Response) ProtoMessage() {} -func (x *CreateTables_Response) ProtoReflect() protoreflect.Message { +func (x *Migrate_Response) ProtoReflect() protoreflect.Message { mi := &file_internal_pb_destination_proto_msgTypes[3] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -177,22 +193,22 @@ func (x *CreateTables_Response) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use CreateTables_Response.ProtoReflect.Descriptor instead. -func (*CreateTables_Response) Descriptor() ([]byte, []int) { +// Deprecated: Use Migrate_Response.ProtoReflect.Descriptor instead. +func (*Migrate_Response) Descriptor() ([]byte, []int) { return file_internal_pb_destination_proto_rawDescGZIP(), []int{0, 1} } -type Save_Request struct { +type Write_Request struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // marshalled []*schema.Resources - Resources []byte `protobuf:"bytes,1,opt,name=resources,proto3" json:"resources,omitempty"` + // marshalled *schema.Resources + Resource []byte `protobuf:"bytes,1,opt,name=resource,proto3" json:"resource,omitempty"` } -func (x *Save_Request) Reset() { - *x = Save_Request{} +func (x *Write_Request) Reset() { + *x = Write_Request{} if protoimpl.UnsafeEnabled { mi := &file_internal_pb_destination_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -200,13 +216,13 @@ func (x *Save_Request) Reset() { } } -func (x *Save_Request) String() string { +func (x *Write_Request) String() string { return protoimpl.X.MessageStringOf(x) } -func (*Save_Request) ProtoMessage() {} +func (*Write_Request) ProtoMessage() {} -func (x *Save_Request) ProtoReflect() protoreflect.Message { +func (x *Write_Request) ProtoReflect() protoreflect.Message { mi := &file_internal_pb_destination_proto_msgTypes[4] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -218,29 +234,29 @@ func (x *Save_Request) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use Save_Request.ProtoReflect.Descriptor instead. -func (*Save_Request) Descriptor() ([]byte, []int) { +// Deprecated: Use Write_Request.ProtoReflect.Descriptor instead. +func (*Write_Request) Descriptor() ([]byte, []int) { return file_internal_pb_destination_proto_rawDescGZIP(), []int{1, 0} } -func (x *Save_Request) GetResources() []byte { +func (x *Write_Request) GetResource() []byte { if x != nil { - return x.Resources + return x.Resource } return nil } -type Save_Response struct { +type Write_Response struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // marshalled goschemajson.Result + // error Error string `protobuf:"bytes,1,opt,name=error,proto3" json:"error,omitempty"` } -func (x *Save_Response) Reset() { - *x = Save_Response{} +func (x *Write_Response) Reset() { + *x = Write_Response{} if protoimpl.UnsafeEnabled { mi := &file_internal_pb_destination_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -248,13 +264,13 @@ func (x *Save_Response) Reset() { } } -func (x *Save_Response) String() string { +func (x *Write_Response) String() string { return protoimpl.X.MessageStringOf(x) } -func (*Save_Response) ProtoMessage() {} +func (*Write_Response) ProtoMessage() {} -func (x *Save_Response) ProtoReflect() protoreflect.Message { +func (x *Write_Response) ProtoReflect() protoreflect.Message { mi := &file_internal_pb_destination_proto_msgTypes[5] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -266,12 +282,12 @@ func (x *Save_Response) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use Save_Response.ProtoReflect.Descriptor instead. -func (*Save_Response) Descriptor() ([]byte, []int) { +// Deprecated: Use Write_Response.ProtoReflect.Descriptor instead. +func (*Write_Response) Descriptor() ([]byte, []int) { return file_internal_pb_destination_proto_rawDescGZIP(), []int{1, 1} } -func (x *Save_Response) GetError() string { +func (x *Write_Response) GetError() string { if x != nil { return x.Error } @@ -284,36 +300,38 @@ var file_internal_pb_destination_proto_rawDesc = []byte{ 0x0a, 0x1d, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x05, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x16, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, - 0x2f, 0x70, 0x62, 0x2f, 0x62, 0x61, 0x73, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x3d, - 0x0a, 0x0c, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x1a, 0x21, - 0x0a, 0x07, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x61, 0x62, - 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x74, 0x61, 0x62, 0x6c, 0x65, - 0x73, 0x1a, 0x0a, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x51, 0x0a, - 0x04, 0x53, 0x61, 0x76, 0x65, 0x1a, 0x27, 0x0a, 0x07, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x1c, 0x0a, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x1a, 0x20, - 0x0a, 0x08, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, - 0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, - 0x32, 0xa6, 0x02, 0x0a, 0x0b, 0x44, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x12, 0x40, 0x0a, 0x09, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x12, 0x18, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x2e, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x55, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x45, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, - 0x65, 0x74, 0x45, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, - 0x47, 0x65, 0x74, 0x45, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x33, 0x0a, 0x04, 0x53, 0x61, 0x76, - 0x65, 0x12, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x61, 0x76, 0x65, 0x2e, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, - 0x61, 0x76, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x12, 0x49, - 0x0a, 0x0c, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x12, 0x1b, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x61, 0x62, - 0x6c, 0x65, 0x73, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, - 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x05, 0x5a, 0x03, 0x2f, 0x70, 0x62, - 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x2f, 0x70, 0x62, 0x2f, 0x62, 0x61, 0x73, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x66, + 0x0a, 0x07, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x65, 0x1a, 0x4f, 0x0a, 0x07, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x06, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x1a, 0x0a, 0x0a, 0x08, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x50, 0x0a, 0x05, 0x57, 0x72, 0x69, 0x74, 0x65, 0x1a, + 0x25, 0x0a, 0x07, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, + 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x72, 0x65, + 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x1a, 0x20, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x32, 0x9a, 0x02, 0x0a, 0x0b, 0x44, 0x65, 0x73, + 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x55, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x45, + 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1f, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x40, 0x0a, 0x09, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x12, 0x18, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x2e, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x3a, 0x0a, 0x07, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x65, 0x12, 0x16, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x65, 0x2e, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x4d, 0x69, 0x67, + 0x72, 0x61, 0x74, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, + 0x05, 0x57, 0x72, 0x69, 0x74, 0x65, 0x12, 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x57, + 0x72, 0x69, 0x74, 0x65, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x57, 0x72, 0x69, 0x74, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x28, 0x01, 0x42, 0x05, 0x5a, 0x03, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -330,26 +348,26 @@ func file_internal_pb_destination_proto_rawDescGZIP() []byte { var file_internal_pb_destination_proto_msgTypes = make([]protoimpl.MessageInfo, 6) var file_internal_pb_destination_proto_goTypes = []interface{}{ - (*CreateTables)(nil), // 0: proto.CreateTables - (*Save)(nil), // 1: proto.Save - (*CreateTables_Request)(nil), // 2: proto.CreateTables.Request - (*CreateTables_Response)(nil), // 3: proto.CreateTables.Response - (*Save_Request)(nil), // 4: proto.Save.Request - (*Save_Response)(nil), // 5: proto.Save.Response - (*Configure_Request)(nil), // 6: proto.Configure.Request - (*GetExampleConfig_Request)(nil), // 7: proto.GetExampleConfig.Request - (*Configure_Response)(nil), // 8: proto.Configure.Response - (*GetExampleConfig_Response)(nil), // 9: proto.GetExampleConfig.Response + (*Migrate)(nil), // 0: proto.Migrate + (*Write)(nil), // 1: proto.Write + (*Migrate_Request)(nil), // 2: proto.Migrate.Request + (*Migrate_Response)(nil), // 3: proto.Migrate.Response + (*Write_Request)(nil), // 4: proto.Write.Request + (*Write_Response)(nil), // 5: proto.Write.Response + (*GetExampleConfig_Request)(nil), // 6: proto.GetExampleConfig.Request + (*Configure_Request)(nil), // 7: proto.Configure.Request + (*GetExampleConfig_Response)(nil), // 8: proto.GetExampleConfig.Response + (*Configure_Response)(nil), // 9: proto.Configure.Response } var file_internal_pb_destination_proto_depIdxs = []int32{ - 6, // 0: proto.Destination.Configure:input_type -> proto.Configure.Request - 7, // 1: proto.Destination.GetExampleConfig:input_type -> proto.GetExampleConfig.Request - 4, // 2: proto.Destination.Save:input_type -> proto.Save.Request - 2, // 3: proto.Destination.CreateTables:input_type -> proto.CreateTables.Request - 8, // 4: proto.Destination.Configure:output_type -> proto.Configure.Response - 9, // 5: proto.Destination.GetExampleConfig:output_type -> proto.GetExampleConfig.Response - 5, // 6: proto.Destination.Save:output_type -> proto.Save.Response - 3, // 7: proto.Destination.CreateTables:output_type -> proto.CreateTables.Response + 6, // 0: proto.Destination.GetExampleConfig:input_type -> proto.GetExampleConfig.Request + 7, // 1: proto.Destination.Configure:input_type -> proto.Configure.Request + 2, // 2: proto.Destination.Migrate:input_type -> proto.Migrate.Request + 4, // 3: proto.Destination.Write:input_type -> proto.Write.Request + 8, // 4: proto.Destination.GetExampleConfig:output_type -> proto.GetExampleConfig.Response + 9, // 5: proto.Destination.Configure:output_type -> proto.Configure.Response + 3, // 6: proto.Destination.Migrate:output_type -> proto.Migrate.Response + 5, // 7: proto.Destination.Write:output_type -> proto.Write.Response 4, // [4:8] is the sub-list for method output_type 0, // [0:4] is the sub-list for method input_type 0, // [0:0] is the sub-list for extension type_name @@ -365,7 +383,7 @@ func file_internal_pb_destination_proto_init() { file_internal_pb_base_proto_init() if !protoimpl.UnsafeEnabled { file_internal_pb_destination_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CreateTables); i { + switch v := v.(*Migrate); i { case 0: return &v.state case 1: @@ -377,7 +395,7 @@ func file_internal_pb_destination_proto_init() { } } file_internal_pb_destination_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Save); i { + switch v := v.(*Write); i { case 0: return &v.state case 1: @@ -389,7 +407,7 @@ func file_internal_pb_destination_proto_init() { } } file_internal_pb_destination_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CreateTables_Request); i { + switch v := v.(*Migrate_Request); i { case 0: return &v.state case 1: @@ -401,7 +419,7 @@ func file_internal_pb_destination_proto_init() { } } file_internal_pb_destination_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CreateTables_Response); i { + switch v := v.(*Migrate_Response); i { case 0: return &v.state case 1: @@ -413,7 +431,7 @@ func file_internal_pb_destination_proto_init() { } } file_internal_pb_destination_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Save_Request); i { + switch v := v.(*Write_Request); i { case 0: return &v.state case 1: @@ -425,7 +443,7 @@ func file_internal_pb_destination_proto_init() { } } file_internal_pb_destination_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Save_Response); i { + switch v := v.(*Write_Response); i { case 0: return &v.state case 1: diff --git a/internal/pb/destination.proto b/internal/pb/destination.proto index 2393932811..c8de802a32 100644 --- a/internal/pb/destination.proto +++ b/internal/pb/destination.proto @@ -4,32 +4,34 @@ option go_package = "/pb"; import "internal/pb/base.proto"; service Destination { - rpc Configure(Configure.Request) returns (Configure.Response); // Get an example configuration for the source plugin rpc GetExampleConfig(GetExampleConfig.Request) returns (GetExampleConfig.Response); - // Save resources - rpc Save(stream Save.Request) returns (Save.Response); - // Create tables - rpc CreateTables(CreateTables.Request) returns (CreateTables.Response); + // Configure the destination plugin with the given credentials and mode + rpc Configure(Configure.Request) returns (Configure.Response); + // Migrate tables to the given source plugin version + rpc Migrate(Migrate.Request) returns (Migrate.Response); + // Write resources + rpc Write(stream Write.Request) returns (Write.Response); } -message CreateTables { +message Migrate { message Request { - // marshalled []*schema.Table - bytes tables = 1; + string name = 1; + string version = 2; + bytes tables = 3; } message Response { } } -message Save { +message Write { message Request { // marshalled *schema.Resources bytes resource = 1; } message Response { - // marshalled goschemajson.Result + // error string error = 1; } } \ No newline at end of file diff --git a/internal/pb/destination_grpc.pb.go b/internal/pb/destination_grpc.pb.go index ecac6dea33..244fd1ec61 100644 --- a/internal/pb/destination_grpc.pb.go +++ b/internal/pb/destination_grpc.pb.go @@ -8,6 +8,7 @@ package pb import ( context "context" + grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" @@ -22,13 +23,14 @@ const _ = grpc.SupportPackageIsVersion7 // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. type DestinationClient interface { - Configure(ctx context.Context, in *Configure_Request, opts ...grpc.CallOption) (*Configure_Response, error) // Get an example configuration for the source plugin GetExampleConfig(ctx context.Context, in *GetExampleConfig_Request, opts ...grpc.CallOption) (*GetExampleConfig_Response, error) - // Save resources - Save(ctx context.Context, opts ...grpc.CallOption) (Destination_SaveClient, error) - // Create tables - CreateTables(ctx context.Context, in *CreateTables_Request, opts ...grpc.CallOption) (*CreateTables_Response, error) + // Configure the destination plugin with the given credentials and mode + Configure(ctx context.Context, in *Configure_Request, opts ...grpc.CallOption) (*Configure_Response, error) + // Migrate tables to the given source plugin version + Migrate(ctx context.Context, in *Migrate_Request, opts ...grpc.CallOption) (*Migrate_Response, error) + // Write resources + Write(ctx context.Context, opts ...grpc.CallOption) (Destination_WriteClient, error) } type destinationClient struct { @@ -39,6 +41,15 @@ func NewDestinationClient(cc grpc.ClientConnInterface) DestinationClient { return &destinationClient{cc} } +func (c *destinationClient) GetExampleConfig(ctx context.Context, in *GetExampleConfig_Request, opts ...grpc.CallOption) (*GetExampleConfig_Response, error) { + out := new(GetExampleConfig_Response) + err := c.cc.Invoke(ctx, "/proto.Destination/GetExampleConfig", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *destinationClient) Configure(ctx context.Context, in *Configure_Request, opts ...grpc.CallOption) (*Configure_Response, error) { out := new(Configure_Response) err := c.cc.Invoke(ctx, "/proto.Destination/Configure", in, out, opts...) @@ -48,69 +59,61 @@ func (c *destinationClient) Configure(ctx context.Context, in *Configure_Request return out, nil } -func (c *destinationClient) GetExampleConfig(ctx context.Context, in *GetExampleConfig_Request, opts ...grpc.CallOption) (*GetExampleConfig_Response, error) { - out := new(GetExampleConfig_Response) - err := c.cc.Invoke(ctx, "/proto.Destination/GetExampleConfig", in, out, opts...) +func (c *destinationClient) Migrate(ctx context.Context, in *Migrate_Request, opts ...grpc.CallOption) (*Migrate_Response, error) { + out := new(Migrate_Response) + err := c.cc.Invoke(ctx, "/proto.Destination/Migrate", in, out, opts...) if err != nil { return nil, err } return out, nil } -func (c *destinationClient) Save(ctx context.Context, opts ...grpc.CallOption) (Destination_SaveClient, error) { - stream, err := c.cc.NewStream(ctx, &Destination_ServiceDesc.Streams[0], "/proto.Destination/Save", opts...) +func (c *destinationClient) Write(ctx context.Context, opts ...grpc.CallOption) (Destination_WriteClient, error) { + stream, err := c.cc.NewStream(ctx, &Destination_ServiceDesc.Streams[0], "/proto.Destination/Write", opts...) if err != nil { return nil, err } - x := &destinationSaveClient{stream} + x := &destinationWriteClient{stream} return x, nil } -type Destination_SaveClient interface { - Send(*Save_Request) error - CloseAndRecv() (*Save_Response, error) +type Destination_WriteClient interface { + Send(*Write_Request) error + CloseAndRecv() (*Write_Response, error) grpc.ClientStream } -type destinationSaveClient struct { +type destinationWriteClient struct { grpc.ClientStream } -func (x *destinationSaveClient) Send(m *Save_Request) error { +func (x *destinationWriteClient) Send(m *Write_Request) error { return x.ClientStream.SendMsg(m) } -func (x *destinationSaveClient) CloseAndRecv() (*Save_Response, error) { +func (x *destinationWriteClient) CloseAndRecv() (*Write_Response, error) { if err := x.ClientStream.CloseSend(); err != nil { return nil, err } - m := new(Save_Response) + m := new(Write_Response) if err := x.ClientStream.RecvMsg(m); err != nil { return nil, err } return m, nil } -func (c *destinationClient) CreateTables(ctx context.Context, in *CreateTables_Request, opts ...grpc.CallOption) (*CreateTables_Response, error) { - out := new(CreateTables_Response) - err := c.cc.Invoke(ctx, "/proto.Destination/CreateTables", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - // DestinationServer is the server API for Destination service. // All implementations must embed UnimplementedDestinationServer // for forward compatibility type DestinationServer interface { - Configure(context.Context, *Configure_Request) (*Configure_Response, error) // Get an example configuration for the source plugin GetExampleConfig(context.Context, *GetExampleConfig_Request) (*GetExampleConfig_Response, error) - // Save resources - Save(Destination_SaveServer) error - // Create tables - CreateTables(context.Context, *CreateTables_Request) (*CreateTables_Response, error) + // Configure the destination plugin with the given credentials and mode + Configure(context.Context, *Configure_Request) (*Configure_Response, error) + // Migrate tables to the given source plugin version + Migrate(context.Context, *Migrate_Request) (*Migrate_Response, error) + // Write resources + Write(Destination_WriteServer) error mustEmbedUnimplementedDestinationServer() } @@ -118,17 +121,17 @@ type DestinationServer interface { type UnimplementedDestinationServer struct { } -func (UnimplementedDestinationServer) Configure(context.Context, *Configure_Request) (*Configure_Response, error) { - return nil, status.Errorf(codes.Unimplemented, "method Configure not implemented") -} func (UnimplementedDestinationServer) GetExampleConfig(context.Context, *GetExampleConfig_Request) (*GetExampleConfig_Response, error) { return nil, status.Errorf(codes.Unimplemented, "method GetExampleConfig not implemented") } -func (UnimplementedDestinationServer) Save(Destination_SaveServer) error { - return status.Errorf(codes.Unimplemented, "method Save not implemented") +func (UnimplementedDestinationServer) Configure(context.Context, *Configure_Request) (*Configure_Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method Configure not implemented") +} +func (UnimplementedDestinationServer) Migrate(context.Context, *Migrate_Request) (*Migrate_Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method Migrate not implemented") } -func (UnimplementedDestinationServer) CreateTables(context.Context, *CreateTables_Request) (*CreateTables_Response, error) { - return nil, status.Errorf(codes.Unimplemented, "method CreateTables not implemented") +func (UnimplementedDestinationServer) Write(Destination_WriteServer) error { + return status.Errorf(codes.Unimplemented, "method Write not implemented") } func (UnimplementedDestinationServer) mustEmbedUnimplementedDestinationServer() {} @@ -143,6 +146,24 @@ func RegisterDestinationServer(s grpc.ServiceRegistrar, srv DestinationServer) { s.RegisterService(&Destination_ServiceDesc, srv) } +func _Destination_GetExampleConfig_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetExampleConfig_Request) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DestinationServer).GetExampleConfig(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/proto.Destination/GetExampleConfig", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DestinationServer).GetExampleConfig(ctx, req.(*GetExampleConfig_Request)) + } + return interceptor(ctx, in, info, handler) +} + func _Destination_Configure_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(Configure_Request) if err := dec(in); err != nil { @@ -161,68 +182,50 @@ func _Destination_Configure_Handler(srv interface{}, ctx context.Context, dec fu return interceptor(ctx, in, info, handler) } -func _Destination_GetExampleConfig_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(GetExampleConfig_Request) +func _Destination_Migrate_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Migrate_Request) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return srv.(DestinationServer).GetExampleConfig(ctx, in) + return srv.(DestinationServer).Migrate(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/proto.Destination/GetExampleConfig", + FullMethod: "/proto.Destination/Migrate", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(DestinationServer).GetExampleConfig(ctx, req.(*GetExampleConfig_Request)) + return srv.(DestinationServer).Migrate(ctx, req.(*Migrate_Request)) } return interceptor(ctx, in, info, handler) } -func _Destination_Save_Handler(srv interface{}, stream grpc.ServerStream) error { - return srv.(DestinationServer).Save(&destinationSaveServer{stream}) +func _Destination_Write_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(DestinationServer).Write(&destinationWriteServer{stream}) } -type Destination_SaveServer interface { - SendAndClose(*Save_Response) error - Recv() (*Save_Request, error) +type Destination_WriteServer interface { + SendAndClose(*Write_Response) error + Recv() (*Write_Request, error) grpc.ServerStream } -type destinationSaveServer struct { +type destinationWriteServer struct { grpc.ServerStream } -func (x *destinationSaveServer) SendAndClose(m *Save_Response) error { +func (x *destinationWriteServer) SendAndClose(m *Write_Response) error { return x.ServerStream.SendMsg(m) } -func (x *destinationSaveServer) Recv() (*Save_Request, error) { - m := new(Save_Request) +func (x *destinationWriteServer) Recv() (*Write_Request, error) { + m := new(Write_Request) if err := x.ServerStream.RecvMsg(m); err != nil { return nil, err } return m, nil } -func _Destination_CreateTables_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(CreateTables_Request) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(DestinationServer).CreateTables(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/proto.Destination/CreateTables", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(DestinationServer).CreateTables(ctx, req.(*CreateTables_Request)) - } - return interceptor(ctx, in, info, handler) -} - // Destination_ServiceDesc is the grpc.ServiceDesc for Destination service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -230,23 +233,23 @@ var Destination_ServiceDesc = grpc.ServiceDesc{ ServiceName: "proto.Destination", HandlerType: (*DestinationServer)(nil), Methods: []grpc.MethodDesc{ - { - MethodName: "Configure", - Handler: _Destination_Configure_Handler, - }, { MethodName: "GetExampleConfig", Handler: _Destination_GetExampleConfig_Handler, }, { - MethodName: "CreateTables", - Handler: _Destination_CreateTables_Handler, + MethodName: "Configure", + Handler: _Destination_Configure_Handler, + }, + { + MethodName: "Migrate", + Handler: _Destination_Migrate_Handler, }, }, Streams: []grpc.StreamDesc{ { - StreamName: "Save", - Handler: _Destination_Save_Handler, + StreamName: "Write", + Handler: _Destination_Write_Handler, ClientStreams: true, }, }, diff --git a/internal/pb/source.pb.go b/internal/pb/source.pb.go index 248c0730a6..00b71e7fc8 100644 --- a/internal/pb/source.pb.go +++ b/internal/pb/source.pb.go @@ -7,10 +7,11 @@ package pb import ( - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" sync "sync" + + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" ) const ( @@ -20,14 +21,14 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -type Fetch struct { +type Sync struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields } -func (x *Fetch) Reset() { - *x = Fetch{} +func (x *Sync) Reset() { + *x = Sync{} if protoimpl.UnsafeEnabled { mi := &file_internal_pb_source_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -35,13 +36,13 @@ func (x *Fetch) Reset() { } } -func (x *Fetch) String() string { +func (x *Sync) String() string { return protoimpl.X.MessageStringOf(x) } -func (*Fetch) ProtoMessage() {} +func (*Sync) ProtoMessage() {} -func (x *Fetch) ProtoReflect() protoreflect.Message { +func (x *Sync) ProtoReflect() protoreflect.Message { mi := &file_internal_pb_source_proto_msgTypes[0] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -53,8 +54,8 @@ func (x *Fetch) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use Fetch.ProtoReflect.Descriptor instead. -func (*Fetch) Descriptor() ([]byte, []int) { +// Deprecated: Use Sync.ProtoReflect.Descriptor instead. +func (*Sync) Descriptor() ([]byte, []int) { return file_internal_pb_source_proto_rawDescGZIP(), []int{0} } @@ -96,14 +97,16 @@ func (*GetTables) Descriptor() ([]byte, []int) { return file_internal_pb_source_proto_rawDescGZIP(), []int{1} } -type Fetch_Request struct { +type Sync_Request struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields + + Spec []byte `protobuf:"bytes,1,opt,name=spec,proto3" json:"spec,omitempty"` } -func (x *Fetch_Request) Reset() { - *x = Fetch_Request{} +func (x *Sync_Request) Reset() { + *x = Sync_Request{} if protoimpl.UnsafeEnabled { mi := &file_internal_pb_source_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -111,13 +114,13 @@ func (x *Fetch_Request) Reset() { } } -func (x *Fetch_Request) String() string { +func (x *Sync_Request) String() string { return protoimpl.X.MessageStringOf(x) } -func (*Fetch_Request) ProtoMessage() {} +func (*Sync_Request) ProtoMessage() {} -func (x *Fetch_Request) ProtoReflect() protoreflect.Message { +func (x *Sync_Request) ProtoReflect() protoreflect.Message { mi := &file_internal_pb_source_proto_msgTypes[2] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -129,12 +132,19 @@ func (x *Fetch_Request) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use Fetch_Request.ProtoReflect.Descriptor instead. -func (*Fetch_Request) Descriptor() ([]byte, []int) { +// Deprecated: Use Sync_Request.ProtoReflect.Descriptor instead. +func (*Sync_Request) Descriptor() ([]byte, []int) { return file_internal_pb_source_proto_rawDescGZIP(), []int{0, 0} } -type Fetch_Response struct { +func (x *Sync_Request) GetSpec() []byte { + if x != nil { + return x.Spec + } + return nil +} + +type Sync_Response struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields @@ -143,8 +153,8 @@ type Fetch_Response struct { Resource []byte `protobuf:"bytes,1,opt,name=resource,proto3" json:"resource,omitempty"` } -func (x *Fetch_Response) Reset() { - *x = Fetch_Response{} +func (x *Sync_Response) Reset() { + *x = Sync_Response{} if protoimpl.UnsafeEnabled { mi := &file_internal_pb_source_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -152,13 +162,13 @@ func (x *Fetch_Response) Reset() { } } -func (x *Fetch_Response) String() string { +func (x *Sync_Response) String() string { return protoimpl.X.MessageStringOf(x) } -func (*Fetch_Response) ProtoMessage() {} +func (*Sync_Response) ProtoMessage() {} -func (x *Fetch_Response) ProtoReflect() protoreflect.Message { +func (x *Sync_Response) ProtoReflect() protoreflect.Message { mi := &file_internal_pb_source_proto_msgTypes[3] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -170,12 +180,12 @@ func (x *Fetch_Response) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use Fetch_Response.ProtoReflect.Descriptor instead. -func (*Fetch_Response) Descriptor() ([]byte, []int) { +// Deprecated: Use Sync_Response.ProtoReflect.Descriptor instead. +func (*Sync_Response) Descriptor() ([]byte, []int) { return file_internal_pb_source_proto_rawDescGZIP(), []int{0, 1} } -func (x *Fetch_Response) GetResource() []byte { +func (x *Sync_Response) GetResource() []byte { if x != nil { return x.Resource } @@ -290,36 +300,33 @@ var file_internal_pb_source_proto_rawDesc = []byte{ 0x0a, 0x18, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x70, 0x62, 0x2f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x05, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x16, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x70, 0x62, 0x2f, 0x62, - 0x61, 0x73, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x3a, 0x0a, 0x05, 0x46, 0x65, 0x74, - 0x63, 0x68, 0x1a, 0x09, 0x0a, 0x07, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x0a, - 0x08, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x72, 0x65, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0x68, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x54, 0x61, 0x62, 0x6c, - 0x65, 0x73, 0x1a, 0x09, 0x0a, 0x07, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x50, 0x0a, - 0x08, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, - 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, - 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x61, 0x62, 0x6c, 0x65, - 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x32, - 0x9b, 0x02, 0x0a, 0x06, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x40, 0x0a, 0x09, 0x47, 0x65, - 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x12, 0x18, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, - 0x47, 0x65, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x19, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x61, 0x62, - 0x6c, 0x65, 0x73, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x55, 0x0a, 0x10, - 0x47, 0x65, 0x74, 0x45, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x12, 0x1f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x78, 0x61, 0x6d, - 0x70, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x20, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x78, 0x61, - 0x6d, 0x70, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x40, 0x0a, 0x09, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, - 0x12, 0x18, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, - 0x72, 0x65, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x2e, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x05, 0x46, 0x65, 0x74, 0x63, 0x68, 0x12, 0x14, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x2e, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46, 0x65, 0x74, - 0x63, 0x68, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x42, 0x05, 0x5a, - 0x03, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x61, 0x73, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x4d, 0x0a, 0x04, 0x53, 0x79, 0x6e, + 0x63, 0x1a, 0x1d, 0x0a, 0x07, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, + 0x73, 0x70, 0x65, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x73, 0x70, 0x65, 0x63, + 0x1a, 0x26, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1a, 0x0a, 0x08, + 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, + 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0x68, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x54, + 0x61, 0x62, 0x6c, 0x65, 0x73, 0x1a, 0x09, 0x0a, 0x07, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x50, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, + 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x61, + 0x62, 0x6c, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x74, 0x61, 0x62, 0x6c, + 0x65, 0x73, 0x32, 0xd6, 0x01, 0x0a, 0x06, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x40, 0x0a, + 0x09, 0x47, 0x65, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x12, 0x18, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x2e, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, + 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x55, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x45, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x12, 0x1f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x45, + 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, + 0x45, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x33, 0x0a, 0x04, 0x53, 0x79, 0x6e, 0x63, 0x12, 0x13, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x2e, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x79, 0x6e, 0x63, + 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x42, 0x05, 0x5a, 0x03, 0x2f, + 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -336,28 +343,24 @@ func file_internal_pb_source_proto_rawDescGZIP() []byte { var file_internal_pb_source_proto_msgTypes = make([]protoimpl.MessageInfo, 6) var file_internal_pb_source_proto_goTypes = []interface{}{ - (*Fetch)(nil), // 0: proto.Fetch + (*Sync)(nil), // 0: proto.Sync (*GetTables)(nil), // 1: proto.GetTables - (*Fetch_Request)(nil), // 2: proto.Fetch.Request - (*Fetch_Response)(nil), // 3: proto.Fetch.Response + (*Sync_Request)(nil), // 2: proto.Sync.Request + (*Sync_Response)(nil), // 3: proto.Sync.Response (*GetTables_Request)(nil), // 4: proto.GetTables.Request (*GetTables_Response)(nil), // 5: proto.GetTables.Response (*GetExampleConfig_Request)(nil), // 6: proto.GetExampleConfig.Request - (*Configure_Request)(nil), // 7: proto.Configure.Request - (*GetExampleConfig_Response)(nil), // 8: proto.GetExampleConfig.Response - (*Configure_Response)(nil), // 9: proto.Configure.Response + (*GetExampleConfig_Response)(nil), // 7: proto.GetExampleConfig.Response } var file_internal_pb_source_proto_depIdxs = []int32{ 4, // 0: proto.Source.GetTables:input_type -> proto.GetTables.Request 6, // 1: proto.Source.GetExampleConfig:input_type -> proto.GetExampleConfig.Request - 7, // 2: proto.Source.Configure:input_type -> proto.Configure.Request - 2, // 3: proto.Source.Fetch:input_type -> proto.Fetch.Request - 5, // 4: proto.Source.GetTables:output_type -> proto.GetTables.Response - 8, // 5: proto.Source.GetExampleConfig:output_type -> proto.GetExampleConfig.Response - 9, // 6: proto.Source.Configure:output_type -> proto.Configure.Response - 3, // 7: proto.Source.Fetch:output_type -> proto.Fetch.Response - 4, // [4:8] is the sub-list for method output_type - 0, // [0:4] is the sub-list for method input_type + 2, // 2: proto.Source.Sync:input_type -> proto.Sync.Request + 5, // 3: proto.Source.GetTables:output_type -> proto.GetTables.Response + 7, // 4: proto.Source.GetExampleConfig:output_type -> proto.GetExampleConfig.Response + 3, // 5: proto.Source.Sync:output_type -> proto.Sync.Response + 3, // [3:6] is the sub-list for method output_type + 0, // [0:3] is the sub-list for method input_type 0, // [0:0] is the sub-list for extension type_name 0, // [0:0] is the sub-list for extension extendee 0, // [0:0] is the sub-list for field type_name @@ -371,7 +374,7 @@ func file_internal_pb_source_proto_init() { file_internal_pb_base_proto_init() if !protoimpl.UnsafeEnabled { file_internal_pb_source_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Fetch); i { + switch v := v.(*Sync); i { case 0: return &v.state case 1: @@ -395,7 +398,7 @@ func file_internal_pb_source_proto_init() { } } file_internal_pb_source_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Fetch_Request); i { + switch v := v.(*Sync_Request); i { case 0: return &v.state case 1: @@ -407,7 +410,7 @@ func file_internal_pb_source_proto_init() { } } file_internal_pb_source_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Fetch_Response); i { + switch v := v.(*Sync_Response); i { case 0: return &v.state case 1: diff --git a/internal/pb/source.proto b/internal/pb/source.proto index 6846fddcc1..a08f87a50d 100644 --- a/internal/pb/source.proto +++ b/internal/pb/source.proto @@ -9,14 +9,13 @@ service Source { rpc GetTables(GetTables.Request) returns (GetTables.Response); // Get an example configuration for the source plugin rpc GetExampleConfig(GetExampleConfig.Request) returns (GetExampleConfig.Response); - // Configure the source plugin with the given spec - rpc Configure(Configure.Request) returns (Configure.Response); // Fetch resources - rpc Fetch(Fetch.Request) returns (stream Fetch.Response); + rpc Sync(Sync.Request) returns (stream Sync.Response); } -message Fetch { +message Sync { message Request { + bytes spec = 1; } message Response { // marshalled *schema.Resources diff --git a/internal/pb/source_grpc.pb.go b/internal/pb/source_grpc.pb.go index 22aa170c0b..07d4670883 100644 --- a/internal/pb/source_grpc.pb.go +++ b/internal/pb/source_grpc.pb.go @@ -8,6 +8,7 @@ package pb import ( context "context" + grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" @@ -26,10 +27,8 @@ type SourceClient interface { GetTables(ctx context.Context, in *GetTables_Request, opts ...grpc.CallOption) (*GetTables_Response, error) // Get an example configuration for the source plugin GetExampleConfig(ctx context.Context, in *GetExampleConfig_Request, opts ...grpc.CallOption) (*GetExampleConfig_Response, error) - // Configure the source plugin with the given spec - Configure(ctx context.Context, in *Configure_Request, opts ...grpc.CallOption) (*Configure_Response, error) // Fetch resources - Fetch(ctx context.Context, in *Fetch_Request, opts ...grpc.CallOption) (Source_FetchClient, error) + Sync(ctx context.Context, in *Sync_Request, opts ...grpc.CallOption) (Source_SyncClient, error) } type sourceClient struct { @@ -58,21 +57,12 @@ func (c *sourceClient) GetExampleConfig(ctx context.Context, in *GetExampleConfi return out, nil } -func (c *sourceClient) Configure(ctx context.Context, in *Configure_Request, opts ...grpc.CallOption) (*Configure_Response, error) { - out := new(Configure_Response) - err := c.cc.Invoke(ctx, "/proto.Source/Configure", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *sourceClient) Fetch(ctx context.Context, in *Fetch_Request, opts ...grpc.CallOption) (Source_FetchClient, error) { - stream, err := c.cc.NewStream(ctx, &Source_ServiceDesc.Streams[0], "/proto.Source/Fetch", opts...) +func (c *sourceClient) Sync(ctx context.Context, in *Sync_Request, opts ...grpc.CallOption) (Source_SyncClient, error) { + stream, err := c.cc.NewStream(ctx, &Source_ServiceDesc.Streams[0], "/proto.Source/Sync", opts...) if err != nil { return nil, err } - x := &sourceFetchClient{stream} + x := &sourceSyncClient{stream} if err := x.ClientStream.SendMsg(in); err != nil { return nil, err } @@ -82,17 +72,17 @@ func (c *sourceClient) Fetch(ctx context.Context, in *Fetch_Request, opts ...grp return x, nil } -type Source_FetchClient interface { - Recv() (*Fetch_Response, error) +type Source_SyncClient interface { + Recv() (*Sync_Response, error) grpc.ClientStream } -type sourceFetchClient struct { +type sourceSyncClient struct { grpc.ClientStream } -func (x *sourceFetchClient) Recv() (*Fetch_Response, error) { - m := new(Fetch_Response) +func (x *sourceSyncClient) Recv() (*Sync_Response, error) { + m := new(Sync_Response) if err := x.ClientStream.RecvMsg(m); err != nil { return nil, err } @@ -107,10 +97,8 @@ type SourceServer interface { GetTables(context.Context, *GetTables_Request) (*GetTables_Response, error) // Get an example configuration for the source plugin GetExampleConfig(context.Context, *GetExampleConfig_Request) (*GetExampleConfig_Response, error) - // Configure the source plugin with the given spec - Configure(context.Context, *Configure_Request) (*Configure_Response, error) // Fetch resources - Fetch(*Fetch_Request, Source_FetchServer) error + Sync(*Sync_Request, Source_SyncServer) error mustEmbedUnimplementedSourceServer() } @@ -124,11 +112,8 @@ func (UnimplementedSourceServer) GetTables(context.Context, *GetTables_Request) func (UnimplementedSourceServer) GetExampleConfig(context.Context, *GetExampleConfig_Request) (*GetExampleConfig_Response, error) { return nil, status.Errorf(codes.Unimplemented, "method GetExampleConfig not implemented") } -func (UnimplementedSourceServer) Configure(context.Context, *Configure_Request) (*Configure_Response, error) { - return nil, status.Errorf(codes.Unimplemented, "method Configure not implemented") -} -func (UnimplementedSourceServer) Fetch(*Fetch_Request, Source_FetchServer) error { - return status.Errorf(codes.Unimplemented, "method Fetch not implemented") +func (UnimplementedSourceServer) Sync(*Sync_Request, Source_SyncServer) error { + return status.Errorf(codes.Unimplemented, "method Sync not implemented") } func (UnimplementedSourceServer) mustEmbedUnimplementedSourceServer() {} @@ -179,42 +164,24 @@ func _Source_GetExampleConfig_Handler(srv interface{}, ctx context.Context, dec return interceptor(ctx, in, info, handler) } -func _Source_Configure_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(Configure_Request) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SourceServer).Configure(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/proto.Source/Configure", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SourceServer).Configure(ctx, req.(*Configure_Request)) - } - return interceptor(ctx, in, info, handler) -} - -func _Source_Fetch_Handler(srv interface{}, stream grpc.ServerStream) error { - m := new(Fetch_Request) +func _Source_Sync_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(Sync_Request) if err := stream.RecvMsg(m); err != nil { return err } - return srv.(SourceServer).Fetch(m, &sourceFetchServer{stream}) + return srv.(SourceServer).Sync(m, &sourceSyncServer{stream}) } -type Source_FetchServer interface { - Send(*Fetch_Response) error +type Source_SyncServer interface { + Send(*Sync_Response) error grpc.ServerStream } -type sourceFetchServer struct { +type sourceSyncServer struct { grpc.ServerStream } -func (x *sourceFetchServer) Send(m *Fetch_Response) error { +func (x *sourceSyncServer) Send(m *Sync_Response) error { return x.ServerStream.SendMsg(m) } @@ -233,15 +200,11 @@ var Source_ServiceDesc = grpc.ServiceDesc{ MethodName: "GetExampleConfig", Handler: _Source_GetExampleConfig_Handler, }, - { - MethodName: "Configure", - Handler: _Source_Configure_Handler, - }, }, Streams: []grpc.StreamDesc{ { - StreamName: "Fetch", - Handler: _Source_Fetch_Handler, + StreamName: "Sync", + Handler: _Source_Sync_Handler, ServerStreams: true, }, }, diff --git a/internal/servers/destinations.go b/internal/servers/destinations.go index cc96e81004..d9ec38a7d5 100644 --- a/internal/servers/destinations.go +++ b/internal/servers/destinations.go @@ -2,6 +2,7 @@ package servers import ( "context" + "encoding/json" "fmt" "io" @@ -11,7 +12,6 @@ import ( "github.com/cloudquery/plugin-sdk/specs" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "gopkg.in/yaml.v3" ) type DestinationServer struct { @@ -20,34 +20,34 @@ type DestinationServer struct { } func (s *DestinationServer) Configure(ctx context.Context, req *pb.Configure_Request) (*pb.Configure_Response, error) { - var spec specs.DestinationSpec - if err := yaml.Unmarshal(req.Config, &spec); err != nil { + var spec specs.Destination + if err := json.Unmarshal(req.Config, &spec); err != nil { return nil, status.Errorf(codes.InvalidArgument, "failed to unmarshal spec: %v", err) } - return &pb.Configure_Response{}, s.Plugin.Configure(ctx, spec) + return &pb.Configure_Response{}, s.Plugin.Initialize(ctx, spec) } func (s *DestinationServer) GetExampleConfig(ctx context.Context, req *pb.GetExampleConfig_Request) (*pb.GetExampleConfig_Response, error) { return &pb.GetExampleConfig_Response{ - Config: s.Plugin.GetExampleConfig(ctx), + Config: s.Plugin.ExampleConfig(), }, nil } -func (s *DestinationServer) Save(msg pb.Destination_SaveServer) error { +func (s *DestinationServer) Write(msg pb.Destination_WriteServer) error { for { r, err := msg.Recv() if err != nil { if err == io.EOF { return nil } - return fmt.Errorf("Save: failed to receive msg: %w", err) + return fmt.Errorf("write: failed to receive msg: %w", err) } - var resources []*schema.Resource - if err := yaml.Unmarshal(r.Resources, &resources); err != nil { + var resource *schema.Resource + if err := json.Unmarshal(r.Resource, &resource); err != nil { return status.Errorf(codes.InvalidArgument, "failed to unmarshal spec: %v", err) } - if err := s.Plugin.Save(context.Background(), resources); err != nil { - return fmt.Errorf("Save: failed to save resources: %w", err) + if err := s.Plugin.Write(context.Background(), resource.TableName, resource.Data); err != nil { + return fmt.Errorf("write: failed to write resource: %w", err) } } } diff --git a/internal/servers/source.go b/internal/servers/source.go index ee9200fbd6..b22de4fab8 100644 --- a/internal/servers/source.go +++ b/internal/servers/source.go @@ -1,15 +1,16 @@ package servers import ( + "bytes" "context" + "encoding/json" + "fmt" "github.com/cloudquery/plugin-sdk/internal/pb" "github.com/cloudquery/plugin-sdk/plugins" "github.com/cloudquery/plugin-sdk/schema" "github.com/cloudquery/plugin-sdk/specs" "github.com/pkg/errors" - "github.com/vmihailenco/msgpack/v5" - "gopkg.in/yaml.v3" ) type SourceServer struct { @@ -18,7 +19,7 @@ type SourceServer struct { } func (s *SourceServer) GetTables(context.Context, *pb.GetTables_Request) (*pb.GetTables_Response, error) { - b, err := msgpack.Marshal(s.Plugin.Tables) + b, err := json.Marshal(s.Plugin.Tables()) if err != nil { return nil, errors.Wrap(err, "failed to marshal tables") } @@ -28,46 +29,41 @@ func (s *SourceServer) GetTables(context.Context, *pb.GetTables_Request) (*pb.Ge } func (s *SourceServer) GetExampleConfig(context.Context, *pb.GetExampleConfig_Request) (*pb.GetExampleConfig_Response, error) { - return &pb.GetExampleConfig_Response{ - Name: s.Plugin.Name, - Version: s.Plugin.Version, - Config: s.Plugin.ExampleConfig}, nil -} - -func (s *SourceServer) Configure(ctx context.Context, req *pb.Configure_Request) (*pb.Configure_Response, error) { - var spec specs.SourceSpec - if err := yaml.Unmarshal(req.Config, &spec); err != nil { - return nil, errors.Wrap(err, "failed to unmarshal config") - } - jsonschemaResult, err := s.Plugin.Init(ctx, spec) - if err != nil { - return nil, errors.Wrap(err, "failed to configure source") - } - b, err := msgpack.Marshal(jsonschemaResult) + exampleConfig, err := s.Plugin.ExampleConfig() if err != nil { - return nil, errors.Wrap(err, "failed to marshal json schema result") + return nil, fmt.Errorf("failed to get example config: %w", err) } - return &pb.Configure_Response{ - JsonschemaResult: b, - }, nil + return &pb.GetExampleConfig_Response{ + Name: s.Plugin.Name(), + Version: s.Plugin.Version(), + Config: exampleConfig}, nil } -func (s *SourceServer) Fetch(req *pb.Fetch_Request, stream pb.Source_FetchServer) error { +func (s *SourceServer) Sync(req *pb.Sync_Request, stream pb.Source_SyncServer) error { resources := make(chan *schema.Resource) var fetchErr error + + var spec specs.Source + dec := json.NewDecoder(bytes.NewReader(req.Spec)) + dec.UseNumber() + dec.DisallowUnknownFields() + if err := dec.Decode(&spec); err != nil { + return fmt.Errorf("failed to decode source spec: %w", err) + } + go func() { defer close(resources) - if err := s.Plugin.Fetch(stream.Context(), resources); err != nil { + if err := s.Plugin.Sync(stream.Context(), spec, resources); err != nil { fetchErr = errors.Wrap(err, "failed to fetch resources") } }() for resource := range resources { - b, err := msgpack.Marshal(resource) + b, err := json.Marshal(resource) if err != nil { return errors.Wrap(err, "failed to marshal resource") } - if err := stream.Send(&pb.Fetch_Response{ + if err := stream.Send(&pb.Sync_Response{ Resource: b, }); err != nil { return errors.Wrap(err, "failed to send resource") diff --git a/plugins/destination.go b/plugins/destination.go index 87b9a2128d..65283aaa41 100644 --- a/plugins/destination.go +++ b/plugins/destination.go @@ -8,13 +8,65 @@ import ( "github.com/rs/zerolog" ) -type DestinationPluginOptions struct { - Logger zerolog.Logger -} +// type DestinationOption func(*DestinationPlugin) + +// type WriteFunc func(ctx context.Context, spec specs.DestinationSpec, tables []*schema.Table, resources <-chan *schema.Resource) error + +// type DestinationPlugin struct { +// // Name of the plugin. +// name string +// // Version is the version of the plugin. +// version string +// // JsonSchema for specific source plugin spec +// jsonSchema string +// // ExampleConfig is the example configuration for this plugin +// exampleConfig string +// // Logger to call, this logger is passed to the serve.Serve Client, if not define Serve will create one instead. +// logger zerolog.Logger +// // Write is a function that get a stream of resources and write them to the configured destination +// // with the configured mode. +// Write WriteFunc +// } type DestinationPlugin interface { - Configure(ctx context.Context, spec specs.DestinationSpec) error - CreateTables(ctx context.Context, table []*schema.Table) error - Save(ctx context.Context, resources []*schema.Resource) error - GetExampleConfig(ctx context.Context) string + Name() string + Version() string + ExampleConfig() string + Initialize(ctx context.Context, spec specs.Destination) error + Migrate(ctx context.Context, tables schema.Tables) error + Write(ctx context.Context, table string, data map[string]interface{}) error + SetLogger(logger zerolog.Logger) } + +// func WithExampleConfig(exampleConfig string) DestinationOption { +// return func(p *DestinationPlugin) { +// p.exampleConfig = exampleConfig +// } +// } + +// func WithJsonSchema(jsonSchema string) DestinationOption { +// return func(p *DestinationPlugin) { +// p.jsonSchema = jsonSchema +// } +// } + +// func WithLogger(logger zerolog.Logger) DestinationOption { +// return func(p *DestinationPlugin) { +// p.logger = logger +// } +// } + +// func NewDestinationClient(name string, version string, writeFunc WriteFunc, opts ...DestinationOption) *DestinationPlugin { +// p := DestinationPlugin{ +// name: name, +// version: version, +// } +// for _, opt := range opts { +// opt(&p) +// } +// return &p +// } + +// func (p *DestinationPlugin) GetExampleConfig() string { +// return p.exampleConfig +// } diff --git a/plugins/source.go b/plugins/source.go index a882540186..72c991fe6c 100644 --- a/plugins/source.go +++ b/plugins/source.go @@ -2,147 +2,208 @@ package plugins import ( "context" - _ "embed" "fmt" "sync" "time" - "github.com/cloudquery/plugin-sdk/helpers" - "github.com/cloudquery/plugin-sdk/helpers/limit" "github.com/cloudquery/plugin-sdk/schema" "github.com/cloudquery/plugin-sdk/specs" "github.com/rs/zerolog" "github.com/thoas/go-funk" - "github.com/xeipuuv/gojsonschema" - "golang.org/x/sync/semaphore" ) +type SourceNewExecutionClientFunc func(context.Context, *SourcePlugin, specs.Source) (schema.ClientMeta, error) + // SourcePlugin is the base structure required to pass to sdk.serve // We take a similar/declerative approach to API here similar to Cobra type SourcePlugin struct { // Name of plugin i.e aws,gcp, azure etc' - Name string + name string // Version of the plugin - Version string + version string + // Classify error and return it's severity and type + ignoreError schema.IgnoreErrorFunc // Called upon configure call to validate and init configuration - Configure func(context.Context, *SourcePlugin, specs.SourceSpec) (schema.ClientMeta, error) + newExecutionClient SourceNewExecutionClientFunc // Tables is all tables supported by this source plugin - Tables []*schema.Table - // JsonSchema for specific source plugin spec - JsonSchema string - // ExampleConfig is the example configuration for this plugin - ExampleConfig string + tables schema.Tables + // exampleConfig + exampleConfig string // Logger to call, this logger is passed to the serve.Serve Client, if not define Serve will create one instead. - Logger zerolog.Logger - - // Internal fields set by configure - clientMeta schema.ClientMeta - spec *specs.SourceSpec + logger zerolog.Logger } +type SourceOption func(*SourcePlugin) + const ExampleSourceConfig = ` # max_goroutines to use when fetching. 0 means default and calculated by CloudQuery # max_goroutines: 0 -# By default cloudquery will fetch all tables in the source plugin +# By default CloudQuery will fetch all tables in the source plugin # tables: ["*"] # skip_tables specify which tables to skip. especially useful when using "*" for tables # skip_tables: [] ` -//go:embed source_schema.json -var sourceSchema string +const minGoRoutines = 5 -func (p *SourcePlugin) Init(ctx context.Context, spec specs.SourceSpec) (*gojsonschema.Result, error) { - res, err := specs.ValidateSpec(sourceSchema, spec) - if err != nil { - return nil, err +func WithSourceExampleConfig(exampleConfig string) SourceOption { + return func(p *SourcePlugin) { + p.exampleConfig = exampleConfig } - if !res.Valid() { - return res, nil +} + +func WithSourceLogger(logger zerolog.Logger) SourceOption { + return func(p *SourcePlugin) { + p.logger = logger } - if p.Configure == nil { - return nil, fmt.Errorf("configure function not defined") +} + +func WithClassifyError(ignoreError schema.IgnoreErrorFunc) SourceOption { + return func(p *SourcePlugin) { + p.ignoreError = ignoreError } - p.clientMeta, err = p.Configure(ctx, p, spec) - if err != nil { - return res, fmt.Errorf("failed to configure source plugin: %w", err) +} + +// Add internal columns +func addInternalColumns(tables []*schema.Table) { + for _, table := range tables { + cqId := schema.CqIdColumn + if len(table.PrimaryKeys()) == 0 { + cqId.CreationOptions.PrimaryKey = true + } + table.Columns = append(table.Columns, cqId, schema.CqFetchTime) + addInternalColumns(table.Relations) + } +} + +func NewSourcePlugin(name string, version string, tables []*schema.Table, newExecutionClient SourceNewExecutionClientFunc, opts ...SourceOption) *SourcePlugin { + p := SourcePlugin{ + name: name, + version: version, + tables: tables, + newExecutionClient: newExecutionClient, } - p.spec = &spec - return res, nil + for _, opt := range opts { + opt(&p) + } + addInternalColumns(p.tables) + if err := p.validate(); err != nil { + panic(err) + } + // add default columns to tables + return &p } -// Fetch fetches data according to source configuration and -func (p *SourcePlugin) Fetch(ctx context.Context, res chan<- *schema.Resource) error { - if p.spec == nil { - return fmt.Errorf("source plugin not initialized") +func (p *SourcePlugin) validate() error { + if p.newExecutionClient == nil { + return fmt.Errorf("newExecutionClient function not defined for source plugin:" + p.name) } - // if resources ["*"] is requested we will fetch all resources - tables, err := p.interpolateAllResources(p.spec.Tables) + + return p.tables.ValidateDuplicateColumns() +} + +func (p *SourcePlugin) Tables() schema.Tables { + return p.tables +} + +func (p *SourcePlugin) ExampleConfig() (string, error) { + return p.exampleConfig, nil +} + +func (p *SourcePlugin) Name() string { + return p.name +} + +func (p *SourcePlugin) Version() string { + return p.version +} + +func (p *SourcePlugin) SetLogger(log zerolog.Logger) { + p.logger = log +} + +// Sync data from source to the given channel +func (p *SourcePlugin) Sync(ctx context.Context, spec specs.Source, res chan<- *schema.Resource) error { + c, err := p.newExecutionClient(ctx, p, spec) if err != nil { - return fmt.Errorf("failed to interpolate resources: %w", err) + return fmt.Errorf("failed to create execution client for source plugin %s: %w", p.name, err) } // limiter used to limit the amount of resources fetched concurrently - maxGoroutines := p.spec.MaxGoRoutines + maxGoroutines := spec.MaxGoRoutines if maxGoroutines == 0 { - maxGoroutines = limit.GetMaxGoRoutines() + maxGoroutines = minGoRoutines } - p.Logger.Info().Uint64("max_goroutines", maxGoroutines).Msg("starting fetch") - goroutinesSem := semaphore.NewWeighted(helpers.Uint64ToInt64(maxGoroutines)) + p.logger.Info().Uint64("max_goroutines", maxGoroutines).Msg("starting fetch") + + // goroutinesSem := semaphore.NewWeighted(helpers.Uint64ToInt64(maxGoroutines)) w := sync.WaitGroup{} - for _, table := range p.Tables { + totalResources := 0 + startTime := time.Now() + tableNames, err := p.interpolateAllResources(p.tables.TableNames()) + if err != nil { + return err + } + + // this is the same fetchtime for all resources + fetchTime := time.Now() + + for _, table := range p.tables { table := table - if funk.ContainsString(p.spec.SkipTables, table.Name) || !funk.ContainsString(tables, table.Name) { - p.Logger.Info().Str("table", table.Name).Msg("skipping table") + if funk.ContainsString(spec.SkipTables, table.Name) || !funk.ContainsString(tableNames, table.Name) { + p.logger.Info().Str("table", table.Name).Msg("skipping table") continue } - clients := []schema.ClientMeta{p.clientMeta} + clients := []schema.ClientMeta{c} if table.Multiplex != nil { - clients = table.Multiplex(p.clientMeta) + clients = table.Multiplex(c) + } + // because table can't import sourceplugin we need to set classifyError if it is not set by table + if table.IgnoreError == nil { + table.IgnoreError = p.ignoreError } // we call this here because we dont know when the following goroutine will be called and we do want an order // of table by table - totalClients := len(clients) - newN := helpers.TryAcquireMax(goroutinesSem, int64(totalClients)) + // totalClients := len(clients) + // newN, err := helpers.TryAcquireMax(ctx, goroutinesSem, int64(totalClients)) + // if err != nil { + // p.logger.Error().Err(err).Msg("failed to TryAcquireMax semaphore. exiting") + // break + // } // goroutinesSem.TryAcquire() w.Add(1) go func() { defer w.Done() - defer goroutinesSem.Release(int64(totalClients) - newN) wg := sync.WaitGroup{} - p.Logger.Info().Str("table", table.Name).Msg("fetch start") - startTime := time.Now() - for i, client := range clients { + p.logger.Info().Str("table", table.Name).Msg("fetch start") + tableStartTime := time.Now() + totalTableResources := 0 + for _, client := range clients { client := client - i := i - // acquire semaphore only if we couldn't acquire it earlier - if newN > 0 && i >= (totalClients-int(newN)) { - if err := goroutinesSem.Acquire(ctx, 1); err != nil { - // this can happen if context was cancelled so we just break out of the loop - p.Logger.Error().Err(err).Msg("failed to acquire semaphore") - return - } - } + + // i := i wg.Add(1) go func() { defer wg.Done() - if newN > 0 && i >= (totalClients-int(newN)) { - defer goroutinesSem.Release(1) - } - table.Resolve(ctx, client, nil, res) + // defer goroutinesSem.Release(1) + totalTableResources += table.Resolve(ctx, client, fetchTime, nil, res) }() } wg.Wait() - p.Logger.Info().Str("table", table.Name).TimeDiff("duration", time.Now(), startTime).Msg("fetch finished") + totalResources += totalTableResources + p.logger.Info().Str("table", table.Name).Int("total_resources", totalTableResources).TimeDiff("duration", time.Now(), tableStartTime).Msg("fetch table finished") }() } w.Wait() - + p.logger.Info().Int("total_resources", totalResources).TimeDiff("duration", time.Now(), startTime).Msg("fetch finished") return nil } func (p *SourcePlugin) interpolateAllResources(tables []string) ([]string, error) { + if tables == nil { + return make([]string, 0), nil + } if !funk.ContainsString(tables, "*") { return tables, nil } @@ -151,8 +212,8 @@ func (p *SourcePlugin) interpolateAllResources(tables []string) ([]string, error return nil, fmt.Errorf("invalid \"*\" resource, with explicit resources") } - allResources := make([]string, 0, len(p.Tables)) - for _, k := range p.Tables { + allResources := make([]string, 0, len(p.tables)) + for _, k := range p.tables { allResources = append(allResources, k.Name) } return allResources, nil diff --git a/plugins/source_test.go b/plugins/source_test.go new file mode 100644 index 0000000000..06b1743348 --- /dev/null +++ b/plugins/source_test.go @@ -0,0 +1,108 @@ +package plugins + +import ( + "context" + "fmt" + "testing" + + "github.com/cloudquery/plugin-sdk/schema" + "github.com/cloudquery/plugin-sdk/specs" + "github.com/rs/zerolog" + "golang.org/x/sync/errgroup" +) + +type testExecutionClient struct { + logger zerolog.Logger +} + +type Account struct { + Name string `json:"name,omitempty"` + Regions []string `json:"regions,omitempty"` +} + +// type testSourceSpec struct { +// Accounts []Account `json:"accounts,omitempty"` +// Regions []string `json:"regions,omitempty"` +// } + +// func newTestSourceSpec() interface{} { +// return &testSourceSpec{} +// } + +var _ schema.ClientMeta = &testExecutionClient{} + +func testTable() *schema.Table { + return &schema.Table{ + Name: "testTable", + Resolver: func(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- interface{}) error { + res <- map[string]interface{}{ + "TestColumn": 3, + } + return nil + }, + Columns: []schema.Column{ + { + Name: "test_column", + Type: schema.TypeInt, + }, + }, + } +} + +func (c *testExecutionClient) Logger() *zerolog.Logger { + return &c.logger +} + +func newTestExecutionClient(context.Context, *SourcePlugin, specs.Source) (schema.ClientMeta, error) { + return &testExecutionClient{}, nil +} + +func TestSync(t *testing.T) { + ctx := context.Background() + plugin := NewSourcePlugin( + "testSourcePlugin", + "1.0.0", + []*schema.Table{testTable()}, + newTestExecutionClient, + WithSourceLogger(zerolog.New(zerolog.NewTestWriter(t))), + ) + + // test round trip: get example config -> sync with example config -> success + exampleConfig, err := plugin.ExampleConfig() + if err != nil { + t.Fatal(err) + } + fmt.Println(exampleConfig) + var spec specs.Spec + if err := specs.SpecUnmarshalYamlStrict([]byte(exampleConfig), &spec); err != nil { + t.Fatal(err) + } + + resources := make(chan *schema.Resource) + g, ctx := errgroup.WithContext(ctx) + g.Go(func() error { + defer close(resources) + err = plugin.Sync(ctx, + *spec.Spec.(*specs.Source), + resources) + return err + }) + + for resource := range resources { + if resource.Table.Name != "testTable" { + t.Fatalf("unexpected resource table name: %s", resource.Table.Name) + } + obj := resource.Get("test_column") + val, ok := obj.(int) + if !ok { + t.Fatalf("unexpected resource column value (expected int): %v", obj) + } + + if val != 3 { + t.Fatalf("unexpected resource column value: %v", val) + } + } + if err := g.Wait(); err != nil { + t.Fatal(err) + } +} diff --git a/plugins/source_test.go.backup b/plugins/source_test.go.backup deleted file mode 100644 index 632fd163cc..0000000000 --- a/plugins/source_test.go.backup +++ /dev/null @@ -1,89 +0,0 @@ -package plugins - -import ( - "context" - "os" - "testing" - - "github.com/cloudquery/plugin-sdk/schema" - "github.com/rs/zerolog" - "github.com/xeipuuv/gojsonschema" -) - -type Account struct { - Name string `yaml:"name"` - Regions []string `yaml:"regions"` -} - -type TestConfig struct { - Accounts []Account `yaml:"accounts"` - Regions []string `yaml:"regions"` -} - -func (TestConfig) Example() string { - return "" -} - -type testSourcePluginClient struct { - logger zerolog.Logger -} - -func (t testSourcePluginClient) Logger() *zerolog.Logger { - return &t.logger -} - -var testSourcePlugin = SourcePlugin{ - Name: "testSourcePlugin", - Version: "1.0.0", - Configure: func(l zerolog.Logger, i interface{}) (schema.ClientMeta, error) { - return testSourcePluginClient{logger: l}, nil - }, - Tables: []*schema.Table{ - { - Name: "testTable", - Resolver: func(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- interface{}) error { - res <- map[string]interface{}{ - "testColumn": 3, - } - return nil - }, - Columns: []schema.Column{ - { - Name: "testColumn", - Type: schema.TypeInt, - }, - }, - }, - }, - Config: func() interface{} { - return &TestConfig{} - }, - Logger: zerolog.New(os.Stderr), -} - -func TestFetch(t *testing.T) { - cfg := ` -tables: - - "*" -configuration: - regions: - - "us-east-1" - accounts: - - name: "testAccount" - regions: - - "us-east-2" -` - resources := make(chan *schema.Resource) - var fetchErr error - var result *gojsonschema.Result - go func() { - defer close(resources) - result, fetchErr = testSourcePlugin.Fetch(context.Background(), []byte(cfg), resources) - }() - for resource := range resources { - t.Logf("%+v", resource) - } - if fetchErr != nil { - t.Errorf("fetch error: %v", fetchErr) - } -} diff --git a/plugins/source_testing.go b/plugins/source_testing.go index b7e99590d2..74ba12fd0d 100644 --- a/plugins/source_testing.go +++ b/plugins/source_testing.go @@ -8,15 +8,11 @@ import ( "github.com/cloudquery/plugin-sdk/schema" "github.com/cloudquery/plugin-sdk/specs" "github.com/georgysavva/scany/pgxscan" - "github.com/xeipuuv/gojsonschema" - "gopkg.in/yaml.v3" ) type ResourceTestCase struct { Plugin *SourcePlugin - Config string - // we want it to be parallel by default - NotParallel bool + Spec specs.Source // ParallelFetchingLimit limits parallel resources fetch at a time ParallelFetchingLimit uint64 // SkipIgnoreInTest flag which detects if schema.Table or schema.Column should be ignored @@ -37,10 +33,9 @@ func init() { // type -func TestResource(t *testing.T, tc ResourceTestCase) { +func TestSourcePluginSync(t *testing.T, plugin *SourcePlugin, spec specs.Source) { t.Parallel() t.Helper() - // No need for configuration or db connection, get it out of the way first // testTableIdentifiersForProvider(t, resource.Provider) @@ -49,33 +44,24 @@ func TestResource(t *testing.T, tc ResourceTestCase) { // resource.Plugin.Logger = l resources := make(chan *schema.Resource) var fetchErr error - var result *gojsonschema.Result - var spec specs.SourceSpec - if err := yaml.Unmarshal([]byte(tc.Config), &spec); err != nil { - t.Fatal("failed to unmarshal source spec:", err) - } - validationResult, err := tc.Plugin.Init(context.Background(), spec) - if err != nil { - t.Fatal("failed to init plugin:", err) - } - if !validationResult.Valid() { - t.Fatal("failed to validate plugin config:", validationResult.Errors()) - } // tc.Plugin.Logger = zerolog.New(zerolog.NewTestWriter(t)) go func() { defer close(resources) - fetchErr = tc.Plugin.Fetch(context.Background(), resources) + fetchErr = plugin.Sync(context.Background(), spec, resources) }() + totalResources := 0 for resource := range resources { + totalResources++ validateResource(t, resource) } + if totalResources == 0 { + t.Fatal("no resources fetched") + } + if fetchErr != nil { t.Fatal(fetchErr) } - if result != nil && !result.Valid() { - t.Errorf("invalid schema: %v", result.Errors()) - } } func validateResource(t *testing.T, resource *schema.Resource) { @@ -86,132 +72,3 @@ func validateResource(t *testing.T, resource *schema.Resource) { } } } - -// func testResource(t *testing.T, resource ResourceTestCase, name string, table *schema.Table, conn execution.QueryExecer) error { -// t.Helper() - -// if createErr := dropAndCreateTable(context.Background(), conn, table); createErr != nil { -// assert.FailNow(t, fmt.Sprintf("failed to create table %s", table.Name), createErr) -// } - -// if !resource.SkipIgnoreInTest && table.IgnoreInTests { -// t.Logf("skipping fetch of resource: %s in tests", name) -// } else { -// if err := fetchResource(t, &resource, name); err != nil { -// return err -// } -// } - -// if verifiers, ok := resource.Verifiers[name]; ok { -// for _, verifier := range verifiers { -// verifier(t, table, conn, resource.SkipIgnoreInTest) -// } -// } else { -// // fallback to default verification -// verifyNoEmptyColumns(t, table, conn, resource.SkipIgnoreInTest) -// } - -// return nil -// } - -// // fetchResource - fetches a resource from the cloud and puts them into database. database config can be specified via DATABASE_URL env variable -// func fetchResource(t *testing.T, resource *ResourceTestCase, resourceName string) error { -// t.Helper() - -// t.Logf("fetch resource %v", resourceName) - -// var resourceSender = &testResourceSender{ -// Errors: []string{}, -// } - -// if err := resource.Provider.FetchResources(context.Background(), -// &cqproto.FetchResourcesRequest{ -// Resources: []string{resourceName}, -// ParallelFetchingLimit: resource.ParallelFetchingLimit, -// }, -// resourceSender, -// ); err != nil { -// return err -// } - -// if len(resourceSender.Errors) > 0 { -// return fmt.Errorf("error/s occurred during test, %s", strings.Join(resourceSender.Errors, ", ")) -// } - -// return nil -// } - -// func verifyNoEmptyColumns(t *testing.T, table *schema.Table, conn pgxscan.Querier, shouldSkipIgnoreInTest bool) { -// t.Helper() -// t.Run(table.Name, func(t *testing.T) { -// t.Helper() - -// if !shouldSkipIgnoreInTest && table.IgnoreInTests { -// t.Skipf("table %s marked as IgnoreInTest. Skipping...", table.Name) -// } -// s := sq.StatementBuilder. -// PlaceholderFormat(sq.Dollar). -// Select(fmt.Sprintf("json_agg(%s)", table.Name)). -// From(table.Name) -// query, args, err := s.ToSql() -// if err != nil { -// t.Fatal(err) -// } -// var data []map[string]interface{} -// if err := pgxscan.Get(context.Background(), conn, &data, query, args...); err != nil { -// t.Fatal(err) -// } - -// if len(data) == 0 { -// t.Errorf("expected to have at least 1 entry at table %s got zero", table.Name) -// return -// } - -// nilColumns := map[string]bool{} -// // mark all columns as nil -// for _, c := range table.Columns { -// if shouldSkipIgnoreInTest || !c.IgnoreInTests { -// nilColumns[c.Name] = true -// } -// } - -// for _, row := range data { -// for c, v := range row { -// if v != nil { -// // as long as we had one row or result with this column not nil it means the resolver worked -// nilColumns[c] = false -// } -// } -// } - -// var nilColumnsArr []string -// for c, v := range nilColumns { -// if v { -// nilColumnsArr = append(nilColumnsArr, c) -// } -// } - -// if len(nilColumnsArr) != 0 { -// t.Errorf("found nil column in table %s. columns=%s", table.Name, strings.Join(nilColumnsArr, ",")) -// } -// for _, childTable := range table.Relations { -// verifyNoEmptyColumns(t, childTable, conn, shouldSkipIgnoreInTest) -// } -// }) -// } - -// func (f *testResourceSender) Send(r *cqproto.FetchResourcesResponse) error { -// if r.Error != "" { -// fmt.Printf(r.Error) -// f.Errors = append(f.Errors, r.Error) -// } - -// return nil -// } - -// func getEnv(key, fallback string) string { -// if value, ok := os.LookupEnv(key); ok { -// return value -// } -// return fallback -// } diff --git a/plugins/verifiers.go b/plugins/verifiers.go deleted file mode 100644 index ff118b0040..0000000000 --- a/plugins/verifiers.go +++ /dev/null @@ -1,103 +0,0 @@ -package plugins - -import ( - "context" - "fmt" - "testing" - - "github.com/cloudquery/faker/v3/support/slice" - "github.com/cloudquery/plugin-sdk/schema" - "github.com/georgysavva/scany/pgxscan" -) - -type Row map[string]interface{} - -// VerifyRowPredicateInTable is a base verifier accepting single row verifier for specific table from schema -func VerifyRowPredicateInTable(tableName string, rowVerifier func(*testing.T, Row)) Verifier { - var verifier Verifier - verifier = func(t *testing.T, table *schema.Table, conn pgxscan.Querier, shouldSkipIgnoreInTest bool) { - if tableName == table.Name { - rows := getRows(t, conn, table, shouldSkipIgnoreInTest) - for _, row := range rows { - rowVerifier(t, row) - } - } - for _, r := range table.Relations { - verifier(t, r, conn, shouldSkipIgnoreInTest) - } - } - return verifier -} - -func getRows(t *testing.T, conn pgxscan.Querier, table *schema.Table, shouldSkipIgnoreInTest bool) []Row { - if shouldSkipIgnoreInTest && table.IgnoreInTests { - t.Skipf("table %s marked as IgnoreInTest. Skipping...", table.Name) - } - - var rows []Row - err := pgxscan.Get( - context.Background(), - conn, - &rows, - fmt.Sprintf("select json_agg(%[1]s) from %[1]s", table.Name), - ) - if err != nil { - t.Fatal(err) - } - - for _, c := range table.Columns { - if shouldSkipIgnoreInTest && c.IgnoreInTests { - for _, row := range rows { - delete(row, c.Name) - } - } - } - - return rows -} - -// VerifyNoEmptyColumnsExcept verifies that for each row in table its columns are not empty except passed -func VerifyNoEmptyColumnsExcept(tableName string, except ...string) Verifier { - return VerifyRowPredicateInTable(tableName, func(t *testing.T, row Row) { - for k, v := range row { - if !slice.Contains(except, k) && v == nil { - t.Fatal("VerifyNoEmptyColumnsExcept failed: illegal row found") - } - } - }) -} - -// VerifyAtMostOneOf verifies that for each row in table at most one column from oneof is not empty -func VerifyAtMostOneOf(tableName string, oneof ...string) Verifier { - return VerifyRowPredicateInTable(tableName, func(t *testing.T, row Row) { - cnt := 0 - for _, k := range oneof { - v, ok := row[k] - if !ok { - t.Fatalf("VerifyAtMostOneOf failed: column %s doesn't exist", k) - } - if v != nil { - cnt++ - } - } - - if cnt > 1 { - t.Fatal("VerifyAtMostOneOf failed: illegal row found") - } - }) -} - -// VerifyAtLeastOneRow verifies that main table from schema has at least one row -func VerifyAtLeastOneRow() Verifier { - return func(t *testing.T, table *schema.Table, conn pgxscan.Querier, _ bool) { - rows, err := conn.Query(context.Background(), fmt.Sprintf("select * from %s;", table.Name)) - if err != nil { - t.Fatal(err) - } - if !rows.Next() { - t.Fatal("VerifyAtLeastOneRow failed: table is empty") - } - - rows.Close() - } -} diff --git a/schema/column.go b/schema/column.go index f36d0d6487..6bb5e9a516 100644 --- a/schema/column.go +++ b/schema/column.go @@ -36,28 +36,29 @@ type ColumnResolver func(ctx context.Context, meta ClientMeta, resource *Resourc // ColumnCreationOptions allow modification of how column is defined when table is created type ColumnCreationOptions struct { - Unique bool - NotNull bool + PrimaryKey bool `json:"primary_key,omitempty"` + Unique bool `json:"unique,omitempty"` + NotNull bool `json:"notnull,omitempty"` } // Column definition for Table type Column struct { // Name of column - Name string + Name string `json:"name,omitempty"` // Value Type of column i.e String, UUID etc' - Type ValueType + Type ValueType `json:"type,omitempty"` // Description about column, this description is added as a comment in the database - Description string + Description string `json:"description,omitempty"` // Column Resolver allows to set you own data based on resolving this can be an API call or setting multiple embedded values etc' - Resolver ColumnResolver `msgpack:"-"` + Resolver ColumnResolver `json:"-"` // Creation options allow modifying how column is defined when table is created - CreationOptions ColumnCreationOptions + CreationOptions ColumnCreationOptions `json:"creation_options,omitempty"` // IgnoreInTests is used to skip verifying the column is non-nil in integration tests. // By default, integration tests perform a fetch for all resources in cloudquery's test account, and // verify all columns are non-nil. // If IgnoreInTests is true, verification is skipped for this column. // Used when it is hard to create a reproducible environment with this column being non-nil (e.g. various error columns). - IgnoreInTests bool + IgnoreInTests bool `json:"-"` // internal is true if this column is managed by the SDK internal bool // meta holds serializable information about the column's resolvers and functions @@ -67,9 +68,7 @@ type Column struct { const ( TypeInvalid ValueType = iota TypeBool - TypeSmallInt TypeInt - TypeBigInt TypeFloat TypeUUID TypeString @@ -91,8 +90,8 @@ func (v ValueType) String() string { switch v { case TypeBool: return "TypeBool" - case TypeInt, TypeBigInt, TypeSmallInt: - return "TypeBigInt" + case TypeInt: + return "TypeInt" case TypeFloat: return "TypeFloat" case TypeUUID: @@ -136,7 +135,7 @@ func ValueTypeFromString(s string) ValueType { case "bool": return TypeBool case "int", "bigint", "smallint": - return TypeBigInt + return TypeInt case "float": return TypeFloat case "uuid": @@ -201,8 +200,7 @@ func (c Column) checkType(v interface{}) bool { switch val := v.(type) { case int8, *int8, uint8, *uint8, int16, *int16, uint16, *uint16, int32, *int32, int, *int, uint32, *uint32, int64, *int64: - // TODO: Deprecate all Int Types in favour of BigInt - return c.Type == TypeBigInt || c.Type == TypeSmallInt || c.Type == TypeInt + return c.Type == TypeInt case []byte: if c.Type == TypeUUID { if _, err := uuid.FromBytes(val); err != nil { @@ -260,13 +258,22 @@ func (c Column) checkType(v interface{}) bool { if kindName == reflect.String && c.Type == TypeString { return true } + switch kindName { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return c.Type == TypeInt + } if kindName == reflect.Slice { itemKind := reflect2.TypeOf(v).Type1().Elem().Kind() if c.Type == TypeStringArray && reflect.String == itemKind { return true } - if c.Type == TypeIntArray && reflect.Int == itemKind { - return true + if c.Type == TypeIntArray { + switch itemKind { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return true + } } if c.Type == TypeJSON && (reflect.Struct == itemKind || reflect.Ptr == itemKind) { return true @@ -278,16 +285,6 @@ func (c Column) checkType(v interface{}) bool { if kindName == reflect.Struct { return c.Type == TypeJSON } - if c.Type == TypeSmallInt && (kindName == reflect.Int8 || kindName == reflect.Int16 || kindName == reflect.Uint8) { - return true - } - - if c.Type == TypeInt && (kindName == reflect.Uint16 || kindName == reflect.Int32) { - return true - } - if c.Type == TypeBigInt && (kindName == reflect.Int || kindName == reflect.Int64 || kindName == reflect.Uint || kindName == reflect.Uint32 || kindName == reflect.Uint64) { - return true - } } return false @@ -328,27 +325,27 @@ func SetColumnMeta(c Column, m *ColumnMeta) Column { } // Sift gets a column list and returns a list of provider columns, and another list of internal columns, cqId column being the very last one -func (c ColumnList) Sift() (providerCols ColumnList, internalCols ColumnList) { - providerCols, internalCols = make(ColumnList, 0, len(c)), make(ColumnList, 0, len(c)) - cqIdColIndex := -1 - for i := range c { - if c[i].internal { - if c[i].Name == cqIdColumn.Name { - cqIdColIndex = len(internalCols) - } +// func (c ColumnList) Sift() (providerCols ColumnList, internalCols ColumnList) { +// providerCols, internalCols = make(ColumnList, 0, len(c)), make(ColumnList, 0, len(c)) +// cqIdColIndex := -1 +// for i := range c { +// if c[i].internal { +// if c[i].Name == cqIdColumn.Name { +// cqIdColIndex = len(internalCols) +// } - internalCols = append(internalCols, c[i]) - } else { - providerCols = append(providerCols, c[i]) - } - } +// internalCols = append(internalCols, c[i]) +// } else { +// providerCols = append(providerCols, c[i]) +// } +// } - // resolve cqId last, as it would need other PKs to be resolved, some might be internal (cq_fetch_date) - if lastIndex := len(internalCols) - 1; cqIdColIndex > -1 && cqIdColIndex != lastIndex { - internalCols[cqIdColIndex], internalCols[lastIndex] = internalCols[lastIndex], internalCols[cqIdColIndex] - } - return providerCols, internalCols -} +// // resolve cqId last, as it would need other PKs to be resolved, some might be internal (cq_fetch_date) +// if lastIndex := len(internalCols) - 1; cqIdColIndex > -1 && cqIdColIndex != lastIndex { +// internalCols[cqIdColIndex], internalCols[lastIndex] = internalCols[lastIndex], internalCols[cqIdColIndex] +// } +// return providerCols, internalCols +// } func (c ColumnList) Names() []string { ret := make([]string, len(c)) diff --git a/schema/column_test.go b/schema/column_test.go index b71e58d7c6..aaf1b45543 100644 --- a/schema/column_test.go +++ b/schema/column_test.go @@ -1,9 +1,11 @@ package schema import ( + "encoding/json" "fmt" "math/rand" "net" + "reflect" "testing" "time" @@ -27,13 +29,13 @@ type SomeInt16 int16 var validateFixtures = []validateFixture{ { - Column: Column{Type: TypeBigInt}, - TestValues: []interface{}{5, 300, funk.PtrOf(555), SomeInt(555)}, - BadValues: []interface{}{"a", funk.PtrOf("abc"), SomeInt16(555)}, + Column: Column{Type: TypeInt}, + TestValues: []interface{}{5, 300, funk.PtrOf(555), SomeInt16(555), SomeInt(555)}, + BadValues: []interface{}{"a", funk.PtrOf("abc")}, }, { - Column: Column{Type: TypeSmallInt}, - TestValues: []interface{}{SomeInt16(555)}, + Column: Column{Type: TypeFloat}, + TestValues: []interface{}{555.5}, BadValues: []interface{}{"a", funk.PtrOf("abc")}, }, { @@ -60,7 +62,7 @@ var validateFixtures = []validateFixture{ }, { Column: Column{Type: TypeIntArray}, - TestValues: []interface{}{[]int{1, 2, 3}, []SomeInt{SomeInt(3)}}, + TestValues: []interface{}{[]int{1, 2, 3}, []SomeInt{SomeInt(3)}, []int16{1, 2, 3}}, BadValues: []interface{}{[]interface{}{1, 2, 3}}, }, { @@ -127,12 +129,14 @@ func GenerateCIDR() *net.IPNet { func TestValidateType(t *testing.T) { for _, f := range validateFixtures { - for _, v := range f.TestValues { - assert.Nil(t, f.Column.ValidateType(v)) - } - for _, v := range f.BadValues { - assert.Error(t, f.Column.ValidateType(v)) - } + t.Run(f.Column.Type.String(), func(t *testing.T) { + for _, v := range f.TestValues { + assert.Nil(t, f.Column.ValidateType(v)) + } + for _, v := range f.BadValues { + assert.Error(t, f.Column.ValidateType(v)) + } + }) } } @@ -141,10 +145,10 @@ func TestValueTypeFromString(t *testing.T) { // case insensitive assert.Equal(t, ValueTypeFromString("Json"), TypeJSON) assert.Equal(t, ValueTypeFromString("JSON"), TypeJSON) - assert.Equal(t, ValueTypeFromString("bigint"), TypeBigInt) + assert.Equal(t, ValueTypeFromString("bigint"), TypeInt) assert.Equal(t, ValueTypeFromString("Blabla"), TypeInvalid) - assert.Equal(t, ValueTypeFromString("TypeBigInt"), TypeBigInt) + assert.Equal(t, ValueTypeFromString("TypeBigInt"), TypeInt) assert.Equal(t, ValueTypeFromString("TypeString"), TypeString) } @@ -176,3 +180,23 @@ func BenchmarkColumn_ValidateTypeMap(b *testing.B) { _ = col.ValidateType(m) } } + +func TestColumnJsonMarshal(t *testing.T) { + // we are testing column json marshalling to make sure + // this can be easily sent over the wire + expected := Column{ + Name: "test", + Type: TypeJSON, + } + b, err := json.Marshal(expected) + if err != nil { + t.Fatal(err) + } + got := Column{} + if err := json.Unmarshal(b, &got); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(expected, got) { + t.Fatalf("expected %v got %v", expected, got) + } +} diff --git a/schema/meta.go b/schema/meta.go index f3c72cd979..8ed69590b0 100644 --- a/schema/meta.go +++ b/schema/meta.go @@ -17,62 +17,10 @@ type Meta struct { const FetchIdMetaKey = "cq_fetch_id" -var ( - // cqMeta = Column{ - // Name: "cq_meta", - // Type: TypeJSON, - // Description: "Meta column holds fetch information", - // Resolver: func(ctx context.Context, meta ClientMeta, resource *Resource, c Column) error { - // mi := Meta{ - // LastUpdate: time.Now().UTC(), - // } - // if val, ok := resource.GetMeta(FetchIdMetaKey); ok { - // if s, ok := val.(string); ok { - // mi.FetchId = s - // } - // } - // b, _ := json.Marshal(mi) - // return resource.Set(c.Name, b) - // }, - // internal: true, - // } - cqIdColumn = Column{ - Name: "cq_id", - Type: TypeUUID, - Description: "Unique CloudQuery Id added to every resource", - // Resolver: func(ctx context.Context, meta ClientMeta, resource *Resource, c Column) error { - // if err := resource.GenerateCQId(); err != nil { - // if resource.Parent == nil { - // return err - // } +var CqIdColumn = Column{Name: "_cq_id", Type: TypeUUID, Description: "Internal CQ ID of the row", CreationOptions: ColumnCreationOptions{Unique: true}, Resolver: CQUUIDResolver()} +var CqFetchTime = Column{Name: "_cq_fetch_time", Type: TypeTimestamp, Description: "Internal CQ row of when fetch was started (this will be the same for all rows in a single fetch)"} - // meta.Logger().Debug("one of the table pk is nil", "table", resource.table.Name) - // } - // return resource.Set(c.Name, resource.Id()) - // }, - CreationOptions: ColumnCreationOptions{ - Unique: true, - NotNull: true, - }, - internal: true, - } - // cqFetchDateColumn = Column{ - // Name: "cq_fetch_date", - // Type: TypeTimestamp, - // Description: "Time of fetch for this resource", - // // Resolver: func(ctx context.Context, meta ClientMeta, resource *Resource, c Column) error { - // // val, ok := resource.GetMeta("cq_fetch_date") - // // if !ok && !resource.executionStart.IsZero() { - // // val = resource.executionStart - // // } - // // if val == nil { - // // return fmt.Errorf("zero cq_fetch date") - // // } - // // return resource.Set(c.Name, val) - // // }, - // CreationOptions: ColumnCreationOptions{ - // NotNull: true, - // }, - // internal: true, - // } -) +var CqColumns = []Column{ + CqIdColumn, + CqFetchTime, +} diff --git a/schema/resolvers.go b/schema/resolvers.go index 3c7489330d..a46cb60583 100644 --- a/schema/resolvers.go +++ b/schema/resolvers.go @@ -7,7 +7,8 @@ import ( "time" "github.com/cloudquery/plugin-sdk/helpers" - "github.com/gofrs/uuid" + // "github.com/gofrs/uuid" + "github.com/google/uuid" "github.com/spf13/cast" "github.com/thoas/go-funk" ) @@ -24,6 +25,13 @@ func PathResolver(path string) ColumnResolver { } } +func CQUUIDResolver() ColumnResolver { + return func(ctx context.Context, meta ClientMeta, r *Resource, c Column) error { + uuidGen := uuid.New() + return r.Set(c.Name, uuidGen) + } +} + // ParentIdResolver resolves the cq_id from the parent // if you want to reference the parent's primary keys use ParentResourceFieldResolver as required. func ParentIdResolver(_ context.Context, _ ClientMeta, r *Resource, c Column) error { @@ -198,8 +206,7 @@ func UUIDResolver(path string) ColumnResolver { if err != nil { return err } - - id, err := uuid.FromString(uuidString) + id, err := uuid.FromBytes([]byte(uuidString)) if err != nil { return err } diff --git a/schema/resource.go b/schema/resource.go index bbedb93ab2..92f7f6df6d 100644 --- a/schema/resource.go +++ b/schema/resource.go @@ -2,9 +2,10 @@ package schema import ( "fmt" + "strings" + "time" "github.com/google/uuid" - "github.com/thoas/go-funk" ) type Resources []*Resource @@ -13,75 +14,73 @@ type Resources []*Resource // generates an Id based on Table's Columns. Resource data can be accessed by the Get and Set methods type Resource struct { // Original resource item that wa from prior resolve - Item interface{} + Item interface{} `json:"-"` // Set if this is an embedded table - Parent *Resource `msgpack:"parent"` + Parent *Resource `json:"-"` // internal fields - Table *Table `msgpack:"table"` - Data map[string]interface{} `msgpack:"data"` - cqId uuid.UUID - metadata map[string]interface{} - CColumns []string `msgpack:"columns"` + Table *Table `json:"-"` + // This is sorted result data by column name + Data map[string]interface{} `json:"data"` + TableName string `json:"table_name"` } -func NewResourceData(t *Table, parent *Resource, item interface{}) *Resource { - return &Resource{ - Item: item, - Parent: parent, - Table: t, - Data: make(map[string]interface{}), - cqId: uuid.New(), - CColumns: t.Columns.Names(), - // metadata: metadata, +func NewResourceData(t *Table, parent *Resource, fetchTime time.Time, item interface{}) *Resource { + r := Resource{ + Item: item, + Parent: parent, + Table: t, + Data: make(map[string]interface{}, len(t.Columns)), + TableName: t.Name, } + r.Data[CqFetchTime.Name] = fetchTime + return &r } -// func (r *Resource) PrimaryKeyValues() []string { -// tablePrimKeys := r.dialect.PrimaryKeys(r.table) -// if len(tablePrimKeys) == 0 { -// return []string{} -// } -// results := make([]string, 0) -// for _, primKey := range tablePrimKeys { -// data := r.Get(primKey) -// if data == nil { -// continue -// } -// // we can have more types, but PKs are usually either ints, strings or a structure -// // hopefully supporting Stringer interface, otherwise we fallback -// switch v := data.(type) { -// case fmt.Stringer: -// results = append(results, v.String()) -// case *string: -// results = append(results, *v) -// case *int: -// results = append(results, fmt.Sprintf("%d", *v)) -// default: -// results = append(results, fmt.Sprintf("%v", v)) -// } -// } -// return results -// } +func (r *Resource) PrimaryKeyValue() string { + pks := r.Table.PrimaryKeys() + if len(pks) == 0 { + return "" + } + var sb strings.Builder + for _, primKey := range pks { + data := r.Get(primKey) + if data == nil { + continue + } + // we can have more types, but PKs are usually either ints, strings or a structure + // hopefully supporting Stringer interface, otherwise we fallback + switch v := data.(type) { + case fmt.Stringer: + sb.WriteString(v.String()) + case *string: + sb.WriteString(*v) + case *int: + sb.WriteString(fmt.Sprintf("%d", *v)) + default: + sb.WriteString(fmt.Sprintf("%d", v)) + } + } + return sb.String() +} func (r *Resource) Get(key string) interface{} { return r.Data[key] } func (r *Resource) Set(key string, value interface{}) error { - columnExists := funk.ContainsString(r.CColumns, key) - if !columnExists { - return fmt.Errorf("column %s does not exist", key) - } r.Data[key] = value return nil } func (r *Resource) Id() uuid.UUID { - return r.cqId + if r.Data[CqIdColumn.Name] == nil { + return uuid.UUID{} + } + return r.Data[CqIdColumn.Name].(uuid.UUID) } func (r *Resource) Columns() []string { - return r.CColumns + return r.Table.Columns.Names() } // func (r *Resource) Values() ([]interface{}, error) { @@ -123,20 +122,13 @@ func (r *Resource) Columns() []string { // return nil // } -func (r *Resource) TableName() string { - if r.Table == nil { - return "" - } - return r.Table.Name -} - -func (r Resource) GetMeta(key string) (interface{}, bool) { - if r.metadata == nil { - return nil, false - } - v, ok := r.metadata[key] - return v, ok -} +// func (r Resource) GetMeta(key string) (interface{}, bool) { +// if r.metadata == nil { +// return nil, false +// } +// v, ok := r.metadata[key] +// return v, ok +// } // func (r Resource) getColumnByName(column string) *Column { // for _, c := range r.Table.Columns { @@ -165,7 +157,7 @@ func (rr Resources) ColumnNames() []string { if len(rr) == 0 { return []string{} } - return rr[0].CColumns + return rr[0].Table.Columns.Names() } // func hashUUID(objs interface{}) (uuid.UUID, error) { diff --git a/schema/table.go b/schema/table.go index 4f57914d77..a407808644 100644 --- a/schema/table.go +++ b/schema/table.go @@ -2,7 +2,10 @@ package schema import ( "context" + "fmt" "runtime/debug" + "sync" + "time" "github.com/cloudquery/plugin-sdk/helpers" "github.com/iancoleman/strcase" @@ -18,43 +21,47 @@ import ( // type TableResolver func(ctx context.Context, meta ClientMeta, parent *Resource, res chan<- interface{}) error -// IgnoreErrorFunc checks if returned error from table resolver should be ignored. -type IgnoreErrorFunc func(err error) bool - type RowResolver func(ctx context.Context, meta ClientMeta, resource *Resource) error +// Classify error and return it's severity and type +type IgnoreErrorFunc func(err error) (bool, string) + +type Tables []*Table + type Table struct { // Name of table - Name string + Name string `json:"name"` // table description - Description string + Description string `json:"description"` // Columns are the set of fields that are part of this table - Columns ColumnList + Columns ColumnList `json:"columns"` // Relations are a set of related tables defines - Relations []*Table + Relations Tables `json:"relations"` // Resolver is the main entry point to fetching table data and - Resolver TableResolver `msgpack:"-"` - // Ignore errors checks if returned error from table resolver should be ignored. - IgnoreError IgnoreErrorFunc `msgpack:"-"` + Resolver TableResolver `json:"-"` + // IgnoreError is a function that classifies error and returns it's severity and type + IgnoreError IgnoreErrorFunc `json:"-"` // Multiplex returns re-purposed meta clients. The sdk will execute the table with each of them - Multiplex func(meta ClientMeta) []ClientMeta `msgpack:"-"` + Multiplex func(meta ClientMeta) []ClientMeta `json:"-"` // Post resource resolver is called after all columns have been resolved, and before resource is inserted to database. - PostResourceResolver RowResolver `msgpack:"-"` + PostResourceResolver RowResolver `json:"-"` // Options allow modification of how the table is defined when created - Options TableCreationOptions + Options TableCreationOptions `json:"options"` // IgnoreInTests is used to exclude a table from integration tests. // By default, integration tests fetch all resources from cloudquery's test account, and verify all tables // have at least one row. // When IgnoreInTests is true, integration tests won't fetch from this table. // Used when it is hard to create a reproducible environment with a row in this table. - IgnoreInTests bool + IgnoreInTests bool `json:"ignore_in_tests"` // Parent is the parent table in case this table is called via parent table (i.e. relation) - Parent *Table + Parent *Table `json:"-"` // Serial is used to force a signature change, which forces new table creation and cascading removal of old table and relations - Serial string + Serial string `json:"-"` + + columnsMap map[string]int } // TableCreationOptions allow modifying how table is created such as defining primary keys, indices, foreign keys and constraints. @@ -63,6 +70,39 @@ type TableCreationOptions struct { PrimaryKeys []string } +func (tt Tables) TableNames() []string { + ret := []string{} + for _, t := range tt { + ret = append(ret, t.TableNames()...) + } + return ret +} + +func (tt Tables) ValidateDuplicateColumns() error { + for _, t := range tt { + if err := t.ValidateDuplicateColumns(); err != nil { + return err + } + } + return nil +} + +func (t Table) ValidateDuplicateColumns() error { + columns := make(map[string]bool, len(t.Columns)) + for _, c := range t.Columns { + if _, ok := columns[c.Name]; ok { + return fmt.Errorf("duplicate column %s in table %s", c.Name, t.Name) + } + columns[c.Name] = true + } + for _, rel := range t.Relations { + if err := rel.ValidateDuplicateColumns(); err != nil { + return err + } + } + return nil +} + func (t Table) Column(name string) *Column { for _, c := range t.Columns { if c.Name == name { @@ -72,6 +112,33 @@ func (t Table) Column(name string) *Column { return nil } +func (t Table) PrimaryKeys() []string { + var primaryKeys []string + for _, c := range t.Columns { + if c.CreationOptions.PrimaryKey { + primaryKeys = append(primaryKeys, c.Name) + } + } + + return primaryKeys +} + +func (t Table) ColumnIndex(name string) int { + var once sync.Once + once.Do(func() { + if t.columnsMap == nil { + t.columnsMap = make(map[string]int) + for i, c := range t.Columns { + t.columnsMap[c.Name] = i + } + } + }) + if index, ok := t.columnsMap[name]; ok { + return index + } + return -1 +} + // func (tco TableCreationOptions) signature() string { // return strings.Join(tco.PrimaryKeys, ";") // } @@ -85,45 +152,64 @@ func (t Table) TableNames() []string { } // Call the table resolver with with all of it's relation for every reolved resource -func (t Table) Resolve(ctx context.Context, meta ClientMeta, parent *Resource, resolvedResources chan<- *Resource) { +func (t Table) Resolve(ctx context.Context, meta ClientMeta, fetchTime time.Time, parent *Resource, resolvedResources chan<- *Resource) int { res := make(chan interface{}) + startTime := time.Now() go func() { defer func() { if r := recover(); r != nil { stack := string(debug.Stack()) - meta.Logger().Error().Str("table_name", t.Name).Str("stack", stack).Msg("table resolver finished with panic") + meta.Logger().Error().Str("table_name", t.Name).TimeDiff("duration", time.Now(), startTime).Str("stack", stack).Msg("table resolver finished with panic") } close(res) }() meta.Logger().Debug().Str("table_name", t.Name).Msg("table resolver started") if err := t.Resolver(ctx, meta, parent, res); err != nil { - meta.Logger().Error().Str("table_name", t.Name).Err(err).Msg("table resolver finished with error") + if t.IgnoreError != nil { + if ignore, errType := t.IgnoreError(err); ignore { + meta.Logger().Debug().Str("table_name", t.Name).TimeDiff("duration", time.Now(), startTime).Str("error_type", errType).Msg("table resolver finished with error") + return + } + } + meta.Logger().Error().Str("table_name", t.Name).TimeDiff("duration", time.Now(), startTime).Err(err).Msg("table resolver finished with error") + return } - meta.Logger().Debug().Str("table_name", t.Name).Msg("table resolver finished successfully") + meta.Logger().Debug().Str("table_name", t.Name).TimeDiff("duration", time.Now(), startTime).Msg("table resolver finished successfully") }() - + totalResources := 0 + // we want to check for data integrity + // in the future we can do that as an optinoal feature via a flag + pks := map[string]bool{} // each result is an array of interface{} for elem := range res { objects := helpers.InterfaceSlice(elem) if len(objects) == 0 { continue } + totalResources += len(objects) for i := range objects { - resource := NewResourceData(&t, parent, objects[i]) + resource := NewResourceData(&t, parent, fetchTime, objects[i]) t.resolveColumns(ctx, meta, resource) if t.PostResourceResolver != nil { meta.Logger().Trace().Str("table_name", t.Name).Msg("post resource resolver started") if err := t.PostResourceResolver(ctx, meta, resource); err != nil { meta.Logger().Error().Str("table_name", t.Name).Err(err).Msg("post resource resolver finished with error") + } else { + meta.Logger().Trace().Str("table_name", t.Name).Msg("post resource resolver finished successfully") } - meta.Logger().Trace().Str("table_name", t.Name).Msg("post resource resolver finished successfully") + } + if pks[resource.PrimaryKeyValue()] { + meta.Logger().Error().Str("table_name", t.Name).Str("primary_key", resource.PrimaryKeyValue()).Msg("duplicate primary key found") + } else { + pks[resource.PrimaryKeyValue()] = true } resolvedResources <- resource for _, rel := range t.Relations { - rel.Resolve(ctx, meta, resource, resolvedResources) + totalResources += rel.Resolve(ctx, meta, fetchTime, resource, resolvedResources) } } } + return totalResources } func (t Table) resolveColumns(ctx context.Context, meta ClientMeta, resource *Resource) { @@ -138,10 +224,14 @@ func (t Table) resolveColumns(ctx context.Context, meta ClientMeta, resource *Re meta.Logger().Trace().Str("colum_name", c.Name).Str("table_name", t.Name).Msg("column resolver default started") // base use case: try to get column with CamelCase name v := funk.Get(resource.Item, strcase.ToCamel(c.Name), funk.WithAllowZero()) - if err := resource.Set(c.Name, v); err != nil { - meta.Logger().Error().Str("colum_name", c.Name).Str("table_name", t.Name).Err(err).Msg("column resolver default finished with error") + if v != nil { + if err := resource.Set(c.Name, v); err != nil { + meta.Logger().Error().Str("colum_name", c.Name).Str("table_name", t.Name).Err(err).Msg("column resolver default finished with error") + } + meta.Logger().Trace().Str("colum_name", c.Name).Str("table_name", t.Name).Msg("column resolver default finished successfully") + } else { + meta.Logger().Trace().Str("colum_name", c.Name).Str("table_name", t.Name).Msg("column resolver default finished successfully with nil") } - meta.Logger().Trace().Str("colum_name", c.Name).Str("table_name", t.Name).Msg("column resolver default finished successfully") } } } diff --git a/schema/validators_test.go b/schema/validators_test.go index d23d02e9db..9d4de169ab 100644 --- a/schema/validators_test.go +++ b/schema/validators_test.go @@ -15,7 +15,7 @@ var testTableValidators = Table{ }, { Name: "zero_int", - Type: TypeBigInt, + Type: TypeInt, }, { Name: "not_zero_bool", diff --git a/serve/doc.go b/serve/doc.go index cc5de916bf..4a3eda2cf6 100644 --- a/serve/doc.go +++ b/serve/doc.go @@ -22,7 +22,7 @@ func newCmdDoc(opts Options) *cobra.Command { return fmt.Errorf("doc generation is only supported for source plugins") } - return schema.GenerateMarkdownTree(opts.SourcePlugin.Tables, args[0]) + return schema.GenerateMarkdownTree(opts.SourcePlugin.Tables(), args[0]) }, } } diff --git a/serve/serve.go b/serve/serve.go index 6c6c148b12..cee4662a6b 100644 --- a/serve/serve.go +++ b/serve/serve.go @@ -16,6 +16,7 @@ import ( "github.com/rs/zerolog/log" "github.com/spf13/cobra" "google.golang.org/grpc" + "google.golang.org/grpc/test/bufconn" ) type Options struct { @@ -24,10 +25,16 @@ type Options struct { DestinationPlugin plugins.DestinationPlugin } +// bufSize used for unit testing grpc server and client +const testBufSize = 1024 * 1024 + const ( serveShort = `Start plugin server` ) +// lis used for unit testing grpc server and client +var testListener *bufconn.Listener + func newCmdServe(opts Options) *cobra.Command { var address string var network string @@ -50,9 +57,15 @@ func newCmdServe(opts Options) *cobra.Command { logger = log.Output(zerolog.ConsoleWriter{Out: os.Stdout}).Level(zerologLevel) } // opts.Plugin.Logger = logger - listener, err := net.Listen(network, address) - if err != nil { - return fmt.Errorf("failed to listen: %w", err) + var listener net.Listener + if network == "test" { + listener = bufconn.Listen(testBufSize) + testListener = listener.(*bufconn.Listener) + } else { + listener, err = net.Listen(network, address) + if err != nil { + return fmt.Errorf("failed to listen %s:%s: %w", network, address, err) + } } // See logging pattern https://github.com/grpc-ecosystem/go-grpc-middleware/blob/v2/providers/zerolog/examples_test.go s := grpc.NewServer( @@ -67,11 +80,11 @@ func newCmdServe(opts Options) *cobra.Command { ) if opts.SourcePlugin != nil { - opts.SourcePlugin.Logger = logger + opts.SourcePlugin.SetLogger(logger) pb.RegisterSourceServer(s, &servers.SourceServer{Plugin: opts.SourcePlugin}) } if opts.DestinationPlugin != nil { - opts.SourcePlugin.Logger = logger + // opts.DestinationPlugin.Logger = logger pb.RegisterDestinationServer(s, &servers.DestinationServer{Plugin: opts.DestinationPlugin}) } diff --git a/serve/serve_test.go b/serve/serve_test.go new file mode 100644 index 0000000000..b4197ff8a7 --- /dev/null +++ b/serve/serve_test.go @@ -0,0 +1,174 @@ +package serve + +import ( + "context" + "net" + "testing" + "time" + + "github.com/cloudquery/plugin-sdk/clients" + "github.com/cloudquery/plugin-sdk/plugins" + "github.com/cloudquery/plugin-sdk/schema" + "github.com/cloudquery/plugin-sdk/specs" + "github.com/google/go-cmp/cmp" + "github.com/rs/zerolog" + "golang.org/x/sync/errgroup" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +type TestSourcePluginSpec struct { + Accounts []string `json:"accounts,omitempty" yaml:"accounts,omitempty"` +} + +type testExecutionClient struct { + logger zerolog.Logger +} + +var _ schema.ClientMeta = &testExecutionClient{} + +var expectedExampleSpecConfig = specs.Spec{ + Kind: specs.KindSource, + Spec: &specs.Source{ + Name: "testSourcePlugin", + Path: "cloudquery/testSourcePlugin", + Version: "v1.0.0", + Tables: []string{"*"}, + Spec: map[string]interface{}{"accounts": []interface{}{"all"}}, + }, +} + +func testTable() *schema.Table { + return &schema.Table{ + Name: "test_table", + Resolver: func(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- interface{}) error { + res <- map[string]interface{}{ + "TestColumn": 3, + } + return nil + }, + Columns: []schema.Column{ + { + Name: "test_column", + Type: schema.TypeInt, + }, + }, + } +} + +func (c *testExecutionClient) Logger() *zerolog.Logger { + return &c.logger +} + +func newTestExecutionClient(context.Context, *plugins.SourcePlugin, specs.Source) (schema.ClientMeta, error) { + return &testExecutionClient{}, nil +} + +// https://stackoverflow.com/questions/32840687/timeout-for-waitgroup-wait +// func waitTimeout(wg *errgroup.Group, timeout time.Duration) (bool, error) { +// c := make(chan struct{}) +// var err error +// go func() { +// defer close(c) +// err = wg.Wait() +// }() +// select { +// case <-c: +// return false, err // completed normally +// case <-time.After(timeout): +// return true, err // timed out +// } +// } + +func bufDialer(context.Context, string) (net.Conn, error) { + return testListener.Dial() +} + +func TestServe(t *testing.T) { + plugin := plugins.NewSourcePlugin( + "testSourcePlugin", + "v1.0.0", + []*schema.Table{testTable()}, + newTestExecutionClient, + plugins.WithSourceExampleConfig(`--- +kind: source +spec: + name: testSourcePlugin + path: cloudquery/testSourcePlugin + spec: + accounts: ["all"] + tables: + - "*" + version: v1.0.0 +`), + plugins.WithSourceLogger(zerolog.New(zerolog.NewTestWriter(t))), + ) + + cmd := newCmdRoot(Options{ + SourcePlugin: plugin, + }) + cmd.SetArgs([]string{"serve", "--network", "test"}) + + var serveErr error + go func() { + serveErr = cmd.Execute() + }() + + // wait for the server to start + for { + if testListener != nil { + break + } + t.Log("waiting for grpc server to start") + time.Sleep(time.Millisecond * 200) + if serveErr != nil { + t.Fatal(serveErr) + } + } + + // https://stackoverflow.com/questions/42102496/testing-a-grpc-service + ctx := context.Background() + conn, err := grpc.DialContext(ctx, "bufnet", grpc.WithContextDialer(bufDialer), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithBlock()) + if err != nil { + t.Fatalf("Failed to dial bufnet: %v", err) + } + c := clients.NewSourceClient(conn) + resources := make(chan *schema.Resource) + wg := errgroup.Group{} + wg.Go(func() error { + defer close(resources) + return c.Sync(ctx, + specs.Source{ + Name: "testSourcePlugin", + Version: "v1.0.0", + Registry: specs.RegistryGithub, + Spec: TestSourcePluginSpec{Accounts: []string{"cloudquery/plugin-sdk"}}, + }, + resources) + }) + for resource := range resources { + if resource.TableName != "test_table" { + t.Fatalf("Expected resource with table name test: %s", resource.TableName) + } + if int(resource.Data["test_column"].(float64)) != 3 { + t.Fatalf("Expected resource {'test_column':3} got: %v", resource.Data) + } + } + if err := wg.Wait(); err != nil { + t.Fatalf("Failed to fetch resources: %v", err) + } + + exampleConfig, err := c.ExampleConfig(ctx) + if err != nil { + t.Fatalf("Failed to get example config: %v", err) + } + var exampleSpec specs.Spec + if err := specs.SpecUnmarshalYamlStrict([]byte(exampleConfig), &exampleSpec); err != nil { + t.Fatalf("Failed to unmarshal example config: %v", err) + } + // skip internal validation for now + + if diff := cmp.Diff(expectedExampleSpecConfig, exampleSpec); diff != "" { + t.Fatalf("Spec mismatch (-want +got):\n%s", diff) + } +} diff --git a/specs/connection.go b/specs/connection.go deleted file mode 100644 index 5cd410ee1c..0000000000 --- a/specs/connection.go +++ /dev/null @@ -1,6 +0,0 @@ -package specs - -type ConnectionSpec struct { - Source string `yaml:"source"` - Destination string `yaml:"destination"` -} diff --git a/specs/destination.go b/specs/destination.go index 04876e85f6..8d5abc7d80 100644 --- a/specs/destination.go +++ b/specs/destination.go @@ -1,11 +1,82 @@ package specs -import "gopkg.in/yaml.v3" - -type DestinationSpec struct { - Name string `yaml:"name"` - Version string `yaml:"version"` - Path string `yaml:"path"` - Registry string `yaml:"registry"` - Spec yaml.Node `yaml:"spec"` +import ( + "bytes" + "encoding/json" + "fmt" + "strings" +) + +type WriteMode int + +type Destination struct { + Name string `json:"name,omitempty"` + Version string `json:"version,omitempty"` + Path string `json:"path,omitempty"` + Registry Registry `json:"registry,omitempty"` + WriteMode WriteMode `json:"write_mode,omitempty"` + Spec interface{} `json:"spec,omitempty"` +} + +const ( + WriteModeAppend WriteMode = iota + WriteModeOverwrite +) + +func (d *Destination) SetDefaults() { + if d.Registry.String() == "" { + d.Registry = RegistryGithub + } + if d.Path == "" { + d.Path = d.Name + } + if d.Version == "" { + d.Version = "latest" + } + if d.Registry == RegistryGithub && !strings.Contains(d.Path, "/") { + d.Path = "cloudquery/" + d.Path + } +} + +func (d *Destination) UnmarshalSpec(out interface{}) error { + b, err := json.Marshal(d.Spec) + if err != nil { + return err + } + dec := json.NewDecoder(nil) + dec.UseNumber() + dec.DisallowUnknownFields() + return json.Unmarshal(b, out) +} + +func (m WriteMode) String() string { + return [...]string{"append", "overwrite"}[m] +} + +func (m WriteMode) MarshalJSON() ([]byte, error) { + buffer := bytes.NewBufferString(`"`) + buffer.WriteString(m.String()) + buffer.WriteString(`"`) + return buffer.Bytes(), nil +} + +func (m *WriteMode) UnmarshalJSON(data []byte) (err error) { + var writeMode string + if err := json.Unmarshal(data, &writeMode); err != nil { + return err + } + if *m, err = WriteModeFromString(writeMode); err != nil { + return err + } + return nil +} + +func WriteModeFromString(s string) (WriteMode, error) { + switch s { + case "append": + return WriteModeAppend, nil + case "overwrite": + return WriteModeOverwrite, nil + } + return 0, fmt.Errorf("invalid write mode: %s", s) } diff --git a/specs/registry.go b/specs/registry.go new file mode 100644 index 0000000000..b18bf3bc39 --- /dev/null +++ b/specs/registry.go @@ -0,0 +1,49 @@ +package specs + +import ( + "bytes" + "encoding/json" + "fmt" +) + +type Registry int + +const ( + RegistryGithub Registry = iota + RegistryLocal + RegistryGrpc +) + +func (r Registry) String() string { + return [...]string{"github", "local", "grpc"}[r] +} +func (r Registry) MarshalJSON() ([]byte, error) { + buffer := bytes.NewBufferString(`"`) + buffer.WriteString(r.String()) + buffer.WriteString(`"`) + return buffer.Bytes(), nil +} + +func (r *Registry) UnmarshalJSON(data []byte) (err error) { + var registry string + if err := json.Unmarshal(data, ®istry); err != nil { + return err + } + if *r, err = RegistryFromString(registry); err != nil { + return err + } + return nil +} + +func RegistryFromString(s string) (Registry, error) { + switch s { + case "github": + return RegistryGithub, nil + case "local": + return RegistryLocal, nil + case "grpc": + return RegistryGrpc, nil + default: + return RegistryGithub, fmt.Errorf("unknown registry %s", s) + } +} diff --git a/specs/registry_test.go b/specs/registry_test.go new file mode 100644 index 0000000000..d7751a27ca --- /dev/null +++ b/specs/registry_test.go @@ -0,0 +1,36 @@ +package specs + +import ( + "encoding/json" + "testing" + + "gopkg.in/yaml.v3" +) + +func TestRegistryJsonMarshalUnmarshal(t *testing.T) { + b, err := json.Marshal(RegistryGrpc) + if err != nil { + t.Fatal("failed to marshal registry:", err) + } + var registry Registry + if err := json.Unmarshal(b, ®istry); err != nil { + t.Fatal("failed to unmarshal registry:", err) + } + if registry != RegistryGrpc { + t.Fatal("expected registry to be github, but got:", registry) + } +} + +func TestRegistryYamlMarshalUnmarsahl(t *testing.T) { + b, err := yaml.Marshal(RegistryGrpc) + if err != nil { + t.Fatal("failed to marshal registry:", err) + } + var registry Registry + if err := yaml.Unmarshal(b, ®istry); err != nil { + t.Fatal("failed to unmarshal registry:", err) + } + if registry != RegistryGrpc { + t.Fatal("expected registry to be github, but got:", registry) + } +} diff --git a/specs/source.go b/specs/source.go index bc1d618a5c..4cd2ba92b4 100644 --- a/specs/source.go +++ b/specs/source.go @@ -1,28 +1,30 @@ package specs import ( + "encoding/json" "strings" - "gopkg.in/yaml.v3" + "github.com/xeipuuv/gojsonschema" ) -// SourceSpec is the shared configuration for all source plugins -type SourceSpec struct { - Name string `json:"name" yaml:"name"` - Version string `json:"version" yaml:"version"` +// Source is the shared configuration for all source plugins +type Source struct { + Name string `json:"name,omitempty"` + Version string `json:"version,omitempty"` // Path is the path in the registry - Path string `json:"path" yaml:"path"` + Path string `json:"path,omitempty"` // Registry can be github,local,grpc. Might support things like https in the future. - Registry string `json:"registry" yaml:"registry"` - MaxGoRoutines uint64 `json:"max_goroutines" yaml:"max_goroutines"` - Tables []string `json:"tables" yaml:"tables"` - SkipTables []string `json:"skip_tables" yaml:"skip_tables"` - // Spec yaml.Node `json:"spec" yaml:"spec"` + Registry Registry `json:"registry,omitempty"` + MaxGoRoutines uint64 `json:"max_goroutines,omitempty"` + Tables []string `json:"tables,omitempty"` + SkipTables []string `json:"skip_tables,omitempty"` + Destinations []string `json:"destinations,omitempty"` + Spec interface{} `json:"spec,omitempty"` } -func SetSourceSpecDefault(s *SourceSpec) { - if s.Registry == "" { - s.Registry = "github" +func (s *Source) SetDefaults() { + if s.Registry.String() == "" { + s.Registry = RegistryGithub } if s.Path == "" { s.Path = s.Name @@ -30,34 +32,22 @@ func SetSourceSpecDefault(s *SourceSpec) { if s.Version == "" { s.Version = "latest" } - if !strings.Contains(s.Path, "/") { + if s.Registry == RegistryGithub && !strings.Contains(s.Path, "/") { s.Path = "cloudquery/" + s.Path } } -func (s *SourceSpec) UnmarshalYAML(n *yaml.Node) error { - type S SourceSpec - type T struct { - *S `yaml:",inline"` - } - // This is a neat trick to avoid recursion and use unmarshal as a one stop shop for default setting - obj := &T{S: (*S)(s)} - if err := n.Decode(&obj); err != nil { +func (s *Source) UnmarshalSpec(out interface{}) error { + b, err := json.Marshal(s.Spec) + if err != nil { return err } + dec := json.NewDecoder(nil) + dec.UseNumber() + dec.DisallowUnknownFields() + return json.Unmarshal(b, out) +} - // set default - if s.Registry == "" { - s.Registry = "github" - } - if s.Path == "" { - s.Path = s.Name - } - if s.Version == "" { - s.Version = "latest" - } - if !strings.Contains(s.Path, "/") { - s.Path = "cloudquery/" + s.Path - } - return nil +func (*Source) Validate() (*gojsonschema.Result, error) { + return nil, nil } diff --git a/plugins/source_schema.json b/specs/source_schema.json similarity index 100% rename from plugins/source_schema.json rename to specs/source_schema.json diff --git a/specs/spec.go b/specs/spec.go index 17573fff41..ec4ccbe40f 100644 --- a/specs/spec.go +++ b/specs/spec.go @@ -1,36 +1,121 @@ package specs import ( + "bytes" + "encoding/json" "fmt" - "gopkg.in/yaml.v3" + "github.com/ghodss/yaml" ) +type Kind int + type Spec struct { - Kind string `yaml:"kind"` - Spec interface{} `yaml:"-"` + Kind Kind `json:"kind"` + Spec interface{} `json:"spec"` } -func (s *Spec) UnmarshalYAML(n *yaml.Node) error { - type S Spec - type T struct { - *S `yaml:",inline"` - Spec yaml.Node `yaml:"spec"` - } +const ( + KindSource Kind = iota + KindDestination +) + +func (k Kind) String() string { + return [...]string{"source", "destination"}[k] +} + +func (k Kind) MarshalJSON() ([]byte, error) { + buffer := bytes.NewBufferString(`"`) + buffer.WriteString(k.String()) + buffer.WriteString(`"`) + return buffer.Bytes(), nil +} - obj := &T{S: (*S)(s)} - if err := n.Decode(obj); err != nil { +func (k *Kind) UnmarshalJSON(data []byte) (err error) { + var kind string + if err := json.Unmarshal(data, &kind); err != nil { return err } - switch s.Kind { + if *k, err = KindFromString(kind); err != nil { + return err + } + return nil +} + +func KindFromString(s string) (Kind, error) { + switch s { case "source": - s.Spec = new(SourceSpec) + return KindSource, nil case "destination": - s.Spec = new(DestinationSpec) - case "connection": - s.Spec = new(ConnectionSpec) + return KindDestination, nil + default: + return KindSource, fmt.Errorf("unknown registry %s", s) + } +} + +func (s *Spec) UnmarshalJSON(data []byte) error { + var t struct { + Kind Kind `json:"kind"` + Spec interface{} `json:"spec"` + // Spec yaml.Node `yaml:"spec"` + } + + if err := json.Unmarshal(data, &t); err != nil { + return err + } + s.Kind = t.Kind + switch s.Kind { + case KindSource: + s.Spec = new(Source) + case KindDestination: + s.Spec = new(Destination) default: return fmt.Errorf("unknown kind %s", s.Kind) } - return obj.Spec.Decode(s.Spec) + b, err := json.Marshal(t.Spec) + if err != nil { + return err + } + return json.Unmarshal(b, s.Spec) +} + +func UnmarshalJsonStrict(b []byte, out interface{}) error { + dec := json.NewDecoder(bytes.NewReader(b)) + dec.DisallowUnknownFields() + dec.UseNumber() + return dec.Decode(out) +} + +func SpecUnmarshalYamlStrict(b []byte, spec *Spec) error { + jb, err := yaml.YAMLToJSON(b) + if err != nil { + return fmt.Errorf("failed to convert yaml to json: %w", err) + } + dec := json.NewDecoder(bytes.NewReader(jb)) + dec.DisallowUnknownFields() + dec.UseNumber() + if err := dec.Decode(spec); err != nil { + return fmt.Errorf("failed to decode json: %w", err) + } + switch spec.Kind { + case KindSource: + spec.Spec.(*Source).SetDefaults() + case KindDestination: + spec.Spec.(*Destination).SetDefaults() + default: + return fmt.Errorf("unknown kind %s", spec.Kind) + } + return nil } + +// func (s Spec) MarshalYAML() (interface{}, error) { +// type T struct { +// Kind Kind `json:"kind,omitempty"` +// Spec interface{} `json:"spec,omitempty"` +// } +// tmp := T{ +// Kind: s.Kind, +// Spec: s.Spec, +// } +// return tmp, nil +// } diff --git a/specs/spec_reader.go b/specs/spec_reader.go index 4da475ced6..40478d0e2f 100644 --- a/specs/spec_reader.go +++ b/specs/spec_reader.go @@ -6,21 +6,17 @@ import ( "os" "path/filepath" "strings" - - "gopkg.in/yaml.v3" ) type SpecReader struct { - sources map[string]SourceSpec - destinations map[string]DestinationSpec - connections map[string]ConnectionSpec + sources map[string]Source + destinations map[string]Destination } func NewSpecReader(directory string) (*SpecReader, error) { reader := SpecReader{ - sources: make(map[string]SourceSpec), - destinations: make(map[string]DestinationSpec), - connections: make(map[string]ConnectionSpec), + sources: make(map[string]Source), + destinations: make(map[string]Destination), } files, err := ioutil.ReadDir(directory) if err != nil { @@ -34,16 +30,14 @@ func NewSpecReader(directory string) (*SpecReader, error) { return nil, fmt.Errorf("failed to read file %s: %w", file.Name(), err) } var s Spec - if err := yaml.Unmarshal(data, &s); err != nil { + if err := SpecUnmarshalYamlStrict(data, &s); err != nil { return nil, fmt.Errorf("failed to unmarshal file %s: %w", file.Name(), err) } switch s.Kind { - case "source": - reader.sources[file.Name()] = *s.Spec.(*SourceSpec) - case "destination": - reader.destinations[file.Name()] = *s.Spec.(*DestinationSpec) - case "connection": - reader.connections[file.Name()] = *s.Spec.(*ConnectionSpec) + case KindSource: + reader.sources[file.Name()] = *s.Spec.(*Source) + case KindDestination: + reader.destinations[file.Name()] = *s.Spec.(*Destination) default: return nil, fmt.Errorf("unknown kind %s", s.Kind) } @@ -52,37 +46,28 @@ func NewSpecReader(directory string) (*SpecReader, error) { return &reader, nil } -func (s *SpecReader) GetSourceByName(name string) SourceSpec { +func (s *SpecReader) GetSources() []Source { + sources := make([]Source, 0, len(s.sources)) for _, spec := range s.sources { - if spec.Name == name { - return spec - } + sources = append(sources, spec) } - return SourceSpec{} + return sources } -func (s *SpecReader) GetDestinatinoByName(name string) DestinationSpec { - for _, spec := range s.destinations { +func (s *SpecReader) GetSourceByName(name string) *Source { + for _, spec := range s.sources { if spec.Name == name { - return spec + return &spec } } - return DestinationSpec{} + return nil } -func (s *SpecReader) GetConnectionByName(name string) ConnectionSpec { - for _, spec := range s.connections { - if spec.Source == name { - return spec +func (s *SpecReader) GetDestinatinoByName(name string) *Destination { + for _, spec := range s.destinations { + if spec.Name == name { + return &spec } } - return ConnectionSpec{} -} - -func (s *SpecReader) Connections() []ConnectionSpec { - connections := make([]ConnectionSpec, 0, len(s.connections)) - for _, spec := range s.connections { - connections = append(connections, spec) - } - return connections + return nil } diff --git a/specs/spec_reader_test.go b/specs/spec_reader_test.go index a728f407d3..f169033ece 100644 --- a/specs/spec_reader_test.go +++ b/specs/spec_reader_test.go @@ -1,32 +1,15 @@ package specs -import ( - "reflect" - "testing" -) - -// This test is testing both unmarshalling and LoadSpecs together -// so to add a new test case add a new file to testdata and an expected unmarshaled objectSpec -var expectedSpecs = map[string]interface{}{ - "aws.cq.yml": SourceSpec{ - Name: "aws", - Path: "cloudquery/aws", - Registry: "github", - Version: "1.0.0", - MaxGoRoutines: 10, - }, -} - -func TestLoadSpecs(t *testing.T) { - specReader, err := NewSpecReader("testdata") - if err != nil { - t.Fatal(err) - } - for fileName, spec := range specReader.sources { - t.Run(fileName, func(t *testing.T) { - if !reflect.DeepEqual(spec, expectedSpecs[fileName]) { - t.Fatalf("expected %v, got %v", expectedSpecs[fileName], spec) - } - }) - } -} +// func TestLoadSpecs(t *testing.T) { +// specReader, err := NewSpecReader("testdata") +// if err != nil { +// t.Fatal(err) +// } +// for fileName, spec := range specReader.sources { +// t.Run(fileName, func(t *testing.T) { +// if !reflect.DeepEqual(spec, testSpecs[fileName]) { +// t.Errorf("expected spec %s to be:\n%v\nbut got:\n%v", fileName, testSpecs[fileName].Spec, spec.Spec) +// } +// }) +// } +// } diff --git a/specs/spec_test.go b/specs/spec_test.go new file mode 100644 index 0000000000..789ba0e385 --- /dev/null +++ b/specs/spec_test.go @@ -0,0 +1,50 @@ +package specs + +import ( + "os" + "reflect" + "testing" +) + +var testSpecs = map[string]Spec{ + "testdata/pg.cq.yml": { + Kind: KindDestination, + Spec: &Destination{ + Name: "postgresql", + Path: "postgresql", + Version: "v1.0.0", + Registry: RegistryGrpc, + WriteMode: WriteModeOverwrite, + }, + }, + // "testdata/aws.cq.yml": { + // Kind: KindSource, + // Spec: &Source{ + // Name: "aws", + // Path: "aws", + // Version: "v1.0.0", + // MaxGoRoutines: 10, + // Registry: RegistryLocal, + // }, + // }, +} + +func TestSpecYamlMarshal(t *testing.T) { + for fileName, expectedSpec := range testSpecs { + t.Run(fileName, func(t *testing.T) { + b, err := os.ReadFile(fileName) + if err != nil { + t.Fatal(err) + } + + var spec Spec + if err := SpecUnmarshalYamlStrict(b, &spec); err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(spec, expectedSpec) { + t.Errorf("expected spec %s to be:\n%+v\nbut got:\n%+v", fileName, expectedSpec.Spec, spec.Spec) + } + }) + } +} diff --git a/specs/testdata/aws.cq.yml b/specs/testdata/aws.cq.yml index 3ca996f012..39c9736c5b 100644 --- a/specs/testdata/aws.cq.yml +++ b/specs/testdata/aws.cq.yml @@ -1,7 +1,8 @@ kind: source spec: name: aws - version: 1.0.0 + version: v1.0.0 max_goroutines: 10 - spec: - regions: ["us-east-1"] \ No newline at end of file + registry: local + + \ No newline at end of file diff --git a/specs/testdata/pg.cq.yml b/specs/testdata/pg.cq.yml new file mode 100644 index 0000000000..e187ad190f --- /dev/null +++ b/specs/testdata/pg.cq.yml @@ -0,0 +1,6 @@ +kind: destination +spec: + name: postgresql + version: v1.0.0 + registry: grpc + write_mode: overwrite