feat: Add publish command

examples/simple_plugin/.gitignore

@@ -0,0 +1 @@
+dist

examples/simple_plugin/client/client.go

@@ -0,0 +1,38 @@
+package client
+
+import (
+	"context"
+
+	"github.com/cloudquery/plugin-sdk/v4/schema"
+	"github.com/rs/zerolog"
+)
+
+type TestClient struct {
+	Logger   zerolog.Logger
+	Spec     Spec
+	ClientID int
+}
+
+func (*TestClient) ID() string {
+	return "TestClient"
+}
+
+func (c *TestClient) withClientID(i int) *TestClient {
+	t := *c
+	t.ClientID = i
+	return &t
+}
+
+func MultiplexBySpec(meta schema.ClientMeta) []schema.ClientMeta {
+	cl := meta.(*TestClient)
+	clients := make([]schema.ClientMeta, cl.Spec.NumClients)
+	for i := 0; i < cl.Spec.NumClients; i++ {
+		clients[i] = cl.withClientID(i)
+	}
+	return clients
+}
+
+func ResolveClientID(_ context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error {
+	cl := meta.(*TestClient)
+	return resource.Set(c.Name, cl.ClientID)
+}

examples/simple_plugin/client/spec.go

@@ -0,0 +1,15 @@
+package client
+
+type Spec struct {
+	NumClients int `json:"num_clients"`
+}
+
+func (s *Spec) SetDefaults() {
+	if s.NumClients <= 0 {
+		s.NumClients = 1
+	}
+}
+
+func (*Spec) Validate() error {
+	return nil
+}

examples/simple_plugin/go.mod

@@ -0,0 +1,64 @@
+module github.com/cloudquery/plugin-sdk/examples/simple_plugin
+
+go 1.20
+
+require (
+	github.com/apache/arrow/go/v13 v13.0.0-20230731205701-112f94971882
+	github.com/cloudquery/plugin-sdk/v4 v4.4.0
+	github.com/rs/zerolog v1.30.0
+)
+
+replace github.com/cloudquery/plugin-sdk/v4 => ../../
+
+require (
+	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
+	github.com/cloudquery/plugin-pb-go v1.9.2 // indirect
+	github.com/cloudquery/plugin-sdk/v2 v2.7.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/getsentry/sentry-go v0.20.0 // indirect
+	github.com/ghodss/yaml v1.0.0 // indirect
+	github.com/go-logr/logr v1.2.4 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/goccy/go-json v0.10.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/flatbuffers v23.5.26+incompatible // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/grpc-ecosystem/go-grpc-middleware/providers/zerolog/v2 v2.0.0-rc.3 // indirect
+	github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.3 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/klauspost/compress v1.16.7 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/pierrec/lz4/v4 v4.1.18 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/spf13/cobra v1.6.1 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/stretchr/testify v1.8.4 // indirect
+	github.com/thoas/go-funk v0.9.3 // indirect
+	github.com/zeebo/xxh3 v1.0.2 // indirect
+	go.opentelemetry.io/otel v1.16.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.16.0 // indirect
+	go.opentelemetry.io/otel/metric v1.16.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.16.0 // indirect
+	go.opentelemetry.io/otel/trace v1.16.0 // indirect
+	go.opentelemetry.io/proto/otlp v0.19.0 // indirect
+	golang.org/x/exp v0.0.0-20230728194245-b0cb94b80691 // indirect
+	golang.org/x/mod v0.11.0 // indirect
+	golang.org/x/net v0.10.0 // indirect
+	golang.org/x/sync v0.1.0 // indirect
+	golang.org/x/sys v0.8.0 // indirect
+	golang.org/x/text v0.9.0 // indirect
+	golang.org/x/tools v0.6.0 // indirect
+	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
+	google.golang.org/genproto v0.0.0-20230731193218-e0aa005b6bdf // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20230731193218-e0aa005b6bdf // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230731193218-e0aa005b6bdf // indirect
+	google.golang.org/grpc v1.57.0 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

examples/simple_plugin/main.go

@@ -0,0 +1,16 @@
+package main
+
+import (
+	"context"
+	"log"
+
+	"github.com/cloudquery/plugin-sdk/examples/simple_plugin/plugin"
+	"github.com/cloudquery/plugin-sdk/v4/serve"
+)
+
+func main() {
+	p := serve.Plugin(plugin.Plugin())
+	if err := p.Serve(context.Background()); err != nil {
+		log.Fatalf("failed to serve plugin: %v", err)
+	}
+}

examples/simple_plugin/plugin/client.go

@@ -0,0 +1,96 @@
+package plugin
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/cloudquery/plugin-sdk/examples/simple_plugin/client"
+	"github.com/cloudquery/plugin-sdk/examples/simple_plugin/services"
+	"github.com/cloudquery/plugin-sdk/v4/message"
+	"github.com/cloudquery/plugin-sdk/v4/plugin"
+	"github.com/cloudquery/plugin-sdk/v4/scheduler"
+	"github.com/cloudquery/plugin-sdk/v4/schema"
+	"github.com/cloudquery/plugin-sdk/v4/transformers"
+	"github.com/rs/zerolog"
+)
+
+type Client struct {
+	logger    zerolog.Logger
+	config    client.Spec
+	tables    schema.Tables
+	scheduler *scheduler.Scheduler
+
+	plugin.UnimplementedDestination
+}
+
+func (c *Client) Logger() *zerolog.Logger {
+	return &c.logger
+}
+
+func (c *Client) Sync(ctx context.Context, options plugin.SyncOptions, res chan<- message.SyncMessage) error {
+	tt, err := c.tables.FilterDfs(options.Tables, options.SkipTables, options.SkipDependentTables)
+	if err != nil {
+		return err
+	}
+
+	schedulerClient := &client.TestClient{
+		Logger: c.logger,
+		Spec:   c.config,
+	}
+
+	return c.scheduler.Sync(ctx, schedulerClient, tt, res, scheduler.WithSyncDeterministicCQID(options.DeterministicCQID))
+}
+
+func (c *Client) Tables(_ context.Context, options plugin.TableOptions) (schema.Tables, error) {
+	tt, err := c.tables.FilterDfs(options.Tables, options.SkipTables, options.SkipDependentTables)
+	if err != nil {
+		return nil, err
+	}
+
+	return tt, nil
+}
+
+func (*Client) Close(_ context.Context) error {
+	return nil
+}
+
+func Configure(_ context.Context, logger zerolog.Logger, spec []byte, opts plugin.NewClientOptions) (plugin.Client, error) {
+	if opts.NoConnection {
+		return &Client{
+			logger: logger,
+			tables: getTables(),
+		}, nil
+	}
+
+	config := &client.Spec{}
+	if err := json.Unmarshal(spec, config); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal spec: %w", err)
+	}
+	config.SetDefaults()
+	if err := config.Validate(); err != nil {
+		return nil, fmt.Errorf("failed to validate spec: %w", err)
+	}
+
+	return &Client{
+		logger: logger,
+		config: *config,
+		scheduler: scheduler.NewScheduler(
+			scheduler.WithLogger(logger),
+		),
+		tables: getTables(),
+	}, nil
+}
+
+func getTables() schema.Tables {
+	tables := schema.Tables{
+		services.TestSomeTable(),
+	}
+	if err := transformers.TransformTables(tables); err != nil {
+		panic(err)
+	}
+	for _, t := range tables {
+		schema.AddCqIDs(t)
+	}
+	return tables
+}

examples/simple_plugin/plugin/plugin.go

@@ -0,0 +1,17 @@
+package plugin
+
+import (
+	"github.com/cloudquery/plugin-sdk/v4/plugin"
+)
+
+var (
+	Version = "development"
+)
+
+func Plugin() *plugin.Plugin {
+	return plugin.NewPlugin(
+		"test",
+		Version,
+		Configure,
+	)
+}

examples/simple_plugin/services/test.go

@@ -0,0 +1,48 @@
+package services
+
+import (
+	"context"
+
+	"github.com/apache/arrow/go/v13/arrow"
+	"github.com/cloudquery/plugin-sdk/examples/simple_plugin/client"
+	"github.com/cloudquery/plugin-sdk/v4/schema"
+)
+
+func TestSomeTable() *schema.Table {
+	return &schema.Table{
+		Name:        "test_some_table",
+		Description: "Test description",
+		Resolver:    fetchSomeTableData,
+		Multiplex:   client.MultiplexBySpec,
+		Columns: []schema.Column{
+			{
+				Name:        "column1",
+				Description: "Test Column 1",
+				Type:        arrow.BinaryTypes.String,
+				PrimaryKey:  true,
+				Resolver:    schema.PathResolver("column1"),
+			},
+			{
+				Name:        "column2",
+				Description: "Test Column 2",
+				Type:        arrow.PrimitiveTypes.Int64,
+				Resolver:    schema.PathResolver("column2"),
+			},
+			{
+				Name:        "client_id",
+				Description: "ID of client",
+				Type:        arrow.PrimitiveTypes.Int64,
+				Resolver:    client.ResolveClientID,
+				PrimaryKey:  true,
+			},
+		},
+	}
+}
+
+func fetchSomeTableData(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error {
+	res <- map[string]any{
+		"column1": "test_project_id",
+		"column2": 123,
+	}
+	return nil
+}

plugin/options.go

@@ -17,6 +17,24 @@ func (m MigrateMode) String() string {
 
 type Option func(*Plugin)
 
+func WithTitle(title string) Option {
+	return func(p *Plugin) {
+		p.title = title
+	}
+}
+
+func WithDescription(description string) Option {
+	return func(p *Plugin) {
+		p.description = description
+	}
+}
+
+func WithShortDescription(shortDescription string) Option {
+	return func(p *Plugin) {
+		p.shortDescription = shortDescription
+	}
+}
+
 type TableOptions struct {
 	Tables              []string
 	SkipTables          []string

plugin/plugin.go

@@ -51,6 +51,16 @@ type Plugin struct {
 	name string
 	// Version of the plugin
 	version string
+	// Title of the plugin as appears in CloudQuery registry
+	title string
+	// Short description of the plugin as appears in CloudQuery registry
+	shortDescription string
+	// Long description of the plugin as appears in CloudQuery registry
+	description string
+	// categories of the plugin as appears in CloudQuery registry
+	categories []string
+	// targets to build plugin for
+	targets []BuildTarget
 	// Called upon init call to validate and init configuration
 	newClient NewClientFunc
 	// Logger to call, this logger is passed to the serve.Serve Client, if not defined Serve will create one instead.
@@ -74,6 +84,9 @@ func NewPlugin(name string, version string, newClient NewClientFunc, options ...
 		version:         version,
 		internalColumns: true,
 		newClient:       newClient,
+		title:           name,
+		categories:      []string{},
+		targets:         buildTargets,
 	}
 	for _, opt := range options {
 		opt(&p)
@@ -91,6 +104,26 @@ func (p *Plugin) Version() string {
 	return p.version
 }
 
+func (p *Plugin) Title() string {
+	return p.title
+}
+
+func (p *Plugin) Description() string {
+	return p.description
+}
+
+func (p *Plugin) ShortDescription() string {
+	return p.shortDescription
+}
+
+func (p *Plugin) Categories() []string {
+	return p.categories
+}
+
+func (p *Plugin) Targets() []BuildTarget {
+	return p.targets
+}
+
 func (p *Plugin) SetLogger(logger zerolog.Logger) {
 	p.logger = logger.With().Str("module", p.name+"-src").Logger()
 }