Update Plugin SDK to v4

.goreleaser.yaml

@@ -9,7 +9,7 @@ builds:
       - CGO_ENABLED=1
       - GO111MODULE=on
     ldflags:
-      - -s -w -X github.com/github.com/dihedron/cq-source-openstack/plugin.Version={{.Version}}
+      - -s -w -X github.com/github.com/dihedron/cq-source-openstack/resources/plugin.Version={{.Version}}
     goos:
       - windows
       - linux

README.md

@@ -3,10 +3,13 @@
 [![test](https://github.com/github.com/dihedron/cq-source-openstack/actions/workflows/test.yaml/badge.svg)](https://github.com/github.com/dihedron/cq-source-openstack/actions/workflows/test.yaml)
 [![Go Report Card](https://goreportcard.com/badge/github.com/github.com/dihedron/cq-source-openstack)](https://goreportcard.com/report/github.com/github.com/dihedron/cq-source-openstack)
 
-A openstack source plugin for CloudQuery that loads data from OpenStack to any database, data warehouse or data lake supported by [CloudQuery](https://www.cloudquery.io/), such as PostgreSQL, BigQuery, Athena, and many more.
+A source plugin for CloudQuery that loads data from OpenStack to any database, data warehouse or data lake supported by [CloudQuery](https://www.cloudquery.io/), such as PostgreSQL, BigQuery, Athena, and many more.
+
+OpenStack is the most widely used open source cloud platform; it powers hundreds of private clouds and some public clouds too;. 
 
 ## Links
 
+ - [OpenStack APIs](https://docs.openstack.org/api-quick-start/)
  - [CloudQuery Quickstart Guide](https://www.cloudquery.io/docs/quickstart)
  - [Supported Tables](docs/tables/README.md)
 

_test/test.md

@@ -0,0 +1,36 @@
+# How to set up the environment for testing
+
+```sql
+SELECT
+  i.id as instance_id,
+  i.name as instance_name,
+  i.power_state_name as instance_power_state,
+  i.tenant_id as instance_project_id,
+  i.user_id as instance_user_id,
+  i.created_at as instance_created_at,
+  i.launched_at as instance_launched_at,
+  i.updated_at as instance_updated_at,
+  i.terminated_at as instance_terminated_at,
+  i.status as instance_status,
+  i.hostid as hypervisor_id,
+  h.hypervisor_hostname as hypervisor_name,
+  h.status as hypervisor_status,
+  h.state as hypervisor_state,
+  h.disk_available_least as hypervisor_disk_available,
+  h.host_ip as hypervisor_ip_address,
+  h.memory_mb as hypervisor_total_ram_mb,
+  h.free_ram_mb as hypervisor_free_ram_mb,
+  h.memory_mb_used as hypervisor_used_ram_mb,
+  h.running_vms as hypervisor_running_vms,
+  h.vcpus as hypervisor_physical_cpus,
+  h.vcpus_used as hypervisor_used_vcpus,
+  i._cq_id as instance_cqid,
+  h._cq_id as hypervisor_cqid
+FROM 
+  openstack_instances i
+JOIN 
+  openstack_hypervisors h on i.hypervisor_hostname = h.hypervisor_hostname
+;
+
+```
+

client/client.go

@@ -5,76 +5,73 @@ import (
 	"fmt"
 	"sync"
 
-	"github.com/cloudquery/plugin-sdk/plugins/source"
-	"github.com/cloudquery/plugin-sdk/schema"
-	"github.com/cloudquery/plugin-sdk/specs"
 	"github.com/dihedron/cq-plugin-utils/format"
 	"github.com/gophercloud/gophercloud"
 	"github.com/gophercloud/gophercloud/openstack"
 	"github.com/rs/zerolog"
 )
 
 type Client struct {
+	Client *gophercloud.ProviderClient
+	Spec   Spec
+
+	logger zerolog.Logger
+	// tables   schema.Tables
 	mutex    sync.RWMutex
 	services map[ServiceType]*gophercloud.ServiceClient
-	Logger   zerolog.Logger
-	Client   *gophercloud.ProviderClient
-	Specs    *Spec
 }
 
 func (c *Client) ID() string {
 	return "github.com/dihedron/cq-source-openstack"
 }
 
-func New(ctx context.Context, logger zerolog.Logger, s specs.Source, opts source.Options) (schema.ClientMeta, error) {
-	var pluginSpec Spec
+func (c *Client) Logger() *zerolog.Logger {
+	return &c.logger
+}
 
-	if err := s.UnmarshalSpec(&pluginSpec); err != nil {
-		return nil, fmt.Errorf("failed to unmarshal plugin spec: %w", err)
-	}
+func New(ctx context.Context, logger zerolog.Logger, spec *Spec) (*Client, error) {
 
-	// TODO: Add your client initialization here
-	logger.Debug().Str("spec", format.ToJSON(pluginSpec)).Msg("Plugin Spec")
+	logger.Debug().Str("spec", format.ToJSON(spec)).Msg("plugin configuration")
 
 	auth := gophercloud.AuthOptions{
 		AllowReauth: true,
 	}
 
-	if pluginSpec.EndpointUrl != nil {
-		auth.IdentityEndpoint = *pluginSpec.EndpointUrl
+	if spec.EndpointUrl != nil {
+		auth.IdentityEndpoint = *spec.EndpointUrl
 	}
-	if pluginSpec.UserID != nil {
-		auth.UserID = *pluginSpec.UserID
+	if spec.UserID != nil {
+		auth.UserID = *spec.UserID
 	}
-	if pluginSpec.Username != nil {
-		auth.Username = *pluginSpec.Username
+	if spec.Username != nil {
+		auth.Username = *spec.Username
 	}
-	if pluginSpec.Password != nil {
-		auth.Password = *pluginSpec.Password
+	if spec.Password != nil {
+		auth.Password = *spec.Password
 	}
-	if pluginSpec.ProjectID != nil {
-		auth.TenantID = *pluginSpec.ProjectID
+	if spec.ProjectID != nil {
+		auth.TenantID = *spec.ProjectID
 	}
-	if pluginSpec.ProjectName != nil {
-		auth.TenantName = *pluginSpec.ProjectName
+	if spec.ProjectName != nil {
+		auth.TenantName = *spec.ProjectName
 	}
-	if pluginSpec.DomainID != nil {
-		auth.DomainID = *pluginSpec.DomainID
+	if spec.DomainID != nil {
+		auth.DomainID = *spec.DomainID
 	}
-	if pluginSpec.DomainName != nil {
-		auth.DomainName = *pluginSpec.DomainName
+	if spec.DomainName != nil {
+		auth.DomainName = *spec.DomainName
 	}
-	if pluginSpec.AccessToken != nil {
-		auth.TokenID = *pluginSpec.AccessToken
+	if spec.AccessToken != nil {
+		auth.TokenID = *spec.AccessToken
 	}
-	if pluginSpec.AppCredentialID != nil {
-		auth.ApplicationCredentialID = *pluginSpec.AppCredentialID
+	if spec.AppCredentialID != nil {
+		auth.ApplicationCredentialID = *spec.AppCredentialID
 	}
-	if pluginSpec.AppCredentialSecret != nil {
-		auth.ApplicationCredentialSecret = *pluginSpec.AppCredentialSecret
+	if spec.AppCredentialSecret != nil {
+		auth.ApplicationCredentialSecret = *spec.AppCredentialSecret
 	}
-	if pluginSpec.AllowReauth != nil {
-		auth.AllowReauth = *pluginSpec.AllowReauth
+	if spec.AllowReauth != nil {
+		auth.AllowReauth = *spec.AllowReauth
 	}
 
 	//
@@ -91,18 +88,18 @@ func New(ctx context.Context, logger zerolog.Logger, s specs.Source, opts source
 	logger.Info().Msg("openstack client created")
 
 	return &Client{
-		Logger:   logger,
+		Spec:     *spec,
+		logger:   logger,
 		Client:   client,
 		services: map[ServiceType]*gophercloud.ServiceClient{},
-		Specs:    &pluginSpec,
 	}, nil
 }
 
 func (c *Client) GetServiceClient(key ServiceType) (*gophercloud.ServiceClient, error) {
 
 	c.mutex.RLock()
 	if service, ok := c.services[key]; ok && service != nil {
-		c.Logger.Info().Str("type", string(key)).Msg("returning existing service client")
+		c.Logger().Info().Str("type", string(key)).Msg("returning existing service client")
 		c.mutex.RUnlock()
 		return service, nil
 	}
@@ -114,31 +111,31 @@ func (c *Client) initServiceClient(key ServiceType) (*gophercloud.ServiceClient,
 
 	// no existing service client, need to initialise one
 	if _, ok := serviceConfigMap[key]; !ok {
-		c.Logger.Error().Str("type", string(key)).Msg("invalid service client type")
+		c.Logger().Error().Str("type", string(key)).Msg("invalid service client type")
 		panic(fmt.Sprintf("invalid service type: %q", key))
 	}
 
-	c.Logger.Info().Str("type", string(key)).Msg("creating new service client")
+	c.Logger().Info().Str("type", string(key)).Msg("creating new service client")
 
 	region := ""
-	if c.Specs.Region != nil {
-		region = *c.Specs.Region
+	if c.Spec.Region != nil {
+		region = *c.Spec.Region
 	}
 
 	client, err := serviceConfigMap[key].newClient(c.Client, gophercloud.EndpointOpts{Region: region})
 
 	if err != nil {
-		c.Logger.Error().Str("type", string(key)).Err(err).Msg("error creating service client")
+		c.Logger().Error().Str("type", string(key)).Err(err).Msg("error creating service client")
 		return nil, err
 	}
-	client.Microversion = serviceConfigMap[key].getMicroversion(c.Specs)
+	client.Microversion = serviceConfigMap[key].getMicroversion(&c.Spec)
 
 	// save to object
 	c.mutex.Lock()
 	defer c.mutex.Unlock()
 	c.services[key] = client
 
-	c.Logger.Info().Str("type", string(key)).Msg("new service client ready")
+	c.Logger().Info().Str("type", string(key)).Msg("new service client ready")
 
 	return client, nil
 }

client/spec.go

@@ -20,3 +20,11 @@ type Spec struct {
 	BlockStorageV3Microversion *string `json:"blockstorage_v3_microversion,omitempty" yaml:"blockstorage_v3_microversion,omitempty"`
 	ImageServiceV2Microversion *string `json:"imageservice_v2_microversion,omitempty" yaml:"imageservice_v2_microversion,omitempty"`
 }
+
+func (s *Spec) Validate() error {
+	// TODO: implement
+	return nil
+}
+
+func (s *Spec) SetDefaults() {
+}

go.mod

@@ -1,41 +1,68 @@
 module github.com/dihedron/cq-source-openstack
 
-go 1.19
+go 1.21
+
+toolchain go1.21.0
 
 require (
-	github.com/cloudquery/plugin-sdk v1.45.0
-	github.com/dihedron/cq-plugin-utils v0.0.0-20230515115130-eb4d7b050d1c
-	github.com/gophercloud/gophercloud v1.3.0
+	github.com/apache/arrow/go/v13 v13.0.0-20230731205701-112f94971882
+	github.com/cloudquery/plugin-sdk/v4 v4.5.0
+	github.com/dihedron/cq-plugin-utils v0.0.0-20230816150611-90ab3d855604
+	github.com/gophercloud/gophercloud v1.5.0
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/rs/zerolog v1.29.1
 )
 
 require (
+	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
+	github.com/cloudquery/plugin-pb-go v1.9.2 // indirect
+	github.com/cloudquery/plugin-sdk/v2 v2.7.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/getsentry/sentry-go v0.20.0 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
+	github.com/go-logr/logr v1.2.4 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/flatbuffers v23.5.26+incompatible // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware/providers/zerolog/v2 v2.0.0-rc.3 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.3 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
 	github.com/hashicorp/errwrap v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/klauspost/compress v1.16.7 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.18 // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/pierrec/lz4/v4 v4.1.18 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/cobra v1.7.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/stretchr/testify v1.8.2 // indirect
+	github.com/stretchr/testify v1.8.4 // indirect
 	github.com/thoas/go-funk v0.9.3 // indirect
-	golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect
-	golang.org/x/net v0.9.0 // indirect
+	github.com/zeebo/xxh3 v1.0.2 // indirect
+	go.opentelemetry.io/otel v1.16.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.16.0 // indirect
+	go.opentelemetry.io/otel/metric v1.16.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.16.0 // indirect
+	go.opentelemetry.io/otel/trace v1.16.0 // indirect
+	go.opentelemetry.io/proto/otlp v0.19.0 // indirect
+	golang.org/x/exp v0.0.0-20230728194245-b0cb94b80691 // indirect
+	golang.org/x/mod v0.11.0 // indirect
+	golang.org/x/net v0.10.0 // indirect
 	golang.org/x/sync v0.1.0 // indirect
-	golang.org/x/sys v0.7.0 // indirect
+	golang.org/x/sys v0.8.0 // indirect
 	golang.org/x/text v0.9.0 // indirect
-	google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
-	google.golang.org/grpc v1.54.0 // indirect
-	google.golang.org/protobuf v1.30.0 // indirect
+	golang.org/x/tools v0.6.0 // indirect
+	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
+	google.golang.org/genproto v0.0.0-20230731193218-e0aa005b6bdf // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20230731193218-e0aa005b6bdf // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230731193218-e0aa005b6bdf // indirect
+	google.golang.org/grpc v1.57.0 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )