fix(oracle)!: Add region & availability_domain to primary keys (#…

…11042)

Refactor as a follow-up to #11037

BEGIN_COMMIT_OVERRIDE
fix(oracle)!: Add `region` & `availability_domain` to primary keys(#11042)
BREAKING-CHANGE:  The following primary key columns were added:
* `oracle_filestorage_export_sets`: `availability_domain`
* `oracle_filestorage_file_systems`: `availability_domain`
* `oracle_filestorage_mount_targets`: `availability_domain`
* `oracle_filestorage_replication_targets`: `availability_domain`
* `oracle_filestorage_replications`: `availability_domain`
* `oracle_identity_compartments`: `region`
* `oracle_identity_cost_tracking_tags`: `region`
* `oracle_identity_dynamic_groups`: `region`
* `oracle_identity_groups`: `region`
* `oracle_identity_network_sources`: `region`
* `oracle_identity_users`: `region`

END_COMMIT_OVERRIDE

plugins/source/oracle/client/client.go

@@ -32,7 +32,7 @@ type Client struct {
 	// By default (if no multiplexer is defined), Region is set to the home region, and CompartmentOcid is set to the tenancy ocid.
 	Region             string
 	CompartmentOcid    string
-	AvailibilityDomain string // For fetches that are multiplexed by availibility domain (and not region)
+	AvailabilityDomain string // For fetches that are multiplexed by availibility domain (and not region)
 
 	logger zerolog.Logger
 }
@@ -98,7 +98,7 @@ func (c *Client) ID() string {
 	idStrings := []string{
 		c.Region,
 		c.CompartmentOcid,
-		c.AvailibilityDomain,
+		c.AvailabilityDomain,
 		c.HomeRegion,
 		c.TenancyOcid,
 	}
@@ -123,10 +123,10 @@ func (c *Client) withCompartment(compartmentOcid string) *Client {
 	return &newClient
 }
 
-func (c *Client) withAvailibilityDomain(availabilityDomain string) *Client {
+func (c *Client) withAvailabilityDomain(availabilityDomain string) *Client {
 	newClient := *c
 	newClient.logger = c.logger.With().Str("availability_domain", availabilityDomain).Logger()
-	newClient.AvailibilityDomain = availabilityDomain
+	newClient.AvailabilityDomain = availabilityDomain
 	return &newClient
 }
 

plugins/source/oracle/client/columns.go

@@ -0,0 +1,27 @@
+package client
+
+import (
+	"github.com/apache/arrow/go/v13/arrow"
+	"github.com/cloudquery/plugin-sdk/v3/schema"
+)
+
+var (
+	RegionColumn = schema.Column{
+		Name:       "region",
+		Type:       arrow.BinaryTypes.String,
+		Resolver:   ResolveOracleRegion,
+		PrimaryKey: true,
+	}
+	CompartmentIDColumn = schema.Column{
+		Name:       "compartment_id",
+		Type:       arrow.BinaryTypes.String,
+		Resolver:   ResolveCompartmentID,
+		PrimaryKey: true,
+	}
+	AvailabilityDomainColumn = schema.Column{
+		Name:       "availability_domain",
+		Type:       arrow.BinaryTypes.String,
+		Resolver:   ResolveAvailabilityDomain,
+		PrimaryKey: true,
+	}
+)

plugins/source/oracle/client/multiplex.go

@@ -40,7 +40,7 @@ func AvailibilityDomainCompartmentMultiplex(meta schema.ClientMeta) []schema.Cli
 				multiplexedClients = append(multiplexedClients, cqClient.
 					withRegion(region).
 					withCompartment(compartmentOcid).
-					withAvailibilityDomain(availibilityDomain))
+					withAvailabilityDomain(availibilityDomain))
 			}
 		}
 	}

plugins/source/oracle/client/resolvers.go

@@ -0,0 +1,25 @@
+package client
+
+import (
+	"context"
+
+	"github.com/cloudquery/plugin-sdk/v3/schema"
+)
+
+// ResolveOracleRegion grabs the region from the cq-client, not from the resource.
+func ResolveOracleRegion(_ context.Context, meta schema.ClientMeta, r *schema.Resource, c schema.Column) error {
+	client := meta.(*Client)
+	return r.Set(c.Name, client.Region)
+}
+
+// ResolveCompartmentID grabs the compartment-id from the cq-client, not from the resource.
+func ResolveCompartmentID(_ context.Context, meta schema.ClientMeta, r *schema.Resource, c schema.Column) error {
+	client := meta.(*Client)
+	return r.Set(c.Name, client.CompartmentOcid)
+}
+
+// ResolveAvailabilityDomain grabs the compartment-id from the cq-client, not from the resource.
+func ResolveAvailabilityDomain(_ context.Context, meta schema.ClientMeta, r *schema.Resource, c schema.Column) error {
+	client := meta.(*Client)
+	return r.Set(c.Name, client.AvailabilityDomain)
+}

plugins/source/oracle/client/transform.go

@@ -4,20 +4,30 @@ import (
 	"reflect"
 
 	"github.com/apache/arrow/go/v13/arrow"
+	"github.com/cloudquery/plugin-sdk/v3/schema"
 	"github.com/cloudquery/plugin-sdk/v3/transformers"
 	"github.com/oracle/oci-go-sdk/v65/common"
 )
 
-func OracleTypeTransformer(field reflect.StructField) (arrow.DataType, error) {
+func typeTransformer(field reflect.StructField) (arrow.DataType, error) {
 	fieldType := field.Type
 
-	if fieldType.Kind() == reflect.Ptr {
+	for fieldType.Kind() == reflect.Pointer {
 		fieldType = fieldType.Elem()
 	}
 
-	if fieldType.Kind() == reflect.Struct && fieldType == reflect.TypeOf(common.SDKTime{}) {
+	if fieldType == reflect.TypeOf(common.SDKTime{}) {
 		return arrow.FixedWidthTypes.Timestamp_us, nil
 	}
 
-	return transformers.DefaultTypeTransformer(field)
+	return nil, nil
+}
+
+var options = []transformers.StructTransformerOption{
+	transformers.WithPrimaryKeys("Id"),
+	transformers.WithTypeTransformer(typeTransformer),
+}
+
+func TransformWithStruct(t any, opts ...transformers.StructTransformerOption) schema.Transform {
+	return transformers.TransformWithStruct(t, append(options, opts...)...)
 }

plugins/source/oracle/resources/services/blockstorage/boot_volume_backups.go

@@ -1,10 +1,8 @@
 package blockstorage
 
 import (
-	"github.com/apache/arrow/go/v13/arrow"
 	"github.com/cloudquery/cloudquery/plugins/source/oracle/client"
 	"github.com/cloudquery/plugin-sdk/v3/schema"
-	"github.com/cloudquery/plugin-sdk/v3/transformers"
 	"github.com/oracle/oci-go-sdk/v65/core"
 )
 
@@ -13,27 +11,7 @@ func BootVolumeBackups() *schema.Table {
 		Name:      "oracle_blockstorage_boot_volume_backups",
 		Resolver:  fetchBootVolumeBackups,
 		Multiplex: client.RegionCompartmentMultiplex,
-		Transform: transformers.TransformWithStruct(&core.BootVolumeBackup{},
-			transformers.WithTypeTransformer(client.OracleTypeTransformer)),
-		Columns: []schema.Column{
-			{
-				Name:       "region",
-				Type:       arrow.BinaryTypes.String,
-				Resolver:   client.ResolveOracleRegion,
-				PrimaryKey: true,
-			},
-			{
-				Name:       "compartment_id",
-				Type:       arrow.BinaryTypes.String,
-				Resolver:   client.ResolveCompartmentId,
-				PrimaryKey: true,
-			},
-			{
-				Name:       "id",
-				Type:       arrow.BinaryTypes.String,
-				Resolver:   schema.PathResolver("Id"),
-				PrimaryKey: true,
-			},
-		},
+		Transform: client.TransformWithStruct(&core.BootVolumeBackup{}),
+		Columns:   schema.ColumnList{client.RegionColumn, client.CompartmentIDColumn},
 	}
 }

plugins/source/oracle/resources/services/blockstorage/boot_volume_replicas.go

@@ -1,10 +1,8 @@
 package blockstorage
 
 import (
-	"github.com/apache/arrow/go/v13/arrow"
 	"github.com/cloudquery/cloudquery/plugins/source/oracle/client"
 	"github.com/cloudquery/plugin-sdk/v3/schema"
-	"github.com/cloudquery/plugin-sdk/v3/transformers"
 	"github.com/oracle/oci-go-sdk/v65/core"
 )
 
@@ -13,27 +11,7 @@ func BootVolumeReplicas() *schema.Table {
 		Name:      "oracle_blockstorage_boot_volume_replicas",
 		Resolver:  fetchBootVolumeReplicas,
 		Multiplex: client.RegionCompartmentMultiplex,
-		Transform: transformers.TransformWithStruct(&core.BootVolumeReplica{},
-			transformers.WithTypeTransformer(client.OracleTypeTransformer)),
-		Columns: []schema.Column{
-			{
-				Name:       "region",
-				Type:       arrow.BinaryTypes.String,
-				Resolver:   client.ResolveOracleRegion,
-				PrimaryKey: true,
-			},
-			{
-				Name:       "compartment_id",
-				Type:       arrow.BinaryTypes.String,
-				Resolver:   client.ResolveCompartmentId,
-				PrimaryKey: true,
-			},
-			{
-				Name:       "id",
-				Type:       arrow.BinaryTypes.String,
-				Resolver:   schema.PathResolver("Id"),
-				PrimaryKey: true,
-			},
-		},
+		Transform: client.TransformWithStruct(&core.BootVolumeReplica{}),
+		Columns:   schema.ColumnList{client.RegionColumn, client.CompartmentIDColumn},
 	}
 }

plugins/source/oracle/resources/services/blockstorage/boot_volumes.go

@@ -1,10 +1,8 @@
 package blockstorage
 
 import (
-	"github.com/apache/arrow/go/v13/arrow"
 	"github.com/cloudquery/cloudquery/plugins/source/oracle/client"
 	"github.com/cloudquery/plugin-sdk/v3/schema"
-	"github.com/cloudquery/plugin-sdk/v3/transformers"
 	"github.com/oracle/oci-go-sdk/v65/core"
 )
 
@@ -13,27 +11,7 @@ func BootVolumes() *schema.Table {
 		Name:      "oracle_blockstorage_boot_volumes",
 		Resolver:  fetchBootVolumes,
 		Multiplex: client.RegionCompartmentMultiplex,
-		Transform: transformers.TransformWithStruct(&core.BootVolume{},
-			transformers.WithTypeTransformer(client.OracleTypeTransformer)),
-		Columns: []schema.Column{
-			{
-				Name:       "region",
-				Type:       arrow.BinaryTypes.String,
-				Resolver:   client.ResolveOracleRegion,
-				PrimaryKey: true,
-			},
-			{
-				Name:       "compartment_id",
-				Type:       arrow.BinaryTypes.String,
-				Resolver:   client.ResolveCompartmentId,
-				PrimaryKey: true,
-			},
-			{
-				Name:       "id",
-				Type:       arrow.BinaryTypes.String,
-				Resolver:   schema.PathResolver("Id"),
-				PrimaryKey: true,
-			},
-		},
+		Transform: client.TransformWithStruct(&core.BootVolume{}),
+		Columns:   schema.ColumnList{client.RegionColumn, client.CompartmentIDColumn},
 	}
 }

plugins/source/oracle/resources/services/blockstorage/volume_backup_policies.go

@@ -1,10 +1,8 @@
 package blockstorage
 
 import (
-	"github.com/apache/arrow/go/v13/arrow"
 	"github.com/cloudquery/cloudquery/plugins/source/oracle/client"
 	"github.com/cloudquery/plugin-sdk/v3/schema"
-	"github.com/cloudquery/plugin-sdk/v3/transformers"
 	"github.com/oracle/oci-go-sdk/v65/core"
 )
 
@@ -13,27 +11,7 @@ func VolumeBackupPolicies() *schema.Table {
 		Name:      "oracle_blockstorage_volume_backup_policies",
 		Resolver:  fetchVolumeBackupPolicies,
 		Multiplex: client.RegionCompartmentMultiplex,
-		Transform: transformers.TransformWithStruct(&core.VolumeBackupPolicy{},
-			transformers.WithTypeTransformer(client.OracleTypeTransformer)),
-		Columns: []schema.Column{
-			{
-				Name:       "region",
-				Type:       arrow.BinaryTypes.String,
-				Resolver:   client.ResolveOracleRegion,
-				PrimaryKey: true,
-			},
-			{
-				Name:       "compartment_id",
-				Type:       arrow.BinaryTypes.String,
-				Resolver:   client.ResolveCompartmentId,
-				PrimaryKey: true,
-			},
-			{
-				Name:       "id",
-				Type:       arrow.BinaryTypes.String,
-				Resolver:   schema.PathResolver("Id"),
-				PrimaryKey: true,
-			},
-		},
+		Transform: client.TransformWithStruct(&core.VolumeBackupPolicy{}),
+		Columns:   schema.ColumnList{client.RegionColumn, client.CompartmentIDColumn},
 	}
 }

plugins/source/oracle/resources/services/blockstorage/volume_backups.go

@@ -1,10 +1,8 @@
 package blockstorage
 
 import (
-	"github.com/apache/arrow/go/v13/arrow"
 	"github.com/cloudquery/cloudquery/plugins/source/oracle/client"
 	"github.com/cloudquery/plugin-sdk/v3/schema"
-	"github.com/cloudquery/plugin-sdk/v3/transformers"
 	"github.com/oracle/oci-go-sdk/v65/core"
 )
 
@@ -13,27 +11,7 @@ func VolumeBackups() *schema.Table {
 		Name:      "oracle_blockstorage_volume_backups",
 		Resolver:  fetchVolumeBackups,
 		Multiplex: client.RegionCompartmentMultiplex,
-		Transform: transformers.TransformWithStruct(&core.VolumeBackup{},
-			transformers.WithTypeTransformer(client.OracleTypeTransformer)),
-		Columns: []schema.Column{
-			{
-				Name:       "region",
-				Type:       arrow.BinaryTypes.String,
-				Resolver:   client.ResolveOracleRegion,
-				PrimaryKey: true,
-			},
-			{
-				Name:       "compartment_id",
-				Type:       arrow.BinaryTypes.String,
-				Resolver:   client.ResolveCompartmentId,
-				PrimaryKey: true,
-			},
-			{
-				Name:       "id",
-				Type:       arrow.BinaryTypes.String,
-				Resolver:   schema.PathResolver("Id"),
-				PrimaryKey: true,
-			},
-		},
+		Transform: client.TransformWithStruct(&core.VolumeBackup{}),
+		Columns:   schema.ColumnList{client.RegionColumn, client.CompartmentIDColumn},
 	}
 }

plugins/source/oracle/resources/services/blockstorage/volume_group_backups.go

@@ -1,10 +1,8 @@
 package blockstorage
 
 import (
-	"github.com/apache/arrow/go/v13/arrow"
 	"github.com/cloudquery/cloudquery/plugins/source/oracle/client"
 	"github.com/cloudquery/plugin-sdk/v3/schema"
-	"github.com/cloudquery/plugin-sdk/v3/transformers"
 	"github.com/oracle/oci-go-sdk/v65/core"
 )
 
@@ -13,27 +11,7 @@ func VolumeGroupBackups() *schema.Table {
 		Name:      "oracle_blockstorage_volume_group_backups",
 		Resolver:  fetchVolumeGroupBackups,
 		Multiplex: client.RegionCompartmentMultiplex,
-		Transform: transformers.TransformWithStruct(&core.VolumeGroupBackup{},
-			transformers.WithTypeTransformer(client.OracleTypeTransformer)),
-		Columns: []schema.Column{
-			{
-				Name:       "region",
-				Type:       arrow.BinaryTypes.String,
-				Resolver:   client.ResolveOracleRegion,
-				PrimaryKey: true,
-			},
-			{
-				Name:       "compartment_id",
-				Type:       arrow.BinaryTypes.String,
-				Resolver:   client.ResolveCompartmentId,
-				PrimaryKey: true,
-			},
-			{
-				Name:       "id",
-				Type:       arrow.BinaryTypes.String,
-				Resolver:   schema.PathResolver("Id"),
-				PrimaryKey: true,
-			},
-		},
+		Transform: client.TransformWithStruct(&core.VolumeGroupBackup{}),
+		Columns:   schema.ColumnList{client.RegionColumn, client.CompartmentIDColumn},
 	}
 }