Add AWS: redshift.cluster_subnet_groups and rds.db_subnet_groups

#16
#17

cmd/config.go

@@ -49,7 +49,9 @@ var awsConfig = `
       - name: kms.keys
       - name: rds.certificates
       - name: rds.clusters
+      - name: rds.db_subnet_groups
       - name: redshift.clusters
+      - name: redshift.cluster_subnet_groups
       - name: s3.buckets`
 
 var gcpConfig = `

providers/aws/provider.go

@@ -121,7 +121,9 @@ var migrateFunctions = []func(*gorm.DB) error{
 	kms.MigrateKeys,
 	rds.MigrateClusters,
 	rds.MigrateCertificates,
+	rds.MigrateDBSubnetGroups,
 	redshift.MigrateClusters,
+	redshift.MigrateClusterSubnetGroups,
 	s3.MigrateBuckets,
 }
 

providers/aws/rds/client.go

@@ -37,7 +37,9 @@ func (c *Client) CollectResource(resource string, config interface{}) error {
 		return c.certificates(config)
 	case "clusters":
 		return c.clusters(config)
+	case "db_subnet_groups":
+		return c.dbSubnetGroups(config)
 	default:
-		return fmt.Errorf("unsupported resource iam.%s", resource)
+		return fmt.Errorf("unsupported resource rds.%s", resource)
 	}
 }

providers/aws/rds/subnet_groups.go

@@ -0,0 +1,107 @@
+package rds
+
+import (
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/rds"
+	"github.com/cloudquery/cloudquery/providers/common"
+	"github.com/mitchellh/mapstructure"
+	"go.uber.org/zap"
+	"gorm.io/gorm"
+)
+
+type DBSubnetGroup struct {
+	ID                       uint `gorm:"primarykey"`
+	AccountID                string
+	Region                   string
+	DBSubnetGroupArn         *string
+	DBSubnetGroupDescription *string
+	DBSubnetGroupName        *string
+	SubnetGroupStatus        *string
+	Subnets                  []*DBSubnetGroupSubnet `gorm:"constraint:OnDelete:CASCADE;"`
+	VpcId                    *string
+}
+
+func (DBSubnetGroup) TableName() string {
+	return "aws_rds_db_subnet_groups"
+}
+
+type DBSubnetGroupSubnet struct {
+	ID                     uint `gorm:"primarykey"`
+	DBSubnetGroupID        uint
+	SubnetAvailabilityZone *rds.AvailabilityZone `gorm:"embedded;embeddedPrefix:availability_zone_"`
+	SubnetIdentifier       *string
+	SubnetOutpost          *rds.Outpost `gorm:"embedded;embeddedPrefix:outpost_"`
+	SubnetStatus           *string
+}
+
+func (DBSubnetGroupSubnet) TableName() string {
+	return "aws_rds_db_subnet_group_subnets"
+}
+
+func (c *Client) transformDBSubnetGroupSubnet(value *rds.Subnet) *DBSubnetGroupSubnet {
+	return &DBSubnetGroupSubnet{
+		SubnetAvailabilityZone: value.SubnetAvailabilityZone,
+		SubnetIdentifier:       value.SubnetIdentifier,
+		SubnetOutpost:          value.SubnetOutpost,
+		SubnetStatus:           value.SubnetStatus,
+	}
+}
+
+func (c *Client) transformDBSubnetGroupSubnets(values []*rds.Subnet) []*DBSubnetGroupSubnet {
+	var tValues []*DBSubnetGroupSubnet
+	for _, v := range values {
+		tValues = append(tValues, c.transformDBSubnetGroupSubnet(v))
+	}
+	return tValues
+}
+
+func (c *Client) transformDBSubnetGroup(value *rds.DBSubnetGroup) *DBSubnetGroup {
+	return &DBSubnetGroup{
+		Region:                   c.region,
+		AccountID:                c.accountID,
+		DBSubnetGroupArn:         value.DBSubnetGroupArn,
+		DBSubnetGroupDescription: value.DBSubnetGroupDescription,
+		DBSubnetGroupName:        value.DBSubnetGroupName,
+		SubnetGroupStatus:        value.SubnetGroupStatus,
+		Subnets:                  c.transformDBSubnetGroupSubnets(value.Subnets),
+		VpcId:                    value.VpcId,
+	}
+}
+
+func (c *Client) transformDBSubnetGroups(values []*rds.DBSubnetGroup) []*DBSubnetGroup {
+	var tValues []*DBSubnetGroup
+	for _, v := range values {
+		tValues = append(tValues, c.transformDBSubnetGroup(v))
+	}
+	return tValues
+}
+
+func MigrateDBSubnetGroups(db *gorm.DB) error {
+	return db.AutoMigrate(
+		&DBSubnetGroup{},
+		&DBSubnetGroupSubnet{},
+	)
+}
+
+func (c *Client) dbSubnetGroups(gConfig interface{}) error {
+	var config rds.DescribeDBSubnetGroupsInput
+	err := mapstructure.Decode(gConfig, &config)
+	if err != nil {
+		return err
+	}
+
+	for {
+		output, err := c.svc.DescribeDBSubnetGroups(&config)
+		if err != nil {
+			return err
+		}
+		c.db.Where("region = ?", c.region).Where("account_id = ?", c.accountID).Delete(&DBSubnetGroup{})
+		common.ChunkedCreate(c.db, c.transformDBSubnetGroups(output.DBSubnetGroups))
+		c.log.Info("Fetched resources", zap.String("resource", "rds.subnet_groups"), zap.Int("count", len(output.DBSubnetGroups)))
+		if aws.StringValue(output.Marker) == "" {
+			break
+		}
+		config.Marker = output.Marker
+	}
+	return nil
+}

providers/aws/redshift/client.go

@@ -35,6 +35,8 @@ func (c *Client) CollectResource(resource string, config interface{}) error {
 	switch resource {
 	case "clusters":
 		return c.clusters(config)
+	case "cluster_subnet_groups":
+		return c.clusterSubnetGroups(config)
 	default:
 		return fmt.Errorf("unsupported resource redshift.%s", resource)
 	}

providers/aws/redshift/subnet_groups.go

@@ -0,0 +1,159 @@
+package redshift
+
+import (
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/redshift"
+	"github.com/cloudquery/cloudquery/providers/common"
+	"github.com/mitchellh/mapstructure"
+	"go.uber.org/zap"
+	"gorm.io/gorm"
+)
+
+type ClusterSubnetGroup struct {
+	ID        uint `gorm:"primarykey"`
+	AccountID string
+	Region    string
+
+	Name        *string
+	Description *string
+	Status      *string
+	Subnets     []*ClusterSubnetGroupSubnet `gorm:"constraint:OnDelete:CASCADE;"`
+	Tags        []*ClusterSubnetGroupTag    `gorm:"constraint:OnDelete:CASCADE;"`
+	VpcId       *string
+}
+
+func (ClusterSubnetGroup) TableName() string {
+	return "aws_redshift_cluster_subnet_groups"
+}
+
+type ClusterSubnetGroupTag struct {
+	ID                   uint `gorm:"primarykey"`
+	ClusterSubnetGroupID uint
+
+	Key   *string
+	Value *string
+}
+
+func (ClusterSubnetGroupTag) TableName() string {
+	return "aws_redshift_cluster_subnet_group_tags"
+}
+
+type ClusterSubnetGroupSubnet struct {
+	ID                   uint `gorm:"primarykey"`
+	ClusterSubnetGroupID uint
+
+	AvailabilityZoneName               *string
+	AvailabilityZoneSupportedPlatforms []*ClusterSubnetGroupSupportedPlatform `gorm:"constraint:OnDelete:CASCADE;"`
+	Identifier                         *string
+	Status                             *string
+}
+
+func (ClusterSubnetGroupSubnet) TableName() string {
+	return "aws_redshift_cluster_subnet_group_subnets"
+}
+
+type ClusterSubnetGroupSupportedPlatform struct {
+	ID                         uint `gorm:"primarykey"`
+	ClusterSubnetGroupSubnetID uint
+
+	Name *string
+}
+
+func (ClusterSubnetGroupSupportedPlatform) TableName() string {
+	return "aws_redshift_cluster_subnet_group_supported_platforms"
+}
+
+func (c *Client) transformClusterSubnetGroups(values []*redshift.ClusterSubnetGroup) []*ClusterSubnetGroup {
+	var tValues []*ClusterSubnetGroup
+	for _, value := range values {
+		tValue := ClusterSubnetGroup{
+			AccountID:   c.accountID,
+			Region:      c.region,
+			Name:        value.ClusterSubnetGroupName,
+			Description: value.Description,
+			Status:      value.SubnetGroupStatus,
+			Subnets:     c.transformClusterSubnetGroupSubnets(value.Subnets),
+			Tags:        c.transformClusterSubnetGroupTags(value.Tags),
+			VpcId:       value.VpcId,
+		}
+		tValues = append(tValues, &tValue)
+	}
+	return tValues
+}
+
+func (c *Client) transformClusterSubnetGroupSupportedPlatforms(values []*redshift.SupportedPlatform) []*ClusterSubnetGroupSupportedPlatform {
+	var tValues []*ClusterSubnetGroupSupportedPlatform
+	for _, value := range values {
+		tValue := ClusterSubnetGroupSupportedPlatform{
+			Name: value.Name,
+		}
+		tValues = append(tValues, &tValue)
+	}
+	return tValues
+}
+
+func (c *Client) transformClusterSubnetGroupSubnets(values []*redshift.Subnet) []*ClusterSubnetGroupSubnet {
+	var tValues []*ClusterSubnetGroupSubnet
+	for _, value := range values {
+		tValue := ClusterSubnetGroupSubnet{
+			Identifier: value.SubnetIdentifier,
+			Status:     value.SubnetStatus,
+		}
+		if value.SubnetAvailabilityZone != nil {
+			tValue.AvailabilityZoneName = value.SubnetAvailabilityZone.Name
+			tValue.AvailabilityZoneSupportedPlatforms = c.transformClusterSubnetGroupSupportedPlatforms(value.SubnetAvailabilityZone.SupportedPlatforms)
+		}
+		tValues = append(tValues, &tValue)
+	}
+	return tValues
+}
+
+func (c *Client) transformClusterSubnetGroupTags(values []*redshift.Tag) []*ClusterSubnetGroupTag {
+	var tValues []*ClusterSubnetGroupTag
+	for _, value := range values {
+		tValue := ClusterSubnetGroupTag{
+			Key:   value.Key,
+			Value: value.Value,
+		}
+		tValues = append(tValues, &tValue)
+	}
+	return tValues
+}
+
+func MigrateClusterSubnetGroups(db *gorm.DB) error {
+	err := db.AutoMigrate(
+		&ClusterSubnetGroup{},
+		&ClusterSubnetGroupSubnet{},
+		&ClusterSubnetGroupSupportedPlatform{},
+		&ClusterSubnetGroupTag{},
+	)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (c *Client) clusterSubnetGroups(gConfig interface{}) error {
+	var config redshift.DescribeClusterSubnetGroupsInput
+	err := mapstructure.Decode(gConfig, &config)
+	if err != nil {
+		return err
+	}
+
+	for {
+		output, err := c.svc.DescribeClusterSubnetGroups(&config)
+		if err != nil {
+			return err
+		}
+		c.db.Where("region = ?", c.region).Where("account_id = ?", c.accountID).Delete(&ClusterSubnetGroup{})
+		common.ChunkedCreate(c.db, c.transformClusterSubnetGroups(output.ClusterSubnetGroups))
+		c.log.Info("Fetched resources", zap.String("resource", "redshift.cluster_subnet_groups"), zap.Int("count", len(output.ClusterSubnetGroups)))
+		if aws.StringValue(output.Marker) == "" {
+			break
+		}
+		config.Marker = output.Marker
+	}
+
+	return nil
+}