Skip to content

Commit

Permalink
feat: Extend sns subscription data (#1424)
Browse files Browse the repository at this point in the history
  • Loading branch information
irmatov committed Aug 17, 2022
1 parent a30f527 commit 63887e7
Show file tree
Hide file tree
Showing 14 changed files with 415 additions and 155 deletions.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 4 additions & 3 deletions plugins/source/aws/client/services.go
Original file line number Diff line number Diff line change
Expand Up @@ -768,12 +768,13 @@ type ShieldClient interface {
ListTagsForResource(ctx context.Context, params *shield.ListTagsForResourceInput, optFns ...func(*shield.Options)) (*shield.ListTagsForResourceOutput, error)
}

//go:generate mockgen -package=mocks -destination=./mocks/mock_sns.go . SnsClient
//go:generate mockgen -package=mocks -destination=./mocks/sns.go . SnsClient
type SnsClient interface {
ListTopics(ctx context.Context, params *sns.ListTopicsInput, optFns ...func(*sns.Options)) (*sns.ListTopicsOutput, error)
ListSubscriptions(ctx context.Context, params *sns.ListSubscriptionsInput, optFns ...func(*sns.Options)) (*sns.ListSubscriptionsOutput, error)
GetSubscriptionAttributes(ctx context.Context, params *sns.GetSubscriptionAttributesInput, optFns ...func(*sns.Options)) (*sns.GetSubscriptionAttributesOutput, error)
GetTopicAttributes(ctx context.Context, params *sns.GetTopicAttributesInput, optFns ...func(*sns.Options)) (*sns.GetTopicAttributesOutput, error)
ListSubscriptions(ctx context.Context, params *sns.ListSubscriptionsInput, optFns ...func(*sns.Options)) (*sns.ListSubscriptionsOutput, error)
ListTagsForResource(ctx context.Context, params *sns.ListTagsForResourceInput, optFns ...func(*sns.Options)) (*sns.ListTagsForResourceOutput, error)
ListTopics(ctx context.Context, params *sns.ListTopicsInput, optFns ...func(*sns.Options)) (*sns.ListTopicsOutput, error)
}

//go:generate mockgen -package=mocks -destination=./mocks/mock_sqs.go . SQSClient
Expand Down
21 changes: 15 additions & 6 deletions plugins/source/aws/docs/tables/aws_sns_subscriptions.md
Original file line number Diff line number Diff line change
@@ -1,13 +1,22 @@

# Table: aws_sns_subscriptions
A wrapper type for the attributes of an Amazon SNS subscription.
Amazon SNS subscription
## Columns
| Name | Type | Description |
| ------------- | ------------- | ----- |
|account_id|text|The AWS Account ID of the resource.|
|region|text|The AWS Region of the resource.|
|endpoint|text|The subscription's endpoint (format depends on the protocol).|
|owner|text|The subscription's owner.|
|protocol|text|The subscription's protocol.|
|arn|text|The subscription's ARN.|
|topic_arn|text|The ARN of the subscription's topic.|
|endpoint|text|The subscription's endpoint (format depends on the protocol)|
|owner|text|The subscription's owner|
|protocol|text|The subscription's protocol|
|arn|text|The subscription's ARN|
|topic_arn|text|The ARN of the subscription's topic|
|confirmation_was_authenticated|boolean|True if the subscription confirmation request was authenticated|
|delivery_policy|jsonb|The JSON serialization of the subscription's delivery policy|
|effective_delivery_policy|jsonb|The JSON serialization of the effective delivery policy that takes into account the topic delivery policy and account system defaults|
|filter_policy|jsonb|The filter policy JSON that is assigned to the subscription|
|pending_confirmation|boolean|True if the subscription hasn't been confirmed|
|raw_message_delivery|boolean|True if raw message delivery is enabled for the subscription|
|redrive_policy|text|When specified, sends undeliverable messages to the specified Amazon SQS dead-letter queue|
|subscription_role_arn|text|The ARN of the IAM role that has permission to write to the Kinesis Data Firehose delivery stream and has Amazon SNS listed as a trusted entity|
|unknown_fields|jsonb|Other subscription attributes|
29 changes: 15 additions & 14 deletions plugins/source/aws/docs/tables/aws_sns_topics.md
Original file line number Diff line number Diff line change
@@ -1,21 +1,22 @@

# Table: aws_sns_topics
AWS SNS topic
Amazon SNS topic
## Columns
| Name | Type | Description |
| ------------- | ------------- | ----- |
|account_id|text|The AWS Account ID of the resource.|
|region|text|The AWS Region of the resource.|
|owner|text|The AWS account ID of the topic's owner.|
|policy|jsonb|The JSON serialization of the topic's access control policy.|
|delivery_policy|jsonb|The JSON serialization of the topic's delivery policy.|
|display_name|text|The human-readable name used in the From field for notifications to email and email-json endpoints.|
|subscriptions_confirmed|bigint|The number of confirmed subscriptions for the topic.|
|subscriptions_deleted|bigint|The number of deleted subscriptions for the topic.|
|subscriptions_pending|bigint|The number of subscriptions pending confirmation for the topic.|
|effective_delivery_policy|jsonb|The JSON serialization of the effective delivery policy, taking system defaults into account.|
|fifo_topic|boolean|When this is set to true, a FIFO topic is created.|
|content_based_deduplication|boolean|Enables content-based deduplication for FIFO topics.|
|kms_master_key_id|text|The ID of an AWS managed customer master key (CMK) for Amazon SNS or a custom CMK|
|arn|text|The topic's ARN.|
|tags|jsonb|Topic tags.|
|tags|jsonb||
|delivery_policy|jsonb|The JSON serialization of the topic's delivery policy|
|display_name|text|The human-readable name used in the From field for notifications to email and email-json endpoints|
|owner|text|The AWS account ID of the topic's owner|
|policy|jsonb|The JSON serialization of the topic's access control policy|
|subscriptions_confirmed|bigint|The number of confirmed subscriptions for the topic|
|subscriptions_deleted|bigint|The number of deleted subscriptions for the topic|
|subscriptions_pending|bigint|The number of subscriptions pending confirmation for the topic|
|arn|text|The Amazon Resource Name (ARN) of the topic|
|effective_delivery_policy|jsonb|The JSON serialization of the effective delivery policy, taking system defaults into account|
|kms_master_key_id|text|The ID of an Amazon Web Services managed customer master key (CMK) for Amazon SNS or a custom CMK|
|fifo_topic|boolean|When this is set to true, a FIFO topic is created|
|content_based_deduplication|boolean|Enables content-based deduplication for FIFO topics|
|unknown_fields|jsonb|Other subscription attributes|
8 changes: 4 additions & 4 deletions plugins/source/aws/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ module github.com/cloudquery/cloudquery/plugins/source/aws
go 1.18

require (
github.com/aws/aws-sdk-go-v2 v1.16.10
github.com/aws/aws-sdk-go-v2 v1.16.11
github.com/aws/aws-sdk-go-v2/config v1.15.14
github.com/aws/aws-sdk-go-v2/credentials v1.12.9
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.20
Expand Down Expand Up @@ -64,7 +64,7 @@ require (
github.com/aws/aws-sdk-go-v2/service/sagemaker v1.34.0
github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.15.12
github.com/aws/aws-sdk-go-v2/service/sesv2 v1.13.8
github.com/aws/aws-sdk-go-v2/service/sns v1.17.8
github.com/aws/aws-sdk-go-v2/service/sns v1.17.13
github.com/aws/aws-sdk-go-v2/service/sqs v1.18.7
github.com/aws/aws-sdk-go-v2/service/ssm v1.27.3
github.com/aws/aws-sdk-go-v2/service/sts v1.16.9
Expand Down Expand Up @@ -110,8 +110,8 @@ require (
github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.3 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.8 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.17 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.11 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.18 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.12 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.15 // indirect
github.com/aws/aws-sdk-go-v2/service/athena v1.16.0
github.com/aws/aws-sdk-go-v2/service/backup v1.16.3
Expand Down
8 changes: 8 additions & 0 deletions plugins/source/aws/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,8 @@ github.com/aws/aws-sdk-go-v2 v1.16.7/go.mod h1:6CpKuLXg2w7If3ABZCl/qZ6rEgwtjZTn4
github.com/aws/aws-sdk-go-v2 v1.16.8/go.mod h1:6CpKuLXg2w7If3ABZCl/qZ6rEgwtjZTn4eAf4RcEyuw=
github.com/aws/aws-sdk-go-v2 v1.16.10 h1:+yDD0tcuHRQZgqONkpDwzepqmElQaSlFPymHRHR9mrc=
github.com/aws/aws-sdk-go-v2 v1.16.10/go.mod h1:WTACcleLz6VZTp7fak4EO5b9Q4foxbn+8PIz3PmyKlo=
github.com/aws/aws-sdk-go-v2 v1.16.11 h1:xM1ZPSvty3xVmdxiGr7ay/wlqv+MWhH0rMlyLdbC0YQ=
github.com/aws/aws-sdk-go-v2 v1.16.11/go.mod h1:WTACcleLz6VZTp7fak4EO5b9Q4foxbn+8PIz3PmyKlo=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.3 h1:S/ZBwevQkr7gv5YxONYpGQxlMFFYSRfz3RMcjsC9Qhk=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.3/go.mod h1:gNsR5CaXKmQSSzrmGxmwmct/r+ZBfbxorAuXYsj/M5Y=
github.com/aws/aws-sdk-go-v2/config v1.15.14 h1:+BqpqlydTq4c2et9Daury7gE+o67P4lbk7eybiCBNc4=
Expand All @@ -74,11 +76,15 @@ github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.14/go.mod h1:kdjrMwHwrC
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.15/go.mod h1:pWrr2OoHlT7M/Pd2y4HV3gJyPb3qj5qMmnPkKSNPYK4=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.17 h1:U8DZvyFFesBmK62dYC6BRXm4Cd/wPP3aPcecu3xv/F4=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.17/go.mod h1:6qtGip7sJEyvgsLjphRZWF9qPe3xJf1mL/MM01E35Wc=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.18 h1:OmiwoVyLKEqqD5GvB683dbSqxiOfvx4U2lDZhG2Esc4=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.18/go.mod h1:348MLhzV1GSlZSMusdwQpXKbhD7X2gbI/TxwAPKkYZQ=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.7/go.mod h1:93Uot80ddyVzSl//xEJreNKMhxntr71WtR3v/A1cRYk=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.8/go.mod h1:ZIV8GYoC6WLBW5KGs+o4rsc65/ozd+eQ0L31XF5VDwk=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.9/go.mod h1:08tUpeSGN33QKSO7fwxXczNfiwCpbj+GxK6XKwqWVv0=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.11 h1:GMp98usVW5tzQhxd26KWhoNQPlR2noIlfbzqjVGBhLU=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.11/go.mod h1:cYAfnB+9ZkmZWpQWmPDsuIGm4EA+6k2ZVtxKjw/XJBY=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.12 h1:5mvQDtNWtI6H56+E4LUnLWEmATMB7oEh+Z9RurtIuC0=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.12/go.mod h1:ckaCVTEdGAxO6KwTGzgskxR1xM+iJW4lxMyDFVda2Fc=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.15 h1:QquxR7NH3ULBsKC+NoTpilzbKKS+5AELfNREInbhvas=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.15/go.mod h1:Tkrthp/0sNBShQQsamR7j/zY4p19tVTAs+nnqhH6R3c=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.5/go.mod h1:aIwFF3dUk95ocCcA3zfk3nhz0oLkpzHFWuMp8l/4nNs=
Expand Down Expand Up @@ -220,6 +226,8 @@ github.com/aws/aws-sdk-go-v2/service/shield v1.16.7 h1:bfyTNq3U7GXyFAr2fSJ1OaV5Z
github.com/aws/aws-sdk-go-v2/service/shield v1.16.7/go.mod h1:T7HfO9ktODwkrs+RlBFSgvOiVhLjn2eEBN8n2266rLY=
github.com/aws/aws-sdk-go-v2/service/sns v1.17.8 h1:Z0LBaDH89pfyBOCZzLCi8tmie3hJyLDV2NDjBWVvPzw=
github.com/aws/aws-sdk-go-v2/service/sns v1.17.8/go.mod h1:Em/qKcKnYe5Y2+xcJ+XI/01K/JtOy7andGwQByLYFXc=
github.com/aws/aws-sdk-go-v2/service/sns v1.17.13 h1:sa8NDFztt68pihEfE31LhX+nJ1wDBJHcFh3T6crluDo=
github.com/aws/aws-sdk-go-v2/service/sns v1.17.13/go.mod h1:yE3hE9v3YRRI9Rsl38kYJ4fyZ6vKSljaZ+28W5xzqgM=
github.com/aws/aws-sdk-go-v2/service/sqs v1.18.7 h1:4inF55jYDETXLUr29ZlAq6Pipq9NTUTxQC5bLQJtjf4=
github.com/aws/aws-sdk-go-v2/service/sqs v1.18.7/go.mod h1:E2OxTDUWA7s1TCdvBm+RDEjyssunta3SuSeqHUdFrCM=
github.com/aws/aws-sdk-go-v2/service/ssm v1.27.3 h1:rujlES62T0e+YDecfhoANcIXCdpLC/+lNNZSlcagf/g=
Expand Down
4 changes: 2 additions & 2 deletions plugins/source/aws/resources/provider/provider.go
Original file line number Diff line number Diff line change
Expand Up @@ -279,8 +279,8 @@ func Provider() *provider.Provider {
"shield.protections_groups": shield.ProtectionGroups(),
"shield.protections": shield.Protections(),
"shield.subscriptions": shield.Subscriptions(),
"sns.subscriptions": sns.SnsSubscriptions(),
"sns.topics": sns.SnsTopics(),
"sns.subscriptions": sns.Subscriptions(),
"sns.topics": sns.Topics(),
"sqs.queues": sqs.SQSQueues(),
"ssm.documents": ssm.SsmDocuments(),
"ssm.instances": ssm.SsmInstances(),
Expand Down
100 changes: 84 additions & 16 deletions plugins/source/aws/resources/services/sns/subscriptions.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,18 +8,19 @@ import (
"github.com/cloudquery/cloudquery/plugins/source/aws/client"
"github.com/cloudquery/cq-provider-sdk/provider/diag"
"github.com/cloudquery/cq-provider-sdk/provider/schema"
"github.com/mitchellh/mapstructure"
)

func SnsSubscriptions() *schema.Table {
//go:generate cq-gen --resource subscriptions --config subscriptions.hcl --output .
func Subscriptions() *schema.Table {
return &schema.Table{
Name: "aws_sns_subscriptions",
Description: "A wrapper type for the attributes of an Amazon SNS subscription.",
Resolver: fetchSnsSubscriptions,
Multiplex: client.ServiceAccountRegionMultiplexer("sns"),
IgnoreError: client.IgnoreCommonErrors,
DeleteFilter: client.DeleteAccountRegionFilter,
Options: schema.TableCreationOptions{PrimaryKeys: []string{"endpoint", "owner", "protocol", "arn", "topic_arn"}},
IgnoreInTests: true,
Name: "aws_sns_subscriptions",
Description: "Amazon SNS subscription",
Resolver: fetchSnsSubscriptions,
Multiplex: client.ServiceAccountRegionMultiplexer("sns"),
IgnoreError: client.IgnoreCommonErrors,
DeleteFilter: client.DeleteAccountRegionFilter,
Options: schema.TableCreationOptions{PrimaryKeys: []string{"arn"}},
Columns: []schema.Column{
{
Name: "account_id",
Expand All @@ -35,29 +36,78 @@ func SnsSubscriptions() *schema.Table {
},
{
Name: "endpoint",
Description: "The subscription's endpoint (format depends on the protocol).",
Description: "The subscription's endpoint (format depends on the protocol)",
Type: schema.TypeString,
Resolver: schema.PathResolver("Subscription.Endpoint"),
},
{
Name: "owner",
Description: "The subscription's owner.",
Description: "The subscription's owner",
Type: schema.TypeString,
Resolver: schema.PathResolver("Subscription.Owner"),
},
{
Name: "protocol",
Description: "The subscription's protocol.",
Description: "The subscription's protocol",
Type: schema.TypeString,
Resolver: schema.PathResolver("Subscription.Protocol"),
},
{
Name: "arn",
Description: "The subscription's ARN.",
Description: "The subscription's ARN",
Type: schema.TypeString,
Resolver: schema.PathResolver("SubscriptionArn"),
Resolver: schema.PathResolver("Subscription.SubscriptionArn"),
},
{
Name: "topic_arn",
Description: "The ARN of the subscription's topic.",
Description: "The ARN of the subscription's topic",
Type: schema.TypeString,
Resolver: schema.PathResolver("Subscription.TopicArn"),
},
{
Name: "confirmation_was_authenticated",
Description: "True if the subscription confirmation request was authenticated",
Type: schema.TypeBool,
},
{
Name: "delivery_policy",
Description: "The JSON serialization of the subscription's delivery policy",
Type: schema.TypeJSON,
},
{
Name: "effective_delivery_policy",
Description: "The JSON serialization of the effective delivery policy that takes into account the topic delivery policy and account system defaults",
Type: schema.TypeJSON,
},
{
Name: "filter_policy",
Description: "The filter policy JSON that is assigned to the subscription",
Type: schema.TypeJSON,
},
{
Name: "pending_confirmation",
Description: "True if the subscription hasn't been confirmed",
Type: schema.TypeBool,
},
{
Name: "raw_message_delivery",
Description: "True if raw message delivery is enabled for the subscription",
Type: schema.TypeBool,
},
{
Name: "redrive_policy",
Description: "When specified, sends undeliverable messages to the specified Amazon SQS dead-letter queue",
Type: schema.TypeString,
},
{
Name: "subscription_role_arn",
Description: "The ARN of the IAM role that has permission to write to the Kinesis Data Firehose delivery stream and has Amazon SNS listed as a trusted entity",
Type: schema.TypeString,
},
{
Name: "unknown_fields",
Description: "Other subscription attributes",
Type: schema.TypeJSON,
},
},
}
Expand All @@ -66,6 +116,7 @@ func SnsSubscriptions() *schema.Table {
// ====================================================================================================================
// Table Resolver Functions
// ====================================================================================================================

func fetchSnsSubscriptions(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- interface{}) error {
c := meta.(*client.Client)
svc := c.Services().SNS
Expand All @@ -75,7 +126,24 @@ func fetchSnsSubscriptions(ctx context.Context, meta schema.ClientMeta, parent *
if err != nil {
return diag.WrapError(err)
}
res <- output.Subscriptions
for _, item := range output.Subscriptions {
attrs, err := svc.GetSubscriptionAttributes(ctx, &sns.GetSubscriptionAttributesInput{SubscriptionArn: item.SubscriptionArn})
if err != nil {
if c.IsNotFoundError(err) {
continue
}
return diag.WrapError(err)
}
s := Subscription{Subscription: item}
dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{WeaklyTypedInput: true, Result: &s})
if err != nil {
return diag.WrapError(err)
}
if err := dec.Decode(attrs.Attributes); err != nil {
return diag.WrapError(err)
}
res <- s
}

if aws.ToString(output.NextToken) == "" {
break
Expand Down
55 changes: 55 additions & 0 deletions plugins/source/aws/resources/services/sns/subscriptions.hcl
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
//check-for-changes
service = "aws"
output_directory = "."
add_generate = true

description_modifier "remove_read_only" {
words = [" This member is required."]
}

resource "aws" "sns" "subscriptions" {
path = "github.com/cloudquery/cloudquery/plugins/source/aws/resources/services/sns.Subscription"
ignoreError "IgnoreCommonErrors" {
path = "github.com/cloudquery/cloudquery/plugins/source/aws/client.IgnoreCommonErrors"
}
deleteFilter "AccountRegionFilter" {
path = "github.com/cloudquery/cloudquery/plugins/source/aws/client.DeleteAccountRegionFilter"
}
multiplex "AwsAccountRegion" {
path = "github.com/cloudquery/cloudquery/plugins/source/aws/client.ServiceAccountRegionMultiplexer"
params = ["sns"]
}
options {
primary_keys = ["arn"]
}
userDefinedColumn "account_id" {
description = "The AWS Account ID of the resource."
type = "string"
resolver "resolveAWSAccount" {
path = "github.com/cloudquery/cloudquery/plugins/source/aws/client.ResolveAWSAccount"
}
}
userDefinedColumn "region" {
type = "string"
description = "The AWS Region of the resource."
resolver "resolveAWSRegion" {
path = "github.com/cloudquery/cloudquery/plugins/source/aws/client.ResolveAWSRegion"
}
}

column "subscription" {
skip_prefix = true
}
column "subscription_arn" {
rename = "arn"
}
column "delivery_policy" {
type = "json"
}
column "effective_delivery_policy" {
type = "json"
}
column "filter_policy" {
type = "json"
}
}

0 comments on commit 63887e7

Please sign in to comment.