Skip to content
This repository has been archived by the owner on Aug 16, 2022. It is now read-only.

feat: Add Glue Workflows resource #1310

Merged
merged 1 commit into from
Jul 28, 2022
Merged

feat: Add Glue Workflows resource #1310

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions client/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ import (
"github.com/aws/aws-sdk-go-v2/service/elasticsearchservice"
"github.com/aws/aws-sdk-go-v2/service/emr"
"github.com/aws/aws-sdk-go-v2/service/fsx"
"github.com/aws/aws-sdk-go-v2/service/glue"
"github.com/aws/aws-sdk-go-v2/service/guardduty"
"github.com/aws/aws-sdk-go-v2/service/iam"
"github.com/aws/aws-sdk-go-v2/service/iot"
Expand Down Expand Up @@ -149,6 +150,7 @@ type Services struct {
ElasticBeanstalk ElasticbeanstalkClient
ElasticSearch ElasticSearch
FSX FsxClient
Glue GlueClient
GuardDuty GuardDutyClient
IAM IamClient
IOT IOTClient
Expand Down Expand Up @@ -645,6 +647,7 @@ func initServices(region string, c aws.Config) Services {
ELBv2: elbv2.NewFromConfig(awsCfg),
EMR: emr.NewFromConfig(awsCfg),
FSX: fsx.NewFromConfig(awsCfg),
Glue: glue.NewFromConfig(awsCfg),
GuardDuty: guardduty.NewFromConfig(awsCfg),
IAM: iam.NewFromConfig(awsCfg),
KMS: kms.NewFromConfig(awsCfg),
Expand Down
1 change: 1 addition & 0 deletions client/helpers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ const (
EC2Service AWSService = "ec2"
EFSService AWSService = "elasticfilesystem"
ElasticLoadBalancingService AWSService = "elasticloadbalancing"
GlueService AWSService = "glue"
GuardDutyService AWSService = "guardduty"
RedshiftService AWSService = "redshift"
Route53Service AWSService = "route53"
Expand Down
96 changes: 96 additions & 0 deletions client/mocks/glue.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 8 additions & 0 deletions client/services.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ import (
"github.com/aws/aws-sdk-go-v2/service/elasticsearchservice"
"github.com/aws/aws-sdk-go-v2/service/emr"
"github.com/aws/aws-sdk-go-v2/service/fsx"
"github.com/aws/aws-sdk-go-v2/service/glue"
"github.com/aws/aws-sdk-go-v2/service/guardduty"
"github.com/aws/aws-sdk-go-v2/service/iam"
"github.com/aws/aws-sdk-go-v2/service/iot"
Expand Down Expand Up @@ -744,3 +745,10 @@ type SESClient interface {
GetEmailTemplate(ctx context.Context, params *sesv2.GetEmailTemplateInput, optFns ...func(*sesv2.Options)) (*sesv2.GetEmailTemplateOutput, error)
ListEmailTemplates(ctx context.Context, params *sesv2.ListEmailTemplatesInput, optFns ...func(*sesv2.Options)) (*sesv2.ListEmailTemplatesOutput, error)
}

//go:generate mockgen -package=mocks -destination=./mocks/glue.go . GlueClient
type GlueClient interface {
GetTags(ctx context.Context, params *glue.GetTagsInput, optFns ...func(*glue.Options)) (*glue.GetTagsOutput, error)
GetWorkflow(ctx context.Context, params *glue.GetWorkflowInput, optFns ...func(*glue.Options)) (*glue.GetWorkflowOutput, error)
ListWorkflows(ctx context.Context, params *glue.ListWorkflowsInput, optFns ...func(*glue.Options)) (*glue.ListWorkflowsOutput, error)
}
34 changes: 34 additions & 0 deletions docs/tables/aws_glue_workflows.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@

# Table: aws_glue_workflows
A workflow is a collection of multiple dependent Glue jobs and crawlers that are run to complete a complex ETL task
## Columns
| Name | Type | Description |
| ------------- | ------------- | ----- |
|account_id|text|The AWS Account ID of the resource.|
|region|text|The AWS Region of the resource.|
|arn|text|The Amazon Resource Name (ARN) of the workflow.|
|tags|jsonb|Resource tags.|
|blueprint_name|text|The name of the blueprint.|
|blueprint_run_id|text|The run ID for this blueprint.|
|created_on|timestamp without time zone|The date and time when the workflow was created.|
|default_run_properties|jsonb|A collection of properties to be used as part of each execution of the workflow. The run properties are made available to each job in the workflow|
|description|text|A description of the workflow.|
|last_modified_on|timestamp without time zone|The date and time when the workflow was last modified.|
|last_run_completed_on|timestamp without time zone|The date and time when the workflow run completed.|
|last_run_error_message|text|This error message describes any error that may have occurred in starting the workflow run|
|last_run_name|text|Name of the workflow that was run.|
|last_run_previous_run_id|text|The ID of the previous workflow run.|
|last_run_started_on|timestamp without time zone|The date and time when the workflow run was started.|
|last_run_starting_event_batch_condition_size|bigint|Number of events in the batch.|
|last_run_starting_event_batch_condition_window|bigint|Duration of the batch window in seconds.|
|last_run_statistics_failed_actions|bigint|Total number of Actions that have failed.|
|last_run_statistics_running_actions|bigint|Total number Actions in running state.|
|last_run_statistics_stopped_actions|bigint|Total number of Actions that have stopped.|
|last_run_statistics_succeeded_actions|bigint|Total number of Actions that have succeeded.|
|last_run_statistics_timeout_actions|bigint|Total number of Actions that timed out.|
|last_run_statistics_total_actions|bigint|Total number of Actions in the workflow run.|
|last_run_status|text|The status of the workflow run.|
|last_run_workflow_run_id|text|The ID of this workflow run.|
|last_run_workflow_run_properties|jsonb|The workflow run properties which were set during the run.|
|max_concurrent_runs|bigint|You can use this parameter to prevent unwanted multiple updates to data, to control costs, or in some cases, to prevent exceeding the maximum number of concurrent runs of any of the component jobs|
|name|text|The name of the workflow.|
1 change: 1 addition & 0 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,7 @@ require (
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.15 // indirect
github.com/aws/aws-sdk-go-v2/service/athena v1.16.0
github.com/aws/aws-sdk-go-v2/service/backup v1.16.3
github.com/aws/aws-sdk-go-v2/service/glue v1.28.1
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.3 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.8 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.8 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -220,6 +220,8 @@ github.com/aws/aws-sdk-go-v2/service/emr v1.20.0 h1:2xjz2hES5SnQLgmW1bBVdVz6j0mj
github.com/aws/aws-sdk-go-v2/service/emr v1.20.0/go.mod h1:OVVv6VrQG33CuCR+V0GiaBuWbwYGogGxWGi9TDrM2yk=
github.com/aws/aws-sdk-go-v2/service/fsx v1.24.2 h1:8ko+AFpvJUbpjtCIEgtaXcXtndkZBi0N7e2ePGocqf8=
github.com/aws/aws-sdk-go-v2/service/fsx v1.24.2/go.mod h1:K3Ym90NBYdXV+BCHvpuiDXCeMAtayFdiGdJ0I1uop5Q=
github.com/aws/aws-sdk-go-v2/service/glue v1.28.1 h1:rG+jzafWyw73tdv+48e4jZYyehihEORcEcqzyBbZUGA=
github.com/aws/aws-sdk-go-v2/service/glue v1.28.1/go.mod h1:JpqCaI8ytHaConkpUXxhWibisAti9SA3KvYR5GLxHXk=
github.com/aws/aws-sdk-go-v2/service/guardduty v1.14.1 h1:5bWqv1hwsELtYvmBpuOtOmfW/PqHYUBVBh3RQ54+PKs=
github.com/aws/aws-sdk-go-v2/service/guardduty v1.14.1/go.mod h1:zeMwk8nXD/85ck1aCJs82TXj+QZcT4wNWmdU62+h9LM=
github.com/aws/aws-sdk-go-v2/service/iam v1.18.8 h1:MYOkrSNwOUokctOnhGUNM9J/yNu87roEmdKcJ74d4eA=
Expand Down
2 changes: 2 additions & 0 deletions resources/provider/provider.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ import (
"github.com/cloudquery/cq-provider-aws/resources/services/elbv2"
"github.com/cloudquery/cq-provider-aws/resources/services/emr"
"github.com/cloudquery/cq-provider-aws/resources/services/fsx"
"github.com/cloudquery/cq-provider-aws/resources/services/glue"
"github.com/cloudquery/cq-provider-aws/resources/services/guardduty"
"github.com/cloudquery/cq-provider-aws/resources/services/iam"
"github.com/cloudquery/cq-provider-aws/resources/services/iot"
Expand Down Expand Up @@ -166,6 +167,7 @@ func Provider() *provider.Provider {
"emr.block_public_access_configs": emr.EmrBlockPublicAccessConfigs(),
"emr.clusters": emr.EmrClusters(),
"fsx.backups": fsx.FsxBackups(),
"glue.workflows": glue.Workflows(),
"guardduty.detectors": guardduty.GuarddutyDetectors(),
"iam.accounts": iam.IamAccounts(),
"iam.groups": iam.IamGroups(),
Expand Down
Loading