Skip to content

Commit

Permalink
fix(aws)!: Step Function Executions And MapRuns (#8130)
Browse files Browse the repository at this point in the history

#### Summary


I have validated that data is able to be populated in all tables:
![Screenshot 2023-02-15 at 9 18 19 AM](https://user-images.githubusercontent.com/30294676/219069727-af4721ec-ac6c-490d-bdea-b387d7b60dca.png)



With no errors or panics
![Screenshot 2023-02-15 at 9 18 53 AM](https://user-images.githubusercontent.com/30294676/219069819-39e342d6-9f91-4526-a6c4-4f5283786ba0.png)
  • Loading branch information
bbernays committed Feb 15, 2023
1 parent 91cab61 commit 21e4ea1
Show file tree
Hide file tree
Showing 18 changed files with 297 additions and 225 deletions.
5 changes: 3 additions & 2 deletions plugins/source/aws/docs/tables/README.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,10 @@ The primary key for this table is **arn**.

## Relations

This table depends on [aws_stepfunctions_state_machines](aws_stepfunctions_state_machines.md).

The following tables depend on aws_stepfunctions_executions:
- [aws_stepfunctions_executions_map_runs](aws_stepfunctions_executions_map_runs.md)
- [aws_stepfunctions_map_runs](aws_stepfunctions_map_runs.md)

## Columns

Expand All @@ -20,9 +22,9 @@ The following tables depend on aws_stepfunctions_executions:
|account_id|String|
|region|String|
|arn (PK)|String|
|state_machine_arn|String|
|execution_arn|String|
|start_date|Timestamp|
|state_machine_arn|String|
|status|String|
|cause|String|
|error|String|
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
# Table: aws_stepfunctions_map_run_executions

https://docs.aws.amazon.com/step-functions/latest/apireference/API_DescribeExecution.html

The primary key for this table is **arn**.

## Relations

This table depends on [aws_stepfunctions_map_runs](aws_stepfunctions_map_runs.md).

## Columns

| Name | Type |
| ------------- | ------------- |
|_cq_source_name|String|
|_cq_sync_time|Timestamp|
|_cq_id|UUID|
|_cq_parent_id|UUID|
|account_id|String|
|region|String|
|arn (PK)|String|
|map_run_arn|String|
|state_machine_arn|String|
|execution_arn|String|
|start_date|Timestamp|
|status|String|
|cause|String|
|error|String|
|input|String|
|input_details|JSON|
|name|String|
|output|String|
|output_details|JSON|
|stop_date|Timestamp|
|trace_header|String|
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Table: aws_stepfunctions_executions_map_runs
# Table: aws_stepfunctions_map_runs

https://docs.aws.amazon.com/step-functions/latest/apireference/API_DescribeMapRun.html

Expand All @@ -8,6 +8,9 @@ The primary key for this table is **arn**.

This table depends on [aws_stepfunctions_executions](aws_stepfunctions_executions.md).

The following tables depend on aws_stepfunctions_map_runs:
- [aws_stepfunctions_map_run_executions](aws_stepfunctions_map_run_executions.md)

## Columns

| Name | Type |
Expand All @@ -19,6 +22,7 @@ This table depends on [aws_stepfunctions_executions](aws_stepfunctions_execution
|account_id|String|
|region|String|
|arn (PK)|String|
|state_machine_arn|String|
|execution_arn|String|
|execution_counts|JSON|
|item_counts|JSON|
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,11 @@ https://docs.aws.amazon.com/step-functions/latest/apireference/API_DescribeState

The primary key for this table is **arn**.

## Relations

The following tables depend on aws_stepfunctions_state_machines:
- [aws_stepfunctions_executions](aws_stepfunctions_executions.md)

## Columns

| Name | Type |
Expand Down
1 change: 0 additions & 1 deletion plugins/source/aws/resources/plugin/tables.go
Original file line number Diff line number Diff line change
Expand Up @@ -427,7 +427,6 @@ func tables() []*schema.Table {
ssm.PatchBaselines(),
ssoadmin.Instances(),
stepfunctions.Activities(),
stepfunctions.Executions(),
stepfunctions.StateMachines(),
timestream.Databases(),
transfer.Servers(),
Expand Down
19 changes: 19 additions & 0 deletions plugins/source/aws/resources/services/stepfunctions/activities.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
package stepfunctions

import (
"context"

"github.com/aws/aws-sdk-go-v2/service/sfn"
"github.com/aws/aws-sdk-go-v2/service/sfn/types"
"github.com/cloudquery/cloudquery/plugins/source/aws/client"
"github.com/cloudquery/plugin-sdk/schema"
Expand Down Expand Up @@ -28,3 +31,19 @@ func Activities() *schema.Table {
},
}
}

func fetchStepfunctionsActivities(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error {
svc := meta.(*client.Client).Services().Sfn
config := sfn.ListActivitiesInput{
MaxResults: 1000,
}
paginator := sfn.NewListActivitiesPaginator(svc, &config)
for paginator.HasMorePages() {
output, err := paginator.NextPage(ctx)
if err != nil {
return err
}
res <- output.Activities
}
return nil
}

This file was deleted.

Original file line number Diff line number Diff line change
@@ -1,13 +1,16 @@
package stepfunctions

import (
"context"

"github.com/aws/aws-sdk-go-v2/service/sfn"
"github.com/aws/aws-sdk-go-v2/service/sfn/types"
"github.com/cloudquery/cloudquery/plugins/source/aws/client"
"github.com/cloudquery/plugin-sdk/schema"
"github.com/cloudquery/plugin-sdk/transformers"
)

func Executions() *schema.Table {
func executions() *schema.Table {
return &schema.Table{
Name: "aws_stepfunctions_executions",
Description: `https://docs.aws.amazon.com/step-functions/latest/apireference/API_DescribeExecution.html`,
Expand All @@ -26,9 +29,46 @@ func Executions() *schema.Table {
PrimaryKey: true,
},
},
{
Name: "state_machine_arn",
Type: schema.TypeString,
Resolver: schema.ParentColumnResolver("arn"),
},
},
Relations: []*schema.Table{
mapRuns(),
},
}
}

func fetchStepfunctionsExecutions(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error {
svc := meta.(*client.Client).Services().Sfn
sfnOutput := parent.Item.(*sfn.DescribeStateMachineOutput)
config := sfn.ListExecutionsInput{
MaxResults: 1000,
StateMachineArn: sfnOutput.StateMachineArn,
}
paginator := sfn.NewListExecutionsPaginator(svc, &config)
for paginator.HasMorePages() {
output, err := paginator.NextPage(ctx)
if err != nil {
return err
}
res <- output.Executions
}
return nil
}

func getExecution(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource) error {
execution := resource.Item.(types.ExecutionListItem)
svc := meta.(*client.Client).Services().Sfn

executionResult, err := svc.DescribeExecution(ctx, &sfn.DescribeExecutionInput{
ExecutionArn: execution.ExecutionArn,
})
if err != nil {
return err
}
resource.Item = executionResult
return nil
}

This file was deleted.

Original file line number Diff line number Diff line change
@@ -1,15 +1,18 @@
package stepfunctions

import (
"context"

"github.com/aws/aws-sdk-go-v2/service/sfn"
"github.com/aws/aws-sdk-go-v2/service/sfn/types"
"github.com/cloudquery/cloudquery/plugins/source/aws/client"
"github.com/cloudquery/plugin-sdk/schema"
"github.com/cloudquery/plugin-sdk/transformers"
)

func mapRuns() *schema.Table {
return &schema.Table{
Name: "aws_stepfunctions_executions_map_runs",
Name: "aws_stepfunctions_map_runs",
Description: `https://docs.aws.amazon.com/step-functions/latest/apireference/API_DescribeMapRun.html`,
Resolver: fetchStepfunctionsMapRuns,
PreResourceResolver: getMapRun,
Expand All @@ -26,7 +29,44 @@ func mapRuns() *schema.Table {
PrimaryKey: true,
},
},
{
Name: "state_machine_arn",
Type: schema.TypeString,
Resolver: schema.ParentColumnResolver("arn"),
},
},
Relations: []*schema.Table{},
Relations: []*schema.Table{
mapRunExecutions(),
},
}
}

func fetchStepfunctionsMapRuns(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error {
svc := meta.(*client.Client).Services().Sfn
config := sfn.ListMapRunsInput{
MaxResults: 1000,
ExecutionArn: parent.Item.(*sfn.DescribeExecutionOutput).ExecutionArn,
}
paginator := sfn.NewListMapRunsPaginator(svc, &config)
for paginator.HasMorePages() {
output, err := paginator.NextPage(ctx)
if err != nil {
return err
}
res <- output.MapRuns
}
return nil
}

func getMapRun(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource) error {
svc := meta.(*client.Client).Services().Sfn
config := sfn.DescribeMapRunInput{
MapRunArn: resource.Item.(types.MapRunListItem).MapRunArn,
}
output, err := svc.DescribeMapRun(ctx, &config)
if err != nil {
return err
}
resource.Item = output
return nil
}

0 comments on commit 21e4ea1

Please sign in to comment.