feat: Added glue table indexes (#1377)

* feat: Added glue table partitions and indexes

* add comment

* added resolver tests and docs

* structure adjusted

* linter

* fix

* docs

* removed partitions

* mock generation

client/resolvers.go

@@ -2,6 +2,7 @@ package client
 
 import (
 	"context"
+	"fmt"
 	"reflect"
 	"time"
 
@@ -82,3 +83,47 @@ func ResolveTimestampField(path string, rfcs ...string) func(_ context.Context,
 		}
 	}
 }
+
+/*
+SliceJsonResolver resolves slice of objects into a map[string]interface{}.
+For example object: SliceJsonStruct{Nested: &SliceJsonStruct{
+				Nested: &SliceJsonStruct{
+					Value: []types1.Tag{{
+						Key:   "k1",
+						Value: "v1",
+					}, {
+						Key:   "k2",
+						Value: "v2",
+					}},
+				},
+			}}
+can be converted to map[string]interface{}{"k1":"v1","k2":"v2"} by setting a resolver with next params:
+SliceJsonResolver("Nested.Nested.Value", "Key", "Value")
+*/
+func SliceJsonResolver(path, keyPath, valuePath string) schema.ColumnResolver {
+	return func(_ context.Context, meta schema.ClientMeta, r *schema.Resource, c schema.Column) error {
+		var j map[string]interface{}
+		field := funk.Get(r.Item, path, funk.WithAllowZero())
+		s := reflect.ValueOf(field)
+		if s.IsNil() {
+			return r.Set(c.Name, j)
+		}
+		j = make(map[string]interface{})
+		if reflect.TypeOf(field).Kind() != reflect.Slice {
+			return diag.WrapError(fmt.Errorf("field: %s is not a slice", path))
+		}
+		for i := 0; i < s.Len(); i++ {
+			key := funk.Get(s.Index(i).Interface(), keyPath, funk.WithAllowZero())
+			value := funk.Get(s.Index(i).Interface(), valuePath, funk.WithAllowZero())
+			k := reflect.ValueOf(key)
+			if k.Kind() == reflect.Ptr {
+				k = k.Elem()
+			}
+			if k.Kind() != reflect.String {
+				return diag.WrapError(fmt.Errorf("key field: %s is not a string", path))
+			}
+			j[k.String()] = value
+		}
+		return r.Set(c.Name, j)
+	}
+}

client/resolvers_test.go

@@ -12,6 +12,11 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+type SliceJsonStruct struct {
+	Value  []types1.Tag
+	Nested *SliceJsonStruct
+}
+
 func TestResolveTags(t *testing.T) {
 	cases := []struct {
 		InputItem    interface{}
@@ -62,3 +67,83 @@ func TestResolveTags(t *testing.T) {
 		assert.Equal(t, tc.ExpectedTags, r.Get(ta.Columns[0].Name))
 	}
 }
+
+func TestResolveSliceJson(t *testing.T) {
+	cases := []struct {
+		InputItem    interface{}
+		ExpectedData map[string]interface{}
+		path         string
+		keyPath      string
+		valuePath    string
+	}{
+		{
+			InputItem: types1.ListWebhookItem{ // non-ptr
+				Tags: []types1.Tag{
+					{
+						Key:   aws.String("k1"),
+						Value: aws.String("v1"),
+					},
+				},
+			},
+			ExpectedData: map[string]interface{}{"k1": aws.String("v1")},
+			path:         "Tags",
+			keyPath:      "Key",
+			valuePath:    "Value",
+		},
+		{
+			InputItem: &types2.EventSubscription{ // ptr
+				Tags: []types2.Tag{
+					{
+						Key:   aws.String("k2"),
+						Value: aws.String("v2"),
+					},
+				},
+			},
+			ExpectedData: map[string]interface{}{"k2": aws.String("v2")},
+			path:         "Tags",
+			keyPath:      "Key",
+			valuePath:    "Value",
+		},
+		{
+			InputItem: SliceJsonStruct{Nested: &SliceJsonStruct{
+				Nested: &SliceJsonStruct{
+					Value: []types1.Tag{{
+						Key:   aws.String("k1"),
+						Value: aws.String("v1"),
+					}, {
+						Key:   aws.String("k2"),
+						Value: aws.String("v2"),
+					}},
+				},
+			}},
+			ExpectedData: map[string]interface{}{"k1": aws.String("v1"), "k2": aws.String("v2")},
+			path:         "Nested.Nested.Value",
+			keyPath:      "Key",
+			valuePath:    "Value",
+		},
+		{
+			InputItem: types1.ListWebhookItem{ // non-ptr, nil
+				Tags: nil,
+			},
+			ExpectedData: nil,
+			path:         "Tags",
+			keyPath:      "Key",
+			valuePath:    "Value",
+		},
+	}
+
+	for _, tc := range cases {
+		ta := &schema.Table{
+			Columns: []schema.Column{
+				{
+					Name: "tags",
+					Type: schema.TypeJSON,
+				},
+			},
+		}
+		r := schema.NewResourceData(schema.PostgresDialect{}, ta, nil, tc.InputItem, nil, time.Now())
+		err := SliceJsonResolver(tc.path, tc.keyPath, tc.valuePath)(context.Background(), nil, r, ta.Columns[0])
+		assert.NoError(t, err)
+		assert.Equal(t, tc.ExpectedData, r.Get(ta.Columns[0].Name))
+	}
+}

client/services.go

@@ -787,6 +787,7 @@ type GlueClient interface {
 	GetDevEndpoints(ctx context.Context, params *glue.GetDevEndpointsInput, optFns ...func(*glue.Options)) (*glue.GetDevEndpointsOutput, error)
 	GetCrawlers(ctx context.Context, params *glue.GetCrawlersInput, optFns ...func(*glue.Options)) (*glue.GetCrawlersOutput, error)
 	GetSecurityConfigurations(ctx context.Context, params *glue.GetSecurityConfigurationsInput, optFns ...func(*glue.Options)) (*glue.GetSecurityConfigurationsOutput, error)
+	GetPartitionIndexes(ctx context.Context, params *glue.GetPartitionIndexesInput, optFns ...func(*glue.Options)) (*glue.GetPartitionIndexesOutput, error)
 	GetClassifiers(ctx context.Context, params *glue.GetClassifiersInput, optFns ...func(*glue.Options)) (*glue.GetClassifiersOutput, error)
 	GetConnections(ctx context.Context, params *glue.GetConnectionsInput, optFns ...func(*glue.Options)) (*glue.GetConnectionsOutput, error)
 }

docs/tables/aws_glue_database_table_columns.md

@@ -0,0 +1,11 @@
+
+# Table: aws_glue_database_table_columns
+A column in a Table
+## Columns
+| Name        | Type           | Description  |
+| ------------- | ------------- | -----  |
+|database_table_cq_id|uuid|Unique CloudQuery ID of aws_glue_database_tables table (FK)|
+|name|text|The name of the Column|
+|comment|text|A free-form text comment|
+|parameters|jsonb|These key-value pairs define properties associated with the column|
+|type|text|The data type of the Column|

docs/tables/aws_glue_database_table_indexes.md

@@ -0,0 +1,11 @@
+
+# Table: aws_glue_database_table_indexes
+A descriptor for a partition index in a table
+## Columns
+| Name        | Type           | Description  |
+| ------------- | ------------- | -----  |
+|database_table_cq_id|uuid|Unique CloudQuery ID of aws_glue_database_tables table (FK)|
+|index_name|text|The name of the partition index|
+|index_status|text|The status of the partition index|
+|keys|jsonb|A list of one or more keys, as KeySchemaElement structures, for the partition index|
+|backfill_errors|jsonb|A list of errors that can occur when registering partition indexes for an existing table|

docs/tables/aws_glue_database_tables.md

@@ -5,6 +5,8 @@ Represents a collection of related data organized in columns and rows
 | Name        | Type           | Description  |
 | ------------- | ------------- | -----  |
 |database_cq_id|uuid|Unique CloudQuery ID of aws_glue_databases table (FK)|
+|parameters|jsonb||
+|storage_parameters|jsonb||
 |name|text|The table name|
 |catalog_id|text|The ID of the Data Catalog in which the table resides|
 |create_time|timestamp without time zone|The time when the table definition was created in the Data Catalog|
@@ -15,9 +17,21 @@ Represents a collection of related data organized in columns and rows
 |last_access_time|timestamp without time zone|The last time that the table was accessed|
 |last_analyzed_time|timestamp without time zone|The last time that column statistics were computed for this table|
 |owner|text|The owner of the table|
-|parameters|jsonb|These key-value pairs define properties associated with the table|
 |retention|bigint|The retention time for this table|
-|storage_descriptor|jsonb|A storage descriptor containing information about the physical storage of this table|
+|additional_locations|text[]|A list of locations that point to the path where a Delta table is located|
+|bucket_columns|text[]|A list of reducer grouping columns, clustering columns, and bucketing columns in the table|
+|compressed|boolean|True if the data in the table is compressed, or False if not|
+|input_format|text|The input format: SequenceFileInputFormat (binary), or TextInputFormat, or a custom format|
+|location|text|The physical location of the table|
+|number_of_buckets|bigint|Must be specified if the table contains any dimension columns|
+|output_format|text|The output format: SequenceFileOutputFormat (binary), or IgnoreKeyTextOutputFormat, or a custom format|
+|schema_reference_schema_id|jsonb|A structure that contains schema identity fields|
+|schema_reference_schema_version_id|text|The unique ID assigned to a version of the schema|
+|schema_reference_schema_version_number|bigint|The version number of the schema|
+|serde_info|jsonb|The serialization/deserialization (SerDe) information|
+|skewed_info|jsonb|The information about values that appear frequently in a column (skewed values)|
+|sort_columns|jsonb|A list specifying the sort order of each bucket in the table|
+|stored_as_sub_directories|boolean|True if the table data is stored in subdirectories, or False if not|
 |table_type|text|The type of this table (EXTERNAL_TABLE, VIRTUAL_VIEW, etc)|
 |target_table_catalog_id|text|The ID of the Data Catalog in which the table resides|
 |target_table_database_name|text|The name of the catalog database that contains the target table|