From 3d025a20656cd79517e1466f1d9f684ee4b000a6 Mon Sep 17 00:00:00 2001 From: erezrokah Date: Wed, 18 Dec 2024 21:00:40 +0000 Subject: [PATCH 1/3] fix: Use field name in json type schema if json tag is missing --- transformers/name.go | 12 ++++++++++++ transformers/struct.go | 2 +- transformers/struct_test.go | 19 +++++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/transformers/name.go b/transformers/name.go index 93faad20da..2144b99319 100644 --- a/transformers/name.go +++ b/transformers/name.go @@ -26,4 +26,16 @@ func DefaultNameTransformer(field reflect.StructField) (string, error) { return defaultCaser.ToSnake(name), nil } +func JsonTagOrFieldName(field reflect.StructField) (string, error) { + name := field.Name + if jsonTag := strings.Split(field.Tag.Get("json"), ",")[0]; len(jsonTag) > 0 { + // return empty string if the field is not related api response + if jsonTag == "-" { + return "", nil + } + return jsonTag, nil + } + return name, nil +} + var _ NameTransformer = DefaultNameTransformer diff --git a/transformers/struct.go b/transformers/struct.go index a85d22aebf..3e33989d47 100644 --- a/transformers/struct.go +++ b/transformers/struct.go @@ -284,7 +284,7 @@ func (t *structTransformer) fieldToJSONSchema(field reflect.StructField, depth i if !structField.IsExported() || isTypeIgnored(structField.Type) { continue } - name, err := t.nameTransformer(structField) + name, err := JsonTagOrFieldName(structField) if err != nil { continue } diff --git a/transformers/struct_test.go b/transformers/struct_test.go index 81234323ca..281ce34eff 100644 --- a/transformers/struct_test.go +++ b/transformers/struct_test.go @@ -12,6 +12,7 @@ import ( "github.com/cloudquery/plugin-sdk/v4/schema" "github.com/cloudquery/plugin-sdk/v4/types" "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" ) type ( @@ -665,6 +666,23 @@ func TestJSONTypeSchema(t *testing.T) { "item": `{"exported":"utf8"}`, }, }, + { + name: "no json tags", + testStruct: struct { + Tags map[string]string + Item struct { + Name string + Tags map[string]string + FlatItems []string + ComplexItems []struct { + Name string + } + } + }{}, + want: map[string]string{ + "item": `{"ComplexItems":[{"Name":"utf8"}],"FlatItems":["utf8"],"Name":"utf8","Tags":{"utf8":"utf8"}}`, + }, + }, } for _, tt := range tests { @@ -684,6 +702,7 @@ func TestJSONTypeSchema(t *testing.T) { } for col, schema := range tt.want { column := table.Column(col) + require.NotNil(t, column, "column %q not found", col) if diff := cmp.Diff(column.TypeSchema, schema); diff != "" { t.Fatalf("table does not match expected. diff (-got, +want): %v", diff) } From 42037ab1fca1232a211d019d148c78ff60a8c0b8 Mon Sep 17 00:00:00 2001 From: erezrokah Date: Wed, 18 Dec 2024 21:15:24 +0000 Subject: [PATCH 2/3] chore: fix lint --- transformers/name.go | 2 +- transformers/struct.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/transformers/name.go b/transformers/name.go index 2144b99319..ad6fb0332c 100644 --- a/transformers/name.go +++ b/transformers/name.go @@ -26,7 +26,7 @@ func DefaultNameTransformer(field reflect.StructField) (string, error) { return defaultCaser.ToSnake(name), nil } -func JsonTagOrFieldName(field reflect.StructField) (string, error) { +func JSONTagOrFieldName(field reflect.StructField) (string, error) { name := field.Name if jsonTag := strings.Split(field.Tag.Get("json"), ",")[0]; len(jsonTag) > 0 { // return empty string if the field is not related api response diff --git a/transformers/struct.go b/transformers/struct.go index 3e33989d47..4e36a497c4 100644 --- a/transformers/struct.go +++ b/transformers/struct.go @@ -284,7 +284,7 @@ func (t *structTransformer) fieldToJSONSchema(field reflect.StructField, depth i if !structField.IsExported() || isTypeIgnored(structField.Type) { continue } - name, err := JsonTagOrFieldName(structField) + name, err := JSONTagOrFieldName(structField) if err != nil { continue } From 503209501d87bfa5bec452bd833219f1366c202d Mon Sep 17 00:00:00 2001 From: erezrokah Date: Thu, 19 Dec 2024 09:58:04 +0000 Subject: [PATCH 3/3] feat: Export as option --- transformers/name.go | 2 +- transformers/options.go | 8 ++++++++ transformers/struct.go | 14 ++++++++------ 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/transformers/name.go b/transformers/name.go index ad6fb0332c..8f47fd2fb8 100644 --- a/transformers/name.go +++ b/transformers/name.go @@ -26,7 +26,7 @@ func DefaultNameTransformer(field reflect.StructField) (string, error) { return defaultCaser.ToSnake(name), nil } -func JSONTagOrFieldName(field reflect.StructField) (string, error) { +func DefaultJSONColumnSchemaNameTransformer(field reflect.StructField) (string, error) { name := field.Name if jsonTag := strings.Split(field.Tag.Get("json"), ",")[0]; len(jsonTag) > 0 { // return empty string if the field is not related api response diff --git a/transformers/options.go b/transformers/options.go index 779a5791c0..ffeec142b3 100644 --- a/transformers/options.go +++ b/transformers/options.go @@ -31,6 +31,14 @@ func WithNameTransformer(transformer NameTransformer) StructTransformerOption { } } +// WithJSONSchemaNameTransformer overrides how column name will be determined. +// DefaultJSONColumnSchemaNameTransformer is used as the default. +func WithJSONSchemaNameTransformer(transformer NameTransformer) StructTransformerOption { + return func(t *structTransformer) { + t.jsonSchemaNameTransformer = transformer + } +} + // WithTypeTransformer overrides how column type will be determined. // DefaultTypeTransformer is used as the default. func WithTypeTransformer(transformer TypeTransformer) StructTransformerOption { diff --git a/transformers/struct.go b/transformers/struct.go index 4e36a497c4..9435c6098a 100644 --- a/transformers/struct.go +++ b/transformers/struct.go @@ -29,6 +29,7 @@ type structTransformer struct { pkFieldsFound []string pkComponentFields []string pkComponentFieldsFound []string + jsonSchemaNameTransformer NameTransformer maxJSONTypeSchemaDepth int } @@ -194,11 +195,12 @@ func (t *structTransformer) addColumnFromField(field reflect.StructField, parent func TransformWithStruct(st any, opts ...StructTransformerOption) schema.Transform { t := &structTransformer{ - nameTransformer: DefaultNameTransformer, - typeTransformer: DefaultTypeTransformer, - resolverTransformer: DefaultResolverTransformer, - ignoreInTestsTransformer: DefaultIgnoreInTestsTransformer, - maxJSONTypeSchemaDepth: DefaultMaxJSONTypeSchemaDepth, + nameTransformer: DefaultNameTransformer, + typeTransformer: DefaultTypeTransformer, + resolverTransformer: DefaultResolverTransformer, + ignoreInTestsTransformer: DefaultIgnoreInTestsTransformer, + jsonSchemaNameTransformer: DefaultJSONColumnSchemaNameTransformer, + maxJSONTypeSchemaDepth: DefaultMaxJSONTypeSchemaDepth, } for _, opt := range opts { opt(t) @@ -284,7 +286,7 @@ func (t *structTransformer) fieldToJSONSchema(field reflect.StructField, depth i if !structField.IsExported() || isTypeIgnored(structField.Type) { continue } - name, err := JSONTagOrFieldName(structField) + name, err := t.jsonSchemaNameTransformer(structField) if err != nil { continue }