From c3d9a1e48754e2699cb8124e9c4a6f59e58d436e Mon Sep 17 00:00:00 2001 From: bbernays Date: Fri, 24 Feb 2023 14:06:30 -0600 Subject: [PATCH 01/20] working --- go.mod | 2 ++ go.sum | 2 ++ plugins/source/plugin_test.go | 2 +- plugins/source/scheduler_dfs.go | 15 +++++++++++++++ schema/meta.go | 11 ----------- schema/resource.go | 30 ++++++++++++++++++++++++++++++ 6 files changed, 50 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index a8383720a3..0bd603b39e 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,8 @@ require ( gopkg.in/yaml.v3 v3.0.1 ) +require github.com/mitchellh/hashstructure/v2 v2.0.2 + require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/golang/protobuf v1.5.2 // indirect diff --git a/go.sum b/go.sum index 51a98e131a..cf8e6756c3 100644 --- a/go.sum +++ b/go.sum @@ -153,6 +153,8 @@ github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWV github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= +github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4= +github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/plugins/source/plugin_test.go b/plugins/source/plugin_test.go index 657a148d3e..947ed2476d 100644 --- a/plugins/source/plugin_test.go +++ b/plugins/source/plugin_test.go @@ -18,7 +18,7 @@ type testExecutionClient struct{} var _ schema.ClientMeta = &testExecutionClient{} -var stableUUID = uuid.MustParse("00000000000040008000000000000000") +var stableUUID = uuid.MustParse("5cdb0df90f0a5215adc56f15aa74a902") func testResolverSuccess(_ context.Context, _ schema.ClientMeta, _ *schema.Resource, res chan<- any) error { res <- map[string]any{ diff --git a/plugins/source/scheduler_dfs.go b/plugins/source/scheduler_dfs.go index e727e19204..f2e3ef4eba 100644 --- a/plugins/source/scheduler_dfs.go +++ b/plugins/source/scheduler_dfs.go @@ -159,6 +159,21 @@ func (p *Plugin) resolveResourcesDfs(ctx context.Context, table *schema.Table, c if resolvedResource == nil { return } + + if err := resolvedResource.CalculateUniqueValue(); err != nil { + tableMetrics := p.metrics.TableClient[table.Name][client.ID()] + p.logger.Error().Err(err).Str("table", table.Name).Str("client", client.ID()).Msg("resource resolver finished with primary key calculation error") + if _, found := sentValidationErrors.LoadOrStore(table.Name, struct{}{}); !found { + // send resource validation errors to Sentry only once per table, + // to avoid sending too many duplicate messages + sentry.WithScope(func(scope *sentry.Scope) { + scope.SetTag("table", table.Name) + sentry.CurrentHub().CaptureMessage(err.Error()) + }) + } + atomic.AddUint64(&tableMetrics.Errors, 1) + return + } if err := resolvedResource.Validate(); err != nil { tableMetrics := p.metrics.TableClient[table.Name][client.ID()] p.logger.Error().Err(err).Str("table", table.Name).Str("client", client.ID()).Msg("resource resolver finished with validation error") diff --git a/schema/meta.go b/schema/meta.go index 59ddc71c55..04b8797996 100644 --- a/schema/meta.go +++ b/schema/meta.go @@ -2,8 +2,6 @@ package schema import ( "context" - - "github.com/google/uuid" ) type ClientMeta interface { @@ -15,7 +13,6 @@ var CqIDColumn = Column{ Name: "_cq_id", Type: TypeUUID, Description: "Internal CQ ID of the row", - Resolver: cqUUIDResolver(), CreationOptions: ColumnCreationOptions{ NotNull: true, Unique: true, @@ -41,14 +38,6 @@ var CqSourceNameColumn = Column{ Description: "Internal CQ row that references the source plugin name data was retrieved", } -func cqUUIDResolver() ColumnResolver { - return func(_ context.Context, _ ClientMeta, r *Resource, c Column) error { - uuidGen := uuid.New() - b, _ := uuidGen.MarshalBinary() - return r.Set(c.Name, b) - } -} - func parentCqUUIDResolver() ColumnResolver { return func(_ context.Context, _ ClientMeta, r *Resource, c Column) error { if r.Parent == nil { diff --git a/schema/resource.go b/schema/resource.go index f2a993b398..99c8a5a190 100644 --- a/schema/resource.go +++ b/schema/resource.go @@ -2,6 +2,10 @@ package schema import ( "fmt" + + "github.com/google/uuid" + "github.com/mitchellh/hashstructure/v2" + "golang.org/x/exp/slices" ) type Resources []*Resource @@ -91,6 +95,32 @@ func (r *Resource) Columns() []string { return r.Table.Columns.Names() } +func (r *Resource) CalculateUniqueValue() error { + names := r.Table.PrimaryKeys() + if len(names) == 0 { + names = r.Table.Columns.Names() + } + slices.Sort(names) + value := make([]any, len(names)) + for i, name := range names { + value[i] = r.Get(name) + } + + hashVal, err := hashstructure.Hash(value, hashstructure.FormatV2, nil) + if err != nil { + return err + } + var uuidVal uuid.UUID + cqIDVal := uuid.NewSHA1(uuidVal, []byte(fmt.Sprintf("%d", hashVal))) + + cqIDValMarshalled, err := cqIDVal.MarshalBinary() + if err != nil { + return err + } + + return r.Set(CqIDColumn.Name, cqIDValMarshalled) +} + // Validates that all primary keys have values. func (r *Resource) Validate() error { var missingPks []string From 29a6e553de2b9ce5ffaf25b3c4dc0631998d525d Mon Sep 17 00:00:00 2001 From: bbernays Date: Mon, 27 Feb 2023 09:20:01 -0600 Subject: [PATCH 02/20] Create resource_test.go --- schema/resource_test.go | 264 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 264 insertions(+) create mode 100644 schema/resource_test.go diff --git a/schema/resource_test.go b/schema/resource_test.go new file mode 100644 index 0000000000..e846bbb159 --- /dev/null +++ b/schema/resource_test.go @@ -0,0 +1,264 @@ +package schema + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" +) + +var calculateUniqueValueTestCases = []struct { + Name string + Resource any + ExpectedValue *UUID + Table *Table +}{ + { + Name: "Nil Value", + Table: &Table{ + Name: "test_table", + Columns: []Column{ + CqIDColumn, + { + Name: "string_column", + Type: TypeString, + }, + }, + }, + Resource: map[string]any{ + "PathResolver": "test", + "PathResolver2": "test2", + }, + ExpectedValue: &UUID{Bytes: [16]uint8{0x86, 0x60, 0x5d, 0xfd, 0x6c, 0xc0, 0x5b, 0x2d, 0x88, 0xa6, 0xf4, 0x9b, 0x65, 0x13, 0x96, 0x22}, Status: 0x2}, + }, { + Name: "Nil Values", + Table: &Table{ + Name: "test_table", + Columns: []Column{ + CqIDColumn, + { + Name: "string_column", + Type: TypeString, + }, + { + Name: "string_column2", + Type: TypeString, + }, + }, + }, + Resource: map[string]any{ + "PathResolver": "test", + "PathResolver2": "test2", + }, + ExpectedValue: &UUID{Bytes: [16]uint8{0xc4, 0xf7, 0xd1, 0xeb, 0x1b, 0xc, 0x54, 0x11, 0x9e, 0x5d, 0xe6, 0x66, 0x79, 0x7f, 0x85, 0xa9}, Status: 0x2}, + }, + { + Name: "Singular Value", + Table: &Table{ + Name: "test_table", + Columns: []Column{ + CqIDColumn, + { + Name: "string_column", + Type: TypeString, + Resolver: PathResolver("PathResolver"), + }, + { + Name: "string_column2", + Type: TypeString, + }, + }, + }, + Resource: map[string]any{ + "PathResolver": "test", + "PathResolver2": "test2", + }, + ExpectedValue: &UUID{Bytes: [16]uint8{0x56, 0xb8, 0xd, 0xe7, 0x7f, 0xd7, 0x54, 0x93, 0x86, 0x5, 0x64, 0xf, 0x48, 0xec, 0xfb, 0x6}, Status: 0x2}, + }, + { + Name: "Change Column Order from Singular Value", + Table: &Table{ + Name: "test_table", + Columns: []Column{ + CqIDColumn, + { + Name: "string_column2", + Type: TypeString, + }, + { + Name: "string_column", + Type: TypeString, + Resolver: PathResolver("PathResolver"), + }, + }, + }, + Resource: map[string]any{ + "PathResolver": "test", + "PathResolver2": "test2", + }, + ExpectedValue: &UUID{Bytes: [16]uint8{0x56, 0xb8, 0xd, 0xe7, 0x7f, 0xd7, 0x54, 0x93, 0x86, 0x5, 0x64, 0xf, 0x48, 0xec, 0xfb, 0x6}, Status: 0x2}, + }, + { + Name: "Multiple Values", + Table: &Table{ + Name: "test_table", + Columns: []Column{ + CqIDColumn, + { + Name: "string_column", + Type: TypeString, + Resolver: PathResolver("PathResolver"), + }, + { + Name: "string_column2", + Type: TypeString, + Resolver: PathResolver("PathResolver2"), + }, + }, + }, + Resource: map[string]any{ + "PathResolver": "test", + "PathResolver2": "test2", + }, + ExpectedValue: &UUID{Bytes: [16]uint8{0x8f, 0x46, 0x30, 0x99, 0x8d, 0xaf, 0x5a, 0xe6, 0xb7, 0xe6, 0xde, 0x28, 0xe0, 0x37, 0xfd, 0xe}, Status: 0x2}, + }, + { + Name: "Change Order From Multiple Values", + Table: &Table{ + Name: "test_table", + Columns: []Column{ + CqIDColumn, + { + Name: "string_column2", + Type: TypeString, + Resolver: PathResolver("PathResolver2"), + }, + { + Name: "string_column", + Type: TypeString, + Resolver: PathResolver("PathResolver"), + }, + }, + }, + Resource: map[string]any{ + "PathResolver": "test", + "PathResolver2": "test2", + }, + ExpectedValue: &UUID{Bytes: [16]uint8{0x8f, 0x46, 0x30, 0x99, 0x8d, 0xaf, 0x5a, 0xe6, 0xb7, 0xe6, 0xde, 0x28, 0xe0, 0x37, 0xfd, 0xe}, Status: 0x2}, + }, + { + Name: "Singular Primary Key", + Table: &Table{ + Name: "test_table", + Columns: []Column{ + CqIDColumn, + { + Name: "string_column", + Type: TypeString, + Resolver: PathResolver("PathResolver"), + CreationOptions: ColumnCreationOptions{ + PrimaryKey: true, + }, + }, + { + Name: "string_column2", + Type: TypeString, + Resolver: PathResolver("PathResolver2"), + }, + }, + }, + Resource: map[string]any{ + "PathResolver": "test", + "PathResolver2": "test2", + }, + ExpectedValue: &UUID{Bytes: [16]uint8{0x51, 0x1e, 0x28, 0xc2, 0x14, 0x6d, 0x5a, 0x58, 0x8e, 0xda, 0xd3, 0x12, 0x50, 0x97, 0xcc, 0xf0}, Status: 0x2}, + }, + { + Name: "Multiple Primary Keys", + Table: &Table{ + Name: "test_table", + Columns: []Column{ + CqIDColumn, + { + Name: "string_column", + Type: TypeString, + Resolver: PathResolver("PathResolver"), + CreationOptions: ColumnCreationOptions{ + PrimaryKey: true, + }, + }, + { + Name: "string_column2", + Type: TypeString, + Resolver: PathResolver("PathResolver2"), + CreationOptions: ColumnCreationOptions{ + PrimaryKey: true, + }, + }, + }, + }, + Resource: map[string]any{ + "PathResolver": "test", + "PathResolver2": "test2", + }, + ExpectedValue: &UUID{Bytes: [16]uint8{0x9b, 0x8e, 0x13, 0x41, 0xa7, 0x23, 0x5c, 0x35, 0x81, 0x8b, 0x58, 0x2e, 0xed, 0x43, 0xfc, 0xb}, Status: 0x2}, + }, + { + Name: "Change Order of Primary Keys", + Table: &Table{ + Name: "test_table", + Columns: []Column{ + CqIDColumn, + { + Name: "string_column2", + Type: TypeString, + Resolver: PathResolver("PathResolver2"), + CreationOptions: ColumnCreationOptions{ + PrimaryKey: true, + }, + }, + { + Name: "string_column", + Type: TypeString, + Resolver: PathResolver("PathResolver"), + CreationOptions: ColumnCreationOptions{ + PrimaryKey: true, + }, + }, + }, + }, + Resource: map[string]any{ + "PathResolver": "test", + "PathResolver2": "test2", + }, + ExpectedValue: &UUID{Bytes: [16]uint8{0x9b, 0x8e, 0x13, 0x41, 0xa7, 0x23, 0x5c, 0x35, 0x81, 0x8b, 0x58, 0x2e, 0xed, 0x43, 0xfc, 0xb}, Status: 0x2}, + }, +} + +func resolveColumns(t *testing.T, resource *Resource, table *Table) { + for _, column := range table.Columns { + if column.Resolver == nil { + continue + } + err := column.Resolver(context.Background(), nil, resource, column) + if err != nil { + t.Errorf("unexpected error: %v", err) + return + } + } +} + +func TestCalculateUniqueValue(t *testing.T) { + for _, tc := range calculateUniqueValueTestCases { + tc := tc + t.Run(tc.Name, func(t *testing.T) { + resource := NewResourceData(tc.Table, nil, tc.Resource) + resolveColumns(t, resource, tc.Table) + err := resource.CalculateUniqueValue() + if err != nil { + t.Errorf("unexpected error: %v", err) + } + assert.Equal(t, tc.ExpectedValue, resource.Get(CqIDColumn.Name)) + }) + } +} From 3e020b8bca1653c8fc1a789e19d2966483d6b7ac Mon Sep 17 00:00:00 2001 From: bbernays Date: Mon, 27 Feb 2023 10:01:59 -0600 Subject: [PATCH 03/20] Update resource_test.go --- schema/resource_test.go | 78 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/schema/resource_test.go b/schema/resource_test.go index e846bbb159..5ecbad7dee 100644 --- a/schema/resource_test.go +++ b/schema/resource_test.go @@ -233,6 +233,84 @@ var calculateUniqueValueTestCases = []struct { }, ExpectedValue: &UUID{Bytes: [16]uint8{0x9b, 0x8e, 0x13, 0x41, 0xa7, 0x23, 0x5c, 0x35, 0x81, 0x8b, 0x58, 0x2e, 0xed, 0x43, 0xfc, 0xb}, Status: 0x2}, }, + { + Name: "Singular JSON Map", + Table: &Table{ + Name: "test_table", + Columns: []Column{ + CqIDColumn, + { + Name: "json_column", + Type: TypeJSON, + Resolver: PathResolver("PathResolver"), + }, + }, + }, + Resource: map[string]any{ + "PathResolver": map[string]any{ + "test": "test", + "testValInt": 1, + }, + }, + ExpectedValue: &UUID{Bytes: [16]uint8{0x3f, 0xb7, 0xc3, 0xac, 0x6b, 0x9a, 0x54, 0xa2, 0xb4, 0xd5, 0xd, 0x73, 0x11, 0xc0, 0x76, 0x6c}, Status: 0x2}, + }, + { + Name: "Singular JSON Map- Values Change order", + Table: &Table{ + Name: "test_table", + Columns: []Column{ + CqIDColumn, + { + Name: "json_column", + Type: TypeJSON, + Resolver: PathResolver("PathResolver"), + }, + }, + }, + Resource: map[string]any{ + "PathResolver": map[string]any{ + "testValInt": 1, + "test": "test", + }, + }, + ExpectedValue: &UUID{Bytes: [16]uint8{0x3f, 0xb7, 0xc3, 0xac, 0x6b, 0x9a, 0x54, 0xa2, 0xb4, 0xd5, 0xd, 0x73, 0x11, 0xc0, 0x76, 0x6c}, Status: 0x2}, + }, + { + Name: "Singular JSON Array", + Table: &Table{ + Name: "test_table", + Columns: []Column{ + CqIDColumn, + { + Name: "json_column", + Type: TypeJSON, + Resolver: PathResolver("PathResolver"), + }, + }, + }, + Resource: map[string]any{ + "PathResolver": []string{"test", "test2", "test3"}, + }, + ExpectedValue: &UUID{Bytes: [16]uint8{0x28, 0xac, 0xa0, 0xec, 0xb, 0x34, 0x5b, 0xe5, 0xb7, 0x9d, 0xc, 0xae, 0xcc, 0x19, 0xa4, 0xeb}, Status: 0x2}, + }, + { + Name: "Singular JSON Array- Values Changes order- And CQ_ID", + Table: &Table{ + Name: "test_table", + Columns: []Column{ + CqIDColumn, + { + Name: "json_column", + Type: TypeJSON, + Resolver: PathResolver("PathResolver"), + }, + }, + }, + Resource: map[string]any{ + "PathResolver": []string{"test3", "test2", "test"}, + }, + ExpectedValue: &UUID{Bytes: [16]uint8{0x30, 0x55, 0x9c, 0x58, 0xfe, 0x8, 0x5f, 0x2a, 0xb9, 0x2f, 0xb3, 0x2d, 0x35, 0x4d, 0x31, 0xca}, Status: 0x2}, + }, } func resolveColumns(t *testing.T, resource *Resource, table *Table) { From 5a227d2a1f1eaf8f12002be3d98fa9a5f6b909c0 Mon Sep 17 00:00:00 2001 From: bbernays Date: Mon, 27 Feb 2023 10:46:28 -0600 Subject: [PATCH 04/20] Update resource_test.go --- schema/resource_test.go | 126 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) diff --git a/schema/resource_test.go b/schema/resource_test.go index 5ecbad7dee..78b9f4ae89 100644 --- a/schema/resource_test.go +++ b/schema/resource_test.go @@ -2,7 +2,9 @@ package schema import ( "context" + "net/netip" "testing" + "time" "github.com/stretchr/testify/assert" ) @@ -311,6 +313,130 @@ var calculateUniqueValueTestCases = []struct { }, ExpectedValue: &UUID{Bytes: [16]uint8{0x30, 0x55, 0x9c, 0x58, 0xfe, 0x8, 0x5f, 0x2a, 0xb9, 0x2f, 0xb3, 0x2d, 0x35, 0x4d, 0x31, 0xca}, Status: 0x2}, }, + + { + Name: "All CQ Types", + Table: &Table{ + Name: "test_table", + Columns: []Column{ + CqIDColumn, + { + Name: "bool_column", + Type: TypeBool, + Resolver: PathResolver("BooleanValue"), + }, + { + Name: "int_column", + Type: TypeInt, + Resolver: PathResolver("IntValue"), + }, + { + Name: "float_column", + Type: TypeFloat, + Resolver: PathResolver("FloatValue"), + }, + { + Name: "uuid_column", + Type: TypeUUID, + Resolver: PathResolver("UUIDValue"), + }, + { + Name: "string_column", + Type: TypeString, + Resolver: PathResolver("StringValue"), + }, + { + Name: "byte_array_column", + Type: TypeByteArray, + Resolver: PathResolver("ByteArrayValue"), + }, + { + Name: "string_array_column", + Type: TypeStringArray, + Resolver: PathResolver("StringArrayValue"), + }, + { + Name: "int_array_column", + Type: TypeIntArray, + Resolver: PathResolver("IntArrayValue"), + }, + { + Name: "timestamp_column", + Type: TypeTimestamp, + Resolver: PathResolver("TimestampValue"), + }, + { + Name: "json_map_column", + Type: TypeJSON, + Resolver: PathResolver("JSONMapValue"), + }, + { + Name: "json_array_column", + Type: TypeJSON, + Resolver: PathResolver("JSONArrayValue"), + }, + { + Name: "uuid_array_column", + Type: TypeUUIDArray, + Resolver: PathResolver("UUIDArrayValue"), + }, + { + Name: "inet_column", + Type: TypeInet, + Resolver: PathResolver("InetValue"), + }, + { + Name: "inet_array_column", + Type: TypeInetArray, + Resolver: PathResolver("InetArrayValue"), + }, + { + Name: "cidr_column", + Type: TypeCIDR, + Resolver: PathResolver("CidrValue"), + }, + { + Name: "cidr_array_column", + Type: TypeCIDRArray, + Resolver: PathResolver("CidrArrayValue"), + }, + { + Name: "mac_address_column", + Type: TypeMacAddr, + Resolver: PathResolver("MacAddressValue"), + }, + { + Name: "mac_address_array_column", + Type: TypeMacAddrArray, + Resolver: PathResolver("MacAddressArrayValue"), + }, + }, + }, + Resource: map[string]any{ + "BooleanValue": true, + "IntValue": 1456, + "FloatValue": 1456.12, + "UUIDValue": "14625e33-4c0a-44e6-909c-0f9865c1b0f9", + "StringValue": "test", + "ByteArrayValue": []byte{'G', 'O', 'L', 'A', 'N', 'G'}, + "StringArrayValue": []string{"test", "test2", "test3"}, + "IntArrayValue": []int{1, 2, 3}, + "TimestampValue": time.Date(2009, 11, 17, 20, 34, 58, 651387237, time.UTC), + "JSONArrayValue": []string{"test3", "test2", "test"}, + "JSONMapValue": map[string]any{ + "testValInt": 1, + "test": "test", + }, + "UUIDArrayValue": []string{"14625e33-4c0a-44e6-909c-0f9865c1b0f9", "14525e33-4c0a-44e6-909c-0f9865c1b0f0"}, + "InetValue": netip.MustParseAddr("192.0.2.1"), + "InetArrayValue": []netip.Addr{netip.MustParseAddr("192.0.2.1"), netip.MustParseAddr("192.0.2.1")}, + "CidrValue": "192.0.2.1/24", + "CidrArrayValue": []string{"192.0.2.1/24", "192.0.2.1/16"}, + "MacAddressValue": "aa:bb:cc:dd:ee:ff", + "MacAddressArrayValue": []string{"aa:bb:cc:dd:ee:ff", "11:22:33:44:55:66"}, + }, + ExpectedValue: &UUID{Bytes: [16]uint8{0xea, 0x45, 0xb4, 0xba, 0xd1, 0x6e, 0x5c, 0xae, 0x8f, 0x71, 0x5, 0xfb, 0xbc, 0x83, 0xfb, 0x22}, Status: 0x2}, + }, } func resolveColumns(t *testing.T, resource *Resource, table *Table) { From 158d253961bffcbd33616482d1aca85327829bc7 Mon Sep 17 00:00:00 2001 From: bbernays Date: Mon, 27 Feb 2023 10:48:53 -0600 Subject: [PATCH 05/20] Update resource_test.go --- schema/resource_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schema/resource_test.go b/schema/resource_test.go index 78b9f4ae89..59348072a9 100644 --- a/schema/resource_test.go +++ b/schema/resource_test.go @@ -435,7 +435,7 @@ var calculateUniqueValueTestCases = []struct { "MacAddressValue": "aa:bb:cc:dd:ee:ff", "MacAddressArrayValue": []string{"aa:bb:cc:dd:ee:ff", "11:22:33:44:55:66"}, }, - ExpectedValue: &UUID{Bytes: [16]uint8{0xea, 0x45, 0xb4, 0xba, 0xd1, 0x6e, 0x5c, 0xae, 0x8f, 0x71, 0x5, 0xfb, 0xbc, 0x83, 0xfb, 0x22}, Status: 0x2}, + ExpectedValue: &UUID{Bytes: [16]uint8{0xa3, 0x99, 0x29, 0x81, 0xb3, 0xf3, 0x5f, 0x8c, 0xaf, 0x31, 0x1d, 0x7c, 0xee, 0x7b, 0x1c, 0x58}, Status: 0x2}, }, } From cb5ee035b2aed70f84106f4e202d7291160adfdc Mon Sep 17 00:00:00 2001 From: bbernays Date: Tue, 28 Feb 2023 10:04:33 -0600 Subject: [PATCH 06/20] Update resource.go --- schema/resource.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/schema/resource.go b/schema/resource.go index 99c8a5a190..fa868b13de 100644 --- a/schema/resource.go +++ b/schema/resource.go @@ -95,7 +95,15 @@ func (r *Resource) Columns() []string { return r.Table.Columns.Names() } -func (r *Resource) CalculateUniqueValue() error { +func (r *Resource) CalculateUniqueValue(consistentID bool) error { + if !consistentID { + uuidGen := uuid.New() + b, err := uuidGen.MarshalBinary() + if err != nil { + return err + } + return r.Set(CqIDColumn.Name, b) + } names := r.Table.PrimaryKeys() if len(names) == 0 { names = r.Table.Columns.Names() From 5aaf2f01d0fe0620a885c27a1f577388a984f7ab Mon Sep 17 00:00:00 2001 From: bbernays Date: Tue, 28 Feb 2023 10:04:37 -0600 Subject: [PATCH 07/20] Update source.go --- specs/source.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/specs/source.go b/specs/source.go index 27d7ce0eb5..361ef6c6d5 100644 --- a/specs/source.go +++ b/specs/source.go @@ -48,6 +48,9 @@ type Source struct { // Spec defines plugin specific configuration // This is different in every source plugin. Spec any `json:"spec,omitempty"` + + // ConsistentID is a flag that indicates whether the source plugin should generate consistent IDs for resources. + ConsistentID bool `json:"consistent_id,omitempty"` } func (s *Source) SetDefaults() { From 0b7cd543c94e2e953a0e743f1e7d5e6ce731e6a7 Mon Sep 17 00:00:00 2001 From: bbernays Date: Tue, 28 Feb 2023 10:04:41 -0600 Subject: [PATCH 08/20] Update resource_test.go --- schema/resource_test.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/schema/resource_test.go b/schema/resource_test.go index 59348072a9..3e4b419a82 100644 --- a/schema/resource_test.go +++ b/schema/resource_test.go @@ -14,6 +14,7 @@ var calculateUniqueValueTestCases = []struct { Resource any ExpectedValue *UUID Table *Table + ConsistentID bool }{ { Name: "Nil Value", @@ -32,6 +33,7 @@ var calculateUniqueValueTestCases = []struct { "PathResolver2": "test2", }, ExpectedValue: &UUID{Bytes: [16]uint8{0x86, 0x60, 0x5d, 0xfd, 0x6c, 0xc0, 0x5b, 0x2d, 0x88, 0xa6, 0xf4, 0x9b, 0x65, 0x13, 0x96, 0x22}, Status: 0x2}, + ConsistentID: true, }, { Name: "Nil Values", Table: &Table{ @@ -53,6 +55,7 @@ var calculateUniqueValueTestCases = []struct { "PathResolver2": "test2", }, ExpectedValue: &UUID{Bytes: [16]uint8{0xc4, 0xf7, 0xd1, 0xeb, 0x1b, 0xc, 0x54, 0x11, 0x9e, 0x5d, 0xe6, 0x66, 0x79, 0x7f, 0x85, 0xa9}, Status: 0x2}, + ConsistentID: true, }, { Name: "Singular Value", @@ -76,6 +79,7 @@ var calculateUniqueValueTestCases = []struct { "PathResolver2": "test2", }, ExpectedValue: &UUID{Bytes: [16]uint8{0x56, 0xb8, 0xd, 0xe7, 0x7f, 0xd7, 0x54, 0x93, 0x86, 0x5, 0x64, 0xf, 0x48, 0xec, 0xfb, 0x6}, Status: 0x2}, + ConsistentID: true, }, { Name: "Change Column Order from Singular Value", @@ -99,6 +103,7 @@ var calculateUniqueValueTestCases = []struct { "PathResolver2": "test2", }, ExpectedValue: &UUID{Bytes: [16]uint8{0x56, 0xb8, 0xd, 0xe7, 0x7f, 0xd7, 0x54, 0x93, 0x86, 0x5, 0x64, 0xf, 0x48, 0xec, 0xfb, 0x6}, Status: 0x2}, + ConsistentID: true, }, { Name: "Multiple Values", @@ -123,6 +128,7 @@ var calculateUniqueValueTestCases = []struct { "PathResolver2": "test2", }, ExpectedValue: &UUID{Bytes: [16]uint8{0x8f, 0x46, 0x30, 0x99, 0x8d, 0xaf, 0x5a, 0xe6, 0xb7, 0xe6, 0xde, 0x28, 0xe0, 0x37, 0xfd, 0xe}, Status: 0x2}, + ConsistentID: true, }, { Name: "Change Order From Multiple Values", @@ -147,6 +153,7 @@ var calculateUniqueValueTestCases = []struct { "PathResolver2": "test2", }, ExpectedValue: &UUID{Bytes: [16]uint8{0x8f, 0x46, 0x30, 0x99, 0x8d, 0xaf, 0x5a, 0xe6, 0xb7, 0xe6, 0xde, 0x28, 0xe0, 0x37, 0xfd, 0xe}, Status: 0x2}, + ConsistentID: true, }, { Name: "Singular Primary Key", @@ -174,6 +181,7 @@ var calculateUniqueValueTestCases = []struct { "PathResolver2": "test2", }, ExpectedValue: &UUID{Bytes: [16]uint8{0x51, 0x1e, 0x28, 0xc2, 0x14, 0x6d, 0x5a, 0x58, 0x8e, 0xda, 0xd3, 0x12, 0x50, 0x97, 0xcc, 0xf0}, Status: 0x2}, + ConsistentID: true, }, { Name: "Multiple Primary Keys", @@ -204,6 +212,7 @@ var calculateUniqueValueTestCases = []struct { "PathResolver2": "test2", }, ExpectedValue: &UUID{Bytes: [16]uint8{0x9b, 0x8e, 0x13, 0x41, 0xa7, 0x23, 0x5c, 0x35, 0x81, 0x8b, 0x58, 0x2e, 0xed, 0x43, 0xfc, 0xb}, Status: 0x2}, + ConsistentID: true, }, { Name: "Change Order of Primary Keys", @@ -234,6 +243,7 @@ var calculateUniqueValueTestCases = []struct { "PathResolver2": "test2", }, ExpectedValue: &UUID{Bytes: [16]uint8{0x9b, 0x8e, 0x13, 0x41, 0xa7, 0x23, 0x5c, 0x35, 0x81, 0x8b, 0x58, 0x2e, 0xed, 0x43, 0xfc, 0xb}, Status: 0x2}, + ConsistentID: true, }, { Name: "Singular JSON Map", @@ -255,6 +265,7 @@ var calculateUniqueValueTestCases = []struct { }, }, ExpectedValue: &UUID{Bytes: [16]uint8{0x3f, 0xb7, 0xc3, 0xac, 0x6b, 0x9a, 0x54, 0xa2, 0xb4, 0xd5, 0xd, 0x73, 0x11, 0xc0, 0x76, 0x6c}, Status: 0x2}, + ConsistentID: true, }, { Name: "Singular JSON Map- Values Change order", @@ -276,6 +287,7 @@ var calculateUniqueValueTestCases = []struct { }, }, ExpectedValue: &UUID{Bytes: [16]uint8{0x3f, 0xb7, 0xc3, 0xac, 0x6b, 0x9a, 0x54, 0xa2, 0xb4, 0xd5, 0xd, 0x73, 0x11, 0xc0, 0x76, 0x6c}, Status: 0x2}, + ConsistentID: true, }, { Name: "Singular JSON Array", @@ -294,6 +306,7 @@ var calculateUniqueValueTestCases = []struct { "PathResolver": []string{"test", "test2", "test3"}, }, ExpectedValue: &UUID{Bytes: [16]uint8{0x28, 0xac, 0xa0, 0xec, 0xb, 0x34, 0x5b, 0xe5, 0xb7, 0x9d, 0xc, 0xae, 0xcc, 0x19, 0xa4, 0xeb}, Status: 0x2}, + ConsistentID: true, }, { Name: "Singular JSON Array- Values Changes order- And CQ_ID", @@ -312,6 +325,7 @@ var calculateUniqueValueTestCases = []struct { "PathResolver": []string{"test3", "test2", "test"}, }, ExpectedValue: &UUID{Bytes: [16]uint8{0x30, 0x55, 0x9c, 0x58, 0xfe, 0x8, 0x5f, 0x2a, 0xb9, 0x2f, 0xb3, 0x2d, 0x35, 0x4d, 0x31, 0xca}, Status: 0x2}, + ConsistentID: true, }, { @@ -436,6 +450,7 @@ var calculateUniqueValueTestCases = []struct { "MacAddressArrayValue": []string{"aa:bb:cc:dd:ee:ff", "11:22:33:44:55:66"}, }, ExpectedValue: &UUID{Bytes: [16]uint8{0xa3, 0x99, 0x29, 0x81, 0xb3, 0xf3, 0x5f, 0x8c, 0xaf, 0x31, 0x1d, 0x7c, 0xee, 0x7b, 0x1c, 0x58}, Status: 0x2}, + ConsistentID: true, }, } @@ -458,7 +473,7 @@ func TestCalculateUniqueValue(t *testing.T) { t.Run(tc.Name, func(t *testing.T) { resource := NewResourceData(tc.Table, nil, tc.Resource) resolveColumns(t, resource, tc.Table) - err := resource.CalculateUniqueValue() + err := resource.CalculateUniqueValue(tc.ConsistentID) if err != nil { t.Errorf("unexpected error: %v", err) } From ba5bdb16190618227eb1af782b05a9aecf6ca905 Mon Sep 17 00:00:00 2001 From: bbernays Date: Tue, 28 Feb 2023 10:04:46 -0600 Subject: [PATCH 09/20] Update scheduler_dfs.go --- plugins/source/scheduler_dfs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/source/scheduler_dfs.go b/plugins/source/scheduler_dfs.go index f2e3ef4eba..bea97b564a 100644 --- a/plugins/source/scheduler_dfs.go +++ b/plugins/source/scheduler_dfs.go @@ -160,7 +160,7 @@ func (p *Plugin) resolveResourcesDfs(ctx context.Context, table *schema.Table, c return } - if err := resolvedResource.CalculateUniqueValue(); err != nil { + if err := resolvedResource.CalculateUniqueValue(p.spec.ConsistentID); err != nil { tableMetrics := p.metrics.TableClient[table.Name][client.ID()] p.logger.Error().Err(err).Str("table", table.Name).Str("client", client.ID()).Msg("resource resolver finished with primary key calculation error") if _, found := sentValidationErrors.LoadOrStore(table.Name, struct{}{}); !found { From 48241a11012eecbb0be011f84d4a8e8b6cbdde3a Mon Sep 17 00:00:00 2001 From: bbernays Date: Tue, 28 Feb 2023 10:04:48 -0600 Subject: [PATCH 10/20] Update plugin_test.go --- plugins/source/plugin_test.go | 77 ++++++++++++++++++++++++++++++----- 1 file changed, 66 insertions(+), 11 deletions(-) diff --git a/plugins/source/plugin_test.go b/plugins/source/plugin_test.go index 947ed2476d..acb8bac67c 100644 --- a/plugins/source/plugin_test.go +++ b/plugins/source/plugin_test.go @@ -18,7 +18,8 @@ type testExecutionClient struct{} var _ schema.ClientMeta = &testExecutionClient{} -var stableUUID = uuid.MustParse("5cdb0df90f0a5215adc56f15aa74a902") +var deterministicStableUUID = uuid.MustParse("5cdb0df90f0a5215adc56f15aa74a902") +var randomStableUUID = uuid.MustParse("00000000000040008000000000000000") func testResolverSuccess(_ context.Context, _ schema.ClientMeta, _ *schema.Resource, res chan<- any) error { res <- map[string]any{ @@ -122,9 +123,10 @@ func newTestExecutionClient(context.Context, zerolog.Logger, specs.Source, Optio } type syncTestCase struct { - table *schema.Table - stats Metrics - data []schema.CQTypes + table *schema.Table + stats Metrics + data []schema.CQTypes + stableUUID bool } var syncTestCases = []syncTestCase{ @@ -141,7 +143,7 @@ var syncTestCases = []syncTestCase{ }, data: []schema.CQTypes{ { - &schema.UUID{Bytes: stableUUID, Status: schema.Present}, + &schema.UUID{Bytes: randomStableUUID, Status: schema.Present}, &schema.UUID{Status: schema.Null}, &schema.Int8{Int: 3, Status: schema.Present}, }, @@ -173,6 +175,56 @@ var syncTestCases = []syncTestCase{ }, data: nil, }, + + { + table: testTableRelationSuccess(), + stats: Metrics{ + TableClient: map[string]map[string]*TableClientMetrics{ + "test_table_relation_success": { + "testExecutionClient": { + Resources: 1, + }, + }, + "test_table_success": { + "testExecutionClient": { + Resources: 1, + }, + }, + }, + }, + data: []schema.CQTypes{ + { + &schema.UUID{Bytes: randomStableUUID, Status: schema.Present}, + &schema.UUID{Status: schema.Null}, + &schema.Int8{Int: 3, Status: schema.Present}, + }, + { + &schema.UUID{Bytes: randomStableUUID, Status: schema.Present}, + &schema.UUID{Bytes: randomStableUUID, Status: schema.Present}, + &schema.Int8{Int: 3, Status: schema.Present}, + }, + }, + }, + { + table: testTableSuccess(), + stats: Metrics{ + TableClient: map[string]map[string]*TableClientMetrics{ + "test_table_success": { + "testExecutionClient": { + Resources: 1, + }, + }, + }, + }, + data: []schema.CQTypes{ + { + &schema.UUID{Bytes: deterministicStableUUID, Status: schema.Present}, + &schema.UUID{Status: schema.Null}, + &schema.Int8{Int: 3, Status: schema.Present}, + }, + }, + stableUUID: true, + }, { table: testTableColumnResolverPanic(), stats: Metrics{ @@ -187,12 +239,13 @@ var syncTestCases = []syncTestCase{ }, data: []schema.CQTypes{ { - &schema.UUID{Bytes: stableUUID, Status: schema.Present}, + &schema.UUID{Bytes: deterministicStableUUID, Status: schema.Present}, &schema.UUID{Status: schema.Null}, &schema.Int8{Int: 3, Status: schema.Present}, &schema.Int8{Status: schema.Undefined}, }, }, + stableUUID: true, }, { table: testTableRelationSuccess(), @@ -212,16 +265,17 @@ var syncTestCases = []syncTestCase{ }, data: []schema.CQTypes{ { - &schema.UUID{Bytes: stableUUID, Status: schema.Present}, + &schema.UUID{Bytes: deterministicStableUUID, Status: schema.Present}, &schema.UUID{Status: schema.Null}, &schema.Int8{Int: 3, Status: schema.Present}, }, { - &schema.UUID{Bytes: stableUUID, Status: schema.Present}, - &schema.UUID{Bytes: stableUUID, Status: schema.Present}, + &schema.UUID{Bytes: deterministicStableUUID, Status: schema.Present}, + &schema.UUID{Bytes: deterministicStableUUID, Status: schema.Present}, &schema.Int8{Int: 3, Status: schema.Present}, }, }, + stableUUID: true, }, } @@ -241,13 +295,13 @@ func TestSync(t *testing.T) { tc := tc tc.table = tc.table.Copy(nil) t.Run(tc.table.Name+"_"+scheduler.String(), func(t *testing.T) { - testSyncTable(t, tc, scheduler) + testSyncTable(t, tc, scheduler, tc.stableUUID) }) } } } -func testSyncTable(t *testing.T, tc syncTestCase, scheduler specs.Scheduler) { +func testSyncTable(t *testing.T, tc syncTestCase, scheduler specs.Scheduler, stableUUID bool) { ctx := context.Background() tables := []*schema.Table{ tc.table, @@ -268,6 +322,7 @@ func testSyncTable(t *testing.T, tc syncTestCase, scheduler specs.Scheduler) { Destinations: []string{"test"}, Concurrency: 1, // choose a very low value to check that we don't run into deadlocks Scheduler: scheduler, + ConsistentID: stableUUID, } if err := plugin.Init(ctx, spec); err != nil { t.Fatal(err) From 6ec4c7b261a89333f4ccdf8c2b6a78f3244201a9 Mon Sep 17 00:00:00 2001 From: bbernays Date: Tue, 28 Feb 2023 10:10:55 -0600 Subject: [PATCH 11/20] Update resource.go --- schema/resource.go | 1 + 1 file changed, 1 insertion(+) diff --git a/schema/resource.go b/schema/resource.go index fa868b13de..608dfbd535 100644 --- a/schema/resource.go +++ b/schema/resource.go @@ -95,6 +95,7 @@ func (r *Resource) Columns() []string { return r.Table.Columns.Names() } +//nolint:revive func (r *Resource) CalculateUniqueValue(consistentID bool) error { if !consistentID { uuidGen := uuid.New() From 296cbfef58383070f2a9fb591b04d20959abcb23 Mon Sep 17 00:00:00 2001 From: bbernays Date: Tue, 28 Feb 2023 10:15:48 -0600 Subject: [PATCH 12/20] renamed --- schema/resource.go | 2 +- schema/resource_test.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/schema/resource.go b/schema/resource.go index 608dfbd535..428c89a415 100644 --- a/schema/resource.go +++ b/schema/resource.go @@ -96,7 +96,7 @@ func (r *Resource) Columns() []string { } //nolint:revive -func (r *Resource) CalculateUniqueValue(consistentID bool) error { +func (r *Resource) CalculateCQID(consistentID bool) error { if !consistentID { uuidGen := uuid.New() b, err := uuidGen.MarshalBinary() diff --git a/schema/resource_test.go b/schema/resource_test.go index 3e4b419a82..3a65295400 100644 --- a/schema/resource_test.go +++ b/schema/resource_test.go @@ -9,7 +9,7 @@ import ( "github.com/stretchr/testify/assert" ) -var calculateUniqueValueTestCases = []struct { +var CalculateCQIDTestCases = []struct { Name string Resource any ExpectedValue *UUID @@ -467,13 +467,13 @@ func resolveColumns(t *testing.T, resource *Resource, table *Table) { } } -func TestCalculateUniqueValue(t *testing.T) { - for _, tc := range calculateUniqueValueTestCases { +func TestCalculateCQID(t *testing.T) { + for _, tc := range calculateCQIDTestCases { tc := tc t.Run(tc.Name, func(t *testing.T) { resource := NewResourceData(tc.Table, nil, tc.Resource) resolveColumns(t, resource, tc.Table) - err := resource.CalculateUniqueValue(tc.ConsistentID) + err := resource.CalculateCQID(tc.ConsistentID) if err != nil { t.Errorf("unexpected error: %v", err) } From 948bfe0eecb0ed5ab36d97655dd0c9f9a29b9629 Mon Sep 17 00:00:00 2001 From: bbernays Date: Tue, 28 Feb 2023 10:17:14 -0600 Subject: [PATCH 13/20] linting --- plugins/source/scheduler_dfs.go | 2 +- schema/resource_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/source/scheduler_dfs.go b/plugins/source/scheduler_dfs.go index bea97b564a..4dbe448610 100644 --- a/plugins/source/scheduler_dfs.go +++ b/plugins/source/scheduler_dfs.go @@ -160,7 +160,7 @@ func (p *Plugin) resolveResourcesDfs(ctx context.Context, table *schema.Table, c return } - if err := resolvedResource.CalculateUniqueValue(p.spec.ConsistentID); err != nil { + if err := resolvedResource.CalculateCQID(p.spec.ConsistentID); err != nil { tableMetrics := p.metrics.TableClient[table.Name][client.ID()] p.logger.Error().Err(err).Str("table", table.Name).Str("client", client.ID()).Msg("resource resolver finished with primary key calculation error") if _, found := sentValidationErrors.LoadOrStore(table.Name, struct{}{}); !found { diff --git a/schema/resource_test.go b/schema/resource_test.go index 3a65295400..1f3f7da4d6 100644 --- a/schema/resource_test.go +++ b/schema/resource_test.go @@ -9,7 +9,7 @@ import ( "github.com/stretchr/testify/assert" ) -var CalculateCQIDTestCases = []struct { +var calculateCQIDTestCases = []struct { Name string Resource any ExpectedValue *UUID From 484a08cdb5e17b6b6c4cf7175adf5766539b7e05 Mon Sep 17 00:00:00 2001 From: bbernays Date: Tue, 28 Feb 2023 10:27:44 -0600 Subject: [PATCH 14/20] more naming --- plugins/source/plugin_test.go | 34 ++++++++--------- plugins/source/scheduler_dfs.go | 2 +- schema/resource.go | 4 +- schema/resource_test.go | 68 ++++++++++++++++----------------- specs/source.go | 5 ++- 5 files changed, 57 insertions(+), 56 deletions(-) diff --git a/plugins/source/plugin_test.go b/plugins/source/plugin_test.go index acb8bac67c..398bf4f9b6 100644 --- a/plugins/source/plugin_test.go +++ b/plugins/source/plugin_test.go @@ -123,10 +123,10 @@ func newTestExecutionClient(context.Context, zerolog.Logger, specs.Source, Optio } type syncTestCase struct { - table *schema.Table - stats Metrics - data []schema.CQTypes - stableUUID bool + table *schema.Table + stats Metrics + data []schema.CQTypes + deterministicCQID bool } var syncTestCases = []syncTestCase{ @@ -223,7 +223,7 @@ var syncTestCases = []syncTestCase{ &schema.Int8{Int: 3, Status: schema.Present}, }, }, - stableUUID: true, + deterministicCQID: true, }, { table: testTableColumnResolverPanic(), @@ -245,7 +245,7 @@ var syncTestCases = []syncTestCase{ &schema.Int8{Status: schema.Undefined}, }, }, - stableUUID: true, + deterministicCQID: true, }, { table: testTableRelationSuccess(), @@ -275,7 +275,7 @@ var syncTestCases = []syncTestCase{ &schema.Int8{Int: 3, Status: schema.Present}, }, }, - stableUUID: true, + deterministicCQID: true, }, } @@ -295,13 +295,13 @@ func TestSync(t *testing.T) { tc := tc tc.table = tc.table.Copy(nil) t.Run(tc.table.Name+"_"+scheduler.String(), func(t *testing.T) { - testSyncTable(t, tc, scheduler, tc.stableUUID) + testSyncTable(t, tc, scheduler, tc.deterministicCQID) }) } } } -func testSyncTable(t *testing.T, tc syncTestCase, scheduler specs.Scheduler, stableUUID bool) { +func testSyncTable(t *testing.T, tc syncTestCase, scheduler specs.Scheduler, deterministicCQID bool) { ctx := context.Background() tables := []*schema.Table{ tc.table, @@ -315,14 +315,14 @@ func testSyncTable(t *testing.T, tc syncTestCase, scheduler specs.Scheduler, sta ) plugin.SetLogger(zerolog.New(zerolog.NewTestWriter(t))) spec := specs.Source{ - Name: "testSource", - Path: "cloudquery/testSource", - Tables: []string{"*"}, - Version: "v1.0.0", - Destinations: []string{"test"}, - Concurrency: 1, // choose a very low value to check that we don't run into deadlocks - Scheduler: scheduler, - ConsistentID: stableUUID, + Name: "testSource", + Path: "cloudquery/testSource", + Tables: []string{"*"}, + Version: "v1.0.0", + Destinations: []string{"test"}, + Concurrency: 1, // choose a very low value to check that we don't run into deadlocks + Scheduler: scheduler, + DeterministicCQID: deterministicCQID, } if err := plugin.Init(ctx, spec); err != nil { t.Fatal(err) diff --git a/plugins/source/scheduler_dfs.go b/plugins/source/scheduler_dfs.go index 4dbe448610..2bf447e447 100644 --- a/plugins/source/scheduler_dfs.go +++ b/plugins/source/scheduler_dfs.go @@ -160,7 +160,7 @@ func (p *Plugin) resolveResourcesDfs(ctx context.Context, table *schema.Table, c return } - if err := resolvedResource.CalculateCQID(p.spec.ConsistentID); err != nil { + if err := resolvedResource.CalculateCQID(p.spec.DeterministicCQID); err != nil { tableMetrics := p.metrics.TableClient[table.Name][client.ID()] p.logger.Error().Err(err).Str("table", table.Name).Str("client", client.ID()).Msg("resource resolver finished with primary key calculation error") if _, found := sentValidationErrors.LoadOrStore(table.Name, struct{}{}); !found { diff --git a/schema/resource.go b/schema/resource.go index 428c89a415..902a7a41e6 100644 --- a/schema/resource.go +++ b/schema/resource.go @@ -96,8 +96,8 @@ func (r *Resource) Columns() []string { } //nolint:revive -func (r *Resource) CalculateCQID(consistentID bool) error { - if !consistentID { +func (r *Resource) CalculateCQID(deterministicCQID bool) error { + if !deterministicCQID { uuidGen := uuid.New() b, err := uuidGen.MarshalBinary() if err != nil { diff --git a/schema/resource_test.go b/schema/resource_test.go index 1f3f7da4d6..50ea2a8d80 100644 --- a/schema/resource_test.go +++ b/schema/resource_test.go @@ -10,11 +10,11 @@ import ( ) var calculateCQIDTestCases = []struct { - Name string - Resource any - ExpectedValue *UUID - Table *Table - ConsistentID bool + Name string + Resource any + ExpectedValue *UUID + Table *Table + DeterministicCQID bool }{ { Name: "Nil Value", @@ -32,8 +32,8 @@ var calculateCQIDTestCases = []struct { "PathResolver": "test", "PathResolver2": "test2", }, - ExpectedValue: &UUID{Bytes: [16]uint8{0x86, 0x60, 0x5d, 0xfd, 0x6c, 0xc0, 0x5b, 0x2d, 0x88, 0xa6, 0xf4, 0x9b, 0x65, 0x13, 0x96, 0x22}, Status: 0x2}, - ConsistentID: true, + ExpectedValue: &UUID{Bytes: [16]uint8{0x86, 0x60, 0x5d, 0xfd, 0x6c, 0xc0, 0x5b, 0x2d, 0x88, 0xa6, 0xf4, 0x9b, 0x65, 0x13, 0x96, 0x22}, Status: 0x2}, + DeterministicCQID: true, }, { Name: "Nil Values", Table: &Table{ @@ -54,8 +54,8 @@ var calculateCQIDTestCases = []struct { "PathResolver": "test", "PathResolver2": "test2", }, - ExpectedValue: &UUID{Bytes: [16]uint8{0xc4, 0xf7, 0xd1, 0xeb, 0x1b, 0xc, 0x54, 0x11, 0x9e, 0x5d, 0xe6, 0x66, 0x79, 0x7f, 0x85, 0xa9}, Status: 0x2}, - ConsistentID: true, + ExpectedValue: &UUID{Bytes: [16]uint8{0xc4, 0xf7, 0xd1, 0xeb, 0x1b, 0xc, 0x54, 0x11, 0x9e, 0x5d, 0xe6, 0x66, 0x79, 0x7f, 0x85, 0xa9}, Status: 0x2}, + DeterministicCQID: true, }, { Name: "Singular Value", @@ -78,8 +78,8 @@ var calculateCQIDTestCases = []struct { "PathResolver": "test", "PathResolver2": "test2", }, - ExpectedValue: &UUID{Bytes: [16]uint8{0x56, 0xb8, 0xd, 0xe7, 0x7f, 0xd7, 0x54, 0x93, 0x86, 0x5, 0x64, 0xf, 0x48, 0xec, 0xfb, 0x6}, Status: 0x2}, - ConsistentID: true, + ExpectedValue: &UUID{Bytes: [16]uint8{0x56, 0xb8, 0xd, 0xe7, 0x7f, 0xd7, 0x54, 0x93, 0x86, 0x5, 0x64, 0xf, 0x48, 0xec, 0xfb, 0x6}, Status: 0x2}, + DeterministicCQID: true, }, { Name: "Change Column Order from Singular Value", @@ -102,8 +102,8 @@ var calculateCQIDTestCases = []struct { "PathResolver": "test", "PathResolver2": "test2", }, - ExpectedValue: &UUID{Bytes: [16]uint8{0x56, 0xb8, 0xd, 0xe7, 0x7f, 0xd7, 0x54, 0x93, 0x86, 0x5, 0x64, 0xf, 0x48, 0xec, 0xfb, 0x6}, Status: 0x2}, - ConsistentID: true, + ExpectedValue: &UUID{Bytes: [16]uint8{0x56, 0xb8, 0xd, 0xe7, 0x7f, 0xd7, 0x54, 0x93, 0x86, 0x5, 0x64, 0xf, 0x48, 0xec, 0xfb, 0x6}, Status: 0x2}, + DeterministicCQID: true, }, { Name: "Multiple Values", @@ -127,8 +127,8 @@ var calculateCQIDTestCases = []struct { "PathResolver": "test", "PathResolver2": "test2", }, - ExpectedValue: &UUID{Bytes: [16]uint8{0x8f, 0x46, 0x30, 0x99, 0x8d, 0xaf, 0x5a, 0xe6, 0xb7, 0xe6, 0xde, 0x28, 0xe0, 0x37, 0xfd, 0xe}, Status: 0x2}, - ConsistentID: true, + ExpectedValue: &UUID{Bytes: [16]uint8{0x8f, 0x46, 0x30, 0x99, 0x8d, 0xaf, 0x5a, 0xe6, 0xb7, 0xe6, 0xde, 0x28, 0xe0, 0x37, 0xfd, 0xe}, Status: 0x2}, + DeterministicCQID: true, }, { Name: "Change Order From Multiple Values", @@ -152,8 +152,8 @@ var calculateCQIDTestCases = []struct { "PathResolver": "test", "PathResolver2": "test2", }, - ExpectedValue: &UUID{Bytes: [16]uint8{0x8f, 0x46, 0x30, 0x99, 0x8d, 0xaf, 0x5a, 0xe6, 0xb7, 0xe6, 0xde, 0x28, 0xe0, 0x37, 0xfd, 0xe}, Status: 0x2}, - ConsistentID: true, + ExpectedValue: &UUID{Bytes: [16]uint8{0x8f, 0x46, 0x30, 0x99, 0x8d, 0xaf, 0x5a, 0xe6, 0xb7, 0xe6, 0xde, 0x28, 0xe0, 0x37, 0xfd, 0xe}, Status: 0x2}, + DeterministicCQID: true, }, { Name: "Singular Primary Key", @@ -180,8 +180,8 @@ var calculateCQIDTestCases = []struct { "PathResolver": "test", "PathResolver2": "test2", }, - ExpectedValue: &UUID{Bytes: [16]uint8{0x51, 0x1e, 0x28, 0xc2, 0x14, 0x6d, 0x5a, 0x58, 0x8e, 0xda, 0xd3, 0x12, 0x50, 0x97, 0xcc, 0xf0}, Status: 0x2}, - ConsistentID: true, + ExpectedValue: &UUID{Bytes: [16]uint8{0x51, 0x1e, 0x28, 0xc2, 0x14, 0x6d, 0x5a, 0x58, 0x8e, 0xda, 0xd3, 0x12, 0x50, 0x97, 0xcc, 0xf0}, Status: 0x2}, + DeterministicCQID: true, }, { Name: "Multiple Primary Keys", @@ -211,8 +211,8 @@ var calculateCQIDTestCases = []struct { "PathResolver": "test", "PathResolver2": "test2", }, - ExpectedValue: &UUID{Bytes: [16]uint8{0x9b, 0x8e, 0x13, 0x41, 0xa7, 0x23, 0x5c, 0x35, 0x81, 0x8b, 0x58, 0x2e, 0xed, 0x43, 0xfc, 0xb}, Status: 0x2}, - ConsistentID: true, + ExpectedValue: &UUID{Bytes: [16]uint8{0x9b, 0x8e, 0x13, 0x41, 0xa7, 0x23, 0x5c, 0x35, 0x81, 0x8b, 0x58, 0x2e, 0xed, 0x43, 0xfc, 0xb}, Status: 0x2}, + DeterministicCQID: true, }, { Name: "Change Order of Primary Keys", @@ -242,8 +242,8 @@ var calculateCQIDTestCases = []struct { "PathResolver": "test", "PathResolver2": "test2", }, - ExpectedValue: &UUID{Bytes: [16]uint8{0x9b, 0x8e, 0x13, 0x41, 0xa7, 0x23, 0x5c, 0x35, 0x81, 0x8b, 0x58, 0x2e, 0xed, 0x43, 0xfc, 0xb}, Status: 0x2}, - ConsistentID: true, + ExpectedValue: &UUID{Bytes: [16]uint8{0x9b, 0x8e, 0x13, 0x41, 0xa7, 0x23, 0x5c, 0x35, 0x81, 0x8b, 0x58, 0x2e, 0xed, 0x43, 0xfc, 0xb}, Status: 0x2}, + DeterministicCQID: true, }, { Name: "Singular JSON Map", @@ -264,8 +264,8 @@ var calculateCQIDTestCases = []struct { "testValInt": 1, }, }, - ExpectedValue: &UUID{Bytes: [16]uint8{0x3f, 0xb7, 0xc3, 0xac, 0x6b, 0x9a, 0x54, 0xa2, 0xb4, 0xd5, 0xd, 0x73, 0x11, 0xc0, 0x76, 0x6c}, Status: 0x2}, - ConsistentID: true, + ExpectedValue: &UUID{Bytes: [16]uint8{0x3f, 0xb7, 0xc3, 0xac, 0x6b, 0x9a, 0x54, 0xa2, 0xb4, 0xd5, 0xd, 0x73, 0x11, 0xc0, 0x76, 0x6c}, Status: 0x2}, + DeterministicCQID: true, }, { Name: "Singular JSON Map- Values Change order", @@ -286,8 +286,8 @@ var calculateCQIDTestCases = []struct { "test": "test", }, }, - ExpectedValue: &UUID{Bytes: [16]uint8{0x3f, 0xb7, 0xc3, 0xac, 0x6b, 0x9a, 0x54, 0xa2, 0xb4, 0xd5, 0xd, 0x73, 0x11, 0xc0, 0x76, 0x6c}, Status: 0x2}, - ConsistentID: true, + ExpectedValue: &UUID{Bytes: [16]uint8{0x3f, 0xb7, 0xc3, 0xac, 0x6b, 0x9a, 0x54, 0xa2, 0xb4, 0xd5, 0xd, 0x73, 0x11, 0xc0, 0x76, 0x6c}, Status: 0x2}, + DeterministicCQID: true, }, { Name: "Singular JSON Array", @@ -305,8 +305,8 @@ var calculateCQIDTestCases = []struct { Resource: map[string]any{ "PathResolver": []string{"test", "test2", "test3"}, }, - ExpectedValue: &UUID{Bytes: [16]uint8{0x28, 0xac, 0xa0, 0xec, 0xb, 0x34, 0x5b, 0xe5, 0xb7, 0x9d, 0xc, 0xae, 0xcc, 0x19, 0xa4, 0xeb}, Status: 0x2}, - ConsistentID: true, + ExpectedValue: &UUID{Bytes: [16]uint8{0x28, 0xac, 0xa0, 0xec, 0xb, 0x34, 0x5b, 0xe5, 0xb7, 0x9d, 0xc, 0xae, 0xcc, 0x19, 0xa4, 0xeb}, Status: 0x2}, + DeterministicCQID: true, }, { Name: "Singular JSON Array- Values Changes order- And CQ_ID", @@ -324,8 +324,8 @@ var calculateCQIDTestCases = []struct { Resource: map[string]any{ "PathResolver": []string{"test3", "test2", "test"}, }, - ExpectedValue: &UUID{Bytes: [16]uint8{0x30, 0x55, 0x9c, 0x58, 0xfe, 0x8, 0x5f, 0x2a, 0xb9, 0x2f, 0xb3, 0x2d, 0x35, 0x4d, 0x31, 0xca}, Status: 0x2}, - ConsistentID: true, + ExpectedValue: &UUID{Bytes: [16]uint8{0x30, 0x55, 0x9c, 0x58, 0xfe, 0x8, 0x5f, 0x2a, 0xb9, 0x2f, 0xb3, 0x2d, 0x35, 0x4d, 0x31, 0xca}, Status: 0x2}, + DeterministicCQID: true, }, { @@ -449,8 +449,8 @@ var calculateCQIDTestCases = []struct { "MacAddressValue": "aa:bb:cc:dd:ee:ff", "MacAddressArrayValue": []string{"aa:bb:cc:dd:ee:ff", "11:22:33:44:55:66"}, }, - ExpectedValue: &UUID{Bytes: [16]uint8{0xa3, 0x99, 0x29, 0x81, 0xb3, 0xf3, 0x5f, 0x8c, 0xaf, 0x31, 0x1d, 0x7c, 0xee, 0x7b, 0x1c, 0x58}, Status: 0x2}, - ConsistentID: true, + ExpectedValue: &UUID{Bytes: [16]uint8{0xa3, 0x99, 0x29, 0x81, 0xb3, 0xf3, 0x5f, 0x8c, 0xaf, 0x31, 0x1d, 0x7c, 0xee, 0x7b, 0x1c, 0x58}, Status: 0x2}, + DeterministicCQID: true, }, } @@ -473,7 +473,7 @@ func TestCalculateCQID(t *testing.T) { t.Run(tc.Name, func(t *testing.T) { resource := NewResourceData(tc.Table, nil, tc.Resource) resolveColumns(t, resource, tc.Table) - err := resource.CalculateCQID(tc.ConsistentID) + err := resource.CalculateCQID(tc.DeterministicCQID) if err != nil { t.Errorf("unexpected error: %v", err) } diff --git a/specs/source.go b/specs/source.go index 361ef6c6d5..5e476fc4ba 100644 --- a/specs/source.go +++ b/specs/source.go @@ -49,8 +49,9 @@ type Source struct { // This is different in every source plugin. Spec any `json:"spec,omitempty"` - // ConsistentID is a flag that indicates whether the source plugin should generate consistent IDs for resources. - ConsistentID bool `json:"consistent_id,omitempty"` + // DeterministicCQID is a flag that indicates whether the source plugin should generate a random UUID as the value of _cq_id + // or whether it should calculate a UUID that is a hash of the primary keys (if they exist) or the entire resource. + DeterministicCQID bool `json:"deterministic_cq_id,omitempty"` } func (s *Source) SetDefaults() { From d407d38c01565b6393cf3d335320e223394cdbb3 Mon Sep 17 00:00:00 2001 From: bbernays Date: Wed, 1 Mar 2023 07:18:48 -0600 Subject: [PATCH 15/20] Remove hashstructure --- go.mod | 2 -- go.sum | 2 -- schema/resource.go | 31 ++++++++++++------------------- 3 files changed, 12 insertions(+), 23 deletions(-) diff --git a/go.mod b/go.mod index 0bd603b39e..a8383720a3 100644 --- a/go.mod +++ b/go.mod @@ -26,8 +26,6 @@ require ( gopkg.in/yaml.v3 v3.0.1 ) -require github.com/mitchellh/hashstructure/v2 v2.0.2 - require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/golang/protobuf v1.5.2 // indirect diff --git a/go.sum b/go.sum index cf8e6756c3..51a98e131a 100644 --- a/go.sum +++ b/go.sum @@ -153,8 +153,6 @@ github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWV github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= -github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4= -github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/schema/resource.go b/schema/resource.go index 902a7a41e6..aad3cdaa06 100644 --- a/schema/resource.go +++ b/schema/resource.go @@ -1,10 +1,10 @@ package schema import ( + "crypto/sha256" "fmt" "github.com/google/uuid" - "github.com/mitchellh/hashstructure/v2" "golang.org/x/exp/slices" ) @@ -98,36 +98,29 @@ func (r *Resource) Columns() []string { //nolint:revive func (r *Resource) CalculateCQID(deterministicCQID bool) error { if !deterministicCQID { - uuidGen := uuid.New() - b, err := uuidGen.MarshalBinary() - if err != nil { - return err - } - return r.Set(CqIDColumn.Name, b) + return r.storeCQID(uuid.New()) } names := r.Table.PrimaryKeys() if len(names) == 0 { names = r.Table.Columns.Names() } slices.Sort(names) - value := make([]any, len(names)) - for i, name := range names { - value[i] = r.Get(name) - } - - hashVal, err := hashstructure.Hash(value, hashstructure.FormatV2, nil) - if err != nil { - return err + byteRepresentation := make([]byte, 0) + for _, name := range names { + byteRepresentation = append(byteRepresentation, []byte(r.Get(name).String())...) } + hsha256 := sha256.Sum256(byteRepresentation) var uuidVal uuid.UUID - cqIDVal := uuid.NewSHA1(uuidVal, []byte(fmt.Sprintf("%d", hashVal))) + cqIDVal := uuid.NewSHA1(uuidVal, []byte(fmt.Sprintf("%s", hsha256))) + return r.storeCQID(cqIDVal) +} - cqIDValMarshalled, err := cqIDVal.MarshalBinary() +func (r *Resource) storeCQID(value uuid.UUID) error { + b, err := value.MarshalBinary() if err != nil { return err } - - return r.Set(CqIDColumn.Name, cqIDValMarshalled) + return r.Set(CqIDColumn.Name, b) } // Validates that all primary keys have values. From 52444d18cdbb8fc7f46f933eb08a4b14d9e8ddfd Mon Sep 17 00:00:00 2001 From: bbernays Date: Wed, 1 Mar 2023 07:59:05 -0600 Subject: [PATCH 16/20] Update resource.go --- schema/resource.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/schema/resource.go b/schema/resource.go index aad3cdaa06..f5e08cbc96 100644 --- a/schema/resource.go +++ b/schema/resource.go @@ -105,14 +105,13 @@ func (r *Resource) CalculateCQID(deterministicCQID bool) error { names = r.Table.Columns.Names() } slices.Sort(names) - byteRepresentation := make([]byte, 0) + h := sha256.New() for _, name := range names { - byteRepresentation = append(byteRepresentation, []byte(r.Get(name).String())...) + // We need to include the column name in the hash because the same value can be present in multiple columns and therefore lead to the same hash + h.Write([]byte(name)) + h.Write([]byte(r.Get(name).String())) } - hsha256 := sha256.Sum256(byteRepresentation) - var uuidVal uuid.UUID - cqIDVal := uuid.NewSHA1(uuidVal, []byte(fmt.Sprintf("%s", hsha256))) - return r.storeCQID(cqIDVal) + return r.storeCQID(uuid.NewSHA1(uuid.UUID{}, h.Sum(nil))) } func (r *Resource) storeCQID(value uuid.UUID) error { From 432f00b60b56968a213ab0a498c4e0d22047fcc0 Mon Sep 17 00:00:00 2001 From: bbernays Date: Wed, 1 Mar 2023 08:51:42 -0600 Subject: [PATCH 17/20] fix tests --- plugins/source/plugin_test.go | 2 +- schema/resource_test.go | 28 ++++++++++++++-------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/plugins/source/plugin_test.go b/plugins/source/plugin_test.go index 398bf4f9b6..cf92d27eb7 100644 --- a/plugins/source/plugin_test.go +++ b/plugins/source/plugin_test.go @@ -18,7 +18,7 @@ type testExecutionClient struct{} var _ schema.ClientMeta = &testExecutionClient{} -var deterministicStableUUID = uuid.MustParse("5cdb0df90f0a5215adc56f15aa74a902") +var deterministicStableUUID = uuid.MustParse("c25c481db0f05865b6d8f07acab8515f") var randomStableUUID = uuid.MustParse("00000000000040008000000000000000") func testResolverSuccess(_ context.Context, _ schema.ClientMeta, _ *schema.Resource, res chan<- any) error { diff --git a/schema/resource_test.go b/schema/resource_test.go index 50ea2a8d80..4c675a9630 100644 --- a/schema/resource_test.go +++ b/schema/resource_test.go @@ -32,7 +32,7 @@ var calculateCQIDTestCases = []struct { "PathResolver": "test", "PathResolver2": "test2", }, - ExpectedValue: &UUID{Bytes: [16]uint8{0x86, 0x60, 0x5d, 0xfd, 0x6c, 0xc0, 0x5b, 0x2d, 0x88, 0xa6, 0xf4, 0x9b, 0x65, 0x13, 0x96, 0x22}, Status: 0x2}, + ExpectedValue: &UUID{Bytes: [16]uint8{0xdb, 0x8e, 0xfb, 0x63, 0x15, 0xa9, 0x5f, 0x82, 0x98, 0x1a, 0x74, 0x98, 0xbf, 0x72, 0x38, 0x67}, Status: 0x2}, DeterministicCQID: true, }, { Name: "Nil Values", @@ -54,7 +54,7 @@ var calculateCQIDTestCases = []struct { "PathResolver": "test", "PathResolver2": "test2", }, - ExpectedValue: &UUID{Bytes: [16]uint8{0xc4, 0xf7, 0xd1, 0xeb, 0x1b, 0xc, 0x54, 0x11, 0x9e, 0x5d, 0xe6, 0x66, 0x79, 0x7f, 0x85, 0xa9}, Status: 0x2}, + ExpectedValue: &UUID{Bytes: [16]uint8{0x33, 0x49, 0x3d, 0xab, 0xce, 0xd6, 0x51, 0xcf, 0x81, 0x7e, 0x22, 0x62, 0x1, 0x93, 0x80, 0x9c}, Status: 0x2}, DeterministicCQID: true, }, { @@ -78,7 +78,7 @@ var calculateCQIDTestCases = []struct { "PathResolver": "test", "PathResolver2": "test2", }, - ExpectedValue: &UUID{Bytes: [16]uint8{0x56, 0xb8, 0xd, 0xe7, 0x7f, 0xd7, 0x54, 0x93, 0x86, 0x5, 0x64, 0xf, 0x48, 0xec, 0xfb, 0x6}, Status: 0x2}, + ExpectedValue: &UUID{Bytes: [16]uint8{0xd6, 0x6c, 0x68, 0xa4, 0x96, 0x3a, 0x53, 0xd3, 0x84, 0x7e, 0xab, 0xf, 0xfb, 0x8f, 0x1, 0x43}, Status: 0x2}, DeterministicCQID: true, }, { @@ -102,7 +102,7 @@ var calculateCQIDTestCases = []struct { "PathResolver": "test", "PathResolver2": "test2", }, - ExpectedValue: &UUID{Bytes: [16]uint8{0x56, 0xb8, 0xd, 0xe7, 0x7f, 0xd7, 0x54, 0x93, 0x86, 0x5, 0x64, 0xf, 0x48, 0xec, 0xfb, 0x6}, Status: 0x2}, + ExpectedValue: &UUID{Bytes: [16]uint8{0xd6, 0x6c, 0x68, 0xa4, 0x96, 0x3a, 0x53, 0xd3, 0x84, 0x7e, 0xab, 0xf, 0xfb, 0x8f, 0x1, 0x43}, Status: 0x2}, DeterministicCQID: true, }, { @@ -127,7 +127,7 @@ var calculateCQIDTestCases = []struct { "PathResolver": "test", "PathResolver2": "test2", }, - ExpectedValue: &UUID{Bytes: [16]uint8{0x8f, 0x46, 0x30, 0x99, 0x8d, 0xaf, 0x5a, 0xe6, 0xb7, 0xe6, 0xde, 0x28, 0xe0, 0x37, 0xfd, 0xe}, Status: 0x2}, + ExpectedValue: &UUID{Bytes: [16]uint8{0x77, 0x70, 0x8a, 0x6e, 0x9d, 0xe1, 0x50, 0xaf, 0xb4, 0x1, 0x96, 0xa, 0x29, 0xc6, 0x40, 0x2a}, Status: 0x2}, DeterministicCQID: true, }, { @@ -152,7 +152,7 @@ var calculateCQIDTestCases = []struct { "PathResolver": "test", "PathResolver2": "test2", }, - ExpectedValue: &UUID{Bytes: [16]uint8{0x8f, 0x46, 0x30, 0x99, 0x8d, 0xaf, 0x5a, 0xe6, 0xb7, 0xe6, 0xde, 0x28, 0xe0, 0x37, 0xfd, 0xe}, Status: 0x2}, + ExpectedValue: &UUID{Bytes: [16]uint8{0x77, 0x70, 0x8a, 0x6e, 0x9d, 0xe1, 0x50, 0xaf, 0xb4, 0x1, 0x96, 0xa, 0x29, 0xc6, 0x40, 0x2a}, Status: 0x2}, DeterministicCQID: true, }, { @@ -180,7 +180,7 @@ var calculateCQIDTestCases = []struct { "PathResolver": "test", "PathResolver2": "test2", }, - ExpectedValue: &UUID{Bytes: [16]uint8{0x51, 0x1e, 0x28, 0xc2, 0x14, 0x6d, 0x5a, 0x58, 0x8e, 0xda, 0xd3, 0x12, 0x50, 0x97, 0xcc, 0xf0}, Status: 0x2}, + ExpectedValue: &UUID{Bytes: [16]uint8{0xa4, 0xe7, 0x20, 0x33, 0xb7, 0x52, 0x58, 0x70, 0x8d, 0x10, 0xe8, 0xa0, 0x54, 0x60, 0x43, 0xec}, Status: 0x2}, DeterministicCQID: true, }, { @@ -211,7 +211,7 @@ var calculateCQIDTestCases = []struct { "PathResolver": "test", "PathResolver2": "test2", }, - ExpectedValue: &UUID{Bytes: [16]uint8{0x9b, 0x8e, 0x13, 0x41, 0xa7, 0x23, 0x5c, 0x35, 0x81, 0x8b, 0x58, 0x2e, 0xed, 0x43, 0xfc, 0xb}, Status: 0x2}, + ExpectedValue: &UUID{Bytes: [16]uint8{0x6e, 0xdd, 0xa8, 0x8a, 0x80, 0xab, 0x50, 0x3a, 0x8c, 0xdc, 0xac, 0x91, 0xbb, 0x2e, 0xa4, 0x37}, Status: 0x2}, DeterministicCQID: true, }, { @@ -242,7 +242,7 @@ var calculateCQIDTestCases = []struct { "PathResolver": "test", "PathResolver2": "test2", }, - ExpectedValue: &UUID{Bytes: [16]uint8{0x9b, 0x8e, 0x13, 0x41, 0xa7, 0x23, 0x5c, 0x35, 0x81, 0x8b, 0x58, 0x2e, 0xed, 0x43, 0xfc, 0xb}, Status: 0x2}, + ExpectedValue: &UUID{Bytes: [16]uint8{0x6e, 0xdd, 0xa8, 0x8a, 0x80, 0xab, 0x50, 0x3a, 0x8c, 0xdc, 0xac, 0x91, 0xbb, 0x2e, 0xa4, 0x37}, Status: 0x2}, DeterministicCQID: true, }, { @@ -264,7 +264,7 @@ var calculateCQIDTestCases = []struct { "testValInt": 1, }, }, - ExpectedValue: &UUID{Bytes: [16]uint8{0x3f, 0xb7, 0xc3, 0xac, 0x6b, 0x9a, 0x54, 0xa2, 0xb4, 0xd5, 0xd, 0x73, 0x11, 0xc0, 0x76, 0x6c}, Status: 0x2}, + ExpectedValue: &UUID{Bytes: [16]uint8{0x9e, 0x21, 0xba, 0xc4, 0xb7, 0xa2, 0x52, 0x38, 0x88, 0x36, 0xc8, 0x80, 0xa9, 0x44, 0xf2, 0xa8}, Status: 0x2}, DeterministicCQID: true, }, { @@ -286,7 +286,7 @@ var calculateCQIDTestCases = []struct { "test": "test", }, }, - ExpectedValue: &UUID{Bytes: [16]uint8{0x3f, 0xb7, 0xc3, 0xac, 0x6b, 0x9a, 0x54, 0xa2, 0xb4, 0xd5, 0xd, 0x73, 0x11, 0xc0, 0x76, 0x6c}, Status: 0x2}, + ExpectedValue: &UUID{Bytes: [16]uint8{0x9e, 0x21, 0xba, 0xc4, 0xb7, 0xa2, 0x52, 0x38, 0x88, 0x36, 0xc8, 0x80, 0xa9, 0x44, 0xf2, 0xa8}, Status: 0x2}, DeterministicCQID: true, }, { @@ -305,7 +305,7 @@ var calculateCQIDTestCases = []struct { Resource: map[string]any{ "PathResolver": []string{"test", "test2", "test3"}, }, - ExpectedValue: &UUID{Bytes: [16]uint8{0x28, 0xac, 0xa0, 0xec, 0xb, 0x34, 0x5b, 0xe5, 0xb7, 0x9d, 0xc, 0xae, 0xcc, 0x19, 0xa4, 0xeb}, Status: 0x2}, + ExpectedValue: &UUID{Bytes: [16]uint8{0x36, 0xee, 0xc3, 0x12, 0x34, 0x3, 0x57, 0xed, 0xbc, 0xe7, 0xa0, 0xe7, 0x64, 0xb8, 0x37, 0x9c}, Status: 0x2}, DeterministicCQID: true, }, { @@ -324,7 +324,7 @@ var calculateCQIDTestCases = []struct { Resource: map[string]any{ "PathResolver": []string{"test3", "test2", "test"}, }, - ExpectedValue: &UUID{Bytes: [16]uint8{0x30, 0x55, 0x9c, 0x58, 0xfe, 0x8, 0x5f, 0x2a, 0xb9, 0x2f, 0xb3, 0x2d, 0x35, 0x4d, 0x31, 0xca}, Status: 0x2}, + ExpectedValue: &UUID{Bytes: [16]uint8{0x41, 0x4d, 0x19, 0x5c, 0xc9, 0x4, 0x56, 0x71, 0xb5, 0x33, 0x62, 0xcd, 0xae, 0x41, 0x41, 0x83}, Status: 0x2}, DeterministicCQID: true, }, @@ -449,7 +449,7 @@ var calculateCQIDTestCases = []struct { "MacAddressValue": "aa:bb:cc:dd:ee:ff", "MacAddressArrayValue": []string{"aa:bb:cc:dd:ee:ff", "11:22:33:44:55:66"}, }, - ExpectedValue: &UUID{Bytes: [16]uint8{0xa3, 0x99, 0x29, 0x81, 0xb3, 0xf3, 0x5f, 0x8c, 0xaf, 0x31, 0x1d, 0x7c, 0xee, 0x7b, 0x1c, 0x58}, Status: 0x2}, + ExpectedValue: &UUID{Bytes: [16]uint8{0x28, 0x22, 0xbd, 0xf8, 0xce, 0xc5, 0x51, 0x8, 0xab, 0xab, 0xae, 0x6b, 0x5c, 0xa1, 0x8b, 0x44}, Status: 0x2}, DeterministicCQID: true, }, } From e0d74b64bd14064c4ac318797994c88ae63daaf3 Mon Sep 17 00:00:00 2001 From: bbernays Date: Wed, 1 Mar 2023 09:40:32 -0600 Subject: [PATCH 18/20] Update resource_test.go --- schema/resource_test.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/schema/resource_test.go b/schema/resource_test.go index 4c675a9630..6667715ebe 100644 --- a/schema/resource_test.go +++ b/schema/resource_test.go @@ -57,6 +57,31 @@ var calculateCQIDTestCases = []struct { ExpectedValue: &UUID{Bytes: [16]uint8{0x33, 0x49, 0x3d, 0xab, 0xce, 0xd6, 0x51, 0xcf, 0x81, 0x7e, 0x22, 0x62, 0x1, 0x93, 0x80, 0x9c}, Status: 0x2}, DeterministicCQID: true, }, + { + Name: "Multiple Identical Values", + Table: &Table{ + Name: "test_table", + Columns: []Column{ + CqIDColumn, + { + Name: "string_column", + Type: TypeString, + Resolver: PathResolver("PathResolver"), + }, + { + Name: "string_column2", + Type: TypeString, + Resolver: PathResolver("PathResolver2"), + }, + }, + }, + Resource: map[string]any{ + "PathResolver": "test", + "PathResolver2": "test", + }, + ExpectedValue: &UUID{Bytes: [16]uint8{0x8a, 0x46, 0xd3, 0x40, 0x6a, 0xe3, 0x58, 0xd0, 0xa5, 0x92, 0x2c, 0xf3, 0xce, 0x9a, 0x6b, 0x2e}, Status: 0x2}, + DeterministicCQID: true, + }, { Name: "Singular Value", Table: &Table{ From f4651befaec2f271d8ab2f180d181b9ee91f7544 Mon Sep 17 00:00:00 2001 From: bbernays Date: Wed, 1 Mar 2023 09:50:26 -0600 Subject: [PATCH 19/20] Update resource_test.go --- schema/resource_test.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/schema/resource_test.go b/schema/resource_test.go index 6667715ebe..36d71dcc65 100644 --- a/schema/resource_test.go +++ b/schema/resource_test.go @@ -82,6 +82,32 @@ var calculateCQIDTestCases = []struct { ExpectedValue: &UUID{Bytes: [16]uint8{0x8a, 0x46, 0xd3, 0x40, 0x6a, 0xe3, 0x58, 0xd0, 0xa5, 0x92, 0x2c, 0xf3, 0xce, 0x9a, 0x6b, 0x2e}, Status: 0x2}, DeterministicCQID: true, }, + { + Name: "Multiple Identical Values- In Different Columns", + Table: &Table{ + Name: "test_table", + Columns: []Column{ + CqIDColumn, + { + Name: "string_column2", + Type: TypeString, + Resolver: PathResolver("PathResolver"), + }, + { + Name: "string_column3", + Type: TypeString, + Resolver: PathResolver("PathResolver2"), + }, + }, + }, + Resource: map[string]any{ + "PathResolver": "test", + "PathResolver2": "test", + }, + // This should be a different value than the previous test case ("Multiple Identical Values") because the column names are different + ExpectedValue: &UUID{Bytes: [16]uint8{0x1f, 0x95, 0x6a, 0xe8, 0xb7, 0x4c, 0x58, 0x33, 0xac, 0x90, 0x67, 0x3b, 0x1, 0xd4, 0xf8, 0x8d}, Status: 0x2}, + DeterministicCQID: true, + }, { Name: "Singular Value", Table: &Table{ From 4328ddb4cc2625cd761a25923dc58a7c034f874d Mon Sep 17 00:00:00 2001 From: bbernays Date: Wed, 1 Mar 2023 13:10:07 -0600 Subject: [PATCH 20/20] Only hash PK fields --- schema/resource.go | 2 +- schema/resource_test.go | 299 ++++++++++++++++++++-------------------- 2 files changed, 148 insertions(+), 153 deletions(-) diff --git a/schema/resource.go b/schema/resource.go index f5e08cbc96..62485e53e4 100644 --- a/schema/resource.go +++ b/schema/resource.go @@ -102,7 +102,7 @@ func (r *Resource) CalculateCQID(deterministicCQID bool) error { } names := r.Table.PrimaryKeys() if len(names) == 0 { - names = r.Table.Columns.Names() + return r.storeCQID(uuid.New()) } slices.Sort(names) h := sha256.New() diff --git a/schema/resource_test.go b/schema/resource_test.go index 36d71dcc65..d882275f09 100644 --- a/schema/resource_test.go +++ b/schema/resource_test.go @@ -9,7 +9,7 @@ import ( "github.com/stretchr/testify/assert" ) -var calculateCQIDTestCases = []struct { +var calculateCQIDPrimaryKeyTestCases = []struct { Name string Resource any ExpectedValue *UUID @@ -17,48 +17,7 @@ var calculateCQIDTestCases = []struct { DeterministicCQID bool }{ { - Name: "Nil Value", - Table: &Table{ - Name: "test_table", - Columns: []Column{ - CqIDColumn, - { - Name: "string_column", - Type: TypeString, - }, - }, - }, - Resource: map[string]any{ - "PathResolver": "test", - "PathResolver2": "test2", - }, - ExpectedValue: &UUID{Bytes: [16]uint8{0xdb, 0x8e, 0xfb, 0x63, 0x15, 0xa9, 0x5f, 0x82, 0x98, 0x1a, 0x74, 0x98, 0xbf, 0x72, 0x38, 0x67}, Status: 0x2}, - DeterministicCQID: true, - }, { - Name: "Nil Values", - Table: &Table{ - Name: "test_table", - Columns: []Column{ - CqIDColumn, - { - Name: "string_column", - Type: TypeString, - }, - { - Name: "string_column2", - Type: TypeString, - }, - }, - }, - Resource: map[string]any{ - "PathResolver": "test", - "PathResolver2": "test2", - }, - ExpectedValue: &UUID{Bytes: [16]uint8{0x33, 0x49, 0x3d, 0xab, 0xce, 0xd6, 0x51, 0xcf, 0x81, 0x7e, 0x22, 0x62, 0x1, 0x93, 0x80, 0x9c}, Status: 0x2}, - DeterministicCQID: true, - }, - { - Name: "Multiple Identical Values", + Name: "Multiple Identical PK Values", Table: &Table{ Name: "test_table", Columns: []Column{ @@ -67,11 +26,17 @@ var calculateCQIDTestCases = []struct { Name: "string_column", Type: TypeString, Resolver: PathResolver("PathResolver"), + CreationOptions: ColumnCreationOptions{ + PrimaryKey: true, + }, }, { Name: "string_column2", Type: TypeString, Resolver: PathResolver("PathResolver2"), + CreationOptions: ColumnCreationOptions{ + PrimaryKey: true, + }, }, }, }, @@ -79,11 +44,11 @@ var calculateCQIDTestCases = []struct { "PathResolver": "test", "PathResolver2": "test", }, - ExpectedValue: &UUID{Bytes: [16]uint8{0x8a, 0x46, 0xd3, 0x40, 0x6a, 0xe3, 0x58, 0xd0, 0xa5, 0x92, 0x2c, 0xf3, 0xce, 0x9a, 0x6b, 0x2e}, Status: 0x2}, + ExpectedValue: &UUID{Bytes: [16]uint8{0x50, 0x3c, 0x31, 0xc6, 0x9, 0x71, 0x5c, 0x89, 0x83, 0x1b, 0x17, 0x74, 0x9e, 0xf, 0xf5, 0xc7}, Status: 0x2}, DeterministicCQID: true, }, { - Name: "Multiple Identical Values- In Different Columns", + Name: "Multiple Identical PK Values- In Different Columns", Table: &Table{ Name: "test_table", Columns: []Column{ @@ -92,11 +57,17 @@ var calculateCQIDTestCases = []struct { Name: "string_column2", Type: TypeString, Resolver: PathResolver("PathResolver"), + CreationOptions: ColumnCreationOptions{ + PrimaryKey: true, + }, }, { Name: "string_column3", Type: TypeString, Resolver: PathResolver("PathResolver2"), + CreationOptions: ColumnCreationOptions{ + PrimaryKey: true, + }, }, }, }, @@ -105,105 +76,7 @@ var calculateCQIDTestCases = []struct { "PathResolver2": "test", }, // This should be a different value than the previous test case ("Multiple Identical Values") because the column names are different - ExpectedValue: &UUID{Bytes: [16]uint8{0x1f, 0x95, 0x6a, 0xe8, 0xb7, 0x4c, 0x58, 0x33, 0xac, 0x90, 0x67, 0x3b, 0x1, 0xd4, 0xf8, 0x8d}, Status: 0x2}, - DeterministicCQID: true, - }, - { - Name: "Singular Value", - Table: &Table{ - Name: "test_table", - Columns: []Column{ - CqIDColumn, - { - Name: "string_column", - Type: TypeString, - Resolver: PathResolver("PathResolver"), - }, - { - Name: "string_column2", - Type: TypeString, - }, - }, - }, - Resource: map[string]any{ - "PathResolver": "test", - "PathResolver2": "test2", - }, - ExpectedValue: &UUID{Bytes: [16]uint8{0xd6, 0x6c, 0x68, 0xa4, 0x96, 0x3a, 0x53, 0xd3, 0x84, 0x7e, 0xab, 0xf, 0xfb, 0x8f, 0x1, 0x43}, Status: 0x2}, - DeterministicCQID: true, - }, - { - Name: "Change Column Order from Singular Value", - Table: &Table{ - Name: "test_table", - Columns: []Column{ - CqIDColumn, - { - Name: "string_column2", - Type: TypeString, - }, - { - Name: "string_column", - Type: TypeString, - Resolver: PathResolver("PathResolver"), - }, - }, - }, - Resource: map[string]any{ - "PathResolver": "test", - "PathResolver2": "test2", - }, - ExpectedValue: &UUID{Bytes: [16]uint8{0xd6, 0x6c, 0x68, 0xa4, 0x96, 0x3a, 0x53, 0xd3, 0x84, 0x7e, 0xab, 0xf, 0xfb, 0x8f, 0x1, 0x43}, Status: 0x2}, - DeterministicCQID: true, - }, - { - Name: "Multiple Values", - Table: &Table{ - Name: "test_table", - Columns: []Column{ - CqIDColumn, - { - Name: "string_column", - Type: TypeString, - Resolver: PathResolver("PathResolver"), - }, - { - Name: "string_column2", - Type: TypeString, - Resolver: PathResolver("PathResolver2"), - }, - }, - }, - Resource: map[string]any{ - "PathResolver": "test", - "PathResolver2": "test2", - }, - ExpectedValue: &UUID{Bytes: [16]uint8{0x77, 0x70, 0x8a, 0x6e, 0x9d, 0xe1, 0x50, 0xaf, 0xb4, 0x1, 0x96, 0xa, 0x29, 0xc6, 0x40, 0x2a}, Status: 0x2}, - DeterministicCQID: true, - }, - { - Name: "Change Order From Multiple Values", - Table: &Table{ - Name: "test_table", - Columns: []Column{ - CqIDColumn, - { - Name: "string_column2", - Type: TypeString, - Resolver: PathResolver("PathResolver2"), - }, - { - Name: "string_column", - Type: TypeString, - Resolver: PathResolver("PathResolver"), - }, - }, - }, - Resource: map[string]any{ - "PathResolver": "test", - "PathResolver2": "test2", - }, - ExpectedValue: &UUID{Bytes: [16]uint8{0x77, 0x70, 0x8a, 0x6e, 0x9d, 0xe1, 0x50, 0xaf, 0xb4, 0x1, 0x96, 0xa, 0x29, 0xc6, 0x40, 0x2a}, Status: 0x2}, + ExpectedValue: &UUID{Bytes: [16]uint8{0x42, 0x1d, 0x8a, 0x42, 0x6a, 0xf0, 0x51, 0x2d, 0xb8, 0x49, 0xc7, 0xaf, 0xb1, 0xaf, 0x56, 0xec}, Status: 0x2}, DeterministicCQID: true, }, { @@ -306,6 +179,9 @@ var calculateCQIDTestCases = []struct { Name: "json_column", Type: TypeJSON, Resolver: PathResolver("PathResolver"), + CreationOptions: ColumnCreationOptions{ + PrimaryKey: true, + }, }, }, }, @@ -315,7 +191,7 @@ var calculateCQIDTestCases = []struct { "testValInt": 1, }, }, - ExpectedValue: &UUID{Bytes: [16]uint8{0x9e, 0x21, 0xba, 0xc4, 0xb7, 0xa2, 0x52, 0x38, 0x88, 0x36, 0xc8, 0x80, 0xa9, 0x44, 0xf2, 0xa8}, Status: 0x2}, + ExpectedValue: &UUID{Bytes: [16]uint8{0x59, 0xa7, 0x9a, 0x5c, 0x66, 0xc2, 0x5f, 0xb3, 0xb9, 0x8a, 0x9e, 0x81, 0xb8, 0x9e, 0x3f, 0xc7}, Status: 0x2}, DeterministicCQID: true, }, { @@ -328,6 +204,9 @@ var calculateCQIDTestCases = []struct { Name: "json_column", Type: TypeJSON, Resolver: PathResolver("PathResolver"), + CreationOptions: ColumnCreationOptions{ + PrimaryKey: true, + }, }, }, }, @@ -337,7 +216,7 @@ var calculateCQIDTestCases = []struct { "test": "test", }, }, - ExpectedValue: &UUID{Bytes: [16]uint8{0x9e, 0x21, 0xba, 0xc4, 0xb7, 0xa2, 0x52, 0x38, 0x88, 0x36, 0xc8, 0x80, 0xa9, 0x44, 0xf2, 0xa8}, Status: 0x2}, + ExpectedValue: &UUID{Bytes: [16]uint8{0x59, 0xa7, 0x9a, 0x5c, 0x66, 0xc2, 0x5f, 0xb3, 0xb9, 0x8a, 0x9e, 0x81, 0xb8, 0x9e, 0x3f, 0xc7}, Status: 0x2}, DeterministicCQID: true, }, { @@ -350,13 +229,16 @@ var calculateCQIDTestCases = []struct { Name: "json_column", Type: TypeJSON, Resolver: PathResolver("PathResolver"), + CreationOptions: ColumnCreationOptions{ + PrimaryKey: true, + }, }, }, }, Resource: map[string]any{ "PathResolver": []string{"test", "test2", "test3"}, }, - ExpectedValue: &UUID{Bytes: [16]uint8{0x36, 0xee, 0xc3, 0x12, 0x34, 0x3, 0x57, 0xed, 0xbc, 0xe7, 0xa0, 0xe7, 0x64, 0xb8, 0x37, 0x9c}, Status: 0x2}, + ExpectedValue: &UUID{Bytes: [16]uint8{0x3, 0xf, 0x3e, 0xb6, 0x65, 0x24, 0x51, 0x1c, 0xac, 0xa, 0x91, 0x4c, 0x7, 0xa4, 0x1f, 0x6c}, Status: 0x2}, DeterministicCQID: true, }, { @@ -369,16 +251,18 @@ var calculateCQIDTestCases = []struct { Name: "json_column", Type: TypeJSON, Resolver: PathResolver("PathResolver"), + CreationOptions: ColumnCreationOptions{ + PrimaryKey: true, + }, }, }, }, Resource: map[string]any{ "PathResolver": []string{"test3", "test2", "test"}, }, - ExpectedValue: &UUID{Bytes: [16]uint8{0x41, 0x4d, 0x19, 0x5c, 0xc9, 0x4, 0x56, 0x71, 0xb5, 0x33, 0x62, 0xcd, 0xae, 0x41, 0x41, 0x83}, Status: 0x2}, + ExpectedValue: &UUID{Bytes: [16]uint8{0x5, 0x44, 0x97, 0x89, 0x98, 0x8c, 0x5b, 0xb, 0x81, 0x12, 0xdd, 0x7e, 0x74, 0xae, 0x70, 0x56}, Status: 0x2}, DeterministicCQID: true, }, - { Name: "All CQ Types", Table: &Table{ @@ -389,91 +273,145 @@ var calculateCQIDTestCases = []struct { Name: "bool_column", Type: TypeBool, Resolver: PathResolver("BooleanValue"), + CreationOptions: ColumnCreationOptions{ + PrimaryKey: true, + }, }, { Name: "int_column", Type: TypeInt, Resolver: PathResolver("IntValue"), + CreationOptions: ColumnCreationOptions{ + PrimaryKey: true, + }, }, { Name: "float_column", Type: TypeFloat, Resolver: PathResolver("FloatValue"), + CreationOptions: ColumnCreationOptions{ + PrimaryKey: true, + }, }, { Name: "uuid_column", Type: TypeUUID, Resolver: PathResolver("UUIDValue"), + CreationOptions: ColumnCreationOptions{ + PrimaryKey: true, + }, }, { Name: "string_column", Type: TypeString, Resolver: PathResolver("StringValue"), + CreationOptions: ColumnCreationOptions{ + PrimaryKey: true, + }, }, { Name: "byte_array_column", Type: TypeByteArray, Resolver: PathResolver("ByteArrayValue"), + CreationOptions: ColumnCreationOptions{ + PrimaryKey: true, + }, }, { Name: "string_array_column", Type: TypeStringArray, Resolver: PathResolver("StringArrayValue"), + CreationOptions: ColumnCreationOptions{ + PrimaryKey: true, + }, }, { Name: "int_array_column", Type: TypeIntArray, Resolver: PathResolver("IntArrayValue"), + CreationOptions: ColumnCreationOptions{ + PrimaryKey: true, + }, }, { Name: "timestamp_column", Type: TypeTimestamp, Resolver: PathResolver("TimestampValue"), + CreationOptions: ColumnCreationOptions{ + PrimaryKey: true, + }, }, { Name: "json_map_column", Type: TypeJSON, Resolver: PathResolver("JSONMapValue"), + CreationOptions: ColumnCreationOptions{ + PrimaryKey: true, + }, }, { Name: "json_array_column", Type: TypeJSON, Resolver: PathResolver("JSONArrayValue"), + CreationOptions: ColumnCreationOptions{ + PrimaryKey: true, + }, }, { Name: "uuid_array_column", Type: TypeUUIDArray, Resolver: PathResolver("UUIDArrayValue"), + CreationOptions: ColumnCreationOptions{ + PrimaryKey: true, + }, }, { Name: "inet_column", Type: TypeInet, Resolver: PathResolver("InetValue"), + CreationOptions: ColumnCreationOptions{ + PrimaryKey: true, + }, }, { Name: "inet_array_column", Type: TypeInetArray, Resolver: PathResolver("InetArrayValue"), + CreationOptions: ColumnCreationOptions{ + PrimaryKey: true, + }, }, { Name: "cidr_column", Type: TypeCIDR, Resolver: PathResolver("CidrValue"), + CreationOptions: ColumnCreationOptions{ + PrimaryKey: true, + }, }, { Name: "cidr_array_column", Type: TypeCIDRArray, Resolver: PathResolver("CidrArrayValue"), + CreationOptions: ColumnCreationOptions{ + PrimaryKey: true, + }, }, { Name: "mac_address_column", Type: TypeMacAddr, Resolver: PathResolver("MacAddressValue"), + CreationOptions: ColumnCreationOptions{ + PrimaryKey: true, + }, }, { Name: "mac_address_array_column", Type: TypeMacAddrArray, Resolver: PathResolver("MacAddressArrayValue"), + CreationOptions: ColumnCreationOptions{ + PrimaryKey: true, + }, }, }, }, @@ -500,7 +438,7 @@ var calculateCQIDTestCases = []struct { "MacAddressValue": "aa:bb:cc:dd:ee:ff", "MacAddressArrayValue": []string{"aa:bb:cc:dd:ee:ff", "11:22:33:44:55:66"}, }, - ExpectedValue: &UUID{Bytes: [16]uint8{0x28, 0x22, 0xbd, 0xf8, 0xce, 0xc5, 0x51, 0x8, 0xab, 0xab, 0xae, 0x6b, 0x5c, 0xa1, 0x8b, 0x44}, Status: 0x2}, + ExpectedValue: &UUID{Bytes: [16]uint8{0xa6, 0x76, 0x76, 0xfb, 0x4c, 0x95, 0x51, 0x2d, 0x9e, 0xcd, 0xf4, 0xc4, 0xae, 0xc1, 0x2a, 0xf5}, Status: 0x2}, DeterministicCQID: true, }, } @@ -510,6 +448,7 @@ func resolveColumns(t *testing.T, resource *Resource, table *Table) { if column.Resolver == nil { continue } + err := column.Resolver(context.Background(), nil, resource, column) if err != nil { t.Errorf("unexpected error: %v", err) @@ -518,8 +457,8 @@ func resolveColumns(t *testing.T, resource *Resource, table *Table) { } } -func TestCalculateCQID(t *testing.T) { - for _, tc := range calculateCQIDTestCases { +func TestCalculateCQIDWithPrimaryKeys(t *testing.T) { + for _, tc := range calculateCQIDPrimaryKeyTestCases { tc := tc t.Run(tc.Name, func(t *testing.T) { resource := NewResourceData(tc.Table, nil, tc.Resource) @@ -532,3 +471,59 @@ func TestCalculateCQID(t *testing.T) { }) } } + +var calculateCQIDNoPrimaryKeyTestCases = []struct { + Name string + Resource any + ExpectedValue *UUID + Table *Table + DeterministicCQID bool +}{ + { + Name: "No Primary Keys", + Table: &Table{ + Name: "test_table", + Columns: []Column{ + CqIDColumn, + { + Name: "string_column2", + Type: TypeString, + Resolver: PathResolver("PathResolver2"), + }, + { + Name: "string_column", + Type: TypeString, + Resolver: PathResolver("PathResolver"), + }, + }, + }, + Resource: map[string]any{ + "PathResolver": "test", + "PathResolver2": "test2", + }, + ExpectedValue: &UUID{Bytes: [16]uint8{0xa5, 0x55, 0x12, 0x57, 0x65, 0x41, 0x4e, 0x2a, 0xa4, 0x78, 0x4c, 0x88, 0x54, 0x4d, 0x38, 0x34}, Status: 0x2}, + DeterministicCQID: true, + }, +} + +// This test is to ensure that the CQID is not deterministic when there are no primary keys +func TestCalculateCQIDNoPrimaryKeys(t *testing.T) { + for _, tc := range calculateCQIDNoPrimaryKeyTestCases { + tc := tc + t.Run(tc.Name, func(t *testing.T) { + resource := NewResourceData(tc.Table, nil, tc.Resource) + resolveColumns(t, resource, tc.Table) + err := resource.CalculateCQID(tc.DeterministicCQID) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + initialCQID := resource.Get(CqIDColumn.Name).String() + + err = resource.CalculateCQID(tc.DeterministicCQID) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + assert.NotEqual(t, initialCQID, resource.Get(CqIDColumn.Name).String()) + }) + } +}