diff --git a/db/migrations/postgres/000002_create_data_table.up.sql b/db/migrations/postgres/000002_create_data_table.up.sql index 0cbe156780..d761c23a24 100644 --- a/db/migrations/postgres/000002_create_data_table.up.sql +++ b/db/migrations/postgres/000002_create_data_table.up.sql @@ -8,7 +8,7 @@ CREATE TABLE data ( datatype_version VARCHAR(64) NOT NULL, hash CHAR(64) NOT NULL, created BIGINT NOT NULL, - value BYTEA NOT NULL, + value TEXT NOT NULL, blobstore BOOLEAN NOT NULL ); CREATE UNIQUE INDEX data_id ON data(id); diff --git a/db/migrations/postgres/000004_create_batches_table.up.sql b/db/migrations/postgres/000004_create_batches_table.up.sql index 2f8a06c561..84973cf5f2 100644 --- a/db/migrations/postgres/000004_create_batches_table.up.sql +++ b/db/migrations/postgres/000004_create_batches_table.up.sql @@ -8,7 +8,7 @@ CREATE TABLE batches ( group_hash CHAR(64), hash CHAR(64), created BIGINT NOT NULL, - payload BYTEA NOT NULL, + payload TEXT NOT NULL, payload_ref CHAR(64), confirmed BIGINT, tx_type VARCHAR(64) NOT NULL, diff --git a/db/migrations/postgres/000005_create_transactions_table.up.sql b/db/migrations/postgres/000005_create_transactions_table.up.sql index 79b4b6d144..3d0a40b0d0 100644 --- a/db/migrations/postgres/000005_create_transactions_table.up.sql +++ b/db/migrations/postgres/000005_create_transactions_table.up.sql @@ -10,7 +10,7 @@ CREATE TABLE transactions ( created BIGINT NOT NULL, protocol_id VARCHAR(256), status VARCHAR(64) NOT NULL, - info BYTEA + info TEXT ); CREATE UNIQUE INDEX transactions_id ON data(id); diff --git a/db/migrations/postgres/000006_create_datatypes_table.up.sql b/db/migrations/postgres/000006_create_datatypes_table.up.sql index 0eeadd7458..4d2ca8cba4 100644 --- a/db/migrations/postgres/000006_create_datatypes_table.up.sql +++ b/db/migrations/postgres/000006_create_datatypes_table.up.sql @@ -9,7 +9,7 @@ CREATE TABLE datatypes ( version VARCHAR(64) NOT NULL, hash CHAR(64) NOT NULL, created BIGINT NOT NULL, - value BYTEA + value TEXT ); CREATE UNIQUE INDEX datatypes_id ON data(id); diff --git a/db/migrations/postgres/000008_create_operations_table.up.sql b/db/migrations/postgres/000008_create_operations_table.up.sql index 6f125b8284..14011cd450 100644 --- a/db/migrations/postgres/000008_create_operations_table.up.sql +++ b/db/migrations/postgres/000008_create_operations_table.up.sql @@ -12,7 +12,7 @@ CREATE TABLE operations ( created BIGINT NOT NULL, updated BIGINT, error VARCHAR NOT NULL, - info BYTEA + info TEXT ); CREATE UNIQUE INDEX operations_id ON operations(id); diff --git a/db/migrations/postgres/000010_create_subscriptions_table.up.sql b/db/migrations/postgres/000010_create_subscriptions_table.up.sql index cdcc026a46..96af8c8509 100644 --- a/db/migrations/postgres/000010_create_subscriptions_table.up.sql +++ b/db/migrations/postgres/000010_create_subscriptions_table.up.sql @@ -9,7 +9,7 @@ CREATE TABLE subscriptions ( filter_topics VARCHAR(256) NOT NULL, filter_tag VARCHAR(256) NOT NULL, filter_group VARCHAR(256) NOT NULL, - options BYTEA NOT NULL, + options TEXT NOT NULL, created BIGINT NOT NULL ); diff --git a/db/migrations/postgres/000013_create_orgs_table.up.sql b/db/migrations/postgres/000013_create_orgs_table.up.sql index 98144d2dcf..edf34ec6b4 100644 --- a/db/migrations/postgres/000013_create_orgs_table.up.sql +++ b/db/migrations/postgres/000013_create_orgs_table.up.sql @@ -7,7 +7,7 @@ CREATE TABLE orgs ( parent VARCHAR(1024), identity VARCHAR(1024) NOT NULL, description VARCHAR(4096) NOT NULL, - profile BYTEA, + profile TEXT, created BIGINT NOT NULL ); diff --git a/db/migrations/postgres/000014_create_nodes_table.up.sql b/db/migrations/postgres/000014_create_nodes_table.up.sql index 89ac79ccda..4e382ef1f2 100644 --- a/db/migrations/postgres/000014_create_nodes_table.up.sql +++ b/db/migrations/postgres/000014_create_nodes_table.up.sql @@ -7,7 +7,7 @@ CREATE TABLE nodes ( name VARCHAR(64) NOT NULL, description VARCHAR(4096) NOT NULL, dx_peer VARCHAR(256), - dx_endpoint BYTEA, + dx_endpoint TEXT, created BIGINT NOT NULL ); diff --git a/db/migrations/postgres/000015_create_config_table.up.sql b/db/migrations/postgres/000015_create_config_table.up.sql index 7b25511af7..663928c435 100644 --- a/db/migrations/postgres/000015_create_config_table.up.sql +++ b/db/migrations/postgres/000015_create_config_table.up.sql @@ -2,7 +2,7 @@ BEGIN; CREATE TABLE config ( seq SERIAL PRIMARY KEY, config_key VARCHAR(512) NOT NULL, - config_value BYTEA NOT NULL + config_value TEXT NOT NULL ); CREATE UNIQUE INDEX config_sequence ON config(seq); CREATE UNIQUE INDEX config_config_key ON config(config_key); diff --git a/db/migrations/postgres/000027_add_operations_input.up.sql b/db/migrations/postgres/000027_add_operations_input.up.sql index 1b1d3bf160..3ae8dd62d3 100644 --- a/db/migrations/postgres/000027_add_operations_input.up.sql +++ b/db/migrations/postgres/000027_add_operations_input.up.sql @@ -1,4 +1,4 @@ BEGIN; ALTER TABLE operations RENAME COLUMN info TO output; -ALTER TABLE operations ADD COLUMN input BYTEA; +ALTER TABLE operations ADD COLUMN input TEXT; COMMIT; diff --git a/db/migrations/sqlite/000002_create_data_table.up.sql b/db/migrations/sqlite/000002_create_data_table.up.sql index bf91dc743c..f420b15d4c 100644 --- a/db/migrations/sqlite/000002_create_data_table.up.sql +++ b/db/migrations/sqlite/000002_create_data_table.up.sql @@ -7,7 +7,7 @@ CREATE TABLE data ( datatype_version VARCHAR(64) NOT NULL, hash CHAR(64) NOT NULL, created BIGINT NOT NULL, - value BYTEA NOT NULL, + value TEXT NOT NULL, blob_hash CHAR(64), blob_public VARCHAR(1024) ); diff --git a/db/migrations/sqlite/000004_create_batches_table.up.sql b/db/migrations/sqlite/000004_create_batches_table.up.sql index 6bce877bed..c5aab6f9e0 100644 --- a/db/migrations/sqlite/000004_create_batches_table.up.sql +++ b/db/migrations/sqlite/000004_create_batches_table.up.sql @@ -7,7 +7,7 @@ CREATE TABLE batches ( group_hash CHAR(64), hash CHAR(64), created BIGINT NOT NULL, - payload BYTEA NOT NULL, + payload TEXT NOT NULL, payload_ref VARCHAR(256), confirmed BIGINT, tx_type VARCHAR(64) NOT NULL, diff --git a/db/migrations/sqlite/000005_create_transactions_table.up.sql b/db/migrations/sqlite/000005_create_transactions_table.up.sql index 2867f7f581..96ce69af3f 100644 --- a/db/migrations/sqlite/000005_create_transactions_table.up.sql +++ b/db/migrations/sqlite/000005_create_transactions_table.up.sql @@ -9,7 +9,7 @@ CREATE TABLE transactions ( created BIGINT NOT NULL, protocol_id VARCHAR(256), status VARCHAR(64) NOT NULL, - info BYTEA + info TEXT ); CREATE UNIQUE INDEX transactions_id ON data(id); diff --git a/db/migrations/sqlite/000006_create_datatypes_table.up.sql b/db/migrations/sqlite/000006_create_datatypes_table.up.sql index 3127e3340b..26570f5d5d 100644 --- a/db/migrations/sqlite/000006_create_datatypes_table.up.sql +++ b/db/migrations/sqlite/000006_create_datatypes_table.up.sql @@ -8,7 +8,7 @@ CREATE TABLE datatypes ( version VARCHAR(64) NOT NULL, hash CHAR(64) NOT NULL, created BIGINT NOT NULL, - value BYTEA + value TEXT ); CREATE UNIQUE INDEX datatypes_id ON data(id); diff --git a/db/migrations/sqlite/000008_create_operations_table.up.sql b/db/migrations/sqlite/000008_create_operations_table.up.sql index e7a60aac0b..ce67be59ce 100644 --- a/db/migrations/sqlite/000008_create_operations_table.up.sql +++ b/db/migrations/sqlite/000008_create_operations_table.up.sql @@ -11,7 +11,7 @@ CREATE TABLE operations ( created BIGINT NOT NULL, updated BIGINT, error VARCHAR NOT NULL, - info BYTEA + info TEXT ); CREATE UNIQUE INDEX operations_id ON operations(id); diff --git a/db/migrations/sqlite/000010_create_subscriptions_table.up.sql b/db/migrations/sqlite/000010_create_subscriptions_table.up.sql index 3d41e99f62..e017bbd4d4 100644 --- a/db/migrations/sqlite/000010_create_subscriptions_table.up.sql +++ b/db/migrations/sqlite/000010_create_subscriptions_table.up.sql @@ -8,7 +8,7 @@ CREATE TABLE subscriptions ( filter_topics VARCHAR(256) NOT NULL, filter_tag VARCHAR(256) NOT NULL, filter_group VARCHAR(256) NOT NULL, - options BYTEA NOT NULL, + options TEXT NOT NULL, created BIGINT NOT NULL ); diff --git a/db/migrations/sqlite/000013_create_orgs_table.up.sql b/db/migrations/sqlite/000013_create_orgs_table.up.sql index 01424d4b1b..1b9711db33 100644 --- a/db/migrations/sqlite/000013_create_orgs_table.up.sql +++ b/db/migrations/sqlite/000013_create_orgs_table.up.sql @@ -6,7 +6,7 @@ CREATE TABLE orgs ( parent VARCHAR(1024), identity VARCHAR(1024) NOT NULL, description VARCHAR(4096) NOT NULL, - profile BYTEA, + profile TEXT, created BIGINT NOT NULL ); diff --git a/db/migrations/sqlite/000014_create_nodes_table.up.sql b/db/migrations/sqlite/000014_create_nodes_table.up.sql index 7680618114..a802abac1e 100644 --- a/db/migrations/sqlite/000014_create_nodes_table.up.sql +++ b/db/migrations/sqlite/000014_create_nodes_table.up.sql @@ -6,7 +6,7 @@ CREATE TABLE nodes ( name VARCHAR(64) NOT NULL, description VARCHAR(4096) NOT NULL, dx_peer VARCHAR(256), - dx_endpoint BYTEA, + dx_endpoint TEXT, created BIGINT NOT NULL ); diff --git a/db/migrations/sqlite/000015_create_config_table.up.sql b/db/migrations/sqlite/000015_create_config_table.up.sql index fed78211eb..1037cba812 100644 --- a/db/migrations/sqlite/000015_create_config_table.up.sql +++ b/db/migrations/sqlite/000015_create_config_table.up.sql @@ -1,7 +1,7 @@ CREATE TABLE config ( seq INTEGER PRIMARY KEY AUTOINCREMENT, config_key VARCHAR(512) NOT NULL, - config_value BYTEA NOT NULL + config_value TEXT NOT NULL ); CREATE UNIQUE INDEX config_sequence ON config(seq); CREATE UNIQUE INDEX config_config_key ON config(config_key); diff --git a/db/migrations/sqlite/000027_add_operations_input.up.sql b/db/migrations/sqlite/000027_add_operations_input.up.sql index d372ba0312..54c76c5dc5 100644 --- a/db/migrations/sqlite/000027_add_operations_input.up.sql +++ b/db/migrations/sqlite/000027_add_operations_input.up.sql @@ -1,2 +1,2 @@ ALTER TABLE operations RENAME COLUMN info TO output; -ALTER TABLE operations ADD COLUMN input BYTEA; +ALTER TABLE operations ADD COLUMN input TEXT; diff --git a/db/migrations/sqlite/000049_add_blobs_size_and_name.up.sql b/db/migrations/sqlite/000049_add_blobs_size_and_name.up.sql index 7dc9069175..db5f02e2e1 100644 --- a/db/migrations/sqlite/000049_add_blobs_size_and_name.up.sql +++ b/db/migrations/sqlite/000049_add_blobs_size_and_name.up.sql @@ -3,5 +3,8 @@ ALTER TABLE blobs ADD size BIGINT; ALTER TABLE data ADD blob_name VARCHAR(1024); ALTER TABLE data ADD blob_size BIGINT; +UPDATE blobs SET size = 0; +UPDATE data SET blob_size = 0, blob_name = ''; + CREATE INDEX data_blob_name ON data(blob_name); CREATE INDEX data_blob_size ON data(blob_size); diff --git a/docs/reference/api_query_syntax.md b/docs/reference/api_query_syntax.md index d64f41fd4e..d7892386cf 100644 --- a/docs/reference/api_query_syntax.md +++ b/docs/reference/api_query_syntax.md @@ -20,6 +20,7 @@ nav_order: 1 REST collections provide filter, `skip`, `limit` and `sort` support. - The field in the message is used as the query parameter + - Syntax: `field=[modifiers][operator]match-string` - When multiple query parameters are supplied these are combined with AND - When the same query parameter is supplied multiple times, these are combined with OR @@ -38,15 +39,52 @@ This states: Table of filter operations, which must be the first character of the query string (after the `=` in the above URL path example) -| Operator | Description | -|----------|-----------------------------------| -| (none) | Equal | -| `!` | Not equal | -| `<` | Less than | -| `<=` | Less than or equal | -| `>` | Greater than | -| `>=` | Greater than or equal | -| `@` | Containing - case sensitive | -| `!@` | Not containing - case sensitive | -| `^` | Containing - case insensitive | -| `!^` | Not containing - case insensitive | +### Operators + +Operators are a type of comparison operation to +perform against the match string. + +| Operator | Description | +|----------|------------------------------------| +| `=` | Equal | +| (none) | Equal (shortcut) | +| `@` | Containing | +| `^` | Starts with | +| `$` | Ends with | +| `<<` | Less than | +| `<` | Less than (shortcut) | +| `<=` | Less than or equal | +| `>>` | Greater than | +| `>` | Greater than (shortcut) | +| `>=` | Greater than or equal | + +> Shortcuts are only safe to use when your match +> string starts with `a-z`, `A-Z`, `0-9`, `-` or `_`. + +### Modifiers + +Modifiers can appear before the operator, to change its +behavior. + +| Modifier | Description | +|----------|------------------------------------| +| `!` | Not - negates the match | +| `:` | Case insensitive | +| `?` | Treat empty match string as null | + +## Detailed examples + +| Example | Description | +|--------------|--------------------------------------------| +| `cat` | Equals "cat" | +| `=cat` | Equals "cat" (same) | +| `!=cat` | Not equal to "cat" | +| `:=cat` | Equal to "CAT", "cat", "CaT etc. | +| `!:cat` | Not equal to "CAT", "cat", "CaT etc. | +| `=!cat` | Equal to "!cat" (! is after operator) | +| `^cats/` | Starts with "cats/" | +| `$_cat` | Ends with with "_cat" | +| `!:^cats/` | Does not start with "cats/", "CATs/" etc. | +| `!$-cat` | Does not end with "-cat" | +| `?=` | Is null | +| `!?=` | Is not null | diff --git a/docs/swagger/swagger.yaml b/docs/swagger/swagger.yaml index db0af644d3..46b8137422 100644 --- a/docs/swagger/swagger.yaml +++ b/docs/swagger/swagger.yaml @@ -479,7 +479,6 @@ paths: validator: type: string value: - format: byte type: string type: object type: array @@ -633,7 +632,6 @@ paths: validator: type: string value: - format: byte type: string type: object type: array @@ -744,7 +742,6 @@ paths: - definition type: string value: - format: byte type: string version: type: string @@ -1070,6 +1067,11 @@ paths: name: validator schema: type: string + - description: 'Data filter field. Prefixes supported: > >= < <= @ ^ ! !@ !^' + in: query + name: value + schema: + type: string - description: Sort field. For multi-field sort use comma separated values (or multiple query values) with '-' prefix for descending in: query @@ -1137,7 +1139,6 @@ paths: validator: type: string value: - format: byte type: string type: object type: array @@ -1190,7 +1191,6 @@ paths: validator: type: string value: - format: byte type: string type: object multipart/form-data: @@ -1247,7 +1247,6 @@ paths: validator: type: string value: - format: byte type: string type: object description: Success @@ -1310,7 +1309,6 @@ paths: validator: type: string value: - format: byte type: string type: object description: Success @@ -1787,7 +1785,6 @@ paths: - definition type: string value: - format: byte type: string version: type: string @@ -1834,7 +1831,6 @@ paths: - definition type: string value: - format: byte type: string version: type: string @@ -1860,7 +1856,6 @@ paths: - definition type: string value: - format: byte type: string version: type: string @@ -1886,7 +1881,6 @@ paths: - definition type: string value: - format: byte type: string version: type: string @@ -1941,7 +1935,6 @@ paths: - definition type: string value: - format: byte type: string version: type: string @@ -2001,7 +1994,6 @@ paths: - definition type: string value: - format: byte type: string version: type: string @@ -2726,7 +2718,6 @@ paths: validator: type: string value: - format: byte type: string type: object type: array diff --git a/internal/apiserver/restfilter.go b/internal/apiserver/restfilter.go index 1e52e03745..dd74237c96 100644 --- a/internal/apiserver/restfilter.go +++ b/internal/apiserver/restfilter.go @@ -1,4 +1,4 @@ -// Copyright © 2021 Kaleido, Inc. +// Copyright © 2022 Kaleido, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -17,6 +17,8 @@ package apiserver import ( + "context" + "database/sql/driver" "net/http" "net/url" "reflect" @@ -35,6 +37,12 @@ type filterResultsWithCount struct { Items interface{} `json:"items"` } +type filterModifiers struct { + negate bool + caseInsensitive bool + emptyIsNull bool +} + func syncRetcode(isSync bool) int { if isSync { return http.StatusOK @@ -75,12 +83,20 @@ func (as *apiServer) buildFilter(req *http.Request, ff database.QueryFactory) (d for _, field := range possibleFields { values := as.getValues(req.Form, field) if len(values) == 1 { - filter.Condition(as.getCondition(fb, field, values[0])) + cond, err := as.getCondition(ctx, fb, field, values[0]) + if err != nil { + return nil, err + } + filter.Condition(cond) } else if len(values) > 0 { sort.Strings(values) fs := make([]database.Filter, len(values)) for i, value := range values { - fs[i] = as.getCondition(fb, field, value) + cond, err := as.getCondition(ctx, fb, field, value) + if err != nil { + return nil, err + } + fs[i] = cond } filter.Condition(fb.Or(fs...)) } @@ -123,27 +139,116 @@ func (as *apiServer) buildFilter(req *http.Request, ff database.QueryFactory) (d return filter, nil } -func (as *apiServer) getCondition(fb database.FilterBuilder, field, value string) database.Filter { - switch { - case strings.HasPrefix(value, ">="): - return fb.Gte(field, value[2:]) - case strings.HasPrefix(value, "<="): - return fb.Lte(field, value[2:]) - case strings.HasPrefix(value, ">"): - return fb.Gt(field, value[1:]) - case strings.HasPrefix(value, "<"): - return fb.Lt(field, value[1:]) - case strings.HasPrefix(value, "@"): - return fb.Contains(field, value[1:]) - case strings.HasPrefix(value, "^"): - return fb.IContains(field, value[1:]) - case strings.HasPrefix(value, "!@"): - return fb.NotContains(field, value[2:]) - case strings.HasPrefix(value, "!^"): - return fb.NotIContains(field, value[2:]) - case strings.HasPrefix(value, "!"): - return fb.Neq(field, value[1:]) +func (as *apiServer) checkNoMods(ctx context.Context, mods filterModifiers, field, op string, filter database.Filter) (database.Filter, error) { + emptyModifiers := filterModifiers{} + if mods != emptyModifiers { + return nil, i18n.NewError(ctx, i18n.MsgQueryOpUnsupportedMod, op, field) + } + return filter, nil +} + +func (as *apiServer) getCondition(ctx context.Context, fb database.FilterBuilder, field, value string) (filter database.Filter, err error) { + + mods := filterModifiers{} + operator := make([]rune, 0, 2) + prefixLength := 0 +opFinder: + for _, r := range value { + switch r { + case '!': + mods.negate = true + prefixLength++ + case ':': + mods.caseInsensitive = true + prefixLength++ + case '?': + mods.emptyIsNull = true + prefixLength++ + case '>', '<': + // Terminates the opFinder if it's the second character + if len(operator) == 1 && operator[0] != r { + // Detected "><" or "<>" - which is a single char operator, followed by beginning of match string + break opFinder + } + operator = append(operator, r) + prefixLength++ + if len(operator) > 1 { + // Detected ">>" or "<<" full operators + break opFinder + } + case '=', '@', '^', '$': + // Always terminates the opFinder + // Could be ">=" or "<=" (due to above logic continuing on '>' or '<' first char) + operator = append(operator, r) + prefixLength++ + break opFinder + default: + // Found a normal character + break opFinder + } + } + + var matchString driver.Value = value[prefixLength:] + if mods.emptyIsNull && prefixLength == len(value) { + matchString = nil + } + return as.mapOperation(ctx, fb, field, matchString, string(operator), mods) +} + +func (as *apiServer) mapOperation(ctx context.Context, fb database.FilterBuilder, field string, matchString driver.Value, op string, mods filterModifiers) (filter database.Filter, err error) { + + switch op { + case ">=": + return as.checkNoMods(ctx, mods, field, op, fb.Gte(field, matchString)) + case "<=": + return as.checkNoMods(ctx, mods, field, op, fb.Lte(field, matchString)) + case ">", ">>": + return as.checkNoMods(ctx, mods, field, op, fb.Gt(field, matchString)) + case "<", "<<": + return as.checkNoMods(ctx, mods, field, op, fb.Lt(field, matchString)) + case "@": + if mods.caseInsensitive { + if mods.negate { + return fb.NotIContains(field, matchString), nil + } + return fb.IContains(field, matchString), nil + } + if mods.negate { + return fb.NotContains(field, matchString), nil + } + return fb.Contains(field, matchString), nil + case "^": + if mods.caseInsensitive { + if mods.negate { + return fb.NotIStartsWith(field, matchString), nil + } + return fb.IStartsWith(field, matchString), nil + } + if mods.negate { + return fb.NotStartsWith(field, matchString), nil + } + return fb.StartsWith(field, matchString), nil + case "$": + if mods.caseInsensitive { + if mods.negate { + return fb.NotIEndsWith(field, matchString), nil + } + return fb.IEndsWith(field, matchString), nil + } + if mods.negate { + return fb.NotEndsWith(field, matchString), nil + } + return fb.EndsWith(field, matchString), nil default: - return fb.Eq(field, value) + if mods.caseInsensitive { + if mods.negate { + return fb.NIeq(field, matchString), nil + } + return fb.IEq(field, matchString), nil + } + if mods.negate { + return fb.Neq(field, matchString), nil + } + return fb.Eq(field, matchString), nil } } diff --git a/internal/apiserver/restfilter_test.go b/internal/apiserver/restfilter_test.go index 897bbd2f71..814a0ea56c 100644 --- a/internal/apiserver/restfilter_test.go +++ b/internal/apiserver/restfilter_test.go @@ -17,6 +17,7 @@ package apiserver import ( + "fmt" "net/http/httptest" "testing" @@ -35,7 +36,82 @@ func TestBuildFilterDescending(t *testing.T) { fi, err := filter.Finalize() assert.NoError(t, err) - assert.Equal(t, "( confirmed != 0 ) && ( created == 0 ) && ( ( tag %! 'abc' ) || ( tag ^! 'abc' ) || ( tag <= 'abc' ) || ( tag < 'abc' ) || ( tag >= 'abc' ) || ( tag > 'abc' ) || ( tag %= 'abc' ) || ( tag ^= 'abc' ) ) sort=-tag,-sequence skip=10 limit=50", fi.String()) + assert.Equal(t, "( confirmed != 0 ) && ( created == 0 ) && ( ( tag !% 'abc' ) || ( tag !^ 'abc' ) || ( tag <= 'abc' ) || ( tag << 'abc' ) || ( tag >= 'abc' ) || ( tag >> 'abc' ) || ( tag %= 'abc' ) || ( tag ^= 'abc' ) ) sort=-tag,-sequence skip=10 limit=50", fi.String()) +} + +func testIndividualFilter(t *testing.T, queryString, expectedToString string) { + as := &apiServer{ + maxFilterLimit: 250, + } + req := httptest.NewRequest("GET", fmt.Sprintf("/things?%s", queryString), nil) + filter, err := as.buildFilter(req, database.MessageQueryFactory) + assert.NoError(t, err) + fi, err := filter.Finalize() + assert.NoError(t, err) + assert.Equal(t, expectedToString, fi.String()) +} + +func TestBuildFilterEachCombo(t *testing.T) { + testIndividualFilter(t, "tag=cat", "( tag == 'cat' )") + testIndividualFilter(t, "tag==cat", "( tag == 'cat' )") + testIndividualFilter(t, "tag===cat", "( tag == '=cat' )") + testIndividualFilter(t, "tag=!cat", "( tag != 'cat' )") + testIndividualFilter(t, "tag=!=cat", "( tag != 'cat' )") + testIndividualFilter(t, "tag=!=!cat", "( tag != '!cat' )") + testIndividualFilter(t, "tag=!==cat", "( tag != '=cat' )") + testIndividualFilter(t, "tag=!:=cat", "( tag ;= 'cat' )") + testIndividualFilter(t, "tag=:!=cat", "( tag ;= 'cat' )") + testIndividualFilter(t, "tag=:=cat", "( tag := 'cat' )") + testIndividualFilter(t, "tag=>cat", "( tag >> 'cat' )") + testIndividualFilter(t, "tag=>>cat", "( tag >> 'cat' )") + testIndividualFilter(t, "tag=>>>cat", "( tag >> '>cat' )") + testIndividualFilter(t, "tag=cat", "( tag << '>cat' )") + testIndividualFilter(t, "tag=>> '=cat", "( tag >= 'cat' )") + testIndividualFilter(t, "tag=<=cat", "( tag <= 'cat' )") + testIndividualFilter(t, "tag=>=>cat", "( tag >= '>cat' )") + testIndividualFilter(t, "tag=>==cat", "( tag >= '=cat' )") + testIndividualFilter(t, "tag=@@cat", "( tag %= '@cat' )") + testIndividualFilter(t, "tag=@cat", "( tag %= 'cat' )") + testIndividualFilter(t, "tag=!@cat", "( tag !% 'cat' )") + testIndividualFilter(t, "tag=:@cat", "( tag :% 'cat' )") + testIndividualFilter(t, "tag=!:@cat", "( tag ;% 'cat' )") + testIndividualFilter(t, "tag=^cat", "( tag ^= 'cat' )") + testIndividualFilter(t, "tag=!^cat", "( tag !^ 'cat' )") + testIndividualFilter(t, "tag=:^cat", "( tag :^ 'cat' )") + testIndividualFilter(t, "tag=!:^cat", "( tag ;^ 'cat' )") + testIndividualFilter(t, "tag=$cat", "( tag $= 'cat' )") + testIndividualFilter(t, "tag=!$cat", "( tag !$ 'cat' )") + testIndividualFilter(t, "tag=:$cat", "( tag :$ 'cat' )") + testIndividualFilter(t, "tag=!:$cat", "( tag ;$ 'cat' )") + testIndividualFilter(t, "tag==", "( tag == '' )") + testIndividualFilter(t, "tag=!=", "( tag != '' )") + testIndividualFilter(t, "tag=:!=", "( tag ;= '' )") + testIndividualFilter(t, "tag=?", "( tag == null )") + testIndividualFilter(t, "tag=!?", "( tag != null )") + testIndividualFilter(t, "tag=?=", "( tag == null )") + testIndividualFilter(t, "tag=!?=", "( tag != null )") + testIndividualFilter(t, "tag=?:!=", "( tag ;= null )") +} + +func testFailFilter(t *testing.T, queryString, errCode string) { + as := &apiServer{ + maxFilterLimit: 250, + } + req := httptest.NewRequest("GET", fmt.Sprintf("/things?%s", queryString), nil) + _, err := as.buildFilter(req, database.MessageQueryFactory) + assert.Regexp(t, errCode, err) +} + +func TestCheckNoMods(t *testing.T) { + testFailFilter(t, "tag=!>=test", "FF10302") + testFailFilter(t, "tag=:>test", "FF10302") + testFailFilter(t, "tag=!= ? AND mt.created <> ? AND mt.seq > ? AND mt.topics LIKE ? AND mt.topics NOT LIKE ? AND mt.topics ILIKE ? AND mt.topics NOT ILIKE ?) ORDER BY mt.seq DESC", sqlFilter) } +func TestSQLQueryFactoryEvenMoreOps(t *testing.T) { + + s, _ := newMockProvider().init() + fb := database.MessageQueryFactory.NewFilter(context.Background()) + u := fftypes.MustParseUUID("4066ABDC-8BBD-4472-9D29-1A55B467F9B9") + f := fb.And( + fb.IEq("id", u), + fb.NIeq("id", nil), + fb.StartsWith("topics", "abc"), + fb.NotStartsWith("topics", "def"), + fb.IStartsWith("topics", "ghi"), + fb.NotIStartsWith("topics", "jkl"), + fb.EndsWith("topics", "mno"), + fb.NotEndsWith("topics", "pqr"), + fb.IEndsWith("topics", "sty"), + fb.NotIEndsWith("topics", "vwx"), + ). + Descending() + + sel := squirrel.Select("*").From("mytable AS mt") + sel, _, _, err := s.filterSelect(context.Background(), "mt", sel, f, nil, []interface{}{"sequence"}) + assert.NoError(t, err) + + sqlFilter, _, err := sel.ToSql() + assert.NoError(t, err) + assert.Equal(t, "SELECT * FROM mytable AS mt WHERE (mt.id ILIKE ? AND mt.id NOT ILIKE ? AND mt.topics LIKE ? AND mt.topics NOT LIKE ? AND mt.topics ILIKE ? AND mt.topics NOT ILIKE ? AND mt.topics LIKE ? AND mt.topics NOT LIKE ? AND mt.topics ILIKE ? AND mt.topics NOT ILIKE ?) ORDER BY mt.seq DESC", sqlFilter) +} + func TestSQLQueryFactoryFinalizeFail(t *testing.T) { s, _ := newMockProvider().init() fb := database.MessageQueryFactory.NewFilter(context.Background()) @@ -169,3 +197,31 @@ func TestSQLQueryFactoryDefaultSortBadType(t *testing.T) { s.filterSelect(context.Background(), "", sel, f, nil, []interface{}{100}) }) } + +func TestILIKE(t *testing.T) { + s, _ := newMockProvider().init() + + s.features.UseILIKE = true + q := s.newILike("test", "value") + sqlString, _, _ := q.ToSql() + assert.Regexp(t, "ILIKE", sqlString) + + s.features.UseILIKE = false + q = s.newILike("test", "value") + sqlString, _, _ = q.ToSql() + assert.Regexp(t, "lower\\(test\\)", sqlString) +} + +func TestNotILIKE(t *testing.T) { + s, _ := newMockProvider().init() + + s.features.UseILIKE = true + q := s.newNotILike("test", "value") + sqlString, _, _ := q.ToSql() + assert.Regexp(t, "ILIKE", sqlString) + + s.features.UseILIKE = false + q = s.newNotILike("test", "value") + sqlString, _, _ = q.ToSql() + assert.Regexp(t, "lower\\(test\\)", sqlString) +} diff --git a/internal/database/sqlcommon/provider.go b/internal/database/sqlcommon/provider.go index 674cb0e0bc..ac6e14b05a 100644 --- a/internal/database/sqlcommon/provider.go +++ b/internal/database/sqlcommon/provider.go @@ -1,4 +1,4 @@ -// Copyright © 2021 Kaleido, Inc. +// Copyright © 2022 Kaleido, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -27,6 +27,18 @@ const ( sequenceColumn = "seq" ) +type SQLFeatures struct { + UseILIKE bool + PlaceholderFormat sq.PlaceholderFormat +} + +func DefaultSQLProviderFeatures() SQLFeatures { + return SQLFeatures{ + UseILIKE: false, + PlaceholderFormat: sq.Dollar, + } +} + // Provider defines the interface an individual provider muse implement to customize the SQLCommon implementation type Provider interface { @@ -42,8 +54,8 @@ type Provider interface { // GetDriver returns the driver implementation GetMigrationDriver(*sql.DB) (migratedb.Driver, error) - // PlaceholderFormat gets the Squirrel placeholder format - PlaceholderFormat() sq.PlaceholderFormat + // Features returns fields + Features() SQLFeatures // UpdateInsertForSequenceReturn updates the INSERT query for returning the Sequence, and returns whether it needs to be run as a query to return the Sequence field UpdateInsertForSequenceReturn(insert sq.InsertBuilder) (updatedInsert sq.InsertBuilder, runAsQuery bool) diff --git a/internal/database/sqlcommon/provider_mock_test.go b/internal/database/sqlcommon/provider_mock_test.go index 19479d1876..8c3ac4a9e3 100644 --- a/internal/database/sqlcommon/provider_mock_test.go +++ b/internal/database/sqlcommon/provider_mock_test.go @@ -67,8 +67,10 @@ func (mp *mockProvider) MigrationsDir() string { return mp.Name() } -func (mp *mockProvider) PlaceholderFormat() sq.PlaceholderFormat { - return sq.Dollar +func (psql *mockProvider) Features() SQLFeatures { + features := DefaultSQLProviderFeatures() + features.UseILIKE = true + return features } func (mp *mockProvider) UpdateInsertForSequenceReturn(insert sq.InsertBuilder) (sq.InsertBuilder, bool) { diff --git a/internal/database/sqlcommon/provider_sqlitego_test.go b/internal/database/sqlcommon/provider_sqlitego_test.go index 45685a40bb..f67cee4da9 100644 --- a/internal/database/sqlcommon/provider_sqlitego_test.go +++ b/internal/database/sqlcommon/provider_sqlitego_test.go @@ -78,8 +78,11 @@ func (tp *sqliteGoTestProvider) MigrationsDir() string { return "sqlite" } -func (tp *sqliteGoTestProvider) PlaceholderFormat() sq.PlaceholderFormat { - return sq.Dollar +func (psql *sqliteGoTestProvider) Features() SQLFeatures { + features := DefaultSQLProviderFeatures() + features.PlaceholderFormat = sq.Dollar + features.UseILIKE = false // Not supported + return features } func (tp *sqliteGoTestProvider) UpdateInsertForSequenceReturn(insert sq.InsertBuilder) (sq.InsertBuilder, bool) { diff --git a/internal/database/sqlcommon/sqlcommon.go b/internal/database/sqlcommon/sqlcommon.go index 8fb49db4a6..d0e85c3761 100644 --- a/internal/database/sqlcommon/sqlcommon.go +++ b/internal/database/sqlcommon/sqlcommon.go @@ -1,4 +1,4 @@ -// Copyright © 2021 Kaleido, Inc. +// Copyright © 2022 Kaleido, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -38,6 +38,7 @@ type SQLCommon struct { capabilities *database.Capabilities callbacks database.Callbacks provider Provider + features SQLFeatures } type txContextKey struct{} @@ -51,7 +52,10 @@ func (s *SQLCommon) Init(ctx context.Context, provider Provider, prefix config.P s.capabilities = capabilities s.callbacks = callbacks s.provider = provider - if s.provider == nil || s.provider.PlaceholderFormat() == nil || sequenceColumn == "" { + if s.provider != nil { + s.features = s.provider.Features() + } + if s.provider == nil || s.features.PlaceholderFormat == nil { log.L(ctx).Errorf("Invalid SQL options from provider '%T'", s.provider) return i18n.NewError(ctx, i18n.MsgDBInitFailed) } @@ -154,7 +158,7 @@ func (s *SQLCommon) queryTx(ctx context.Context, tx *txWrapper, q sq.SelectBuild } l := log.L(ctx) - sqlQuery, args, err := q.PlaceholderFormat(s.provider.PlaceholderFormat()).ToSql() + sqlQuery, args, err := q.PlaceholderFormat(s.features.PlaceholderFormat).ToSql() if err != nil { return nil, tx, i18n.WrapError(ctx, err, i18n.MsgDBQueryBuildFailed) } @@ -190,7 +194,7 @@ func (s *SQLCommon) countQuery(ctx context.Context, tx *txWrapper, tableName str countExpr = "*" } q := sq.Select(fmt.Sprintf("COUNT(%s)", countExpr)).From(tableName).Where(fop) - sqlQuery, args, err := q.PlaceholderFormat(s.provider.PlaceholderFormat()).ToSql() + sqlQuery, args, err := q.PlaceholderFormat(s.features.PlaceholderFormat).ToSql() if err != nil { return count, i18n.WrapError(ctx, err, i18n.MsgDBQueryBuildFailed) } @@ -233,7 +237,7 @@ func (s *SQLCommon) insertTx(ctx context.Context, tx *txWrapper, q sq.InsertBuil l := log.L(ctx) q, useQuery := s.provider.UpdateInsertForSequenceReturn(q) - sqlQuery, args, err := q.PlaceholderFormat(s.provider.PlaceholderFormat()).ToSql() + sqlQuery, args, err := q.PlaceholderFormat(s.features.PlaceholderFormat).ToSql() if err != nil { return -1, i18n.WrapError(ctx, err, i18n.MsgDBQueryBuildFailed) } @@ -264,7 +268,7 @@ func (s *SQLCommon) insertTx(ctx context.Context, tx *txWrapper, q sq.InsertBuil func (s *SQLCommon) deleteTx(ctx context.Context, tx *txWrapper, q sq.DeleteBuilder, postCommit func()) error { l := log.L(ctx) - sqlQuery, args, err := q.PlaceholderFormat(s.provider.PlaceholderFormat()).ToSql() + sqlQuery, args, err := q.PlaceholderFormat(s.features.PlaceholderFormat).ToSql() if err != nil { return i18n.WrapError(ctx, err, i18n.MsgDBQueryBuildFailed) } @@ -289,7 +293,7 @@ func (s *SQLCommon) deleteTx(ctx context.Context, tx *txWrapper, q sq.DeleteBuil func (s *SQLCommon) updateTx(ctx context.Context, tx *txWrapper, q sq.UpdateBuilder, postCommit func()) (int64, error) { l := log.L(ctx) - sqlQuery, args, err := q.PlaceholderFormat(s.provider.PlaceholderFormat()).ToSql() + sqlQuery, args, err := q.PlaceholderFormat(s.features.PlaceholderFormat).ToSql() if err != nil { return -1, i18n.WrapError(ctx, err, i18n.MsgDBQueryBuildFailed) } diff --git a/internal/database/sqlite3/sqlite3.go b/internal/database/sqlite3/sqlite3.go index dd21962135..90e4cf3463 100644 --- a/internal/database/sqlite3/sqlite3.go +++ b/internal/database/sqlite3/sqlite3.go @@ -1,4 +1,4 @@ -// Copyright © 2021 Kaleido, Inc. +// Copyright © 2022 Kaleido, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -32,15 +32,29 @@ import ( "github.com/hyperledger/firefly/pkg/database" // Import the derivation of SQLite3 CGO suported by golang-migrate - _ "github.com/mattn/go-sqlite3" + "github.com/mattn/go-sqlite3" ) +var ffSQLiteRegistered = false + type SQLite3 struct { sqlcommon.SQLCommon } +func connHook(conn *sqlite3.SQLiteConn) error { + _, err := conn.Exec("PRAGMA case_sensitive_like=ON;", nil) + return err +} + func (sqlite *SQLite3) Init(ctx context.Context, prefix config.Prefix, callbacks database.Callbacks) error { capabilities := &database.Capabilities{} + if !ffSQLiteRegistered { + sql.Register("sqlite3_ff", + &sqlite3.SQLiteDriver{ + ConnectHook: connHook, + }) + ffSQLiteRegistered = true + } return sqlite.SQLCommon.Init(ctx, sqlite, prefix, callbacks, capabilities) } @@ -52,8 +66,11 @@ func (sqlite *SQLite3) MigrationsDir() string { return "sqlite" } -func (sqlite *SQLite3) PlaceholderFormat() sq.PlaceholderFormat { - return sq.Dollar +func (sqlite *SQLite3) Features() sqlcommon.SQLFeatures { + features := sqlcommon.DefaultSQLProviderFeatures() + features.PlaceholderFormat = sq.Dollar + features.UseILIKE = false // Not supported + return features } func (sqlite *SQLite3) UpdateInsertForSequenceReturn(insert sq.InsertBuilder) (sq.InsertBuilder, bool) { @@ -61,7 +78,7 @@ func (sqlite *SQLite3) UpdateInsertForSequenceReturn(insert sq.InsertBuilder) (s } func (sqlite *SQLite3) Open(url string) (*sql.DB, error) { - return sql.Open("sqlite3", url) + return sql.Open("sqlite3_ff", url) } func (sqlite *SQLite3) GetMigrationDriver(db *sql.DB) (migratedb.Driver, error) { diff --git a/internal/database/sqlite3/sqlite3_test.go b/internal/database/sqlite3/sqlite3_test.go index a5390829f6..26d10ae5fe 100644 --- a/internal/database/sqlite3/sqlite3_test.go +++ b/internal/database/sqlite3/sqlite3_test.go @@ -40,8 +40,14 @@ func TestSQLite3GoProvider(t *testing.T) { _, err = sqlite.GetMigrationDriver(sqlite.DB()) assert.Error(t, err) + db, err := sqlite.Open("file::memory:") + assert.NoError(t, err) + conn, err := db.Conn(context.Background()) + assert.NoError(t, err) + conn.Close() + assert.Equal(t, "sqlite3", sqlite.Name()) - assert.Equal(t, sq.Dollar, sqlite.PlaceholderFormat()) + assert.Equal(t, sq.Dollar, sqlite.Features().PlaceholderFormat) insert := sq.Insert("test").Columns("col1").Values("val1") insert, query := sqlite.UpdateInsertForSequenceReturn(insert) diff --git a/internal/dataexchange/dxhttps/dxhttps.go b/internal/dataexchange/dxhttps/dxhttps.go index 352e792694..cc9812dc71 100644 --- a/internal/dataexchange/dxhttps/dxhttps.go +++ b/internal/dataexchange/dxhttps/dxhttps.go @@ -258,7 +258,7 @@ func (h *HTTPS) eventLoop() { case messageDelivered: err = h.callbacks.TransferResult(msg.RequestID, fftypes.OpStatusSucceeded, "", nil) case messageReceived: - err = h.callbacks.MessageReceived(msg.Sender, fftypes.Byteable(msg.Message)) + err = h.callbacks.MessageReceived(msg.Sender, []byte(msg.Message)) case blobFailed: err = h.callbacks.TransferResult(msg.RequestID, fftypes.OpStatusFailed, msg.Error, nil) case blobDelivered: diff --git a/internal/definitions/definition_handler.go b/internal/definitions/definition_handler.go index d13f75e86e..d2029fdba4 100644 --- a/internal/definitions/definition_handler.go +++ b/internal/definitions/definition_handler.go @@ -1,4 +1,4 @@ -// Copyright © 2021 Kaleido, Inc. +// Copyright © 2022 Kaleido, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -122,7 +122,7 @@ func (dh *definitionHandlers) getSystemBroadcastPayload(ctx context.Context, msg l.Warnf("Unable to process system broadcast %s - expecting 1 attachement, found %d", msg.Header.ID, len(data)) return false } - err := json.Unmarshal(data[0].Value, &res) + err := json.Unmarshal(data[0].Value.Bytes(), &res) if err != nil { l.Warnf("Unable to process system broadcast %s - unmarshal failed: %s", msg.Header.ID, err) return false diff --git a/internal/definitions/definition_handler_datatype_test.go b/internal/definitions/definition_handler_datatype_test.go index 2a53bf737f..5933e1dabc 100644 --- a/internal/definitions/definition_handler_datatype_test.go +++ b/internal/definitions/definition_handler_datatype_test.go @@ -38,13 +38,13 @@ func TestHandleDefinitionBroadcastDatatypeOk(t *testing.T) { Namespace: "ns1", Name: "name1", Version: "ver1", - Value: fftypes.Byteable(`{}`), + Value: fftypes.JSONAnyPtr(`{}`), } dt.Hash = dt.Value.Hash() b, err := json.Marshal(&dt) assert.NoError(t, err) data := &fftypes.Data{ - Value: fftypes.Byteable(b), + Value: fftypes.JSONAnyPtrBytes(b), } mdm := dh.data.(*datamocks.Manager) @@ -74,13 +74,13 @@ func TestHandleDefinitionBroadcastDatatypeEventFail(t *testing.T) { Namespace: "ns1", Name: "name1", Version: "ver1", - Value: fftypes.Byteable(`{}`), + Value: fftypes.JSONAnyPtr(`{}`), } dt.Hash = dt.Value.Hash() b, err := json.Marshal(&dt) assert.NoError(t, err) data := &fftypes.Data{ - Value: fftypes.Byteable(b), + Value: fftypes.JSONAnyPtrBytes(b), } mdm := dh.data.(*datamocks.Manager) @@ -109,13 +109,13 @@ func TestHandleDefinitionBroadcastDatatypeMissingID(t *testing.T) { Namespace: "ns1", Name: "name1", Version: "ver1", - Value: fftypes.Byteable(`{}`), + Value: fftypes.JSONAnyPtr(`{}`), } dt.Hash = dt.Value.Hash() b, err := json.Marshal(&dt) assert.NoError(t, err) data := &fftypes.Data{ - Value: fftypes.Byteable(b), + Value: fftypes.JSONAnyPtrBytes(b), } action, err := dh.HandleSystemBroadcast(context.Background(), &fftypes.Message{ @@ -136,13 +136,13 @@ func TestHandleDefinitionBroadcastBadSchema(t *testing.T) { Namespace: "ns1", Name: "name1", Version: "ver1", - Value: fftypes.Byteable(`{}`), + Value: fftypes.JSONAnyPtr(`{}`), } dt.Hash = dt.Value.Hash() b, err := json.Marshal(&dt) assert.NoError(t, err) data := &fftypes.Data{ - Value: fftypes.Byteable(b), + Value: fftypes.JSONAnyPtrBytes(b), } mdm := dh.data.(*datamocks.Manager) @@ -167,7 +167,7 @@ func TestHandleDefinitionBroadcastMissingData(t *testing.T) { Namespace: "ns1", Name: "name1", Version: "ver1", - Value: fftypes.Byteable(`{}`), + Value: fftypes.JSONAnyPtr(`{}`), } dt.Hash = dt.Value.Hash() @@ -189,13 +189,13 @@ func TestHandleDefinitionBroadcastDatatypeLookupFail(t *testing.T) { Namespace: "ns1", Name: "name1", Version: "ver1", - Value: fftypes.Byteable(`{}`), + Value: fftypes.JSONAnyPtr(`{}`), } dt.Hash = dt.Value.Hash() b, err := json.Marshal(&dt) assert.NoError(t, err) data := &fftypes.Data{ - Value: fftypes.Byteable(b), + Value: fftypes.JSONAnyPtrBytes(b), } mdm := dh.data.(*datamocks.Manager) @@ -224,13 +224,13 @@ func TestHandleDefinitionBroadcastUpsertFail(t *testing.T) { Namespace: "ns1", Name: "name1", Version: "ver1", - Value: fftypes.Byteable(`{}`), + Value: fftypes.JSONAnyPtr(`{}`), } dt.Hash = dt.Value.Hash() b, err := json.Marshal(&dt) assert.NoError(t, err) data := &fftypes.Data{ - Value: fftypes.Byteable(b), + Value: fftypes.JSONAnyPtrBytes(b), } mdm := dh.data.(*datamocks.Manager) @@ -259,13 +259,13 @@ func TestHandleDefinitionBroadcastDatatypeDuplicate(t *testing.T) { Namespace: "ns1", Name: "name1", Version: "ver1", - Value: fftypes.Byteable(`{}`), + Value: fftypes.JSONAnyPtr(`{}`), } dt.Hash = dt.Value.Hash() b, err := json.Marshal(&dt) assert.NoError(t, err) data := &fftypes.Data{ - Value: fftypes.Byteable(b), + Value: fftypes.JSONAnyPtrBytes(b), } mdm := dh.data.(*datamocks.Manager) diff --git a/internal/definitions/definition_handler_namespace_test.go b/internal/definitions/definition_handler_namespace_test.go index 87c306b206..a40bee861f 100644 --- a/internal/definitions/definition_handler_namespace_test.go +++ b/internal/definitions/definition_handler_namespace_test.go @@ -38,7 +38,7 @@ func TestHandleDefinitionBroadcastNSOk(t *testing.T) { b, err := json.Marshal(&ns) assert.NoError(t, err) data := &fftypes.Data{ - Value: fftypes.Byteable(b), + Value: fftypes.JSONAnyPtrBytes(b), } mdi := dh.database.(*databasemocks.Plugin) @@ -66,7 +66,7 @@ func TestHandleDefinitionBroadcastNSEventFail(t *testing.T) { b, err := json.Marshal(&ns) assert.NoError(t, err) data := &fftypes.Data{ - Value: fftypes.Byteable(b), + Value: fftypes.JSONAnyPtrBytes(b), } mdi := dh.database.(*databasemocks.Plugin) @@ -94,7 +94,7 @@ func TestHandleDefinitionBroadcastNSUpsertFail(t *testing.T) { b, err := json.Marshal(&ns) assert.NoError(t, err) data := &fftypes.Data{ - Value: fftypes.Byteable(b), + Value: fftypes.JSONAnyPtrBytes(b), } mdi := dh.database.(*databasemocks.Plugin) @@ -130,7 +130,7 @@ func TestHandleDefinitionBroadcastNSBadID(t *testing.T) { b, err := json.Marshal(&ns) assert.NoError(t, err) data := &fftypes.Data{ - Value: fftypes.Byteable(b), + Value: fftypes.JSONAnyPtrBytes(b), } action, err := dh.HandleSystemBroadcast(context.Background(), &fftypes.Message{ @@ -146,7 +146,7 @@ func TestHandleDefinitionBroadcastNSBadData(t *testing.T) { dh := newTestDefinitionHandlers(t) data := &fftypes.Data{ - Value: fftypes.Byteable(`!{json`), + Value: fftypes.JSONAnyPtr(`!{json`), } action, err := dh.HandleSystemBroadcast(context.Background(), &fftypes.Message{ @@ -168,7 +168,7 @@ func TestHandleDefinitionBroadcastDuplicate(t *testing.T) { b, err := json.Marshal(&ns) assert.NoError(t, err) data := &fftypes.Data{ - Value: fftypes.Byteable(b), + Value: fftypes.JSONAnyPtrBytes(b), } mdi := dh.database.(*databasemocks.Plugin) @@ -195,7 +195,7 @@ func TestHandleDefinitionBroadcastDuplicateOverrideLocal(t *testing.T) { b, err := json.Marshal(&ns) assert.NoError(t, err) data := &fftypes.Data{ - Value: fftypes.Byteable(b), + Value: fftypes.JSONAnyPtrBytes(b), } mdi := dh.database.(*databasemocks.Plugin) @@ -225,7 +225,7 @@ func TestHandleDefinitionBroadcastDuplicateOverrideLocalFail(t *testing.T) { b, err := json.Marshal(&ns) assert.NoError(t, err) data := &fftypes.Data{ - Value: fftypes.Byteable(b), + Value: fftypes.JSONAnyPtrBytes(b), } mdi := dh.database.(*databasemocks.Plugin) @@ -252,7 +252,7 @@ func TestHandleDefinitionBroadcastDupCheckFail(t *testing.T) { b, err := json.Marshal(&ns) assert.NoError(t, err) data := &fftypes.Data{ - Value: fftypes.Byteable(b), + Value: fftypes.JSONAnyPtrBytes(b), } mdi := dh.database.(*databasemocks.Plugin) diff --git a/internal/definitions/definition_handler_network_node_test.go b/internal/definitions/definition_handler_network_node_test.go index d670c235ab..5c6b98efb2 100644 --- a/internal/definitions/definition_handler_network_node_test.go +++ b/internal/definitions/definition_handler_network_node_test.go @@ -45,7 +45,7 @@ func TestHandleDefinitionBroadcastNodeOk(t *testing.T) { b, err := json.Marshal(&node) assert.NoError(t, err) data := &fftypes.Data{ - Value: fftypes.Byteable(b), + Value: fftypes.JSONAnyPtrBytes(b), } mdi := dh.database.(*databasemocks.Plugin) @@ -87,7 +87,7 @@ func TestHandleDefinitionBroadcastNodeUpsertFail(t *testing.T) { b, err := json.Marshal(&node) assert.NoError(t, err) data := &fftypes.Data{ - Value: fftypes.Byteable(b), + Value: fftypes.JSONAnyPtrBytes(b), } mdi := dh.database.(*databasemocks.Plugin) @@ -127,7 +127,7 @@ func TestHandleDefinitionBroadcastNodeAddPeerFail(t *testing.T) { b, err := json.Marshal(&node) assert.NoError(t, err) data := &fftypes.Data{ - Value: fftypes.Byteable(b), + Value: fftypes.JSONAnyPtrBytes(b), } mdi := dh.database.(*databasemocks.Plugin) @@ -169,7 +169,7 @@ func TestHandleDefinitionBroadcastNodeDupMismatch(t *testing.T) { b, err := json.Marshal(&node) assert.NoError(t, err) data := &fftypes.Data{ - Value: fftypes.Byteable(b), + Value: fftypes.JSONAnyPtrBytes(b), } mdi := dh.database.(*databasemocks.Plugin) @@ -207,7 +207,7 @@ func TestHandleDefinitionBroadcastNodeDupOK(t *testing.T) { b, err := json.Marshal(&node) assert.NoError(t, err) data := &fftypes.Data{ - Value: fftypes.Byteable(b), + Value: fftypes.JSONAnyPtrBytes(b), } mdi := dh.database.(*databasemocks.Plugin) @@ -248,7 +248,7 @@ func TestHandleDefinitionBroadcastNodeGetFail(t *testing.T) { b, err := json.Marshal(&node) assert.NoError(t, err) data := &fftypes.Data{ - Value: fftypes.Byteable(b), + Value: fftypes.JSONAnyPtrBytes(b), } mdi := dh.database.(*databasemocks.Plugin) @@ -286,7 +286,7 @@ func TestHandleDefinitionBroadcastNodeBadAuthor(t *testing.T) { b, err := json.Marshal(&node) assert.NoError(t, err) data := &fftypes.Data{ - Value: fftypes.Byteable(b), + Value: fftypes.JSONAnyPtrBytes(b), } mdi := dh.database.(*databasemocks.Plugin) @@ -323,7 +323,7 @@ func TestHandleDefinitionBroadcastNodeGetOrgNotFound(t *testing.T) { b, err := json.Marshal(&node) assert.NoError(t, err) data := &fftypes.Data{ - Value: fftypes.Byteable(b), + Value: fftypes.JSONAnyPtrBytes(b), } mdi := dh.database.(*databasemocks.Plugin) @@ -360,7 +360,7 @@ func TestHandleDefinitionBroadcastNodeGetOrgFail(t *testing.T) { b, err := json.Marshal(&node) assert.NoError(t, err) data := &fftypes.Data{ - Value: fftypes.Byteable(b), + Value: fftypes.JSONAnyPtrBytes(b), } mdi := dh.database.(*databasemocks.Plugin) @@ -397,7 +397,7 @@ func TestHandleDefinitionBroadcastNodeValidateFail(t *testing.T) { b, err := json.Marshal(&node) assert.NoError(t, err) data := &fftypes.Data{ - Value: fftypes.Byteable(b), + Value: fftypes.JSONAnyPtrBytes(b), } action, err := dh.HandleSystemBroadcast(context.Background(), &fftypes.Message{ @@ -418,7 +418,7 @@ func TestHandleDefinitionBroadcastNodeUnmarshalFail(t *testing.T) { dh := newTestDefinitionHandlers(t) data := &fftypes.Data{ - Value: fftypes.Byteable(`!json`), + Value: fftypes.JSONAnyPtr(`!json`), } action, err := dh.HandleSystemBroadcast(context.Background(), &fftypes.Message{ diff --git a/internal/definitions/definition_handler_network_org_test.go b/internal/definitions/definition_handler_network_org_test.go index 18b7379baa..12b665129d 100644 --- a/internal/definitions/definition_handler_network_org_test.go +++ b/internal/definitions/definition_handler_network_org_test.go @@ -49,7 +49,7 @@ func TestHandleDefinitionBroadcastChildOrgOk(t *testing.T) { b, err := json.Marshal(&org) assert.NoError(t, err) data := &fftypes.Data{ - Value: fftypes.Byteable(b), + Value: fftypes.JSONAnyPtrBytes(b), } mdi := dh.database.(*databasemocks.Plugin) @@ -95,7 +95,7 @@ func TestHandleDefinitionBroadcastChildOrgDupOk(t *testing.T) { b, err := json.Marshal(&org) assert.NoError(t, err) data := &fftypes.Data{ - Value: fftypes.Byteable(b), + Value: fftypes.JSONAnyPtrBytes(b), } mdi := dh.database.(*databasemocks.Plugin) @@ -139,7 +139,7 @@ func TestHandleDefinitionBroadcastChildOrgBadKey(t *testing.T) { b, err := json.Marshal(&org) assert.NoError(t, err) data := &fftypes.Data{ - Value: fftypes.Byteable(b), + Value: fftypes.JSONAnyPtrBytes(b), } mdi := dh.database.(*databasemocks.Plugin) @@ -174,7 +174,7 @@ func TestHandleDefinitionBroadcastOrgDupMismatch(t *testing.T) { b, err := json.Marshal(&org) assert.NoError(t, err) data := &fftypes.Data{ - Value: fftypes.Byteable(b), + Value: fftypes.JSONAnyPtrBytes(b), } mdi := dh.database.(*databasemocks.Plugin) @@ -208,7 +208,7 @@ func TestHandleDefinitionBroadcastOrgUpsertFail(t *testing.T) { b, err := json.Marshal(&org) assert.NoError(t, err) data := &fftypes.Data{ - Value: fftypes.Byteable(b), + Value: fftypes.JSONAnyPtrBytes(b), } mdi := dh.database.(*databasemocks.Plugin) @@ -244,7 +244,7 @@ func TestHandleDefinitionBroadcastOrgGetOrgFail(t *testing.T) { b, err := json.Marshal(&org) assert.NoError(t, err) data := &fftypes.Data{ - Value: fftypes.Byteable(b), + Value: fftypes.JSONAnyPtrBytes(b), } mdi := dh.database.(*databasemocks.Plugin) @@ -277,7 +277,7 @@ func TestHandleDefinitionBroadcastOrgAuthorMismatch(t *testing.T) { b, err := json.Marshal(&org) assert.NoError(t, err) data := &fftypes.Data{ - Value: fftypes.Byteable(b), + Value: fftypes.JSONAnyPtrBytes(b), } mdi := dh.database.(*databasemocks.Plugin) @@ -312,7 +312,7 @@ func TestHandleDefinitionBroadcastGetParentFail(t *testing.T) { b, err := json.Marshal(&org) assert.NoError(t, err) data := &fftypes.Data{ - Value: fftypes.Byteable(b), + Value: fftypes.JSONAnyPtrBytes(b), } mdi := dh.database.(*databasemocks.Plugin) @@ -347,7 +347,7 @@ func TestHandleDefinitionBroadcastGetParentNotFound(t *testing.T) { b, err := json.Marshal(&org) assert.NoError(t, err) data := &fftypes.Data{ - Value: fftypes.Byteable(b), + Value: fftypes.JSONAnyPtrBytes(b), } mdi := dh.database.(*databasemocks.Plugin) @@ -380,7 +380,7 @@ func TestHandleDefinitionBroadcastValidateFail(t *testing.T) { b, err := json.Marshal(&org) assert.NoError(t, err) data := &fftypes.Data{ - Value: fftypes.Byteable(b), + Value: fftypes.JSONAnyPtrBytes(b), } action, err := dh.HandleSystemBroadcast(context.Background(), &fftypes.Message{ @@ -401,7 +401,7 @@ func TestHandleDefinitionBroadcastUnmarshalFail(t *testing.T) { dh := newTestDefinitionHandlers(t) data := &fftypes.Data{ - Value: fftypes.Byteable(`!json`), + Value: fftypes.JSONAnyPtr(`!json`), } action, err := dh.HandleSystemBroadcast(context.Background(), &fftypes.Message{ diff --git a/internal/definitions/definition_handler_tokenpool_test.go b/internal/definitions/definition_handler_tokenpool_test.go index 449a851846..d0203c7f42 100644 --- a/internal/definitions/definition_handler_tokenpool_test.go +++ b/internal/definitions/definition_handler_tokenpool_test.go @@ -61,7 +61,7 @@ func buildPoolDefinitionMessage(announce *fftypes.TokenPoolAnnouncement) (*fftyp return nil, nil, err } data := []*fftypes.Data{{ - Value: fftypes.Byteable(b), + Value: fftypes.JSONAnyPtrBytes(b), }} return msg, data, nil } diff --git a/internal/events/batch_pin_complete_test.go b/internal/events/batch_pin_complete_test.go index c9a4e46764..fbcec13f5a 100644 --- a/internal/events/batch_pin_complete_test.go +++ b/internal/events/batch_pin_complete_test.go @@ -397,7 +397,7 @@ func TestPersistBatchGoodDataUpsertOptimizeExistingFail(t *testing.T) { ID: fftypes.NewUUID(), }, Data: []*fftypes.Data{ - {ID: fftypes.NewUUID(), Value: fftypes.Byteable(`"test"`)}, + {ID: fftypes.NewUUID(), Value: fftypes.JSONAnyPtr(`"test"`)}, }, }, } @@ -430,7 +430,7 @@ func TestPersistBatchGoodDataUpsertOptimizeNewFail(t *testing.T) { ID: fftypes.NewUUID(), }, Data: []*fftypes.Data{ - {ID: fftypes.NewUUID(), Value: fftypes.Byteable(`"test"`)}, + {ID: fftypes.NewUUID(), Value: fftypes.JSONAnyPtr(`"test"`)}, }, }, } @@ -544,7 +544,7 @@ func TestPersistBatchDataBadHash(t *testing.T) { } data := &fftypes.Data{ ID: fftypes.NewUUID(), - Value: fftypes.Byteable(`"test"`), + Value: fftypes.JSONAnyPtr(`"test"`), Hash: fftypes.NewRandB32(), } err := em.persistBatchData(context.Background(), batch, 0, data, database.UpsertOptimizationSkip) @@ -558,7 +558,7 @@ func TestPersistBatchDataUpsertHashMismatch(t *testing.T) { ID: fftypes.NewUUID(), } - data := &fftypes.Data{ID: fftypes.NewUUID(), Value: fftypes.Byteable(`"test"`)} + data := &fftypes.Data{ID: fftypes.NewUUID(), Value: fftypes.JSONAnyPtr(`"test"`)} data.Hash = data.Value.Hash() mdi := em.database.(*databasemocks.Plugin) @@ -576,7 +576,7 @@ func TestPersistBatchDataUpsertDataError(t *testing.T) { ID: fftypes.NewUUID(), } - data := &fftypes.Data{ID: fftypes.NewUUID(), Value: fftypes.Byteable(`"test"`)} + data := &fftypes.Data{ID: fftypes.NewUUID(), Value: fftypes.JSONAnyPtr(`"test"`)} data.Hash = data.Value.Hash() mdi := em.database.(*databasemocks.Plugin) @@ -593,7 +593,7 @@ func TestPersistBatchDataOk(t *testing.T) { ID: fftypes.NewUUID(), } - data := &fftypes.Data{ID: fftypes.NewUUID(), Value: fftypes.Byteable(`"test"`)} + data := &fftypes.Data{ID: fftypes.NewUUID(), Value: fftypes.JSONAnyPtr(`"test"`)} data.Hash = data.Value.Hash() mdi := em.database.(*databasemocks.Plugin) diff --git a/internal/events/dx_callbacks_test.go b/internal/events/dx_callbacks_test.go index df0f5b22e8..cb1497ee92 100644 --- a/internal/events/dx_callbacks_test.go +++ b/internal/events/dx_callbacks_test.go @@ -707,7 +707,7 @@ func TestMessageReceiveMessagePersistDataFail(t *testing.T) { } data := &fftypes.Data{ ID: fftypes.NewUUID(), - Value: fftypes.Byteable(`{}`), + Value: fftypes.JSONAnyPtr(`{}`), } err := msg.Seal(em.ctx) assert.NoError(t, err) @@ -757,7 +757,7 @@ func TestMessageReceiveMessagePersistEventFail(t *testing.T) { } data := &fftypes.Data{ ID: fftypes.NewUUID(), - Value: fftypes.Byteable(`{}`), + Value: fftypes.JSONAnyPtr(`{}`), } err := msg.Seal(em.ctx) assert.NoError(t, err) @@ -809,7 +809,7 @@ func TestMessageReceiveMessageEnsureLocalGroupFail(t *testing.T) { } data := &fftypes.Data{ ID: fftypes.NewUUID(), - Value: fftypes.Byteable(`{}`), + Value: fftypes.JSONAnyPtr(`{}`), } err := msg.Seal(em.ctx) assert.NoError(t, err) @@ -851,7 +851,7 @@ func TestMessageReceiveMessageEnsureLocalGroupReject(t *testing.T) { } data := &fftypes.Data{ ID: fftypes.NewUUID(), - Value: fftypes.Byteable(`{}`), + Value: fftypes.JSONAnyPtr(`{}`), } err := msg.Seal(em.ctx) assert.NoError(t, err) diff --git a/internal/events/event_dispatcher_test.go b/internal/events/event_dispatcher_test.go index 079a1a9dd5..3feb780b14 100644 --- a/internal/events/event_dispatcher_test.go +++ b/internal/events/event_dispatcher_test.go @@ -884,7 +884,7 @@ func TestEventDispatcherWithReply(t *testing.T) { }, }, InlineData: fftypes.InlineData{ - {Value: fftypes.Byteable(`"my reply"`)}, + {Value: fftypes.JSONAnyPtr(`"my reply"`)}, }, }, }) @@ -900,7 +900,7 @@ func TestEventDispatcherWithReply(t *testing.T) { }, }, InlineData: fftypes.InlineData{ - {Value: fftypes.Byteable(`"my reply"`)}, + {Value: fftypes.JSONAnyPtr(`"my reply"`)}, }, }, }) diff --git a/internal/events/persist_batch.go b/internal/events/persist_batch.go index 95ac9929a6..54bfd98686 100644 --- a/internal/events/persist_batch.go +++ b/internal/events/persist_batch.go @@ -1,4 +1,4 @@ -// Copyright © 2021 Kaleido, Inc. +// Copyright © 2022 Kaleido, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -73,7 +73,7 @@ func (em *eventManager) isRootOrgBroadcast(batch *fftypes.Batch) bool { if batchDataItem.ID.Equals(messageDataItem.ID) { if batchDataItem.Validator == fftypes.MessageTypeDefinition { var org *fftypes.Organization - err := json.Unmarshal(batchDataItem.Value, &org) + err := json.Unmarshal(batchDataItem.Value.Bytes(), &org) if err != nil { return false } diff --git a/internal/events/persist_batch_test.go b/internal/events/persist_batch_test.go index ba5af86050..65b439b433 100644 --- a/internal/events/persist_batch_test.go +++ b/internal/events/persist_batch_test.go @@ -48,7 +48,7 @@ func TestPersistBatchFromBroadcastRootOrg(t *testing.T) { assert.NoError(t, err) data := &fftypes.Data{ ID: fftypes.NewUUID(), - Value: orgBytes, + Value: fftypes.JSONAnyPtrBytes(orgBytes), Validator: fftypes.MessageTypeDefinition, } @@ -103,7 +103,7 @@ func TestPersistBatchFromBroadcastRootOrgBadData(t *testing.T) { data := &fftypes.Data{ ID: fftypes.NewUUID(), - Value: []byte("!badness"), + Value: fftypes.JSONAnyPtr("!badness"), Validator: fftypes.MessageTypeDefinition, } diff --git a/internal/events/webhooks/webhooks.go b/internal/events/webhooks/webhooks.go index 863fd6893d..d952207186 100644 --- a/internal/events/webhooks/webhooks.go +++ b/internal/events/webhooks/webhooks.go @@ -1,4 +1,4 @@ -// Copyright © 2021 Kaleido, Inc. +// Copyright © 2022 Kaleido, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -56,7 +56,7 @@ type whRequest struct { type whResponse struct { Status int `json:"status"` Headers fftypes.JSONObject `json:"headers"` - Body fftypes.Byteable `json:"body"` + Body *fftypes.JSONAny `json:"body"` } func (wh *WebHooks) Name() string { return "webhooks" } @@ -263,7 +263,7 @@ func (wh *WebHooks) ValidateOptions(options *fftypes.SubscriptionOptions) error func (wh *WebHooks) attemptRequest(sub *fftypes.Subscription, event *fftypes.EventDelivery, data []*fftypes.Data) (req *whRequest, res *whResponse, err error) { withData := sub.Options.WithData != nil && *sub.Options.WithData - allData := make([]fftypes.Byteable, 0, len(data)) + allData := make([]*fftypes.JSONAny, 0, len(data)) var firstData fftypes.JSONObject var valid bool if withData { @@ -334,7 +334,8 @@ func (wh *WebHooks) attemptRequest(sub *fftypes.Subscription, event *fftypes.Eve if err != nil { return nil, nil, i18n.WrapError(wh.ctx, err, i18n.MsgWebhooksReplyBadJSON) } - res.Body, _ = json.Marshal(&resData) // we know we can re-marshal it + b, _ := json.Marshal(&resData) // we know we can re-marshal It + res.Body = fftypes.JSONAnyPtrBytes(b) } else { // Anything other than JSON, gets returned as a JSON string in base64 encoding buf := &bytes.Buffer{} @@ -343,7 +344,7 @@ func (wh *WebHooks) attemptRequest(sub *fftypes.Subscription, event *fftypes.Eve _, _ = io.Copy(b64Encoder, resp.RawBody()) _ = b64Encoder.Close() buf.WriteByte('"') - res.Body = buf.Bytes() + res.Body = fftypes.JSONAnyPtrBytes(buf.Bytes()) } return req, res, nil @@ -363,7 +364,7 @@ func (wh *WebHooks) doDelivery(connID string, reply bool, sub *fftypes.Subscript Headers: fftypes.JSONObject{ "Content-Type": "application/json", }, - Body: b, + Body: fftypes.JSONAnyPtrBytes(b), } } b, _ := json.Marshal(&res) @@ -390,7 +391,7 @@ func (wh *WebHooks) doDelivery(connID string, reply bool, sub *fftypes.Subscript }, }, InlineData: fftypes.InlineData{ - {Value: b}, + {Value: fftypes.JSONAnyPtrBytes(b)}, }, }, }) diff --git a/internal/events/webhooks/webhooks_test.go b/internal/events/webhooks/webhooks_test.go index 075d4ebb1b..6c65bc3f56 100644 --- a/internal/events/webhooks/webhooks_test.go +++ b/internal/events/webhooks/webhooks_test.go @@ -189,7 +189,7 @@ func TestRequestWithBodyReplyEndToEnd(t *testing.T) { } data := &fftypes.Data{ ID: dataID, - Value: fftypes.Byteable(`{ + Value: fftypes.JSONAnyPtr(`{ "in_body": { "inputfield": "inputvalue" }, @@ -294,7 +294,7 @@ func TestRequestWithEmptyStringBodyReplyEndToEnd(t *testing.T) { } data := &fftypes.Data{ ID: dataID, - Value: fftypes.Byteable(`{ + Value: fftypes.JSONAnyPtr(`{ "in_body": { "inputfield": "" }, @@ -371,7 +371,7 @@ func TestRequestNoBodyNoReply(t *testing.T) { } data := &fftypes.Data{ ID: dataID, - Value: fftypes.Byteable(`{ + Value: fftypes.JSONAnyPtr(`{ "inputfield": "inputvalue" }`), } @@ -541,8 +541,8 @@ func TestRequestReplyDataArrayBadStatusB64(t *testing.T) { })).Return(nil) err := wh.DeliveryRequest(mock.Anything, sub, event, []*fftypes.Data{ - {ID: fftypes.NewUUID(), Value: fftypes.Byteable(`"value1"`)}, - {ID: fftypes.NewUUID(), Value: fftypes.Byteable(`"value2"`)}, + {ID: fftypes.NewUUID(), Value: fftypes.JSONAnyPtr(`"value1"`)}, + {ID: fftypes.NewUUID(), Value: fftypes.JSONAnyPtr(`"value2"`)}, }) assert.NoError(t, err) assert.True(t, called) @@ -589,8 +589,8 @@ func TestRequestReplyDataArrayError(t *testing.T) { })).Return(nil) err := wh.DeliveryRequest(mock.Anything, sub, event, []*fftypes.Data{ - {ID: fftypes.NewUUID(), Value: fftypes.Byteable(`"value1"`)}, - {ID: fftypes.NewUUID(), Value: fftypes.Byteable(`"value2"`)}, + {ID: fftypes.NewUUID(), Value: fftypes.JSONAnyPtr(`"value1"`)}, + {ID: fftypes.NewUUID(), Value: fftypes.JSONAnyPtr(`"value2"`)}, }) assert.NoError(t, err) @@ -639,8 +639,8 @@ func TestRequestReplyBuildRequestFailFastAsk(t *testing.T) { } err := wh.DeliveryRequest(mock.Anything, sub, event, []*fftypes.Data{ - {ID: fftypes.NewUUID(), Value: fftypes.Byteable(`"value1"`)}, - {ID: fftypes.NewUUID(), Value: fftypes.Byteable(`"value2"`)}, + {ID: fftypes.NewUUID(), Value: fftypes.JSONAnyPtr(`"value1"`)}, + {ID: fftypes.NewUUID(), Value: fftypes.JSONAnyPtr(`"value2"`)}, }) assert.NoError(t, err) <-waiter diff --git a/internal/i18n/en_translations.go b/internal/i18n/en_translations.go index 70775f2aac..021347be5a 100644 --- a/internal/i18n/en_translations.go +++ b/internal/i18n/en_translations.go @@ -219,6 +219,9 @@ var ( MsgInvalidChartNumberParam = ffm("FF10299", "Invalid %s. Must be a number.", 400) MsgHistogramInvalidTimes = ffm("FF10300", "Start time must be before end time", 400) MsgUnsupportedCollection = ffm("FF10301", "%s collection is not supported", 400) - MsgDXBadSize = ffm("FF10302", "Unexpected size returned from data exchange upload. Size=%d Expected=%d") - MsgBlobMismatchSealingData = ffm("FF10303", "Blob mismatch when sealing data") + MsgQueryOpUnsupportedMod = ffm("FF10302", "Operation '%s' on '%s' does not support modifiers", 400) + MsgDXBadSize = ffm("FF10303", "Unexpected size returned from data exchange upload. Size=%d Expected=%d") + MsgBlobMismatchSealingData = ffm("FF10304", "Blob mismatch when sealing data") + MsgFieldTypeNoStringMatching = ffm("FF10305", "Field '%s' of type '%s' does not support partial or case-insensitive string matching", 400) + MsgFieldMatchNoNull = ffm("FF10306", "Comparison operator for field '%s' cannot accept a null value", 400) ) diff --git a/internal/orchestrator/config.go b/internal/orchestrator/config.go index fffd989543..4e9a7b3e94 100644 --- a/internal/orchestrator/config.go +++ b/internal/orchestrator/config.go @@ -1,4 +1,4 @@ -// Copyright © 2021 Kaleido, Inc. +// Copyright © 2022 Kaleido, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -36,7 +36,7 @@ func (or *orchestrator) GetConfigRecords(ctx context.Context, filter database.An return or.database.GetConfigRecords(ctx, filter) } -func (or *orchestrator) PutConfigRecord(ctx context.Context, key string, value fftypes.Byteable) (outputValue fftypes.Byteable, err error) { +func (or *orchestrator) PutConfigRecord(ctx context.Context, key string, value *fftypes.JSONAny) (outputValue *fftypes.JSONAny, err error) { configRecord := &fftypes.ConfigRecord{ Key: key, Value: value, diff --git a/internal/orchestrator/config_test.go b/internal/orchestrator/config_test.go index 1103e41b38..d8b1af825b 100644 --- a/internal/orchestrator/config_test.go +++ b/internal/orchestrator/config_test.go @@ -31,12 +31,12 @@ func TestGetConfigRecord(t *testing.T) { or := newTestOrchestrator() or.mdi.On("GetConfigRecord", mock.Anything, mock.Anything).Return(&fftypes.ConfigRecord{ Key: "foobar", - Value: []byte(`{"foo": "bar"}`), + Value: fftypes.JSONAnyPtr(`{"foo": "bar"}`), }, nil) ctx := context.Background() configRecord, err := or.GetConfigRecord(ctx, "foo") assert.NoError(t, err) - assert.Equal(t, fftypes.Byteable(`{"foo": "bar"}`), configRecord.Value) + assert.Equal(t, fftypes.JSONAnyPtr(`{"foo": "bar"}`), configRecord.Value) } func TestGetConfigRecords(t *testing.T) { @@ -44,17 +44,17 @@ func TestGetConfigRecords(t *testing.T) { or.mdi.On("GetConfigRecords", mock.Anything, mock.Anything).Return([]*fftypes.ConfigRecord{ { Key: "foobar", - Value: []byte(`{"foo": "bar"}`), + Value: fftypes.JSONAnyPtr(`{"foo": "bar"}`), }, }, nil, nil) ctx := context.Background() configRecords, _, err := or.GetConfigRecords(ctx, nil) assert.NoError(t, err) - assert.Equal(t, fftypes.Byteable(`{"foo": "bar"}`), configRecords[0].Value) + assert.Equal(t, fftypes.JSONAnyPtr(`{"foo": "bar"}`), configRecords[0].Value) } func TestPutConfigRecord(t *testing.T) { - testValue := fftypes.Byteable(`{"foo": "bar"}`) + testValue := fftypes.JSONAnyPtr(`{"foo": "bar"}`) or := newTestOrchestrator() or.mdi.On("UpsertConfigRecord", mock.Anything, mock.Anything, mock.Anything).Return(nil) ctx := context.Background() @@ -64,7 +64,7 @@ func TestPutConfigRecord(t *testing.T) { } func TestPutConfigRecordFail(t *testing.T) { - testValue := fftypes.Byteable(`{"foo": "bar"}`) + testValue := fftypes.JSONAnyPtr(`{"foo": "bar"}`) or := newTestOrchestrator() or.mdi.On("UpsertConfigRecord", mock.Anything, mock.Anything, mock.Anything).Return(fmt.Errorf("pop")) ctx := context.Background() diff --git a/internal/orchestrator/data_query_test.go b/internal/orchestrator/data_query_test.go index bc6d6413c6..2658f10800 100644 --- a/internal/orchestrator/data_query_test.go +++ b/internal/orchestrator/data_query_test.go @@ -123,8 +123,8 @@ func TestGetMessageByIDWithDataOk(t *testing.T) { } or.mdi.On("GetMessageByID", mock.Anything, mock.MatchedBy(func(u *fftypes.UUID) bool { return u.Equals(msgID) })).Return(msg, nil) or.mdm.On("GetMessageData", mock.Anything, mock.Anything, true).Return([]*fftypes.Data{ - {ID: fftypes.NewUUID(), Hash: fftypes.NewRandB32(), Value: fftypes.Byteable("{}")}, - {ID: fftypes.NewUUID(), Hash: fftypes.NewRandB32(), Value: fftypes.Byteable("{}")}, + {ID: fftypes.NewUUID(), Hash: fftypes.NewRandB32(), Value: fftypes.JSONAnyPtr("{}")}, + {ID: fftypes.NewUUID(), Hash: fftypes.NewRandB32(), Value: fftypes.JSONAnyPtr("{}")}, }, true, nil) msgI, err := or.GetMessageByIDWithData(context.Background(), "ns1", msgID.String()) diff --git a/internal/orchestrator/orchestrator.go b/internal/orchestrator/orchestrator.go index cc7d06d589..97427b6dd0 100644 --- a/internal/orchestrator/orchestrator.go +++ b/internal/orchestrator/orchestrator.go @@ -1,4 +1,4 @@ -// Copyright © 2021 Kaleido, Inc. +// Copyright © 2022 Kaleido, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -115,7 +115,7 @@ type Orchestrator interface { GetConfig(ctx context.Context) fftypes.JSONObject GetConfigRecord(ctx context.Context, key string) (*fftypes.ConfigRecord, error) GetConfigRecords(ctx context.Context, filter database.AndFilter) ([]*fftypes.ConfigRecord, *database.FilterResult, error) - PutConfigRecord(ctx context.Context, key string, configRecord fftypes.Byteable) (outputValue fftypes.Byteable, err error) + PutConfigRecord(ctx context.Context, key string, configRecord *fftypes.JSONAny) (outputValue *fftypes.JSONAny, err error) DeleteConfigRecord(ctx context.Context, key string) (err error) ResetConfig(ctx context.Context) diff --git a/internal/orchestrator/orchestrator_test.go b/internal/orchestrator/orchestrator_test.go index 661899601a..d79fb7d164 100644 --- a/internal/orchestrator/orchestrator_test.go +++ b/internal/orchestrator/orchestrator_test.go @@ -208,7 +208,7 @@ func TestBlockchaiInitMergeConfigRecordsFail(t *testing.T) { or.mdi.On("GetConfigRecords", mock.Anything, mock.Anything, mock.Anything).Return([]*fftypes.ConfigRecord{ { Key: "pizza.toppings", - Value: []byte("cheese, pepperoni, mushrooms"), + Value: fftypes.JSONAnyPtr("cheese, pepperoni, mushrooms"), }, }, nil, nil) or.mii.On("Init", mock.Anything, mock.Anything, mock.Anything).Return(nil) diff --git a/internal/privatemessaging/groupmanager.go b/internal/privatemessaging/groupmanager.go index 354f4e50e5..c9d2be9973 100644 --- a/internal/privatemessaging/groupmanager.go +++ b/internal/privatemessaging/groupmanager.go @@ -82,8 +82,9 @@ func (gm *groupManager) groupInit(ctx context.Context, signer *fftypes.Identity, Namespace: group.Namespace, // must go in the same ordering context as the message Created: fftypes.Now(), } - data.Value, err = json.Marshal(&group) + b, err := json.Marshal(&group) if err == nil { + data.Value = fftypes.JSONAnyPtrBytes(b) err = group.Validate(ctx, true) if err == nil { err = data.Seal(ctx, nil) @@ -207,7 +208,7 @@ func (gm *groupManager) ResolveInitGroup(ctx context.Context, msg *fftypes.Messa return nil, err } var newGroup fftypes.Group - err = json.Unmarshal(data[0].Value, &newGroup) + err = json.Unmarshal(data[0].Value.Bytes(), &newGroup) if err != nil { log.L(ctx).Warnf("Group %s definition in message %s invalid: %s", msg.Header.Group, msg.Header.ID, err) return nil, nil diff --git a/internal/privatemessaging/groupmanager_test.go b/internal/privatemessaging/groupmanager_test.go index 29e2aa5cfd..5993dc01f9 100644 --- a/internal/privatemessaging/groupmanager_test.go +++ b/internal/privatemessaging/groupmanager_test.go @@ -110,7 +110,7 @@ func TestResolveInitGroupBadData(t *testing.T) { mdm := pm.data.(*datamocks.Manager) mdm.On("GetMessageData", pm.ctx, mock.Anything, true).Return([]*fftypes.Data{ - {ID: fftypes.NewUUID(), Value: fftypes.Byteable(`!json`)}, + {ID: fftypes.NewUUID(), Value: fftypes.JSONAnyPtr(`!json`)}, }, true, nil) _, err := pm.ResolveInitGroup(pm.ctx, &fftypes.Message{ @@ -135,7 +135,7 @@ func TestResolveInitGroupBadValidation(t *testing.T) { mdm := pm.data.(*datamocks.Manager) mdm.On("GetMessageData", pm.ctx, mock.Anything, true).Return([]*fftypes.Data{ - {ID: fftypes.NewUUID(), Value: fftypes.Byteable(`{}`)}, + {ID: fftypes.NewUUID(), Value: fftypes.JSONAnyPtr(`{}`)}, }, true, nil) _, err := pm.ResolveInitGroup(pm.ctx, &fftypes.Message{ @@ -173,7 +173,7 @@ func TestResolveInitGroupBadGroupID(t *testing.T) { mdm := pm.data.(*datamocks.Manager) mdm.On("GetMessageData", pm.ctx, mock.Anything, true).Return([]*fftypes.Data{ - {ID: fftypes.NewUUID(), Value: fftypes.Byteable(b)}, + {ID: fftypes.NewUUID(), Value: fftypes.JSONAnyPtrBytes(b)}, }, true, nil) _, err := pm.ResolveInitGroup(pm.ctx, &fftypes.Message{ @@ -211,7 +211,7 @@ func TestResolveInitGroupUpsertFail(t *testing.T) { mdm := pm.data.(*datamocks.Manager) mdm.On("GetMessageData", pm.ctx, mock.Anything, true).Return([]*fftypes.Data{ - {ID: fftypes.NewUUID(), Value: fftypes.Byteable(b)}, + {ID: fftypes.NewUUID(), Value: fftypes.JSONAnyPtrBytes(b)}, }, true, nil) mdi := pm.database.(*databasemocks.Plugin) mdi.On("UpsertGroup", pm.ctx, mock.Anything, true).Return(fmt.Errorf("pop")) @@ -251,7 +251,7 @@ func TestResolveInitGroupNewOk(t *testing.T) { mdm := pm.data.(*datamocks.Manager) mdm.On("GetMessageData", pm.ctx, mock.Anything, true).Return([]*fftypes.Data{ - {ID: fftypes.NewUUID(), Value: fftypes.Byteable(b)}, + {ID: fftypes.NewUUID(), Value: fftypes.JSONAnyPtrBytes(b)}, }, true, nil) mdi := pm.database.(*databasemocks.Plugin) mdi.On("UpsertGroup", pm.ctx, mock.Anything, true).Return(nil) diff --git a/internal/privatemessaging/message.go b/internal/privatemessaging/message.go index b06ebf6179..1644a108e2 100644 --- a/internal/privatemessaging/message.go +++ b/internal/privatemessaging/message.go @@ -1,4 +1,4 @@ -// Copyright © 2021 Kaleido, Inc. +// Copyright © 2022 Kaleido, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -219,5 +219,5 @@ func (s *messageSender) sendUnpinned(ctx context.Context) (err error) { return i18n.WrapError(ctx, err, i18n.MsgSerializationFailed) } - return s.mgr.sendData(ctx, "message", s.msg.Header.ID, s.msg.Header.Group, s.namespace, nodes, payload, nil, data) + return s.mgr.sendData(ctx, "message", s.msg.Header.ID, s.msg.Header.Group, s.namespace, nodes, fftypes.JSONAnyPtrBytes(payload), nil, data) } diff --git a/internal/privatemessaging/message_test.go b/internal/privatemessaging/message_test.go index d60ef5e308..6c76b38de2 100644 --- a/internal/privatemessaging/message_test.go +++ b/internal/privatemessaging/message_test.go @@ -81,7 +81,7 @@ func TestSendConfirmMessageE2EOk(t *testing.T) { msg, err := pm.SendMessage(pm.ctx, "ns1", &fftypes.MessageInOut{ InlineData: fftypes.InlineData{ - {Value: fftypes.Byteable(`{"some": "data"}`)}, + {Value: fftypes.JSONAnyPtr(`{"some": "data"}`)}, }, Group: &fftypes.InputGroup{ Members: []fftypes.MemberInput{ @@ -116,7 +116,7 @@ func TestSendUnpinnedMessageE2EOk(t *testing.T) { {ID: dataID, Hash: fftypes.NewRandB32()}, }, nil) mdm.On("GetMessageData", pm.ctx, mock.Anything, true).Return([]*fftypes.Data{ - {ID: dataID, Value: fftypes.Byteable(`{"some": "data"}`)}, + {ID: dataID, Value: fftypes.JSONAnyPtr(`{"some": "data"}`)}, }, true, nil).Once() mdi := pm.database.(*databasemocks.Plugin) @@ -149,7 +149,7 @@ func TestSendUnpinnedMessageE2EOk(t *testing.T) { }, }, InlineData: fftypes.InlineData{ - {Value: fftypes.Byteable(`{"some": "data"}`)}, + {Value: fftypes.JSONAnyPtr(`{"some": "data"}`)}, }, Group: &fftypes.InputGroup{ Members: []fftypes.MemberInput{ @@ -177,7 +177,7 @@ func TestSendMessageBadGroup(t *testing.T) { _, err := pm.SendMessage(pm.ctx, "ns1", &fftypes.MessageInOut{ InlineData: fftypes.InlineData{ - {Value: fftypes.Byteable(`{"some": "data"}`)}, + {Value: fftypes.JSONAnyPtr(`{"some": "data"}`)}, }, Group: &fftypes.InputGroup{}, }, true) @@ -197,7 +197,7 @@ func TestSendMessageBadIdentity(t *testing.T) { _, err := pm.SendMessage(pm.ctx, "ns1", &fftypes.MessageInOut{ InlineData: fftypes.InlineData{ - {Value: fftypes.Byteable(`{"some": "data"}`)}, + {Value: fftypes.JSONAnyPtr(`{"some": "data"}`)}, }, Group: &fftypes.InputGroup{ Members: []fftypes.MemberInput{ @@ -245,7 +245,7 @@ func TestSendMessageFail(t *testing.T) { _, err := pm.SendMessage(pm.ctx, "ns1", &fftypes.MessageInOut{ InlineData: fftypes.InlineData{ - {Value: fftypes.Byteable(`{"some": "data"}`)}, + {Value: fftypes.JSONAnyPtr(`{"some": "data"}`)}, }, Group: &fftypes.InputGroup{ Members: []fftypes.MemberInput{ @@ -401,7 +401,7 @@ func TestSendUnpinnedMessageMarshalFail(t *testing.T) { nodeID2 := fftypes.NewUUID() mdm := pm.data.(*datamocks.Manager) mdm.On("GetMessageData", pm.ctx, mock.Anything, true).Return([]*fftypes.Data{ - {ID: fftypes.NewUUID(), Value: fftypes.Byteable(`!Invalid JSON`)}, + {ID: fftypes.NewUUID(), Value: fftypes.JSONAnyPtr(`!Invalid JSON`)}, }, true, nil).Once() mdi := pm.database.(*databasemocks.Plugin) @@ -555,7 +555,7 @@ func TestSendUnpinnedMessageInsertFail(t *testing.T) { }, }, InlineData: fftypes.InlineData{ - {Value: fftypes.Byteable(`{"some": "data"}`)}, + {Value: fftypes.JSONAnyPtr(`{"some": "data"}`)}, }, Group: &fftypes.InputGroup{ Members: []fftypes.MemberInput{ @@ -628,7 +628,7 @@ func TestSendUnpinnedMessageResolveGroupFail(t *testing.T) { }, }, InlineData: fftypes.InlineData{ - {Value: fftypes.Byteable(`{"some": "data"}`)}, + {Value: fftypes.JSONAnyPtr(`{"some": "data"}`)}, }, Group: &fftypes.InputGroup{ Members: []fftypes.MemberInput{ @@ -662,7 +662,7 @@ func TestSendUnpinnedMessageEventFail(t *testing.T) { {ID: dataID, Hash: fftypes.NewRandB32()}, }, nil) mdm.On("GetMessageData", pm.ctx, mock.Anything, true).Return([]*fftypes.Data{ - {ID: dataID, Value: fftypes.Byteable(`{"some": "data"}`)}, + {ID: dataID, Value: fftypes.JSONAnyPtr(`{"some": "data"}`)}, }, true, nil).Once() mdi := pm.database.(*databasemocks.Plugin) @@ -695,7 +695,7 @@ func TestSendUnpinnedMessageEventFail(t *testing.T) { }, }, InlineData: fftypes.InlineData{ - {Value: fftypes.Byteable(`{"some": "data"}`)}, + {Value: fftypes.JSONAnyPtr(`{"some": "data"}`)}, }, Group: &fftypes.InputGroup{ Members: []fftypes.MemberInput{ diff --git a/internal/privatemessaging/privatemessaging.go b/internal/privatemessaging/privatemessaging.go index de68b41576..697b077ee8 100644 --- a/internal/privatemessaging/privatemessaging.go +++ b/internal/privatemessaging/privatemessaging.go @@ -1,4 +1,4 @@ -// Copyright © 2021 Kaleido, Inc. +// Copyright © 2022 Kaleido, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -135,7 +135,7 @@ func (pm *privateMessaging) dispatchBatch(ctx context.Context, batch *fftypes.Ba } return pm.database.RunAsGroup(ctx, func(ctx context.Context) error { - return pm.sendAndSubmitBatch(ctx, batch, nodes, payload, contexts) + return pm.sendAndSubmitBatch(ctx, batch, nodes, fftypes.JSONAnyPtrBytes(payload), contexts) }) } @@ -174,7 +174,7 @@ func (pm *privateMessaging) transferBlobs(ctx context.Context, data []*fftypes.D return nil } -func (pm *privateMessaging) sendData(ctx context.Context, mType string, mID *fftypes.UUID, group *fftypes.Bytes32, ns string, nodes []*fftypes.Node, payload fftypes.Byteable, txid *fftypes.UUID, data []*fftypes.Data) (err error) { +func (pm *privateMessaging) sendData(ctx context.Context, mType string, mID *fftypes.UUID, group *fftypes.Bytes32, ns string, nodes []*fftypes.Node, payload *fftypes.JSONAny, txid *fftypes.UUID, data []*fftypes.Data) (err error) { l := log.L(ctx) // TODO: move to using DIDs consistently as the way to reference the node/organization (i.e. node.Owner becomes a DID) @@ -199,7 +199,7 @@ func (pm *privateMessaging) sendData(ctx context.Context, mType string, mID *fft } // Send the payload itself - trackingID, err := pm.exchange.SendMessage(ctx, node.DX.Peer, payload) + trackingID, err := pm.exchange.SendMessage(ctx, node.DX.Peer, payload.Bytes()) if err != nil { return err } @@ -222,7 +222,7 @@ func (pm *privateMessaging) sendData(ctx context.Context, mType string, mID *fft return nil } -func (pm *privateMessaging) sendAndSubmitBatch(ctx context.Context, batch *fftypes.Batch, nodes []*fftypes.Node, payload fftypes.Byteable, contexts []*fftypes.Bytes32) (err error) { +func (pm *privateMessaging) sendAndSubmitBatch(ctx context.Context, batch *fftypes.Batch, nodes []*fftypes.Node, payload *fftypes.JSONAny, contexts []*fftypes.Bytes32) (err error) { if err = pm.sendData(ctx, "batch", batch.ID, batch.Group, batch.Namespace, nodes, payload, batch.Payload.TX.ID, batch.Payload.Data); err != nil { return err } diff --git a/internal/privatemessaging/privatemessaging_test.go b/internal/privatemessaging/privatemessaging_test.go index 53e6b96297..c1b9072564 100644 --- a/internal/privatemessaging/privatemessaging_test.go +++ b/internal/privatemessaging/privatemessaging_test.go @@ -184,7 +184,7 @@ func TestDispatchBatchBadData(t *testing.T) { err := pm.dispatchBatch(pm.ctx, &fftypes.Batch{ Payload: fftypes.BatchPayload{ Data: []*fftypes.Data{ - {Value: fftypes.Byteable(`{!json}`)}, + {Value: fftypes.JSONAnyPtr(`{!json}`)}, }, }, }, []*fftypes.Bytes32{}) @@ -223,7 +223,7 @@ func TestSendAndSubmitBatchBadID(t *testing.T) { Identity: fftypes.Identity{ Author: "badauthor", }, - }, []*fftypes.Node{}, fftypes.Byteable(`{}`), []*fftypes.Bytes32{}) + }, []*fftypes.Node{}, fftypes.JSONAnyPtr(`{}`), []*fftypes.Bytes32{}) assert.Regexp(t, "pop", err) } @@ -241,7 +241,7 @@ func TestSendAndSubmitBatchUnregisteredNode(t *testing.T) { Identity: fftypes.Identity{ Author: "badauthor", }, - }, []*fftypes.Node{}, fftypes.Byteable(`{}`), []*fftypes.Bytes32{}) + }, []*fftypes.Node{}, fftypes.JSONAnyPtr(`{}`), []*fftypes.Bytes32{}) assert.Regexp(t, "pop", err) } @@ -266,7 +266,7 @@ func TestSendImmediateFail(t *testing.T) { Endpoint: fftypes.JSONObject{"url": "https://node1.example.com"}, }, }, - }, fftypes.Byteable(`{}`), []*fftypes.Bytes32{}) + }, fftypes.JSONAnyPtr(`{}`), []*fftypes.Bytes32{}) assert.Regexp(t, "pop", err) } @@ -299,7 +299,7 @@ func TestSendSubmitInsertOperationFail(t *testing.T) { Endpoint: fftypes.JSONObject{"url": "https://node1.example.com"}, }, }, - }, fftypes.Byteable(`{}`), []*fftypes.Bytes32{}) + }, fftypes.JSONAnyPtr(`{}`), []*fftypes.Bytes32{}) assert.Regexp(t, "pop", err) } @@ -329,7 +329,7 @@ func TestSendSubmitBlobTransferFail(t *testing.T) { Endpoint: fftypes.JSONObject{"url": "https://node1.example.com"}, }, }, - }, fftypes.Byteable(`{}`), []*fftypes.Bytes32{}) + }, fftypes.JSONAnyPtr(`{}`), []*fftypes.Bytes32{}) assert.Regexp(t, "pop", err) } diff --git a/internal/privatemessaging/recipients_test.go b/internal/privatemessaging/recipients_test.go index 03a64f956d..c79fd2d78e 100644 --- a/internal/privatemessaging/recipients_test.go +++ b/internal/privatemessaging/recipients_test.go @@ -63,7 +63,7 @@ func TestResolveMemberListNewGroupE2E(t *testing.T) { assert.Equal(t, fftypes.ValidatorTypeSystemDefinition, data.Validator) assert.Equal(t, "ns1", data.Namespace) var group fftypes.Group - err := json.Unmarshal(data.Value, &group) + err := json.Unmarshal(data.Value.Bytes(), &group) assert.NoError(t, err) assert.Len(t, group.Members, 2) // Group identiy is sorted by group members DIDs so check them in that order diff --git a/internal/syncasync/sync_async_bridge_test.go b/internal/syncasync/sync_async_bridge_test.go index 1f53b55a43..b24d74c8a6 100644 --- a/internal/syncasync/sync_async_bridge_test.go +++ b/internal/syncasync/sync_async_bridge_test.go @@ -69,7 +69,7 @@ func TestRequestReplyOk(t *testing.T) { mdm := sa.data.(*datamocks.Manager) mdm.On("GetMessageData", sa.ctx, mock.Anything, true).Return([]*fftypes.Data{ - {ID: dataID, Value: fftypes.Byteable(`"response data"`)}, + {ID: dataID, Value: fftypes.JSONAnyPtr(`"response data"`)}, }, true, nil) reply, err := sa.WaitForReply(sa.ctx, "ns1", requestID, func(ctx context.Context) error { @@ -87,7 +87,7 @@ func TestRequestReplyOk(t *testing.T) { }) assert.NoError(t, err) assert.Equal(t, *replyID, *reply.Header.ID) - assert.Equal(t, `"response data"`, string(reply.InlineData[0].Value)) + assert.Equal(t, `"response data"`, reply.InlineData[0].Value.String()) } @@ -116,7 +116,7 @@ func TestAwaitConfirmationOk(t *testing.T) { mdm := sa.data.(*datamocks.Manager) mdm.On("GetMessageData", sa.ctx, mock.Anything, true).Return([]*fftypes.Data{ - {ID: dataID, Value: fftypes.Byteable(`"response data"`)}, + {ID: dataID, Value: fftypes.JSONAnyPtr(`"response data"`)}, }, true, nil) reply, err := sa.WaitForMessage(sa.ctx, "ns1", requestID, func(ctx context.Context) error { @@ -162,7 +162,7 @@ func TestAwaitConfirmationRejected(t *testing.T) { mdm := sa.data.(*datamocks.Manager) mdm.On("GetMessageData", sa.ctx, mock.Anything, true).Return([]*fftypes.Data{ - {ID: dataID, Value: fftypes.Byteable(`"response data"`)}, + {ID: dataID, Value: fftypes.JSONAnyPtr(`"response data"`)}, }, true, nil) _, err := sa.WaitForMessage(sa.ctx, "ns1", requestID, func(ctx context.Context) error { diff --git a/mocks/orchestratormocks/orchestrator.go b/mocks/orchestratormocks/orchestrator.go index e950ae5a6d..6c54a71e98 100644 --- a/mocks/orchestratormocks/orchestrator.go +++ b/mocks/orchestratormocks/orchestrator.go @@ -1091,20 +1091,20 @@ func (_m *Orchestrator) PrivateMessaging() privatemessaging.Manager { } // PutConfigRecord provides a mock function with given fields: ctx, key, configRecord -func (_m *Orchestrator) PutConfigRecord(ctx context.Context, key string, configRecord fftypes.Byteable) (fftypes.Byteable, error) { +func (_m *Orchestrator) PutConfigRecord(ctx context.Context, key string, configRecord *fftypes.JSONAny) (*fftypes.JSONAny, error) { ret := _m.Called(ctx, key, configRecord) - var r0 fftypes.Byteable - if rf, ok := ret.Get(0).(func(context.Context, string, fftypes.Byteable) fftypes.Byteable); ok { + var r0 *fftypes.JSONAny + if rf, ok := ret.Get(0).(func(context.Context, string, *fftypes.JSONAny) *fftypes.JSONAny); ok { r0 = rf(ctx, key, configRecord) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(fftypes.Byteable) + r0 = ret.Get(0).(*fftypes.JSONAny) } } var r1 error - if rf, ok := ret.Get(1).(func(context.Context, string, fftypes.Byteable) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, string, *fftypes.JSONAny) error); ok { r1 = rf(ctx, key, configRecord) } else { r1 = ret.Error(1) diff --git a/pkg/database/filter.go b/pkg/database/filter.go index 16bbe1b2bf..57741f4165 100644 --- a/pkg/database/filter.go +++ b/pkg/database/filter.go @@ -1,4 +1,4 @@ -// Copyright © 2021 Kaleido, Inc. +// Copyright © 2022 Kaleido, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -24,6 +24,7 @@ import ( "strings" "github.com/hyperledger/firefly/internal/i18n" + "github.com/hyperledger/firefly/pkg/fftypes" ) // Filter is the output of the builder @@ -68,6 +69,7 @@ type OrFilter interface{ MultiConditionFilter } // used in the core string formatting method (for logging etc.) type FilterOp string +// The character pairs in this are not used anywhere externally, just in a to-string representation of queries const ( // FilterOpAnd and FilterOpAnd FilterOp = "&&" @@ -75,16 +77,20 @@ const ( FilterOpOr FilterOp = "||" // FilterOpEq equal FilterOpEq FilterOp = "==" + // FilterOpIEq equal + FilterOpIEq FilterOp = ":=" // FilterOpNe not equal - FilterOpNe FilterOp = "!=" + FilterOpNeq FilterOp = "!=" + // FilterOpNIeq not equal + FilterOpNIeq FilterOp = ";=" // FilterOpIn in list of values FilterOpIn FilterOp = "IN" // FilterOpNotIn not in list of values FilterOpNotIn FilterOp = "NI" // FilterOpGt greater than - FilterOpGt FilterOp = ">" + FilterOpGt FilterOp = ">>" // FilterOpLt less than - FilterOpLt FilterOp = "<" + FilterOpLt FilterOp = "<<" // FilterOpGte greater than or equal FilterOpGte FilterOp = ">=" // FilterOpLte less than or equal @@ -92,13 +98,51 @@ const ( // FilterOpCont contains the specified text, case sensitive FilterOpCont FilterOp = "%=" // FilterOpNotCont does not contain the specified text, case sensitive - FilterOpNotCont FilterOp = "%!" + FilterOpNotCont FilterOp = "!%" // FilterOpICont contains the specified text, case insensitive - FilterOpICont FilterOp = "^=" + FilterOpICont FilterOp = ":%" // FilterOpNotICont does not contain the specified text, case insensitive - FilterOpNotICont FilterOp = "^!" + FilterOpNotICont FilterOp = ";%" + // FilterOpStartsWith contains the specified text, case sensitive + FilterOpStartsWith FilterOp = "^=" + // FilterOpNotCont does not contain the specified text, case sensitive + FilterOpNotStartsWith FilterOp = "!^" + // FilterOpICont contains the specified text, case insensitive + FilterOpIStartsWith FilterOp = ":^" + // FilterOpNotICont does not contain the specified text, case insensitive + FilterOpNotIStartsWith FilterOp = ";^" + // FilterOpEndsWith contains the specified text, case sensitive + FilterOpEndsWith FilterOp = "$=" + // FilterOpNotCont does not contain the specified text, case sensitive + FilterOpNotEndsWith FilterOp = "!$" + // FilterOpICont contains the specified text, case insensitive + FilterOpIEndsWith FilterOp = ":$" + // FilterOpNotICont does not contain the specified text, case insensitive + FilterOpNotIEndsWith FilterOp = ";$" ) +func filterOpIsStringMatch(op FilterOp) bool { + for _, r := range string(op) { + switch r { + case '%', '^', '$', ':': + // Partial or case-insensitive matches all need a string + return true + } + } + return false +} + +func filterCannotAcceptNull(op FilterOp) bool { + for _, r := range string(op) { + switch r { + case '%', '^', '$', ':', '>', '<': + // string based matching, or gt/lt cannot accept null + return true + } + } + return false +} + // FilterBuilder is the syntax used to build the filter, where And() and Or() can be nested type FilterBuilder interface { // Fields is the list of available fields @@ -107,10 +151,14 @@ type FilterBuilder interface { And(and ...Filter) AndFilter // Or requires any of the sub-filters to match Or(and ...Filter) OrFilter - // Eq equal + // Eq equal - case sensitive Eq(name string, value driver.Value) Filter - // Neq not equal + // Neq not equal - case sensitive Neq(name string, value driver.Value) Filter + // IEq equal - case insensitive + IEq(name string, value driver.Value) Filter + // INeq not equal - case insensitive + NIeq(name string, value driver.Value) Filter // In one of an array of values In(name string, value []driver.Value) Filter // NotIn not one of an array of values @@ -127,10 +175,26 @@ type FilterBuilder interface { Contains(name string, value driver.Value) Filter // NotContains disallows the string anywhere - case sensitive NotContains(name string, value driver.Value) Filter - // IContains allows the string anywhere - case sensitive + // IContains allows the string anywhere - case insensitive IContains(name string, value driver.Value) Filter - // INotContains disallows the string anywhere - case sensitive + // INotContains disallows the string anywhere - case insensitive NotIContains(name string, value driver.Value) Filter + // StartsWith allows the string at the start - case sensitive + StartsWith(name string, value driver.Value) Filter + // NotStartsWith disallows the string at the start - case sensitive + NotStartsWith(name string, value driver.Value) Filter + // IStartsWith allows the string at the start - case insensitive + IStartsWith(name string, value driver.Value) Filter + // NotIStartsWith disallows the string att the start - case insensitive + NotIStartsWith(name string, value driver.Value) Filter + // EndsWith allows the string at the end - case sensitive + EndsWith(name string, value driver.Value) Filter + // NotEndsWith disallows the string at the end - case sensitive + NotEndsWith(name string, value driver.Value) Filter + // IEndsWith allows the string at the end - case insensitive + IEndsWith(name string, value driver.Value) Filter + // NotIEndsWith disallows the string att the end - case insensitive + NotIEndsWith(name string, value driver.Value) Filter } // NullBehavior specifies whether to sort nulls first or last in a query @@ -173,11 +237,8 @@ func valueString(f FieldSerialization) string { v, _ := f.Value() switch tv := v.(type) { case nil: - return "null" + return fftypes.NullString case []byte: - if tv == nil { - return "null" - } return fmt.Sprintf("'%s'", tv) case int64: return strconv.FormatInt(tv, 10) @@ -302,9 +363,30 @@ func (f *baseFilter) Finalize() (fi *FilterInfo, err error) { if !ok { return nil, i18n.NewError(f.fb.ctx, i18n.MsgInvalidFilterField, name) } - value = field.getSerialization() - if err = value.Scan(f.value); err != nil { - return nil, i18n.WrapError(f.fb.ctx, err, i18n.MsgInvalidValueForFilterField, name) + skipScan := false + switch f.value.(type) { + case nil: + if filterCannotAcceptNull(f.op) { + return nil, i18n.NewError(f.fb.ctx, i18n.MsgFieldMatchNoNull, f.op, name) + } + value = &nullField{} + skipScan = true + case string: + switch { + case field.filterAsString(): + value = &stringField{} + case filterOpIsStringMatch(f.op): + return nil, i18n.NewError(f.fb.ctx, i18n.MsgFieldTypeNoStringMatching, name, field.description()) + default: + value = field.getSerialization() + } + default: + value = field.getSerialization() + } + if !skipScan { + if err = value.Scan(f.value); err != nil { + return nil, i18n.WrapError(f.fb.ctx, err, i18n.MsgInvalidValueForFilterField, name) + } } } @@ -416,7 +498,15 @@ func (fb *filterBuilder) Eq(name string, value driver.Value) Filter { } func (fb *filterBuilder) Neq(name string, value driver.Value) Filter { - return fb.fieldFilter(FilterOpNe, name, value) + return fb.fieldFilter(FilterOpNeq, name, value) +} + +func (fb *filterBuilder) IEq(name string, value driver.Value) Filter { + return fb.fieldFilter(FilterOpIEq, name, value) +} + +func (fb *filterBuilder) NIeq(name string, value driver.Value) Filter { + return fb.fieldFilter(FilterOpNIeq, name, value) } func (fb *filterBuilder) In(name string, values []driver.Value) Filter { @@ -459,6 +549,38 @@ func (fb *filterBuilder) NotIContains(name string, value driver.Value) Filter { return fb.fieldFilter(FilterOpNotICont, name, value) } +func (fb *filterBuilder) StartsWith(name string, value driver.Value) Filter { + return fb.fieldFilter(FilterOpStartsWith, name, value) +} + +func (fb *filterBuilder) NotStartsWith(name string, value driver.Value) Filter { + return fb.fieldFilter(FilterOpNotStartsWith, name, value) +} + +func (fb *filterBuilder) IStartsWith(name string, value driver.Value) Filter { + return fb.fieldFilter(FilterOpIStartsWith, name, value) +} + +func (fb *filterBuilder) NotIStartsWith(name string, value driver.Value) Filter { + return fb.fieldFilter(FilterOpNotIStartsWith, name, value) +} + +func (fb *filterBuilder) EndsWith(name string, value driver.Value) Filter { + return fb.fieldFilter(FilterOpEndsWith, name, value) +} + +func (fb *filterBuilder) NotEndsWith(name string, value driver.Value) Filter { + return fb.fieldFilter(FilterOpNotEndsWith, name, value) +} + +func (fb *filterBuilder) IEndsWith(name string, value driver.Value) Filter { + return fb.fieldFilter(FilterOpIEndsWith, name, value) +} + +func (fb *filterBuilder) NotIEndsWith(name string, value driver.Value) Filter { + return fb.fieldFilter(FilterOpNotIEndsWith, name, value) +} + func (fb *filterBuilder) fieldFilter(op FilterOp, name string, value interface{}) Filter { return &fieldFilter{ baseFilter: baseFilter{ diff --git a/pkg/database/filter_test.go b/pkg/database/filter_test.go index 2243d0038a..fb9d3f9a12 100644 --- a/pkg/database/filter_test.go +++ b/pkg/database/filter_test.go @@ -42,7 +42,7 @@ func TestBuildMessageFilter(t *testing.T) { Descending(). Finalize() assert.NoError(t, err) - assert.Equal(t, "( namespace == 'ns1' ) && ( ( id == '35c11cba-adff-4a4d-970a-02e3a0858dc8' ) || ( id == 'caefb9d1-9fc9-4d6a-a155-514d3139adf7' ) ) && ( sequence > 12345 ) && ( confirmed == null ) sort=-namespace skip=50 limit=25 count=true", f.String()) + assert.Equal(t, "( namespace == 'ns1' ) && ( ( id == '35c11cba-adff-4a4d-970a-02e3a0858dc8' ) || ( id == 'caefb9d1-9fc9-4d6a-a155-514d3139adf7' ) ) && ( sequence >> 12345 ) && ( confirmed == null ) sort=-namespace skip=50 limit=25 count=true", f.String()) } func TestBuildMessageFilter2(t *testing.T) { @@ -53,7 +53,7 @@ func TestBuildMessageFilter2(t *testing.T) { Finalize() assert.NoError(t, err) - assert.Equal(t, "sequence > 0 sort=sequence", f.String()) + assert.Equal(t, "sequence >> 0 sort=sequence", f.String()) } func TestBuildMessageFilter3(t *testing.T) { @@ -76,7 +76,25 @@ func TestBuildMessageFilter3(t *testing.T) { Sort("-sequence"). Finalize() assert.NoError(t, err) - assert.Equal(t, "( created IN [1000000000,2000000000,3000000000] ) && ( created NI [1000000000,2000000000,3000000000] ) && ( created < 0 ) && ( created <= 0 ) && ( created >= 0 ) && ( created != 0 ) && ( sequence > 12345 ) && ( topics %= 'abc' ) && ( topics %! 'def' ) && ( topics ^= 'ghi' ) && ( topics ^! 'jkl' ) sort=-created,topics,-sequence", f.String()) + assert.Equal(t, "( created IN [1000000000,2000000000,3000000000] ) && ( created NI [1000000000,2000000000,3000000000] ) && ( created << 0 ) && ( created <= 0 ) && ( created >= 0 ) && ( created != 0 ) && ( sequence >> 12345 ) && ( topics %= 'abc' ) && ( topics !% 'def' ) && ( topics :% 'ghi' ) && ( topics ;% 'jkl' ) sort=-created,topics,-sequence", f.String()) +} + +func TestBuildMessageFilter4(t *testing.T) { + fb := MessageQueryFactory.NewFilter(context.Background()) + f, err := fb.And( + fb.IEq("topics", "abc"), + fb.NIeq("topics", "bcd"), + fb.StartsWith("topics", "cde"), + fb.IStartsWith("topics", "def"), + fb.NotStartsWith("topics", "efg"), + fb.NotIStartsWith("topics", "fgh"), + fb.EndsWith("topics", "hij"), + fb.IEndsWith("topics", "ikl"), + fb.NotEndsWith("topics", "lmn"), + fb.NotIEndsWith("topics", "mno"), + ).Finalize() + assert.NoError(t, err) + assert.Equal(t, "( topics := 'abc' ) && ( topics ;= 'bcd' ) && ( topics ^= 'cde' ) && ( topics :^ 'def' ) && ( topics !^ 'efg' ) && ( topics ;^ 'fgh' ) && ( topics $= 'hij' ) && ( topics :$ 'ikl' ) && ( topics !$ 'lmn' ) && ( topics ;$ 'mno' )", f.String()) } func TestBuildMessageBadInFilterField(t *testing.T) { @@ -112,7 +130,7 @@ func TestBuildMessageUUIDConvert(t *testing.T) { fb.Eq("id", nilB32), ).Finalize() assert.NoError(t, err) - assert.Equal(t, "( id == '4066abdc-8bbd-4472-9d29-1a55b467f9b9' ) && ( id == '4066abdc-8bbd-4472-9d29-1a55b467f9b9' ) && ( id IN ['4066abdc-8bbd-4472-9d29-1a55b467f9b9'] ) && ( id == '4066abdc-8bbd-4472-9d29-1a55b467f9b9' ) && ( id != null ) && ( id == '4066abdc-8bbd-4472-9d29-1a55b467f9b9' ) && ( id != '4066abdc-8bbd-4472-9d29-1a55b467f9b9' ) && ( id == null ) && ( id == null )", f.String()) + assert.Equal(t, "( id == '4066abdc-8bbd-4472-9d29-1a55b467f9b9' ) && ( id == '4066abdc-8bbd-4472-9d29-1a55b467f9b9' ) && ( id IN ['4066abdc-8bbd-4472-9d29-1a55b467f9b9'] ) && ( id == '4066abdc-8bbd-4472-9d29-1a55b467f9b9' ) && ( id != null ) && ( id == '4066abdc-8bbd-4472-9d29-1a55b467f9b9' ) && ( id != '4066abdc-8bbd-4472-9d29-1a55b467f9b9' ) && ( id == '' ) && ( id == null )", f.String()) } func TestBuildMessageBytes32Convert(t *testing.T) { @@ -129,7 +147,7 @@ func TestBuildMessageBytes32Convert(t *testing.T) { fb.Eq("hash", nilB32), ).Finalize() assert.NoError(t, err) - assert.Equal(t, "( hash == '7f4806535f8b3d9bf178af053d2bbdb46047365466ed16bbb0732a71492bdaf0' ) && ( hash == '7f4806535f8b3d9bf178af053d2bbdb46047365466ed16bbb0732a71492bdaf0' ) && ( hash IN ['7f4806535f8b3d9bf178af053d2bbdb46047365466ed16bbb0732a71492bdaf0'] ) && ( hash == '7f4806535f8b3d9bf178af053d2bbdb46047365466ed16bbb0732a71492bdaf0' ) && ( hash != null ) && ( hash == null ) && ( hash == null )", f.String()) + assert.Equal(t, "( hash == '7f4806535f8b3d9bf178af053d2bbdb46047365466ed16bbb0732a71492bdaf0' ) && ( hash == '7f4806535f8b3d9bf178af053d2bbdb46047365466ed16bbb0732a71492bdaf0' ) && ( hash IN ['7f4806535f8b3d9bf178af053d2bbdb46047365466ed16bbb0732a71492bdaf0'] ) && ( hash == '7f4806535f8b3d9bf178af053d2bbdb46047365466ed16bbb0732a71492bdaf0' ) && ( hash != null ) && ( hash == '' ) && ( hash == null )", f.String()) } func TestBuildMessageIntConvert(t *testing.T) { fb := MessageQueryFactory.NewFilter(context.Background()) @@ -142,7 +160,7 @@ func TestBuildMessageIntConvert(t *testing.T) { fb.Lt("sequence", uint64(666)), ).Finalize() assert.NoError(t, err) - assert.Equal(t, "( sequence < 111 ) && ( sequence < 222 ) && ( sequence < 333 ) && ( sequence < 444 ) && ( sequence < 555 ) && ( sequence < 666 )", f.String()) + assert.Equal(t, "( sequence << 111 ) && ( sequence << 222 ) && ( sequence << 333 ) && ( sequence << 444 ) && ( sequence << 555 ) && ( sequence << 666 )", f.String()) } func TestBuildMessageTimeConvert(t *testing.T) { @@ -156,7 +174,7 @@ func TestBuildMessageTimeConvert(t *testing.T) { fb.Lt("created", *fftypes.UnixTime(1621112824)), ).Finalize() assert.NoError(t, err) - assert.Equal(t, "( created > 1621112824000000000 ) && ( created > 0 ) && ( created == 1621112874123456789 ) && ( created == null ) && ( created < 1621112824000000000 ) && ( created < 1621112824000000000 )", f.String()) + assert.Equal(t, "( created >> 1621112824000000000 ) && ( created >> 0 ) && ( created == 1621112874123456789 ) && ( created == null ) && ( created << 1621112824000000000 ) && ( created << 1621112824000000000 )", f.String()) } func TestBuildMessageStringConvert(t *testing.T) { @@ -170,14 +188,13 @@ func TestBuildMessageStringConvert(t *testing.T) { fb.Lt("namespace", uint(444)), fb.Lt("namespace", uint32(555)), fb.Lt("namespace", uint64(666)), - fb.Lt("namespace", nil), fb.Lt("namespace", *u), fb.Lt("namespace", u), fb.Lt("namespace", *b32), fb.Lt("namespace", b32), ).Finalize() assert.NoError(t, err) - assert.Equal(t, "( namespace < '111' ) && ( namespace < '222' ) && ( namespace < '333' ) && ( namespace < '444' ) && ( namespace < '555' ) && ( namespace < '666' ) && ( namespace < '' ) && ( namespace < '3f96e0d5-a10e-47c6-87a0-f2e7604af179' ) && ( namespace < '3f96e0d5-a10e-47c6-87a0-f2e7604af179' ) && ( namespace < '3f96e0d5a10e47c687a0f2e7604af17900000000000000000000000000000000' ) && ( namespace < '3f96e0d5a10e47c687a0f2e7604af17900000000000000000000000000000000' )", f.String()) + assert.Equal(t, "( namespace << '111' ) && ( namespace << '222' ) && ( namespace << '333' ) && ( namespace << '444' ) && ( namespace << '555' ) && ( namespace << '666' ) && ( namespace << '3f96e0d5-a10e-47c6-87a0-f2e7604af179' ) && ( namespace << '3f96e0d5-a10e-47c6-87a0-f2e7604af179' ) && ( namespace << '3f96e0d5a10e47c687a0f2e7604af17900000000000000000000000000000000' ) && ( namespace << '3f96e0d5a10e47c687a0f2e7604af17900000000000000000000000000000000' )", f.String()) } func TestBuildMessageBoolConvert(t *testing.T) { @@ -198,7 +215,7 @@ func TestBuildMessageBoolConvert(t *testing.T) { fb.Eq("masked", nil), ).Finalize() assert.NoError(t, err) - assert.Equal(t, "( masked == false ) && ( masked == true ) && ( masked == false ) && ( masked == true ) && ( masked == true ) && ( masked == false ) && ( masked == true ) && ( masked == true ) && ( masked == true ) && ( masked == true ) && ( masked == true ) && ( masked == true ) && ( masked == false )", f.String()) + assert.Equal(t, "( masked == false ) && ( masked == true ) && ( masked == false ) && ( masked == true ) && ( masked == true ) && ( masked == false ) && ( masked == true ) && ( masked == true ) && ( masked == true ) && ( masked == true ) && ( masked == true ) && ( masked == true ) && ( masked == null )", f.String()) } func TestBuildMessageJSONConvert(t *testing.T) { @@ -221,7 +238,7 @@ func TestBuildFFNameArrayConvert(t *testing.T) { fb.Eq("topics", []byte(`test2`)), ).Finalize() assert.NoError(t, err) - assert.Equal(t, `( topics == '' ) && ( topics == 'test1' ) && ( topics == 'test2' )`, f.String()) + assert.Equal(t, `( topics == null ) && ( topics == 'test1' ) && ( topics == 'test2' )`, f.String()) } func TestBuildMessageFailStringConvert(t *testing.T) { @@ -280,6 +297,22 @@ func TestQueryFactoryBadNestedValue(t *testing.T) { assert.Regexp(t, "FF10149.*sequence", err) } +func TestQueryFactoryStringMatchNonString(t *testing.T) { + fb := MessageQueryFactory.NewFilter(context.Background()) + _, err := fb.And( + fb.Contains("sequence", "stuff"), + ).Finalize() + assert.Regexp(t, "FF10305", err) +} + +func TestQueryFactoryNullGreaterThan(t *testing.T) { + fb := DataQueryFactory.NewFilter(context.Background()) + _, err := fb.And( + fb.Gt("created", nil), + ).Finalize() + assert.Regexp(t, "FF10306", err) +} + func TestQueryFactoryGetFields(t *testing.T) { fb := MessageQueryFactory.NewFilter(context.Background()) assert.NotNil(t, fb.Fields()) diff --git a/pkg/database/plugin.go b/pkg/database/plugin.go index dea35386e4..cfe0479f40 100644 --- a/pkg/database/plugin.go +++ b/pkg/database/plugin.go @@ -646,6 +646,7 @@ var DataQueryFactory = &queryFields{ "blob.name": &StringField{}, "blob.size": &Int64Field{}, "created": &TimeField{}, + "value": &JSONField{}, } // DatatypeQueryFactory filter fields for data definitions diff --git a/pkg/database/query_fields.go b/pkg/database/query_fields.go index 90088212a4..96c768a52d 100644 --- a/pkg/database/query_fields.go +++ b/pkg/database/query_fields.go @@ -1,4 +1,4 @@ -// Copyright © 2021 Kaleido, Inc. +// Copyright © 2022 Kaleido, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -68,8 +68,19 @@ type FieldSerialization interface { type Field interface { getSerialization() FieldSerialization + description() string + filterAsString() bool } +// nullField is a special FieldSerialization used to represent nil in queries +type nullField struct{} + +func (f *nullField) Scan(src interface{}) error { + return nil +} +func (f *nullField) Value() (driver.Value, error) { return nil, nil } +func (f *nullField) String() string { return fftypes.NullString } + type StringField struct{} type stringField struct{ s string } @@ -102,6 +113,7 @@ func (f *stringField) Scan(src interface{}) error { case fftypes.Bytes32: f.s = tv.String() case nil: + f.s = "" default: if reflect.TypeOf(tv).Kind() == reflect.String { // This is helpful for status enums @@ -115,6 +127,8 @@ func (f *stringField) Scan(src interface{}) error { func (f *stringField) Value() (driver.Value, error) { return f.s, nil } func (f *stringField) String() string { return f.s } func (f *StringField) getSerialization() FieldSerialization { return &stringField{} } +func (f *StringField) filterAsString() bool { return true } +func (f *StringField) description() string { return "String" } type UUIDField struct{} type uuidField struct{ u *fftypes.UUID } @@ -146,6 +160,7 @@ func (f *uuidField) Scan(src interface{}) (err error) { copy(u[:], tv[0:16]) f.u = &u case nil: + f.u = nil default: return i18n.NewError(context.Background(), i18n.MsgScanFailed, src, f.u) } @@ -154,6 +169,8 @@ func (f *uuidField) Scan(src interface{}) (err error) { func (f *uuidField) Value() (driver.Value, error) { return f.u.Value() } func (f *uuidField) String() string { return fmt.Sprintf("%v", f.u) } func (f *UUIDField) getSerialization() FieldSerialization { return &uuidField{} } +func (f *UUIDField) filterAsString() bool { return true } +func (f *UUIDField) description() string { return "UUID" } type Bytes32Field struct{} type bytes32Field struct{ b32 *fftypes.Bytes32 } @@ -173,6 +190,7 @@ func (f *bytes32Field) Scan(src interface{}) (err error) { b32 := tv f.b32 = &b32 case nil: + f.b32 = nil default: return i18n.NewError(context.Background(), i18n.MsgScanFailed, src, f.b32) } @@ -181,6 +199,8 @@ func (f *bytes32Field) Scan(src interface{}) (err error) { func (f *bytes32Field) Value() (driver.Value, error) { return f.b32.Value() } func (f *bytes32Field) String() string { return fmt.Sprintf("%v", f.b32) } func (f *Bytes32Field) getSerialization() FieldSerialization { return &bytes32Field{} } +func (f *Bytes32Field) filterAsString() bool { return true } +func (f *Bytes32Field) description() string { return "Byte-Array" } type Int64Field struct{} type int64Field struct{ i int64 } @@ -204,6 +224,8 @@ func (f *int64Field) Scan(src interface{}) (err error) { if err != nil { return i18n.WrapError(context.Background(), err, i18n.MsgScanFailed, src, int64(0)) } + case nil: + f.i = 0 default: return i18n.NewError(context.Background(), i18n.MsgScanFailed, src, f.i) } @@ -212,6 +234,8 @@ func (f *int64Field) Scan(src interface{}) (err error) { func (f *int64Field) Value() (driver.Value, error) { return f.i, nil } func (f *int64Field) String() string { return fmt.Sprintf("%d", f.i) } func (f *Int64Field) getSerialization() FieldSerialization { return &int64Field{} } +func (f *Int64Field) filterAsString() bool { return false } +func (f *Int64Field) description() string { return "Integer" } type TimeField struct{} type timeField struct{ t *fftypes.FFTime } @@ -223,7 +247,7 @@ func (f *timeField) Scan(src interface{}) (err error) { case int64: f.t = fftypes.UnixTime(tv) case string: - f.t, err = fftypes.ParseString(tv) + f.t, err = fftypes.ParseTimeString(tv) return err case fftypes.FFTime: f.t = &tv @@ -233,7 +257,6 @@ func (f *timeField) Scan(src interface{}) (err error) { return nil case nil: f.t = nil - return nil default: return i18n.NewError(context.Background(), i18n.MsgScanFailed, src, f.t) } @@ -247,6 +270,8 @@ func (f *timeField) Value() (driver.Value, error) { } func (f *timeField) String() string { return fmt.Sprintf("%v", f.t) } func (f *TimeField) getSerialization() FieldSerialization { return &timeField{} } +func (f *TimeField) filterAsString() bool { return false } +func (f *TimeField) description() string { return "Date-time" } type JSONField struct{} type jsonField struct{ b []byte } @@ -269,6 +294,8 @@ func (f *jsonField) Scan(src interface{}) (err error) { func (f *jsonField) Value() (driver.Value, error) { return f.b, nil } func (f *jsonField) String() string { return string(f.b) } func (f *JSONField) getSerialization() FieldSerialization { return &jsonField{} } +func (f *JSONField) filterAsString() bool { return true } +func (f *JSONField) description() string { return "JSON-blob" } type FFNameArrayField struct{} type ffNameArrayField struct{ na fftypes.FFNameArray } @@ -279,6 +306,8 @@ func (f *ffNameArrayField) Scan(src interface{}) (err error) { func (f *ffNameArrayField) Value() (driver.Value, error) { return f.na.String(), nil } func (f *ffNameArrayField) String() string { return f.na.String() } func (f *FFNameArrayField) getSerialization() FieldSerialization { return &ffNameArrayField{} } +func (f *FFNameArrayField) filterAsString() bool { return true } +func (f *FFNameArrayField) description() string { return "String-array" } type BoolField struct{} type boolField struct{ b bool } @@ -311,3 +340,5 @@ func (f *boolField) Scan(src interface{}) (err error) { func (f *boolField) Value() (driver.Value, error) { return f.b, nil } func (f *boolField) String() string { return fmt.Sprintf("%t", f.b) } func (f *BoolField) getSerialization() FieldSerialization { return &boolField{} } +func (f *BoolField) filterAsString() bool { return false } +func (f *BoolField) description() string { return "Boolean" } diff --git a/pkg/database/query_fields_test.go b/pkg/database/query_fields_test.go new file mode 100644 index 0000000000..82e5510098 --- /dev/null +++ b/pkg/database/query_fields_test.go @@ -0,0 +1,216 @@ +// Copyright © 2022 Kaleido, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package database + +import ( + "testing" + "time" + + "github.com/hyperledger/firefly/pkg/fftypes" + "github.com/stretchr/testify/assert" +) + +func TestNullField(t *testing.T) { + + f := nullField{} + v, err := f.Value() + assert.NoError(t, err) + assert.Nil(t, v) + + err = f.Scan("anything") + assert.NoError(t, err) + v, err = f.Value() + assert.NoError(t, err) + assert.Nil(t, v) + + assert.Equal(t, "null", f.String()) +} + +func TestStringField(t *testing.T) { + + fd := &StringField{} + assert.NotEmpty(t, fd.description()) + f := stringField{} + + err := f.Scan("test") + assert.NoError(t, err) + v, err := f.Value() + assert.NoError(t, err) + assert.Equal(t, "test", v) + + err = f.Scan(nil) + assert.NoError(t, err) + v, err = f.Value() + assert.NoError(t, err) + assert.Equal(t, "", v) + +} + +func TestUUIDField(t *testing.T) { + + fd := &UUIDField{} + assert.NotEmpty(t, fd.description()) + f := uuidField{} + + err := f.Scan("") + assert.NoError(t, err) + v, err := f.Value() + assert.NoError(t, err) + assert.Nil(t, v) + + u1 := fftypes.NewUUID() + err = f.Scan(u1.String()) + assert.NoError(t, err) + v, err = f.Value() + assert.NoError(t, err) + assert.Equal(t, v, u1.String()) + + err = f.Scan(nil) + assert.NoError(t, err) + v, err = f.Value() + assert.NoError(t, err) + assert.Nil(t, v) + +} + +func TestBytes32Field(t *testing.T) { + + fd := &Bytes32Field{} + assert.NotEmpty(t, fd.description()) + f := bytes32Field{} + + err := f.Scan("") + assert.NoError(t, err) + v, err := f.Value() + assert.NoError(t, err) + assert.Nil(t, v) + + b1 := fftypes.NewRandB32() + err = f.Scan(b1.String()) + assert.NoError(t, err) + v, err = f.Value() + assert.NoError(t, err) + assert.Equal(t, v, b1.String()) + + err = f.Scan(nil) + assert.NoError(t, err) + v, err = f.Value() + assert.NoError(t, err) + assert.Nil(t, v) + +} + +func TestInt64Field(t *testing.T) { + + fd := &Int64Field{} + assert.NotEmpty(t, fd.description()) + f := int64Field{} + + err := f.Scan("12345") + assert.NoError(t, err) + v, err := f.Value() + assert.NoError(t, err) + assert.Equal(t, int64(12345), v) + + err = f.Scan(nil) + assert.NoError(t, err) + v, err = f.Value() + assert.NoError(t, err) + assert.Equal(t, int64(0), v) + +} + +func TestTimeField(t *testing.T) { + + fd := &TimeField{} + assert.NotEmpty(t, fd.description()) + f := timeField{} + + now := time.Now() + err := f.Scan(now.Format(time.RFC3339Nano)) + assert.NoError(t, err) + v, err := f.Value() + assert.NoError(t, err) + assert.Equal(t, v, now.UnixNano()) + + err = f.Scan(nil) + assert.NoError(t, err) + v, err = f.Value() + assert.NoError(t, err) + assert.Nil(t, v) + +} + +func TestJSONField(t *testing.T) { + + fd := &JSONField{} + assert.NotEmpty(t, fd.description()) + f := jsonField{} + + err := f.Scan("{}") + assert.NoError(t, err) + v, err := f.Value() + assert.NoError(t, err) + assert.Equal(t, v, []byte("{}")) + + err = f.Scan(nil) + assert.NoError(t, err) + v, err = f.Value() + assert.NoError(t, err) + assert.Nil(t, v) + +} + +func TestBoolField(t *testing.T) { + + fd := &BoolField{} + assert.NotEmpty(t, fd.description()) + f := boolField{} + + err := f.Scan("true") + assert.NoError(t, err) + v, err := f.Value() + assert.NoError(t, err) + assert.True(t, v.(bool)) + + err = f.Scan(nil) + assert.NoError(t, err) + v, err = f.Value() + assert.NoError(t, err) + assert.False(t, v.(bool)) + +} + +func TestFFNameArrayField(t *testing.T) { + + fd := &FFNameArrayField{} + assert.NotEmpty(t, fd.description()) + f := ffNameArrayField{} + + err := f.Scan("a,b") + assert.NoError(t, err) + v, err := f.Value() + assert.NoError(t, err) + assert.Equal(t, v, "a,b") + + err = f.Scan(nil) + assert.NoError(t, err) + v, err = f.Value() + assert.NoError(t, err) + assert.Equal(t, "", v) + +} diff --git a/pkg/fftypes/config.go b/pkg/fftypes/config.go index ad83565a90..d316da8601 100644 --- a/pkg/fftypes/config.go +++ b/pkg/fftypes/config.go @@ -1,4 +1,4 @@ -// Copyright © 2021 Kaleido, Inc. +// Copyright © 2022 Kaleido, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -18,5 +18,5 @@ package fftypes type ConfigRecord struct { Key string `json:"key,omitempty"` - Value Byteable `json:"value,omitempty"` + Value *JSONAny `json:"value,omitempty"` } diff --git a/pkg/fftypes/data.go b/pkg/fftypes/data.go index e65d978894..59ec9387cd 100644 --- a/pkg/fftypes/data.go +++ b/pkg/fftypes/data.go @@ -44,7 +44,7 @@ type Data struct { Hash *Bytes32 `json:"hash,omitempty"` Created *FFTime `json:"created,omitempty"` Datatype *DatatypeRef `json:"datatype,omitempty"` - Value Byteable `json:"value"` + Value *JSONAny `json:"value"` Blob *BlobRef `json:"blob,omitempty"` } @@ -60,7 +60,7 @@ type DatatypeRef struct { func (dr *DatatypeRef) String() string { if dr == nil { - return nullString + return NullString } return fmt.Sprintf("%s/%s", dr.Name, dr.Version) } @@ -84,9 +84,9 @@ func CheckValidatorType(ctx context.Context, validator ValidatorType) error { func (d *Data) CalcHash(ctx context.Context) (*Bytes32, error) { if d.Value == nil { - d.Value = Byteable(nullString) + d.Value = JSONAnyPtr(NullString) } - valueIsNull := d.Value.String() == nullString + valueIsNull := d.Value.String() == NullString if valueIsNull && (d.Blob == nil || d.Blob.Hash == nil) { return nil, i18n.NewError(ctx, i18n.MsgDataValueIsNull) } diff --git a/pkg/fftypes/data_test.go b/pkg/fftypes/data_test.go index 4a2f487c17..ce7c7901ed 100644 --- a/pkg/fftypes/data_test.go +++ b/pkg/fftypes/data_test.go @@ -28,7 +28,7 @@ import ( func TestDatatypeReference(t *testing.T) { var dr *DatatypeRef - assert.Equal(t, nullString, dr.String()) + assert.Equal(t, NullString, dr.String()) dr = &DatatypeRef{ Name: "customer", Version: "0.0.1", @@ -50,7 +50,7 @@ func TestSealNoData(t *testing.T) { func TestSealValueOnly(t *testing.T) { d := &Data{ - Value: []byte("{}"), + Value: JSONAnyPtr("{}"), Blob: &BlobRef{}, } err := d.Seal(context.Background(), nil) @@ -78,7 +78,7 @@ func TestSealBlobExplictlyNamed(t *testing.T) { Blob: &BlobRef{ Hash: blobHash, }, - Value: Byteable(`{ + Value: JSONAnyPtr(`{ "name": "use this", "filename": "ignore this", "path": "ignore this too" @@ -98,7 +98,7 @@ func TestSealBlobPathNamed(t *testing.T) { Blob: &BlobRef{ Hash: blobHash, }, - Value: Byteable(`{ + Value: JSONAnyPtr(`{ "filename": "file.ext", "path": "/path/to" }`), @@ -117,7 +117,7 @@ func TestSealBlobFileNamed(t *testing.T) { Blob: &BlobRef{ Hash: blobHash, }, - Value: Byteable(`{ + Value: JSONAnyPtr(`{ "filename": "file.ext" }`), } @@ -139,7 +139,7 @@ func TestSealBlobMismatch1(t *testing.T) { err := d.Seal(context.Background(), &Blob{ Hash: NewRandB32(), }) - assert.Regexp(t, "FF10303", err) + assert.Regexp(t, "FF10304", err) } func TestSealBlobMismatch2(t *testing.T) { @@ -147,7 +147,7 @@ func TestSealBlobMismatch2(t *testing.T) { Blob: &BlobRef{Hash: NewRandB32()}, } err := d.Seal(context.Background(), nil) - assert.Regexp(t, "FF10303", err) + assert.Regexp(t, "FF10304", err) } func TestSealBlobAndHashOnly(t *testing.T) { @@ -156,7 +156,7 @@ func TestSealBlobAndHashOnly(t *testing.T) { Blob: &BlobRef{ Hash: blobHash, }, - Value: []byte("{}"), + Value: JSONAnyPtr("{}"), } h := sha256.Sum256([]byte(`44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a22440fcf4ee9ac8c1a83de36c3a9ef39f838d960971dc79b274718392f1735f9`)) err := d.Seal(context.Background(), &Blob{ diff --git a/pkg/fftypes/datatype.go b/pkg/fftypes/datatype.go index ccd04585e6..60da473f09 100644 --- a/pkg/fftypes/datatype.go +++ b/pkg/fftypes/datatype.go @@ -1,4 +1,4 @@ -// Copyright © 2021 Kaleido, Inc. +// Copyright © 2022 Kaleido, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -43,7 +43,7 @@ type Datatype struct { Version string `json:"version,omitempty"` Hash *Bytes32 `json:"hash,omitempty"` Created *FFTime `json:"created,omitempty"` - Value Byteable `json:"value,omitempty"` + Value *JSONAny `json:"value,omitempty"` } func (dt *Datatype) Validate(ctx context.Context, existing bool) (err error) { @@ -59,7 +59,7 @@ func (dt *Datatype) Validate(ctx context.Context, existing bool) (err error) { if err = ValidateFFNameField(ctx, dt.Version, "version"); err != nil { return err } - if len(dt.Value) == 0 { + if dt.Value == nil || len(*dt.Value) == 0 { return i18n.NewError(ctx, i18n.MsgMissingRequiredField, "value") } if existing { diff --git a/pkg/fftypes/datatype_test.go b/pkg/fftypes/datatype_test.go index e4a238cd3f..f325fcc8fd 100644 --- a/pkg/fftypes/datatype_test.go +++ b/pkg/fftypes/datatype_test.go @@ -64,7 +64,7 @@ func TestDatatypeValidation(t *testing.T) { Namespace: "ok", Name: "ok", Version: "ok", - Value: Byteable(`{}`), + Value: JSONAnyPtr(`{}`), } assert.NoError(t, dt.Validate(context.Background(), false)) diff --git a/pkg/fftypes/byteable.go b/pkg/fftypes/jsonany.go similarity index 67% rename from pkg/fftypes/byteable.go rename to pkg/fftypes/jsonany.go index 7b4edf1316..cf13c1ad1e 100644 --- a/pkg/fftypes/byteable.go +++ b/pkg/fftypes/jsonany.go @@ -26,44 +26,66 @@ import ( ) const ( - nullString = "null" + NullString = "null" ) -// Byteable uses raw encode/decode to preserve field order, and can handle any types of field. +// JSONAny uses raw encode/decode to preserve field order, and can handle any types of field. // It validates the JSON can be unmarshalled, but does not change the order. // It does however trim out whitespace -type Byteable []byte +type JSONAny string -func (h *Byteable) UnmarshalJSON(b []byte) error { +func JSONAnyPtr(str string) *JSONAny { + return (*JSONAny)(&str) +} + +func JSONAnyPtrBytes(b []byte) *JSONAny { + if b == nil { + return nil + } + ja := JSONAny(b) + return &ja +} + +func (h *JSONAny) UnmarshalJSON(b []byte) error { var flattener json.RawMessage err := json.Unmarshal(b, &flattener) if err != nil { return err } - *h, err = json.Marshal(flattener) + standardizedBytes, err := json.Marshal(flattener) + if err == nil { + *h = JSONAny(standardizedBytes) + } return err } -func (h Byteable) MarshalJSON() ([]byte, error) { - if h == nil { - return []byte(nullString), nil +func (h JSONAny) MarshalJSON() ([]byte, error) { + if h == "" { + h = NullString } - return h, nil + return []byte(h), nil } -func (h Byteable) Hash() *Bytes32 { +func (h JSONAny) Hash() *Bytes32 { var b32 Bytes32 = sha256.Sum256([]byte(h)) return &b32 } -func (h Byteable) String() string { +func (h JSONAny) String() string { b, _ := h.MarshalJSON() return string(b) } -func (h Byteable) JSONObjectOk(noWarn ...bool) (JSONObject, bool) { +func (h *JSONAny) Bytes() []byte { + if h == nil { + return nil + } + return []byte(*h) +} + +func (h JSONAny) JSONObjectOk(noWarn ...bool) (JSONObject, bool) { var jo JSONObject - err := json.Unmarshal(h, &jo) + err := json.Unmarshal([]byte(h), &jo) if err != nil { if len(noWarn) == 0 || !noWarn[0] { log.L(context.Background()).Warnf("Unable to deserialize as JSON object: %s", string(h)) @@ -76,24 +98,23 @@ func (h Byteable) JSONObjectOk(noWarn ...bool) (JSONObject, bool) { // JSONObject attempts to de-serailize the contained structure as a JSON Object (map) // Safe and will never return nil // Will return an empty object if the type is array, string, bool, number etc. -func (h Byteable) JSONObject() JSONObject { +func (h JSONAny) JSONObject() JSONObject { jo, _ := h.JSONObjectOk() return jo } // JSONObjectNowarn acts the same as JSONObject, but does not warn if the value cannot // be parsed as an object -func (h Byteable) JSONObjectNowarn() JSONObject { +func (h JSONAny) JSONObjectNowarn() JSONObject { jo, _ := h.JSONObjectOk(true) return jo } // Scan implements sql.Scanner -func (h *Byteable) Scan(src interface{}) error { +func (h *JSONAny) Scan(src interface{}) error { switch src := src.(type) { case nil: - nullVal := []byte(nullString) - *h = nullVal + *h = NullString return nil case []byte: return h.UnmarshalJSON(src) diff --git a/pkg/fftypes/byteable_test.go b/pkg/fftypes/jsonany_test.go similarity index 79% rename from pkg/fftypes/byteable_test.go rename to pkg/fftypes/jsonany_test.go index 3c5f848653..292e23d7a0 100644 --- a/pkg/fftypes/byteable_test.go +++ b/pkg/fftypes/jsonany_test.go @@ -23,11 +23,11 @@ import ( "github.com/stretchr/testify/assert" ) -func TestByteableSerializeNull(t *testing.T) { +func TestJSONAnySerializeNull(t *testing.T) { type testStruct struct { - Prop1 *Byteable `json:"prop1"` - Prop2 *Byteable `json:"prop2,omitempty"` + Prop1 *JSONAny `json:"prop1"` + Prop2 *JSONAny `json:"prop2,omitempty"` } ts := &testStruct{} @@ -41,10 +41,10 @@ func TestByteableSerializeNull(t *testing.T) { } -func TestByteableSerializeObjects(t *testing.T) { +func TestJSONAnySerializeObjects(t *testing.T) { type testStruct struct { - Prop *Byteable `json:"prop,omitempty"` + Prop *JSONAny `json:"prop,omitempty"` } ts := &testStruct{} @@ -72,17 +72,17 @@ func TestByteableSerializeObjects(t *testing.T) { } -func TestByteableMarshalNull(t *testing.T) { +func TestJSONAnyMarshalNull(t *testing.T) { - var pb Byteable + var pb JSONAny b, err := pb.MarshalJSON() assert.NoError(t, err) - assert.Equal(t, nullString, string(b)) + assert.Equal(t, NullString, string(b)) } -func TestByteableUnmarshalFail(t *testing.T) { +func TestJSONAnyUnmarshalFail(t *testing.T) { - var b Byteable + var b JSONAny err := b.UnmarshalJSON([]byte(`!json`)) assert.Error(t, err) @@ -92,9 +92,9 @@ func TestByteableUnmarshalFail(t *testing.T) { func TestScan(t *testing.T) { - var h Byteable + var h JSONAny assert.NoError(t, h.Scan(nil)) - assert.Equal(t, []byte(nullString), []byte(h)) + assert.Equal(t, []byte(NullString), []byte(h)) assert.NoError(t, h.Scan(`{"some": "stuff"}`)) assert.Equal(t, "stuff", h.JSONObject().GetString("some")) @@ -107,4 +107,10 @@ func TestScan(t *testing.T) { assert.Regexp(t, "FF10125", h.Scan(12345)) + assert.Equal(t, "test", JSONAnyPtrBytes([]byte(`{"val": "test"}`)).JSONObject().GetString("val")) + assert.Nil(t, JSONAnyPtrBytes(nil)) + + assert.Nil(t, JSONAnyPtrBytes(nil).Bytes()) + assert.NotEmpty(t, JSONAnyPtr("{}").Bytes()) + } diff --git a/pkg/fftypes/jsondata.go b/pkg/fftypes/jsonobject.go similarity index 84% rename from pkg/fftypes/jsondata.go rename to pkg/fftypes/jsonobject.go index 489cae0d37..0b8729e06b 100644 --- a/pkg/fftypes/jsondata.go +++ b/pkg/fftypes/jsonobject.go @@ -1,4 +1,4 @@ -// Copyright © 2021 Kaleido, Inc. +// Copyright © 2022 Kaleido, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -183,42 +183,3 @@ func (jd JSONObject) Hash(jsonDesc string) (*Bytes32, error) { var b32 Bytes32 = sha256.Sum256(b) return &b32, nil } - -// JSONObjectArray is an array of JSONObject -type JSONObjectArray []JSONObject - -// Scan implements sql.Scanner -func (jd *JSONObjectArray) Scan(src interface{}) error { - switch src := src.(type) { - case nil: - return nil - - case string, []byte: - if src == "" { - return nil - } - return json.Unmarshal(src.([]byte), &jd) - - default: - return i18n.NewError(context.Background(), i18n.MsgScanFailed, src, jd) - } - -} - -func (jd JSONObjectArray) Value() (driver.Value, error) { - return json.Marshal(&jd) -} - -func (jd JSONObjectArray) String() string { - b, _ := json.Marshal(&jd) - return string(b) -} - -func (jd JSONObjectArray) Hash(jsonDesc string) (*Bytes32, error) { - b, err := json.Marshal(&jd) - if err != nil { - return nil, i18n.NewError(context.Background(), i18n.MsgJSONObjectParseFailed, jsonDesc) - } - var b32 Bytes32 = sha256.Sum256(b) - return &b32, nil -} diff --git a/pkg/fftypes/jsondata_test.go b/pkg/fftypes/jsonobject_test.go similarity index 88% rename from pkg/fftypes/jsondata_test.go rename to pkg/fftypes/jsonobject_test.go index 8fb50ae6ac..52501cf873 100644 --- a/pkg/fftypes/jsondata_test.go +++ b/pkg/fftypes/jsonobject_test.go @@ -84,26 +84,6 @@ func TestJSONObject(t *testing.T) { assert.Equal(t, "", v) } -func TestJSONObjectArray(t *testing.T) { - - data := Byteable(`{ - "field1": true, - "field2": false, - "field3": "True", - "field4": "not true", - "field5": { "not": "boolable" }, - "field6": null - }`) - dataJSON := data.JSONObject() - assert.True(t, dataJSON.GetBool("field1")) - assert.False(t, dataJSON.GetBool("field2")) - assert.True(t, dataJSON.GetBool("field3")) - assert.False(t, dataJSON.GetBool("field4")) - assert.False(t, dataJSON.GetBool("field5")) - assert.False(t, dataJSON.GetBool("field6")) - assert.False(t, dataJSON.GetBool("field7")) -} - func TestJSONObjectBool(t *testing.T) { data := JSONObjectArray{ @@ -112,7 +92,7 @@ func TestJSONObjectBool(t *testing.T) { b, err := data.Value() assert.NoError(t, err) - assert.IsType(t, []byte{}, b) + assert.Equal(t, "[{\"some\":\"data\"}]", b) var dataRead JSONObjectArray err = dataRead.Scan(b) diff --git a/pkg/fftypes/jsonobjectarray.go b/pkg/fftypes/jsonobjectarray.go new file mode 100644 index 0000000000..c24cf12662 --- /dev/null +++ b/pkg/fftypes/jsonobjectarray.go @@ -0,0 +1,78 @@ +// Copyright © 2022 Kaleido, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fftypes + +import ( + "context" + "crypto/sha256" + "database/sql/driver" + "encoding/json" + + "github.com/hyperledger/firefly/internal/i18n" +) + +// JSONObjectArray is an array of JSONObject +type JSONObjectArray []JSONObject + +// Scan implements sql.Scanner +func (jd *JSONObjectArray) Scan(src interface{}) error { + switch src := src.(type) { + case nil: + *jd = JSONObjectArray{} + return nil + + case []byte: + if src == nil { + *jd = JSONObjectArray{} + return nil + } + return json.Unmarshal(src, &jd) + + case string: + if src == "" { + *jd = JSONObjectArray{} + return nil + } + return json.Unmarshal([]byte(src), &jd) + + default: + return i18n.NewError(context.Background(), i18n.MsgScanFailed, src, jd) + } + +} + +func (jd JSONObjectArray) Value() (driver.Value, error) { + b, err := json.Marshal(&jd) + if err != nil { + return nil, err + } + return string(b), err +} + +func (jd JSONObjectArray) String() string { + b, _ := json.Marshal(&jd) + return string(b) +} + +func (jd JSONObjectArray) Hash(jsonDesc string) (*Bytes32, error) { + b, err := json.Marshal(&jd) + if err != nil { + return nil, i18n.NewError(context.Background(), i18n.MsgJSONObjectParseFailed, jsonDesc) + } + var b32 Bytes32 = sha256.Sum256(b) + return &b32, nil +} diff --git a/pkg/fftypes/jsonobjectarray_test.go b/pkg/fftypes/jsonobjectarray_test.go new file mode 100644 index 0000000000..2329ff6cf3 --- /dev/null +++ b/pkg/fftypes/jsonobjectarray_test.go @@ -0,0 +1,77 @@ +// Copyright © 2021 Kaleido, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fftypes + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestJSONObjectArray(t *testing.T) { + + data := JSONAnyPtr(`{ + "field1": true, + "field2": false, + "field3": "True", + "field4": "not true", + "field5": { "not": "boolable" }, + "field6": null + }`) + dataJSON := data.JSONObject() + assert.True(t, dataJSON.GetBool("field1")) + assert.False(t, dataJSON.GetBool("field2")) + assert.True(t, dataJSON.GetBool("field3")) + assert.False(t, dataJSON.GetBool("field4")) + assert.False(t, dataJSON.GetBool("field5")) + assert.False(t, dataJSON.GetBool("field6")) + assert.False(t, dataJSON.GetBool("field7")) +} + +func TestJSONObjectArrayScan(t *testing.T) { + + var joa JSONObjectArray + + err := joa.Scan(`[{"test": 1}]`) + assert.NoError(t, err) + assert.Equal(t, "1", joa[0].GetString("test")) + + err = joa.Scan([]byte(`[{"test": 1}]`)) + assert.NoError(t, err) + assert.Equal(t, "1", joa[0].GetString("test")) + + err = joa.Scan(nil) + assert.NoError(t, err) + assert.Empty(t, joa) + + err = joa.Scan("") + assert.NoError(t, err) + assert.Empty(t, joa) + + err = joa.Scan([]byte(nil)) + assert.NoError(t, err) + assert.Empty(t, joa) + + joa = JSONObjectArray([]JSONObject{ + JSONObject(map[string]interface{}{ + "bad": map[bool]bool{false: true}, + }), + }) + _, err = joa.Value() + assert.Error(t, err) + +} diff --git a/pkg/fftypes/message.go b/pkg/fftypes/message.go index bac967494b..28b3f697a8 100644 --- a/pkg/fftypes/message.go +++ b/pkg/fftypes/message.go @@ -1,4 +1,4 @@ -// Copyright © 2021 Kaleido, Inc. +// Copyright © 2022 Kaleido, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -118,7 +118,7 @@ type DataRefOrValue struct { Validator ValidatorType `json:"validator,omitempty"` Datatype *DatatypeRef `json:"datatype,omitempty"` - Value Byteable `json:"value,omitempty"` + Value *JSONAny `json:"value,omitempty"` Blob *BlobRef `json:"blob,omitempty"` } diff --git a/pkg/fftypes/message_test.go b/pkg/fftypes/message_test.go index f13132ceb2..63101c4833 100644 --- a/pkg/fftypes/message_test.go +++ b/pkg/fftypes/message_test.go @@ -202,7 +202,7 @@ func TestSealKnownMessage(t *testing.T) { func TestSetInlineData(t *testing.T) { msg := &MessageInOut{} msg.SetInlineData([]*Data{ - {ID: NewUUID(), Value: Byteable(`"some data"`)}, + {ID: NewUUID(), Value: JSONAnyPtr(`"some data"`)}, }) b, err := json.Marshal(&msg) assert.NoError(t, err) diff --git a/pkg/fftypes/namearray.go b/pkg/fftypes/namearray.go index c3e7b9089d..2bda91ace6 100644 --- a/pkg/fftypes/namearray.go +++ b/pkg/fftypes/namearray.go @@ -1,4 +1,4 @@ -// Copyright © 2021 Kaleido, Inc. +// Copyright © 2022 Kaleido, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -59,6 +59,7 @@ func (na *FFNameArray) Scan(src interface{}) error { *na = st return nil case nil: + *na = []string{} return nil default: return i18n.NewError(context.Background(), i18n.MsgScanFailed, src, na) diff --git a/pkg/fftypes/timeutils.go b/pkg/fftypes/timeutils.go index 92e2ae13a3..cfb40d212b 100644 --- a/pkg/fftypes/timeutils.go +++ b/pkg/fftypes/timeutils.go @@ -1,4 +1,4 @@ -// Copyright © 2021 Kaleido, Inc. +// Copyright © 2022 Kaleido, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -67,7 +67,7 @@ func (ft *FFTime) MarshalJSON() ([]byte, error) { return json.Marshal(ft.String()) } -func ParseString(str string) (*FFTime, error) { +func ParseTimeString(str string) (*FFTime, error) { t, err := time.Parse(time.RFC3339Nano, str) if err != nil { var unixTime int64 @@ -96,7 +96,7 @@ func (ft *FFTime) UnixNano() int64 { } func (ft *FFTime) UnmarshalText(b []byte) error { - t, err := ParseString(string(b)) + t, err := ParseTimeString(string(b)) if err != nil { return err } @@ -112,7 +112,7 @@ func (ft *FFTime) Scan(src interface{}) error { return nil case string: - t, err := ParseString(src) + t, err := ParseTimeString(src) if err != nil { return err } diff --git a/test/e2e/onchain_offchain_test.go b/test/e2e/onchain_offchain_test.go index 15d20a66ec..ad30c6b2c4 100644 --- a/test/e2e/onchain_offchain_test.go +++ b/test/e2e/onchain_offchain_test.go @@ -50,7 +50,7 @@ func (suite *OnChainOffChainTestSuite) TestE2EBroadcast() { received2, changes2 := wsReader(suite.T(), suite.testState.ws2) var resp *resty.Response - value := fftypes.Byteable(`"Hello"`) + value := fftypes.JSONAnyPtr(`"Hello"`) data := fftypes.DataRefOrValue{ Value: value, } @@ -78,7 +78,7 @@ func (suite *OnChainOffChainTestSuite) TestStrongDatatypesBroadcast() { received2, changes2 := wsReader(suite.T(), suite.testState.ws2) var resp *resty.Response - value := fftypes.Byteable(`"Hello"`) + value := fftypes.JSONAnyPtr(`"Hello"`) randVer, _ := rand.Int(rand.Reader, big.NewInt(100000000)) version := fmt.Sprintf("0.0.%d", randVer.Int64()) data := fftypes.DataRefOrValue{ @@ -98,7 +98,7 @@ func (suite *OnChainOffChainTestSuite) TestStrongDatatypesBroadcast() { dt := &fftypes.Datatype{ Name: "widget", Version: version, - Value: widgetSchemaJSON, + Value: fftypes.JSONAnyPtrBytes(widgetSchemaJSON), } dt = CreateDatatype(suite.T(), suite.testState.client1, dt, true) @@ -107,7 +107,7 @@ func (suite *OnChainOffChainTestSuite) TestStrongDatatypesBroadcast() { assert.Equal(suite.T(), 400, resp.StatusCode()) assert.Contains(suite.T(), resp.String(), "FF10198") // does not conform - data.Value = fftypes.Byteable(`{ + data.Value = fftypes.JSONAnyPtr(`{ "id": "widget12345", "name": "mywidget" }`) @@ -129,7 +129,7 @@ func (suite *OnChainOffChainTestSuite) TestStrongDatatypesPrivate() { received2, changes2 := wsReader(suite.T(), suite.testState.ws2) var resp *resty.Response - value := fftypes.Byteable(`{"foo":"bar"}`) + value := fftypes.JSONAnyPtr(`{"foo":"bar"}`) randVer, _ := rand.Int(rand.Reader, big.NewInt(100000000)) version := fmt.Sprintf("0.0.%d", randVer.Int64()) data := fftypes.DataRefOrValue{ @@ -152,7 +152,7 @@ func (suite *OnChainOffChainTestSuite) TestStrongDatatypesPrivate() { dt := &fftypes.Datatype{ Name: "widget", Version: version, - Value: widgetSchemaJSON, + Value: fftypes.JSONAnyPtrBytes(widgetSchemaJSON), } dt = CreateDatatype(suite.T(), suite.testState.client1, dt, true) @@ -164,7 +164,7 @@ func (suite *OnChainOffChainTestSuite) TestStrongDatatypesPrivate() { assert.Equal(suite.T(), 400, resp.StatusCode()) assert.Contains(suite.T(), resp.String(), "FF10198") // does not conform - data.Value = fftypes.Byteable(`{ + data.Value = fftypes.JSONAnyPtr(`{ "id": "widget12345", "name": "mywidget" }`) @@ -189,7 +189,7 @@ func (suite *OnChainOffChainTestSuite) TestE2EPrivate() { received2, _ := wsReader(suite.T(), suite.testState.ws2) var resp *resty.Response - value := fftypes.Byteable(`"Hello"`) + value := fftypes.JSONAnyPtr(`"Hello"`) data := fftypes.DataRefOrValue{ Value: value, } @@ -224,13 +224,13 @@ func (suite *OnChainOffChainTestSuite) TestE2EBroadcastBlob() { waitForMessageConfirmed(suite.T(), received1, fftypes.MessageTypeBroadcast) val1 := validateReceivedMessages(suite.testState, suite.testState.client1, fftypes.MessageTypeBroadcast, fftypes.TransactionTypeBatchPin, 1, 0) - assert.Regexp(suite.T(), "myfile.txt", string(val1.Value)) + assert.Regexp(suite.T(), "myfile.txt", val1.Value.String()) assert.Equal(suite.T(), "myfile.txt", val1.Blob.Name) assert.Equal(suite.T(), data.Blob.Size, val1.Blob.Size) waitForMessageConfirmed(suite.T(), received2, fftypes.MessageTypeBroadcast) val2 := validateReceivedMessages(suite.testState, suite.testState.client2, fftypes.MessageTypeBroadcast, fftypes.TransactionTypeBatchPin, 1, 0) - assert.Regexp(suite.T(), "myfile.txt", string(val2.Value)) + assert.Regexp(suite.T(), "myfile.txt", val2.Value.String()) assert.Equal(suite.T(), "myfile.txt", val2.Blob.Name) assert.Equal(suite.T(), data.Blob.Size, val2.Blob.Size) @@ -292,7 +292,7 @@ func (suite *OnChainOffChainTestSuite) TestE2EWebhookExchange() { assert.NotNil(suite.T(), sub.ID) data := fftypes.DataRefOrValue{ - Value: fftypes.Byteable(`{}`), + Value: fftypes.JSONAnyPtr(`{}`), } var resp *resty.Response @@ -345,7 +345,7 @@ func (suite *OnChainOffChainTestSuite) TestE2EWebhookRequestReplyNoTx() { assert.NotNil(suite.T(), sub.ID) data := fftypes.DataRefOrValue{ - Value: fftypes.Byteable(`{}`), + Value: fftypes.JSONAnyPtr(`{}`), } reply := RequestReply(suite.T(), suite.testState.client1, &data, []string{ diff --git a/test/e2e/tokens_test.go b/test/e2e/tokens_test.go index f50566ff95..7e5727547f 100644 --- a/test/e2e/tokens_test.go +++ b/test/e2e/tokens_test.go @@ -107,7 +107,7 @@ func (suite *TokensTestSuite) TestE2EFungibleTokensAsync() { Message: &fftypes.MessageInOut{ InlineData: fftypes.InlineData{ { - Value: fftypes.Byteable(`"payment for data"`), + Value: fftypes.JSONAnyPtr(`"payment for data"`), }, }, }, @@ -238,7 +238,7 @@ func (suite *TokensTestSuite) TestE2ENonFungibleTokensSync() { Message: &fftypes.MessageInOut{ InlineData: fftypes.InlineData{ { - Value: fftypes.Byteable(`"ownership change"`), + Value: fftypes.JSONAnyPtr(`"ownership change"`), }, }, },