Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: support contains to compare json objects #3687

Merged
merged 6 commits into from
Feb 26, 2024
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
24 changes: 24 additions & 0 deletions docs/docs/concepts/expressions.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ Tracetest allows you to add expressions when writing your tests. They are a nice
* Arithmetic operations
* String interpolation
* Filters
* JSON comparison

### Reference Span Attributes

Expand Down Expand Up @@ -139,3 +140,26 @@ attr:myapp.operations | get_index 2 | type = "string"
# check if attribute is either a number of a string
["number", "string"] contains attr:my_attribute | type
```

### JSON Comparison

When working with APIs, it's very common for developers to want to check if the response contains some data while ignoring all the noise a response can have. For this, you can use the `contains` comparator in JSON objects. It works just like [Jest's toMatchObject](https://jestjs.io/pt-BR/docs/expect#tomatchobjectobject).
mathnogueira marked this conversation as resolved.
Show resolved Hide resolved

The order of the attributes doesn't matter and the left side of the expression can contain more attributes than the right side.

Some examples:

```css
# If user is 32 years old, it passes
'{"name": "john", "age": 32}' contains '{"age": 32}'
```

```css
# If any of the users is called "maria", it passes
'[{"name": "john", "age": 32}, {"name": "maria", "age": 63}]' contains '{"name": "maria"}'
```

```css
# In this case, both ages must be part of the JSON array
'[{"name": "john", "age": 32}, {"name": "maria", "age": 63}]' contains '[{"age": 63}, {"age": 32}]'
```
100 changes: 100 additions & 0 deletions server/assertions/comparator/basic.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package comparator

import (
"encoding/json"
"fmt"
"strconv"
"strings"
Expand All @@ -18,6 +19,7 @@ var (
NotContains,
StartsWith,
EndsWith,
JsonContains,
}
)

Expand Down Expand Up @@ -228,3 +230,101 @@ func (c endsWith) Compare(expected, actual string) error {
func (c endsWith) String() string {
return "endsWith"
}

// JsonContains
var JsonContains Comparator = jsonContains{}

type jsonContains struct{}

func (c jsonContains) Compare(right, left string) error {
supersetMap, err := c.parseJson(left)
if err != nil {
return fmt.Errorf("left side is not a JSON object")
}

subsetMap, err := c.parseJson(right)
if err != nil {
return fmt.Errorf("left side is not a JSON object")
}

return c.compare(supersetMap, subsetMap)
}

func (c jsonContains) parseJson(input string) ([]map[string]any, error) {
if strings.HasPrefix(input, "[") && strings.HasSuffix(input, "]") {
output := []map[string]any{}
err := json.Unmarshal([]byte(input), &output)
if err != nil {
return []map[string]any{}, fmt.Errorf("invalid JSON array")
}

return output, nil
}

object := map[string]any{}
err := json.Unmarshal([]byte(input), &object)
if err != nil {
return []map[string]any{}, fmt.Errorf("invalid JSON object")
}

return []map[string]any{object}, nil
}

func (c jsonContains) compare(left, right []map[string]any) error {
for i, expected := range right {
err := c.anyMatches(left, expected)
if err != nil {
return fmt.Errorf("left side array doesn't match item %d of right side: %w", i, ErrNoMatch)
}
}

return nil
}

func (c jsonContains) anyMatches(array []map[string]any, expected map[string]any) error {
for _, left := range array {
err := c.compareObjects(left, expected)
if err == nil {
return nil
}
}

return ErrNoMatch
}

func (c jsonContains) compareObjects(left, right map[string]any) error {
for key, value := range right {
leftValue, ok := left[key]
if !ok {
return fmt.Errorf(`field "%s" not found on left side: %w`, key, ErrNoMatch)
}

if leftValueMap, ok := leftValue.(map[string]any); ok {
rightValueMap, ok := value.(map[string]any)
if !ok {
return fmt.Errorf(`field: "%s": %w`, key, ErrWrongType)
}

err := c.compareObjects(leftValueMap, rightValueMap)
if err != nil {
return fmt.Errorf(`field "%s": %w`, key, err)
}

return nil
}

// This minimizes the change of a panic due to comparing two []interface{} types
leftValueString, _ := json.Marshal(leftValue)
rightValueString, _ := json.Marshal(value)

if string(leftValueString) != string(rightValueString) {
return fmt.Errorf(`field "%s": %w`, key, ErrNoMatch)
}
}

return nil
}

func (c jsonContains) String() string {
return "json-contains"
}
5 changes: 3 additions & 2 deletions server/assertions/comparator/comparators.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,9 @@ type Comparator interface {
}

var (
ErrNoMatch = fmt.Errorf("no match")
ErrNotFound = fmt.Errorf("not found")
ErrNoMatch = fmt.Errorf("no match")
ErrNotFound = fmt.Errorf("not found")
ErrWrongType = fmt.Errorf("wrong type")
)

type Registry interface {
Expand Down
14 changes: 8 additions & 6 deletions server/expression/comparison.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,17 @@ import (
"fmt"

"github.com/kubeshop/tracetest/server/assertions/comparator"
"github.com/kubeshop/tracetest/server/expression/types"
"github.com/kubeshop/tracetest/server/expression/value"
)

func compare(comparatorName string, leftValue, rightValue value.Value) error {
if rightValue.Value().Type == types.TypeJson && comparatorName == "contains" {
if leftValue.Value().Type == types.TypeJson || leftValue.Value().Type == types.TypeArray {
comparatorName = "json-contains"
}
}
Comment on lines +12 to +16
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I thought this was cleaner to write than a longer if statement containing all four conditions.


comparatorFunction, err := comparator.DefaultRegistry().Get(comparatorName)
if err != nil {
return fmt.Errorf("comparator not supported: %w", err)
Expand All @@ -17,12 +24,7 @@ func compare(comparatorName string, leftValue, rightValue value.Value) error {
return compareArrayContains(leftValue, rightValue)
}

err = comparatorFunction.Compare(rightValue.String(), leftValue.String())
if err == comparator.ErrNoMatch {
return ErrNoMatch
}

return nil
return comparatorFunction.Compare(rightValue.String(), leftValue.String())
}

func compareArrayContains(array, expected value.Value) error {
Expand Down
63 changes: 63 additions & 0 deletions server/expression/executor_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -373,6 +373,69 @@ func TestArrayExecution(t *testing.T) {
Query: `[31,35,39] contains 42`,
ShouldPass: false,
},
{
Name: "should_identify_array_instead_of_json",
Query: `["{}", "{}", "{}"] | type = "array"`,
ShouldPass: true,
},
}

executeTestCases(t, testCases)
}

func TestJSONExecution(t *testing.T) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does it make sense to add a comparison for escaped JSON? Something like:

'{"name": "john", "age": 32, "email": "john@company.com"}' contains '{\"email\": \"john@company.com\" }'

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Escaping is not stable in the engine. As this is a problem in the core of the engine and not in the feature, I'll skip this and wait for users to ask for a fix. Not sure if that will happen.

testCases := []executorTestCase{
{
Name: "should_identify_json_input",
Query: `'{"name": "john", "age": 32, "email": "john@company.com"}' | type = "json"`,
ShouldPass: true,
},
{
Name: "should_be_able_to_compare_with_subset",
Query: `'{"name": "john", "age": 32, "email": "john@company.com"}' contains '{"email": "john@company.com", "name": "john"}'`,
ShouldPass: true,
},
Comment on lines +393 to +397
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would add another case here as a sanity check: comparing the same object, but with different attribute orders, like:

{"name": "john", "age": 32, "email": "john@company.com"}' contains '{"email": "john@company.com", "name": "john", "age": "32"}

{
Name: "should_be_able_to_compare_with_subset_ignoring_order",
Query: `'{"name": "john", "age": 32, "email": "john@company.com"}' contains '{"email": "john@company.com", "name": "john", "age": 32}'`,
ShouldPass: true,
},
{
Name: "should_be_able_to_compare_deep_objects_in_subset",
Query: `'{"name": "john", "age": 32, "email": "john@company.com", "company": {"name": "Company", "address": "1234 Agora Street"}}' contains '{"email": "john@company.com", "name": "john", "company": {"name": "Company"}}'`,
ShouldPass: true,
Comment on lines +404 to +406
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe a test with deep objects and arrays?

{"name": "john", "age": 32, "email": "john@company.com", "company": {"name": "Company", "address": "1234 Agora Street", "telephones": ["01", "02", "03"]}}' contains '{"email": "john@company.com", "name": "john", "company": {"name": "Company", "telephones": ["01", "02", "03"]}}

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added

},
{
Name: "should_be_able_to_compare_array_of_json_objects",
Query: `'[{"name": "john", "age": 32}, {"name": "Maria", "age": 63}]' contains '[{"age": 63}, {"age": 32}]'`,
ShouldPass: true,
},
{
Name: "should_match_complete_arrays",
Query: `'{"numbers": [0,1,2,3,4]}' contains '{"numbers": [0,1,2,3,4]}'`,
ShouldPass: true,
},
Comment on lines +413 to +417
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Based on this case, I would add a case to test arrays that seems to be equal, but doesn't, like:

{"numbers": [0,1,2,3,4]}' contains '{"numbers": [0,1,"2","3",4]}

{
Name: "should_fail_when_array_doesnt_match_size_and_order",
Query: `'{"numbers": [0,1,2,3,4]}' contains '{"numbers": [0,1]}'`,
ShouldPass: false,
},
{
Name: "should_fail_when_array_contains_same_elements_but_different_types",
Query: `'{"numbers": [0,1,2,3,4]}' contains '{"numbers": [0,1,"2","3",4]}'`,
ShouldPass: false,
},
{
Name: "should_identify_json_input_from_attribute",
Query: `attr:tracetest.response.body contains '{"name": "john"}'`,
ShouldPass: true,
AttributeDataStore: expression.AttributeDataStore{
Span: traces.Span{
ID: id.NewRandGenerator().SpanID(),
Attributes: traces.NewAttributes().Set("tracetest.response.body", `{"name": "john", "age": 32, "email": "john@company.com"}`),
},
},
},
}

executeTestCases(t, testCases)
Expand Down
16 changes: 15 additions & 1 deletion server/expression/types/types.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
package types

import "regexp"
import (
"encoding/json"
"regexp"
)

type Type uint

Expand All @@ -12,11 +15,13 @@ const (
TypeDuration
TypeVariable
TypeArray
TypeJson
)

var typeNames = map[Type]string{
TypeNil: "nil",
TypeString: "string",
TypeJson: "json",
TypeNumber: "number",
TypeAttribute: "attribute",
TypeDuration: "duration",
Expand All @@ -37,7 +42,16 @@ func GetType(value string) Type {
return TypeDuration
}

if err := json.Unmarshal([]byte(value), &map[string]any{}); err == nil {
return TypeJson
}

if arrayRegex.Match([]byte(value)) {
var jsonArray = []map[string]any{}
if err := json.Unmarshal([]byte(value), &jsonArray); err == nil {
return TypeJson
}

return TypeArray
}

Expand Down
4 changes: 3 additions & 1 deletion server/traces/span_entitiess.go
Original file line number Diff line number Diff line change
Expand Up @@ -108,10 +108,12 @@ func (a Attributes) Get(key string) string {
return v
}

func (a Attributes) Set(key, value string) {
func (a Attributes) Set(key, value string) Attributes {
a.lock()
defer a.unlock()
a.values[key] = value

return a
}

func (a Attributes) Delete(key string) {
Expand Down