Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cleanup old validation #1179

Merged
merged 40 commits into from
Oct 12, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
40 commits
Select commit Hold shift + click to select a range
ce6b9a5
Potential place for validation
rumyantseva Sep 26, 2022
c54ec40
more thoughts
rumyantseva Sep 26, 2022
33669f6
validation things
rumyantseva Sep 27, 2022
67db6f5
Merge branch 'main' into issue-693-validation
rumyantseva Sep 28, 2022
94a7b2d
Merge branch 'issue-693-validation' of https://github.com/rumyantseva…
rumyantseva Sep 28, 2022
9b4aa1a
wip
rumyantseva Sep 28, 2022
82711a5
Merge branch 'main' into issue-693-validation
rumyantseva Sep 28, 2022
648803f
Merge branch 'main' into issue-693-validation
rumyantseva Sep 29, 2022
0c5a30c
Merge branch 'main' into issue-693-validation
rumyantseva Oct 6, 2022
fe5e417
Merge branch 'main' into issue-693-validation
rumyantseva Oct 10, 2022
fa07513
thoughts...
rumyantseva Oct 10, 2022
7876d86
thoughts...
rumyantseva Oct 10, 2022
bcf7eaf
thoughts...
rumyantseva Oct 10, 2022
e3f703b
wip
rumyantseva Oct 10, 2022
ae04620
wip
rumyantseva Oct 10, 2022
8fd988f
wip
rumyantseva Oct 10, 2022
a64ad30
no validation yet
rumyantseva Oct 10, 2022
56eb4b7
wip
rumyantseva Oct 10, 2022
1f01a4b
wip
rumyantseva Oct 10, 2022
10f09ab
wip
rumyantseva Oct 10, 2022
a47b6e2
wip
rumyantseva Oct 10, 2022
8dc12cb
wip
rumyantseva Oct 10, 2022
72877e2
wip
rumyantseva Oct 10, 2022
e1e7970
Merge branch 'main' into issue-693-validation
rumyantseva Oct 10, 2022
b2bca76
wip
rumyantseva Oct 10, 2022
90ca99c
wip
rumyantseva Oct 10, 2022
191e72b
wip
rumyantseva Oct 10, 2022
ac2f49a
dlete all validation related things
rumyantseva Oct 10, 2022
cadaf6d
delete all validation related things
rumyantseva Oct 10, 2022
fa8b884
fmt
rumyantseva Oct 10, 2022
f33116c
fmt
rumyantseva Oct 10, 2022
cdf95b5
wip
rumyantseva Oct 10, 2022
3003792
wip
rumyantseva Oct 10, 2022
fc8a10c
Merge branch 'main' into issue-693-validation
rumyantseva Oct 11, 2022
f19a7a0
wip
rumyantseva Oct 11, 2022
a256865
Merge branch 'issue-693-validation' of https://github.com/rumyantseva…
rumyantseva Oct 11, 2022
71ccfa8
fix a bug with nil
rumyantseva Oct 11, 2022
65f8c88
Merge branch 'main' into issue-693-validation
rumyantseva Oct 11, 2022
629f365
Merge branch 'main' into issue-693-validation
rumyantseva Oct 11, 2022
e8a77f1
Merge branch 'main' into issue-693-validation
AlekSi Oct 12, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
16 changes: 0 additions & 16 deletions internal/types/array.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,12 +39,6 @@ func MakeArray(capacity int) *Array {

// NewArray creates an array with the given values.
func NewArray(values ...any) (*Array, error) {
for i, value := range values {
if err := validateValue(value); err != nil {
return nil, fmt.Errorf("types.NewArray: index %d: %w", i, err)
}
}

return &Array{s: values}, nil
}

Expand Down Expand Up @@ -88,22 +82,12 @@ func (a *Array) Set(index int, value any) error {
return fmt.Errorf("types.Array.Set: index %d is out of bounds [0-%d)", index, l)
}

if err := validateValue(value); err != nil {
return fmt.Errorf("types.Array.Set: %w", err)
}

a.s[index] = value
return nil
}

// Append appends given values to the array.
func (a *Array) Append(values ...any) error {
for _, value := range values {
if err := validateValue(value); err != nil {
return fmt.Errorf("types.Array.Append: %w", err)
}
}

if a.s == nil {
a.s = values
return nil
Expand Down
14 changes: 0 additions & 14 deletions internal/types/array_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,20 +50,6 @@ func TestArray(t *testing.T) {
value, err := a.Get(0)
assert.NoError(t, err)
assert.Equal(t, Null, value)

err = a.Append(42)
assert.EqualError(t, err, `types.Array.Append: types.validateValue: unsupported type: int (42)`)

err = a.Append(nil)
assert.EqualError(t, err, `types.Array.Append: types.validateValue: unsupported type: <nil> (<nil>)`)
})

t.Run("NewArray", func(t *testing.T) {
t.Parallel()

a, err := NewArray(int32(42), 42)
assert.Nil(t, a)
assert.EqualError(t, err, `types.NewArray: index 1: types.validateValue: unsupported type: int (42)`)
})

t.Run("DeepCopy", func(t *testing.T) {
Expand Down
86 changes: 4 additions & 82 deletions internal/types/document.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ package types
import (
"fmt"
"strconv"
"unicode/utf8"

"golang.org/x/exp/slices"

Expand All @@ -40,10 +39,10 @@ type Document struct {
keys []string
}

// ConvertDocument converts bson.Document to *types.Document and validates it.
// ConvertDocument converts bson.Document to *types.Document.
// It references the same data without copying it.
//
// TODO Remove this function.
// TODO Remove this function: https://github.com/FerretDB/FerretDB/issues/260
func ConvertDocument(d document) (*Document, error) {
if d == nil {
panic("types.ConvertDocument: d is nil")
Expand All @@ -54,10 +53,6 @@ func ConvertDocument(d document) (*Document, error) {
keys: d.Keys(),
}

if err := doc.validate(); err != nil {
return doc, fmt.Errorf("types.ConvertDocument: %w", err)
}

return doc, nil
}

Expand Down Expand Up @@ -98,10 +93,6 @@ func NewDocument(pairs ...any) (*Document, error) {
}
}

if err := doc.validate(); err != nil {
return nil, fmt.Errorf("types.NewDocument: %w", err)
}

return doc, nil
}

Expand All @@ -115,59 +106,6 @@ func (d *Document) DeepCopy() *Document {
return deepCopy(d).(*Document)
}

// isValidKey returns false if key is not a valid document field key.
//
// TODO That function should be removed once we have separate validation for command and data documents.
func isValidKey(key string) bool {
if key == "" {
// TODO that should be valid only for command documents, not for data documents
return true
}

// forbid keys like $k (used by pjson representation), but allow $db (used by many commands)
if key[0] == '$' && len(key) <= 2 {
return false
}

return utf8.ValidString(key)
}

// validate checks if the document is valid.
func (d *Document) validate() error {
if d == nil {
panic("types.Document.validate: d is nil")
}

if len(d.m) != len(d.keys) {
return fmt.Errorf("types.Document.validate: keys and values count mismatch: %d != %d", len(d.m), len(d.keys))
}

// TODO check that _id is not regex or array

prevKeys := make(map[string]struct{}, len(d.keys))
for _, key := range d.keys {
if !isValidKey(key) {
return fmt.Errorf("types.Document.validate: invalid key: %q", key)
}

value, ok := d.m[key]
if !ok {
return fmt.Errorf("types.Document.validate: key not found: %q", key)
}

if _, ok := prevKeys[key]; ok {
return fmt.Errorf("types.Document.validate: duplicate key: %q", key)
}
prevKeys[key] = struct{}{}

if err := validateValue(value); err != nil {
return fmt.Errorf("types.Document.validate: %w", err)
}
}

return nil
}

// Len returns the number of elements in the document.
//
// It returns 0 for nil Document.
Expand Down Expand Up @@ -216,17 +154,9 @@ func (d *Document) add(key string, value any) error {
return fmt.Errorf("types.Document.add: key already present: %q", key)
}

if !isValidKey(key) {
return fmt.Errorf("types.Document.add: invalid key: %q", key)
}

if err := validateValue(value); err != nil {
return fmt.Errorf("types.Document.validate: %w", err)
}

// update keys slice
if key == "_id" {
// TODO check that value is not regex or array
// TODO check that value is not regex or array: https://github.com/FerretDB/FerretDB/issues/1235

// ensure that _id is the first field
d.keys = slices.Insert(d.keys, 0, key)
Expand Down Expand Up @@ -258,17 +188,9 @@ func (d *Document) Get(key string) (any, error) {
//
// As a special case, _id always becomes the first key.
func (d *Document) Set(key string, value any) error {
if !isValidKey(key) {
return fmt.Errorf("types.Document.Set: invalid key: %q", key)
}

if err := validateValue(value); err != nil {
return fmt.Errorf("types.Document.validate: %w", err)
}

// update keys slice
if key == "_id" {
// TODO check that value is not regex or array
// TODO check that value is not regex or array: https://github.com/FerretDB/FerretDB/issues/1235

// ensure that _id is the first field
if i := slices.Index(d.keys, key); i >= 0 {
Expand Down
79 changes: 0 additions & 79 deletions internal/types/document_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
package types

import (
"fmt"
"testing"

"github.com/stretchr/testify/assert"
Expand Down Expand Up @@ -55,13 +54,6 @@ func TestDocument(t *testing.T) {
value, err := doc.Get("foo")
assert.NoError(t, err)
assert.Equal(t, Null, value)

err = doc.Set("bar", 42)
assert.EqualError(t, err, `types.Document.validate: types.validateValue: unsupported type: int (42)`)

err = doc.Set("bar", nil)
assert.EqualError(t, err, `types.Document.validate: types.validateValue: unsupported type: <nil> (<nil>)`)

assert.Equal(t, "foo", doc.Command())
})

Expand All @@ -71,10 +63,6 @@ func TestDocument(t *testing.T) {
doc, err := NewDocument(42, 42)
assert.Nil(t, doc)
assert.EqualError(t, err, `types.NewDocument: invalid key type: int`)

doc, err = NewDocument("foo", 42)
assert.Nil(t, doc)
assert.EqualError(t, err, `types.NewDocument: types.Document.validate: types.validateValue: unsupported type: int (42)`)
})

t.Run("DeepCopy", func(t *testing.T) {
Expand Down Expand Up @@ -109,73 +97,6 @@ func TestDocument(t *testing.T) {
assert.Equal(t, []string{"_id", "foo"}, doc.keys)
})

t.Run("Validate", func(t *testing.T) {
t.Parallel()

for _, tc := range []struct {
name string
doc Document
err error
}{{
name: "normal",
doc: Document{
keys: []string{"0"},
m: map[string]any{"0": "foo"},
},
}, {
name: "empty",
doc: Document{},
}, {
name: "different keys",
doc: Document{
keys: []string{"0"},
m: map[string]any{"1": "foo"},
},
err: fmt.Errorf(`types.Document.validate: key not found: "0"`),
}, {
name: "duplicate keys",
doc: Document{
keys: []string{"0", "0"},
m: map[string]any{"0": "foo"},
},
err: fmt.Errorf("types.Document.validate: keys and values count mismatch: 1 != 2"),
}, {
name: "duplicate and different keys",
doc: Document{
keys: []string{"0", "0"},
m: map[string]any{"0": "foo", "1": "bar"},
},
err: fmt.Errorf(`types.Document.validate: duplicate key: "0"`),
}, {
name: "pjson keys",
doc: Document{
keys: []string{"$k"},
m: map[string]any{"$k": "foo"},
},
err: fmt.Errorf(`types.Document.validate: invalid key: "$k"`),
}, {
name: "dollar keys",
doc: Document{
keys: []string{"$db"},
m: map[string]any{"$db": "foo"},
},
}, {
name: "empty key",
doc: Document{
keys: []string{""},
m: map[string]any{"": ""},
},
}} {
tc := tc
t.Run(tc.name, func(t *testing.T) {
t.Parallel()

err := tc.doc.validate()
assert.Equal(t, tc.err, err)
})
}
})

t.Run("SetByPath", func(t *testing.T) {
for _, tc := range []struct {
name string
Expand Down
38 changes: 0 additions & 38 deletions internal/types/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -93,44 +93,6 @@ type (
// Null represents BSON value Null.
var Null = NullType{}

// validateValue validates value.
//
// TODO https://github.com/FerretDB/FerretDB/issues/260
func validateValue(value any) error {
switch value := value.(type) {
case *Document:
return value.validate()
case *Array:
// It is impossible to construct invalid Array using exported function, methods, or type conversions,
// so no need to revalidate it.
return nil
case float64:
return nil
case string:
return nil
case Binary:
return nil
case ObjectID:
return nil
case bool:
return nil
case time.Time:
return nil
case NullType:
return nil
case Regex:
return nil
case int32:
return nil
case Timestamp:
return nil
case int64:
return nil
default:
return fmt.Errorf("types.validateValue: unsupported type: %[1]T (%[1]v)", value)
}
}

// deepCopy returns a deep copy of the given value.
func deepCopy(value any) any {
if value == nil {
Expand Down
2 changes: 1 addition & 1 deletion internal/util/testutil/equal.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ func AssertEqual[T types.Type](tb testing.TB, expected, actual T) bool {
return assert.Fail(tb, msg)
}

// AssertEqual asserts that two BSON slices are equal.
// AssertEqualSlices asserts that two BSON slices are equal.
func AssertEqualSlices[T types.Type](tb testing.TB, expected, actual []T) bool {
tb.Helper()

Expand Down
4 changes: 0 additions & 4 deletions internal/wire/op_msg_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -143,10 +143,6 @@ var msgTestCases = []testCase{{
},
}},
},
}, {
name: "dollar_dot",
expectedB: testutil.MustParseDumpFile("testdata", "dollar_dot.hex"),
err: `types.Document.validate: invalid key: "$."`,
}, {
name: "msg_fuzz1",
expectedB: testutil.MustParseDumpFile("testdata", "msg_fuzz1.hex"),
Expand Down
1 change: 1 addition & 0 deletions internal/wire/op_query.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ type OpQuery struct {

func (query *OpQuery) msgbody() {}

// readFrom composes an OpQuery from a buffered reader.
func (query *OpQuery) readFrom(bufr *bufio.Reader) error {
if err := binary.Read(bufr, binary.LittleEndian, &query.Flags); err != nil {
return lazyerrors.Errorf("wire.OpQuery.ReadFrom (binary.Read): %w", err)
Expand Down