diff --git a/node/basicnode/bytes.go b/node/basicnode/bytes.go index 7c45d208..566c32fe 100644 --- a/node/basicnode/bytes.go +++ b/node/basicnode/bytes.go @@ -106,7 +106,7 @@ func (nb *plainBytes__Builder) Reset() { // -- NodeAssembler --> type plainBytes__Assembler struct { - w *plainBytes + w datamodel.Node } func (plainBytes__Assembler) BeginMap(sizeHint int64) (datamodel.MapAssembler, error) { @@ -131,17 +131,24 @@ func (plainBytes__Assembler) AssignString(string) error { return mixins.BytesAssembler{TypeName: "bytes"}.AssignString("") } func (na *plainBytes__Assembler) AssignBytes(v []byte) error { - *na.w = plainBytes(v) + na.w = datamodel.Node(plainBytes(v)) return nil } func (plainBytes__Assembler) AssignLink(datamodel.Link) error { return mixins.BytesAssembler{TypeName: "bytes"}.AssignLink(nil) } func (na *plainBytes__Assembler) AssignNode(v datamodel.Node) error { + if lb, ok := v.(datamodel.LargeBytesNode); ok { + lbn, err := lb.AsLargeBytes() + if err == nil { + na.w = streamBytes{lbn} + return nil + } + } if v2, err := v.AsBytes(); err != nil { return err } else { - *na.w = plainBytes(v2) + na.w = plainBytes(v2) return nil } } diff --git a/node/basicnode/bytes_stream.go b/node/basicnode/bytes_stream.go new file mode 100644 index 00000000..ad238bcf --- /dev/null +++ b/node/basicnode/bytes_stream.go @@ -0,0 +1,81 @@ +package basicnode + +import ( + "io" + + "github.com/ipld/go-ipld-prime/datamodel" + "github.com/ipld/go-ipld-prime/node/mixins" +) + +var ( + _ datamodel.Node = streamBytes{nil} + _ datamodel.NodePrototype = Prototype__Bytes{} + _ datamodel.NodeBuilder = &plainBytes__Builder{} + _ datamodel.NodeAssembler = &plainBytes__Assembler{} +) + +func NewBytesFromReader(rs io.ReadSeeker) datamodel.Node { + return streamBytes{rs} +} + +// streamBytes is a boxed reader that complies with datamodel.Node. +type streamBytes struct { + io.ReadSeeker +} + +// -- Node interface methods --> + +func (streamBytes) Kind() datamodel.Kind { + return datamodel.Kind_Bytes +} +func (streamBytes) LookupByString(string) (datamodel.Node, error) { + return mixins.Bytes{TypeName: "bytes"}.LookupByString("") +} +func (streamBytes) LookupByNode(key datamodel.Node) (datamodel.Node, error) { + return mixins.Bytes{TypeName: "bytes"}.LookupByNode(nil) +} +func (streamBytes) LookupByIndex(idx int64) (datamodel.Node, error) { + return mixins.Bytes{TypeName: "bytes"}.LookupByIndex(0) +} +func (streamBytes) LookupBySegment(seg datamodel.PathSegment) (datamodel.Node, error) { + return mixins.Bytes{TypeName: "bytes"}.LookupBySegment(seg) +} +func (streamBytes) MapIterator() datamodel.MapIterator { + return nil +} +func (streamBytes) ListIterator() datamodel.ListIterator { + return nil +} +func (streamBytes) Length() int64 { + return -1 +} +func (streamBytes) IsAbsent() bool { + return false +} +func (streamBytes) IsNull() bool { + return false +} +func (streamBytes) AsBool() (bool, error) { + return mixins.Bytes{TypeName: "bytes"}.AsBool() +} +func (streamBytes) AsInt() (int64, error) { + return mixins.Bytes{TypeName: "bytes"}.AsInt() +} +func (streamBytes) AsFloat() (float64, error) { + return mixins.Bytes{TypeName: "bytes"}.AsFloat() +} +func (streamBytes) AsString() (string, error) { + return mixins.Bytes{TypeName: "bytes"}.AsString() +} +func (n streamBytes) AsBytes() ([]byte, error) { + return io.ReadAll(n) +} +func (streamBytes) AsLink() (datamodel.Link, error) { + return mixins.Bytes{TypeName: "bytes"}.AsLink() +} +func (streamBytes) Prototype() datamodel.NodePrototype { + return Prototype__Bytes{} +} +func (n streamBytes) AsLargeBytes() (io.ReadSeeker, error) { + return n.ReadSeeker, nil +} diff --git a/node/basicnode/bytes_test.go b/node/basicnode/bytes_test.go new file mode 100644 index 00000000..ca408484 --- /dev/null +++ b/node/basicnode/bytes_test.go @@ -0,0 +1,12 @@ +package basicnode_test + +import ( + "testing" + + "github.com/ipld/go-ipld-prime/node/basicnode" + "github.com/ipld/go-ipld-prime/node/tests" +) + +func TestBytes(t *testing.T) { + tests.SpecTestBytes(t, basicnode.Prototype__Bytes{}) +} diff --git a/node/tests/byteSpecs.go b/node/tests/byteSpecs.go new file mode 100644 index 00000000..69bf494b --- /dev/null +++ b/node/tests/byteSpecs.go @@ -0,0 +1,35 @@ +package tests + +import ( + "io" + "testing" + + qt "github.com/frankban/quicktest" + + "github.com/ipld/go-ipld-prime/datamodel" +) + +func SpecTestBytes(t *testing.T, np datamodel.NodePrototype) { + t.Run("byte node", func(t *testing.T) { + nb := np.NewBuilder() + err := nb.AssignBytes([]byte("asdf")) + qt.Check(t, err, qt.IsNil) + n := nb.Build() + + qt.Check(t, n.Kind(), qt.Equals, datamodel.Kind_Bytes) + qt.Check(t, n.IsNull(), qt.IsFalse) + x, err := n.AsBytes() + qt.Check(t, err, qt.IsNil) + qt.Check(t, x, qt.DeepEquals, []byte("asdf")) + + lbn, ok := n.(datamodel.LargeBytesNode) + if ok { + str, err := lbn.AsLargeBytes() + qt.Check(t, err, qt.IsNil) + bytes, err := io.ReadAll(str) + qt.Check(t, err, qt.IsNil) + qt.Check(t, bytes, qt.DeepEquals, []byte("asdf")) + } + + }) +} diff --git a/traversal/selector/builder/builder.go b/traversal/selector/builder/builder.go index 80067758..6cc69e3c 100644 --- a/traversal/selector/builder/builder.go +++ b/traversal/selector/builder/builder.go @@ -32,6 +32,7 @@ type SelectorSpecBuilder interface { ExploreFields(ExploreFieldsSpecBuildingClosure) SelectorSpec ExploreInterpretAs(as string, next SelectorSpec) SelectorSpec Matcher() SelectorSpec + MatcherSubset(from, to int64) SelectorSpec } // ExploreFieldsSpecBuildingClosure is a function that provided to SelectorSpecBuilder's @@ -170,6 +171,19 @@ func (ssb *selectorSpecBuilder) Matcher() SelectorSpec { } } +func (ssb *selectorSpecBuilder) MatcherSubset(from, to int64) SelectorSpec { + return selectorSpec{ + fluent.MustBuildMap(ssb.np, 1, func(na fluent.MapAssembler) { + na.AssembleEntry(selector.SelectorKey_Matcher).CreateMap(1, func(na fluent.MapAssembler) { + na.AssembleEntry(selector.SelectorKey_Subset).CreateMap(2, func(na fluent.MapAssembler) { + na.AssembleEntry(selector.SelectorKey_From).AssignInt(from) + na.AssembleEntry(selector.SelectorKey_To).AssignInt(to) + }) + }) + }), + } +} + type exploreFieldsSpecBuilder struct { na fluent.MapAssembler } diff --git a/traversal/selector/exploreAll.go b/traversal/selector/exploreAll.go index a7749f8e..7d889d7e 100644 --- a/traversal/selector/exploreAll.go +++ b/traversal/selector/exploreAll.go @@ -27,6 +27,11 @@ func (s ExploreAll) Decide(n datamodel.Node) bool { return false } +// Match always returns false because this is not a matcher +func (s ExploreAll) Match(node datamodel.Node) (datamodel.Node, error) { + return nil, nil +} + // ParseExploreAll assembles a Selector from a ExploreAll selector node func (pc ParseContext) ParseExploreAll(n datamodel.Node) (Selector, error) { if n.Kind() != datamodel.Kind_Map { diff --git a/traversal/selector/exploreFields.go b/traversal/selector/exploreFields.go index 5d2abdd2..b2796ff2 100644 --- a/traversal/selector/exploreFields.go +++ b/traversal/selector/exploreFields.go @@ -38,6 +38,11 @@ func (s ExploreFields) Decide(n datamodel.Node) bool { return false } +// Match always returns false because this is not a matcher +func (s ExploreFields) Match(node datamodel.Node) (datamodel.Node, error) { + return nil, nil +} + // ParseExploreFields assembles a Selector // from a ExploreFields selector node func (pc ParseContext) ParseExploreFields(n datamodel.Node) (Selector, error) { diff --git a/traversal/selector/exploreIndex.go b/traversal/selector/exploreIndex.go index 0c8edf16..56a2badc 100644 --- a/traversal/selector/exploreIndex.go +++ b/traversal/selector/exploreIndex.go @@ -37,6 +37,11 @@ func (s ExploreIndex) Decide(n datamodel.Node) bool { return false } +// Match always returns false because this is not a matcher +func (s ExploreIndex) Match(node datamodel.Node) (datamodel.Node, error) { + return nil, nil +} + // ParseExploreIndex assembles a Selector // from a ExploreIndex selector node func (pc ParseContext) ParseExploreIndex(n datamodel.Node) (Selector, error) { diff --git a/traversal/selector/exploreInterpretAs.go b/traversal/selector/exploreInterpretAs.go index 25cf045b..6f4aed67 100644 --- a/traversal/selector/exploreInterpretAs.go +++ b/traversal/selector/exploreInterpretAs.go @@ -27,6 +27,11 @@ func (s ExploreInterpretAs) Decide(n datamodel.Node) bool { return false } +// Match always returns false because this is not a matcher +func (s ExploreInterpretAs) Match(node datamodel.Node) (datamodel.Node, error) { + return nil, nil +} + // NamedReifier indicates how this selector expects to Reify the current datamodel.Node. func (s ExploreInterpretAs) NamedReifier() string { return s.adl diff --git a/traversal/selector/exploreRange.go b/traversal/selector/exploreRange.go index 73e356ba..e9e041bc 100644 --- a/traversal/selector/exploreRange.go +++ b/traversal/selector/exploreRange.go @@ -41,6 +41,11 @@ func (s ExploreRange) Decide(n datamodel.Node) bool { return false } +// Match always returns false because this is not a matcher +func (s ExploreRange) Match(node datamodel.Node) (datamodel.Node, error) { + return nil, nil +} + // ParseExploreRange assembles a Selector // from a ExploreRange selector node func (pc ParseContext) ParseExploreRange(n datamodel.Node) (Selector, error) { diff --git a/traversal/selector/exploreRecursive.go b/traversal/selector/exploreRecursive.go index 17614025..28a9bedc 100644 --- a/traversal/selector/exploreRecursive.go +++ b/traversal/selector/exploreRecursive.go @@ -176,6 +176,11 @@ func (s ExploreRecursive) Decide(n datamodel.Node) bool { return s.current.Decide(n) } +// Match always returns false because this is not a matcher +func (s ExploreRecursive) Match(node datamodel.Node) (datamodel.Node, error) { + return s.current.Match(node) +} + type exploreRecursiveContext struct { edgesFound int } diff --git a/traversal/selector/exploreRecursiveEdge.go b/traversal/selector/exploreRecursiveEdge.go index 65438572..534c73b8 100644 --- a/traversal/selector/exploreRecursiveEdge.go +++ b/traversal/selector/exploreRecursiveEdge.go @@ -31,6 +31,11 @@ func (s ExploreRecursiveEdge) Decide(n datamodel.Node) bool { return false } +// Match always returns false because this is not a matcher +func (s ExploreRecursiveEdge) Match(node datamodel.Node) (datamodel.Node, error) { + return nil, nil +} + // ParseExploreRecursiveEdge assembles a Selector // from a exploreRecursiveEdge selector node func (pc ParseContext) ParseExploreRecursiveEdge(n datamodel.Node) (Selector, error) { diff --git a/traversal/selector/exploreUnion.go b/traversal/selector/exploreUnion.go index 0af42cd6..21594954 100644 --- a/traversal/selector/exploreUnion.go +++ b/traversal/selector/exploreUnion.go @@ -74,6 +74,18 @@ func (s ExploreUnion) Decide(n datamodel.Node) bool { return false } +// Match returns true for a Union selector based on the matched union. +func (s ExploreUnion) Match(n datamodel.Node) (datamodel.Node, error) { + for _, m := range s.Members { + if mn, err := m.Match(n); mn != nil { + return mn, nil + } else if err != nil { + return nil, err + } + } + return nil, nil +} + // ParseExploreUnion assembles a Selector // from an ExploreUnion selector node func (pc ParseContext) ParseExploreUnion(n datamodel.Node) (Selector, error) { diff --git a/traversal/selector/fieldKeys.go b/traversal/selector/fieldKeys.go index 58024b61..39ec61e6 100644 --- a/traversal/selector/fieldKeys.go +++ b/traversal/selector/fieldKeys.go @@ -23,5 +23,8 @@ const ( SelectorKey_StopAt = "!" SelectorKey_Condition = "&" SelectorKey_As = "as" + SelectorKey_Subset = "subset" + SelectorKey_From = "[" + SelectorKey_To = "]" // not filling conditional keys since it's not complete ) diff --git a/traversal/selector/matcher.go b/traversal/selector/matcher.go index b8f64f68..ff2596c1 100644 --- a/traversal/selector/matcher.go +++ b/traversal/selector/matcher.go @@ -2,8 +2,10 @@ package selector import ( "fmt" + "io" "github.com/ipld/go-ipld-prime/datamodel" + "github.com/ipld/go-ipld-prime/node/basicnode" ) // Matcher marks a node to be included in the "result" set. @@ -16,7 +18,64 @@ import ( // A selector tree with only "explore*"-type selectors and no Matcher selectors // is valid; it will just generate a "covered" set of nodes and no "result" set. // TODO: From spec: implement conditions and labels -type Matcher struct{} +type Matcher struct { + *Slice +} + +// Slice limits a result node to a subset of the node. +// The returned node will be limited based on slicing the specified range of the +// node into a new node, or making use of the `AsLargeBytes` io.ReadSeeker to +// restrict response with a SectionReader. +type Slice struct { + From int64 + To int64 +} + +func (s Slice) Slice(n datamodel.Node) (datamodel.Node, error) { + var from, to int64 + switch n.Kind() { + case datamodel.Kind_String: + str, err := n.AsString() + if err != nil { + return nil, err + } + to = s.To + if len(str) < int(to) { + to = int64(len(str)) + } + from = s.From + if len(str) < int(from) { + from = int64(len(str)) + } + return basicnode.NewString(str[from:to]), nil + case datamodel.Kind_Bytes: + if lbn, ok := n.(datamodel.LargeBytesNode); ok { + rdr, err := lbn.AsLargeBytes() + if err != nil { + return nil, err + } + + sr := io.NewSectionReader(readerat{rdr}, s.From, s.To-s.From) + return basicnode.NewBytesFromReader(sr), nil + } + bytes, err := n.AsBytes() + if err != nil { + return nil, err + } + to = s.To + if len(bytes) < int(to) { + to = int64(len(bytes)) + } + from = s.From + if len(bytes) < int(from) { + from = int64(len(bytes)) + } + + return basicnode.NewBytes(bytes[from:to]), nil + default: + return nil, fmt.Errorf("selector slice rejected on %s: subset match must be over string or bytes", n.Kind()) + } +} // Interests are empty for a matcher (for now) because // It is always just there to match, not explore further @@ -30,11 +89,19 @@ func (s Matcher) Explore(n datamodel.Node, p datamodel.PathSegment) (Selector, e } // Decide is always true for a match cause it's in the result set -// TODO: Implement boolean logic for conditionals +// Deprecated: use Match instead func (s Matcher) Decide(n datamodel.Node) bool { return true } +// Match is always true for a match cause it's in the result set +func (s Matcher) Match(node datamodel.Node) (datamodel.Node, error) { + if s.Slice != nil { + return s.Slice.Slice(node) + } + return node, nil +} + // ParseMatcher assembles a Selector // from a matcher selector node // TODO: Parse labels and conditions @@ -42,5 +109,38 @@ func (pc ParseContext) ParseMatcher(n datamodel.Node) (Selector, error) { if n.Kind() != datamodel.Kind_Map { return nil, fmt.Errorf("selector spec parse rejected: selector body must be a map") } + + // check if a slice is specified + if subset, err := n.LookupByString("subset"); err == nil { + if subset.Kind() != datamodel.Kind_Map { + return nil, fmt.Errorf("selector spec parse rejected: subset body must be a map") + } + from, err := subset.LookupByString("[") + if err != nil { + return nil, fmt.Errorf("selector spec parse rejected: selector body must be a map with a from '[' key") + } + fromN, err := from.AsInt() + if err != nil { + return nil, fmt.Errorf("selector spec parse rejected: selector body must be a map with a 'from' key that is a number") + } + to, err := subset.LookupByString("]") + if err != nil { + return nil, fmt.Errorf("selector spec parse rejected: selector body must be a map with a to ']' key") + } + toN, err := to.AsInt() + if err != nil { + return nil, fmt.Errorf("selector spec parse rejected: selector body must be a map with a 'to' key that is a number") + } + if fromN > toN { + return nil, fmt.Errorf("selector spec parse rejected: selector body must be a map with a 'from' key that is less than or equal to the 'to' key") + } + if fromN < 0 || toN < 0 { + return nil, fmt.Errorf("selector spec parse rejected: selector body must be a map with keys not less than 0") + } + return Matcher{&Slice{ + From: fromN, + To: toN, + }}, nil + } return Matcher{}, nil } diff --git a/traversal/selector/matcher_util.go b/traversal/selector/matcher_util.go new file mode 100644 index 00000000..237c5e71 --- /dev/null +++ b/traversal/selector/matcher_util.go @@ -0,0 +1,20 @@ +package selector + +import "io" + +type readerat struct { + io.ReadSeeker +} + +// ReadAt provides the io.ReadAt method over a ReadSeeker. +// This implementation does not support concurrent calls to `ReadAt`, +// as specified by the ReaderAt interface, and so must only be used +// in non-concurrent use cases. +func (r readerat) ReadAt(p []byte, off int64) (n int, err error) { + // TODO: consider keeping track of current offset. + _, err = r.Seek(off, 0) + if err != nil { + return 0, err + } + return r.Read(p) +} diff --git a/traversal/selector/selector.go b/traversal/selector/selector.go index 4d0a7814..b58fcd86 100644 --- a/traversal/selector/selector.go +++ b/traversal/selector/selector.go @@ -106,6 +106,12 @@ type Selector interface { // Only "Matcher" clauses actually implement this in a way that ever returns "true". // See the Selector specs for discussion on "matched" vs "reached"/"visited" nodes. Decide(node datamodel.Node) bool + + // Match is an extension to Decide allowing the matcher to `decide` a transformation of + // the matched node. This is used for `Subset` match behavior. If the node is matched, + // the first argument will be the matched node. If it is not matched, the first argument + // will be null. If there is an error, the first argument will be null. + Match(node datamodel.Node) (datamodel.Node, error) } // REVIEW: do ParsedParent and ParseContext need to be exported? They're mostly used during the compilation process. diff --git a/traversal/selector/spec_test.go b/traversal/selector/spec_test.go index 215bfd6b..7d309d05 100644 --- a/traversal/selector/spec_test.go +++ b/traversal/selector/spec_test.go @@ -2,6 +2,7 @@ package selector_test import ( "bytes" + "io" "os" "testing" @@ -14,7 +15,7 @@ import ( "github.com/ipld/go-ipld-prime/fluent/qp" "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/traversal" - "github.com/ipld/go-ipld-prime/traversal/selector/parse" + selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" ) func TestSpecFixtures(t *testing.T) { @@ -81,6 +82,18 @@ func testOneSpecFixtureFile(t *testing.T, filename string) { })) qp.MapEntry(ma, "matched", qp.Bool(reason == traversal.VisitReason_SelectionMatch)) }) + if reason == traversal.VisitReason_SelectionMatch && n.Kind() == datamodel.Kind_Bytes { + if lbn, ok := n.(datamodel.LargeBytesNode); ok { + rdr, err := lbn.AsLargeBytes() + if err == nil { + io.Copy(io.Discard, rdr) + } + } + _, err := n.AsBytes() + if err != nil { + panic("insanity at a deeper level than this test's target") + } + } if err != nil { panic("insanity at a deeper level than this test's target") } diff --git a/traversal/walk.go b/traversal/walk.go index 1656de15..8db783f0 100644 --- a/traversal/walk.go +++ b/traversal/walk.go @@ -174,6 +174,7 @@ func (prog Progress) walkAdv(n datamodel.Node, s selector.Selector, fn AdvVisitF if err != nil { return fmt.Errorf("failed to reify node as %q: %w", adl, err) } + // explore into the `InterpretAs` clause to the child selector. s, err = s.Explore(n, datamodel.PathSegment{}) if err != nil { return err @@ -183,10 +184,12 @@ func (prog Progress) walkAdv(n datamodel.Node, s selector.Selector, fn AdvVisitF if prog.Path.Len() >= prog.Cfg.StartAtPath.Len() || !prog.PastStartAtPath { // Decide if this node is matched -- do callbacks as appropriate. - if s.Decide(n) { - if err := fn(prog, n, VisitReason_SelectionMatch); err != nil { + if match, err := s.Match(n); match != nil { + if err := fn(prog, match, VisitReason_SelectionMatch); err != nil { return err } + } else if err != nil { + return err } else { if err := fn(prog, n, VisitReason_SelectionCandidate); err != nil { return err