From 101960bbcd62556c4dabe5621e8de3a6873c1b55 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 13 Aug 2021 01:24:24 -0400 Subject: [PATCH 1/3] feat: do not special case downloading the first block in the blockservice fetcher --- impl/blockservice/fetcher.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/impl/blockservice/fetcher.go b/impl/blockservice/fetcher.go index 237f3f9..3327a4e 100644 --- a/impl/blockservice/fetcher.go +++ b/impl/blockservice/fetcher.go @@ -97,10 +97,14 @@ func (f *fetcherSession) NodeMatching(ctx context.Context, node ipld.Node, match } func (f *fetcherSession) BlockMatchingOfType(ctx context.Context, root ipld.Link, match ipld.Node, - ptype ipld.NodePrototype, cb fetcher.FetchCallback) error { + _ ipld.NodePrototype, cb fetcher.FetchCallback) error { // retrieve first node - node, err := f.BlockOfType(ctx, root, ptype) + prototype, err := f.PrototypeFromLink(root) + if err != nil { + return err + } + node, err := f.BlockOfType(ctx, root, prototype) if err != nil { return err } From e91ab79237f6545ce880e2ae7a69aa07161051fb Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 13 Aug 2021 01:28:50 -0400 Subject: [PATCH 2/3] fix: delete BlockAllOfType helper and fix BlockAll helper so that the selector now always compiles --- helpers/traversal.go | 22 +++++----------------- impl/blockservice/fetcher_test.go | 2 +- 2 files changed, 6 insertions(+), 18 deletions(-) diff --git a/helpers/traversal.go b/helpers/traversal.go index 37feeeb..87b631d 100644 --- a/helpers/traversal.go +++ b/helpers/traversal.go @@ -5,6 +5,7 @@ import ( "github.com/ipfs/go-fetcher" "github.com/ipld/go-ipld-prime" + basicnode "github.com/ipld/go-ipld-prime/node/basic" "github.com/ipld/go-ipld-prime/traversal/selector" "github.com/ipld/go-ipld-prime/traversal/selector/builder" ) @@ -21,30 +22,17 @@ func Block(ctx context.Context, f fetcher.Fetcher, link ipld.Link) (ipld.Node, e // BlockMatching traverses a schemaless node graph starting with the given link using the given selector and possibly crossing // block boundaries. Each matched node is sent to the FetchResult channel. func BlockMatching(ctx context.Context, f fetcher.Fetcher, root ipld.Link, match ipld.Node, cb fetcher.FetchCallback) error { - prototype, err := f.PrototypeFromLink(root) - if err != nil { - return err - } - return f.BlockMatchingOfType(ctx, root, match, prototype, cb) + return f.BlockMatchingOfType(ctx, root, match, nil, cb) } // BlockAll traverses all nodes in the graph linked by root. The nodes will be untyped and send over the results // channel. func BlockAll(ctx context.Context, f fetcher.Fetcher, root ipld.Link, cb fetcher.FetchCallback) error { - prototype, err := f.PrototypeFromLink(root) - if err != nil { - return err - } - return BlockAllOfType(ctx, f, root, prototype, cb) -} - -// BlockAllOfType traverses all nodes in the graph linked by root. The nodes will typed according to ptype -// and send over the results channel. -func BlockAllOfType(ctx context.Context, f fetcher.Fetcher, root ipld.Link, ptype ipld.NodePrototype, cb fetcher.FetchCallback) error { - ssb := builder.NewSelectorSpecBuilder(ptype) + ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) allSelector := ssb.ExploreRecursive(selector.RecursionLimitNone(), ssb.ExploreUnion( ssb.Matcher(), ssb.ExploreAll(ssb.ExploreRecursiveEdge()), )).Node() - return f.BlockMatchingOfType(ctx, root, allSelector, ptype, cb) + + return f.BlockMatchingOfType(ctx, root, allSelector, nil, cb) } diff --git a/impl/blockservice/fetcher_test.go b/impl/blockservice/fetcher_test.go index f8c2d00..42d31d4 100644 --- a/impl/blockservice/fetcher_test.go +++ b/impl/blockservice/fetcher_test.go @@ -269,7 +269,7 @@ func TestHelpers(t *testing.T) { defer cancel() results := []fetcher.FetchResult{} - err = helpers.BlockAllOfType(ctx, session, cidlink.Link{Cid: block1.Cid()}, basicnode.Prototype__Any{}, func(res fetcher.FetchResult) error { + err = helpers.BlockAll(ctx, session, cidlink.Link{Cid: block1.Cid()}, func(res fetcher.FetchResult) error { results = append(results, res) return nil }) From 760b004af53a8be83ca201343becdd26a392cd6a Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 13 Aug 2021 01:32:00 -0400 Subject: [PATCH 3/3] feat: precompile the matchAll selector --- helpers/traversal.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/helpers/traversal.go b/helpers/traversal.go index 87b631d..0bc42ac 100644 --- a/helpers/traversal.go +++ b/helpers/traversal.go @@ -10,6 +10,16 @@ import ( "github.com/ipld/go-ipld-prime/traversal/selector/builder" ) +var matchAllSelector ipld.Node + +func init() { + ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) + matchAllSelector = ssb.ExploreRecursive(selector.RecursionLimitNone(), ssb.ExploreUnion( + ssb.Matcher(), + ssb.ExploreAll(ssb.ExploreRecursiveEdge()), + )).Node() +} + // Block fetches a schemaless node graph corresponding to single block by link. func Block(ctx context.Context, f fetcher.Fetcher, link ipld.Link) (ipld.Node, error) { prototype, err := f.PrototypeFromLink(link) @@ -28,11 +38,5 @@ func BlockMatching(ctx context.Context, f fetcher.Fetcher, root ipld.Link, match // BlockAll traverses all nodes in the graph linked by root. The nodes will be untyped and send over the results // channel. func BlockAll(ctx context.Context, f fetcher.Fetcher, root ipld.Link, cb fetcher.FetchCallback) error { - ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) - allSelector := ssb.ExploreRecursive(selector.RecursionLimitNone(), ssb.ExploreUnion( - ssb.Matcher(), - ssb.ExploreAll(ssb.ExploreRecursiveEdge()), - )).Node() - - return f.BlockMatchingOfType(ctx, root, allSelector, nil, cb) + return f.BlockMatchingOfType(ctx, root, matchAllSelector, nil, cb) }