From e87c333328aa7c5fe847cbf9943cb31f060a653d Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Sat, 21 Apr 2018 22:07:50 +0200 Subject: [PATCH 1/5] extract path/resolver --- core/builder.go | 2 +- core/commands/files.go | 2 +- core/commands/ls.go | 2 +- core/commands/pin.go | 2 +- core/commands/unixfs/ls.go | 2 +- core/core.go | 2 +- core/coreapi/coreapi.go | 2 +- core/corehttp/gateway_handler.go | 2 +- core/corerepo/pinning.go | 2 +- core/coreunix/cat.go | 2 +- core/pathresolver.go | 2 +- path/resolver/resolver.go | 175 ------------------------------- path/resolver/resolver_test.go | 70 ------------- 13 files changed, 11 insertions(+), 256 deletions(-) delete mode 100644 path/resolver/resolver.go delete mode 100644 path/resolver/resolver_test.go diff --git a/core/builder.go b/core/builder.go index 8541584e6e9..8bcb6f5259f 100644 --- a/core/builder.go +++ b/core/builder.go @@ -9,10 +9,10 @@ import ( "syscall" "time" + resolver "github.com/ipfs/go-ipfs-path-resolver" bserv "github.com/ipfs/go-ipfs/blockservice" filestore "github.com/ipfs/go-ipfs/filestore" dag "github.com/ipfs/go-ipfs/merkledag" - resolver "github.com/ipfs/go-ipfs/path/resolver" pin "github.com/ipfs/go-ipfs/pin" repo "github.com/ipfs/go-ipfs/repo" cfg "github.com/ipfs/go-ipfs/repo/config" diff --git a/core/commands/files.go b/core/commands/files.go index b8a7e4c2f8d..c6cfec1d6a4 100644 --- a/core/commands/files.go +++ b/core/commands/files.go @@ -10,6 +10,7 @@ import ( gopath "path" "strings" + resolver "github.com/ipfs/go-ipfs-path-resolver" bservice "github.com/ipfs/go-ipfs/blockservice" oldcmds "github.com/ipfs/go-ipfs/commands" lgc "github.com/ipfs/go-ipfs/commands/legacy" @@ -18,7 +19,6 @@ import ( dag "github.com/ipfs/go-ipfs/merkledag" mfs "github.com/ipfs/go-ipfs/mfs" path "github.com/ipfs/go-ipfs/path" - resolver "github.com/ipfs/go-ipfs/path/resolver" ft "github.com/ipfs/go-ipfs/unixfs" uio "github.com/ipfs/go-ipfs/unixfs/io" diff --git a/core/commands/ls.go b/core/commands/ls.go index a4a38cad802..4f64c4a1eba 100644 --- a/core/commands/ls.go +++ b/core/commands/ls.go @@ -6,13 +6,13 @@ import ( "io" "text/tabwriter" + resolver "github.com/ipfs/go-ipfs-path-resolver" blockservice "github.com/ipfs/go-ipfs/blockservice" cmds "github.com/ipfs/go-ipfs/commands" core "github.com/ipfs/go-ipfs/core" e "github.com/ipfs/go-ipfs/core/commands/e" merkledag "github.com/ipfs/go-ipfs/merkledag" path "github.com/ipfs/go-ipfs/path" - resolver "github.com/ipfs/go-ipfs/path/resolver" unixfs "github.com/ipfs/go-ipfs/unixfs" uio "github.com/ipfs/go-ipfs/unixfs/io" unixfspb "github.com/ipfs/go-ipfs/unixfs/pb" diff --git a/core/commands/pin.go b/core/commands/pin.go index 1d1a3c7febf..7d0afde7fdb 100644 --- a/core/commands/pin.go +++ b/core/commands/pin.go @@ -7,6 +7,7 @@ import ( "io" "time" + resolver "github.com/ipfs/go-ipfs-path-resolver" bserv "github.com/ipfs/go-ipfs/blockservice" cmds "github.com/ipfs/go-ipfs/commands" core "github.com/ipfs/go-ipfs/core" @@ -14,7 +15,6 @@ import ( corerepo "github.com/ipfs/go-ipfs/core/corerepo" dag "github.com/ipfs/go-ipfs/merkledag" path "github.com/ipfs/go-ipfs/path" - resolver "github.com/ipfs/go-ipfs/path/resolver" pin "github.com/ipfs/go-ipfs/pin" "github.com/ipfs/go-ipfs/thirdparty/verifcid" uio "github.com/ipfs/go-ipfs/unixfs/io" diff --git a/core/commands/unixfs/ls.go b/core/commands/unixfs/ls.go index 0c989a2f8c1..8b15a2c67af 100644 --- a/core/commands/unixfs/ls.go +++ b/core/commands/unixfs/ls.go @@ -9,12 +9,12 @@ import ( cmdkit "gx/ipfs/QmceUdzxkimdYsgtX733uNgzf1DLHyBKN6ehGSp85ayppM/go-ipfs-cmdkit" + resolver "github.com/ipfs/go-ipfs-path-resolver" cmds "github.com/ipfs/go-ipfs/commands" core "github.com/ipfs/go-ipfs/core" e "github.com/ipfs/go-ipfs/core/commands/e" merkledag "github.com/ipfs/go-ipfs/merkledag" path "github.com/ipfs/go-ipfs/path" - resolver "github.com/ipfs/go-ipfs/path/resolver" unixfs "github.com/ipfs/go-ipfs/unixfs" uio "github.com/ipfs/go-ipfs/unixfs/io" unixfspb "github.com/ipfs/go-ipfs/unixfs/pb" diff --git a/core/core.go b/core/core.go index 890a2f7bfbc..d6dbf1b8870 100644 --- a/core/core.go +++ b/core/core.go @@ -21,6 +21,7 @@ import ( "strings" "time" + "github.com/ipfs/go-ipfs-path-resolver" bserv "github.com/ipfs/go-ipfs/blockservice" bitswap "github.com/ipfs/go-ipfs/exchange/bitswap" bsnet "github.com/ipfs/go-ipfs/exchange/bitswap/network" @@ -32,7 +33,6 @@ import ( namesys "github.com/ipfs/go-ipfs/namesys" ipnsrp "github.com/ipfs/go-ipfs/namesys/republisher" p2p "github.com/ipfs/go-ipfs/p2p" - "github.com/ipfs/go-ipfs/path/resolver" pin "github.com/ipfs/go-ipfs/pin" repo "github.com/ipfs/go-ipfs/repo" config "github.com/ipfs/go-ipfs/repo/config" diff --git a/core/coreapi/coreapi.go b/core/coreapi/coreapi.go index a82320a3cf9..9a098d1dfdf 100644 --- a/core/coreapi/coreapi.go +++ b/core/coreapi/coreapi.go @@ -3,11 +3,11 @@ package coreapi import ( "context" + resolver "github.com/ipfs/go-ipfs-path-resolver" core "github.com/ipfs/go-ipfs/core" coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface" namesys "github.com/ipfs/go-ipfs/namesys" ipfspath "github.com/ipfs/go-ipfs/path" - resolver "github.com/ipfs/go-ipfs/path/resolver" uio "github.com/ipfs/go-ipfs/unixfs/io" cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go index 842d30d46bf..833a2646a9c 100644 --- a/core/corehttp/gateway_handler.go +++ b/core/corehttp/gateway_handler.go @@ -12,6 +12,7 @@ import ( "strings" "time" + resolver "github.com/ipfs/go-ipfs-path-resolver" core "github.com/ipfs/go-ipfs/core" coreapi "github.com/ipfs/go-ipfs/core/coreapi" coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface" @@ -19,7 +20,6 @@ import ( dag "github.com/ipfs/go-ipfs/merkledag" dagutils "github.com/ipfs/go-ipfs/merkledag/utils" path "github.com/ipfs/go-ipfs/path" - resolver "github.com/ipfs/go-ipfs/path/resolver" ft "github.com/ipfs/go-ipfs/unixfs" uio "github.com/ipfs/go-ipfs/unixfs/io" diff --git a/core/corerepo/pinning.go b/core/corerepo/pinning.go index 96af8604ab7..8c95c8a3686 100644 --- a/core/corerepo/pinning.go +++ b/core/corerepo/pinning.go @@ -17,9 +17,9 @@ import ( "context" "fmt" + resolver "github.com/ipfs/go-ipfs-path-resolver" "github.com/ipfs/go-ipfs/core" path "github.com/ipfs/go-ipfs/path" - resolver "github.com/ipfs/go-ipfs/path/resolver" uio "github.com/ipfs/go-ipfs/unixfs/io" cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" diff --git a/core/coreunix/cat.go b/core/coreunix/cat.go index 084d7dee690..e1e2bf71744 100644 --- a/core/coreunix/cat.go +++ b/core/coreunix/cat.go @@ -3,9 +3,9 @@ package coreunix import ( "context" + resolver "github.com/ipfs/go-ipfs-path-resolver" core "github.com/ipfs/go-ipfs/core" path "github.com/ipfs/go-ipfs/path" - resolver "github.com/ipfs/go-ipfs/path/resolver" uio "github.com/ipfs/go-ipfs/unixfs/io" ) diff --git a/core/pathresolver.go b/core/pathresolver.go index 6c71b0b5f74..634e23a78f3 100644 --- a/core/pathresolver.go +++ b/core/pathresolver.go @@ -5,9 +5,9 @@ import ( "errors" "strings" + resolver "github.com/ipfs/go-ipfs-path-resolver" namesys "github.com/ipfs/go-ipfs/namesys" path "github.com/ipfs/go-ipfs/path" - resolver "github.com/ipfs/go-ipfs/path/resolver" logging "gx/ipfs/QmRb5jh8z2E8hMGN2tkvs1yHynUanqnZ3UeKwgN1i9P1F8/go-log" cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" diff --git a/path/resolver/resolver.go b/path/resolver/resolver.go deleted file mode 100644 index 203fe9ce953..00000000000 --- a/path/resolver/resolver.go +++ /dev/null @@ -1,175 +0,0 @@ -// Package resolver implements utilities for resolving paths within ipfs. -package resolver - -import ( - "context" - "errors" - "fmt" - "time" - - dag "github.com/ipfs/go-ipfs/merkledag" - path "github.com/ipfs/go-ipfs/path" - - logging "gx/ipfs/QmRb5jh8z2E8hMGN2tkvs1yHynUanqnZ3UeKwgN1i9P1F8/go-log" - cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" - ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" -) - -var log = logging.Logger("pathresolv") - -// ErrNoComponents is used when Paths after a protocol -// do not contain at least one component -var ErrNoComponents = errors.New( - "path must contain at least one component") - -// ErrNoLink is returned when a link is not found in a path -type ErrNoLink struct { - Name string - Node *cid.Cid -} - -// Error implements the Error interface for ErrNoLink with a useful -// human readable message. -func (e ErrNoLink) Error() string { - return fmt.Sprintf("no link named %q under %s", e.Name, e.Node.String()) -} - -// Resolver provides path resolution to IPFS -// It has a pointer to a DAGService, which is uses to resolve nodes. -// TODO: now that this is more modular, try to unify this code with the -// the resolvers in namesys -type Resolver struct { - DAG ipld.NodeGetter - - ResolveOnce func(ctx context.Context, ds ipld.NodeGetter, nd ipld.Node, names []string) (*ipld.Link, []string, error) -} - -// NewBasicResolver constructs a new basic resolver. -func NewBasicResolver(ds ipld.DAGService) *Resolver { - return &Resolver{ - DAG: ds, - ResolveOnce: ResolveSingle, - } -} - -// ResolveToLastNode walks the given path and returns the ipld.Node -// referenced by the last element in it. -func (r *Resolver) ResolveToLastNode(ctx context.Context, fpath path.Path) (ipld.Node, []string, error) { - c, p, err := path.SplitAbsPath(fpath) - if err != nil { - return nil, nil, err - } - - nd, err := r.DAG.Get(ctx, c) - if err != nil { - return nil, nil, err - } - - for len(p) > 0 { - val, rest, err := nd.Resolve(p) - if err != nil { - return nil, nil, err - } - - switch val := val.(type) { - case *ipld.Link: - next, err := val.GetNode(ctx, r.DAG) - if err != nil { - return nil, nil, err - } - nd = next - p = rest - default: - return nd, p, nil - } - } - - return nd, nil, nil -} - -// ResolvePath fetches the node for given path. It returns the last item -// returned by ResolvePathComponents. -func (r *Resolver) ResolvePath(ctx context.Context, fpath path.Path) (ipld.Node, error) { - // validate path - if err := fpath.IsValid(); err != nil { - return nil, err - } - - nodes, err := r.ResolvePathComponents(ctx, fpath) - if err != nil || nodes == nil { - return nil, err - } - return nodes[len(nodes)-1], err -} - -// ResolveSingle simply resolves one hop of a path through a graph with no -// extra context (does not opaquely resolve through sharded nodes) -func ResolveSingle(ctx context.Context, ds ipld.NodeGetter, nd ipld.Node, names []string) (*ipld.Link, []string, error) { - return nd.ResolveLink(names) -} - -// ResolvePathComponents fetches the nodes for each segment of the given path. -// It uses the first path component as a hash (key) of the first node, then -// resolves all other components walking the links, with ResolveLinks. -func (r *Resolver) ResolvePathComponents(ctx context.Context, fpath path.Path) ([]ipld.Node, error) { - evt := log.EventBegin(ctx, "resolvePathComponents", logging.LoggableMap{"fpath": fpath}) - defer evt.Done() - - h, parts, err := path.SplitAbsPath(fpath) - if err != nil { - evt.Append(logging.LoggableMap{"error": err.Error()}) - return nil, err - } - - log.Debug("resolve dag get") - nd, err := r.DAG.Get(ctx, h) - if err != nil { - evt.Append(logging.LoggableMap{"error": err.Error()}) - return nil, err - } - - return r.ResolveLinks(ctx, nd, parts) -} - -// ResolveLinks iteratively resolves names by walking the link hierarchy. -// Every node is fetched from the DAGService, resolving the next name. -// Returns the list of nodes forming the path, starting with ndd. This list is -// guaranteed never to be empty. -// -// ResolveLinks(nd, []string{"foo", "bar", "baz"}) -// would retrieve "baz" in ("bar" in ("foo" in nd.Links).Links).Links -func (r *Resolver) ResolveLinks(ctx context.Context, ndd ipld.Node, names []string) ([]ipld.Node, error) { - - evt := log.EventBegin(ctx, "resolveLinks", logging.LoggableMap{"names": names}) - defer evt.Done() - result := make([]ipld.Node, 0, len(names)+1) - result = append(result, ndd) - nd := ndd // dup arg workaround - - // for each of the path components - for len(names) > 0 { - var cancel context.CancelFunc - ctx, cancel = context.WithTimeout(ctx, time.Minute) - defer cancel() - - lnk, rest, err := r.ResolveOnce(ctx, r.DAG, nd, names) - if err == dag.ErrLinkNotFound { - evt.Append(logging.LoggableMap{"error": err.Error()}) - return result, ErrNoLink{Name: names[0], Node: nd.Cid()} - } else if err != nil { - evt.Append(logging.LoggableMap{"error": err.Error()}) - return result, err - } - - nextnode, err := lnk.GetNode(ctx, r.DAG) - if err != nil { - evt.Append(logging.LoggableMap{"error": err.Error()}) - return result, err - } - - nd = nextnode - result = append(result, nextnode) - names = rest - } - return result, nil -} diff --git a/path/resolver/resolver_test.go b/path/resolver/resolver_test.go deleted file mode 100644 index 79a857cb6c6..00000000000 --- a/path/resolver/resolver_test.go +++ /dev/null @@ -1,70 +0,0 @@ -package resolver_test - -import ( - "context" - "fmt" - "testing" - - merkledag "github.com/ipfs/go-ipfs/merkledag" - dagmock "github.com/ipfs/go-ipfs/merkledag/test" - path "github.com/ipfs/go-ipfs/path" - "github.com/ipfs/go-ipfs/path/resolver" - - util "gx/ipfs/QmNiJuT8Ja3hMVpBHXv3Q6dwmperaQ6JjLtpMQgMCD7xvx/go-ipfs-util" - ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" -) - -func randNode() *merkledag.ProtoNode { - node := new(merkledag.ProtoNode) - node.SetData(make([]byte, 32)) - util.NewTimeSeededRand().Read(node.Data()) - return node -} - -func TestRecurivePathResolution(t *testing.T) { - ctx := context.Background() - dagService := dagmock.Mock() - - a := randNode() - b := randNode() - c := randNode() - - err := b.AddNodeLink("grandchild", c) - if err != nil { - t.Fatal(err) - } - - err = a.AddNodeLink("child", b) - if err != nil { - t.Fatal(err) - } - - for _, n := range []ipld.Node{a, b, c} { - err = dagService.Add(ctx, n) - if err != nil { - t.Fatal(err) - } - } - - aKey := a.Cid() - - segments := []string{aKey.String(), "child", "grandchild"} - p, err := path.FromSegments("/ipfs/", segments...) - if err != nil { - t.Fatal(err) - } - - resolver := resolver.NewBasicResolver(dagService) - node, err := resolver.ResolvePath(ctx, p) - if err != nil { - t.Fatal(err) - } - - cKey := c.Cid() - key := node.Cid() - if key.String() != cKey.String() { - t.Fatal(fmt.Errorf( - "recursive path resolution failed for %s: %s != %s", - p.String(), key.String(), cKey.String())) - } -} From c6213420c7d885b8601af0907c7b206e62366796 Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Sat, 21 Apr 2018 22:19:21 +0200 Subject: [PATCH 2/5] extract path --- core/commands/dag/dag.go | 2 +- core/commands/dht.go | 2 +- core/commands/files.go | 2 +- core/commands/get.go | 2 +- core/commands/ls.go | 2 +- core/commands/object/diff.go | 5 +- core/commands/object/object.go | 2 +- core/commands/object/patch.go | 2 +- core/commands/pin.go | 2 +- core/commands/publish.go | 4 +- core/commands/refs.go | 2 +- core/commands/resolve.go | 2 +- core/commands/tar.go | 2 +- core/commands/unixfs/ls.go | 2 +- core/coreapi/coreapi.go | 2 +- core/coreapi/key.go | 2 +- core/coreapi/name.go | 2 +- core/coreapi/name_test.go | 2 +- core/corehttp/commands.go | 2 +- core/corehttp/gateway_handler.go | 2 +- core/corehttp/gateway_test.go | 2 +- core/corerepo/pinning.go | 2 +- core/coreunix/cat.go | 2 +- core/pathresolver.go | 2 +- core/pathresolver_test.go | 2 +- fuse/ipns/common.go | 5 +- fuse/ipns/ipns_unix.go | 2 +- fuse/readonly/readonly_unix.go | 2 +- merkledag/utils/utils.go | 2 +- merkledag/utils/utils_test.go | 2 +- mfs/mfs_test.go | 2 +- mfs/ops.go | 2 +- namesys/base.go | 2 +- namesys/dns.go | 5 +- namesys/interface.go | 2 +- namesys/ipns_select_test.go | 2 +- namesys/ipns_validate_test.go | 2 +- namesys/namesys.go | 2 +- namesys/namesys_test.go | 2 +- namesys/proquint.go | 5 +- namesys/publisher.go | 2 +- namesys/publisher_test.go | 2 +- namesys/pubsub.go | 2 +- namesys/pubsub_test.go | 2 +- namesys/republisher/repub.go | 2 +- namesys/republisher/repub_test.go | 2 +- namesys/resolve_test.go | 2 +- namesys/routing.go | 2 +- path/path.go | 180 ------------------------------ path/path_test.go | 79 ------------- tar/format.go | 2 +- 51 files changed, 58 insertions(+), 313 deletions(-) delete mode 100644 path/path.go delete mode 100644 path/path_test.go diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index 2e069a60f72..b1929246be7 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -7,10 +7,10 @@ import ( "math" "strings" + path "github.com/ipfs/go-ipfs-path" cmds "github.com/ipfs/go-ipfs/commands" e "github.com/ipfs/go-ipfs/core/commands/e" coredag "github.com/ipfs/go-ipfs/core/coredag" - path "github.com/ipfs/go-ipfs/path" pin "github.com/ipfs/go-ipfs/pin" mh "gx/ipfs/QmZyZDi491cCNTLfAhwcaDii2Kg4pwKRkhqQzURGDvY6ua/go-multihash" diff --git a/core/commands/dht.go b/core/commands/dht.go index aceba7cea14..216fe8d11cc 100644 --- a/core/commands/dht.go +++ b/core/commands/dht.go @@ -8,10 +8,10 @@ import ( "io" "time" + path "github.com/ipfs/go-ipfs-path" cmds "github.com/ipfs/go-ipfs/commands" e "github.com/ipfs/go-ipfs/core/commands/e" dag "github.com/ipfs/go-ipfs/merkledag" - path "github.com/ipfs/go-ipfs/path" routing "gx/ipfs/QmTiWLZ6Fo5j4KcTVutZJ5KWRRJrbxzmxA4td8NfEdrPh7/go-libp2p-routing" notif "gx/ipfs/QmTiWLZ6Fo5j4KcTVutZJ5KWRRJrbxzmxA4td8NfEdrPh7/go-libp2p-routing/notifications" diff --git a/core/commands/files.go b/core/commands/files.go index c6cfec1d6a4..a767f403a27 100644 --- a/core/commands/files.go +++ b/core/commands/files.go @@ -10,6 +10,7 @@ import ( gopath "path" "strings" + path "github.com/ipfs/go-ipfs-path" resolver "github.com/ipfs/go-ipfs-path-resolver" bservice "github.com/ipfs/go-ipfs/blockservice" oldcmds "github.com/ipfs/go-ipfs/commands" @@ -18,7 +19,6 @@ import ( e "github.com/ipfs/go-ipfs/core/commands/e" dag "github.com/ipfs/go-ipfs/merkledag" mfs "github.com/ipfs/go-ipfs/mfs" - path "github.com/ipfs/go-ipfs/path" ft "github.com/ipfs/go-ipfs/unixfs" uio "github.com/ipfs/go-ipfs/unixfs/io" diff --git a/core/commands/get.go b/core/commands/get.go index 752c9924a3e..8a841f7972c 100644 --- a/core/commands/get.go +++ b/core/commands/get.go @@ -9,10 +9,10 @@ import ( "path/filepath" "strings" + path "github.com/ipfs/go-ipfs-path" core "github.com/ipfs/go-ipfs/core" e "github.com/ipfs/go-ipfs/core/commands/e" dag "github.com/ipfs/go-ipfs/merkledag" - path "github.com/ipfs/go-ipfs/path" uarchive "github.com/ipfs/go-ipfs/unixfs/archive" tar "gx/ipfs/QmQine7gvHncNevKtG9QXxf3nXcwSj6aDDmMm52mHofEEp/tar-utils" diff --git a/core/commands/ls.go b/core/commands/ls.go index 4f64c4a1eba..7385e8ab0b5 100644 --- a/core/commands/ls.go +++ b/core/commands/ls.go @@ -6,13 +6,13 @@ import ( "io" "text/tabwriter" + path "github.com/ipfs/go-ipfs-path" resolver "github.com/ipfs/go-ipfs-path-resolver" blockservice "github.com/ipfs/go-ipfs/blockservice" cmds "github.com/ipfs/go-ipfs/commands" core "github.com/ipfs/go-ipfs/core" e "github.com/ipfs/go-ipfs/core/commands/e" merkledag "github.com/ipfs/go-ipfs/merkledag" - path "github.com/ipfs/go-ipfs/path" unixfs "github.com/ipfs/go-ipfs/unixfs" uio "github.com/ipfs/go-ipfs/unixfs/io" unixfspb "github.com/ipfs/go-ipfs/unixfs/pb" diff --git a/core/commands/object/diff.go b/core/commands/object/diff.go index db3303d3294..199951ea30b 100644 --- a/core/commands/object/diff.go +++ b/core/commands/object/diff.go @@ -5,12 +5,13 @@ import ( "fmt" "io" + cmdkit "gx/ipfs/QmceUdzxkimdYsgtX733uNgzf1DLHyBKN6ehGSp85ayppM/go-ipfs-cmdkit" + + path "github.com/ipfs/go-ipfs-path" cmds "github.com/ipfs/go-ipfs/commands" core "github.com/ipfs/go-ipfs/core" e "github.com/ipfs/go-ipfs/core/commands/e" dagutils "github.com/ipfs/go-ipfs/merkledag/utils" - path "github.com/ipfs/go-ipfs/path" - cmdkit "gx/ipfs/QmceUdzxkimdYsgtX733uNgzf1DLHyBKN6ehGSp85ayppM/go-ipfs-cmdkit" ) type Changes struct { diff --git a/core/commands/object/object.go b/core/commands/object/object.go index 73352bf8445..ef1dc3e060f 100644 --- a/core/commands/object/object.go +++ b/core/commands/object/object.go @@ -13,12 +13,12 @@ import ( "strings" "text/tabwriter" + path "github.com/ipfs/go-ipfs-path" oldcmds "github.com/ipfs/go-ipfs/commands" lgc "github.com/ipfs/go-ipfs/commands/legacy" core "github.com/ipfs/go-ipfs/core" e "github.com/ipfs/go-ipfs/core/commands/e" dag "github.com/ipfs/go-ipfs/merkledag" - path "github.com/ipfs/go-ipfs/path" pin "github.com/ipfs/go-ipfs/pin" ft "github.com/ipfs/go-ipfs/unixfs" diff --git a/core/commands/object/patch.go b/core/commands/object/patch.go index 6355ccd10e9..623408273cf 100644 --- a/core/commands/object/patch.go +++ b/core/commands/object/patch.go @@ -6,13 +6,13 @@ import ( "io/ioutil" "strings" + path "github.com/ipfs/go-ipfs-path" oldcmds "github.com/ipfs/go-ipfs/commands" lgc "github.com/ipfs/go-ipfs/commands/legacy" core "github.com/ipfs/go-ipfs/core" e "github.com/ipfs/go-ipfs/core/commands/e" dag "github.com/ipfs/go-ipfs/merkledag" dagutils "github.com/ipfs/go-ipfs/merkledag/utils" - path "github.com/ipfs/go-ipfs/path" ft "github.com/ipfs/go-ipfs/unixfs" logging "gx/ipfs/QmRb5jh8z2E8hMGN2tkvs1yHynUanqnZ3UeKwgN1i9P1F8/go-log" diff --git a/core/commands/pin.go b/core/commands/pin.go index 7d0afde7fdb..80c8b895fb8 100644 --- a/core/commands/pin.go +++ b/core/commands/pin.go @@ -7,6 +7,7 @@ import ( "io" "time" + path "github.com/ipfs/go-ipfs-path" resolver "github.com/ipfs/go-ipfs-path-resolver" bserv "github.com/ipfs/go-ipfs/blockservice" cmds "github.com/ipfs/go-ipfs/commands" @@ -14,7 +15,6 @@ import ( e "github.com/ipfs/go-ipfs/core/commands/e" corerepo "github.com/ipfs/go-ipfs/core/corerepo" dag "github.com/ipfs/go-ipfs/merkledag" - path "github.com/ipfs/go-ipfs/path" pin "github.com/ipfs/go-ipfs/pin" "github.com/ipfs/go-ipfs/thirdparty/verifcid" uio "github.com/ipfs/go-ipfs/unixfs/io" diff --git a/core/commands/publish.go b/core/commands/publish.go index bf9d9719c1a..499267e1b4a 100644 --- a/core/commands/publish.go +++ b/core/commands/publish.go @@ -8,11 +8,11 @@ import ( "strings" "time" + path "github.com/ipfs/go-ipfs-path" cmds "github.com/ipfs/go-ipfs/commands" core "github.com/ipfs/go-ipfs/core" e "github.com/ipfs/go-ipfs/core/commands/e" keystore "github.com/ipfs/go-ipfs/keystore" - path "github.com/ipfs/go-ipfs/path" peer "gx/ipfs/QmZoWKhxUmZ2seW4BzX6fJkNR8hh9PsGModr7q171yq2SS/go-libp2p-peer" crypto "gx/ipfs/QmaPbCnUMBohSGo3KnxEa2bHqyJVVeEEcwtqJAYxerieBo/go-libp2p-crypto" @@ -52,7 +52,7 @@ Publish an with another name, added by an 'ipfs key' command: > ipfs name publish --key=mykey /ipfs/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy Published to QmSrPmbaUKA3ZodhzPWZnpFgcPMFWF4QsxXbkWfEptTBJd: /ipfs/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy -Alternatively, publish an using a valid PeerID (as listed by +Alternatively, publish an using a valid PeerID (as listed by 'ipfs key list -l'): > ipfs name publish --key=QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n /ipfs/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy diff --git a/core/commands/refs.go b/core/commands/refs.go index 294ef9bd70d..ad1a4aa0577 100644 --- a/core/commands/refs.go +++ b/core/commands/refs.go @@ -7,10 +7,10 @@ import ( "io" "strings" + path "github.com/ipfs/go-ipfs-path" cmds "github.com/ipfs/go-ipfs/commands" "github.com/ipfs/go-ipfs/core" e "github.com/ipfs/go-ipfs/core/commands/e" - path "github.com/ipfs/go-ipfs/path" cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" "gx/ipfs/QmceUdzxkimdYsgtX733uNgzf1DLHyBKN6ehGSp85ayppM/go-ipfs-cmdkit" diff --git a/core/commands/resolve.go b/core/commands/resolve.go index 48452da0da1..e923e7e0254 100644 --- a/core/commands/resolve.go +++ b/core/commands/resolve.go @@ -6,12 +6,12 @@ import ( "strings" "time" + path "github.com/ipfs/go-ipfs-path" cmds "github.com/ipfs/go-ipfs/commands" "github.com/ipfs/go-ipfs/core" e "github.com/ipfs/go-ipfs/core/commands/e" ns "github.com/ipfs/go-ipfs/namesys" nsopts "github.com/ipfs/go-ipfs/namesys/opts" - path "github.com/ipfs/go-ipfs/path" "gx/ipfs/QmceUdzxkimdYsgtX733uNgzf1DLHyBKN6ehGSp85ayppM/go-ipfs-cmdkit" ) diff --git a/core/commands/tar.go b/core/commands/tar.go index d95974d448c..6516f83d060 100644 --- a/core/commands/tar.go +++ b/core/commands/tar.go @@ -4,12 +4,12 @@ import ( "io" "strings" + path "github.com/ipfs/go-ipfs-path" cmds "github.com/ipfs/go-ipfs/commands" core "github.com/ipfs/go-ipfs/core" e "github.com/ipfs/go-ipfs/core/commands/e" "github.com/ipfs/go-ipfs/core/coreunix" dag "github.com/ipfs/go-ipfs/merkledag" - path "github.com/ipfs/go-ipfs/path" tar "github.com/ipfs/go-ipfs/tar" "gx/ipfs/QmceUdzxkimdYsgtX733uNgzf1DLHyBKN6ehGSp85ayppM/go-ipfs-cmdkit" diff --git a/core/commands/unixfs/ls.go b/core/commands/unixfs/ls.go index 8b15a2c67af..ead1536c0eb 100644 --- a/core/commands/unixfs/ls.go +++ b/core/commands/unixfs/ls.go @@ -9,12 +9,12 @@ import ( cmdkit "gx/ipfs/QmceUdzxkimdYsgtX733uNgzf1DLHyBKN6ehGSp85ayppM/go-ipfs-cmdkit" + path "github.com/ipfs/go-ipfs-path" resolver "github.com/ipfs/go-ipfs-path-resolver" cmds "github.com/ipfs/go-ipfs/commands" core "github.com/ipfs/go-ipfs/core" e "github.com/ipfs/go-ipfs/core/commands/e" merkledag "github.com/ipfs/go-ipfs/merkledag" - path "github.com/ipfs/go-ipfs/path" unixfs "github.com/ipfs/go-ipfs/unixfs" uio "github.com/ipfs/go-ipfs/unixfs/io" unixfspb "github.com/ipfs/go-ipfs/unixfs/pb" diff --git a/core/coreapi/coreapi.go b/core/coreapi/coreapi.go index 9a098d1dfdf..d7cd25f1e43 100644 --- a/core/coreapi/coreapi.go +++ b/core/coreapi/coreapi.go @@ -3,11 +3,11 @@ package coreapi import ( "context" + ipfspath "github.com/ipfs/go-ipfs-path" resolver "github.com/ipfs/go-ipfs-path-resolver" core "github.com/ipfs/go-ipfs/core" coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface" namesys "github.com/ipfs/go-ipfs/namesys" - ipfspath "github.com/ipfs/go-ipfs/path" uio "github.com/ipfs/go-ipfs/unixfs/io" cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" diff --git a/core/coreapi/key.go b/core/coreapi/key.go index 7e8268e6cbd..5412808a75a 100644 --- a/core/coreapi/key.go +++ b/core/coreapi/key.go @@ -6,9 +6,9 @@ import ( "fmt" "sort" + ipfspath "github.com/ipfs/go-ipfs-path" coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface" caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - ipfspath "github.com/ipfs/go-ipfs/path" peer "gx/ipfs/QmZoWKhxUmZ2seW4BzX6fJkNR8hh9PsGModr7q171yq2SS/go-libp2p-peer" crypto "gx/ipfs/QmaPbCnUMBohSGo3KnxEa2bHqyJVVeEEcwtqJAYxerieBo/go-libp2p-crypto" diff --git a/core/coreapi/name.go b/core/coreapi/name.go index 7559560eaaa..a404a6917e8 100644 --- a/core/coreapi/name.go +++ b/core/coreapi/name.go @@ -7,13 +7,13 @@ import ( "strings" "time" + ipath "github.com/ipfs/go-ipfs-path" core "github.com/ipfs/go-ipfs/core" coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface" caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" keystore "github.com/ipfs/go-ipfs/keystore" namesys "github.com/ipfs/go-ipfs/namesys" nsopts "github.com/ipfs/go-ipfs/namesys/opts" - ipath "github.com/ipfs/go-ipfs/path" offline "gx/ipfs/QmXtoXbu9ReyV6Q4kDQ5CF9wXQNDY1PdHc4HhfxRR5AHB3/go-ipfs-routing/offline" peer "gx/ipfs/QmZoWKhxUmZ2seW4BzX6fJkNR8hh9PsGModr7q171yq2SS/go-libp2p-peer" diff --git a/core/coreapi/name_test.go b/core/coreapi/name_test.go index bfb1802df8e..93bcc56f961 100644 --- a/core/coreapi/name_test.go +++ b/core/coreapi/name_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - ipath "github.com/ipfs/go-ipfs/path" + ipath "github.com/ipfs/go-ipfs-path" coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface" opt "github.com/ipfs/go-ipfs/core/coreapi/interface/options" diff --git a/core/corehttp/commands.go b/core/corehttp/commands.go index f01d9233fd7..b03400e8342 100644 --- a/core/corehttp/commands.go +++ b/core/corehttp/commands.go @@ -9,10 +9,10 @@ import ( "strconv" "strings" + path "github.com/ipfs/go-ipfs-path" oldcmds "github.com/ipfs/go-ipfs/commands" core "github.com/ipfs/go-ipfs/core" corecommands "github.com/ipfs/go-ipfs/core/commands" - path "github.com/ipfs/go-ipfs/path" config "github.com/ipfs/go-ipfs/repo/config" cmds "gx/ipfs/QmfAkMSt9Fwzk48QDJecPcwCUjnf2uG7MLnmCGTp4C6ouL/go-ipfs-cmds" diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go index 833a2646a9c..84dba4666b3 100644 --- a/core/corehttp/gateway_handler.go +++ b/core/corehttp/gateway_handler.go @@ -12,6 +12,7 @@ import ( "strings" "time" + path "github.com/ipfs/go-ipfs-path" resolver "github.com/ipfs/go-ipfs-path-resolver" core "github.com/ipfs/go-ipfs/core" coreapi "github.com/ipfs/go-ipfs/core/coreapi" @@ -19,7 +20,6 @@ import ( "github.com/ipfs/go-ipfs/importer" dag "github.com/ipfs/go-ipfs/merkledag" dagutils "github.com/ipfs/go-ipfs/merkledag/utils" - path "github.com/ipfs/go-ipfs/path" ft "github.com/ipfs/go-ipfs/unixfs" uio "github.com/ipfs/go-ipfs/unixfs/io" diff --git a/core/corehttp/gateway_test.go b/core/corehttp/gateway_test.go index af7a2b72694..3c81f1457f2 100644 --- a/core/corehttp/gateway_test.go +++ b/core/corehttp/gateway_test.go @@ -10,12 +10,12 @@ import ( "testing" "time" + path "github.com/ipfs/go-ipfs-path" core "github.com/ipfs/go-ipfs/core" coreunix "github.com/ipfs/go-ipfs/core/coreunix" dag "github.com/ipfs/go-ipfs/merkledag" namesys "github.com/ipfs/go-ipfs/namesys" nsopts "github.com/ipfs/go-ipfs/namesys/opts" - path "github.com/ipfs/go-ipfs/path" repo "github.com/ipfs/go-ipfs/repo" config "github.com/ipfs/go-ipfs/repo/config" diff --git a/core/corerepo/pinning.go b/core/corerepo/pinning.go index 8c95c8a3686..5c26d09c85a 100644 --- a/core/corerepo/pinning.go +++ b/core/corerepo/pinning.go @@ -17,9 +17,9 @@ import ( "context" "fmt" + path "github.com/ipfs/go-ipfs-path" resolver "github.com/ipfs/go-ipfs-path-resolver" "github.com/ipfs/go-ipfs/core" - path "github.com/ipfs/go-ipfs/path" uio "github.com/ipfs/go-ipfs/unixfs/io" cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" diff --git a/core/coreunix/cat.go b/core/coreunix/cat.go index e1e2bf71744..615e114f04d 100644 --- a/core/coreunix/cat.go +++ b/core/coreunix/cat.go @@ -3,9 +3,9 @@ package coreunix import ( "context" + path "github.com/ipfs/go-ipfs-path" resolver "github.com/ipfs/go-ipfs-path-resolver" core "github.com/ipfs/go-ipfs/core" - path "github.com/ipfs/go-ipfs/path" uio "github.com/ipfs/go-ipfs/unixfs/io" ) diff --git a/core/pathresolver.go b/core/pathresolver.go index 634e23a78f3..91f855e8f1f 100644 --- a/core/pathresolver.go +++ b/core/pathresolver.go @@ -5,9 +5,9 @@ import ( "errors" "strings" + path "github.com/ipfs/go-ipfs-path" resolver "github.com/ipfs/go-ipfs-path-resolver" namesys "github.com/ipfs/go-ipfs/namesys" - path "github.com/ipfs/go-ipfs/path" logging "gx/ipfs/QmRb5jh8z2E8hMGN2tkvs1yHynUanqnZ3UeKwgN1i9P1F8/go-log" cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" diff --git a/core/pathresolver_test.go b/core/pathresolver_test.go index 3f5f5a25e7f..68cbd0a8fb4 100644 --- a/core/pathresolver_test.go +++ b/core/pathresolver_test.go @@ -3,9 +3,9 @@ package core_test import ( "testing" + path "github.com/ipfs/go-ipfs-path" core "github.com/ipfs/go-ipfs/core" coremock "github.com/ipfs/go-ipfs/core/mock" - path "github.com/ipfs/go-ipfs/path" ) func TestResolveNoComponents(t *testing.T) { diff --git a/fuse/ipns/common.go b/fuse/ipns/common.go index 4c9b0d341e9..8ea34910e49 100644 --- a/fuse/ipns/common.go +++ b/fuse/ipns/common.go @@ -3,11 +3,12 @@ package ipns import ( "context" + ci "gx/ipfs/QmaPbCnUMBohSGo3KnxEa2bHqyJVVeEEcwtqJAYxerieBo/go-libp2p-crypto" + + path "github.com/ipfs/go-ipfs-path" "github.com/ipfs/go-ipfs/core" nsys "github.com/ipfs/go-ipfs/namesys" - path "github.com/ipfs/go-ipfs/path" ft "github.com/ipfs/go-ipfs/unixfs" - ci "gx/ipfs/QmaPbCnUMBohSGo3KnxEa2bHqyJVVeEEcwtqJAYxerieBo/go-libp2p-crypto" ) // InitializeKeyspace sets the ipns record for the given key to diff --git a/fuse/ipns/ipns_unix.go b/fuse/ipns/ipns_unix.go index e4d95c31a72..52868cbfae4 100644 --- a/fuse/ipns/ipns_unix.go +++ b/fuse/ipns/ipns_unix.go @@ -11,11 +11,11 @@ import ( "io" "os" + path "github.com/ipfs/go-ipfs-path" core "github.com/ipfs/go-ipfs/core" dag "github.com/ipfs/go-ipfs/merkledag" mfs "github.com/ipfs/go-ipfs/mfs" namesys "github.com/ipfs/go-ipfs/namesys" - path "github.com/ipfs/go-ipfs/path" ft "github.com/ipfs/go-ipfs/unixfs" logging "gx/ipfs/QmRb5jh8z2E8hMGN2tkvs1yHynUanqnZ3UeKwgN1i9P1F8/go-log" diff --git a/fuse/readonly/readonly_unix.go b/fuse/readonly/readonly_unix.go index 710fe8721ca..7313bef9db4 100644 --- a/fuse/readonly/readonly_unix.go +++ b/fuse/readonly/readonly_unix.go @@ -10,9 +10,9 @@ import ( "os" "syscall" + path "github.com/ipfs/go-ipfs-path" core "github.com/ipfs/go-ipfs/core" mdag "github.com/ipfs/go-ipfs/merkledag" - path "github.com/ipfs/go-ipfs/path" uio "github.com/ipfs/go-ipfs/unixfs/io" ftpb "github.com/ipfs/go-ipfs/unixfs/pb" diff --git a/merkledag/utils/utils.go b/merkledag/utils/utils.go index 4416fb15d21..284598732df 100644 --- a/merkledag/utils/utils.go +++ b/merkledag/utils/utils.go @@ -4,9 +4,9 @@ import ( "context" "errors" + path "github.com/ipfs/go-ipfs-path" bserv "github.com/ipfs/go-ipfs/blockservice" dag "github.com/ipfs/go-ipfs/merkledag" - path "github.com/ipfs/go-ipfs/path" offline "gx/ipfs/QmWM5HhdG5ZQNyHQ5XhMdGmV9CvLpFynQfGpTxN2MEM7Lc/go-ipfs-exchange-offline" ds "gx/ipfs/QmXRKBQA4wXP7xWbFiZsR1GP4HV6wMDQ1aWFxZZ4uBcPX9/go-datastore" diff --git a/merkledag/utils/utils_test.go b/merkledag/utils/utils_test.go index eb440c39cf7..81af6ddb4f6 100644 --- a/merkledag/utils/utils_test.go +++ b/merkledag/utils/utils_test.go @@ -4,9 +4,9 @@ import ( "context" "testing" + path "github.com/ipfs/go-ipfs-path" dag "github.com/ipfs/go-ipfs/merkledag" mdtest "github.com/ipfs/go-ipfs/merkledag/test" - path "github.com/ipfs/go-ipfs/path" cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" diff --git a/mfs/mfs_test.go b/mfs/mfs_test.go index f7f8c877bc2..b7f984db08a 100644 --- a/mfs/mfs_test.go +++ b/mfs/mfs_test.go @@ -14,10 +14,10 @@ import ( "testing" "time" + "github.com/ipfs/go-ipfs-path" bserv "github.com/ipfs/go-ipfs/blockservice" importer "github.com/ipfs/go-ipfs/importer" dag "github.com/ipfs/go-ipfs/merkledag" - "github.com/ipfs/go-ipfs/path" ft "github.com/ipfs/go-ipfs/unixfs" uio "github.com/ipfs/go-ipfs/unixfs/io" diff --git a/mfs/ops.go b/mfs/ops.go index e6ad1a3bea1..ce68d04370b 100644 --- a/mfs/ops.go +++ b/mfs/ops.go @@ -7,7 +7,7 @@ import ( gopath "path" "strings" - path "github.com/ipfs/go-ipfs/path" + path "github.com/ipfs/go-ipfs-path" cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" diff --git a/namesys/base.go b/namesys/base.go index a301a5a6120..996bc200cf7 100644 --- a/namesys/base.go +++ b/namesys/base.go @@ -5,8 +5,8 @@ import ( context "context" + path "github.com/ipfs/go-ipfs-path" opts "github.com/ipfs/go-ipfs/namesys/opts" - path "github.com/ipfs/go-ipfs/path" ) type resolver interface { diff --git a/namesys/dns.go b/namesys/dns.go index 6d74e52216b..a2c6430dffd 100644 --- a/namesys/dns.go +++ b/namesys/dns.go @@ -6,9 +6,10 @@ import ( "net" "strings" - opts "github.com/ipfs/go-ipfs/namesys/opts" - path "github.com/ipfs/go-ipfs/path" isd "gx/ipfs/QmZmmuAXgX73UQmX1jRKjTGmjzq24Jinqkq8vzkBtno4uX/go-is-domain" + + path "github.com/ipfs/go-ipfs-path" + opts "github.com/ipfs/go-ipfs/namesys/opts" ) type LookupTXTFunc func(name string) (txt []string, err error) diff --git a/namesys/interface.go b/namesys/interface.go index 4def9b1d713..e3fc3cf0e15 100644 --- a/namesys/interface.go +++ b/namesys/interface.go @@ -35,8 +35,8 @@ import ( context "context" + path "github.com/ipfs/go-ipfs-path" opts "github.com/ipfs/go-ipfs/namesys/opts" - path "github.com/ipfs/go-ipfs/path" ci "gx/ipfs/QmaPbCnUMBohSGo3KnxEa2bHqyJVVeEEcwtqJAYxerieBo/go-libp2p-crypto" ) diff --git a/namesys/ipns_select_test.go b/namesys/ipns_select_test.go index 7489f139b16..710cf52f320 100644 --- a/namesys/ipns_select_test.go +++ b/namesys/ipns_select_test.go @@ -6,8 +6,8 @@ import ( "testing" "time" + path "github.com/ipfs/go-ipfs-path" pb "github.com/ipfs/go-ipfs/namesys/pb" - path "github.com/ipfs/go-ipfs/path" u "gx/ipfs/QmNiJuT8Ja3hMVpBHXv3Q6dwmperaQ6JjLtpMQgMCD7xvx/go-ipfs-util" proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto" diff --git a/namesys/ipns_validate_test.go b/namesys/ipns_validate_test.go index ac3fb847014..324d32d40f3 100644 --- a/namesys/ipns_validate_test.go +++ b/namesys/ipns_validate_test.go @@ -6,8 +6,8 @@ import ( "testing" "time" + path "github.com/ipfs/go-ipfs-path" opts "github.com/ipfs/go-ipfs/namesys/opts" - path "github.com/ipfs/go-ipfs/path" u "gx/ipfs/QmNiJuT8Ja3hMVpBHXv3Q6dwmperaQ6JjLtpMQgMCD7xvx/go-ipfs-util" routing "gx/ipfs/QmTiWLZ6Fo5j4KcTVutZJ5KWRRJrbxzmxA4td8NfEdrPh7/go-libp2p-routing" diff --git a/namesys/namesys.go b/namesys/namesys.go index 9b4cdf3f729..9ae092b6cb1 100644 --- a/namesys/namesys.go +++ b/namesys/namesys.go @@ -7,8 +7,8 @@ import ( "sync" "time" + path "github.com/ipfs/go-ipfs-path" opts "github.com/ipfs/go-ipfs/namesys/opts" - path "github.com/ipfs/go-ipfs/path" p2phost "gx/ipfs/QmNmJZL7FQySMtE2BQuLMuZg2EB2CLEunJJUSVSc9YnnbV/go-libp2p-host" floodsub "gx/ipfs/QmSFihvoND3eDaAYRCeLgLPt62yCPgMZs1NSZmKFEtJQQw/go-libp2p-floodsub" diff --git a/namesys/namesys_test.go b/namesys/namesys_test.go index 9b27cf32204..a9ff12d4f84 100644 --- a/namesys/namesys_test.go +++ b/namesys/namesys_test.go @@ -6,8 +6,8 @@ import ( context "context" + path "github.com/ipfs/go-ipfs-path" opts "github.com/ipfs/go-ipfs/namesys/opts" - path "github.com/ipfs/go-ipfs/path" "github.com/ipfs/go-ipfs/unixfs" ds "gx/ipfs/QmXRKBQA4wXP7xWbFiZsR1GP4HV6wMDQ1aWFxZZ4uBcPX9/go-datastore" diff --git a/namesys/proquint.go b/namesys/proquint.go index 2c61c98d3aa..e3583497082 100644 --- a/namesys/proquint.go +++ b/namesys/proquint.go @@ -5,9 +5,10 @@ import ( context "context" - opts "github.com/ipfs/go-ipfs/namesys/opts" - path "github.com/ipfs/go-ipfs/path" proquint "gx/ipfs/QmYnf27kzqR2cxt6LFZdrAFJuQd6785fTkBvMuEj9EeRxM/proquint" + + path "github.com/ipfs/go-ipfs-path" + opts "github.com/ipfs/go-ipfs/namesys/opts" ) type ProquintResolver struct{} diff --git a/namesys/publisher.go b/namesys/publisher.go index b485b24b21f..72582bdf417 100644 --- a/namesys/publisher.go +++ b/namesys/publisher.go @@ -6,8 +6,8 @@ import ( "fmt" "time" + path "github.com/ipfs/go-ipfs-path" pb "github.com/ipfs/go-ipfs/namesys/pb" - path "github.com/ipfs/go-ipfs/path" pin "github.com/ipfs/go-ipfs/pin" ft "github.com/ipfs/go-ipfs/unixfs" diff --git a/namesys/publisher_test.go b/namesys/publisher_test.go index 20f774fa3c2..b614487b9b0 100644 --- a/namesys/publisher_test.go +++ b/namesys/publisher_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - path "github.com/ipfs/go-ipfs/path" + path "github.com/ipfs/go-ipfs-path" dshelp "gx/ipfs/QmTmqJGRQfuH8eKWD1FjThwPRipt1QhqJQNZ8MpzmfAAxo/go-ipfs-ds-help" testutil "gx/ipfs/QmVvkK7s5imCiq3JVbL3pGfnhcCnf3LrFJPF4GE2sAoGZf/go-testutil" diff --git a/namesys/pubsub.go b/namesys/pubsub.go index 9171b16ccff..2d6f7cff0fd 100644 --- a/namesys/pubsub.go +++ b/namesys/pubsub.go @@ -8,9 +8,9 @@ import ( "sync" "time" + path "github.com/ipfs/go-ipfs-path" opts "github.com/ipfs/go-ipfs/namesys/opts" pb "github.com/ipfs/go-ipfs/namesys/pb" - path "github.com/ipfs/go-ipfs/path" u "gx/ipfs/QmNiJuT8Ja3hMVpBHXv3Q6dwmperaQ6JjLtpMQgMCD7xvx/go-ipfs-util" p2phost "gx/ipfs/QmNmJZL7FQySMtE2BQuLMuZg2EB2CLEunJJUSVSc9YnnbV/go-libp2p-host" diff --git a/namesys/pubsub_test.go b/namesys/pubsub_test.go index 16dae39fc99..0f81e22b635 100644 --- a/namesys/pubsub_test.go +++ b/namesys/pubsub_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - path "github.com/ipfs/go-ipfs/path" + path "github.com/ipfs/go-ipfs-path" p2phost "gx/ipfs/QmNmJZL7FQySMtE2BQuLMuZg2EB2CLEunJJUSVSc9YnnbV/go-libp2p-host" bhost "gx/ipfs/QmQr1j6UvdhpponAaqSdswqRpdzsFwNop2N8kXLNw8afem/go-libp2p-blankhost" diff --git a/namesys/republisher/repub.go b/namesys/republisher/repub.go index 0aabf37382f..e0b86aae37b 100644 --- a/namesys/republisher/repub.go +++ b/namesys/republisher/repub.go @@ -5,10 +5,10 @@ import ( "errors" "time" + path "github.com/ipfs/go-ipfs-path" keystore "github.com/ipfs/go-ipfs/keystore" namesys "github.com/ipfs/go-ipfs/namesys" pb "github.com/ipfs/go-ipfs/namesys/pb" - path "github.com/ipfs/go-ipfs/path" logging "gx/ipfs/QmRb5jh8z2E8hMGN2tkvs1yHynUanqnZ3UeKwgN1i9P1F8/go-log" goprocess "gx/ipfs/QmSF8fPo3jgVBAy8fpdjjYqgG87dkJgUprRBHRd2tmfgpP/goprocess" diff --git a/namesys/republisher/repub_test.go b/namesys/republisher/repub_test.go index 580b708de7d..8a2b9d93088 100644 --- a/namesys/republisher/repub_test.go +++ b/namesys/republisher/repub_test.go @@ -6,11 +6,11 @@ import ( "testing" "time" + path "github.com/ipfs/go-ipfs-path" "github.com/ipfs/go-ipfs/core" mock "github.com/ipfs/go-ipfs/core/mock" namesys "github.com/ipfs/go-ipfs/namesys" . "github.com/ipfs/go-ipfs/namesys/republisher" - path "github.com/ipfs/go-ipfs/path" mocknet "gx/ipfs/QmNh1kGFFdsPu79KNSaL4NUKUPb4Eiz4KHdMtFY6664RDp/go-libp2p/p2p/net/mock" goprocess "gx/ipfs/QmSF8fPo3jgVBAy8fpdjjYqgG87dkJgUprRBHRd2tmfgpP/goprocess" diff --git a/namesys/resolve_test.go b/namesys/resolve_test.go index 9999801e402..c531e19748e 100644 --- a/namesys/resolve_test.go +++ b/namesys/resolve_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - path "github.com/ipfs/go-ipfs/path" + path "github.com/ipfs/go-ipfs-path" testutil "gx/ipfs/QmVvkK7s5imCiq3JVbL3pGfnhcCnf3LrFJPF4GE2sAoGZf/go-testutil" ds "gx/ipfs/QmXRKBQA4wXP7xWbFiZsR1GP4HV6wMDQ1aWFxZZ4uBcPX9/go-datastore" diff --git a/namesys/routing.go b/namesys/routing.go index ccf30506679..684fa2acc00 100644 --- a/namesys/routing.go +++ b/namesys/routing.go @@ -5,9 +5,9 @@ import ( "strings" "time" + path "github.com/ipfs/go-ipfs-path" opts "github.com/ipfs/go-ipfs/namesys/opts" pb "github.com/ipfs/go-ipfs/namesys/pb" - path "github.com/ipfs/go-ipfs/path" u "gx/ipfs/QmNiJuT8Ja3hMVpBHXv3Q6dwmperaQ6JjLtpMQgMCD7xvx/go-ipfs-util" logging "gx/ipfs/QmRb5jh8z2E8hMGN2tkvs1yHynUanqnZ3UeKwgN1i9P1F8/go-log" diff --git a/path/path.go b/path/path.go deleted file mode 100644 index ac9348b560c..00000000000 --- a/path/path.go +++ /dev/null @@ -1,180 +0,0 @@ -// Package path contains utilities to work with ipfs paths. -package path - -import ( - "errors" - "path" - "strings" - - cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" -) - -var ( - // ErrBadPath is returned when a given path is incorrectly formatted - ErrBadPath = errors.New("invalid 'ipfs ref' path") - - // ErrNoComponents is used when Paths after a protocol - // do not contain at least one component - ErrNoComponents = errors.New( - "path must contain at least one component") -) - -// A Path represents an ipfs content path: -// * //path/to/file -// * /ipfs/ -// * /ipns//path/to/folder -// * etc -type Path string - -// ^^^ -// TODO: debate making this a private struct wrapped in a public interface -// would allow us to control creation, and cache segments. - -// FromString safely converts a string type to a Path type. -func FromString(s string) Path { - return Path(s) -} - -// FromCid safely converts a cid.Cid type to a Path type. -func FromCid(c *cid.Cid) Path { - return Path("/ipfs/" + c.String()) -} - -// Segments returns the different elements of a path -// (elements are delimited by a /). -func (p Path) Segments() []string { - cleaned := path.Clean(string(p)) - segments := strings.Split(cleaned, "/") - - // Ignore leading slash - if len(segments[0]) == 0 { - segments = segments[1:] - } - - return segments -} - -// String converts a path to string. -func (p Path) String() string { - return string(p) -} - -// IsJustAKey returns true if the path is of the form or /ipfs/. -func (p Path) IsJustAKey() bool { - parts := p.Segments() - return len(parts) == 2 && parts[0] == "ipfs" -} - -// PopLastSegment returns a new Path without its final segment, and the final -// segment, separately. If there is no more to pop (the path is just a key), -// the original path is returned. -func (p Path) PopLastSegment() (Path, string, error) { - - if p.IsJustAKey() { - return p, "", nil - } - - segs := p.Segments() - newPath, err := ParsePath("/" + strings.Join(segs[:len(segs)-1], "/")) - if err != nil { - return "", "", err - } - - return newPath, segs[len(segs)-1], nil -} - -// FromSegments returns a path given its different segments. -func FromSegments(prefix string, seg ...string) (Path, error) { - return ParsePath(prefix + strings.Join(seg, "/")) -} - -// ParsePath returns a well-formed ipfs Path. -// The returned path will always be prefixed with /ipfs/ or /ipns/. -// The prefix will be added if not present in the given string. -// This function will return an error when the given string is -// not a valid ipfs path. -func ParsePath(txt string) (Path, error) { - parts := strings.Split(txt, "/") - if len(parts) == 1 { - kp, err := ParseCidToPath(txt) - if err == nil { - return kp, nil - } - } - - // if the path doesnt begin with a '/' - // we expect this to start with a hash, and be an 'ipfs' path - if parts[0] != "" { - if _, err := ParseCidToPath(parts[0]); err != nil { - return "", ErrBadPath - } - // The case when the path starts with hash without a protocol prefix - return Path("/ipfs/" + txt), nil - } - - if len(parts) < 3 { - return "", ErrBadPath - } - - if parts[1] == "ipfs" { - if _, err := ParseCidToPath(parts[2]); err != nil { - return "", err - } - } else if parts[1] != "ipns" { - return "", ErrBadPath - } - - return Path(txt), nil -} - -// ParseCidToPath takes a CID in string form and returns a valid ipfs Path. -func ParseCidToPath(txt string) (Path, error) { - if txt == "" { - return "", ErrNoComponents - } - - c, err := cid.Decode(txt) - if err != nil { - return "", err - } - - return FromCid(c), nil -} - -// IsValid checks if a path is a valid ipfs Path. -func (p *Path) IsValid() error { - _, err := ParsePath(p.String()) - return err -} - -// Join joins strings slices using / -func Join(pths []string) string { - return strings.Join(pths, "/") -} - -// SplitList splits strings usings / -func SplitList(pth string) []string { - return strings.Split(pth, "/") -} - -// SplitAbsPath clean up and split fpath. It extracts the first component (which -// must be a Multihash) and return it separately. -func SplitAbsPath(fpath Path) (*cid.Cid, []string, error) { - parts := fpath.Segments() - if parts[0] == "ipfs" { - parts = parts[1:] - } - - // if nothing, bail. - if len(parts) == 0 { - return nil, nil, ErrNoComponents - } - - c, err := cid.Decode(parts[0]) - // first element in the path is a cid - if err != nil { - return nil, nil, err - } - - return c, parts[1:], nil -} diff --git a/path/path_test.go b/path/path_test.go deleted file mode 100644 index b095ffd98e7..00000000000 --- a/path/path_test.go +++ /dev/null @@ -1,79 +0,0 @@ -package path - -import ( - "testing" -) - -func TestPathParsing(t *testing.T) { - cases := map[string]bool{ - "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": true, - "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a": true, - "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b/c/d/e/f": true, - "/ipns/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b/c/d/e/f": true, - "/ipns/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": true, - "QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b/c/d/e/f": true, - "QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": true, - "/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": false, - "/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a": false, - "/ipfs/": false, - "ipfs/": false, - "ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": false, - } - - for p, expected := range cases { - _, err := ParsePath(p) - valid := err == nil - if valid != expected { - t.Fatalf("expected %s to have valid == %t", p, expected) - } - } -} - -func TestIsJustAKey(t *testing.T) { - cases := map[string]bool{ - "QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": true, - "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": true, - "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a": false, - "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b": false, - "/ipns/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": false, - } - - for p, expected := range cases { - path, err := ParsePath(p) - if err != nil { - t.Fatalf("ParsePath failed to parse \"%s\", but should have succeeded", p) - } - result := path.IsJustAKey() - if result != expected { - t.Fatalf("expected IsJustAKey(%s) to return %v, not %v", p, expected, result) - } - } -} - -func TestPopLastSegment(t *testing.T) { - cases := map[string][]string{ - "QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": []string{"/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n", ""}, - "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": []string{"/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n", ""}, - "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a": []string{"/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n", "a"}, - "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b": []string{"/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a", "b"}, - "/ipns/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/x/y/z": []string{"/ipns/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/x/y", "z"}, - } - - for p, expected := range cases { - path, err := ParsePath(p) - if err != nil { - t.Fatalf("ParsePath failed to parse \"%s\", but should have succeeded", p) - } - head, tail, err := path.PopLastSegment() - if err != nil { - t.Fatalf("PopLastSegment failed, but should have succeeded: %s", err) - } - headStr := head.String() - if headStr != expected[0] { - t.Fatalf("expected head of PopLastSegment(%s) to return %v, not %v", p, expected[0], headStr) - } - if tail != expected[1] { - t.Fatalf("expected tail of PopLastSegment(%s) to return %v, not %v", p, expected[1], tail) - } - } -} diff --git a/tar/format.go b/tar/format.go index cf06180f865..d00051f1188 100644 --- a/tar/format.go +++ b/tar/format.go @@ -8,10 +8,10 @@ import ( "io" "strings" + path "github.com/ipfs/go-ipfs-path" importer "github.com/ipfs/go-ipfs/importer" dag "github.com/ipfs/go-ipfs/merkledag" dagutil "github.com/ipfs/go-ipfs/merkledag/utils" - path "github.com/ipfs/go-ipfs/path" uio "github.com/ipfs/go-ipfs/unixfs/io" logging "gx/ipfs/QmRb5jh8z2E8hMGN2tkvs1yHynUanqnZ3UeKwgN1i9P1F8/go-log" From d9b3d5afc04bb31381757e7437554c4ad6bccec1 Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Sat, 21 Apr 2018 22:24:40 +0200 Subject: [PATCH 3/5] extract merkledag --- Rules.mk | 3 - cmd/ipfs/Rules.mk | 4 +- core/builder.go | 2 +- core/commands/add.go | 4 +- core/commands/dht.go | 2 +- core/commands/files.go | 2 +- core/commands/get.go | 2 +- core/commands/ls.go | 2 +- core/commands/object/diff.go | 2 +- core/commands/object/object.go | 2 +- core/commands/object/patch.go | 4 +- core/commands/pin.go | 2 +- core/commands/tar.go | 2 +- core/commands/unixfs/ls.go | 2 +- core/core.go | 2 +- core/coreapi/object.go | 4 +- core/coreapi/pin.go | 2 +- core/coreapi/unixfs_test.go | 2 +- core/coredag/dagpb.go | 2 +- core/coredag/raw.go | 2 +- core/corehttp/gateway_handler.go | 4 +- core/corehttp/gateway_test.go | 2 +- core/coreunix/add.go | 2 +- core/coreunix/add_test.go | 2 +- core/coreunix/metadata.go | 5 +- core/coreunix/metadata_test.go | 2 +- exchange/reprovide/providers.go | 2 +- filestore/filestore_test.go | 2 +- fuse/ipns/ipns_unix.go | 2 +- fuse/readonly/ipfs_test.go | 2 +- fuse/readonly/readonly_unix.go | 2 +- importer/balanced/balanced_test.go | 4 +- importer/helpers/dagbuilder.go | 2 +- importer/helpers/helpers.go | 2 +- importer/importer_test.go | 2 +- importer/trickle/trickle_test.go | 4 +- importer/trickle/trickledag.go | 2 +- merkledag/coding.go | 137 ----- merkledag/errservice.go | 47 -- merkledag/merkledag.go | 392 -------------- merkledag/merkledag_test.go | 705 ------------------------ merkledag/node.go | 380 ------------- merkledag/node_test.go | 165 ------ merkledag/pb/Rules.mk | 8 - merkledag/pb/merkledag.pb.go | 801 ---------------------------- merkledag/pb/merkledag.proto | 39 -- merkledag/pb/merkledagpb_test.go | 450 ---------------- merkledag/raw.go | 101 ---- merkledag/readonly.go | 20 - merkledag/readonly_test.go | 64 --- merkledag/rwservice.go | 47 -- merkledag/session.go | 21 - merkledag/test/utils.go | 23 - merkledag/traverse/traverse.go | 223 -------- merkledag/traverse/traverse_test.go | 418 --------------- merkledag/utils/diff.go | 210 -------- merkledag/utils/diffenum.go | 99 ---- merkledag/utils/diffenum_test.go | 249 --------- merkledag/utils/utils.go | 234 -------- merkledag/utils/utils_test.go | 114 ---- mfs/dir.go | 2 +- mfs/file.go | 2 +- mfs/mfs_test.go | 2 +- mfs/system.go | 2 +- pin/gc/gc.go | 2 +- pin/pin.go | 4 +- pin/pin_test.go | 2 +- pin/set.go | 2 +- pin/set_test.go | 2 +- tar/format.go | 4 +- unixfs/archive/tar/writer.go | 2 +- unixfs/hamt/hamt.go | 2 +- unixfs/hamt/hamt_stress_test.go | 2 +- unixfs/hamt/hamt_test.go | 6 +- unixfs/io/dagreader.go | 2 +- unixfs/io/dagreader_test.go | 2 +- unixfs/io/dirbuilder.go | 2 +- unixfs/io/dirbuilder_test.go | 2 +- unixfs/io/pbdagreader.go | 2 +- unixfs/io/resolve.go | 2 +- unixfs/mod/dagmodifier.go | 2 +- unixfs/test/utils.go | 4 +- unixfs/unixfs.go | 2 +- 83 files changed, 73 insertions(+), 5022 deletions(-) delete mode 100644 merkledag/coding.go delete mode 100644 merkledag/errservice.go delete mode 100644 merkledag/merkledag.go delete mode 100644 merkledag/merkledag_test.go delete mode 100644 merkledag/node.go delete mode 100644 merkledag/node_test.go delete mode 100644 merkledag/pb/Rules.mk delete mode 100644 merkledag/pb/merkledag.pb.go delete mode 100644 merkledag/pb/merkledag.proto delete mode 100644 merkledag/pb/merkledagpb_test.go delete mode 100644 merkledag/raw.go delete mode 100644 merkledag/readonly.go delete mode 100644 merkledag/readonly_test.go delete mode 100644 merkledag/rwservice.go delete mode 100644 merkledag/session.go delete mode 100644 merkledag/test/utils.go delete mode 100644 merkledag/traverse/traverse.go delete mode 100644 merkledag/traverse/traverse_test.go delete mode 100644 merkledag/utils/diff.go delete mode 100644 merkledag/utils/diffenum.go delete mode 100644 merkledag/utils/diffenum_test.go delete mode 100644 merkledag/utils/utils.go delete mode 100644 merkledag/utils/utils_test.go diff --git a/Rules.mk b/Rules.mk index 864002d4913..df6e6543958 100644 --- a/Rules.mk +++ b/Rules.mk @@ -53,9 +53,6 @@ include $(dir)/Rules.mk dir := unixfs/pb include $(dir)/Rules.mk -dir := merkledag/pb -include $(dir)/Rules.mk - dir := exchange/bitswap/message/pb include $(dir)/Rules.mk diff --git a/cmd/ipfs/Rules.mk b/cmd/ipfs/Rules.mk index 2a5b07edfb9..422593862a9 100644 --- a/cmd/ipfs/Rules.mk +++ b/cmd/ipfs/Rules.mk @@ -10,7 +10,7 @@ PATH := $(realpath $(d)):$(PATH) # disabled for now # depend on *.pb.go files in the repo as Order Only (as they shouldn't be rebuilt if exist) # DPES_OO_$(d) := diagnostics/pb/diagnostics.pb.go exchange/bitswap/message/pb/message.pb.go -# DEPS_OO_$(d) += merkledag/pb/merkledag.pb.go namesys/pb/namesys.pb.go +# DEPS_OO_$(d) += namesys/pb/namesys.pb.go # DEPS_OO_$(d) += pin/internal/pb/header.pb.go unixfs/pb/unixfs.pb.go $(d)_flags =-ldflags="-X "github.com/ipfs/go-ipfs/repo/config".CurrentCommit=$(git-hash)" @@ -33,7 +33,7 @@ $(TRY_BUILD_$(d)): $(d) $$(DEPS_GO) ALWAYS .PHONY: $(TRY_BUILD_$(d)) $(d)-install: GOFLAGS += $(cmd/ipfs_flags) -$(d)-install: $(d) $$(DEPS_GO) ALWAYS +$(d)-install: $(d) $$(DEPS_GO) ALWAYS $(GOCC) install $(go-flags-with-tags) ./cmd/ipfs .PHONY: $(d)-install diff --git a/core/builder.go b/core/builder.go index 8bcb6f5259f..4e1d2d0a3a0 100644 --- a/core/builder.go +++ b/core/builder.go @@ -9,10 +9,10 @@ import ( "syscall" "time" + dag "github.com/ipfs/go-ipfs-merkledag" resolver "github.com/ipfs/go-ipfs-path-resolver" bserv "github.com/ipfs/go-ipfs/blockservice" filestore "github.com/ipfs/go-ipfs/filestore" - dag "github.com/ipfs/go-ipfs/merkledag" pin "github.com/ipfs/go-ipfs/pin" repo "github.com/ipfs/go-ipfs/repo" cfg "github.com/ipfs/go-ipfs/repo/config" diff --git a/core/commands/add.go b/core/commands/add.go index d2e3cbe28e2..7fa359a0630 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -7,11 +7,11 @@ import ( "os" "strings" + dag "github.com/ipfs/go-ipfs-merkledag" + dagtest "github.com/ipfs/go-ipfs-merkledag/test" blockservice "github.com/ipfs/go-ipfs/blockservice" core "github.com/ipfs/go-ipfs/core" "github.com/ipfs/go-ipfs/core/coreunix" - dag "github.com/ipfs/go-ipfs/merkledag" - dagtest "github.com/ipfs/go-ipfs/merkledag/test" mfs "github.com/ipfs/go-ipfs/mfs" ft "github.com/ipfs/go-ipfs/unixfs" diff --git a/core/commands/dht.go b/core/commands/dht.go index 216fe8d11cc..0541802b9f3 100644 --- a/core/commands/dht.go +++ b/core/commands/dht.go @@ -8,10 +8,10 @@ import ( "io" "time" + dag "github.com/ipfs/go-ipfs-merkledag" path "github.com/ipfs/go-ipfs-path" cmds "github.com/ipfs/go-ipfs/commands" e "github.com/ipfs/go-ipfs/core/commands/e" - dag "github.com/ipfs/go-ipfs/merkledag" routing "gx/ipfs/QmTiWLZ6Fo5j4KcTVutZJ5KWRRJrbxzmxA4td8NfEdrPh7/go-libp2p-routing" notif "gx/ipfs/QmTiWLZ6Fo5j4KcTVutZJ5KWRRJrbxzmxA4td8NfEdrPh7/go-libp2p-routing/notifications" diff --git a/core/commands/files.go b/core/commands/files.go index a767f403a27..33771cf353f 100644 --- a/core/commands/files.go +++ b/core/commands/files.go @@ -10,6 +10,7 @@ import ( gopath "path" "strings" + dag "github.com/ipfs/go-ipfs-merkledag" path "github.com/ipfs/go-ipfs-path" resolver "github.com/ipfs/go-ipfs-path-resolver" bservice "github.com/ipfs/go-ipfs/blockservice" @@ -17,7 +18,6 @@ import ( lgc "github.com/ipfs/go-ipfs/commands/legacy" core "github.com/ipfs/go-ipfs/core" e "github.com/ipfs/go-ipfs/core/commands/e" - dag "github.com/ipfs/go-ipfs/merkledag" mfs "github.com/ipfs/go-ipfs/mfs" ft "github.com/ipfs/go-ipfs/unixfs" uio "github.com/ipfs/go-ipfs/unixfs/io" diff --git a/core/commands/get.go b/core/commands/get.go index 8a841f7972c..f5e0a7b21c9 100644 --- a/core/commands/get.go +++ b/core/commands/get.go @@ -9,10 +9,10 @@ import ( "path/filepath" "strings" + dag "github.com/ipfs/go-ipfs-merkledag" path "github.com/ipfs/go-ipfs-path" core "github.com/ipfs/go-ipfs/core" e "github.com/ipfs/go-ipfs/core/commands/e" - dag "github.com/ipfs/go-ipfs/merkledag" uarchive "github.com/ipfs/go-ipfs/unixfs/archive" tar "gx/ipfs/QmQine7gvHncNevKtG9QXxf3nXcwSj6aDDmMm52mHofEEp/tar-utils" diff --git a/core/commands/ls.go b/core/commands/ls.go index 7385e8ab0b5..f3b586f4ff8 100644 --- a/core/commands/ls.go +++ b/core/commands/ls.go @@ -6,13 +6,13 @@ import ( "io" "text/tabwriter" + merkledag "github.com/ipfs/go-ipfs-merkledag" path "github.com/ipfs/go-ipfs-path" resolver "github.com/ipfs/go-ipfs-path-resolver" blockservice "github.com/ipfs/go-ipfs/blockservice" cmds "github.com/ipfs/go-ipfs/commands" core "github.com/ipfs/go-ipfs/core" e "github.com/ipfs/go-ipfs/core/commands/e" - merkledag "github.com/ipfs/go-ipfs/merkledag" unixfs "github.com/ipfs/go-ipfs/unixfs" uio "github.com/ipfs/go-ipfs/unixfs/io" unixfspb "github.com/ipfs/go-ipfs/unixfs/pb" diff --git a/core/commands/object/diff.go b/core/commands/object/diff.go index 199951ea30b..5b80b891fec 100644 --- a/core/commands/object/diff.go +++ b/core/commands/object/diff.go @@ -7,11 +7,11 @@ import ( cmdkit "gx/ipfs/QmceUdzxkimdYsgtX733uNgzf1DLHyBKN6ehGSp85ayppM/go-ipfs-cmdkit" + dagutils "github.com/ipfs/go-ipfs-merkledag/utils" path "github.com/ipfs/go-ipfs-path" cmds "github.com/ipfs/go-ipfs/commands" core "github.com/ipfs/go-ipfs/core" e "github.com/ipfs/go-ipfs/core/commands/e" - dagutils "github.com/ipfs/go-ipfs/merkledag/utils" ) type Changes struct { diff --git a/core/commands/object/object.go b/core/commands/object/object.go index ef1dc3e060f..202cd278cd9 100644 --- a/core/commands/object/object.go +++ b/core/commands/object/object.go @@ -13,12 +13,12 @@ import ( "strings" "text/tabwriter" + dag "github.com/ipfs/go-ipfs-merkledag" path "github.com/ipfs/go-ipfs-path" oldcmds "github.com/ipfs/go-ipfs/commands" lgc "github.com/ipfs/go-ipfs/commands/legacy" core "github.com/ipfs/go-ipfs/core" e "github.com/ipfs/go-ipfs/core/commands/e" - dag "github.com/ipfs/go-ipfs/merkledag" pin "github.com/ipfs/go-ipfs/pin" ft "github.com/ipfs/go-ipfs/unixfs" diff --git a/core/commands/object/patch.go b/core/commands/object/patch.go index 623408273cf..a5e3d9fa73a 100644 --- a/core/commands/object/patch.go +++ b/core/commands/object/patch.go @@ -6,13 +6,13 @@ import ( "io/ioutil" "strings" + dag "github.com/ipfs/go-ipfs-merkledag" + dagutils "github.com/ipfs/go-ipfs-merkledag/utils" path "github.com/ipfs/go-ipfs-path" oldcmds "github.com/ipfs/go-ipfs/commands" lgc "github.com/ipfs/go-ipfs/commands/legacy" core "github.com/ipfs/go-ipfs/core" e "github.com/ipfs/go-ipfs/core/commands/e" - dag "github.com/ipfs/go-ipfs/merkledag" - dagutils "github.com/ipfs/go-ipfs/merkledag/utils" ft "github.com/ipfs/go-ipfs/unixfs" logging "gx/ipfs/QmRb5jh8z2E8hMGN2tkvs1yHynUanqnZ3UeKwgN1i9P1F8/go-log" diff --git a/core/commands/pin.go b/core/commands/pin.go index 80c8b895fb8..eb6dd26c8d5 100644 --- a/core/commands/pin.go +++ b/core/commands/pin.go @@ -7,6 +7,7 @@ import ( "io" "time" + dag "github.com/ipfs/go-ipfs-merkledag" path "github.com/ipfs/go-ipfs-path" resolver "github.com/ipfs/go-ipfs-path-resolver" bserv "github.com/ipfs/go-ipfs/blockservice" @@ -14,7 +15,6 @@ import ( core "github.com/ipfs/go-ipfs/core" e "github.com/ipfs/go-ipfs/core/commands/e" corerepo "github.com/ipfs/go-ipfs/core/corerepo" - dag "github.com/ipfs/go-ipfs/merkledag" pin "github.com/ipfs/go-ipfs/pin" "github.com/ipfs/go-ipfs/thirdparty/verifcid" uio "github.com/ipfs/go-ipfs/unixfs/io" diff --git a/core/commands/tar.go b/core/commands/tar.go index 6516f83d060..6e6bb693bc4 100644 --- a/core/commands/tar.go +++ b/core/commands/tar.go @@ -4,12 +4,12 @@ import ( "io" "strings" + dag "github.com/ipfs/go-ipfs-merkledag" path "github.com/ipfs/go-ipfs-path" cmds "github.com/ipfs/go-ipfs/commands" core "github.com/ipfs/go-ipfs/core" e "github.com/ipfs/go-ipfs/core/commands/e" "github.com/ipfs/go-ipfs/core/coreunix" - dag "github.com/ipfs/go-ipfs/merkledag" tar "github.com/ipfs/go-ipfs/tar" "gx/ipfs/QmceUdzxkimdYsgtX733uNgzf1DLHyBKN6ehGSp85ayppM/go-ipfs-cmdkit" diff --git a/core/commands/unixfs/ls.go b/core/commands/unixfs/ls.go index ead1536c0eb..da342decad0 100644 --- a/core/commands/unixfs/ls.go +++ b/core/commands/unixfs/ls.go @@ -9,12 +9,12 @@ import ( cmdkit "gx/ipfs/QmceUdzxkimdYsgtX733uNgzf1DLHyBKN6ehGSp85ayppM/go-ipfs-cmdkit" + merkledag "github.com/ipfs/go-ipfs-merkledag" path "github.com/ipfs/go-ipfs-path" resolver "github.com/ipfs/go-ipfs-path-resolver" cmds "github.com/ipfs/go-ipfs/commands" core "github.com/ipfs/go-ipfs/core" e "github.com/ipfs/go-ipfs/core/commands/e" - merkledag "github.com/ipfs/go-ipfs/merkledag" unixfs "github.com/ipfs/go-ipfs/unixfs" uio "github.com/ipfs/go-ipfs/unixfs/io" unixfspb "github.com/ipfs/go-ipfs/unixfs/pb" diff --git a/core/core.go b/core/core.go index d6dbf1b8870..d7429580324 100644 --- a/core/core.go +++ b/core/core.go @@ -21,6 +21,7 @@ import ( "strings" "time" + merkledag "github.com/ipfs/go-ipfs-merkledag" "github.com/ipfs/go-ipfs-path-resolver" bserv "github.com/ipfs/go-ipfs/blockservice" bitswap "github.com/ipfs/go-ipfs/exchange/bitswap" @@ -28,7 +29,6 @@ import ( rp "github.com/ipfs/go-ipfs/exchange/reprovide" filestore "github.com/ipfs/go-ipfs/filestore" mount "github.com/ipfs/go-ipfs/fuse/mount" - merkledag "github.com/ipfs/go-ipfs/merkledag" mfs "github.com/ipfs/go-ipfs/mfs" namesys "github.com/ipfs/go-ipfs/namesys" ipnsrp "github.com/ipfs/go-ipfs/namesys/republisher" diff --git a/core/coreapi/object.go b/core/coreapi/object.go index 32c30edb2a6..3f6ea4c127f 100644 --- a/core/coreapi/object.go +++ b/core/coreapi/object.go @@ -11,10 +11,10 @@ import ( "io" "io/ioutil" + dag "github.com/ipfs/go-ipfs-merkledag" + dagutils "github.com/ipfs/go-ipfs-merkledag/utils" coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface" caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - dag "github.com/ipfs/go-ipfs/merkledag" - dagutils "github.com/ipfs/go-ipfs/merkledag/utils" ft "github.com/ipfs/go-ipfs/unixfs" cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" diff --git a/core/coreapi/pin.go b/core/coreapi/pin.go index 1268cf93ff5..6170c76de8b 100644 --- a/core/coreapi/pin.go +++ b/core/coreapi/pin.go @@ -4,11 +4,11 @@ import ( "context" "fmt" + merkledag "github.com/ipfs/go-ipfs-merkledag" bserv "github.com/ipfs/go-ipfs/blockservice" coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface" caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" corerepo "github.com/ipfs/go-ipfs/core/corerepo" - merkledag "github.com/ipfs/go-ipfs/merkledag" pin "github.com/ipfs/go-ipfs/pin" offline "gx/ipfs/QmWM5HhdG5ZQNyHQ5XhMdGmV9CvLpFynQfGpTxN2MEM7Lc/go-ipfs-exchange-offline" diff --git a/core/coreapi/unixfs_test.go b/core/coreapi/unixfs_test.go index 3ccaa1e4c5b..d05300b1533 100644 --- a/core/coreapi/unixfs_test.go +++ b/core/coreapi/unixfs_test.go @@ -9,12 +9,12 @@ import ( "strings" "testing" + mdag "github.com/ipfs/go-ipfs-merkledag" core "github.com/ipfs/go-ipfs/core" coreapi "github.com/ipfs/go-ipfs/core/coreapi" coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface" coreunix "github.com/ipfs/go-ipfs/core/coreunix" keystore "github.com/ipfs/go-ipfs/keystore" - mdag "github.com/ipfs/go-ipfs/merkledag" repo "github.com/ipfs/go-ipfs/repo" config "github.com/ipfs/go-ipfs/repo/config" unixfs "github.com/ipfs/go-ipfs/unixfs" diff --git a/core/coredag/dagpb.go b/core/coredag/dagpb.go index 6ea5d4c4319..6bfe366da01 100644 --- a/core/coredag/dagpb.go +++ b/core/coredag/dagpb.go @@ -5,7 +5,7 @@ import ( "io/ioutil" "math" - "github.com/ipfs/go-ipfs/merkledag" + "github.com/ipfs/go-ipfs-merkledag" mh "gx/ipfs/QmZyZDi491cCNTLfAhwcaDii2Kg4pwKRkhqQzURGDvY6ua/go-multihash" cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" diff --git a/core/coredag/raw.go b/core/coredag/raw.go index 2537618a4e0..effdf0bf883 100644 --- a/core/coredag/raw.go +++ b/core/coredag/raw.go @@ -5,7 +5,7 @@ import ( "io/ioutil" "math" - "github.com/ipfs/go-ipfs/merkledag" + "github.com/ipfs/go-ipfs-merkledag" mh "gx/ipfs/QmZyZDi491cCNTLfAhwcaDii2Kg4pwKRkhqQzURGDvY6ua/go-multihash" cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go index 84dba4666b3..62788829ff8 100644 --- a/core/corehttp/gateway_handler.go +++ b/core/corehttp/gateway_handler.go @@ -12,14 +12,14 @@ import ( "strings" "time" + dag "github.com/ipfs/go-ipfs-merkledag" + dagutils "github.com/ipfs/go-ipfs-merkledag/utils" path "github.com/ipfs/go-ipfs-path" resolver "github.com/ipfs/go-ipfs-path-resolver" core "github.com/ipfs/go-ipfs/core" coreapi "github.com/ipfs/go-ipfs/core/coreapi" coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface" "github.com/ipfs/go-ipfs/importer" - dag "github.com/ipfs/go-ipfs/merkledag" - dagutils "github.com/ipfs/go-ipfs/merkledag/utils" ft "github.com/ipfs/go-ipfs/unixfs" uio "github.com/ipfs/go-ipfs/unixfs/io" diff --git a/core/corehttp/gateway_test.go b/core/corehttp/gateway_test.go index 3c81f1457f2..6b9fc56a909 100644 --- a/core/corehttp/gateway_test.go +++ b/core/corehttp/gateway_test.go @@ -10,10 +10,10 @@ import ( "testing" "time" + dag "github.com/ipfs/go-ipfs-merkledag" path "github.com/ipfs/go-ipfs-path" core "github.com/ipfs/go-ipfs/core" coreunix "github.com/ipfs/go-ipfs/core/coreunix" - dag "github.com/ipfs/go-ipfs/merkledag" namesys "github.com/ipfs/go-ipfs/namesys" nsopts "github.com/ipfs/go-ipfs/namesys/opts" repo "github.com/ipfs/go-ipfs/repo" diff --git a/core/coreunix/add.go b/core/coreunix/add.go index 03f9c86217c..59cf11f33e4 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -9,11 +9,11 @@ import ( gopath "path" "strconv" + dag "github.com/ipfs/go-ipfs-merkledag" core "github.com/ipfs/go-ipfs/core" balanced "github.com/ipfs/go-ipfs/importer/balanced" ihelper "github.com/ipfs/go-ipfs/importer/helpers" trickle "github.com/ipfs/go-ipfs/importer/trickle" - dag "github.com/ipfs/go-ipfs/merkledag" mfs "github.com/ipfs/go-ipfs/mfs" "github.com/ipfs/go-ipfs/pin" unixfs "github.com/ipfs/go-ipfs/unixfs" diff --git a/core/coreunix/add_test.go b/core/coreunix/add_test.go index a2f6349ecaa..6e1eb4e87a8 100644 --- a/core/coreunix/add_test.go +++ b/core/coreunix/add_test.go @@ -10,9 +10,9 @@ import ( "testing" "time" + dag "github.com/ipfs/go-ipfs-merkledag" "github.com/ipfs/go-ipfs/blockservice" "github.com/ipfs/go-ipfs/core" - dag "github.com/ipfs/go-ipfs/merkledag" "github.com/ipfs/go-ipfs/pin/gc" "github.com/ipfs/go-ipfs/repo" "github.com/ipfs/go-ipfs/repo/config" diff --git a/core/coreunix/metadata.go b/core/coreunix/metadata.go index c704db98148..8071ab86c07 100644 --- a/core/coreunix/metadata.go +++ b/core/coreunix/metadata.go @@ -1,10 +1,11 @@ package coreunix import ( + cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" + + dag "github.com/ipfs/go-ipfs-merkledag" core "github.com/ipfs/go-ipfs/core" - dag "github.com/ipfs/go-ipfs/merkledag" ft "github.com/ipfs/go-ipfs/unixfs" - cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" ) func AddMetadataTo(n *core.IpfsNode, skey string, m *ft.Metadata) (string, error) { diff --git a/core/coreunix/metadata_test.go b/core/coreunix/metadata_test.go index ad42ee9fc31..dcf66e5c180 100644 --- a/core/coreunix/metadata_test.go +++ b/core/coreunix/metadata_test.go @@ -6,10 +6,10 @@ import ( "io/ioutil" "testing" + merkledag "github.com/ipfs/go-ipfs-merkledag" bserv "github.com/ipfs/go-ipfs/blockservice" core "github.com/ipfs/go-ipfs/core" importer "github.com/ipfs/go-ipfs/importer" - merkledag "github.com/ipfs/go-ipfs/merkledag" ft "github.com/ipfs/go-ipfs/unixfs" uio "github.com/ipfs/go-ipfs/unixfs/io" diff --git a/exchange/reprovide/providers.go b/exchange/reprovide/providers.go index ec9f7012fb2..7b0c26e234a 100644 --- a/exchange/reprovide/providers.go +++ b/exchange/reprovide/providers.go @@ -3,7 +3,7 @@ package reprovide import ( "context" - merkledag "github.com/ipfs/go-ipfs/merkledag" + merkledag "github.com/ipfs/go-ipfs-merkledag" pin "github.com/ipfs/go-ipfs/pin" blocks "gx/ipfs/QmaG4DZ4JaqEfvPWt5nPPgoTzhc1tr1T3f4Nu9Jpdm8ymY/go-ipfs-blockstore" diff --git a/filestore/filestore_test.go b/filestore/filestore_test.go index 4bb8bfa04f6..8af021fb477 100644 --- a/filestore/filestore_test.go +++ b/filestore/filestore_test.go @@ -7,7 +7,7 @@ import ( "math/rand" "testing" - dag "github.com/ipfs/go-ipfs/merkledag" + dag "github.com/ipfs/go-ipfs-merkledag" ds "gx/ipfs/QmXRKBQA4wXP7xWbFiZsR1GP4HV6wMDQ1aWFxZZ4uBcPX9/go-datastore" blockstore "gx/ipfs/QmaG4DZ4JaqEfvPWt5nPPgoTzhc1tr1T3f4Nu9Jpdm8ymY/go-ipfs-blockstore" diff --git a/fuse/ipns/ipns_unix.go b/fuse/ipns/ipns_unix.go index 52868cbfae4..96563d0408c 100644 --- a/fuse/ipns/ipns_unix.go +++ b/fuse/ipns/ipns_unix.go @@ -11,9 +11,9 @@ import ( "io" "os" + dag "github.com/ipfs/go-ipfs-merkledag" path "github.com/ipfs/go-ipfs-path" core "github.com/ipfs/go-ipfs/core" - dag "github.com/ipfs/go-ipfs/merkledag" mfs "github.com/ipfs/go-ipfs/mfs" namesys "github.com/ipfs/go-ipfs/namesys" ft "github.com/ipfs/go-ipfs/unixfs" diff --git a/fuse/readonly/ipfs_test.go b/fuse/readonly/ipfs_test.go index b4815e749f3..b1607ec026e 100644 --- a/fuse/readonly/ipfs_test.go +++ b/fuse/readonly/ipfs_test.go @@ -13,11 +13,11 @@ import ( "sync" "testing" + dag "github.com/ipfs/go-ipfs-merkledag" core "github.com/ipfs/go-ipfs/core" coreunix "github.com/ipfs/go-ipfs/core/coreunix" coremock "github.com/ipfs/go-ipfs/core/mock" importer "github.com/ipfs/go-ipfs/importer" - dag "github.com/ipfs/go-ipfs/merkledag" uio "github.com/ipfs/go-ipfs/unixfs/io" u "gx/ipfs/QmNiJuT8Ja3hMVpBHXv3Q6dwmperaQ6JjLtpMQgMCD7xvx/go-ipfs-util" diff --git a/fuse/readonly/readonly_unix.go b/fuse/readonly/readonly_unix.go index 7313bef9db4..0190626d661 100644 --- a/fuse/readonly/readonly_unix.go +++ b/fuse/readonly/readonly_unix.go @@ -10,9 +10,9 @@ import ( "os" "syscall" + mdag "github.com/ipfs/go-ipfs-merkledag" path "github.com/ipfs/go-ipfs-path" core "github.com/ipfs/go-ipfs/core" - mdag "github.com/ipfs/go-ipfs/merkledag" uio "github.com/ipfs/go-ipfs/unixfs/io" ftpb "github.com/ipfs/go-ipfs/unixfs/pb" diff --git a/importer/balanced/balanced_test.go b/importer/balanced/balanced_test.go index 7f24b34874b..a70339e09c3 100644 --- a/importer/balanced/balanced_test.go +++ b/importer/balanced/balanced_test.go @@ -9,9 +9,9 @@ import ( mrand "math/rand" "testing" + dag "github.com/ipfs/go-ipfs-merkledag" + mdtest "github.com/ipfs/go-ipfs-merkledag/test" h "github.com/ipfs/go-ipfs/importer/helpers" - dag "github.com/ipfs/go-ipfs/merkledag" - mdtest "github.com/ipfs/go-ipfs/merkledag/test" uio "github.com/ipfs/go-ipfs/unixfs/io" u "gx/ipfs/QmNiJuT8Ja3hMVpBHXv3Q6dwmperaQ6JjLtpMQgMCD7xvx/go-ipfs-util" diff --git a/importer/helpers/dagbuilder.go b/importer/helpers/dagbuilder.go index d0532aa227c..005631081f2 100644 --- a/importer/helpers/dagbuilder.go +++ b/importer/helpers/dagbuilder.go @@ -5,7 +5,7 @@ import ( "io" "os" - dag "github.com/ipfs/go-ipfs/merkledag" + dag "github.com/ipfs/go-ipfs-merkledag" ft "github.com/ipfs/go-ipfs/unixfs" chunker "gx/ipfs/QmWo8jYc19ppG7YoTsrr2kEtLRbARTJho5oNXFTR6B7Peq/go-ipfs-chunker" diff --git a/importer/helpers/helpers.go b/importer/helpers/helpers.go index 4e4390ca998..f0babe807e6 100644 --- a/importer/helpers/helpers.go +++ b/importer/helpers/helpers.go @@ -5,7 +5,7 @@ import ( "fmt" "os" - dag "github.com/ipfs/go-ipfs/merkledag" + dag "github.com/ipfs/go-ipfs-merkledag" ft "github.com/ipfs/go-ipfs/unixfs" pi "gx/ipfs/Qmb3jLEFAQrqdVgWUajqEyuuDoavkSq1XQXz6tWdFWF995/go-ipfs-posinfo" diff --git a/importer/importer_test.go b/importer/importer_test.go index 44a4f790b4b..2adef3889b9 100644 --- a/importer/importer_test.go +++ b/importer/importer_test.go @@ -7,7 +7,7 @@ import ( "io/ioutil" "testing" - mdtest "github.com/ipfs/go-ipfs/merkledag/test" + mdtest "github.com/ipfs/go-ipfs-merkledag/test" uio "github.com/ipfs/go-ipfs/unixfs/io" u "gx/ipfs/QmNiJuT8Ja3hMVpBHXv3Q6dwmperaQ6JjLtpMQgMCD7xvx/go-ipfs-util" diff --git a/importer/trickle/trickle_test.go b/importer/trickle/trickle_test.go index c0fe536613a..e78173d771e 100644 --- a/importer/trickle/trickle_test.go +++ b/importer/trickle/trickle_test.go @@ -9,9 +9,9 @@ import ( mrand "math/rand" "testing" + merkledag "github.com/ipfs/go-ipfs-merkledag" + mdtest "github.com/ipfs/go-ipfs-merkledag/test" h "github.com/ipfs/go-ipfs/importer/helpers" - merkledag "github.com/ipfs/go-ipfs/merkledag" - mdtest "github.com/ipfs/go-ipfs/merkledag/test" ft "github.com/ipfs/go-ipfs/unixfs" uio "github.com/ipfs/go-ipfs/unixfs/io" diff --git a/importer/trickle/trickledag.go b/importer/trickle/trickledag.go index 29252d23360..72830ddc401 100644 --- a/importer/trickle/trickledag.go +++ b/importer/trickle/trickledag.go @@ -20,8 +20,8 @@ import ( "errors" "fmt" + dag "github.com/ipfs/go-ipfs-merkledag" h "github.com/ipfs/go-ipfs/importer/helpers" - dag "github.com/ipfs/go-ipfs/merkledag" ft "github.com/ipfs/go-ipfs/unixfs" cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" diff --git a/merkledag/coding.go b/merkledag/coding.go deleted file mode 100644 index f76631b2e04..00000000000 --- a/merkledag/coding.go +++ /dev/null @@ -1,137 +0,0 @@ -package merkledag - -import ( - "fmt" - "sort" - "strings" - - "gx/ipfs/Qmej7nf81hi2x2tvjRBF3mcp74sQyuDH4VMYDGd1YtXjb2/go-block-format" - - pb "github.com/ipfs/go-ipfs/merkledag/pb" - - cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" - ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" -) - -// for now, we use a PBNode intermediate thing. -// because native go objects are nice. - -// unmarshal decodes raw data into a *Node instance. -// The conversion uses an intermediate PBNode. -func (n *ProtoNode) unmarshal(encoded []byte) error { - var pbn pb.PBNode - if err := pbn.Unmarshal(encoded); err != nil { - return fmt.Errorf("unmarshal failed. %v", err) - } - - pbnl := pbn.GetLinks() - n.links = make([]*ipld.Link, len(pbnl)) - for i, l := range pbnl { - n.links[i] = &ipld.Link{Name: l.GetName(), Size: l.GetTsize()} - c, err := cid.Cast(l.GetHash()) - if err != nil { - return fmt.Errorf("link hash #%d is not valid multihash. %v", i, err) - } - n.links[i].Cid = c - } - sort.Stable(LinkSlice(n.links)) // keep links sorted - - n.data = pbn.GetData() - n.encoded = encoded - return nil -} - -// Marshal encodes a *Node instance into a new byte slice. -// The conversion uses an intermediate PBNode. -func (n *ProtoNode) Marshal() ([]byte, error) { - pbn := n.getPBNode() - data, err := pbn.Marshal() - if err != nil { - return data, fmt.Errorf("marshal failed. %v", err) - } - return data, nil -} - -func (n *ProtoNode) getPBNode() *pb.PBNode { - pbn := &pb.PBNode{} - if len(n.links) > 0 { - pbn.Links = make([]*pb.PBLink, len(n.links)) - } - - sort.Stable(LinkSlice(n.links)) // keep links sorted - for i, l := range n.links { - pbn.Links[i] = &pb.PBLink{} - pbn.Links[i].Name = &l.Name - pbn.Links[i].Tsize = &l.Size - if l.Cid != nil { - pbn.Links[i].Hash = l.Cid.Bytes() - } - } - - if len(n.data) > 0 { - pbn.Data = n.data - } - return pbn -} - -// EncodeProtobuf returns the encoded raw data version of a Node instance. -// It may use a cached encoded version, unless the force flag is given. -func (n *ProtoNode) EncodeProtobuf(force bool) ([]byte, error) { - sort.Stable(LinkSlice(n.links)) // keep links sorted - if n.encoded == nil || force { - n.cached = nil - var err error - n.encoded, err = n.Marshal() - if err != nil { - return nil, err - } - } - - if n.cached == nil { - if n.Prefix.Codec == 0 { // unset - n.Prefix = v0CidPrefix - } - c, err := n.Prefix.Sum(n.encoded) - if err != nil { - return nil, err - } - - n.cached = c - } - - return n.encoded, nil -} - -// DecodeProtobuf decodes raw data and returns a new Node instance. -func DecodeProtobuf(encoded []byte) (*ProtoNode, error) { - n := new(ProtoNode) - err := n.unmarshal(encoded) - if err != nil { - return nil, fmt.Errorf("incorrectly formatted merkledag node: %s", err) - } - return n, nil -} - -// DecodeProtobufBlock is a block decoder for protobuf IPLD nodes conforming to -// node.DecodeBlockFunc -func DecodeProtobufBlock(b blocks.Block) (ipld.Node, error) { - c := b.Cid() - if c.Type() != cid.DagProtobuf { - return nil, fmt.Errorf("this function can only decode protobuf nodes") - } - - decnd, err := DecodeProtobuf(b.RawData()) - if err != nil { - if strings.Contains(err.Error(), "Unmarshal failed") { - return nil, fmt.Errorf("the block referred to by '%s' was not a valid merkledag node", c) - } - return nil, fmt.Errorf("failed to decode Protocol Buffers: %v", err) - } - - decnd.cached = c - decnd.Prefix = c.Prefix() - return decnd, nil -} - -// Type assertion -var _ ipld.DecodeBlockFunc = DecodeProtobufBlock diff --git a/merkledag/errservice.go b/merkledag/errservice.go deleted file mode 100644 index 499e54f59c1..00000000000 --- a/merkledag/errservice.go +++ /dev/null @@ -1,47 +0,0 @@ -package merkledag - -import ( - "context" - - cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" - ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" -) - -// ErrorService implements ipld.DAGService, returning 'Err' for every call. -type ErrorService struct { - Err error -} - -var _ ipld.DAGService = (*ErrorService)(nil) - -// Add returns the cs.Err. -func (cs *ErrorService) Add(ctx context.Context, nd ipld.Node) error { - return cs.Err -} - -// AddMany returns the cs.Err. -func (cs *ErrorService) AddMany(ctx context.Context, nds []ipld.Node) error { - return cs.Err -} - -// Get returns the cs.Err. -func (cs *ErrorService) Get(ctx context.Context, c *cid.Cid) (ipld.Node, error) { - return nil, cs.Err -} - -// GetMany many returns the cs.Err. -func (cs *ErrorService) GetMany(ctx context.Context, cids []*cid.Cid) <-chan *ipld.NodeOption { - ch := make(chan *ipld.NodeOption) - close(ch) - return ch -} - -// Remove returns the cs.Err. -func (cs *ErrorService) Remove(ctx context.Context, c *cid.Cid) error { - return cs.Err -} - -// RemoveMany returns the cs.Err. -func (cs *ErrorService) RemoveMany(ctx context.Context, cids []*cid.Cid) error { - return cs.Err -} diff --git a/merkledag/merkledag.go b/merkledag/merkledag.go deleted file mode 100644 index d8b1336584f..00000000000 --- a/merkledag/merkledag.go +++ /dev/null @@ -1,392 +0,0 @@ -// Package merkledag implements the IPFS Merkle DAG data structures. -package merkledag - -import ( - "context" - "fmt" - "sync" - - bserv "github.com/ipfs/go-ipfs/blockservice" - - ipldcbor "gx/ipfs/QmNRz7BDWfdFNVLt7AVvmRefkrURD25EeoipcXqo6yoXU1/go-ipld-cbor" - cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" - ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" - blocks "gx/ipfs/Qmej7nf81hi2x2tvjRBF3mcp74sQyuDH4VMYDGd1YtXjb2/go-block-format" -) - -// TODO: We should move these registrations elsewhere. Really, most of the IPLD -// functionality should go in a `go-ipld` repo but that will take a lot of work -// and design. -func init() { - ipld.Register(cid.DagProtobuf, DecodeProtobufBlock) - ipld.Register(cid.Raw, DecodeRawBlock) - ipld.Register(cid.DagCBOR, ipldcbor.DecodeBlock) -} - -// contextKey is a type to use as value for the ProgressTracker contexts. -type contextKey string - -const progressContextKey contextKey = "progress" - -// NewDAGService constructs a new DAGService (using the default implementation). -// Note that the default implementation is also an ipld.LinkGetter. -func NewDAGService(bs bserv.BlockService) *dagService { - return &dagService{Blocks: bs} -} - -// dagService is an IPFS Merkle DAG service. -// - the root is virtual (like a forest) -// - stores nodes' data in a BlockService -// TODO: should cache Nodes that are in memory, and be -// able to free some of them when vm pressure is high -type dagService struct { - Blocks bserv.BlockService -} - -// Add adds a node to the dagService, storing the block in the BlockService -func (n *dagService) Add(ctx context.Context, nd ipld.Node) error { - if n == nil { // FIXME remove this assertion. protect with constructor invariant - return fmt.Errorf("dagService is nil") - } - - return n.Blocks.AddBlock(nd) -} - -func (n *dagService) AddMany(ctx context.Context, nds []ipld.Node) error { - blks := make([]blocks.Block, len(nds)) - for i, nd := range nds { - blks[i] = nd - } - return n.Blocks.AddBlocks(blks) -} - -// Get retrieves a node from the dagService, fetching the block in the BlockService -func (n *dagService) Get(ctx context.Context, c *cid.Cid) (ipld.Node, error) { - if n == nil { - return nil, fmt.Errorf("dagService is nil") - } - - ctx, cancel := context.WithCancel(ctx) - defer cancel() - - b, err := n.Blocks.GetBlock(ctx, c) - if err != nil { - if err == bserv.ErrNotFound { - return nil, ipld.ErrNotFound - } - return nil, fmt.Errorf("failed to get block for %s: %v", c, err) - } - - return ipld.Decode(b) -} - -// GetLinks return the links for the node, the node doesn't necessarily have -// to exist locally. -func (n *dagService) GetLinks(ctx context.Context, c *cid.Cid) ([]*ipld.Link, error) { - if c.Type() == cid.Raw { - return nil, nil - } - node, err := n.Get(ctx, c) - if err != nil { - return nil, err - } - return node.Links(), nil -} - -func (n *dagService) Remove(ctx context.Context, c *cid.Cid) error { - return n.Blocks.DeleteBlock(c) -} - -// RemoveMany removes multiple nodes from the DAG. It will likely be faster than -// removing them individually. -// -// This operation is not atomic. If it returns an error, some nodes may or may -// not have been removed. -func (n *dagService) RemoveMany(ctx context.Context, cids []*cid.Cid) error { - // TODO(#4608): make this batch all the way down. - for _, c := range cids { - if err := n.Blocks.DeleteBlock(c); err != nil { - return err - } - } - return nil -} - -// GetLinksDirect creates a function to get the links for a node, from -// the node, bypassing the LinkService. If the node does not exist -// locally (and can not be retrieved) an error will be returned. -func GetLinksDirect(serv ipld.NodeGetter) GetLinks { - return func(ctx context.Context, c *cid.Cid) ([]*ipld.Link, error) { - nd, err := serv.Get(ctx, c) - if err != nil { - if err == bserv.ErrNotFound { - err = ipld.ErrNotFound - } - return nil, err - } - return nd.Links(), nil - } -} - -type sesGetter struct { - bs *bserv.Session -} - -// Get gets a single node from the DAG. -func (sg *sesGetter) Get(ctx context.Context, c *cid.Cid) (ipld.Node, error) { - blk, err := sg.bs.GetBlock(ctx, c) - switch err { - case bserv.ErrNotFound: - return nil, ipld.ErrNotFound - default: - return nil, err - case nil: - // noop - } - - return ipld.Decode(blk) -} - -// GetMany gets many nodes at once, batching the request if possible. -func (sg *sesGetter) GetMany(ctx context.Context, keys []*cid.Cid) <-chan *ipld.NodeOption { - return getNodesFromBG(ctx, sg.bs, keys) -} - -// Session returns a NodeGetter using a new session for block fetches. -func (n *dagService) Session(ctx context.Context) ipld.NodeGetter { - return &sesGetter{bserv.NewSession(ctx, n.Blocks)} -} - -// FetchGraph fetches all nodes that are children of the given node -func FetchGraph(ctx context.Context, root *cid.Cid, serv ipld.DAGService) error { - var ng ipld.NodeGetter = serv - ds, ok := serv.(*dagService) - if ok { - ng = &sesGetter{bserv.NewSession(ctx, ds.Blocks)} - } - - v, _ := ctx.Value(progressContextKey).(*ProgressTracker) - if v == nil { - return EnumerateChildrenAsync(ctx, GetLinksDirect(ng), root, cid.NewSet().Visit) - } - set := cid.NewSet() - visit := func(c *cid.Cid) bool { - if set.Visit(c) { - v.Increment() - return true - } - return false - } - return EnumerateChildrenAsync(ctx, GetLinksDirect(ng), root, visit) -} - -// GetMany gets many nodes from the DAG at once. -// -// This method may not return all requested nodes (and may or may not return an -// error indicating that it failed to do so. It is up to the caller to verify -// that it received all nodes. -func (n *dagService) GetMany(ctx context.Context, keys []*cid.Cid) <-chan *ipld.NodeOption { - return getNodesFromBG(ctx, n.Blocks, keys) -} - -func dedupKeys(keys []*cid.Cid) []*cid.Cid { - set := cid.NewSet() - for _, c := range keys { - set.Add(c) - } - if set.Len() == len(keys) { - return keys - } - return set.Keys() -} - -func getNodesFromBG(ctx context.Context, bs bserv.BlockGetter, keys []*cid.Cid) <-chan *ipld.NodeOption { - keys = dedupKeys(keys) - - out := make(chan *ipld.NodeOption, len(keys)) - blocks := bs.GetBlocks(ctx, keys) - var count int - - go func() { - defer close(out) - for { - select { - case b, ok := <-blocks: - if !ok { - if count != len(keys) { - out <- &ipld.NodeOption{Err: fmt.Errorf("failed to fetch all nodes")} - } - return - } - - nd, err := ipld.Decode(b) - if err != nil { - out <- &ipld.NodeOption{Err: err} - return - } - - out <- &ipld.NodeOption{Node: nd} - count++ - - case <-ctx.Done(): - out <- &ipld.NodeOption{Err: ctx.Err()} - return - } - } - }() - return out -} - -// GetLinks is the type of function passed to the EnumerateChildren function(s) -// for getting the children of an IPLD node. -type GetLinks func(context.Context, *cid.Cid) ([]*ipld.Link, error) - -// GetLinksWithDAG returns a GetLinks function that tries to use the given -// NodeGetter as a LinkGetter to get the children of a given IPLD node. This may -// allow us to traverse the DAG without actually loading and parsing the node in -// question (if we already have the links cached). -func GetLinksWithDAG(ng ipld.NodeGetter) GetLinks { - return func(ctx context.Context, c *cid.Cid) ([]*ipld.Link, error) { - return ipld.GetLinks(ctx, ng, c) - } -} - -// EnumerateChildren will walk the dag below the given root node and add all -// unseen children to the passed in set. -// TODO: parallelize to avoid disk latency perf hits? -func EnumerateChildren(ctx context.Context, getLinks GetLinks, root *cid.Cid, visit func(*cid.Cid) bool) error { - links, err := getLinks(ctx, root) - if err != nil { - return err - } - for _, lnk := range links { - c := lnk.Cid - if visit(c) { - err = EnumerateChildren(ctx, getLinks, c, visit) - if err != nil { - return err - } - } - } - return nil -} - -// ProgressTracker is used to show progress when fetching nodes. -type ProgressTracker struct { - Total int - lk sync.Mutex -} - -// DeriveContext returns a new context with value "progress" derived from -// the given one. -func (p *ProgressTracker) DeriveContext(ctx context.Context) context.Context { - return context.WithValue(ctx, progressContextKey, p) -} - -// Increment adds one to the total progress. -func (p *ProgressTracker) Increment() { - p.lk.Lock() - defer p.lk.Unlock() - p.Total++ -} - -// Value returns the current progress. -func (p *ProgressTracker) Value() int { - p.lk.Lock() - defer p.lk.Unlock() - return p.Total -} - -// FetchGraphConcurrency is total number of concurrent fetches that -// 'fetchNodes' will start at a time -var FetchGraphConcurrency = 8 - -// EnumerateChildrenAsync is equivalent to EnumerateChildren *except* that it -// fetches children in parallel. -// -// NOTE: It *does not* make multiple concurrent calls to the passed `visit` function. -func EnumerateChildrenAsync(ctx context.Context, getLinks GetLinks, c *cid.Cid, visit func(*cid.Cid) bool) error { - feed := make(chan *cid.Cid) - out := make(chan []*ipld.Link) - done := make(chan struct{}) - - var setlk sync.Mutex - - errChan := make(chan error) - fetchersCtx, cancel := context.WithCancel(ctx) - - defer cancel() - - for i := 0; i < FetchGraphConcurrency; i++ { - go func() { - for ic := range feed { - setlk.Lock() - shouldVisit := visit(ic) - setlk.Unlock() - - if shouldVisit { - links, err := getLinks(ctx, ic) - if err != nil { - errChan <- err - return - } - - select { - case out <- links: - case <-fetchersCtx.Done(): - return - } - } - select { - case done <- struct{}{}: - case <-fetchersCtx.Done(): - } - } - }() - } - defer close(feed) - - send := feed - var todobuffer []*cid.Cid - var inProgress int - - next := c - for { - select { - case send <- next: - inProgress++ - if len(todobuffer) > 0 { - next = todobuffer[0] - todobuffer = todobuffer[1:] - } else { - next = nil - send = nil - } - case <-done: - inProgress-- - if inProgress == 0 && next == nil { - return nil - } - case links := <-out: - for _, lnk := range links { - if next == nil { - next = lnk.Cid - send = feed - } else { - todobuffer = append(todobuffer, lnk.Cid) - } - } - case err := <-errChan: - return err - - case <-ctx.Done(): - return ctx.Err() - } - } - -} - -var _ ipld.LinkGetter = &dagService{} -var _ ipld.NodeGetter = &dagService{} -var _ ipld.NodeGetter = &sesGetter{} -var _ ipld.DAGService = &dagService{} diff --git a/merkledag/merkledag_test.go b/merkledag/merkledag_test.go deleted file mode 100644 index bcc207d895a..00000000000 --- a/merkledag/merkledag_test.go +++ /dev/null @@ -1,705 +0,0 @@ -package merkledag_test - -import ( - "bytes" - "context" - "errors" - "fmt" - "io" - "io/ioutil" - "math/rand" - "strings" - "sync" - "testing" - "time" - - bserv "github.com/ipfs/go-ipfs/blockservice" - bstest "github.com/ipfs/go-ipfs/blockservice/test" - . "github.com/ipfs/go-ipfs/merkledag" - mdpb "github.com/ipfs/go-ipfs/merkledag/pb" - dstest "github.com/ipfs/go-ipfs/merkledag/test" - - u "gx/ipfs/QmNiJuT8Ja3hMVpBHXv3Q6dwmperaQ6JjLtpMQgMCD7xvx/go-ipfs-util" - offline "gx/ipfs/QmWM5HhdG5ZQNyHQ5XhMdGmV9CvLpFynQfGpTxN2MEM7Lc/go-ipfs-exchange-offline" - cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" - ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" - blocks "gx/ipfs/Qmej7nf81hi2x2tvjRBF3mcp74sQyuDH4VMYDGd1YtXjb2/go-block-format" -) - -func TestNode(t *testing.T) { - - n1 := NodeWithData([]byte("beep")) - n2 := NodeWithData([]byte("boop")) - n3 := NodeWithData([]byte("beep boop")) - if err := n3.AddNodeLink("beep-link", n1); err != nil { - t.Error(err) - } - if err := n3.AddNodeLink("boop-link", n2); err != nil { - t.Error(err) - } - - printn := func(name string, n *ProtoNode) { - fmt.Println(">", name) - fmt.Println("data:", string(n.Data())) - - fmt.Println("links:") - for _, l := range n.Links() { - fmt.Println("-", l.Name, l.Size, l.Cid) - } - - e, err := n.EncodeProtobuf(false) - if err != nil { - t.Error(err) - } else { - fmt.Println("encoded:", e) - } - - h := n.Multihash() - k := n.Cid().Hash() - if k.String() != h.String() { - t.Error("Key is not equivalent to multihash") - } else { - fmt.Println("key: ", k) - } - - SubtestNodeStat(t, n) - } - - printn("beep", n1) - printn("boop", n2) - printn("beep boop", n3) -} - -func SubtestNodeStat(t *testing.T, n *ProtoNode) { - enc, err := n.EncodeProtobuf(true) - if err != nil { - t.Error("n.EncodeProtobuf(true) failed") - return - } - - cumSize, err := n.Size() - if err != nil { - t.Error("n.Size() failed") - return - } - - k := n.Cid() - - expected := ipld.NodeStat{ - NumLinks: len(n.Links()), - BlockSize: len(enc), - LinksSize: len(enc) - len(n.Data()), // includes framing. - DataSize: len(n.Data()), - CumulativeSize: int(cumSize), - Hash: k.String(), - } - - actual, err := n.Stat() - if err != nil { - t.Error("n.Stat() failed") - return - } - - if expected != *actual { - t.Errorf("n.Stat incorrect.\nexpect: %s\nactual: %s", expected, actual) - } else { - fmt.Printf("n.Stat correct: %s\n", actual) - } -} - -type devZero struct{} - -func (_ devZero) Read(b []byte) (int, error) { - for i := range b { - b[i] = 0 - } - return len(b), nil -} - -func TestBatchFetch(t *testing.T) { - read := io.LimitReader(u.NewTimeSeededRand(), 1024*32) - runBatchFetchTest(t, read) -} - -func TestBatchFetchDupBlock(t *testing.T) { - read := io.LimitReader(devZero{}, 1024*32) - runBatchFetchTest(t, read) -} - -// makeTestDAG creates a simple DAG from the data in a reader. -// First, a node is created from each 512 bytes of data from the reader -// (like a the Size chunker would do). Then all nodes are added as children -// to a root node, which is returned. -func makeTestDAG(t *testing.T, read io.Reader, ds ipld.DAGService) ipld.Node { - p := make([]byte, 512) - nodes := []*ProtoNode{} - - for { - n, err := io.ReadFull(read, p) - if err == io.EOF { - break - } - - if err != nil { - t.Fatal(err) - } - - if n != len(p) { - t.Fatal("should have read 512 bytes from the reader") - } - - protoNode := NodeWithData(p) - nodes = append(nodes, protoNode) - } - - ctx := context.Background() - // Add a root referencing all created nodes - root := NodeWithData(nil) - for _, n := range nodes { - root.AddNodeLink(n.Cid().String(), n) - err := ds.Add(ctx, n) - if err != nil { - t.Fatal(err) - } - } - err := ds.Add(ctx, root) - if err != nil { - t.Fatal(err) - } - return root -} - -// makeTestDAGReader takes the root node as returned by makeTestDAG and -// provides a reader that reads all the RawData from that node and its children. -func makeTestDAGReader(t *testing.T, root ipld.Node, ds ipld.DAGService) io.Reader { - ctx := context.Background() - buf := new(bytes.Buffer) - buf.Write(root.RawData()) - for _, l := range root.Links() { - n, err := ds.Get(ctx, l.Cid) - if err != nil { - t.Fatal(err) - } - _, err = buf.Write(n.RawData()) - if err != nil { - t.Fatal(err) - } - } - return buf -} - -func runBatchFetchTest(t *testing.T, read io.Reader) { - ctx := context.Background() - var dagservs []ipld.DAGService - for _, bsi := range bstest.Mocks(5) { - dagservs = append(dagservs, NewDAGService(bsi)) - } - - root := makeTestDAG(t, read, dagservs[0]) - - t.Log("finished setup.") - - dagr := makeTestDAGReader(t, root, dagservs[0]) - - expected, err := ioutil.ReadAll(dagr) - if err != nil { - t.Fatal(err) - } - - err = dagservs[0].Add(ctx, root) - if err != nil { - t.Fatal(err) - } - - t.Log("Added file to first node.") - - c := root.Cid() - - wg := sync.WaitGroup{} - errs := make(chan error) - - for i := 1; i < len(dagservs); i++ { - wg.Add(1) - go func(i int) { - defer wg.Done() - first, err := dagservs[i].Get(ctx, c) - if err != nil { - errs <- err - } - fmt.Println("Got first node back.") - - firstpb, ok := first.(*ProtoNode) - if !ok { - errs <- ErrNotProtobuf - } - read := makeTestDAGReader(t, firstpb, dagservs[i]) - datagot, err := ioutil.ReadAll(read) - if err != nil { - errs <- err - } - - if !bytes.Equal(datagot, expected) { - errs <- errors.New("got bad data back") - } - }(i) - } - - go func() { - wg.Wait() - close(errs) - }() - - for err := range errs { - if err != nil { - t.Fatal(err) - } - } -} - -func TestCantGet(t *testing.T) { - ds := dstest.Mock() - a := NodeWithData([]byte("A")) - - c := a.Cid() - _, err := ds.Get(context.Background(), c) - if !strings.Contains(err.Error(), "not found") { - t.Fatal("expected err not found, got: ", err) - } -} - -func TestFetchGraph(t *testing.T) { - var dservs []ipld.DAGService - bsis := bstest.Mocks(2) - for _, bsi := range bsis { - dservs = append(dservs, NewDAGService(bsi)) - } - - read := io.LimitReader(u.NewTimeSeededRand(), 1024*32) - root := makeTestDAG(t, read, dservs[0]) - - err := FetchGraph(context.TODO(), root.Cid(), dservs[1]) - if err != nil { - t.Fatal(err) - } - - // create an offline dagstore and ensure all blocks were fetched - bs := bserv.New(bsis[1].Blockstore(), offline.Exchange(bsis[1].Blockstore())) - - offlineDS := NewDAGService(bs) - - err = EnumerateChildren(context.Background(), offlineDS.GetLinks, root.Cid(), func(_ *cid.Cid) bool { return true }) - if err != nil { - t.Fatal(err) - } -} - -func TestEnumerateChildren(t *testing.T) { - bsi := bstest.Mocks(1) - ds := NewDAGService(bsi[0]) - - read := io.LimitReader(u.NewTimeSeededRand(), 1024*1024) - root := makeTestDAG(t, read, ds) - - set := cid.NewSet() - - err := EnumerateChildren(context.Background(), ds.GetLinks, root.Cid(), set.Visit) - if err != nil { - t.Fatal(err) - } - - var traverse func(n ipld.Node) - traverse = func(n ipld.Node) { - // traverse dag and check - for _, lnk := range n.Links() { - c := lnk.Cid - if !set.Has(c) { - t.Fatal("missing key in set! ", lnk.Cid.String()) - } - child, err := ds.Get(context.Background(), c) - if err != nil { - t.Fatal(err) - } - traverse(child) - } - } - - traverse(root) -} - -func TestFetchFailure(t *testing.T) { - ctx := context.Background() - - ds := dstest.Mock() - ds_bad := dstest.Mock() - - top := new(ProtoNode) - for i := 0; i < 10; i++ { - nd := NodeWithData([]byte{byte('a' + i)}) - err := ds.Add(ctx, nd) - if err != nil { - t.Fatal(err) - } - - err = top.AddNodeLink(fmt.Sprintf("AA%d", i), nd) - if err != nil { - t.Fatal(err) - } - } - - for i := 0; i < 10; i++ { - nd := NodeWithData([]byte{'f', 'a' + byte(i)}) - err := ds_bad.Add(ctx, nd) - if err != nil { - t.Fatal(err) - } - - err = top.AddNodeLink(fmt.Sprintf("BB%d", i), nd) - if err != nil { - t.Fatal(err) - } - } - - getters := ipld.GetDAG(ctx, ds, top) - for i, getter := range getters { - _, err := getter.Get(ctx) - if err != nil && i < 10 { - t.Fatal(err) - } - if err == nil && i >= 10 { - t.Fatal("should have failed request") - } - } -} - -func TestUnmarshalFailure(t *testing.T) { - badData := []byte("hello world") - - _, err := DecodeProtobuf(badData) - if err == nil { - t.Fatal("shouldnt succeed to parse this") - } - - // now with a bad link - pbn := &mdpb.PBNode{Links: []*mdpb.PBLink{{Hash: []byte("not a multihash")}}} - badlink, err := pbn.Marshal() - if err != nil { - t.Fatal(err) - } - - _, err = DecodeProtobuf(badlink) - if err == nil { - t.Fatal("should have failed to parse node with bad link") - } - - n := &ProtoNode{} - n.Marshal() -} - -func TestBasicAddGet(t *testing.T) { - ctx := context.Background() - - ds := dstest.Mock() - nd := new(ProtoNode) - - err := ds.Add(ctx, nd) - if err != nil { - t.Fatal(err) - } - - out, err := ds.Get(ctx, nd.Cid()) - if err != nil { - t.Fatal(err) - } - - if !nd.Cid().Equals(out.Cid()) { - t.Fatal("output didnt match input") - } -} - -func TestGetRawNodes(t *testing.T) { - ctx := context.Background() - - rn := NewRawNode([]byte("test")) - - ds := dstest.Mock() - - err := ds.Add(ctx, rn) - if err != nil { - t.Fatal(err) - } - - if !rn.Cid().Equals(rn.Cid()) { - t.Fatal("output cids didnt match") - } - - out, err := ds.Get(ctx, rn.Cid()) - if err != nil { - t.Fatal(err) - } - - if !bytes.Equal(out.RawData(), []byte("test")) { - t.Fatal("raw block should match input data") - } - - if out.Links() != nil { - t.Fatal("raw blocks shouldnt have links") - } - - if out.Tree("", -1) != nil { - t.Fatal("tree should return no paths in a raw block") - } - - size, err := out.Size() - if err != nil { - t.Fatal(err) - } - if size != 4 { - t.Fatal("expected size to be 4") - } - - ns, err := out.Stat() - if err != nil { - t.Fatal(err) - } - - if ns.DataSize != 4 { - t.Fatal("expected size to be 4, got: ", ns.DataSize) - } - - _, _, err = out.Resolve([]string{"foo"}) - if err != ErrLinkNotFound { - t.Fatal("shouldnt find links under raw blocks") - } -} - -func TestProtoNodeResolve(t *testing.T) { - - nd := new(ProtoNode) - nd.SetLinks([]*ipld.Link{{Name: "foo"}}) - - lnk, left, err := nd.ResolveLink([]string{"foo", "bar"}) - if err != nil { - t.Fatal(err) - } - - if len(left) != 1 || left[0] != "bar" { - t.Fatal("expected the single path element 'bar' to remain") - } - - if lnk.Name != "foo" { - t.Fatal("how did we get anything else?") - } - - tvals := nd.Tree("", -1) - if len(tvals) != 1 || tvals[0] != "foo" { - t.Fatal("expected tree to return []{\"foo\"}") - } -} - -func TestCidRetention(t *testing.T) { - ctx := context.Background() - - nd := new(ProtoNode) - nd.SetData([]byte("fooooo")) - - pref := nd.Cid().Prefix() - pref.Version = 1 - - c2, err := pref.Sum(nd.RawData()) - if err != nil { - t.Fatal(err) - } - - blk, err := blocks.NewBlockWithCid(nd.RawData(), c2) - if err != nil { - t.Fatal(err) - } - - bs := dstest.Bserv() - err = bs.AddBlock(blk) - if err != nil { - t.Fatal(err) - } - - ds := NewDAGService(bs) - out, err := ds.Get(ctx, c2) - if err != nil { - t.Fatal(err) - } - - if !out.Cid().Equals(c2) { - t.Fatal("output cid didnt match") - } -} - -func TestCidRawDoesnNeedData(t *testing.T) { - srv := NewDAGService(dstest.Bserv()) - nd := NewRawNode([]byte("somedata")) - - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - // there is no data for this node in the blockservice - // so dag service can't load it - links, err := srv.GetLinks(ctx, nd.Cid()) - if err != nil { - t.Fatal(err) - } - if len(links) != 0 { - t.Fatal("raw node shouldn't have any links") - } -} - -func TestGetManyDuplicate(t *testing.T) { - ctx := context.Background() - - srv := NewDAGService(dstest.Bserv()) - - nd := NodeWithData([]byte("foo")) - if err := srv.Add(ctx, nd); err != nil { - t.Fatal(err) - } - nds := srv.GetMany(ctx, []*cid.Cid{nd.Cid(), nd.Cid(), nd.Cid()}) - out, ok := <-nds - if !ok { - t.Fatal("expecting node foo") - } - if out.Err != nil { - t.Fatal(out.Err) - } - if !out.Node.Cid().Equals(nd.Cid()) { - t.Fatal("got wrong node") - } - out, ok = <-nds - if ok { - if out.Err != nil { - t.Fatal(out.Err) - } else { - t.Fatal("expecting no more nodes") - } - } -} - -func TestEnumerateAsyncFailsNotFound(t *testing.T) { - ctx := context.Background() - - a := NodeWithData([]byte("foo1")) - b := NodeWithData([]byte("foo2")) - c := NodeWithData([]byte("foo3")) - d := NodeWithData([]byte("foo4")) - - ds := dstest.Mock() - for _, n := range []ipld.Node{a, b, c} { - err := ds.Add(ctx, n) - if err != nil { - t.Fatal(err) - } - } - - parent := new(ProtoNode) - if err := parent.AddNodeLink("a", a); err != nil { - t.Fatal(err) - } - - if err := parent.AddNodeLink("b", b); err != nil { - t.Fatal(err) - } - - if err := parent.AddNodeLink("c", c); err != nil { - t.Fatal(err) - } - - if err := parent.AddNodeLink("d", d); err != nil { - t.Fatal(err) - } - - err := ds.Add(ctx, parent) - if err != nil { - t.Fatal(err) - } - - cset := cid.NewSet() - err = EnumerateChildrenAsync(ctx, GetLinksDirect(ds), parent.Cid(), cset.Visit) - if err == nil { - t.Fatal("this should have failed") - } -} - -func TestProgressIndicator(t *testing.T) { - testProgressIndicator(t, 5) -} - -func TestProgressIndicatorNoChildren(t *testing.T) { - testProgressIndicator(t, 0) -} - -func testProgressIndicator(t *testing.T, depth int) { - ds := dstest.Mock() - - top, numChildren := mkDag(ds, depth) - - v := new(ProgressTracker) - ctx := v.DeriveContext(context.Background()) - - err := FetchGraph(ctx, top, ds) - if err != nil { - t.Fatal(err) - } - - if v.Value() != numChildren+1 { - t.Errorf("wrong number of children reported in progress indicator, expected %d, got %d", - numChildren+1, v.Value()) - } -} - -func mkDag(ds ipld.DAGService, depth int) (*cid.Cid, int) { - ctx := context.Background() - - totalChildren := 0 - f := func() *ProtoNode { - p := new(ProtoNode) - buf := make([]byte, 16) - rand.Read(buf) - - p.SetData(buf) - err := ds.Add(ctx, p) - if err != nil { - panic(err) - } - return p - } - - for i := 0; i < depth; i++ { - thisf := f - f = func() *ProtoNode { - pn := mkNodeWithChildren(thisf, 10) - err := ds.Add(ctx, pn) - if err != nil { - panic(err) - } - totalChildren += 10 - return pn - } - } - - nd := f() - err := ds.Add(ctx, nd) - if err != nil { - panic(err) - } - - return nd.Cid(), totalChildren -} - -func mkNodeWithChildren(getChild func() *ProtoNode, width int) *ProtoNode { - cur := new(ProtoNode) - - for i := 0; i < width; i++ { - c := getChild() - if err := cur.AddNodeLink(fmt.Sprint(i), c); err != nil { - panic(err) - } - } - - return cur -} diff --git a/merkledag/node.go b/merkledag/node.go deleted file mode 100644 index 5ff66e76c7f..00000000000 --- a/merkledag/node.go +++ /dev/null @@ -1,380 +0,0 @@ -package merkledag - -import ( - "context" - "encoding/json" - "fmt" - - mh "gx/ipfs/QmZyZDi491cCNTLfAhwcaDii2Kg4pwKRkhqQzURGDvY6ua/go-multihash" - cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" - ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" -) - -// Common errors -var ( - ErrNotProtobuf = fmt.Errorf("expected protobuf dag node") - ErrLinkNotFound = fmt.Errorf("no link by that name") -) - -// ProtoNode represents a node in the IPFS Merkle DAG. -// nodes have opaque data and a set of navigable links. -type ProtoNode struct { - links []*ipld.Link - data []byte - - // cache encoded/marshaled value - encoded []byte - - cached *cid.Cid - - // Prefix specifies cid version and hashing function - Prefix cid.Prefix -} - -var v0CidPrefix = cid.Prefix{ - Codec: cid.DagProtobuf, - MhLength: -1, - MhType: mh.SHA2_256, - Version: 0, -} - -var v1CidPrefix = cid.Prefix{ - Codec: cid.DagProtobuf, - MhLength: -1, - MhType: mh.SHA2_256, - Version: 1, -} - -// V0CidPrefix returns a prefix for CIDv0 -func V0CidPrefix() cid.Prefix { return v0CidPrefix } - -// V1CidPrefix returns a prefix for CIDv1 with the default settings -func V1CidPrefix() cid.Prefix { return v1CidPrefix } - -// PrefixForCidVersion returns the Protobuf prefix for a given CID version -func PrefixForCidVersion(version int) (cid.Prefix, error) { - switch version { - case 0: - return v0CidPrefix, nil - case 1: - return v1CidPrefix, nil - default: - return cid.Prefix{}, fmt.Errorf("unknown CID version: %d", version) - } -} - -// SetPrefix sets the CID prefix if it is non nil, if prefix is nil then -// it resets it the default value -func (n *ProtoNode) SetPrefix(prefix *cid.Prefix) { - if prefix == nil { - n.Prefix = v0CidPrefix - } else { - n.Prefix = *prefix - n.Prefix.Codec = cid.DagProtobuf - n.encoded = nil - n.cached = nil - } -} - -// LinkSlice is a slice of ipld.Links -type LinkSlice []*ipld.Link - -func (ls LinkSlice) Len() int { return len(ls) } -func (ls LinkSlice) Swap(a, b int) { ls[a], ls[b] = ls[b], ls[a] } -func (ls LinkSlice) Less(a, b int) bool { return ls[a].Name < ls[b].Name } - -// NodeWithData builds a new Protonode with the given data. -func NodeWithData(d []byte) *ProtoNode { - return &ProtoNode{data: d} -} - -// AddNodeLink adds a link to another node. -func (n *ProtoNode) AddNodeLink(name string, that ipld.Node) error { - n.encoded = nil - - lnk, err := ipld.MakeLink(that) - if err != nil { - return err - } - - lnk.Name = name - - n.AddRawLink(name, lnk) - - return nil -} - -// AddRawLink adds a copy of a link to this node -func (n *ProtoNode) AddRawLink(name string, l *ipld.Link) error { - n.encoded = nil - n.links = append(n.links, &ipld.Link{ - Name: name, - Size: l.Size, - Cid: l.Cid, - }) - - return nil -} - -// RemoveNodeLink removes a link on this node by the given name. -func (n *ProtoNode) RemoveNodeLink(name string) error { - n.encoded = nil - good := make([]*ipld.Link, 0, len(n.links)) - var found bool - - for _, l := range n.links { - if l.Name != name { - good = append(good, l) - } else { - found = true - } - } - n.links = good - - if !found { - return ipld.ErrNotFound - } - - return nil -} - -// GetNodeLink returns a copy of the link with the given name. -func (n *ProtoNode) GetNodeLink(name string) (*ipld.Link, error) { - for _, l := range n.links { - if l.Name == name { - return &ipld.Link{ - Name: l.Name, - Size: l.Size, - Cid: l.Cid, - }, nil - } - } - return nil, ErrLinkNotFound -} - -// GetLinkedProtoNode returns a copy of the ProtoNode with the given name. -func (n *ProtoNode) GetLinkedProtoNode(ctx context.Context, ds ipld.DAGService, name string) (*ProtoNode, error) { - nd, err := n.GetLinkedNode(ctx, ds, name) - if err != nil { - return nil, err - } - - pbnd, ok := nd.(*ProtoNode) - if !ok { - return nil, ErrNotProtobuf - } - - return pbnd, nil -} - -// GetLinkedNode returns a copy of the IPLD Node with the given name. -func (n *ProtoNode) GetLinkedNode(ctx context.Context, ds ipld.DAGService, name string) (ipld.Node, error) { - lnk, err := n.GetNodeLink(name) - if err != nil { - return nil, err - } - - return lnk.GetNode(ctx, ds) -} - -// Copy returns a copy of the node. -// NOTE: Does not make copies of Node objects in the links. -func (n *ProtoNode) Copy() ipld.Node { - nnode := new(ProtoNode) - if len(n.data) > 0 { - nnode.data = make([]byte, len(n.data)) - copy(nnode.data, n.data) - } - - if len(n.links) > 0 { - nnode.links = make([]*ipld.Link, len(n.links)) - copy(nnode.links, n.links) - } - - nnode.Prefix = n.Prefix - - return nnode -} - -// RawData returns the protobuf-encoded version of the node. -func (n *ProtoNode) RawData() []byte { - out, _ := n.EncodeProtobuf(false) - return out -} - -// Data returns the data stored by this node. -func (n *ProtoNode) Data() []byte { - return n.data -} - -// SetData stores data in this nodes. -func (n *ProtoNode) SetData(d []byte) { - n.encoded = nil - n.cached = nil - n.data = d -} - -// UpdateNodeLink return a copy of the node with the link name set to point to -// that. If a link of the same name existed, it is removed. -func (n *ProtoNode) UpdateNodeLink(name string, that *ProtoNode) (*ProtoNode, error) { - newnode := n.Copy().(*ProtoNode) - _ = newnode.RemoveNodeLink(name) // ignore error - err := newnode.AddNodeLink(name, that) - return newnode, err -} - -// Size returns the total size of the data addressed by node, -// including the total sizes of references. -func (n *ProtoNode) Size() (uint64, error) { - b, err := n.EncodeProtobuf(false) - if err != nil { - return 0, err - } - - s := uint64(len(b)) - for _, l := range n.links { - s += l.Size - } - return s, nil -} - -// Stat returns statistics on the node. -func (n *ProtoNode) Stat() (*ipld.NodeStat, error) { - enc, err := n.EncodeProtobuf(false) - if err != nil { - return nil, err - } - - cumSize, err := n.Size() - if err != nil { - return nil, err - } - - return &ipld.NodeStat{ - Hash: n.Cid().String(), - NumLinks: len(n.links), - BlockSize: len(enc), - LinksSize: len(enc) - len(n.data), // includes framing. - DataSize: len(n.data), - CumulativeSize: int(cumSize), - }, nil -} - -// Loggable implements the ipfs/go-log.Loggable interface. -func (n *ProtoNode) Loggable() map[string]interface{} { - return map[string]interface{}{ - "node": n.String(), - } -} - -// UnmarshalJSON reads the node fields from a JSON-encoded byte slice. -func (n *ProtoNode) UnmarshalJSON(b []byte) error { - s := struct { - Data []byte `json:"data"` - Links []*ipld.Link `json:"links"` - }{} - - err := json.Unmarshal(b, &s) - if err != nil { - return err - } - - n.data = s.Data - n.links = s.Links - return nil -} - -// MarshalJSON returns a JSON representation of the node. -func (n *ProtoNode) MarshalJSON() ([]byte, error) { - out := map[string]interface{}{ - "data": n.data, - "links": n.links, - } - - return json.Marshal(out) -} - -// Cid returns the node's Cid, calculated according to its prefix -// and raw data contents. -func (n *ProtoNode) Cid() *cid.Cid { - if n.encoded != nil && n.cached != nil { - return n.cached - } - - if n.Prefix.Codec == 0 { - n.SetPrefix(nil) - } - - c, err := n.Prefix.Sum(n.RawData()) - if err != nil { - // programmer error - err = fmt.Errorf("invalid CID of length %d: %x: %v", len(n.RawData()), n.RawData(), err) - panic(err) - } - - n.cached = c - return c -} - -// String prints the node's Cid. -func (n *ProtoNode) String() string { - return n.Cid().String() -} - -// Multihash hashes the encoded data of this node. -func (n *ProtoNode) Multihash() mh.Multihash { - // NOTE: EncodeProtobuf generates the hash and puts it in n.cached. - _, err := n.EncodeProtobuf(false) - if err != nil { - // Note: no possibility exists for an error to be returned through here - panic(err) - } - - return n.cached.Hash() -} - -// Links returns the node links. -func (n *ProtoNode) Links() []*ipld.Link { - return n.links -} - -// SetLinks replaces the node links with the given ones. -func (n *ProtoNode) SetLinks(links []*ipld.Link) { - n.links = links -} - -// Resolve is an alias for ResolveLink. -func (n *ProtoNode) Resolve(path []string) (interface{}, []string, error) { - return n.ResolveLink(path) -} - -// ResolveLink consumes the first element of the path and obtains the link -// corresponding to it from the node. It returns the link -// and the path without the consumed element. -func (n *ProtoNode) ResolveLink(path []string) (*ipld.Link, []string, error) { - if len(path) == 0 { - return nil, nil, fmt.Errorf("end of path, no more links to resolve") - } - - lnk, err := n.GetNodeLink(path[0]) - if err != nil { - return nil, nil, err - } - - return lnk, path[1:], nil -} - -// Tree returns the link names of the ProtoNode. -// ProtoNodes are only ever one path deep, so anything different than an empty -// string for p results in nothing. The depth parameter is ignored. -func (n *ProtoNode) Tree(p string, depth int) []string { - if p != "" { - return nil - } - - out := make([]string, 0, len(n.links)) - for _, lnk := range n.links { - out = append(out, lnk.Name) - } - return out -} diff --git a/merkledag/node_test.go b/merkledag/node_test.go deleted file mode 100644 index 273aaa5b492..00000000000 --- a/merkledag/node_test.go +++ /dev/null @@ -1,165 +0,0 @@ -package merkledag_test - -import ( - "bytes" - "context" - "testing" - - . "github.com/ipfs/go-ipfs/merkledag" - mdtest "github.com/ipfs/go-ipfs/merkledag/test" - - ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" -) - -func TestRemoveLink(t *testing.T) { - nd := &ProtoNode{} - nd.SetLinks([]*ipld.Link{ - {Name: "a"}, - {Name: "b"}, - {Name: "a"}, - {Name: "a"}, - {Name: "c"}, - {Name: "a"}, - }) - - err := nd.RemoveNodeLink("a") - if err != nil { - t.Fatal(err) - } - - if len(nd.Links()) != 2 { - t.Fatal("number of links incorrect") - } - - if nd.Links()[0].Name != "b" { - t.Fatal("link order wrong") - } - - if nd.Links()[1].Name != "c" { - t.Fatal("link order wrong") - } - - // should fail - err = nd.RemoveNodeLink("a") - if err != ipld.ErrNotFound { - t.Fatal("should have failed to remove link") - } - - // ensure nothing else got touched - if len(nd.Links()) != 2 { - t.Fatal("number of links incorrect") - } - - if nd.Links()[0].Name != "b" { - t.Fatal("link order wrong") - } - - if nd.Links()[1].Name != "c" { - t.Fatal("link order wrong") - } -} - -func TestFindLink(t *testing.T) { - ctx := context.Background() - - ds := mdtest.Mock() - ndEmpty := new(ProtoNode) - err := ds.Add(ctx, ndEmpty) - if err != nil { - t.Fatal(err) - } - - kEmpty := ndEmpty.Cid() - - nd := &ProtoNode{} - nd.SetLinks([]*ipld.Link{ - {Name: "a", Cid: kEmpty}, - {Name: "c", Cid: kEmpty}, - {Name: "b", Cid: kEmpty}, - }) - - err = ds.Add(ctx, nd) - if err != nil { - t.Fatal(err) - } - - lnk, err := nd.GetNodeLink("b") - if err != nil { - t.Fatal(err) - } - - if lnk.Name != "b" { - t.Fatal("got wrong link back") - } - - _, err = nd.GetNodeLink("f") - if err != ErrLinkNotFound { - t.Fatal("shouldnt have found link") - } - - _, err = nd.GetLinkedNode(context.Background(), ds, "b") - if err != nil { - t.Fatal(err) - } - - outnd, err := nd.UpdateNodeLink("b", nd) - if err != nil { - t.Fatal(err) - } - - olnk, err := outnd.GetNodeLink("b") - if err != nil { - t.Fatal(err) - } - - if olnk.Cid.String() == kEmpty.String() { - t.Fatal("new link should have different hash") - } -} - -func TestNodeCopy(t *testing.T) { - nd := &ProtoNode{} - nd.SetLinks([]*ipld.Link{ - {Name: "a"}, - {Name: "c"}, - {Name: "b"}, - }) - - nd.SetData([]byte("testing")) - - ond := nd.Copy().(*ProtoNode) - ond.SetData(nil) - - if nd.Data() == nil { - t.Fatal("should be different objects") - } -} - -func TestJsonRoundtrip(t *testing.T) { - nd := new(ProtoNode) - nd.SetLinks([]*ipld.Link{ - {Name: "a"}, - {Name: "c"}, - {Name: "b"}, - }) - nd.SetData([]byte("testing")) - - jb, err := nd.MarshalJSON() - if err != nil { - t.Fatal(err) - } - - nn := new(ProtoNode) - err = nn.UnmarshalJSON(jb) - if err != nil { - t.Fatal(err) - } - - if !bytes.Equal(nn.Data(), nd.Data()) { - t.Fatal("data wasnt the same") - } - - if !nn.Cid().Equals(nd.Cid()) { - t.Fatal("objects differed after marshaling") - } -} diff --git a/merkledag/pb/Rules.mk b/merkledag/pb/Rules.mk deleted file mode 100644 index 505f70e7541..00000000000 --- a/merkledag/pb/Rules.mk +++ /dev/null @@ -1,8 +0,0 @@ -include mk/header.mk - -PB_$(d) = $(wildcard $(d)/*.proto) -TGTS_$(d) = $(PB_$(d):.proto=.pb.go) - -#DEPS_GO += $(TGTS_$(d)) - -include mk/footer.mk diff --git a/merkledag/pb/merkledag.pb.go b/merkledag/pb/merkledag.pb.go deleted file mode 100644 index a4c73580f93..00000000000 --- a/merkledag/pb/merkledag.pb.go +++ /dev/null @@ -1,801 +0,0 @@ -// Code generated by protoc-gen-gogo. -// source: merkledag.proto -// DO NOT EDIT! - -/* - Package merkledag_pb is a generated protocol buffer package. - - It is generated from these files: - merkledag.proto - - It has these top-level messages: - PBLink - PBNode -*/ -package merkledag_pb - -import proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto" -import math "math" - -// discarding unused import gogoproto "code.google.com/p/gogoprotobuf/gogoproto/gogo.pb" - -import io "io" -import fmt "fmt" -import github_com_gogo_protobuf_proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto" - -import strings "strings" -import reflect "reflect" - -import sort "sort" -import strconv "strconv" - -import bytes "bytes" - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = math.Inf - -// An IPFS MerkleDAG Link -type PBLink struct { - // multihash of the target object - Hash []byte `protobuf:"bytes,1,opt" json:"Hash,omitempty"` - // utf string name. should be unique per object - Name *string `protobuf:"bytes,2,opt" json:"Name,omitempty"` - // cumulative size of target object - Tsize *uint64 `protobuf:"varint,3,opt" json:"Tsize,omitempty"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *PBLink) Reset() { *m = PBLink{} } -func (*PBLink) ProtoMessage() {} - -func (m *PBLink) GetHash() []byte { - if m != nil { - return m.Hash - } - return nil -} - -func (m *PBLink) GetName() string { - if m != nil && m.Name != nil { - return *m.Name - } - return "" -} - -func (m *PBLink) GetTsize() uint64 { - if m != nil && m.Tsize != nil { - return *m.Tsize - } - return 0 -} - -// An IPFS MerkleDAG Node -type PBNode struct { - // refs to other objects - Links []*PBLink `protobuf:"bytes,2,rep" json:"Links,omitempty"` - // opaque user data - Data []byte `protobuf:"bytes,1,opt" json:"Data,omitempty"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *PBNode) Reset() { *m = PBNode{} } -func (*PBNode) ProtoMessage() {} - -func (m *PBNode) GetLinks() []*PBLink { - if m != nil { - return m.Links - } - return nil -} - -func (m *PBNode) GetData() []byte { - if m != nil { - return m.Data - } - return nil -} - -func init() { -} -func (m *PBLink) Unmarshal(data []byte) error { - l := len(data) - index := 0 - for index < l { - var wire uint64 - for shift := uint(0); ; shift += 7 { - if index >= l { - return io.ErrUnexpectedEOF - } - b := data[index] - index++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Hash", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if index >= l { - return io.ErrUnexpectedEOF - } - b := data[index] - index++ - byteLen |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - postIndex := index + byteLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Hash = append([]byte{}, data[index:postIndex]...) - index = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if index >= l { - return io.ErrUnexpectedEOF - } - b := data[index] - index++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - postIndex := index + int(stringLen) - if postIndex > l { - return io.ErrUnexpectedEOF - } - s := string(data[index:postIndex]) - m.Name = &s - index = postIndex - case 3: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Tsize", wireType) - } - var v uint64 - for shift := uint(0); ; shift += 7 { - if index >= l { - return io.ErrUnexpectedEOF - } - b := data[index] - index++ - v |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - m.Tsize = &v - default: - var sizeOfWire int - for { - sizeOfWire++ - wire >>= 7 - if wire == 0 { - break - } - } - index -= sizeOfWire - skippy, err := github_com_gogo_protobuf_proto.Skip(data[index:]) - if err != nil { - return err - } - if (index + skippy) > l { - return io.ErrUnexpectedEOF - } - m.XXX_unrecognized = append(m.XXX_unrecognized, data[index:index+skippy]...) - index += skippy - } - } - return nil -} -func (m *PBNode) Unmarshal(data []byte) error { - l := len(data) - index := 0 - for index < l { - var wire uint64 - for shift := uint(0); ; shift += 7 { - if index >= l { - return io.ErrUnexpectedEOF - } - b := data[index] - index++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - switch fieldNum { - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Links", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if index >= l { - return io.ErrUnexpectedEOF - } - b := data[index] - index++ - msglen |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - postIndex := index + msglen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Links = append(m.Links, &PBLink{}) - m.Links[len(m.Links)-1].Unmarshal(data[index:postIndex]) - index = postIndex - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if index >= l { - return io.ErrUnexpectedEOF - } - b := data[index] - index++ - byteLen |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - postIndex := index + byteLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Data = append([]byte{}, data[index:postIndex]...) - index = postIndex - default: - var sizeOfWire int - for { - sizeOfWire++ - wire >>= 7 - if wire == 0 { - break - } - } - index -= sizeOfWire - skippy, err := github_com_gogo_protobuf_proto.Skip(data[index:]) - if err != nil { - return err - } - if (index + skippy) > l { - return io.ErrUnexpectedEOF - } - m.XXX_unrecognized = append(m.XXX_unrecognized, data[index:index+skippy]...) - index += skippy - } - } - return nil -} -func (this *PBLink) String() string { - if this == nil { - return "nil" - } - s := strings.Join([]string{`&PBLink{`, - `Hash:` + valueToStringMerkledag(this.Hash) + `,`, - `Name:` + valueToStringMerkledag(this.Name) + `,`, - `Tsize:` + valueToStringMerkledag(this.Tsize) + `,`, - `XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`, - `}`, - }, "") - return s -} -func (this *PBNode) String() string { - if this == nil { - return "nil" - } - s := strings.Join([]string{`&PBNode{`, - `Links:` + strings.Replace(fmt.Sprintf("%v", this.Links), "PBLink", "PBLink", 1) + `,`, - `Data:` + valueToStringMerkledag(this.Data) + `,`, - `XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`, - `}`, - }, "") - return s -} -func valueToStringMerkledag(v interface{}) string { - rv := reflect.ValueOf(v) - if rv.IsNil() { - return "nil" - } - pv := reflect.Indirect(rv).Interface() - return fmt.Sprintf("*%v", pv) -} -func (m *PBLink) Size() (n int) { - var l int - _ = l - if m.Hash != nil { - l = len(m.Hash) - n += 1 + l + sovMerkledag(uint64(l)) - } - if m.Name != nil { - l = len(*m.Name) - n += 1 + l + sovMerkledag(uint64(l)) - } - if m.Tsize != nil { - n += 1 + sovMerkledag(uint64(*m.Tsize)) - } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } - return n -} - -func (m *PBNode) Size() (n int) { - var l int - _ = l - if len(m.Links) > 0 { - for _, e := range m.Links { - l = e.Size() - n += 1 + l + sovMerkledag(uint64(l)) - } - } - if m.Data != nil { - l = len(m.Data) - n += 1 + l + sovMerkledag(uint64(l)) - } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } - return n -} - -func sovMerkledag(x uint64) (n int) { - for { - n++ - x >>= 7 - if x == 0 { - break - } - } - return n -} -func sozMerkledag(x uint64) (n int) { - return sovMerkledag(uint64((x << 1) ^ uint64((int64(x) >> 63)))) -} -func NewPopulatedPBLink(r randyMerkledag, easy bool) *PBLink { - this := &PBLink{} - if r.Intn(10) != 0 { - v1 := r.Intn(100) - this.Hash = make([]byte, v1) - for i := 0; i < v1; i++ { - this.Hash[i] = byte(r.Intn(256)) - } - } - if r.Intn(10) != 0 { - v2 := randStringMerkledag(r) - this.Name = &v2 - } - if r.Intn(10) != 0 { - v3 := uint64(r.Uint32()) - this.Tsize = &v3 - } - if !easy && r.Intn(10) != 0 { - this.XXX_unrecognized = randUnrecognizedMerkledag(r, 4) - } - return this -} - -func NewPopulatedPBNode(r randyMerkledag, easy bool) *PBNode { - this := &PBNode{} - if r.Intn(10) != 0 { - v4 := r.Intn(10) - this.Links = make([]*PBLink, v4) - for i := 0; i < v4; i++ { - this.Links[i] = NewPopulatedPBLink(r, easy) - } - } - if r.Intn(10) != 0 { - v5 := r.Intn(100) - this.Data = make([]byte, v5) - for i := 0; i < v5; i++ { - this.Data[i] = byte(r.Intn(256)) - } - } - if !easy && r.Intn(10) != 0 { - this.XXX_unrecognized = randUnrecognizedMerkledag(r, 3) - } - return this -} - -type randyMerkledag interface { - Float32() float32 - Float64() float64 - Int63() int64 - Int31() int32 - Uint32() uint32 - Intn(n int) int -} - -func randUTF8RuneMerkledag(r randyMerkledag) rune { - return rune(r.Intn(126-43) + 43) -} -func randStringMerkledag(r randyMerkledag) string { - v6 := r.Intn(100) - tmps := make([]rune, v6) - for i := 0; i < v6; i++ { - tmps[i] = randUTF8RuneMerkledag(r) - } - return string(tmps) -} -func randUnrecognizedMerkledag(r randyMerkledag, maxFieldNumber int) (data []byte) { - l := r.Intn(5) - for i := 0; i < l; i++ { - wire := r.Intn(4) - if wire == 3 { - wire = 5 - } - fieldNumber := maxFieldNumber + r.Intn(100) - data = randFieldMerkledag(data, r, fieldNumber, wire) - } - return data -} -func randFieldMerkledag(data []byte, r randyMerkledag, fieldNumber int, wire int) []byte { - key := uint32(fieldNumber)<<3 | uint32(wire) - switch wire { - case 0: - data = encodeVarintPopulateMerkledag(data, uint64(key)) - v7 := r.Int63() - if r.Intn(2) == 0 { - v7 *= -1 - } - data = encodeVarintPopulateMerkledag(data, uint64(v7)) - case 1: - data = encodeVarintPopulateMerkledag(data, uint64(key)) - data = append(data, byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256))) - case 2: - data = encodeVarintPopulateMerkledag(data, uint64(key)) - ll := r.Intn(100) - data = encodeVarintPopulateMerkledag(data, uint64(ll)) - for j := 0; j < ll; j++ { - data = append(data, byte(r.Intn(256))) - } - default: - data = encodeVarintPopulateMerkledag(data, uint64(key)) - data = append(data, byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256))) - } - return data -} -func encodeVarintPopulateMerkledag(data []byte, v uint64) []byte { - for v >= 1<<7 { - data = append(data, uint8(uint64(v)&0x7f|0x80)) - v >>= 7 - } - data = append(data, uint8(v)) - return data -} -func (m *PBLink) Marshal() (data []byte, err error) { - size := m.Size() - data = make([]byte, size) - n, err := m.MarshalTo(data) - if err != nil { - return nil, err - } - return data[:n], nil -} - -func (m *PBLink) MarshalTo(data []byte) (n int, err error) { - var i int - _ = i - var l int - _ = l - if m.Hash != nil { - data[i] = 0xa - i++ - i = encodeVarintMerkledag(data, i, uint64(len(m.Hash))) - i += copy(data[i:], m.Hash) - } - if m.Name != nil { - data[i] = 0x12 - i++ - i = encodeVarintMerkledag(data, i, uint64(len(*m.Name))) - i += copy(data[i:], *m.Name) - } - if m.Tsize != nil { - data[i] = 0x18 - i++ - i = encodeVarintMerkledag(data, i, uint64(*m.Tsize)) - } - if m.XXX_unrecognized != nil { - i += copy(data[i:], m.XXX_unrecognized) - } - return i, nil -} - -func (m *PBNode) Marshal() (data []byte, err error) { - size := m.Size() - data = make([]byte, size) - n, err := m.MarshalTo(data) - if err != nil { - return nil, err - } - return data[:n], nil -} - -func (m *PBNode) MarshalTo(data []byte) (n int, err error) { - var i int - _ = i - var l int - _ = l - if len(m.Links) > 0 { - for _, msg := range m.Links { - data[i] = 0x12 - i++ - i = encodeVarintMerkledag(data, i, uint64(msg.Size())) - n, err := msg.MarshalTo(data[i:]) - if err != nil { - return 0, err - } - i += n - } - } - if m.Data != nil { - data[i] = 0xa - i++ - i = encodeVarintMerkledag(data, i, uint64(len(m.Data))) - i += copy(data[i:], m.Data) - } - if m.XXX_unrecognized != nil { - i += copy(data[i:], m.XXX_unrecognized) - } - return i, nil -} - -func encodeFixed64Merkledag(data []byte, offset int, v uint64) int { - data[offset] = uint8(v) - data[offset+1] = uint8(v >> 8) - data[offset+2] = uint8(v >> 16) - data[offset+3] = uint8(v >> 24) - data[offset+4] = uint8(v >> 32) - data[offset+5] = uint8(v >> 40) - data[offset+6] = uint8(v >> 48) - data[offset+7] = uint8(v >> 56) - return offset + 8 -} -func encodeFixed32Merkledag(data []byte, offset int, v uint32) int { - data[offset] = uint8(v) - data[offset+1] = uint8(v >> 8) - data[offset+2] = uint8(v >> 16) - data[offset+3] = uint8(v >> 24) - return offset + 4 -} -func encodeVarintMerkledag(data []byte, offset int, v uint64) int { - for v >= 1<<7 { - data[offset] = uint8(v&0x7f | 0x80) - v >>= 7 - offset++ - } - data[offset] = uint8(v) - return offset + 1 -} -func (this *PBLink) GoString() string { - if this == nil { - return "nil" - } - s := strings.Join([]string{`&merkledag_pb.PBLink{` + - `Hash:` + valueToGoStringMerkledag(this.Hash, "byte"), - `Name:` + valueToGoStringMerkledag(this.Name, "string"), - `Tsize:` + valueToGoStringMerkledag(this.Tsize, "uint64"), - `XXX_unrecognized:` + fmt.Sprintf("%#v", this.XXX_unrecognized) + `}`}, ", ") - return s -} -func (this *PBNode) GoString() string { - if this == nil { - return "nil" - } - s := strings.Join([]string{`&merkledag_pb.PBNode{` + - `Links:` + fmt.Sprintf("%#v", this.Links), - `Data:` + valueToGoStringMerkledag(this.Data, "byte"), - `XXX_unrecognized:` + fmt.Sprintf("%#v", this.XXX_unrecognized) + `}`}, ", ") - return s -} -func valueToGoStringMerkledag(v interface{}, typ string) string { - rv := reflect.ValueOf(v) - if rv.IsNil() { - return "nil" - } - pv := reflect.Indirect(rv).Interface() - return fmt.Sprintf("func(v %v) *%v { return &v } ( %#v )", typ, typ, pv) -} -func extensionToGoStringMerkledag(e map[int32]github_com_gogo_protobuf_proto.Extension) string { - if e == nil { - return "nil" - } - s := "map[int32]proto.Extension{" - keys := make([]int, 0, len(e)) - for k := range e { - keys = append(keys, int(k)) - } - sort.Ints(keys) - ss := []string{} - for _, k := range keys { - ss = append(ss, strconv.Itoa(k)+": "+e[int32(k)].GoString()) - } - s += strings.Join(ss, ",") + "}" - return s -} -func (this *PBLink) VerboseEqual(that interface{}) error { - if that == nil { - if this == nil { - return nil - } - return fmt.Errorf("that == nil && this != nil") - } - - that1, ok := that.(*PBLink) - if !ok { - return fmt.Errorf("that is not of type *PBLink") - } - if that1 == nil { - if this == nil { - return nil - } - return fmt.Errorf("that is type *PBLink but is nil && this != nil") - } else if this == nil { - return fmt.Errorf("that is type *PBLinkbut is not nil && this == nil") - } - if !bytes.Equal(this.Hash, that1.Hash) { - return fmt.Errorf("Hash this(%v) Not Equal that(%v)", this.Hash, that1.Hash) - } - if this.Name != nil && that1.Name != nil { - if *this.Name != *that1.Name { - return fmt.Errorf("Name this(%v) Not Equal that(%v)", *this.Name, *that1.Name) - } - } else if this.Name != nil { - return fmt.Errorf("this.Name == nil && that.Name != nil") - } else if that1.Name != nil { - return fmt.Errorf("Name this(%v) Not Equal that(%v)", this.Name, that1.Name) - } - if this.Tsize != nil && that1.Tsize != nil { - if *this.Tsize != *that1.Tsize { - return fmt.Errorf("Tsize this(%v) Not Equal that(%v)", *this.Tsize, *that1.Tsize) - } - } else if this.Tsize != nil { - return fmt.Errorf("this.Tsize == nil && that.Tsize != nil") - } else if that1.Tsize != nil { - return fmt.Errorf("Tsize this(%v) Not Equal that(%v)", this.Tsize, that1.Tsize) - } - if !bytes.Equal(this.XXX_unrecognized, that1.XXX_unrecognized) { - return fmt.Errorf("XXX_unrecognized this(%v) Not Equal that(%v)", this.XXX_unrecognized, that1.XXX_unrecognized) - } - return nil -} -func (this *PBLink) Equal(that interface{}) bool { - if that == nil { - if this == nil { - return true - } - return false - } - - that1, ok := that.(*PBLink) - if !ok { - return false - } - if that1 == nil { - if this == nil { - return true - } - return false - } else if this == nil { - return false - } - if !bytes.Equal(this.Hash, that1.Hash) { - return false - } - if this.Name != nil && that1.Name != nil { - if *this.Name != *that1.Name { - return false - } - } else if this.Name != nil { - return false - } else if that1.Name != nil { - return false - } - if this.Tsize != nil && that1.Tsize != nil { - if *this.Tsize != *that1.Tsize { - return false - } - } else if this.Tsize != nil { - return false - } else if that1.Tsize != nil { - return false - } - if !bytes.Equal(this.XXX_unrecognized, that1.XXX_unrecognized) { - return false - } - return true -} -func (this *PBNode) VerboseEqual(that interface{}) error { - if that == nil { - if this == nil { - return nil - } - return fmt.Errorf("that == nil && this != nil") - } - - that1, ok := that.(*PBNode) - if !ok { - return fmt.Errorf("that is not of type *PBNode") - } - if that1 == nil { - if this == nil { - return nil - } - return fmt.Errorf("that is type *PBNode but is nil && this != nil") - } else if this == nil { - return fmt.Errorf("that is type *PBNodebut is not nil && this == nil") - } - if len(this.Links) != len(that1.Links) { - return fmt.Errorf("Links this(%v) Not Equal that(%v)", len(this.Links), len(that1.Links)) - } - for i := range this.Links { - if !this.Links[i].Equal(that1.Links[i]) { - return fmt.Errorf("Links this[%v](%v) Not Equal that[%v](%v)", i, this.Links[i], i, that1.Links[i]) - } - } - if !bytes.Equal(this.Data, that1.Data) { - return fmt.Errorf("Data this(%v) Not Equal that(%v)", this.Data, that1.Data) - } - if !bytes.Equal(this.XXX_unrecognized, that1.XXX_unrecognized) { - return fmt.Errorf("XXX_unrecognized this(%v) Not Equal that(%v)", this.XXX_unrecognized, that1.XXX_unrecognized) - } - return nil -} -func (this *PBNode) Equal(that interface{}) bool { - if that == nil { - if this == nil { - return true - } - return false - } - - that1, ok := that.(*PBNode) - if !ok { - return false - } - if that1 == nil { - if this == nil { - return true - } - return false - } else if this == nil { - return false - } - if len(this.Links) != len(that1.Links) { - return false - } - for i := range this.Links { - if !this.Links[i].Equal(that1.Links[i]) { - return false - } - } - if !bytes.Equal(this.Data, that1.Data) { - return false - } - if !bytes.Equal(this.XXX_unrecognized, that1.XXX_unrecognized) { - return false - } - return true -} diff --git a/merkledag/pb/merkledag.proto b/merkledag/pb/merkledag.proto deleted file mode 100644 index d0d47f5a319..00000000000 --- a/merkledag/pb/merkledag.proto +++ /dev/null @@ -1,39 +0,0 @@ -package merkledag.pb; - -import "code.google.com/p/gogoprotobuf/gogoproto/gogo.proto"; - -option (gogoproto.gostring_all) = true; -option (gogoproto.equal_all) = true; -option (gogoproto.verbose_equal_all) = true; -option (gogoproto.goproto_stringer_all) = false; -option (gogoproto.stringer_all) = true; -option (gogoproto.populate_all) = true; -option (gogoproto.testgen_all) = true; -option (gogoproto.benchgen_all) = true; -option (gogoproto.marshaler_all) = true; -option (gogoproto.sizer_all) = true; -option (gogoproto.unmarshaler_all) = true; - - -// An IPFS MerkleDAG Link -message PBLink { - - // multihash of the target object - optional bytes Hash = 1; - - // utf string name. should be unique per object - optional string Name = 2; - - // cumulative size of target object - optional uint64 Tsize = 3; -} - -// An IPFS MerkleDAG Node -message PBNode { - - // refs to other objects - repeated PBLink Links = 2; - - // opaque user data - optional bytes Data = 1; -} diff --git a/merkledag/pb/merkledagpb_test.go b/merkledag/pb/merkledagpb_test.go deleted file mode 100644 index b59fca7faf9..00000000000 --- a/merkledag/pb/merkledagpb_test.go +++ /dev/null @@ -1,450 +0,0 @@ -// Code generated by protoc-gen-gogo. -// source: merkledag.proto -// DO NOT EDIT! - -/* -Package merkledag_pb is a generated protocol buffer package. - -It is generated from these files: - merkledag.proto - -It has these top-level messages: - PBLink - PBNode -*/ -package merkledag_pb - -import testing "testing" -import math_rand "math/rand" -import time "time" -import github_com_gogo_protobuf_proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto" -import encoding_json "encoding/json" -import fmt "fmt" -import go_parser "go/parser" - -func TestPBLinkProto(t *testing.T) { - popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano())) - p := NewPopulatedPBLink(popr, false) - data, err := github_com_gogo_protobuf_proto.Marshal(p) - if err != nil { - panic(err) - } - msg := &PBLink{} - if err := github_com_gogo_protobuf_proto.Unmarshal(data, msg); err != nil { - panic(err) - } - for i := range data { - data[i] = byte(popr.Intn(256)) - } - if err := p.VerboseEqual(msg); err != nil { - t.Fatalf("%#v !VerboseProto %#v, since %v", msg, p, err) - } - if !p.Equal(msg) { - t.Fatalf("%#v !Proto %#v", msg, p) - } -} - -func TestPBLinkMarshalTo(t *testing.T) { - popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano())) - p := NewPopulatedPBLink(popr, false) - size := p.Size() - data := make([]byte, size) - for i := range data { - data[i] = byte(popr.Intn(256)) - } - _, err := p.MarshalTo(data) - if err != nil { - panic(err) - } - msg := &PBLink{} - if err := github_com_gogo_protobuf_proto.Unmarshal(data, msg); err != nil { - panic(err) - } - for i := range data { - data[i] = byte(popr.Intn(256)) - } - if err := p.VerboseEqual(msg); err != nil { - t.Fatalf("%#v !VerboseProto %#v, since %v", msg, p, err) - } - if !p.Equal(msg) { - t.Fatalf("%#v !Proto %#v", msg, p) - } -} - -func BenchmarkPBLinkProtoMarshal(b *testing.B) { - popr := math_rand.New(math_rand.NewSource(616)) - total := 0 - pops := make([]*PBLink, 10000) - for i := 0; i < 10000; i++ { - pops[i] = NewPopulatedPBLink(popr, false) - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - data, err := github_com_gogo_protobuf_proto.Marshal(pops[i%10000]) - if err != nil { - panic(err) - } - total += len(data) - } - b.SetBytes(int64(total / b.N)) -} - -func BenchmarkPBLinkProtoUnmarshal(b *testing.B) { - popr := math_rand.New(math_rand.NewSource(616)) - total := 0 - datas := make([][]byte, 10000) - for i := 0; i < 10000; i++ { - data, err := github_com_gogo_protobuf_proto.Marshal(NewPopulatedPBLink(popr, false)) - if err != nil { - panic(err) - } - datas[i] = data - } - msg := &PBLink{} - b.ResetTimer() - for i := 0; i < b.N; i++ { - total += len(datas[i%10000]) - if err := github_com_gogo_protobuf_proto.Unmarshal(datas[i%10000], msg); err != nil { - panic(err) - } - } - b.SetBytes(int64(total / b.N)) -} - -func TestPBNodeProto(t *testing.T) { - popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano())) - p := NewPopulatedPBNode(popr, false) - data, err := github_com_gogo_protobuf_proto.Marshal(p) - if err != nil { - panic(err) - } - msg := &PBNode{} - if err := github_com_gogo_protobuf_proto.Unmarshal(data, msg); err != nil { - panic(err) - } - for i := range data { - data[i] = byte(popr.Intn(256)) - } - if err := p.VerboseEqual(msg); err != nil { - t.Fatalf("%#v !VerboseProto %#v, since %v", msg, p, err) - } - if !p.Equal(msg) { - t.Fatalf("%#v !Proto %#v", msg, p) - } -} - -func TestPBNodeMarshalTo(t *testing.T) { - popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano())) - p := NewPopulatedPBNode(popr, false) - size := p.Size() - data := make([]byte, size) - for i := range data { - data[i] = byte(popr.Intn(256)) - } - _, err := p.MarshalTo(data) - if err != nil { - panic(err) - } - msg := &PBNode{} - if err := github_com_gogo_protobuf_proto.Unmarshal(data, msg); err != nil { - panic(err) - } - for i := range data { - data[i] = byte(popr.Intn(256)) - } - if err := p.VerboseEqual(msg); err != nil { - t.Fatalf("%#v !VerboseProto %#v, since %v", msg, p, err) - } - if !p.Equal(msg) { - t.Fatalf("%#v !Proto %#v", msg, p) - } -} - -func BenchmarkPBNodeProtoMarshal(b *testing.B) { - popr := math_rand.New(math_rand.NewSource(616)) - total := 0 - pops := make([]*PBNode, 10000) - for i := 0; i < 10000; i++ { - pops[i] = NewPopulatedPBNode(popr, false) - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - data, err := github_com_gogo_protobuf_proto.Marshal(pops[i%10000]) - if err != nil { - panic(err) - } - total += len(data) - } - b.SetBytes(int64(total / b.N)) -} - -func BenchmarkPBNodeProtoUnmarshal(b *testing.B) { - popr := math_rand.New(math_rand.NewSource(616)) - total := 0 - datas := make([][]byte, 10000) - for i := 0; i < 10000; i++ { - data, err := github_com_gogo_protobuf_proto.Marshal(NewPopulatedPBNode(popr, false)) - if err != nil { - panic(err) - } - datas[i] = data - } - msg := &PBNode{} - b.ResetTimer() - for i := 0; i < b.N; i++ { - total += len(datas[i%10000]) - if err := github_com_gogo_protobuf_proto.Unmarshal(datas[i%10000], msg); err != nil { - panic(err) - } - } - b.SetBytes(int64(total / b.N)) -} - -func TestPBLinkJSON(t *testing.T) { - popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano())) - p := NewPopulatedPBLink(popr, true) - jsondata, err := encoding_json.Marshal(p) - if err != nil { - panic(err) - } - msg := &PBLink{} - err = encoding_json.Unmarshal(jsondata, msg) - if err != nil { - panic(err) - } - if err := p.VerboseEqual(msg); err != nil { - t.Fatalf("%#v !VerboseProto %#v, since %v", msg, p, err) - } - if !p.Equal(msg) { - t.Fatalf("%#v !Json Equal %#v", msg, p) - } -} -func TestPBNodeJSON(t *testing.T) { - popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano())) - p := NewPopulatedPBNode(popr, true) - jsondata, err := encoding_json.Marshal(p) - if err != nil { - panic(err) - } - msg := &PBNode{} - err = encoding_json.Unmarshal(jsondata, msg) - if err != nil { - panic(err) - } - if err := p.VerboseEqual(msg); err != nil { - t.Fatalf("%#v !VerboseProto %#v, since %v", msg, p, err) - } - if !p.Equal(msg) { - t.Fatalf("%#v !Json Equal %#v", msg, p) - } -} -func TestPBLinkProtoText(t *testing.T) { - popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano())) - p := NewPopulatedPBLink(popr, true) - data := github_com_gogo_protobuf_proto.MarshalTextString(p) - msg := &PBLink{} - if err := github_com_gogo_protobuf_proto.UnmarshalText(data, msg); err != nil { - panic(err) - } - if err := p.VerboseEqual(msg); err != nil { - t.Fatalf("%#v !VerboseProto %#v, since %v", msg, p, err) - } - if !p.Equal(msg) { - t.Fatalf("%#v !Proto %#v", msg, p) - } -} - -func TestPBLinkProtoCompactText(t *testing.T) { - popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano())) - p := NewPopulatedPBLink(popr, true) - data := github_com_gogo_protobuf_proto.CompactTextString(p) - msg := &PBLink{} - if err := github_com_gogo_protobuf_proto.UnmarshalText(data, msg); err != nil { - panic(err) - } - if err := p.VerboseEqual(msg); err != nil { - t.Fatalf("%#v !VerboseProto %#v, since %v", msg, p, err) - } - if !p.Equal(msg) { - t.Fatalf("%#v !Proto %#v", msg, p) - } -} - -func TestPBNodeProtoText(t *testing.T) { - popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano())) - p := NewPopulatedPBNode(popr, true) - data := github_com_gogo_protobuf_proto.MarshalTextString(p) - msg := &PBNode{} - if err := github_com_gogo_protobuf_proto.UnmarshalText(data, msg); err != nil { - panic(err) - } - if err := p.VerboseEqual(msg); err != nil { - t.Fatalf("%#v !VerboseProto %#v, since %v", msg, p, err) - } - if !p.Equal(msg) { - t.Fatalf("%#v !Proto %#v", msg, p) - } -} - -func TestPBNodeProtoCompactText(t *testing.T) { - popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano())) - p := NewPopulatedPBNode(popr, true) - data := github_com_gogo_protobuf_proto.CompactTextString(p) - msg := &PBNode{} - if err := github_com_gogo_protobuf_proto.UnmarshalText(data, msg); err != nil { - panic(err) - } - if err := p.VerboseEqual(msg); err != nil { - t.Fatalf("%#v !VerboseProto %#v, since %v", msg, p, err) - } - if !p.Equal(msg) { - t.Fatalf("%#v !Proto %#v", msg, p) - } -} - -func TestPBLinkStringer(t *testing.T) { - popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano())) - p := NewPopulatedPBLink(popr, false) - s1 := p.String() - s2 := fmt.Sprintf("%v", p) - if s1 != s2 { - t.Fatalf("String want %v got %v", s1, s2) - } -} -func TestPBNodeStringer(t *testing.T) { - popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano())) - p := NewPopulatedPBNode(popr, false) - s1 := p.String() - s2 := fmt.Sprintf("%v", p) - if s1 != s2 { - t.Fatalf("String want %v got %v", s1, s2) - } -} -func TestPBLinkSize(t *testing.T) { - popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano())) - p := NewPopulatedPBLink(popr, true) - size2 := github_com_gogo_protobuf_proto.Size(p) - data, err := github_com_gogo_protobuf_proto.Marshal(p) - if err != nil { - panic(err) - } - size := p.Size() - if len(data) != size { - t.Fatalf("size %v != marshalled size %v", size, len(data)) - } - if size2 != size { - t.Fatalf("size %v != before marshal proto.Size %v", size, size2) - } - size3 := github_com_gogo_protobuf_proto.Size(p) - if size3 != size { - t.Fatalf("size %v != after marshal proto.Size %v", size, size3) - } -} - -func BenchmarkPBLinkSize(b *testing.B) { - popr := math_rand.New(math_rand.NewSource(616)) - total := 0 - pops := make([]*PBLink, 1000) - for i := 0; i < 1000; i++ { - pops[i] = NewPopulatedPBLink(popr, false) - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - total += pops[i%1000].Size() - } - b.SetBytes(int64(total / b.N)) -} - -func TestPBNodeSize(t *testing.T) { - popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano())) - p := NewPopulatedPBNode(popr, true) - size2 := github_com_gogo_protobuf_proto.Size(p) - data, err := github_com_gogo_protobuf_proto.Marshal(p) - if err != nil { - panic(err) - } - size := p.Size() - if len(data) != size { - t.Fatalf("size %v != marshalled size %v", size, len(data)) - } - if size2 != size { - t.Fatalf("size %v != before marshal proto.Size %v", size, size2) - } - size3 := github_com_gogo_protobuf_proto.Size(p) - if size3 != size { - t.Fatalf("size %v != after marshal proto.Size %v", size, size3) - } -} - -func BenchmarkPBNodeSize(b *testing.B) { - popr := math_rand.New(math_rand.NewSource(616)) - total := 0 - pops := make([]*PBNode, 1000) - for i := 0; i < 1000; i++ { - pops[i] = NewPopulatedPBNode(popr, false) - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - total += pops[i%1000].Size() - } - b.SetBytes(int64(total / b.N)) -} - -func TestPBLinkGoString(t *testing.T) { - popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano())) - p := NewPopulatedPBLink(popr, false) - s1 := p.GoString() - s2 := fmt.Sprintf("%#v", p) - if s1 != s2 { - t.Fatalf("GoString want %v got %v", s1, s2) - } - _, err := go_parser.ParseExpr(s1) - if err != nil { - panic(err) - } -} -func TestPBNodeGoString(t *testing.T) { - popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano())) - p := NewPopulatedPBNode(popr, false) - s1 := p.GoString() - s2 := fmt.Sprintf("%#v", p) - if s1 != s2 { - t.Fatalf("GoString want %v got %v", s1, s2) - } - _, err := go_parser.ParseExpr(s1) - if err != nil { - panic(err) - } -} -func TestPBLinkVerboseEqual(t *testing.T) { - popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano())) - p := NewPopulatedPBLink(popr, false) - data, err := github_com_gogo_protobuf_proto.Marshal(p) - if err != nil { - panic(err) - } - msg := &PBLink{} - if err := github_com_gogo_protobuf_proto.Unmarshal(data, msg); err != nil { - panic(err) - } - if err := p.VerboseEqual(msg); err != nil { - t.Fatalf("%#v !VerboseEqual %#v, since %v", msg, p, err) - } -} -func TestPBNodeVerboseEqual(t *testing.T) { - popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano())) - p := NewPopulatedPBNode(popr, false) - data, err := github_com_gogo_protobuf_proto.Marshal(p) - if err != nil { - panic(err) - } - msg := &PBNode{} - if err := github_com_gogo_protobuf_proto.Unmarshal(data, msg); err != nil { - panic(err) - } - if err := p.VerboseEqual(msg); err != nil { - t.Fatalf("%#v !VerboseEqual %#v, since %v", msg, p, err) - } -} - -//These tests are generated by github.com/gogo/protobuf/plugin/testgen diff --git a/merkledag/raw.go b/merkledag/raw.go deleted file mode 100644 index d0d37b2e629..00000000000 --- a/merkledag/raw.go +++ /dev/null @@ -1,101 +0,0 @@ -package merkledag - -import ( - "fmt" - "gx/ipfs/Qmej7nf81hi2x2tvjRBF3mcp74sQyuDH4VMYDGd1YtXjb2/go-block-format" - - u "gx/ipfs/QmNiJuT8Ja3hMVpBHXv3Q6dwmperaQ6JjLtpMQgMCD7xvx/go-ipfs-util" - cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" - ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" -) - -// RawNode represents a node which only contains data. -type RawNode struct { - blocks.Block -} - -// NewRawNode creates a RawNode using the default sha2-256 hash function. -func NewRawNode(data []byte) *RawNode { - h := u.Hash(data) - c := cid.NewCidV1(cid.Raw, h) - blk, _ := blocks.NewBlockWithCid(data, c) - - return &RawNode{blk} -} - -// DecodeRawBlock is a block decoder for raw IPLD nodes conforming to `node.DecodeBlockFunc`. -func DecodeRawBlock(block blocks.Block) (ipld.Node, error) { - if block.Cid().Type() != cid.Raw { - return nil, fmt.Errorf("raw nodes cannot be decoded from non-raw blocks: %d", block.Cid().Type()) - } - // Once you "share" a block, it should be immutable. Therefore, we can just use this block as-is. - return &RawNode{block}, nil -} - -var _ ipld.DecodeBlockFunc = DecodeRawBlock - -// NewRawNodeWPrefix creates a RawNode with the hash function -// specified in prefix. -func NewRawNodeWPrefix(data []byte, prefix cid.Prefix) (*RawNode, error) { - prefix.Codec = cid.Raw - if prefix.Version == 0 { - prefix.Version = 1 - } - c, err := prefix.Sum(data) - if err != nil { - return nil, err - } - blk, err := blocks.NewBlockWithCid(data, c) - if err != nil { - return nil, err - } - return &RawNode{blk}, nil -} - -// Links returns nil. -func (rn *RawNode) Links() []*ipld.Link { - return nil -} - -// ResolveLink returns an error. -func (rn *RawNode) ResolveLink(path []string) (*ipld.Link, []string, error) { - return nil, nil, ErrLinkNotFound -} - -// Resolve returns an error. -func (rn *RawNode) Resolve(path []string) (interface{}, []string, error) { - return nil, nil, ErrLinkNotFound -} - -// Tree returns nil. -func (rn *RawNode) Tree(p string, depth int) []string { - return nil -} - -// Copy performs a deep copy of this node and returns it as an ipld.Node -func (rn *RawNode) Copy() ipld.Node { - copybuf := make([]byte, len(rn.RawData())) - copy(copybuf, rn.RawData()) - nblk, err := blocks.NewBlockWithCid(rn.RawData(), rn.Cid()) - if err != nil { - // programmer error - panic("failure attempting to clone raw block: " + err.Error()) - } - - return &RawNode{nblk} -} - -// Size returns the size of this node -func (rn *RawNode) Size() (uint64, error) { - return uint64(len(rn.RawData())), nil -} - -// Stat returns some Stats about this node. -func (rn *RawNode) Stat() (*ipld.NodeStat, error) { - return &ipld.NodeStat{ - CumulativeSize: len(rn.RawData()), - DataSize: len(rn.RawData()), - }, nil -} - -var _ ipld.Node = (*RawNode)(nil) diff --git a/merkledag/readonly.go b/merkledag/readonly.go deleted file mode 100644 index 1fd48eff95c..00000000000 --- a/merkledag/readonly.go +++ /dev/null @@ -1,20 +0,0 @@ -package merkledag - -import ( - "fmt" - - ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" -) - -// ErrReadOnly is used when a read-only datastructure is written to. -var ErrReadOnly = fmt.Errorf("cannot write to readonly DAGService") - -// NewReadOnlyDagService takes a NodeGetter, and returns a full DAGService -// implementation that returns ErrReadOnly when its 'write' methods are -// invoked. -func NewReadOnlyDagService(ng ipld.NodeGetter) ipld.DAGService { - return &ComboService{ - Read: ng, - Write: &ErrorService{ErrReadOnly}, - } -} diff --git a/merkledag/readonly_test.go b/merkledag/readonly_test.go deleted file mode 100644 index 86ea86cdad9..00000000000 --- a/merkledag/readonly_test.go +++ /dev/null @@ -1,64 +0,0 @@ -package merkledag_test - -import ( - "context" - "testing" - - . "github.com/ipfs/go-ipfs/merkledag" - dstest "github.com/ipfs/go-ipfs/merkledag/test" - - cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" - ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" -) - -func TestReadonlyProperties(t *testing.T) { - ds := dstest.Mock() - ro := NewReadOnlyDagService(ds) - - ctx := context.Background() - nds := []ipld.Node{ - NewRawNode([]byte("foo1")), - NewRawNode([]byte("foo2")), - NewRawNode([]byte("foo3")), - NewRawNode([]byte("foo4")), - } - cids := []*cid.Cid{ - nds[0].Cid(), - nds[1].Cid(), - nds[2].Cid(), - nds[3].Cid(), - } - - // add to the actual underlying datastore - if err := ds.Add(ctx, nds[2]); err != nil { - t.Fatal(err) - } - if err := ds.Add(ctx, nds[3]); err != nil { - t.Fatal(err) - } - - if err := ro.Add(ctx, nds[0]); err != ErrReadOnly { - t.Fatal("expected ErrReadOnly") - } - if err := ro.Add(ctx, nds[2]); err != ErrReadOnly { - t.Fatal("expected ErrReadOnly") - } - - if err := ro.AddMany(ctx, nds[0:1]); err != ErrReadOnly { - t.Fatal("expected ErrReadOnly") - } - - if err := ro.Remove(ctx, cids[3]); err != ErrReadOnly { - t.Fatal("expected ErrReadOnly") - } - if err := ro.RemoveMany(ctx, cids[1:2]); err != ErrReadOnly { - t.Fatal("expected ErrReadOnly") - } - - if _, err := ro.Get(ctx, cids[0]); err != ipld.ErrNotFound { - t.Fatal("expected ErrNotFound") - } - if _, err := ro.Get(ctx, cids[3]); err != nil { - t.Fatal(err) - } -} diff --git a/merkledag/rwservice.go b/merkledag/rwservice.go deleted file mode 100644 index eb0c19b1c9a..00000000000 --- a/merkledag/rwservice.go +++ /dev/null @@ -1,47 +0,0 @@ -package merkledag - -import ( - "context" - - cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" - ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" -) - -// ComboService implements ipld.DAGService, using 'Read' for all fetch methods, -// and 'Write' for all methods that add new objects. -type ComboService struct { - Read ipld.NodeGetter - Write ipld.DAGService -} - -var _ ipld.DAGService = (*ComboService)(nil) - -// Add writes a new node using the Write DAGService. -func (cs *ComboService) Add(ctx context.Context, nd ipld.Node) error { - return cs.Write.Add(ctx, nd) -} - -// AddMany adds nodes using the Write DAGService. -func (cs *ComboService) AddMany(ctx context.Context, nds []ipld.Node) error { - return cs.Write.AddMany(ctx, nds) -} - -// Get fetches a node using the Read DAGService. -func (cs *ComboService) Get(ctx context.Context, c *cid.Cid) (ipld.Node, error) { - return cs.Read.Get(ctx, c) -} - -// GetMany fetches nodes using the Read DAGService. -func (cs *ComboService) GetMany(ctx context.Context, cids []*cid.Cid) <-chan *ipld.NodeOption { - return cs.Read.GetMany(ctx, cids) -} - -// Remove deletes a node using the Write DAGService. -func (cs *ComboService) Remove(ctx context.Context, c *cid.Cid) error { - return cs.Write.Remove(ctx, c) -} - -// RemoveMany deletes nodes using the Write DAGService. -func (cs *ComboService) RemoveMany(ctx context.Context, cids []*cid.Cid) error { - return cs.Write.RemoveMany(ctx, cids) -} diff --git a/merkledag/session.go b/merkledag/session.go deleted file mode 100644 index fe0df24d0ec..00000000000 --- a/merkledag/session.go +++ /dev/null @@ -1,21 +0,0 @@ -package merkledag - -import ( - "context" - - ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" -) - -// SessionMaker is an object that can generate a new fetching session. -type SessionMaker interface { - Session(context.Context) ipld.NodeGetter -} - -// NewSession returns a session backed NodeGetter if the given NodeGetter -// implements SessionMaker. -func NewSession(ctx context.Context, g ipld.NodeGetter) ipld.NodeGetter { - if sm, ok := g.(SessionMaker); ok { - return sm.Session(ctx) - } - return g -} diff --git a/merkledag/test/utils.go b/merkledag/test/utils.go deleted file mode 100644 index ebba914bc2a..00000000000 --- a/merkledag/test/utils.go +++ /dev/null @@ -1,23 +0,0 @@ -package mdutils - -import ( - bsrv "github.com/ipfs/go-ipfs/blockservice" - dag "github.com/ipfs/go-ipfs/merkledag" - - offline "gx/ipfs/QmWM5HhdG5ZQNyHQ5XhMdGmV9CvLpFynQfGpTxN2MEM7Lc/go-ipfs-exchange-offline" - ds "gx/ipfs/QmXRKBQA4wXP7xWbFiZsR1GP4HV6wMDQ1aWFxZZ4uBcPX9/go-datastore" - dssync "gx/ipfs/QmXRKBQA4wXP7xWbFiZsR1GP4HV6wMDQ1aWFxZZ4uBcPX9/go-datastore/sync" - blockstore "gx/ipfs/QmaG4DZ4JaqEfvPWt5nPPgoTzhc1tr1T3f4Nu9Jpdm8ymY/go-ipfs-blockstore" - ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" -) - -// Mock returns a new thread-safe, mock DAGService. -func Mock() ipld.DAGService { - return dag.NewDAGService(Bserv()) -} - -// Bserv returns a new, thread-safe, mock BlockService. -func Bserv() bsrv.BlockService { - bstore := blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())) - return bsrv.New(bstore, offline.Exchange(bstore)) -} diff --git a/merkledag/traverse/traverse.go b/merkledag/traverse/traverse.go deleted file mode 100644 index 5561fe0261e..00000000000 --- a/merkledag/traverse/traverse.go +++ /dev/null @@ -1,223 +0,0 @@ -// Package traverse provides merkledag traversal functions -package traverse - -import ( - "context" - "errors" - - ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" -) - -// Order is an identifier for traversal algorithm orders -type Order int - -// These constants define different traversing methods -const ( - // DFSPre defines depth-first pre-order - DFSPre Order = iota - // DFSPost defines depth-first post-order - DFSPost - // BFS defines breadth-first order - BFS -) - -// Options specifies a series of traversal options -type Options struct { - DAG ipld.NodeGetter // the dagservice to fetch nodes - Order Order // what order to traverse in - Func Func // the function to perform at each step - ErrFunc ErrFunc // see ErrFunc. Optional - - SkipDuplicates bool // whether to skip duplicate nodes -} - -// State is a current traversal state -type State struct { - Node ipld.Node - Depth int -} - -type traversal struct { - opts Options - seen map[string]struct{} -} - -func (t *traversal) shouldSkip(n ipld.Node) (bool, error) { - if t.opts.SkipDuplicates { - k := n.Cid() - if _, found := t.seen[k.KeyString()]; found { - return true, nil - } - t.seen[k.KeyString()] = struct{}{} - } - - return false, nil -} - -func (t *traversal) callFunc(next State) error { - return t.opts.Func(next) -} - -// getNode returns the node for link. If it return an error, -// stop processing. if it returns a nil node, just skip it. -// -// the error handling is a little complicated. -func (t *traversal) getNode(link *ipld.Link) (ipld.Node, error) { - - getNode := func(l *ipld.Link) (ipld.Node, error) { - next, err := l.GetNode(context.TODO(), t.opts.DAG) - if err != nil { - return nil, err - } - - skip, err := t.shouldSkip(next) - if skip { - next = nil - } - return next, err - } - - next, err := getNode(link) - if err != nil && t.opts.ErrFunc != nil { // attempt recovery. - err = t.opts.ErrFunc(err) - next = nil // skip regardless - } - return next, err -} - -// Func is the type of the function called for each dag.Node visited by Traverse. -// The traversal argument contains the current traversal state. -// If an error is returned, processing stops. -type Func func(current State) error - -// ErrFunc is provided to handle problems when walking to the Node. Traverse -// will call ErrFunc with the error encountered. ErrFunc can decide how to -// handle that error, and return an error back to Traversal with how to proceed: -// * nil - skip the Node and its children, but continue processing -// * all other errors halt processing immediately. -// -// If ErrFunc is nil, Traversal will stop, as if: -// -// opts.ErrFunc = func(err error) { return err } -// -type ErrFunc func(err error) error - -// Traverse initiates a DAG traversal with the given options starting at -// the given root. -func Traverse(root ipld.Node, o Options) error { - t := traversal{ - opts: o, - seen: map[string]struct{}{}, - } - - state := State{ - Node: root, - Depth: 0, - } - - switch o.Order { - default: - return dfsPreTraverse(state, &t) - case DFSPre: - return dfsPreTraverse(state, &t) - case DFSPost: - return dfsPostTraverse(state, &t) - case BFS: - return bfsTraverse(state, &t) - } -} - -type dfsFunc func(state State, t *traversal) error - -func dfsPreTraverse(state State, t *traversal) error { - if err := t.callFunc(state); err != nil { - return err - } - return dfsDescend(dfsPreTraverse, state, t) -} - -func dfsPostTraverse(state State, t *traversal) error { - if err := dfsDescend(dfsPostTraverse, state, t); err != nil { - return err - } - return t.callFunc(state) -} - -func dfsDescend(df dfsFunc, curr State, t *traversal) error { - for _, l := range curr.Node.Links() { - node, err := t.getNode(l) - if err != nil { - return err - } - if node == nil { // skip - continue - } - - next := State{ - Node: node, - Depth: curr.Depth + 1, - } - if err := df(next, t); err != nil { - return err - } - } - return nil -} - -func bfsTraverse(root State, t *traversal) error { - - if skip, err := t.shouldSkip(root.Node); skip || err != nil { - return err - } - - var q queue - q.enq(root) - for q.len() > 0 { - curr := q.deq() - if curr.Node == nil { - return errors.New("failed to dequeue though queue not empty") - } - - // call user's func - if err := t.callFunc(curr); err != nil { - return err - } - - for _, l := range curr.Node.Links() { - node, err := t.getNode(l) - if err != nil { - return err - } - if node == nil { // skip - continue - } - - q.enq(State{ - Node: node, - Depth: curr.Depth + 1, - }) - } - } - return nil -} - -type queue struct { - s []State -} - -func (q *queue) enq(n State) { - q.s = append(q.s, n) -} - -func (q *queue) deq() State { - if len(q.s) < 1 { - return State{} - } - n := q.s[0] - q.s = q.s[1:] - return n -} - -func (q *queue) len() int { - return len(q.s) -} diff --git a/merkledag/traverse/traverse_test.go b/merkledag/traverse/traverse_test.go deleted file mode 100644 index c9fcb4136aa..00000000000 --- a/merkledag/traverse/traverse_test.go +++ /dev/null @@ -1,418 +0,0 @@ -package traverse - -import ( - "bytes" - "context" - "fmt" - "testing" - - mdag "github.com/ipfs/go-ipfs/merkledag" - mdagtest "github.com/ipfs/go-ipfs/merkledag/test" - - ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" -) - -func TestDFSPreNoSkip(t *testing.T) { - ds := mdagtest.Mock() - opts := Options{Order: DFSPre, DAG: ds} - - testWalkOutputs(t, newFan(t, ds), opts, []byte(` -0 /a -1 /a/aa -1 /a/ab -1 /a/ac -1 /a/ad -`)) - - testWalkOutputs(t, newLinkedList(t, ds), opts, []byte(` -0 /a -1 /a/aa -2 /a/aa/aaa -3 /a/aa/aaa/aaaa -4 /a/aa/aaa/aaaa/aaaaa -`)) - - testWalkOutputs(t, newBinaryTree(t, ds), opts, []byte(` -0 /a -1 /a/aa -2 /a/aa/aaa -2 /a/aa/aab -1 /a/ab -2 /a/ab/aba -2 /a/ab/abb -`)) - - testWalkOutputs(t, newBinaryDAG(t, ds), opts, []byte(` -0 /a -1 /a/aa -2 /a/aa/aaa -3 /a/aa/aaa/aaaa -4 /a/aa/aaa/aaaa/aaaaa -4 /a/aa/aaa/aaaa/aaaaa -3 /a/aa/aaa/aaaa -4 /a/aa/aaa/aaaa/aaaaa -4 /a/aa/aaa/aaaa/aaaaa -2 /a/aa/aaa -3 /a/aa/aaa/aaaa -4 /a/aa/aaa/aaaa/aaaaa -4 /a/aa/aaa/aaaa/aaaaa -3 /a/aa/aaa/aaaa -4 /a/aa/aaa/aaaa/aaaaa -4 /a/aa/aaa/aaaa/aaaaa -1 /a/aa -2 /a/aa/aaa -3 /a/aa/aaa/aaaa -4 /a/aa/aaa/aaaa/aaaaa -4 /a/aa/aaa/aaaa/aaaaa -3 /a/aa/aaa/aaaa -4 /a/aa/aaa/aaaa/aaaaa -4 /a/aa/aaa/aaaa/aaaaa -2 /a/aa/aaa -3 /a/aa/aaa/aaaa -4 /a/aa/aaa/aaaa/aaaaa -4 /a/aa/aaa/aaaa/aaaaa -3 /a/aa/aaa/aaaa -4 /a/aa/aaa/aaaa/aaaaa -4 /a/aa/aaa/aaaa/aaaaa -`)) -} - -func TestDFSPreSkip(t *testing.T) { - ds := mdagtest.Mock() - opts := Options{Order: DFSPre, SkipDuplicates: true, DAG: ds} - - testWalkOutputs(t, newFan(t, ds), opts, []byte(` -0 /a -1 /a/aa -1 /a/ab -1 /a/ac -1 /a/ad -`)) - - testWalkOutputs(t, newLinkedList(t, ds), opts, []byte(` -0 /a -1 /a/aa -2 /a/aa/aaa -3 /a/aa/aaa/aaaa -4 /a/aa/aaa/aaaa/aaaaa -`)) - - testWalkOutputs(t, newBinaryTree(t, ds), opts, []byte(` -0 /a -1 /a/aa -2 /a/aa/aaa -2 /a/aa/aab -1 /a/ab -2 /a/ab/aba -2 /a/ab/abb -`)) - - testWalkOutputs(t, newBinaryDAG(t, ds), opts, []byte(` -0 /a -1 /a/aa -2 /a/aa/aaa -3 /a/aa/aaa/aaaa -4 /a/aa/aaa/aaaa/aaaaa -`)) -} - -func TestDFSPostNoSkip(t *testing.T) { - ds := mdagtest.Mock() - opts := Options{Order: DFSPost, DAG: ds} - - testWalkOutputs(t, newFan(t, ds), opts, []byte(` -1 /a/aa -1 /a/ab -1 /a/ac -1 /a/ad -0 /a -`)) - - testWalkOutputs(t, newLinkedList(t, ds), opts, []byte(` -4 /a/aa/aaa/aaaa/aaaaa -3 /a/aa/aaa/aaaa -2 /a/aa/aaa -1 /a/aa -0 /a -`)) - - testWalkOutputs(t, newBinaryTree(t, ds), opts, []byte(` -2 /a/aa/aaa -2 /a/aa/aab -1 /a/aa -2 /a/ab/aba -2 /a/ab/abb -1 /a/ab -0 /a -`)) - - testWalkOutputs(t, newBinaryDAG(t, ds), opts, []byte(` -4 /a/aa/aaa/aaaa/aaaaa -4 /a/aa/aaa/aaaa/aaaaa -3 /a/aa/aaa/aaaa -4 /a/aa/aaa/aaaa/aaaaa -4 /a/aa/aaa/aaaa/aaaaa -3 /a/aa/aaa/aaaa -2 /a/aa/aaa -4 /a/aa/aaa/aaaa/aaaaa -4 /a/aa/aaa/aaaa/aaaaa -3 /a/aa/aaa/aaaa -4 /a/aa/aaa/aaaa/aaaaa -4 /a/aa/aaa/aaaa/aaaaa -3 /a/aa/aaa/aaaa -2 /a/aa/aaa -1 /a/aa -4 /a/aa/aaa/aaaa/aaaaa -4 /a/aa/aaa/aaaa/aaaaa -3 /a/aa/aaa/aaaa -4 /a/aa/aaa/aaaa/aaaaa -4 /a/aa/aaa/aaaa/aaaaa -3 /a/aa/aaa/aaaa -2 /a/aa/aaa -4 /a/aa/aaa/aaaa/aaaaa -4 /a/aa/aaa/aaaa/aaaaa -3 /a/aa/aaa/aaaa -4 /a/aa/aaa/aaaa/aaaaa -4 /a/aa/aaa/aaaa/aaaaa -3 /a/aa/aaa/aaaa -2 /a/aa/aaa -1 /a/aa -0 /a -`)) -} - -func TestDFSPostSkip(t *testing.T) { - ds := mdagtest.Mock() - opts := Options{Order: DFSPost, SkipDuplicates: true, DAG: ds} - - testWalkOutputs(t, newFan(t, ds), opts, []byte(` -1 /a/aa -1 /a/ab -1 /a/ac -1 /a/ad -0 /a -`)) - - testWalkOutputs(t, newLinkedList(t, ds), opts, []byte(` -4 /a/aa/aaa/aaaa/aaaaa -3 /a/aa/aaa/aaaa -2 /a/aa/aaa -1 /a/aa -0 /a -`)) - - testWalkOutputs(t, newBinaryTree(t, ds), opts, []byte(` -2 /a/aa/aaa -2 /a/aa/aab -1 /a/aa -2 /a/ab/aba -2 /a/ab/abb -1 /a/ab -0 /a -`)) - - testWalkOutputs(t, newBinaryDAG(t, ds), opts, []byte(` -4 /a/aa/aaa/aaaa/aaaaa -3 /a/aa/aaa/aaaa -2 /a/aa/aaa -1 /a/aa -0 /a -`)) -} - -func TestBFSNoSkip(t *testing.T) { - ds := mdagtest.Mock() - opts := Options{Order: BFS, DAG: ds} - - testWalkOutputs(t, newFan(t, ds), opts, []byte(` -0 /a -1 /a/aa -1 /a/ab -1 /a/ac -1 /a/ad -`)) - - testWalkOutputs(t, newLinkedList(t, ds), opts, []byte(` -0 /a -1 /a/aa -2 /a/aa/aaa -3 /a/aa/aaa/aaaa -4 /a/aa/aaa/aaaa/aaaaa -`)) - - testWalkOutputs(t, newBinaryTree(t, ds), opts, []byte(` -0 /a -1 /a/aa -1 /a/ab -2 /a/aa/aaa -2 /a/aa/aab -2 /a/ab/aba -2 /a/ab/abb -`)) - - testWalkOutputs(t, newBinaryDAG(t, ds), opts, []byte(` -0 /a -1 /a/aa -1 /a/aa -2 /a/aa/aaa -2 /a/aa/aaa -2 /a/aa/aaa -2 /a/aa/aaa -3 /a/aa/aaa/aaaa -3 /a/aa/aaa/aaaa -3 /a/aa/aaa/aaaa -3 /a/aa/aaa/aaaa -3 /a/aa/aaa/aaaa -3 /a/aa/aaa/aaaa -3 /a/aa/aaa/aaaa -3 /a/aa/aaa/aaaa -4 /a/aa/aaa/aaaa/aaaaa -4 /a/aa/aaa/aaaa/aaaaa -4 /a/aa/aaa/aaaa/aaaaa -4 /a/aa/aaa/aaaa/aaaaa -4 /a/aa/aaa/aaaa/aaaaa -4 /a/aa/aaa/aaaa/aaaaa -4 /a/aa/aaa/aaaa/aaaaa -4 /a/aa/aaa/aaaa/aaaaa -4 /a/aa/aaa/aaaa/aaaaa -4 /a/aa/aaa/aaaa/aaaaa -4 /a/aa/aaa/aaaa/aaaaa -4 /a/aa/aaa/aaaa/aaaaa -4 /a/aa/aaa/aaaa/aaaaa -4 /a/aa/aaa/aaaa/aaaaa -4 /a/aa/aaa/aaaa/aaaaa -4 /a/aa/aaa/aaaa/aaaaa -`)) -} - -func TestBFSSkip(t *testing.T) { - ds := mdagtest.Mock() - opts := Options{Order: BFS, SkipDuplicates: true, DAG: ds} - - testWalkOutputs(t, newFan(t, ds), opts, []byte(` -0 /a -1 /a/aa -1 /a/ab -1 /a/ac -1 /a/ad -`)) - - testWalkOutputs(t, newLinkedList(t, ds), opts, []byte(` -0 /a -1 /a/aa -2 /a/aa/aaa -3 /a/aa/aaa/aaaa -4 /a/aa/aaa/aaaa/aaaaa -`)) - - testWalkOutputs(t, newBinaryTree(t, ds), opts, []byte(` -0 /a -1 /a/aa -1 /a/ab -2 /a/aa/aaa -2 /a/aa/aab -2 /a/ab/aba -2 /a/ab/abb -`)) - - testWalkOutputs(t, newBinaryDAG(t, ds), opts, []byte(` -0 /a -1 /a/aa -2 /a/aa/aaa -3 /a/aa/aaa/aaaa -4 /a/aa/aaa/aaaa/aaaaa -`)) -} - -func testWalkOutputs(t *testing.T, root ipld.Node, opts Options, expect []byte) { - expect = bytes.TrimLeft(expect, "\n") - - buf := new(bytes.Buffer) - walk := func(current State) error { - s := fmt.Sprintf("%d %s\n", current.Depth, current.Node.(*mdag.ProtoNode).Data()) - t.Logf("walk: %s", s) - buf.Write([]byte(s)) - return nil - } - - opts.Func = walk - if err := Traverse(root, opts); err != nil { - t.Error(err) - return - } - - actual := buf.Bytes() - if !bytes.Equal(actual, expect) { - t.Error("error: outputs differ") - t.Logf("expect:\n%s", expect) - t.Logf("actual:\n%s", actual) - } else { - t.Logf("expect matches actual:\n%s", expect) - } -} - -func newFan(t *testing.T, ds ipld.DAGService) ipld.Node { - a := mdag.NodeWithData([]byte("/a")) - addLink(t, ds, a, child(t, ds, a, "aa")) - addLink(t, ds, a, child(t, ds, a, "ab")) - addLink(t, ds, a, child(t, ds, a, "ac")) - addLink(t, ds, a, child(t, ds, a, "ad")) - return a -} - -func newLinkedList(t *testing.T, ds ipld.DAGService) ipld.Node { - a := mdag.NodeWithData([]byte("/a")) - aa := child(t, ds, a, "aa") - aaa := child(t, ds, aa, "aaa") - aaaa := child(t, ds, aaa, "aaaa") - aaaaa := child(t, ds, aaaa, "aaaaa") - addLink(t, ds, aaaa, aaaaa) - addLink(t, ds, aaa, aaaa) - addLink(t, ds, aa, aaa) - addLink(t, ds, a, aa) - return a -} - -func newBinaryTree(t *testing.T, ds ipld.DAGService) ipld.Node { - a := mdag.NodeWithData([]byte("/a")) - aa := child(t, ds, a, "aa") - ab := child(t, ds, a, "ab") - addLink(t, ds, aa, child(t, ds, aa, "aaa")) - addLink(t, ds, aa, child(t, ds, aa, "aab")) - addLink(t, ds, ab, child(t, ds, ab, "aba")) - addLink(t, ds, ab, child(t, ds, ab, "abb")) - addLink(t, ds, a, aa) - addLink(t, ds, a, ab) - return a -} - -func newBinaryDAG(t *testing.T, ds ipld.DAGService) ipld.Node { - a := mdag.NodeWithData([]byte("/a")) - aa := child(t, ds, a, "aa") - aaa := child(t, ds, aa, "aaa") - aaaa := child(t, ds, aaa, "aaaa") - aaaaa := child(t, ds, aaaa, "aaaaa") - addLink(t, ds, aaaa, aaaaa) - addLink(t, ds, aaaa, aaaaa) - addLink(t, ds, aaa, aaaa) - addLink(t, ds, aaa, aaaa) - addLink(t, ds, aa, aaa) - addLink(t, ds, aa, aaa) - addLink(t, ds, a, aa) - addLink(t, ds, a, aa) - return a -} - -func addLink(t *testing.T, ds ipld.DAGService, a, b ipld.Node) { - to := string(a.(*mdag.ProtoNode).Data()) + "2" + string(b.(*mdag.ProtoNode).Data()) - if err := ds.Add(context.Background(), b); err != nil { - t.Error(err) - } - if err := a.(*mdag.ProtoNode).AddNodeLink(to, b.(*mdag.ProtoNode)); err != nil { - t.Error(err) - } -} - -func child(t *testing.T, ds ipld.DAGService, a ipld.Node, name string) ipld.Node { - return mdag.NodeWithData([]byte(string(a.(*mdag.ProtoNode).Data()) + "/" + name)) -} diff --git a/merkledag/utils/diff.go b/merkledag/utils/diff.go deleted file mode 100644 index 4ba0f48c5df..00000000000 --- a/merkledag/utils/diff.go +++ /dev/null @@ -1,210 +0,0 @@ -package dagutils - -import ( - "context" - "fmt" - "path" - - dag "github.com/ipfs/go-ipfs/merkledag" - - cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" - ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" -) - -// These constants define the changes that can be applied to a DAG. -const ( - Add = iota - Remove - Mod -) - -// Change represents a change to a DAG and contains a reference to the old and -// new CIDs. -type Change struct { - Type int - Path string - Before *cid.Cid - After *cid.Cid -} - -// String prints a human-friendly line about a change. -func (c *Change) String() string { - switch c.Type { - case Add: - return fmt.Sprintf("Added %s at %s", c.After.String(), c.Path) - case Remove: - return fmt.Sprintf("Removed %s from %s", c.Before.String(), c.Path) - case Mod: - return fmt.Sprintf("Changed %s to %s at %s", c.Before.String(), c.After.String(), c.Path) - default: - panic("nope") - } -} - -// ApplyChange applies the requested changes to the given node in the given dag. -func ApplyChange(ctx context.Context, ds ipld.DAGService, nd *dag.ProtoNode, cs []*Change) (*dag.ProtoNode, error) { - e := NewDagEditor(nd, ds) - for _, c := range cs { - switch c.Type { - case Add: - child, err := ds.Get(ctx, c.After) - if err != nil { - return nil, err - } - - childpb, ok := child.(*dag.ProtoNode) - if !ok { - return nil, dag.ErrNotProtobuf - } - - err = e.InsertNodeAtPath(ctx, c.Path, childpb, nil) - if err != nil { - return nil, err - } - - case Remove: - err := e.RmLink(ctx, c.Path) - if err != nil { - return nil, err - } - - case Mod: - err := e.RmLink(ctx, c.Path) - if err != nil { - return nil, err - } - child, err := ds.Get(ctx, c.After) - if err != nil { - return nil, err - } - - childpb, ok := child.(*dag.ProtoNode) - if !ok { - return nil, dag.ErrNotProtobuf - } - - err = e.InsertNodeAtPath(ctx, c.Path, childpb, nil) - if err != nil { - return nil, err - } - } - } - - return e.Finalize(ctx, ds) -} - -// Diff returns a set of changes that transform node 'a' into node 'b' -func Diff(ctx context.Context, ds ipld.DAGService, a, b ipld.Node) ([]*Change, error) { - // Base case where both nodes are leaves, just compare - // their CIDs. - if len(a.Links()) == 0 && len(b.Links()) == 0 { - if a.Cid().Equals(b.Cid()) { - return []*Change{}, nil - } - return []*Change{ - &Change{ - Type: Mod, - Before: a.Cid(), - After: b.Cid(), - }, - }, nil - } - - var out []*Change - cleanA := a.Copy().(*dag.ProtoNode) - cleanB := b.Copy().(*dag.ProtoNode) - - // strip out unchanged stuff - for _, lnk := range a.Links() { - l, _, err := b.ResolveLink([]string{lnk.Name}) - if err == nil { - if l.Cid.Equals(lnk.Cid) { - // no change... ignore it - } else { - anode, err := lnk.GetNode(ctx, ds) - if err != nil { - return nil, err - } - - bnode, err := l.GetNode(ctx, ds) - if err != nil { - return nil, err - } - - anodepb, ok := anode.(*dag.ProtoNode) - if !ok { - return nil, dag.ErrNotProtobuf - } - - bnodepb, ok := bnode.(*dag.ProtoNode) - if !ok { - return nil, dag.ErrNotProtobuf - } - - sub, err := Diff(ctx, ds, anodepb, bnodepb) - if err != nil { - return nil, err - } - - for _, subc := range sub { - subc.Path = path.Join(lnk.Name, subc.Path) - out = append(out, subc) - } - } - cleanA.RemoveNodeLink(l.Name) - cleanB.RemoveNodeLink(l.Name) - } - } - - for _, lnk := range cleanA.Links() { - out = append(out, &Change{ - Type: Remove, - Path: lnk.Name, - Before: lnk.Cid, - }) - } - for _, lnk := range cleanB.Links() { - out = append(out, &Change{ - Type: Add, - Path: lnk.Name, - After: lnk.Cid, - }) - } - - return out, nil -} - -// Conflict represents two incompatible changes and is returned by MergeDiffs(). -type Conflict struct { - A *Change - B *Change -} - -// MergeDiffs takes two slice of changes and adds them to a single slice. -// When a Change from b happens to the same path of an existing change in a, -// a conflict is created and b is not added to the merged slice. -// A slice of Conflicts is returned and contains pointers to the -// Changes involved (which share the same path). -func MergeDiffs(a, b []*Change) ([]*Change, []Conflict) { - var out []*Change - var conflicts []Conflict - paths := make(map[string]*Change) - for _, c := range a { - paths[c.Path] = c - } - - for _, c := range b { - if ca, ok := paths[c.Path]; ok { - conflicts = append(conflicts, Conflict{ - A: ca, - B: c, - }) - } else { - out = append(out, c) - } - } - for _, c := range paths { - out = append(out, c) - } - return out, conflicts -} diff --git a/merkledag/utils/diffenum.go b/merkledag/utils/diffenum.go deleted file mode 100644 index d815102e715..00000000000 --- a/merkledag/utils/diffenum.go +++ /dev/null @@ -1,99 +0,0 @@ -package dagutils - -import ( - "context" - "fmt" - - mdag "github.com/ipfs/go-ipfs/merkledag" - - cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" - ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" -) - -// DiffEnumerate fetches every object in the graph pointed to by 'to' that is -// not in 'from'. This can be used to more efficiently fetch a graph if you can -// guarantee you already have the entirety of 'from' -func DiffEnumerate(ctx context.Context, dserv ipld.NodeGetter, from, to *cid.Cid) error { - fnd, err := dserv.Get(ctx, from) - if err != nil { - return fmt.Errorf("get %s: %s", from, err) - } - - tnd, err := dserv.Get(ctx, to) - if err != nil { - return fmt.Errorf("get %s: %s", to, err) - } - - diff := getLinkDiff(fnd, tnd) - - sset := cid.NewSet() - for _, c := range diff { - // Since we're already assuming we have everything in the 'from' graph, - // add all those cids to our 'already seen' set to avoid potentially - // enumerating them later - if c.bef != nil { - sset.Add(c.bef) - } - } - for _, c := range diff { - if c.bef == nil { - if sset.Has(c.aft) { - continue - } - err := mdag.EnumerateChildrenAsync(ctx, mdag.GetLinksDirect(dserv), c.aft, sset.Visit) - if err != nil { - return err - } - } else { - err := DiffEnumerate(ctx, dserv, c.bef, c.aft) - if err != nil { - return err - } - } - } - - return nil -} - -// if both bef and aft are not nil, then that signifies bef was replaces with aft. -// if bef is nil and aft is not, that means aft was newly added -// if aft is nil and bef is not, that means bef was deleted -type diffpair struct { - bef, aft *cid.Cid -} - -// getLinkDiff returns a changeset between nodes 'a' and 'b'. Currently does -// not log deletions as our usecase doesnt call for this. -func getLinkDiff(a, b ipld.Node) []diffpair { - ina := make(map[string]*ipld.Link) - inb := make(map[string]*ipld.Link) - var aonly []*cid.Cid - for _, l := range b.Links() { - inb[l.Cid.KeyString()] = l - } - for _, l := range a.Links() { - var key = l.Cid.KeyString() - ina[key] = l - if inb[key] == nil { - aonly = append(aonly, l.Cid) - } - } - - var out []diffpair - var aindex int - - for _, l := range b.Links() { - if ina[l.Cid.KeyString()] != nil { - continue - } - - if aindex < len(aonly) { - out = append(out, diffpair{bef: aonly[aindex], aft: l.Cid}) - aindex++ - } else { - out = append(out, diffpair{aft: l.Cid}) - continue - } - } - return out -} diff --git a/merkledag/utils/diffenum_test.go b/merkledag/utils/diffenum_test.go deleted file mode 100644 index bc9728fe7b8..00000000000 --- a/merkledag/utils/diffenum_test.go +++ /dev/null @@ -1,249 +0,0 @@ -package dagutils - -import ( - "context" - "fmt" - "testing" - - dag "github.com/ipfs/go-ipfs/merkledag" - mdtest "github.com/ipfs/go-ipfs/merkledag/test" - - cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" - ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" -) - -func buildNode(name string, desc map[string]ndesc, out map[string]ipld.Node) ipld.Node { - this := desc[name] - nd := new(dag.ProtoNode) - nd.SetData([]byte(name)) - for k, v := range this { - child, ok := out[v] - if !ok { - child = buildNode(v, desc, out) - out[v] = child - } - - if err := nd.AddNodeLink(k, child); err != nil { - panic(err) - } - } - - return nd -} - -type ndesc map[string]string - -func mkGraph(desc map[string]ndesc) map[string]ipld.Node { - out := make(map[string]ipld.Node) - for name := range desc { - if _, ok := out[name]; ok { - continue - } - - out[name] = buildNode(name, desc, out) - } - return out -} - -var tg1 = map[string]ndesc{ - "a1": ndesc{ - "foo": "b", - }, - "b": ndesc{}, - "a2": ndesc{ - "foo": "b", - "bar": "c", - }, - "c": ndesc{}, -} - -var tg2 = map[string]ndesc{ - "a1": ndesc{ - "foo": "b", - }, - "b": ndesc{}, - "a2": ndesc{ - "foo": "b", - "bar": "c", - }, - "c": ndesc{"baz": "d"}, - "d": ndesc{}, -} - -var tg3 = map[string]ndesc{ - "a1": ndesc{ - "foo": "b", - "bar": "c", - }, - "b": ndesc{}, - "a2": ndesc{ - "foo": "b", - "bar": "d", - }, - "c": ndesc{}, - "d": ndesc{}, -} - -var tg4 = map[string]ndesc{ - "a1": ndesc{ - "key1": "b", - "key2": "c", - }, - "a2": ndesc{ - "key1": "b", - "key2": "d", - }, -} - -var tg5 = map[string]ndesc{ - "a1": ndesc{ - "key1": "a", - "key2": "b", - }, - "a2": ndesc{ - "key1": "c", - "key2": "d", - }, -} - -func TestNameMatching(t *testing.T) { - nds := mkGraph(tg4) - - diff := getLinkDiff(nds["a1"], nds["a2"]) - if len(diff) != 1 { - t.Fatal(fmt.Errorf("node diff didn't match by name")) - } -} - -func TestNameMatching2(t *testing.T) { - nds := mkGraph(tg5) - - diff := getLinkDiff(nds["a1"], nds["a2"]) - if len(diff) != 2 { - t.Fatal(fmt.Errorf("incorrect number of link diff elements")) - } - if !(diff[0].bef.Equals(nds["a1"].Links()[0].Cid) && diff[0].aft.Equals(nds["a2"].Links()[0].Cid)) { - t.Fatal(fmt.Errorf("node diff didn't match by name")) - } -} - -func TestDiffEnumBasic(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - nds := mkGraph(tg1) - - ds := mdtest.Mock() - lgds := &getLogger{ds: ds} - - for _, nd := range nds { - err := ds.Add(ctx, nd) - if err != nil { - t.Fatal(err) - } - } - - err := DiffEnumerate(ctx, lgds, nds["a1"].Cid(), nds["a2"].Cid()) - if err != nil { - t.Fatal(err) - } - - err = assertCidList(lgds.log, []*cid.Cid{nds["a1"].Cid(), nds["a2"].Cid(), nds["c"].Cid()}) - if err != nil { - t.Fatal(err) - } -} - -type getLogger struct { - ds ipld.NodeGetter - log []*cid.Cid -} - -func (gl *getLogger) Get(ctx context.Context, c *cid.Cid) (ipld.Node, error) { - nd, err := gl.ds.Get(ctx, c) - if err != nil { - return nil, err - } - gl.log = append(gl.log, c) - return nd, nil -} - -func (gl *getLogger) GetMany(ctx context.Context, cids []*cid.Cid) <-chan *ipld.NodeOption { - outCh := make(chan *ipld.NodeOption, len(cids)) - nds := gl.ds.GetMany(ctx, cids) - for no := range nds { - if no.Err == nil { - gl.log = append(gl.log, no.Node.Cid()) - } - select { - case outCh <- no: - default: - panic("too many responses") - } - } - return nds -} - -func assertCidList(a, b []*cid.Cid) error { - if len(a) != len(b) { - return fmt.Errorf("got different number of cids than expected") - } - for i, c := range a { - if !c.Equals(b[i]) { - return fmt.Errorf("expected %s, got %s", c, b[i]) - } - } - return nil -} - -func TestDiffEnumFail(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - nds := mkGraph(tg2) - - ds := mdtest.Mock() - lgds := &getLogger{ds: ds} - - for _, s := range []string{"a1", "a2", "b", "c"} { - err := ds.Add(ctx, nds[s]) - if err != nil { - t.Fatal(err) - } - } - - err := DiffEnumerate(ctx, lgds, nds["a1"].Cid(), nds["a2"].Cid()) - if err != ipld.ErrNotFound { - t.Fatal("expected err not found") - } - - err = assertCidList(lgds.log, []*cid.Cid{nds["a1"].Cid(), nds["a2"].Cid(), nds["c"].Cid()}) - if err != nil { - t.Fatal(err) - } - -} - -func TestDiffEnumRecurse(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - nds := mkGraph(tg3) - - ds := mdtest.Mock() - lgds := &getLogger{ds: ds} - - for _, s := range []string{"a1", "a2", "b", "c", "d"} { - err := ds.Add(ctx, nds[s]) - if err != nil { - t.Fatal(err) - } - } - - err := DiffEnumerate(ctx, lgds, nds["a1"].Cid(), nds["a2"].Cid()) - if err != nil { - t.Fatal(err) - } - - err = assertCidList(lgds.log, []*cid.Cid{nds["a1"].Cid(), nds["a2"].Cid(), nds["c"].Cid(), nds["d"].Cid()}) - if err != nil { - t.Fatal(err) - } -} diff --git a/merkledag/utils/utils.go b/merkledag/utils/utils.go deleted file mode 100644 index 284598732df..00000000000 --- a/merkledag/utils/utils.go +++ /dev/null @@ -1,234 +0,0 @@ -package dagutils - -import ( - "context" - "errors" - - path "github.com/ipfs/go-ipfs-path" - bserv "github.com/ipfs/go-ipfs/blockservice" - dag "github.com/ipfs/go-ipfs/merkledag" - - offline "gx/ipfs/QmWM5HhdG5ZQNyHQ5XhMdGmV9CvLpFynQfGpTxN2MEM7Lc/go-ipfs-exchange-offline" - ds "gx/ipfs/QmXRKBQA4wXP7xWbFiZsR1GP4HV6wMDQ1aWFxZZ4uBcPX9/go-datastore" - syncds "gx/ipfs/QmXRKBQA4wXP7xWbFiZsR1GP4HV6wMDQ1aWFxZZ4uBcPX9/go-datastore/sync" - bstore "gx/ipfs/QmaG4DZ4JaqEfvPWt5nPPgoTzhc1tr1T3f4Nu9Jpdm8ymY/go-ipfs-blockstore" - ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" -) - -// Editor represents a ProtoNode tree editor and provides methods to -// modify it. -type Editor struct { - root *dag.ProtoNode - - // tmp is a temporary in memory (for now) dagstore for all of the - // intermediary nodes to be stored in - tmp ipld.DAGService - - // src is the dagstore with *all* of the data on it, it is used to pull - // nodes from for modification (nil is a valid value) - src ipld.DAGService -} - -// NewMemoryDagService returns a new, thread-safe in-memory DAGService. -func NewMemoryDagService() ipld.DAGService { - // build mem-datastore for editor's intermediary nodes - bs := bstore.NewBlockstore(syncds.MutexWrap(ds.NewMapDatastore())) - bsrv := bserv.New(bs, offline.Exchange(bs)) - return dag.NewDAGService(bsrv) -} - -// NewDagEditor returns an ProtoNode editor. -// -// * root is the node to be modified -// * source is the dagstore to pull nodes from (optional) -func NewDagEditor(root *dag.ProtoNode, source ipld.DAGService) *Editor { - return &Editor{ - root: root, - tmp: NewMemoryDagService(), - src: source, - } -} - -// GetNode returns the a copy of the root node being edited. -func (e *Editor) GetNode() *dag.ProtoNode { - return e.root.Copy().(*dag.ProtoNode) -} - -// GetDagService returns the DAGService used by this editor. -func (e *Editor) GetDagService() ipld.DAGService { - return e.tmp -} - -func addLink(ctx context.Context, ds ipld.DAGService, root *dag.ProtoNode, childname string, childnd ipld.Node) (*dag.ProtoNode, error) { - if childname == "" { - return nil, errors.New("cannot create link with no name") - } - - // ensure that the node we are adding is in the dagservice - err := ds.Add(ctx, childnd) - if err != nil { - return nil, err - } - - _ = ds.Remove(ctx, root.Cid()) - - // ensure no link with that name already exists - _ = root.RemoveNodeLink(childname) // ignore error, only option is ErrNotFound - - if err := root.AddNodeLink(childname, childnd); err != nil { - return nil, err - } - - if err := ds.Add(ctx, root); err != nil { - return nil, err - } - return root, nil -} - -// InsertNodeAtPath inserts a new node in the tree and replaces the current root with the new one. -func (e *Editor) InsertNodeAtPath(ctx context.Context, pth string, toinsert ipld.Node, create func() *dag.ProtoNode) error { - splpath := path.SplitList(pth) - nd, err := e.insertNodeAtPath(ctx, e.root, splpath, toinsert, create) - if err != nil { - return err - } - e.root = nd - return nil -} - -func (e *Editor) insertNodeAtPath(ctx context.Context, root *dag.ProtoNode, path []string, toinsert ipld.Node, create func() *dag.ProtoNode) (*dag.ProtoNode, error) { - if len(path) == 1 { - return addLink(ctx, e.tmp, root, path[0], toinsert) - } - - nd, err := root.GetLinkedProtoNode(ctx, e.tmp, path[0]) - if err != nil { - // if 'create' is true, we create directories on the way down as needed - if err == dag.ErrLinkNotFound && create != nil { - nd = create() - err = nil // no longer an error case - } else if err == ipld.ErrNotFound { - // try finding it in our source dagstore - nd, err = root.GetLinkedProtoNode(ctx, e.src, path[0]) - } - - // if we receive an ErrNotFound, then our second 'GetLinkedNode' call - // also fails, we want to error out - if err != nil { - return nil, err - } - } - - ndprime, err := e.insertNodeAtPath(ctx, nd, path[1:], toinsert, create) - if err != nil { - return nil, err - } - - _ = e.tmp.Remove(ctx, root.Cid()) - - _ = root.RemoveNodeLink(path[0]) - err = root.AddNodeLink(path[0], ndprime) - if err != nil { - return nil, err - } - - err = e.tmp.Add(ctx, root) - if err != nil { - return nil, err - } - - return root, nil -} - -// RmLink removes the link with the given name and updates the root node of -// the editor. -func (e *Editor) RmLink(ctx context.Context, pth string) error { - splpath := path.SplitList(pth) - nd, err := e.rmLink(ctx, e.root, splpath) - if err != nil { - return err - } - e.root = nd - return nil -} - -func (e *Editor) rmLink(ctx context.Context, root *dag.ProtoNode, path []string) (*dag.ProtoNode, error) { - if len(path) == 1 { - // base case, remove node in question - err := root.RemoveNodeLink(path[0]) - if err != nil { - return nil, err - } - - err = e.tmp.Add(ctx, root) - if err != nil { - return nil, err - } - - return root, nil - } - - // search for node in both tmp dagstore and source dagstore - nd, err := root.GetLinkedProtoNode(ctx, e.tmp, path[0]) - if err == ipld.ErrNotFound { - nd, err = root.GetLinkedProtoNode(ctx, e.src, path[0]) - } - - if err != nil { - return nil, err - } - - nnode, err := e.rmLink(ctx, nd, path[1:]) - if err != nil { - return nil, err - } - - e.tmp.Remove(ctx, root.Cid()) - - _ = root.RemoveNodeLink(path[0]) - err = root.AddNodeLink(path[0], nnode) - if err != nil { - return nil, err - } - - err = e.tmp.Add(ctx, root) - if err != nil { - return nil, err - } - - return root, nil -} - -// Finalize writes the new DAG to the given DAGService and returns the modified -// root node. -func (e *Editor) Finalize(ctx context.Context, ds ipld.DAGService) (*dag.ProtoNode, error) { - nd := e.GetNode() - err := copyDag(ctx, nd, e.tmp, ds) - return nd, err -} - -func copyDag(ctx context.Context, nd ipld.Node, from, to ipld.DAGService) error { - // TODO(#4609): make this batch. - err := to.Add(ctx, nd) - if err != nil { - return err - } - - for _, lnk := range nd.Links() { - child, err := lnk.GetNode(ctx, from) - if err != nil { - if err == ipld.ErrNotFound { - // not found means we didnt modify it, and it should - // already be in the target datastore - continue - } - return err - } - - err = copyDag(ctx, child, from, to) - if err != nil { - return err - } - } - return nil -} diff --git a/merkledag/utils/utils_test.go b/merkledag/utils/utils_test.go deleted file mode 100644 index 81af6ddb4f6..00000000000 --- a/merkledag/utils/utils_test.go +++ /dev/null @@ -1,114 +0,0 @@ -package dagutils - -import ( - "context" - "testing" - - path "github.com/ipfs/go-ipfs-path" - dag "github.com/ipfs/go-ipfs/merkledag" - mdtest "github.com/ipfs/go-ipfs/merkledag/test" - - cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" - ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" -) - -func TestAddLink(t *testing.T) { - ctx, context := context.WithCancel(context.Background()) - defer context() - - ds := mdtest.Mock() - fishnode := dag.NodeWithData([]byte("fishcakes!")) - - err := ds.Add(ctx, fishnode) - if err != nil { - t.Fatal(err) - } - - nd := new(dag.ProtoNode) - nnode, err := addLink(ctx, ds, nd, "fish", fishnode) - if err != nil { - t.Fatal(err) - } - - fnprime, err := nnode.GetLinkedNode(ctx, ds, "fish") - if err != nil { - t.Fatal(err) - } - - fnpkey := fnprime.Cid() - if !fnpkey.Equals(fishnode.Cid()) { - t.Fatal("wrong child node found!") - } -} - -func assertNodeAtPath(t *testing.T, ds ipld.DAGService, root *dag.ProtoNode, pth string, exp *cid.Cid) { - parts := path.SplitList(pth) - cur := root - for _, e := range parts { - nxt, err := cur.GetLinkedProtoNode(context.Background(), ds, e) - if err != nil { - t.Fatal(err) - } - - cur = nxt - } - - curc := cur.Cid() - if !curc.Equals(exp) { - t.Fatal("node not as expected at end of path") - } -} - -func TestInsertNode(t *testing.T) { - root := new(dag.ProtoNode) - e := NewDagEditor(root, nil) - - testInsert(t, e, "a", "anodefortesting", false, "") - testInsert(t, e, "a/b", "data", false, "") - testInsert(t, e, "a/b/c/d/e", "blah", false, "no link by that name") - testInsert(t, e, "a/b/c/d/e", "foo", true, "") - testInsert(t, e, "a/b/c/d/f", "baz", true, "") - testInsert(t, e, "a/b/c/d/f", "bar", true, "") - - testInsert(t, e, "", "bar", true, "cannot create link with no name") - testInsert(t, e, "////", "slashes", true, "cannot create link with no name") - - c := e.GetNode().Cid() - - if c.String() != "QmZ8yeT9uD6ouJPNAYt62XffYuXBT6b4mP4obRSE9cJrSt" { - t.Fatal("output was different than expected: ", c) - } -} - -func testInsert(t *testing.T, e *Editor, path, data string, create bool, experr string) { - child := dag.NodeWithData([]byte(data)) - err := e.tmp.Add(context.Background(), child) - if err != nil { - t.Fatal(err) - } - - var c func() *dag.ProtoNode - if create { - c = func() *dag.ProtoNode { - return &dag.ProtoNode{} - } - } - - err = e.InsertNodeAtPath(context.Background(), path, child, c) - if experr != "" { - var got string - if err != nil { - got = err.Error() - } - if got != experr { - t.Fatalf("expected '%s' but got '%s'", experr, got) - } - return - } - - if err != nil { - t.Fatal(err, path, data, create, experr) - } - - assertNodeAtPath(t, e.tmp, e.root, path, child.Cid()) -} diff --git a/mfs/dir.go b/mfs/dir.go index 0cc6e6568c8..19a4e6c76e2 100644 --- a/mfs/dir.go +++ b/mfs/dir.go @@ -10,7 +10,7 @@ import ( "sync" "time" - dag "github.com/ipfs/go-ipfs/merkledag" + dag "github.com/ipfs/go-ipfs-merkledag" ft "github.com/ipfs/go-ipfs/unixfs" uio "github.com/ipfs/go-ipfs/unixfs/io" ufspb "github.com/ipfs/go-ipfs/unixfs/pb" diff --git a/mfs/file.go b/mfs/file.go index 11d4a2a75e6..59fbfb2b9b1 100644 --- a/mfs/file.go +++ b/mfs/file.go @@ -5,7 +5,7 @@ import ( "fmt" "sync" - dag "github.com/ipfs/go-ipfs/merkledag" + dag "github.com/ipfs/go-ipfs-merkledag" ft "github.com/ipfs/go-ipfs/unixfs" mod "github.com/ipfs/go-ipfs/unixfs/mod" diff --git a/mfs/mfs_test.go b/mfs/mfs_test.go index b7f984db08a..d95453a104b 100644 --- a/mfs/mfs_test.go +++ b/mfs/mfs_test.go @@ -14,10 +14,10 @@ import ( "testing" "time" + dag "github.com/ipfs/go-ipfs-merkledag" "github.com/ipfs/go-ipfs-path" bserv "github.com/ipfs/go-ipfs/blockservice" importer "github.com/ipfs/go-ipfs/importer" - dag "github.com/ipfs/go-ipfs/merkledag" ft "github.com/ipfs/go-ipfs/unixfs" uio "github.com/ipfs/go-ipfs/unixfs/io" diff --git a/mfs/system.go b/mfs/system.go index d93af7cfdcc..a40be6404b6 100644 --- a/mfs/system.go +++ b/mfs/system.go @@ -16,7 +16,7 @@ import ( "sync" "time" - dag "github.com/ipfs/go-ipfs/merkledag" + dag "github.com/ipfs/go-ipfs-merkledag" ft "github.com/ipfs/go-ipfs/unixfs" logging "gx/ipfs/QmRb5jh8z2E8hMGN2tkvs1yHynUanqnZ3UeKwgN1i9P1F8/go-log" diff --git a/pin/gc/gc.go b/pin/gc/gc.go index 1606a01799e..6b16d0fdeb9 100644 --- a/pin/gc/gc.go +++ b/pin/gc/gc.go @@ -7,8 +7,8 @@ import ( "fmt" "strings" + dag "github.com/ipfs/go-ipfs-merkledag" bserv "github.com/ipfs/go-ipfs/blockservice" - dag "github.com/ipfs/go-ipfs/merkledag" pin "github.com/ipfs/go-ipfs/pin" "github.com/ipfs/go-ipfs/thirdparty/verifcid" diff --git a/pin/pin.go b/pin/pin.go index ce777988359..c4393eed3c0 100644 --- a/pin/pin.go +++ b/pin/pin.go @@ -9,8 +9,8 @@ import ( "sync" "time" - mdag "github.com/ipfs/go-ipfs/merkledag" - dutils "github.com/ipfs/go-ipfs/merkledag/utils" + mdag "github.com/ipfs/go-ipfs-merkledag" + dutils "github.com/ipfs/go-ipfs-merkledag/utils" logging "gx/ipfs/QmRb5jh8z2E8hMGN2tkvs1yHynUanqnZ3UeKwgN1i9P1F8/go-log" ds "gx/ipfs/QmXRKBQA4wXP7xWbFiZsR1GP4HV6wMDQ1aWFxZZ4uBcPX9/go-datastore" diff --git a/pin/pin_test.go b/pin/pin_test.go index e65f8b63ebc..2c2c1d4ed82 100644 --- a/pin/pin_test.go +++ b/pin/pin_test.go @@ -5,8 +5,8 @@ import ( "testing" "time" + mdag "github.com/ipfs/go-ipfs-merkledag" bs "github.com/ipfs/go-ipfs/blockservice" - mdag "github.com/ipfs/go-ipfs/merkledag" util "gx/ipfs/QmNiJuT8Ja3hMVpBHXv3Q6dwmperaQ6JjLtpMQgMCD7xvx/go-ipfs-util" offline "gx/ipfs/QmWM5HhdG5ZQNyHQ5XhMdGmV9CvLpFynQfGpTxN2MEM7Lc/go-ipfs-exchange-offline" diff --git a/pin/set.go b/pin/set.go index d239859eac8..9389357c67a 100644 --- a/pin/set.go +++ b/pin/set.go @@ -9,7 +9,7 @@ import ( "hash/fnv" "sort" - "github.com/ipfs/go-ipfs/merkledag" + "github.com/ipfs/go-ipfs-merkledag" "github.com/ipfs/go-ipfs/pin/internal/pb" "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto" diff --git a/pin/set_test.go b/pin/set_test.go index f7d6d0ede44..6abcf1de709 100644 --- a/pin/set_test.go +++ b/pin/set_test.go @@ -5,8 +5,8 @@ import ( "encoding/binary" "testing" + dag "github.com/ipfs/go-ipfs-merkledag" bserv "github.com/ipfs/go-ipfs/blockservice" - dag "github.com/ipfs/go-ipfs/merkledag" offline "gx/ipfs/QmWM5HhdG5ZQNyHQ5XhMdGmV9CvLpFynQfGpTxN2MEM7Lc/go-ipfs-exchange-offline" ds "gx/ipfs/QmXRKBQA4wXP7xWbFiZsR1GP4HV6wMDQ1aWFxZZ4uBcPX9/go-datastore" diff --git a/tar/format.go b/tar/format.go index d00051f1188..cf1cb058b0e 100644 --- a/tar/format.go +++ b/tar/format.go @@ -8,10 +8,10 @@ import ( "io" "strings" + dag "github.com/ipfs/go-ipfs-merkledag" + dagutil "github.com/ipfs/go-ipfs-merkledag/utils" path "github.com/ipfs/go-ipfs-path" importer "github.com/ipfs/go-ipfs/importer" - dag "github.com/ipfs/go-ipfs/merkledag" - dagutil "github.com/ipfs/go-ipfs/merkledag/utils" uio "github.com/ipfs/go-ipfs/unixfs/io" logging "gx/ipfs/QmRb5jh8z2E8hMGN2tkvs1yHynUanqnZ3UeKwgN1i9P1F8/go-log" diff --git a/unixfs/archive/tar/writer.go b/unixfs/archive/tar/writer.go index 1c00ce67421..f22ac622478 100644 --- a/unixfs/archive/tar/writer.go +++ b/unixfs/archive/tar/writer.go @@ -10,7 +10,7 @@ import ( "path" "time" - mdag "github.com/ipfs/go-ipfs/merkledag" + mdag "github.com/ipfs/go-ipfs-merkledag" ft "github.com/ipfs/go-ipfs/unixfs" uio "github.com/ipfs/go-ipfs/unixfs/io" upb "github.com/ipfs/go-ipfs/unixfs/pb" diff --git a/unixfs/hamt/hamt.go b/unixfs/hamt/hamt.go index c83ec1ceafc..926bb1366d1 100644 --- a/unixfs/hamt/hamt.go +++ b/unixfs/hamt/hamt.go @@ -25,7 +25,7 @@ import ( "fmt" "os" - dag "github.com/ipfs/go-ipfs/merkledag" + dag "github.com/ipfs/go-ipfs-merkledag" format "github.com/ipfs/go-ipfs/unixfs" upb "github.com/ipfs/go-ipfs/unixfs/pb" diff --git a/unixfs/hamt/hamt_stress_test.go b/unixfs/hamt/hamt_stress_test.go index 185e385e1d3..a8d17058d80 100644 --- a/unixfs/hamt/hamt_stress_test.go +++ b/unixfs/hamt/hamt_stress_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - mdtest "github.com/ipfs/go-ipfs/merkledag/test" + mdtest "github.com/ipfs/go-ipfs-merkledag/test" ft "github.com/ipfs/go-ipfs/unixfs" ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" diff --git a/unixfs/hamt/hamt_test.go b/unixfs/hamt/hamt_test.go index 9a0e172aca7..18f49c4438f 100644 --- a/unixfs/hamt/hamt_test.go +++ b/unixfs/hamt/hamt_test.go @@ -9,9 +9,9 @@ import ( "testing" "time" - dag "github.com/ipfs/go-ipfs/merkledag" - mdtest "github.com/ipfs/go-ipfs/merkledag/test" - dagutils "github.com/ipfs/go-ipfs/merkledag/utils" + dag "github.com/ipfs/go-ipfs-merkledag" + mdtest "github.com/ipfs/go-ipfs-merkledag/test" + dagutils "github.com/ipfs/go-ipfs-merkledag/utils" ft "github.com/ipfs/go-ipfs/unixfs" ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" diff --git a/unixfs/io/dagreader.go b/unixfs/io/dagreader.go index b1841bd74ef..4db93bed9d6 100644 --- a/unixfs/io/dagreader.go +++ b/unixfs/io/dagreader.go @@ -6,7 +6,7 @@ import ( "fmt" "io" - mdag "github.com/ipfs/go-ipfs/merkledag" + mdag "github.com/ipfs/go-ipfs-merkledag" ft "github.com/ipfs/go-ipfs/unixfs" ftpb "github.com/ipfs/go-ipfs/unixfs/pb" diff --git a/unixfs/io/dagreader_test.go b/unixfs/io/dagreader_test.go index e3d3d042b23..ae1be744778 100644 --- a/unixfs/io/dagreader_test.go +++ b/unixfs/io/dagreader_test.go @@ -8,7 +8,7 @@ import ( "strings" "testing" - mdag "github.com/ipfs/go-ipfs/merkledag" + mdag "github.com/ipfs/go-ipfs-merkledag" "github.com/ipfs/go-ipfs/unixfs" context "context" diff --git a/unixfs/io/dirbuilder.go b/unixfs/io/dirbuilder.go index e8cbff52d2a..1524300290c 100644 --- a/unixfs/io/dirbuilder.go +++ b/unixfs/io/dirbuilder.go @@ -5,7 +5,7 @@ import ( "fmt" "os" - mdag "github.com/ipfs/go-ipfs/merkledag" + mdag "github.com/ipfs/go-ipfs-merkledag" format "github.com/ipfs/go-ipfs/unixfs" hamt "github.com/ipfs/go-ipfs/unixfs/hamt" diff --git a/unixfs/io/dirbuilder_test.go b/unixfs/io/dirbuilder_test.go index 3518972968f..f5340c5c002 100644 --- a/unixfs/io/dirbuilder_test.go +++ b/unixfs/io/dirbuilder_test.go @@ -5,7 +5,7 @@ import ( "fmt" "testing" - mdtest "github.com/ipfs/go-ipfs/merkledag/test" + mdtest "github.com/ipfs/go-ipfs-merkledag/test" ft "github.com/ipfs/go-ipfs/unixfs" ) diff --git a/unixfs/io/pbdagreader.go b/unixfs/io/pbdagreader.go index a194dccf430..c79e7a19669 100644 --- a/unixfs/io/pbdagreader.go +++ b/unixfs/io/pbdagreader.go @@ -6,7 +6,7 @@ import ( "fmt" "io" - mdag "github.com/ipfs/go-ipfs/merkledag" + mdag "github.com/ipfs/go-ipfs-merkledag" ft "github.com/ipfs/go-ipfs/unixfs" ftpb "github.com/ipfs/go-ipfs/unixfs/pb" diff --git a/unixfs/io/resolve.go b/unixfs/io/resolve.go index 26d360bb377..04c163d90d8 100644 --- a/unixfs/io/resolve.go +++ b/unixfs/io/resolve.go @@ -3,7 +3,7 @@ package io import ( "context" - dag "github.com/ipfs/go-ipfs/merkledag" + dag "github.com/ipfs/go-ipfs-merkledag" ft "github.com/ipfs/go-ipfs/unixfs" hamt "github.com/ipfs/go-ipfs/unixfs/hamt" diff --git a/unixfs/mod/dagmodifier.go b/unixfs/mod/dagmodifier.go index f8a616ddae5..12aa86bedfa 100644 --- a/unixfs/mod/dagmodifier.go +++ b/unixfs/mod/dagmodifier.go @@ -8,9 +8,9 @@ import ( "errors" "io" + mdag "github.com/ipfs/go-ipfs-merkledag" help "github.com/ipfs/go-ipfs/importer/helpers" trickle "github.com/ipfs/go-ipfs/importer/trickle" - mdag "github.com/ipfs/go-ipfs/merkledag" ft "github.com/ipfs/go-ipfs/unixfs" uio "github.com/ipfs/go-ipfs/unixfs/io" diff --git a/unixfs/test/utils.go b/unixfs/test/utils.go index 574b7ec9724..ecf7958034f 100644 --- a/unixfs/test/utils.go +++ b/unixfs/test/utils.go @@ -8,10 +8,10 @@ import ( "io/ioutil" "testing" + mdag "github.com/ipfs/go-ipfs-merkledag" + mdagmock "github.com/ipfs/go-ipfs-merkledag/test" h "github.com/ipfs/go-ipfs/importer/helpers" trickle "github.com/ipfs/go-ipfs/importer/trickle" - mdag "github.com/ipfs/go-ipfs/merkledag" - mdagmock "github.com/ipfs/go-ipfs/merkledag/test" ft "github.com/ipfs/go-ipfs/unixfs" u "gx/ipfs/QmNiJuT8Ja3hMVpBHXv3Q6dwmperaQ6JjLtpMQgMCD7xvx/go-ipfs-util" diff --git a/unixfs/unixfs.go b/unixfs/unixfs.go index 654de7ff82e..69cafa10e8e 100644 --- a/unixfs/unixfs.go +++ b/unixfs/unixfs.go @@ -8,7 +8,7 @@ import ( proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto" - dag "github.com/ipfs/go-ipfs/merkledag" + dag "github.com/ipfs/go-ipfs-merkledag" pb "github.com/ipfs/go-ipfs/unixfs/pb" ) From d94db49fe5cc6c293a1f6f7a7fdf48826ed50a7c Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Sat, 21 Apr 2018 22:32:27 +0200 Subject: [PATCH 4/5] extract blockservice --- blockservice/blockservice.go | 336 ------------------------------ blockservice/blockservice_test.go | 48 ----- blockservice/test/blocks_test.go | 97 --------- blockservice/test/mock.go | 24 --- core/builder.go | 2 +- core/commands/add.go | 2 +- core/commands/files.go | 2 +- core/commands/ls.go | 2 +- core/commands/pin.go | 2 +- core/core.go | 2 +- core/coreapi/pin.go | 2 +- core/coreunix/add_test.go | 2 +- core/coreunix/metadata_test.go | 2 +- mfs/mfs_test.go | 2 +- pin/gc/gc.go | 2 +- pin/pin_test.go | 2 +- pin/set_test.go | 2 +- 17 files changed, 13 insertions(+), 518 deletions(-) delete mode 100644 blockservice/blockservice.go delete mode 100644 blockservice/blockservice_test.go delete mode 100644 blockservice/test/blocks_test.go delete mode 100644 blockservice/test/mock.go diff --git a/blockservice/blockservice.go b/blockservice/blockservice.go deleted file mode 100644 index b40c6647341..00000000000 --- a/blockservice/blockservice.go +++ /dev/null @@ -1,336 +0,0 @@ -// package blockservice implements a BlockService interface that provides -// a single GetBlock/AddBlock interface that seamlessly retrieves data either -// locally or from a remote peer through the exchange. -package blockservice - -import ( - "context" - "errors" - "fmt" - "io" - - "github.com/ipfs/go-ipfs/thirdparty/verifcid" - - logging "gx/ipfs/QmRb5jh8z2E8hMGN2tkvs1yHynUanqnZ3UeKwgN1i9P1F8/go-log" - blockstore "gx/ipfs/QmaG4DZ4JaqEfvPWt5nPPgoTzhc1tr1T3f4Nu9Jpdm8ymY/go-ipfs-blockstore" - cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" - exchange "gx/ipfs/QmdcAXgEHUueP4A7b5hjabKn2EooeHgMreMvFC249dGCgc/go-ipfs-exchange-interface" - blocks "gx/ipfs/Qmej7nf81hi2x2tvjRBF3mcp74sQyuDH4VMYDGd1YtXjb2/go-block-format" -) - -var log = logging.Logger("blockservice") - -var ErrNotFound = errors.New("blockservice: key not found") - -// BlockGetter is the common interface shared between blockservice sessions and -// the blockservice. -type BlockGetter interface { - // GetBlock gets the requested block. - GetBlock(ctx context.Context, c *cid.Cid) (blocks.Block, error) - - // GetBlocks does a batch request for the given cids, returning blocks as - // they are found, in no particular order. - // - // It may not be able to find all requested blocks (or the context may - // be canceled). In that case, it will close the channel early. It is up - // to the consumer to detect this situation and keep track which blocks - // it has received and which it hasn't. - GetBlocks(ctx context.Context, ks []*cid.Cid) <-chan blocks.Block -} - -// BlockService is a hybrid block datastore. It stores data in a local -// datastore and may retrieve data from a remote Exchange. -// It uses an internal `datastore.Datastore` instance to store values. -type BlockService interface { - io.Closer - BlockGetter - - // Blockstore returns a reference to the underlying blockstore - Blockstore() blockstore.Blockstore - - // Exchange returns a reference to the underlying exchange (usually bitswap) - Exchange() exchange.Interface - - // AddBlock puts a given block to the underlying datastore - AddBlock(o blocks.Block) error - - // AddBlocks adds a slice of blocks at the same time using batching - // capabilities of the underlying datastore whenever possible. - AddBlocks(bs []blocks.Block) error - - // DeleteBlock deletes the given block from the blockservice. - DeleteBlock(o *cid.Cid) error -} - -type blockService struct { - blockstore blockstore.Blockstore - exchange exchange.Interface - // If checkFirst is true then first check that a block doesn't - // already exist to avoid republishing the block on the exchange. - checkFirst bool -} - -// NewBlockService creates a BlockService with given datastore instance. -func New(bs blockstore.Blockstore, rem exchange.Interface) BlockService { - if rem == nil { - log.Warning("blockservice running in local (offline) mode.") - } - - return &blockService{ - blockstore: bs, - exchange: rem, - checkFirst: true, - } -} - -// NewWriteThrough ceates a BlockService that guarantees writes will go -// through to the blockstore and are not skipped by cache checks. -func NewWriteThrough(bs blockstore.Blockstore, rem exchange.Interface) BlockService { - if rem == nil { - log.Warning("blockservice running in local (offline) mode.") - } - - return &blockService{ - blockstore: bs, - exchange: rem, - checkFirst: false, - } -} - -// Blockstore returns the blockstore behind this blockservice. -func (s *blockService) Blockstore() blockstore.Blockstore { - return s.blockstore -} - -// Exchange returns the exchange behind this blockservice. -func (s *blockService) Exchange() exchange.Interface { - return s.exchange -} - -// NewSession creates a new session that allows for -// controlled exchange of wantlists to decrease the bandwidth overhead. -// If the current exchange is a SessionExchange, a new exchange -// session will be created. Otherwise, the current exchange will be used -// directly. -func NewSession(ctx context.Context, bs BlockService) *Session { - exch := bs.Exchange() - if sessEx, ok := exch.(exchange.SessionExchange); ok { - ses := sessEx.NewSession(ctx) - return &Session{ - ses: ses, - bs: bs.Blockstore(), - } - } - return &Session{ - ses: exch, - bs: bs.Blockstore(), - } -} - -// AddBlock adds a particular block to the service, Putting it into the datastore. -// TODO pass a context into this if the remote.HasBlock is going to remain here. -func (s *blockService) AddBlock(o blocks.Block) error { - c := o.Cid() - // hash security - err := verifcid.ValidateCid(c) - if err != nil { - return err - } - if s.checkFirst { - if has, err := s.blockstore.Has(c); has || err != nil { - return err - } - } - - if err := s.blockstore.Put(o); err != nil { - return err - } - - log.Event(context.TODO(), "BlockService.BlockAdded", c) - - if err := s.exchange.HasBlock(o); err != nil { - // TODO(#4623): really an error? - return errors.New("blockservice is closed") - } - - return nil -} - -func (s *blockService) AddBlocks(bs []blocks.Block) error { - // hash security - for _, b := range bs { - err := verifcid.ValidateCid(b.Cid()) - if err != nil { - return err - } - } - var toput []blocks.Block - if s.checkFirst { - toput = make([]blocks.Block, 0, len(bs)) - for _, b := range bs { - has, err := s.blockstore.Has(b.Cid()) - if err != nil { - return err - } - if !has { - toput = append(toput, b) - } - } - } else { - toput = bs - } - - err := s.blockstore.PutMany(toput) - if err != nil { - return err - } - - for _, o := range toput { - log.Event(context.TODO(), "BlockService.BlockAdded", o.Cid()) - if err := s.exchange.HasBlock(o); err != nil { - // TODO(#4623): Should this really *return*? - return fmt.Errorf("blockservice is closed (%s)", err) - } - } - return nil -} - -// GetBlock retrieves a particular block from the service, -// Getting it from the datastore using the key (hash). -func (s *blockService) GetBlock(ctx context.Context, c *cid.Cid) (blocks.Block, error) { - log.Debugf("BlockService GetBlock: '%s'", c) - - var f exchange.Fetcher - if s.exchange != nil { - f = s.exchange - } - - return getBlock(ctx, c, s.blockstore, f) // hash security -} - -func getBlock(ctx context.Context, c *cid.Cid, bs blockstore.Blockstore, f exchange.Fetcher) (blocks.Block, error) { - err := verifcid.ValidateCid(c) // hash security - if err != nil { - return nil, err - } - - block, err := bs.Get(c) - if err == nil { - return block, nil - } - - if err == blockstore.ErrNotFound && f != nil { - // TODO be careful checking ErrNotFound. If the underlying - // implementation changes, this will break. - log.Debug("Blockservice: Searching bitswap") - blk, err := f.GetBlock(ctx, c) - if err != nil { - if err == blockstore.ErrNotFound { - return nil, ErrNotFound - } - return nil, err - } - log.Event(ctx, "BlockService.BlockFetched", c) - return blk, nil - } - - log.Debug("Blockservice GetBlock: Not found") - if err == blockstore.ErrNotFound { - return nil, ErrNotFound - } - - return nil, err -} - -// GetBlocks gets a list of blocks asynchronously and returns through -// the returned channel. -// NB: No guarantees are made about order. -func (s *blockService) GetBlocks(ctx context.Context, ks []*cid.Cid) <-chan blocks.Block { - return getBlocks(ctx, ks, s.blockstore, s.exchange) // hash security -} - -func getBlocks(ctx context.Context, ks []*cid.Cid, bs blockstore.Blockstore, f exchange.Fetcher) <-chan blocks.Block { - out := make(chan blocks.Block) - - go func() { - defer close(out) - - k := 0 - for _, c := range ks { - // hash security - if err := verifcid.ValidateCid(c); err == nil { - ks[k] = c - k++ - } else { - log.Errorf("unsafe CID (%s) passed to blockService.GetBlocks: %s", c, err) - } - } - ks = ks[:k] - - var misses []*cid.Cid - for _, c := range ks { - hit, err := bs.Get(c) - if err != nil { - misses = append(misses, c) - continue - } - select { - case out <- hit: - case <-ctx.Done(): - return - } - } - - if len(misses) == 0 { - return - } - - rblocks, err := f.GetBlocks(ctx, misses) - if err != nil { - log.Debugf("Error with GetBlocks: %s", err) - return - } - - for b := range rblocks { - log.Event(ctx, "BlockService.BlockFetched", b.Cid()) - select { - case out <- b: - case <-ctx.Done(): - return - } - } - }() - return out -} - -// DeleteBlock deletes a block in the blockservice from the datastore -func (s *blockService) DeleteBlock(c *cid.Cid) error { - err := s.blockstore.DeleteBlock(c) - if err == nil { - log.Event(context.TODO(), "BlockService.BlockDeleted", c) - } - return err -} - -func (s *blockService) Close() error { - log.Debug("blockservice is shutting down...") - return s.exchange.Close() -} - -// Session is a helper type to provide higher level access to bitswap sessions -type Session struct { - bs blockstore.Blockstore - ses exchange.Fetcher -} - -// GetBlock gets a block in the context of a request session -func (s *Session) GetBlock(ctx context.Context, c *cid.Cid) (blocks.Block, error) { - return getBlock(ctx, c, s.bs, s.ses) // hash security -} - -// GetBlocks gets blocks in the context of a request session -func (s *Session) GetBlocks(ctx context.Context, ks []*cid.Cid) <-chan blocks.Block { - return getBlocks(ctx, ks, s.bs, s.ses) // hash security -} - -var _ BlockGetter = (*Session)(nil) diff --git a/blockservice/blockservice_test.go b/blockservice/blockservice_test.go deleted file mode 100644 index eda9497c137..00000000000 --- a/blockservice/blockservice_test.go +++ /dev/null @@ -1,48 +0,0 @@ -package blockservice - -import ( - "testing" - - offline "gx/ipfs/QmWM5HhdG5ZQNyHQ5XhMdGmV9CvLpFynQfGpTxN2MEM7Lc/go-ipfs-exchange-offline" - ds "gx/ipfs/QmXRKBQA4wXP7xWbFiZsR1GP4HV6wMDQ1aWFxZZ4uBcPX9/go-datastore" - dssync "gx/ipfs/QmXRKBQA4wXP7xWbFiZsR1GP4HV6wMDQ1aWFxZZ4uBcPX9/go-datastore/sync" - blockstore "gx/ipfs/QmaG4DZ4JaqEfvPWt5nPPgoTzhc1tr1T3f4Nu9Jpdm8ymY/go-ipfs-blockstore" - blocks "gx/ipfs/Qmej7nf81hi2x2tvjRBF3mcp74sQyuDH4VMYDGd1YtXjb2/go-block-format" - butil "gx/ipfs/Qmf951DP11mCoctpyF3ZppPZdo2oAxuNi2vnkVDgHJ8Fqk/go-ipfs-blocksutil" -) - -func TestWriteThroughWorks(t *testing.T) { - bstore := &PutCountingBlockstore{ - blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())), - 0, - } - bstore2 := blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())) - exch := offline.Exchange(bstore2) - bserv := NewWriteThrough(bstore, exch) - bgen := butil.NewBlockGenerator() - - block := bgen.Next() - - t.Logf("PutCounter: %d", bstore.PutCounter) - bserv.AddBlock(block) - if bstore.PutCounter != 1 { - t.Fatalf("expected just one Put call, have: %d", bstore.PutCounter) - } - - bserv.AddBlock(block) - if bstore.PutCounter != 2 { - t.Fatalf("Put should have called again, should be 2 is: %d", bstore.PutCounter) - } -} - -var _ blockstore.Blockstore = (*PutCountingBlockstore)(nil) - -type PutCountingBlockstore struct { - blockstore.Blockstore - PutCounter int -} - -func (bs *PutCountingBlockstore) Put(block blocks.Block) error { - bs.PutCounter++ - return bs.Blockstore.Put(block) -} diff --git a/blockservice/test/blocks_test.go b/blockservice/test/blocks_test.go deleted file mode 100644 index 978bc8658c9..00000000000 --- a/blockservice/test/blocks_test.go +++ /dev/null @@ -1,97 +0,0 @@ -package bstest - -import ( - "bytes" - "context" - "fmt" - "testing" - "time" - - . "github.com/ipfs/go-ipfs/blockservice" - - u "gx/ipfs/QmNiJuT8Ja3hMVpBHXv3Q6dwmperaQ6JjLtpMQgMCD7xvx/go-ipfs-util" - offline "gx/ipfs/QmWM5HhdG5ZQNyHQ5XhMdGmV9CvLpFynQfGpTxN2MEM7Lc/go-ipfs-exchange-offline" - ds "gx/ipfs/QmXRKBQA4wXP7xWbFiZsR1GP4HV6wMDQ1aWFxZZ4uBcPX9/go-datastore" - dssync "gx/ipfs/QmXRKBQA4wXP7xWbFiZsR1GP4HV6wMDQ1aWFxZZ4uBcPX9/go-datastore/sync" - blockstore "gx/ipfs/QmaG4DZ4JaqEfvPWt5nPPgoTzhc1tr1T3f4Nu9Jpdm8ymY/go-ipfs-blockstore" - cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" - blocks "gx/ipfs/Qmej7nf81hi2x2tvjRBF3mcp74sQyuDH4VMYDGd1YtXjb2/go-block-format" -) - -func newObject(data []byte) blocks.Block { - return blocks.NewBlock(data) -} - -func TestBlocks(t *testing.T) { - bstore := blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())) - bs := New(bstore, offline.Exchange(bstore)) - defer bs.Close() - - o := newObject([]byte("beep boop")) - h := cid.NewCidV0(u.Hash([]byte("beep boop"))) - if !o.Cid().Equals(h) { - t.Error("Block key and data multihash key not equal") - } - - err := bs.AddBlock(o) - if err != nil { - t.Error("failed to add block to BlockService", err) - return - } - - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - b2, err := bs.GetBlock(ctx, o.Cid()) - if err != nil { - t.Error("failed to retrieve block from BlockService", err) - return - } - - if !o.Cid().Equals(b2.Cid()) { - t.Error("Block keys not equal.") - } - - if !bytes.Equal(o.RawData(), b2.RawData()) { - t.Error("Block data is not equal.") - } -} - -func makeObjects(n int) []blocks.Block { - var out []blocks.Block - for i := 0; i < n; i++ { - out = append(out, newObject([]byte(fmt.Sprintf("object %d", i)))) - } - return out -} - -func TestGetBlocksSequential(t *testing.T) { - var servs = Mocks(4) - for _, s := range servs { - defer s.Close() - } - objs := makeObjects(50) - - var cids []*cid.Cid - for _, o := range objs { - cids = append(cids, o.Cid()) - servs[0].AddBlock(o) - } - - t.Log("one instance at a time, get blocks concurrently") - - for i := 1; i < len(servs); i++ { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*50) - defer cancel() - out := servs[i].GetBlocks(ctx, cids) - gotten := make(map[string]blocks.Block) - for blk := range out { - if _, ok := gotten[blk.Cid().KeyString()]; ok { - t.Fatal("Got duplicate block!") - } - gotten[blk.Cid().KeyString()] = blk - } - if len(gotten) != len(objs) { - t.Fatalf("Didnt get enough blocks back: %d/%d", len(gotten), len(objs)) - } - } -} diff --git a/blockservice/test/mock.go b/blockservice/test/mock.go deleted file mode 100644 index a9c34d42d7c..00000000000 --- a/blockservice/test/mock.go +++ /dev/null @@ -1,24 +0,0 @@ -package bstest - -import ( - . "github.com/ipfs/go-ipfs/blockservice" - bitswap "github.com/ipfs/go-ipfs/exchange/bitswap" - tn "github.com/ipfs/go-ipfs/exchange/bitswap/testnet" - - delay "gx/ipfs/QmRJVNatYJwTAHgdSM1Xef9QVQ1Ch3XHdmcrykjP5Y4soL/go-ipfs-delay" - mockrouting "gx/ipfs/QmXtoXbu9ReyV6Q4kDQ5CF9wXQNDY1PdHc4HhfxRR5AHB3/go-ipfs-routing/mock" -) - -// Mocks returns |n| connected mock Blockservices -func Mocks(n int) []BlockService { - net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(0)) - sg := bitswap.NewTestSessionGenerator(net) - - instances := sg.Instances(n) - - var servs []BlockService - for _, i := range instances { - servs = append(servs, New(i.Blockstore(), i.Exchange)) - } - return servs -} diff --git a/core/builder.go b/core/builder.go index 4e1d2d0a3a0..edaf8590cab 100644 --- a/core/builder.go +++ b/core/builder.go @@ -9,9 +9,9 @@ import ( "syscall" "time" + bserv "github.com/ipfs/go-ipfs-blockservice" dag "github.com/ipfs/go-ipfs-merkledag" resolver "github.com/ipfs/go-ipfs-path-resolver" - bserv "github.com/ipfs/go-ipfs/blockservice" filestore "github.com/ipfs/go-ipfs/filestore" pin "github.com/ipfs/go-ipfs/pin" repo "github.com/ipfs/go-ipfs/repo" diff --git a/core/commands/add.go b/core/commands/add.go index 7fa359a0630..c4ce9f8f631 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -7,9 +7,9 @@ import ( "os" "strings" + blockservice "github.com/ipfs/go-ipfs-blockservice" dag "github.com/ipfs/go-ipfs-merkledag" dagtest "github.com/ipfs/go-ipfs-merkledag/test" - blockservice "github.com/ipfs/go-ipfs/blockservice" core "github.com/ipfs/go-ipfs/core" "github.com/ipfs/go-ipfs/core/coreunix" mfs "github.com/ipfs/go-ipfs/mfs" diff --git a/core/commands/files.go b/core/commands/files.go index 33771cf353f..9bb37293f08 100644 --- a/core/commands/files.go +++ b/core/commands/files.go @@ -10,10 +10,10 @@ import ( gopath "path" "strings" + bservice "github.com/ipfs/go-ipfs-blockservice" dag "github.com/ipfs/go-ipfs-merkledag" path "github.com/ipfs/go-ipfs-path" resolver "github.com/ipfs/go-ipfs-path-resolver" - bservice "github.com/ipfs/go-ipfs/blockservice" oldcmds "github.com/ipfs/go-ipfs/commands" lgc "github.com/ipfs/go-ipfs/commands/legacy" core "github.com/ipfs/go-ipfs/core" diff --git a/core/commands/ls.go b/core/commands/ls.go index f3b586f4ff8..9b9de288b7b 100644 --- a/core/commands/ls.go +++ b/core/commands/ls.go @@ -6,10 +6,10 @@ import ( "io" "text/tabwriter" + blockservice "github.com/ipfs/go-ipfs-blockservice" merkledag "github.com/ipfs/go-ipfs-merkledag" path "github.com/ipfs/go-ipfs-path" resolver "github.com/ipfs/go-ipfs-path-resolver" - blockservice "github.com/ipfs/go-ipfs/blockservice" cmds "github.com/ipfs/go-ipfs/commands" core "github.com/ipfs/go-ipfs/core" e "github.com/ipfs/go-ipfs/core/commands/e" diff --git a/core/commands/pin.go b/core/commands/pin.go index eb6dd26c8d5..4c687dc1454 100644 --- a/core/commands/pin.go +++ b/core/commands/pin.go @@ -7,10 +7,10 @@ import ( "io" "time" + bserv "github.com/ipfs/go-ipfs-blockservice" dag "github.com/ipfs/go-ipfs-merkledag" path "github.com/ipfs/go-ipfs-path" resolver "github.com/ipfs/go-ipfs-path-resolver" - bserv "github.com/ipfs/go-ipfs/blockservice" cmds "github.com/ipfs/go-ipfs/commands" core "github.com/ipfs/go-ipfs/core" e "github.com/ipfs/go-ipfs/core/commands/e" diff --git a/core/core.go b/core/core.go index d7429580324..c91ba6428ad 100644 --- a/core/core.go +++ b/core/core.go @@ -21,9 +21,9 @@ import ( "strings" "time" + bserv "github.com/ipfs/go-ipfs-blockservice" merkledag "github.com/ipfs/go-ipfs-merkledag" "github.com/ipfs/go-ipfs-path-resolver" - bserv "github.com/ipfs/go-ipfs/blockservice" bitswap "github.com/ipfs/go-ipfs/exchange/bitswap" bsnet "github.com/ipfs/go-ipfs/exchange/bitswap/network" rp "github.com/ipfs/go-ipfs/exchange/reprovide" diff --git a/core/coreapi/pin.go b/core/coreapi/pin.go index 6170c76de8b..7da639c447c 100644 --- a/core/coreapi/pin.go +++ b/core/coreapi/pin.go @@ -4,8 +4,8 @@ import ( "context" "fmt" + bserv "github.com/ipfs/go-ipfs-blockservice" merkledag "github.com/ipfs/go-ipfs-merkledag" - bserv "github.com/ipfs/go-ipfs/blockservice" coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface" caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" corerepo "github.com/ipfs/go-ipfs/core/corerepo" diff --git a/core/coreunix/add_test.go b/core/coreunix/add_test.go index 6e1eb4e87a8..7c77bcf3a98 100644 --- a/core/coreunix/add_test.go +++ b/core/coreunix/add_test.go @@ -10,8 +10,8 @@ import ( "testing" "time" + "github.com/ipfs/go-ipfs-blockservice" dag "github.com/ipfs/go-ipfs-merkledag" - "github.com/ipfs/go-ipfs/blockservice" "github.com/ipfs/go-ipfs/core" "github.com/ipfs/go-ipfs/pin/gc" "github.com/ipfs/go-ipfs/repo" diff --git a/core/coreunix/metadata_test.go b/core/coreunix/metadata_test.go index dcf66e5c180..5e9b1908ced 100644 --- a/core/coreunix/metadata_test.go +++ b/core/coreunix/metadata_test.go @@ -6,8 +6,8 @@ import ( "io/ioutil" "testing" + bserv "github.com/ipfs/go-ipfs-blockservice" merkledag "github.com/ipfs/go-ipfs-merkledag" - bserv "github.com/ipfs/go-ipfs/blockservice" core "github.com/ipfs/go-ipfs/core" importer "github.com/ipfs/go-ipfs/importer" ft "github.com/ipfs/go-ipfs/unixfs" diff --git a/mfs/mfs_test.go b/mfs/mfs_test.go index d95453a104b..bf3261e6293 100644 --- a/mfs/mfs_test.go +++ b/mfs/mfs_test.go @@ -14,9 +14,9 @@ import ( "testing" "time" + bserv "github.com/ipfs/go-ipfs-blockservice" dag "github.com/ipfs/go-ipfs-merkledag" "github.com/ipfs/go-ipfs-path" - bserv "github.com/ipfs/go-ipfs/blockservice" importer "github.com/ipfs/go-ipfs/importer" ft "github.com/ipfs/go-ipfs/unixfs" uio "github.com/ipfs/go-ipfs/unixfs/io" diff --git a/pin/gc/gc.go b/pin/gc/gc.go index 6b16d0fdeb9..98793d68234 100644 --- a/pin/gc/gc.go +++ b/pin/gc/gc.go @@ -7,8 +7,8 @@ import ( "fmt" "strings" + bserv "github.com/ipfs/go-ipfs-blockservice" dag "github.com/ipfs/go-ipfs-merkledag" - bserv "github.com/ipfs/go-ipfs/blockservice" pin "github.com/ipfs/go-ipfs/pin" "github.com/ipfs/go-ipfs/thirdparty/verifcid" diff --git a/pin/pin_test.go b/pin/pin_test.go index 2c2c1d4ed82..2676a3b2cf9 100644 --- a/pin/pin_test.go +++ b/pin/pin_test.go @@ -5,8 +5,8 @@ import ( "testing" "time" + bs "github.com/ipfs/go-ipfs-blockservice" mdag "github.com/ipfs/go-ipfs-merkledag" - bs "github.com/ipfs/go-ipfs/blockservice" util "gx/ipfs/QmNiJuT8Ja3hMVpBHXv3Q6dwmperaQ6JjLtpMQgMCD7xvx/go-ipfs-util" offline "gx/ipfs/QmWM5HhdG5ZQNyHQ5XhMdGmV9CvLpFynQfGpTxN2MEM7Lc/go-ipfs-exchange-offline" diff --git a/pin/set_test.go b/pin/set_test.go index 6abcf1de709..298d6795cea 100644 --- a/pin/set_test.go +++ b/pin/set_test.go @@ -5,8 +5,8 @@ import ( "encoding/binary" "testing" + bserv "github.com/ipfs/go-ipfs-blockservice" dag "github.com/ipfs/go-ipfs-merkledag" - bserv "github.com/ipfs/go-ipfs/blockservice" offline "gx/ipfs/QmWM5HhdG5ZQNyHQ5XhMdGmV9CvLpFynQfGpTxN2MEM7Lc/go-ipfs-exchange-offline" ds "gx/ipfs/QmXRKBQA4wXP7xWbFiZsR1GP4HV6wMDQ1aWFxZZ4uBcPX9/go-datastore" From b6d36a3c96c5178cce1c434fa0f5f20bd00f5a26 Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Sat, 21 Apr 2018 22:38:51 +0200 Subject: [PATCH 5/5] extract thirdparty/verifcid --- core/commands/pin.go | 2 +- exchange/reprovide/reprovide.go | 4 +- pin/gc/gc.go | 2 +- thirdparty/verifbs/verifbs.go | 2 +- thirdparty/verifcid/validate.go | 62 ---------------------------- thirdparty/verifcid/validate_test.go | 59 -------------------------- 6 files changed, 5 insertions(+), 126 deletions(-) delete mode 100644 thirdparty/verifcid/validate.go delete mode 100644 thirdparty/verifcid/validate_test.go diff --git a/core/commands/pin.go b/core/commands/pin.go index 4c687dc1454..26b7646c135 100644 --- a/core/commands/pin.go +++ b/core/commands/pin.go @@ -16,8 +16,8 @@ import ( e "github.com/ipfs/go-ipfs/core/commands/e" corerepo "github.com/ipfs/go-ipfs/core/corerepo" pin "github.com/ipfs/go-ipfs/pin" - "github.com/ipfs/go-ipfs/thirdparty/verifcid" uio "github.com/ipfs/go-ipfs/unixfs/io" + "github.com/ipfs/go-verifcid" u "gx/ipfs/QmNiJuT8Ja3hMVpBHXv3Q6dwmperaQ6JjLtpMQgMCD7xvx/go-ipfs-util" offline "gx/ipfs/QmWM5HhdG5ZQNyHQ5XhMdGmV9CvLpFynQfGpTxN2MEM7Lc/go-ipfs-exchange-offline" diff --git a/exchange/reprovide/reprovide.go b/exchange/reprovide/reprovide.go index 8049bfaf5e0..c24d8c0f060 100644 --- a/exchange/reprovide/reprovide.go +++ b/exchange/reprovide/reprovide.go @@ -5,12 +5,12 @@ import ( "fmt" "time" - "github.com/ipfs/go-ipfs/thirdparty/verifcid" - backoff "gx/ipfs/QmPJUtEJsm5YLUWhF6imvyCH8KZXRJa9Wup7FDMwTy5Ufz/backoff" logging "gx/ipfs/QmRb5jh8z2E8hMGN2tkvs1yHynUanqnZ3UeKwgN1i9P1F8/go-log" routing "gx/ipfs/QmTiWLZ6Fo5j4KcTVutZJ5KWRRJrbxzmxA4td8NfEdrPh7/go-libp2p-routing" cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" + + verifcid "github.com/ipfs/go-verifcid" ) var log = logging.Logger("reprovider") diff --git a/pin/gc/gc.go b/pin/gc/gc.go index 98793d68234..29bd4879ef9 100644 --- a/pin/gc/gc.go +++ b/pin/gc/gc.go @@ -10,7 +10,7 @@ import ( bserv "github.com/ipfs/go-ipfs-blockservice" dag "github.com/ipfs/go-ipfs-merkledag" pin "github.com/ipfs/go-ipfs/pin" - "github.com/ipfs/go-ipfs/thirdparty/verifcid" + "github.com/ipfs/go-verifcid" logging "gx/ipfs/QmRb5jh8z2E8hMGN2tkvs1yHynUanqnZ3UeKwgN1i9P1F8/go-log" offline "gx/ipfs/QmWM5HhdG5ZQNyHQ5XhMdGmV9CvLpFynQfGpTxN2MEM7Lc/go-ipfs-exchange-offline" diff --git a/thirdparty/verifbs/verifbs.go b/thirdparty/verifbs/verifbs.go index 5055d968c90..12181a29f41 100644 --- a/thirdparty/verifbs/verifbs.go +++ b/thirdparty/verifbs/verifbs.go @@ -1,7 +1,7 @@ package verifbs import ( - "github.com/ipfs/go-ipfs/thirdparty/verifcid" + "github.com/ipfs/go-verifcid" bstore "gx/ipfs/QmaG4DZ4JaqEfvPWt5nPPgoTzhc1tr1T3f4Nu9Jpdm8ymY/go-ipfs-blockstore" cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" diff --git a/thirdparty/verifcid/validate.go b/thirdparty/verifcid/validate.go deleted file mode 100644 index 4af24b4c43a..00000000000 --- a/thirdparty/verifcid/validate.go +++ /dev/null @@ -1,62 +0,0 @@ -package verifcid - -import ( - "fmt" - - mh "gx/ipfs/QmZyZDi491cCNTLfAhwcaDii2Kg4pwKRkhqQzURGDvY6ua/go-multihash" - cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" -) - -var ErrPossiblyInsecureHashFunction = fmt.Errorf("potentially insecure hash functions not allowed") -var ErrBelowMinimumHashLength = fmt.Errorf("hashes must be at %d least bytes long", minimumHashLength) - -const minimumHashLength = 20 - -var goodset = map[uint64]bool{ - mh.SHA2_256: true, - mh.SHA2_512: true, - mh.SHA3_224: true, - mh.SHA3_256: true, - mh.SHA3_384: true, - mh.SHA3_512: true, - mh.SHAKE_256: true, - mh.DBL_SHA2_256: true, - mh.KECCAK_224: true, - mh.KECCAK_256: true, - mh.KECCAK_384: true, - mh.KECCAK_512: true, - mh.ID: true, - - mh.SHA1: true, // not really secure but still useful -} - -func IsGoodHash(code uint64) bool { - good, found := goodset[code] - if good { - return true - } - - if !found { - if code >= mh.BLAKE2B_MIN+19 && code <= mh.BLAKE2B_MAX { - return true - } - if code >= mh.BLAKE2S_MIN+19 && code <= mh.BLAKE2S_MAX { - return true - } - } - - return false -} - -func ValidateCid(c *cid.Cid) error { - pref := c.Prefix() - if !IsGoodHash(pref.MhType) { - return ErrPossiblyInsecureHashFunction - } - - if pref.MhType != mh.ID && pref.MhLength < minimumHashLength { - return ErrBelowMinimumHashLength - } - - return nil -} diff --git a/thirdparty/verifcid/validate_test.go b/thirdparty/verifcid/validate_test.go deleted file mode 100644 index 21ab0902154..00000000000 --- a/thirdparty/verifcid/validate_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package verifcid - -import ( - "testing" - - mh "gx/ipfs/QmZyZDi491cCNTLfAhwcaDii2Kg4pwKRkhqQzURGDvY6ua/go-multihash" - - cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" -) - -func TestValidateCids(t *testing.T) { - assertTrue := func(v bool) { - t.Helper() - if !v { - t.Fatal("expected success") - } - } - assertFalse := func(v bool) { - t.Helper() - if v { - t.Fatal("expected failure") - } - } - - assertTrue(IsGoodHash(mh.SHA2_256)) - assertTrue(IsGoodHash(mh.BLAKE2B_MIN + 32)) - assertTrue(IsGoodHash(mh.DBL_SHA2_256)) - assertTrue(IsGoodHash(mh.KECCAK_256)) - assertTrue(IsGoodHash(mh.SHA3)) - - assertTrue(IsGoodHash(mh.SHA1)) - - assertFalse(IsGoodHash(mh.BLAKE2B_MIN + 5)) - - mhcid := func(code uint64, length int) *cid.Cid { - mhash, err := mh.Sum([]byte{}, code, length) - if err != nil { - t.Fatal(err) - } - return cid.NewCidV1(cid.DagCBOR, mhash) - } - - cases := []struct { - cid *cid.Cid - err error - }{ - {mhcid(mh.SHA2_256, 32), nil}, - {mhcid(mh.SHA2_256, 16), ErrBelowMinimumHashLength}, - {mhcid(mh.MURMUR3, 4), ErrPossiblyInsecureHashFunction}, - } - - for i, cas := range cases { - if ValidateCid(cas.cid) != cas.err { - t.Errorf("wrong result in case of %s (index %d). Expected: %s, got %s", - cas.cid, i, cas.err, ValidateCid(cas.cid)) - } - } - -}