Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Support for listing only cached links in refs command #2446

Closed
wants to merge 4 commits into from
Closed

Support for listing only cached links in refs command #2446

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
de84da9
Support for listing only cached links in refs command
sivachandran Mar 5, 2016
affc15e
Added final period in usage instruction.
sivachandran Mar 8, 2016
eee9549
Now offline BlockService is used if listing only cached refs.
sivachandran Mar 11, 2016
60bc015
Uses DAG node get result to determine link cached status
sivachandran Apr 15, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
42 changes: 34 additions & 8 deletions core/commands/refs.go
View file
Open in desktop
Expand Up @@ -5,11 +5,14 @@ import (
"errors"
"fmt"
"io"
"strconv"
"strings"

key "github.com/ipfs/go-ipfs/blocks/key"
bserv "github.com/ipfs/go-ipfs/blockservice"
cmds "github.com/ipfs/go-ipfs/commands"
"github.com/ipfs/go-ipfs/core"
offline "github.com/ipfs/go-ipfs/exchange/offline"
dag "github.com/ipfs/go-ipfs/merkledag"
path "github.com/ipfs/go-ipfs/path"
u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
Expand Down Expand Up @@ -49,10 +52,11 @@ Note: List all references recursively by using the flag '-r'.
cmds.StringArg("ipfs-path", true, true, "Path to the object(s) to list refs from.").EnableStdin(),
},
Options: []cmds.Option{
cmds.StringOption("format", "Emit edges with given format. Available tokens: <src> <dst> <linkname>."),
cmds.StringOption("format", "Emit edges with given format. Available tokens: <src> <dst> <linkname> <linksize> <linkcached>"),
cmds.BoolOption("edges", "e", "Emit edge format: `<from> -> <to>`."),
cmds.BoolOption("unique", "u", "Omit duplicate refs from output."),
cmds.BoolOption("recursive", "r", "Recursively list links of child nodes."),
cmds.BoolOption("cached", "List only links of child nodes cached locally."),
},
Run: func(req cmds.Request, res cmds.Response) {
ctx := req.Context()
Expand Down Expand Up @@ -86,6 +90,17 @@ Note: List all references recursively by using the flag '-r'.
return
}

cached, _, err := req.Option("cached").Bool()
if err != nil {
res.SetError(err, cmds.ErrNormal)
return
}

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

here, i would do something like:

dserv := n.DAG
if cached {
    dserv = makeOfflineDatastore(n.Blockstore)
}

and then for makeOfflineDatastore, do something like we do in merkledag/test/util.go to create a dagservice with only a blockstore and an offline exchange.

Then, pass that dserv to the RefWriter instead of n.DAG

That way we avoid having split conditionals further down the codepath.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@whyrusleeping incorporated your suggestion.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

now that you have that, you don't need any of the Cached logic in the RefWriter

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

or at least, you might need less of it. In this case since you want a full listing of all local refs, you would just use the Cached flag to determine whether or not to error out on a dag.ErrNotFound type error from the dagservice Get method.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@whyrusleeping I am not sure how can I do without the Cached logic. The NodeGetter.Get hangs when I use with BlockService with offline exchange and the ref is not cached locally. Looks like I've to change quite a lot if I want to make nodePromise(i.e. NodeGetter) to respond even if the ref is not locally available.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh wow. thank you for reminding me that I hadnt merged #2257 yet.

That PR will make all this much easier

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see that the PR is still having pending comments. When do you think you can merge the PR #2257?

ds := n.DAG
if cached {
ds = dag.NewDAGService(bserv.New(n.Blockstore, offline.Exchange(n.Blockstore)))
}

objs, err := objectsForPaths(ctx, n, req.Arguments())
if err != nil {
res.SetError(err, cmds.ErrNormal)
Expand All @@ -100,12 +115,13 @@ Note: List all references recursively by using the flag '-r'.

rw := RefWriter{
out: out,
DAG: n.DAG,
DAG: ds,
Ctx: ctx,
Unique: unique,
PrintEdge: edges,
PrintFmt: format,
Recursive: recursive,
Cached: cached,
}

for _, o := range objs {
Expand Down Expand Up @@ -212,6 +228,7 @@ type RefWriter struct {

Unique bool
Recursive bool
Cached bool
PrintEdge bool
PrintFmt string

Expand Down Expand Up @@ -239,13 +256,19 @@ func (rw *RefWriter) writeRefsRecursive(n *dag.Node) (int, error) {
continue
}

if err := rw.WriteEdge(nkey, lk, n.Links[i].Name); err != nil {
return count, err
nd, err := ng.Get(rw.Ctx)

linkCached := (nd != nil)
if werr := rw.WriteEdge(nkey, lk, n.Links[i].Name, n.Links[i].Size, linkCached); werr != nil {
return count, werr
}

nd, err := ng.Get(rw.Ctx)
if err != nil {
return count, err
if rw.Cached {
continue
} else {
return count, err
}
}

c, err := rw.writeRefsRecursive(nd)
Expand All @@ -254,6 +277,7 @@ func (rw *RefWriter) writeRefsRecursive(n *dag.Node) (int, error) {
return count, err
}
}

return count, nil
}

Expand All @@ -275,7 +299,7 @@ func (rw *RefWriter) writeRefsSingle(n *dag.Node) (int, error) {
continue
}

if err := rw.WriteEdge(nkey, lk, l.Name); err != nil {
if err := rw.WriteEdge(nkey, lk, l.Name, l.Size, true); err != nil {
return count, err
}
count++
Expand All @@ -301,7 +325,7 @@ func (rw *RefWriter) skip(k key.Key) bool {
}

// Write one edge
func (rw *RefWriter) WriteEdge(from, to key.Key, linkname string) error {
func (rw *RefWriter) WriteEdge(from, to key.Key, linkname string, linksize uint64, linkcached bool) error {
if rw.Ctx != nil {
select {
case <-rw.Ctx.Done(): // just in case.
Expand All @@ -317,6 +341,8 @@ func (rw *RefWriter) WriteEdge(from, to key.Key, linkname string) error {
s = strings.Replace(s, "<src>", from.B58String(), -1)
s = strings.Replace(s, "<dst>", to.B58String(), -1)
s = strings.Replace(s, "<linkname>", linkname, -1)
s = strings.Replace(s, "<linksize>", strconv.FormatUint(linksize, 10), -1)
s = strings.Replace(s, "<linkcached>", strconv.FormatBool(linkcached), -1)
case rw.PrintEdge:
s = from.B58String() + " -> " + to.B58String()
default:
Expand Down