From e23dbd291be024b369c43709357d85b51c14ddd4 Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Sat, 13 May 2017 00:35:22 -0400 Subject: [PATCH 1/2] Add support for using an alternative hash function with raw nodes. License: MIT Signed-off-by: Kevin Atkinson --- importer/helpers/dagbuilder.go | 19 +++++++++++++++---- merkledag/merkledag.go | 2 +- merkledag/raw.go | 20 ++++++++++++++++++++ 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/importer/helpers/dagbuilder.go b/importer/helpers/dagbuilder.go index a06b38c595a..de33563efb2 100644 --- a/importer/helpers/dagbuilder.go +++ b/importer/helpers/dagbuilder.go @@ -165,10 +165,21 @@ func (db *DagBuilderHelper) GetNextDataNode() (*UnixfsNode, error) { } if db.rawLeaves { - return &UnixfsNode{ - rawnode: dag.NewRawNode(data), - raw: true, - }, nil + if db.prefix == nil { + return &UnixfsNode{ + rawnode: dag.NewRawNode(data), + raw: true, + }, nil + } else { + rawnode, err := dag.NewRawNodeWPrefix(data, *db.prefix) + if err != nil { + return nil, err + } + return &UnixfsNode{ + rawnode: rawnode, + raw: true, + }, nil + } } else { blk := db.NewUnixfsBlock() blk.SetData(data) diff --git a/merkledag/merkledag.go b/merkledag/merkledag.go index e0dc5c74ae1..5509bcaebcd 100644 --- a/merkledag/merkledag.go +++ b/merkledag/merkledag.go @@ -116,7 +116,7 @@ func decodeBlock(b blocks.Block) (node.Node, error) { decnd.Prefix = b.Cid().Prefix() return decnd, nil case cid.Raw: - return NewRawNode(b.RawData()), nil + return NewRawNodeWPrefix(b.RawData(), b.Cid().Prefix()) case cid.DagCBOR: return ipldcbor.Decode(b.RawData()) default: diff --git a/merkledag/raw.go b/merkledag/raw.go index 30fe639af99..7c5ba56af07 100644 --- a/merkledag/raw.go +++ b/merkledag/raw.go @@ -12,6 +12,8 @@ type RawNode struct { blocks.Block } +// NewRawNode creates a RawNode using the default sha2-256 hash +// funcition. func NewRawNode(data []byte) *RawNode { h := u.Hash(data) c := cid.NewCidV1(cid.Raw, h) @@ -20,6 +22,24 @@ func NewRawNode(data []byte) *RawNode { return &RawNode{blk} } +// NewRawNodeWPrefix creates a RawNode with the hash function +// specified in prefix. +func NewRawNodeWPrefix(data []byte, prefix cid.Prefix) (*RawNode, error) { + prefix.Codec = cid.Raw + if prefix.Version == 0 { + prefix.Version = 1 + } + c, err := prefix.Sum(data) + if err != nil { + return nil, err + } + blk, err := blocks.NewBlockWithCid(data, c) + if err != nil { + return nil, err + } + return &RawNode{blk}, nil +} + func (rn *RawNode) Links() []*node.Link { return nil } From 4bbd00185c6c7cb42e63a1b3f04dcc0c52d6ccea Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Sat, 13 May 2017 01:51:29 -0400 Subject: [PATCH 2/2] adder: add support for specifying the hash function License: MIT Signed-off-by: Kevin Atkinson --- core/commands/add.go | 18 ++++++++++++++++++ test/sharness/t0040-add-and-cat.sh | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/core/commands/add.go b/core/commands/add.go index a16817db935..7a6b6100340 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "io" + "strings" bstore "github.com/ipfs/go-ipfs/blocks/blockstore" blockservice "github.com/ipfs/go-ipfs/blockservice" @@ -17,6 +18,7 @@ import ( mfs "github.com/ipfs/go-ipfs/mfs" ft "github.com/ipfs/go-ipfs/unixfs" + mh "gx/ipfs/QmVGtdTZdTFaLsaj2RwdVG8jcjNNcp1DE914DKZ2kHmXHw/go-multihash" u "gx/ipfs/QmWbjfz3u6HkAdPh34dgPchGbQjob6LXLhAeCGii2TX69n/go-ipfs-util" "gx/ipfs/QmeWjRodbcZFKe5tMN7poEx3izym6osrLSnTLf9UjJZBbs/pb" ) @@ -39,6 +41,7 @@ const ( noCopyOptionName = "nocopy" fstoreCacheOptionName = "fscache" cidVersionOptionName = "cid-version" + hashOptionName = "hash" ) const adderOutChanSize = 8 @@ -90,6 +93,7 @@ You can now refer to the added file in a gateway, like so: cmds.BoolOption(noCopyOptionName, "Add the file using filestore. (experimental)"), cmds.BoolOption(fstoreCacheOptionName, "Check the filestore for pre-existing blocks. (experimental)"), cmds.IntOption(cidVersionOptionName, "Cid version. Non-zero value will change default of 'raw-leaves' to true. (experimental)").Default(0), + cmds.StringOption(hashOptionName, "Hash function to use. Will set Cid version to 1 if used. (experimental)").Default("sha2-256"), }, PreRun: func(req cmds.Request) error { quiet, _, _ := req.Option(quietOptionName).Bool() @@ -164,6 +168,7 @@ You can now refer to the added file in a gateway, like so: nocopy, _, _ := req.Option(noCopyOptionName).Bool() fscache, _, _ := req.Option(fstoreCacheOptionName).Bool() cidVer, _, _ := req.Option(cidVersionOptionName).Int() + hashFunStr, hfset, _ := req.Option(hashOptionName).String() if nocopy && !cfg.Experimental.FilestoreEnabled { res.SetError(errors.New("filestore is not enabled, see https://git.io/vy4XN"), @@ -180,6 +185,10 @@ You can now refer to the added file in a gateway, like so: return } + if hfset && cidVer == 0 { + cidVer = 1 + } + if cidVer >= 1 && !rbset { rawblks = true } @@ -190,6 +199,15 @@ You can now refer to the added file in a gateway, like so: return } + hashFunCode, ok := mh.Names[strings.ToLower(hashFunStr)] + if !ok { + res.SetError(fmt.Errorf("unrecognized hash function: %s", strings.ToLower(hashFunStr)), cmds.ErrNormal) + return + } + + prefix.MhType = hashFunCode + prefix.MhLength = -1 + if hash { nilnode, err := core.NewNode(n.Context(), &core.BuildCfg{ //TODO: need this to be true or all files diff --git a/test/sharness/t0040-add-and-cat.sh b/test/sharness/t0040-add-and-cat.sh index 4b2d636a480..52a3833dd31 100755 --- a/test/sharness/t0040-add-and-cat.sh +++ b/test/sharness/t0040-add-and-cat.sh @@ -403,6 +403,10 @@ MARS="zdj7WWx6fGNrNGkdpkuTAxCjKbQ1pPtarqA6VQhedhLTZu34J" VENUS="zdj7WbB1BUF8WejmVpQCmMLd1RbPnxJtvAj1Lep6eTmXRFbrz" add_directory '--cid-version=1 --raw-leaves=false' +PLANETS="zDMZof1kqxDAx9myQbXsyWwyWP9qRPsXsWH7XuTz6isT7Rh1S6nM" +MARS="zCT5htkdz1ZBHYVQXFQn51ngPXLVqaHSWoae87V1d6e9qWpSAjXw" +VENUS="zCT5htke5JcdoMM4WhmUKXWf2QC3TnQToqGZHH1WsZERv6kPhFPg" +add_directory '--hash=blake2b-256' test_expect_success "'ipfs add -rn' succeeds" ' mkdir -p mountdir/moons/jupiter && @@ -446,12 +450,26 @@ test_add_cat_5MB --cid-version=1 "zdj7WiiaedqVBXjX4SNqx3jfuZideDqdLYnDzCDJ66JDMK # format (i.e. not raw) test_add_cat_5MB '--cid-version=1 --raw-leaves=false' "zdj7WfgEsj897BBZj2mcfsRLhaPZcCixPV2G7DkWgF1Wdr64P" +# note: --hash=blake2b-256 implies --cid-version=1 which implies --raw-leaves=true +# the specified hash represents the leaf nodes stored as raw leaves and +# encoded with the blake2b-256 hash funtion +test_add_cat_5MB '--hash=blake2b-256' "zDMZof1kuxn7ebvKyvmkYLPvocSvFYxxAWT1yQBN1wWiXXr7w5mY" + +# the specified hash represents the leaf nodes stored as protoful nodes and +# encoded with the blake2b-256 hash funtion +test_add_cat_5MB '--hash=blake2b-256 --raw-leaves=false' "zDMZof1krz3SFTyhboRyWZyUP2qNgVdn9wjtaX211aHJ8WgeyT9v" + test_add_cat_expensive "" "QmU9SWAPPmNEKZB8umYMmjYvN7VyHqABNvdA6GUi4MMEz3" # note: the specified hash implies that internal nodes are stored # using CidV1 and leaves are stored using raw blocks test_add_cat_expensive "--cid-version=1" "zdj7WcatQrtuE4WMkS4XsfsMixuQN2po4irkYhqxeJyW1wgCq" +# note: --hash=blake2b-256 implies --cid-version=1 which implies --raw-leaves=true +# the specified hash represents the leaf nodes stored as raw leaves and +# encoded with the blake2b-256 hash funtion +test_add_cat_expensive '--hash=blake2b-256' "zDMZof1kwndounDzQCANUHjiE3zt1mPEgx7RE3JTHoZrRRa79xcv" + test_add_named_pipe " Post http://$API_ADDR/api/v0/add?encoding=json&progress=true&r=true&stream-channels=true:" test_add_pwd_is_symlink