IPLD Prime In IPFS: Target Merge Branch

.circleci/config.yml

@@ -229,7 +229,7 @@ jobs:
       - run:
           name: Cloning
           command: |
-            git clone https://github.com/ipfs/go-ipfs-api.git
+            git clone -b ipld-in-ipfs https://github.com/ipfs/go-ipfs-api.git
             git -C go-ipfs-api log -1
       - run:
           name: Starting the daemon

core/commands/dag/dag.go

@@ -77,10 +77,10 @@ into an object of the specified format.
 		cmds.FileArg("object data", true, true, "The object to put").EnableStdin(),
 	},
 	Options: []cmds.Option{
-		cmds.StringOption("format", "f", "Format that the object will be added as.").WithDefault("cbor"),
-		cmds.StringOption("input-enc", "Format that the input object will be.").WithDefault("json"),
+		cmds.StringOption("format", "f", "Format that the object will be added as.").WithDefault("dag-cbor"),
+		cmds.StringOption("input-enc", "Format that the input object will be.").WithDefault("dag-json"),
 		cmds.BoolOption("pin", "Pin this object when adding."),
-		cmds.StringOption("hash", "Hash function to use").WithDefault(""),
+		cmds.StringOption("hash", "Hash function to use").WithDefault("sha2-256"),
 	},
 	Run:  dagPut,
 	Type: OutputObject{},
@@ -108,6 +108,9 @@ format.
 	Arguments: []cmds.Argument{
 		cmds.StringArg("ref", true, false, "The object to get").EnableStdin(),
 	},
+	Options: []cmds.Option{
+		cmds.StringOption("format", "f", "Format that the object will be serialized as.").WithDefault("dag-json"),
+	},
 	Run: dagGet,
 }
 

core/commands/dag/get.go

@@ -1,11 +1,18 @@
 package dagcmd
 
 import (
-	"strings"
+	"fmt"
+	"io"
 
 	"github.com/ipfs/go-ipfs/core/commands/cmdenv"
+	ipldlegacy "github.com/ipfs/go-ipld-legacy"
 	"github.com/ipfs/interface-go-ipfs-core/path"
 
+	"github.com/ipld/go-ipld-prime"
+	"github.com/ipld/go-ipld-prime/multicodec"
+	"github.com/ipld/go-ipld-prime/traversal"
+	mc "github.com/multiformats/go-multicodec"
+
 	cmds "github.com/ipfs/go-ipfs-cmds"
 )
 
@@ -15,6 +22,12 @@ func dagGet(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) e
 		return err
 	}
 
+	format, _ := req.Options["format"].(string)
+	var fCodec mc.Code
+	if err := fCodec.Set(format); err != nil {
+		return err
+	}
+
 	rp, err := api.ResolvePath(req.Context, path.New(req.Arguments[0]))
 	if err != nil {
 		return err
@@ -25,14 +38,34 @@ func dagGet(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) e
 		return err
 	}
 
-	var out interface{} = obj
+	universal, ok := obj.(ipldlegacy.UniversalNode)
+	if !ok {
+		return fmt.Errorf("%T is not a valid IPLD node", obj)
+	}
+
+	finalNode := universal.(ipld.Node)
+
 	if len(rp.Remainder()) > 0 {
-		rem := strings.Split(rp.Remainder(), "/")
-		final, _, err := obj.Resolve(rem)
+		remainderPath := ipld.ParsePath(rp.Remainder())
+
+		finalNode, err = traversal.Get(finalNode, remainderPath)
 		if err != nil {
 			return err
 		}
-		out = final
 	}
-	return cmds.EmitOnce(res, &out)
+
+	encoder, err := multicodec.LookupEncoder(uint64(fCodec))
+	if err != nil {
+		return fmt.Errorf("invalid encoding: %s - %s", format, err)
+	}
+
+	r, w := io.Pipe()
+	go func() {
+		defer w.Close()
+		if err := encoder(finalNode, w); err != nil {
+			_ = res.CloseWithError(err)
+		}
+	}()
+
+	return res.Emit(r)
 }

core/commands/dag/put.go

@@ -1,16 +1,28 @@
 package dagcmd
 
 import (
+	"bytes"
 	"fmt"
-	"math"
 
+	blocks "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-ipfs/core/commands/cmdenv"
-	"github.com/ipfs/go-ipfs/core/coredag"
+	ipldlegacy "github.com/ipfs/go-ipld-legacy"
+	"github.com/ipld/go-ipld-prime/multicodec"
+	basicnode "github.com/ipld/go-ipld-prime/node/basic"
 
 	cmds "github.com/ipfs/go-ipfs-cmds"
 	files "github.com/ipfs/go-ipfs-files"
 	ipld "github.com/ipfs/go-ipld-format"
-	mh "github.com/multiformats/go-multihash"
+	mc "github.com/multiformats/go-multicodec"
+
+	// Expected minimal set of available format/ienc codecs.
+	_ "github.com/ipld/go-codec-dagpb"
+	_ "github.com/ipld/go-ipld-prime/codec/cbor"
+	_ "github.com/ipld/go-ipld-prime/codec/dagcbor"
+	_ "github.com/ipld/go-ipld-prime/codec/dagjson"
+	_ "github.com/ipld/go-ipld-prime/codec/json"
+	_ "github.com/ipld/go-ipld-prime/codec/raw"
 )
 
 func dagPut(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
@@ -24,16 +36,33 @@ func dagPut(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) e
 	hash, _ := req.Options["hash"].(string)
 	dopin, _ := req.Options["pin"].(bool)
 
-	// mhType tells inputParser which hash should be used. MaxUint64 means 'use
-	// default hash' (sha256 for cbor, sha1 for git..)
-	mhType := uint64(math.MaxUint64)
+	var icodec mc.Code
+	if err := icodec.Set(ienc); err != nil {
+		return err
+	}
+	var fcodec mc.Code
+	if err := fcodec.Set(format); err != nil {
+		return err
+	}
+	var mhType mc.Code
+	if err := mhType.Set(hash); err != nil {
+		return err
+	}
+
+	cidPrefix := cid.Prefix{
+		Version:  1,
+		Codec:    uint64(fcodec),
+		MhType:   uint64(mhType),
+		MhLength: -1,
+	}
 
-	if hash != "" {
-		var ok bool
-		mhType, ok = mh.Names[hash]
-		if !ok {
-			return fmt.Errorf("%s in not a valid multihash name", hash)
-		}
+	decoder, err := multicodec.LookupDecoder(uint64(icodec))
+	if err != nil {
+		return err
+	}
+	encoder, err := multicodec.LookupEncoder(uint64(fcodec))
+	if err != nil {
+		return err
 	}
 
 	var adder ipld.NodeAdder = api.Dag()
@@ -48,22 +77,44 @@ func dagPut(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) e
 		if file == nil {
 			return fmt.Errorf("expected a regular file")
 		}
-		nds, err := coredag.ParseInputs(ienc, format, file, mhType, -1)
+
+		node := basicnode.Prototype.Any.NewBuilder()
+		if err := decoder(node, file); err != nil {
+			return err
+		}
+		n := node.Build()
+
+		bd := bytes.NewBuffer([]byte{})
+		if err := encoder(n, bd); err != nil {
+			return err
+		}
+
+		blockCid, err := cidPrefix.Sum(bd.Bytes())
+		if err != nil {
+			return err
+		}
+		blk, err := blocks.NewBlockWithCid(bd.Bytes(), blockCid)
 		if err != nil {
 			return err
 		}
-		if len(nds) == 0 {
-			return fmt.Errorf("no node returned from ParseInputs")
+		ln := ipldlegacy.LegacyNode{
+			Block: blk,
+			Node:  n,
 		}
 
-		for _, nd := range nds {
-			err := b.Add(req.Context, nd)
-			if err != nil {
-				return err
-			}
+		if err := b.Add(req.Context, &ln); err != nil {
+			return err
 		}
+		/*
+			for _, nd := range nds {
+				err := b.Add(req.Context, nd)
+				if err != nil {
+					return err
+				}
+			}
+		*/
 
-		cid := nds[0].Cid()
+		cid := ln.Cid()
 		if err := res.Emit(&OutputObject{Cid: cid}); err != nil {
 			return err
 		}

core/core.go

@@ -17,6 +17,7 @@ import (
 	"github.com/ipfs/go-ipfs-pinner"
 
 	bserv "github.com/ipfs/go-blockservice"
+	"github.com/ipfs/go-fetcher"
 	"github.com/ipfs/go-graphsync"
 	bstore "github.com/ipfs/go-ipfs-blockstore"
 	exchange "github.com/ipfs/go-ipfs-exchange-interface"
@@ -77,6 +78,7 @@ type IpfsNode struct {
 	Blocks          bserv.BlockService        // the block service, get/add blocks.
 	DAG             ipld.DAGService           // the merkle dag service, get/add objects.
 	Resolver        *resolver.Resolver        // the path resolution system
+	FetcherFactory  fetcher.Factory           // an implementation of the fetcher
 	Reporter        *metrics.BandwidthCounter `optional:"true"`
 	Discovery       discovery.Service         `optional:"true"`
 	FilesRoot       *mfs.Root

core/coreapi/coreapi.go

@@ -19,14 +19,18 @@ import (
 	"fmt"
 
 	bserv "github.com/ipfs/go-blockservice"
-	"github.com/ipfs/go-ipfs-blockstore"
-	"github.com/ipfs/go-ipfs-exchange-interface"
+	"github.com/ipfs/go-dagwriter"
+	bsdagwriter "github.com/ipfs/go-dagwriter/impl/blockservice"
+	"github.com/ipfs/go-fetcher"
+	blockstore "github.com/ipfs/go-ipfs-blockstore"
+	exchange "github.com/ipfs/go-ipfs-exchange-interface"
 	offlinexch "github.com/ipfs/go-ipfs-exchange-offline"
-	"github.com/ipfs/go-ipfs-pinner"
-	"github.com/ipfs/go-ipfs-provider"
+	pin "github.com/ipfs/go-ipfs-pinner"
+	provider "github.com/ipfs/go-ipfs-provider"
 	offlineroute "github.com/ipfs/go-ipfs-routing/offline"
 	ipld "github.com/ipfs/go-ipld-format"
 	dag "github.com/ipfs/go-merkledag"
+	"github.com/ipfs/go-path/resolver"
 	coreiface "github.com/ipfs/interface-go-ipfs-core"
 	"github.com/ipfs/interface-go-ipfs-core/options"
 	ci "github.com/libp2p/go-libp2p-core/crypto"
@@ -54,9 +58,11 @@ type CoreAPI struct {
 	baseBlocks blockstore.Blockstore
 	pinning    pin.Pinner
 
-	blocks bserv.BlockService
-	dag    ipld.DAGService
-
+	blocks          bserv.BlockService
+	dag             ipld.DAGService
+	fetcherFactory  fetcher.Factory
+	dagWriter       dagwriter.DagWritingService
+	resolver        *resolver.Resolver
 	peerstore       pstore.Peerstore
 	peerHost        p2phost.Host
 	recordValidator record.Validator
@@ -105,6 +111,19 @@ func (api *CoreAPI) Dag() coreiface.APIDagService {
 	}
 }
 
+type nodeAPI struct {
+	fetcher.Factory
+	dagwriter.DagWritingService
+}
+
+// Node returns the Node interface implementation backed by the go-ipfs node
+func (api *CoreAPI) Node() coreiface.NodeAPI {
+	return &nodeAPI{
+		api.fetcherFactory,
+		api.dagWriter,
+	}
+}
+
 // Name returns the NameAPI interface implementation backed by the go-ipfs node
 func (api *CoreAPI) Name() coreiface.NameAPI {
 	return (*NameAPI)(api)
@@ -165,8 +184,10 @@ func (api *CoreAPI) WithOptions(opts ...options.ApiOption) (coreiface.CoreAPI, e
 		baseBlocks: n.BaseBlocks,
 		pinning:    n.Pinning,
 
-		blocks: n.Blocks,
-		dag:    n.DAG,
+		blocks:         n.Blocks,
+		dag:            n.DAG,
+		resolver:       n.Resolver,
+		fetcherFactory: n.FetcherFactory,
 
 		peerstore:       n.Peerstore,
 		peerHost:        n.PeerHost,
@@ -183,6 +204,8 @@ func (api *CoreAPI) WithOptions(opts ...options.ApiOption) (coreiface.CoreAPI, e
 		parentOpts: settings,
 	}
 
+	subApi.dagWriter = bsdagwriter.NewDagWriter(subApi.blocks)
+
 	subApi.checkOnline = func(allowOffline bool) error {
 		if !n.IsOnline && !allowOffline {
 			return coreiface.ErrOffline

core/coreapi/path.go

@@ -10,8 +10,6 @@ import (
 	"github.com/ipfs/go-cid"
 	ipld "github.com/ipfs/go-ipld-format"
 	ipfspath "github.com/ipfs/go-path"
-	"github.com/ipfs/go-path/resolver"
-	uio "github.com/ipfs/go-unixfs/io"
 	coreiface "github.com/ipfs/interface-go-ipfs-core"
 	path "github.com/ipfs/interface-go-ipfs-core/path"
 )
@@ -49,23 +47,11 @@ func (api *CoreAPI) ResolvePath(ctx context.Context, p path.Path) (path.Resolved
 		return nil, err
 	}
 
-	var resolveOnce resolver.ResolveOnce
-
-	switch ipath.Segments()[0] {
-	case "ipfs":
-		resolveOnce = uio.ResolveUnixfsOnce
-	case "ipld":
-		resolveOnce = resolver.ResolveSingle
-	default:
+	if ipath.Segments()[0] != "ipfs" && ipath.Segments()[0] != "ipld" {
 		return nil, fmt.Errorf("unsupported path namespace: %s", p.Namespace())
 	}
 
-	r := &resolver.Resolver{
-		DAG:         api.dag,
-		ResolveOnce: resolveOnce,
-	}
-
-	node, rest, err := r.ResolveToLastNode(ctx, ipath)
+	node, rest, err := api.resolver.ResolveToLastNode(ctx, ipath)
 	if err != nil {
 		return nil, err
 	}