Skip to content

Commit

Permalink
fix and improve the writable gateway
Browse files Browse the repository at this point in the history 
1. Fix a panic due to an unchecked type assertion.
2. Fix handling of PUT. The simple implementation was the correct
   implementation, I have no idea what was going on here.
3. Use MFS everywhere to reduce code duplication and add support for sharded
   directories.
4. _Correctly_ block IPNS.
5. Remove the dependency on `core.IpfsNode`.
  • Loading branch information
@Stebalien
Stebalien committed Jul 26, 2019
1 parent 876d5ba commit 91a1324
Show file tree
Hide file tree
Showing 2 changed files with 109 additions and 154 deletions.
2 changes: 1 addition & 1 deletion core/corehttp/gateway.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ func GatewayOption(writable bool, paths ...string) ServeOption {
"X-Stream-Output",
}, headers[ACEHeadersName]...))

gateway := newGatewayHandler(n, GatewayConfig{
gateway := newGatewayHandler(GatewayConfig{
Headers: headers,
Writable: writable,
PathPrefixes: cfg.Gateway.PathPrefixes,
Expand Down
261 changes: 108 additions & 153 deletions core/corehttp/gateway_handler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package corehttp

import (
"context"
"errors"
"fmt"
"io"
"net/http"
Expand All @@ -12,20 +11,13 @@ import (
"strings"
"time"

"github.com/ipfs/go-ipfs/core"
"github.com/ipfs/go-ipfs/dagutils"
"github.com/ipfs/go-ipfs/namesys/resolve"

"github.com/dustin/go-humanize"
"github.com/ipfs/go-cid"
chunker "github.com/ipfs/go-ipfs-chunker"
files "github.com/ipfs/go-ipfs-files"
ipld "github.com/ipfs/go-ipld-format"
dag "github.com/ipfs/go-merkledag"
"github.com/ipfs/go-mfs"
"github.com/ipfs/go-path"
"github.com/ipfs/go-path/resolver"
ft "github.com/ipfs/go-unixfs"
"github.com/ipfs/go-unixfs/importer"
coreiface "github.com/ipfs/interface-go-ipfs-core"
ipath "github.com/ipfs/interface-go-ipfs-core/path"
routing "github.com/libp2p/go-libp2p-core/routing"
Expand All @@ -40,27 +32,36 @@ const (
// gatewayHandler is a HTTP handler that serves IPFS objects (accessible by default at /ipfs/<path>)
// (it serves requests like GET /ipfs/QmVRzPKPzNtSrEzBFm2UZfxmPAgnaLke4DMcerbsGGSaFe/link)
type gatewayHandler struct {
node *core.IpfsNode
config GatewayConfig
api coreiface.CoreAPI
}

func newGatewayHandler(n *core.IpfsNode, c GatewayConfig, api coreiface.CoreAPI) *gatewayHandler {
func newGatewayHandler(c GatewayConfig, api coreiface.CoreAPI) *gatewayHandler {
i := &gatewayHandler{
node: n,
config: c,
api: api,
}
return i
}

// TODO(cryptix): find these helpers somewhere else
func (i *gatewayHandler) newDagFromReader(r io.Reader) (ipld.Node, error) {
// TODO(cryptix): change and remove this helper once PR1136 is merged
// return ufs.AddFromReader(i.node, r.Body)
return importer.BuildDagFromReader(
i.node.DAG,
chunker.DefaultSplitter(r))
func parseIpfsPath(p string) (cid.Cid, string, error) {
rootPath, err := path.ParsePath(p)
if err != nil {
return cid.Cid{}, "", err
}

// Check the path.
rsegs := rootPath.Segments()
if rsegs[0] != "ipfs" {
return cid.Cid{}, "", fmt.Errorf("WritableGateway: only ipfs paths supported", rsegs[0])
}

rootCid, err := cid.Decode(rsegs[1])
if err != nil {
return cid.Cid{}, "", err
}

return rootCid, path.Join(rsegs[2:]), nil
}

func (i *gatewayHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
Expand Down Expand Up @@ -160,10 +161,12 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request

// Resolve path to the final DAG node for the ETag
resolvedPath, err := i.api.ResolvePath(r.Context(), parsedPath)
if err == coreiface.ErrOffline && !i.node.IsOnline {
switch err {
case nil:
case coreiface.ErrOffline:
webError(w, "ipfs resolve -r "+escapedURLPath, err, http.StatusServiceUnavailable)
return
} else if err != nil {
default:
webError(w, "ipfs resolve -r "+escapedURLPath, err, http.StatusNotFound)
return
}
Expand Down Expand Up @@ -395,194 +398,146 @@ func (i *gatewayHandler) postHandler(w http.ResponseWriter, r *http.Request) {
}

func (i *gatewayHandler) putHandler(w http.ResponseWriter, r *http.Request) {
rootPath, err := path.ParsePath(r.URL.Path)
ctx := r.Context()
ds := i.api.Dag()

// Parse the path
rootCid, newPath, err := parseIpfsPath(r.URL.Path)
if err != nil {
webError(w, "putHandler: IPFS path not valid", err, http.StatusBadRequest)
webError(w, "WritableGateway: failed to parse the path", err, http.StatusBadRequest)
return
}

rsegs := rootPath.Segments()
if rsegs[0] == ipnsPathPrefix {
webError(w, "putHandler: updating named entries not supported", errors.New("WritableGateway: ipns put not supported"), http.StatusBadRequest)
if newPath == "" || newPath == "/" {
http.Error(w, "WritableGateway: empty path", http.StatusBadRequest)
return
}
newDirectory, _ := gopath.Split(newPath)

var newnode ipld.Node
if rsegs[len(rsegs)-1] == "QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn" {
newnode = ft.EmptyDirNode()
} else {
putNode, err := i.newDagFromReader(r.Body)
if err != nil {
webError(w, "putHandler: Could not create DAG from request", err, http.StatusInternalServerError)
return
}
newnode = putNode
}
// Resolve the old root.

var newPath string
if len(rsegs) > 1 {
newPath = path.Join(rsegs[2:])
rnode, err := ds.Get(ctx, rootCid)
if err != nil {
webError(w, "WritableGateway: Could not create DAG from request", err, http.StatusInternalServerError)
return
}

var newcid cid.Cid
rnode, err := resolve.Resolve(r.Context(), i.node.Namesys, i.node.Resolver, rootPath)
switch ev := err.(type) {
case resolver.ErrNoLink:
// ev.Node < node where resolve failed
// ev.Name < new link
// but we need to patch from the root
c, err := cid.Decode(rsegs[1])
if err != nil {
webError(w, "putHandler: bad input path", err, http.StatusBadRequest)
return
}

rnode, err := i.node.DAG.Get(r.Context(), c)
if err != nil {
webError(w, "putHandler: Could not create DAG from request", err, http.StatusInternalServerError)
return
}

pbnd, ok := rnode.(*dag.ProtoNode)
if !ok {
webError(w, "Cannot read non protobuf nodes through gateway", dag.ErrNotProtobuf, http.StatusBadRequest)
return
}

e := dagutils.NewDagEditor(pbnd, i.node.DAG)
err = e.InsertNodeAtPath(r.Context(), newPath, newnode, ft.EmptyDirNode)
if err != nil {
webError(w, "putHandler: InsertNodeAtPath failed", err, http.StatusInternalServerError)
return
}

nnode, err := e.Finalize(r.Context(), i.node.DAG)
if err != nil {
webError(w, "putHandler: could not get node", err, http.StatusInternalServerError)
return
}
pbnd, ok := rnode.(*dag.ProtoNode)
if !ok {
webError(w, "Cannot read non protobuf nodes through gateway", dag.ErrNotProtobuf, http.StatusBadRequest)
return
}

newcid = nnode.Cid()
// Create the new file.
newFilePath, err := i.api.Unixfs().Add(ctx, files.NewReaderFile(r.Body))
if err != nil {
webError(w, "WritableGateway: could not create DAG from request", err, http.StatusInternalServerError)
return
}

case nil:
pbnd, ok := rnode.(*dag.ProtoNode)
if !ok {
webError(w, "Cannot read non protobuf nodes through gateway", dag.ErrNotProtobuf, http.StatusBadRequest)
return
}
newFile, err := ds.Get(ctx, newFilePath.Cid())
if err != nil {
webError(w, "WritableGateway: failed to resolve new file", err, http.StatusInternalServerError)
return
}

pbnewnode, ok := newnode.(*dag.ProtoNode)
if !ok {
webError(w, "Cannot read non protobuf nodes through gateway", dag.ErrNotProtobuf, http.StatusBadRequest)
return
}
// Patch the new file into the old root.

// object set-data case
pbnd.SetData(pbnewnode.Data())
root, err := mfs.NewRoot(ctx, ds, pbnd, nil)
if err != nil {
webError(w, "WritableGateway: failed to create MFS root", err, http.StatusBadRequest)
return
}

newcid = pbnd.Cid()
err = i.node.DAG.Add(r.Context(), pbnd)
if newDirectory != "" {
err := mfs.Mkdir(root, newDirectory, mfs.MkdirOpts{Mkparents: true, Flush: false})
if err != nil {
nnk := newnode.Cid()
webError(w, fmt.Sprintf("putHandler: Could not add newnode(%q) to root(%q)", nnk.String(), newcid.String()), err, http.StatusInternalServerError)
webError(w, "WritableGateway: failed to create MFS directory", err, http.StatusInternalServerError)
return
}
default:
webError(w, "could not resolve root DAG", ev, http.StatusInternalServerError)
return
}
err = mfs.PutNode(root, newPath, newFile)
if err != nil {
webError(w, "WritableGateway: failed to link file into directory", err, http.StatusInternalServerError)
}
nnode, err := root.GetDirectory().GetNode()
if err != nil {
webError(w, "WritableGateway: failed to finalize", err, http.StatusInternalServerError)
}
newcid := nnode.Cid()

i.addUserHeaders(w) // ok, _now_ write user's headers.
w.Header().Set("IPFS-Hash", newcid.String())
http.Redirect(w, r, gopath.Join(ipfsPathPrefix, newcid.String(), newPath), http.StatusCreated)
}

func (i *gatewayHandler) deleteHandler(w http.ResponseWriter, r *http.Request) {
urlPath := r.URL.Path
ctx := r.Context()

// parse the path

p, err := path.ParsePath(urlPath)
rootCid, newPath, err := parseIpfsPath(r.URL.Path)
if err != nil {
webError(w, "failed to parse path", err, http.StatusBadRequest)
webError(w, "WritableGateway: failed to parse the path", err, http.StatusBadRequest)
return
}

c, components, err := path.SplitAbsPath(p)
if err != nil {
webError(w, "Could not split path", err, http.StatusInternalServerError)
if newPath == "" || newPath == "/" {
http.Error(w, "WritableGateway: empty path", http.StatusBadRequest)
return
}
directory, filename := gopath.Split(newPath)

// lookup the root

pathNodes, err := i.resolvePathComponents(r.Context(), c, components)
rootNodeIPLD, err := i.api.Dag().Get(ctx, rootCid)
if err != nil {
webError(w, "Could not resolve path components", err, http.StatusBadRequest)
webError(w, "WritableGateway: failed to resolve root CID", err, http.StatusInternalServerError)
return
}

pbnd, ok := pathNodes[len(pathNodes)-1].(*dag.ProtoNode)
rootNode, ok := rootNodeIPLD.(*dag.ProtoNode)
if !ok {
webError(w, "Cannot read non protobuf nodes through gateway", dag.ErrNotProtobuf, http.StatusBadRequest)
http.Error(w, "WritableGateway: empty path", http.StatusInternalServerError)
return
}

// TODO(cyrptix): assumes len(pathNodes) > 1 - not found is an error above?
err = pbnd.RemoveNodeLink(components[len(components)-1])
// construct the mfs root

root, err := mfs.NewRoot(ctx, i.api.Dag(), rootNode, nil)
if err != nil {
webError(w, "Could not delete link", err, http.StatusBadRequest)
webError(w, "WritableGateway: failed to construct the MFS root", err, http.StatusBadRequest)
return
}

var newnode *dag.ProtoNode = pbnd
for j := len(pathNodes) - 2; j >= 0; j-- {
if err := i.node.DAG.Add(r.Context(), newnode); err != nil {
webError(w, "Could not add node", err, http.StatusInternalServerError)
return
}

pathpb, ok := pathNodes[j].(*dag.ProtoNode)
if !ok {
webError(w, "Cannot read non protobuf nodes through gateway", dag.ErrNotProtobuf, http.StatusBadRequest)
return
}
// lookup the parent directory

newnode, err = pathpb.UpdateNodeLink(components[j], newnode)
if err != nil {
webError(w, "Could not update node links", err, http.StatusInternalServerError)
return
}
parentNode, err := mfs.Lookup(root, directory)
if err != nil {
webError(w, "WritableGateway: failed to look up parent", err, http.StatusInternalServerError)
return
}

if err := i.node.DAG.Add(r.Context(), newnode); err != nil {
webError(w, "Could not add root node", err, http.StatusInternalServerError)
parent, ok := parentNode.(*mfs.Directory)
if !ok {
http.Error(w, "WritableGateway: parent is not a directory", http.StatusInternalServerError)
return
}

// Redirect to new path
ncid := newnode.Cid()

i.addUserHeaders(w) // ok, _now_ write user's headers.
w.Header().Set("IPFS-Hash", ncid.String())
http.Redirect(w, r, gopath.Join(ipfsPathPrefix+ncid.String(), path.Join(components[:len(components)-1])), http.StatusCreated)
}

func (i *gatewayHandler) resolvePathComponents(
ctx context.Context,
c cid.Cid,
components []string,
) ([]ipld.Node, error) {
tctx, cancel := context.WithTimeout(ctx, time.Minute)
defer cancel()
// delete the file

rootnd, err := i.node.Resolver.DAG.Get(tctx, c)
err = parent.Unlink(filename)
if err != nil {
return nil, fmt.Errorf("Could not resolve root object: %s", err)
webError(w, "WritableGateway: failed to remove file", err, http.StatusInternalServerError)
return
}

pathNodes, err := i.node.Resolver.ResolveLinks(tctx, rootnd, components[:len(components)-1])
nnode, err := root.GetDirectory().GetNode()
if err != nil {
return nil, fmt.Errorf("Could not resolve parent object: %s", err)
webError(w, "WritableGateway: failed to finalize", err, http.StatusInternalServerError)
}
ncid := nnode.Cid()

return pathNodes, nil
i.addUserHeaders(w) // ok, _now_ write user's headers.
w.Header().Set("IPFS-Hash", ncid.String())
http.Redirect(w, r, gopath.Join(ipfsPathPrefix+ncid.String(), directory), http.StatusCreated)
}

func (i *gatewayHandler) addUserHeaders(w http.ResponseWriter) {
Expand Down

0 comments on commit 91a1324

Please sign in to comment.