From 5f26f3ca2a8f1ed8fd579c2e0d17670ef9b7ae9e Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Fri, 1 Mar 2019 13:11:08 -0500 Subject: [PATCH 1/2] Initial mount draft/WIP License: MIT Signed-off-by: Dominic Della Valle --- cmd/ipfs/Rules.mk | 7 + cmd/ipfs/daemon.go | 49 +- core/commands/{mount_unix.go => mount.go} | 88 +-- core/commands/mount/cache.go | 97 ++++ core/commands/mount/create.go | 311 +++++++++++ core/commands/mount/delete.go | 137 +++++ core/commands/mount/dirio.go | 340 ++++++++++++ core/commands/mount/event.go | 53 ++ core/commands/mount/filesystem.go | 519 ++++++++++++++++++ core/commands/mount/index.go | 343 ++++++++++++ core/commands/mount/indexNodes.go | 73 +++ core/commands/mount/interface.go | 187 +++++++ core/commands/mount/interface/interface.go | 13 + core/commands/mount/io.go | 442 +++++++++++++++ core/commands/mount/modify.go | 139 +++++ core/commands/mount/probe.go | 222 ++++++++ core/commands/mount/rootNodes.go | 33 ++ core/commands/mount/system_darwin.go | 28 + core/commands/mount/system_linux.go | 27 + core/commands/mount/system_windows.go | 86 +++ core/commands/mount/utils.go | 181 +++++++ core/commands/mount/write.go | 66 +++ core/commands/mount_nofuse.go | 2 +- core/commands/mount_windows.go | 19 - core/core.go | 25 +- core/coreapi/coreapi.go | 3 - fuse/ipns/common.go | 34 -- fuse/ipns/ipns_test.go | 486 ----------------- fuse/ipns/ipns_unix.go | 594 --------------------- fuse/ipns/link_unix.go | 28 - fuse/ipns/mount_unix.go | 26 - fuse/mount/fuse.go | 162 ------ fuse/mount/mount.go | 107 ---- fuse/node/mount_darwin.go | 243 --------- fuse/node/mount_nofuse.go | 13 - fuse/node/mount_test.go | 86 --- fuse/node/mount_unix.go | 114 ---- fuse/node/mount_windows.go | 11 - fuse/readonly/doc.go | 3 - fuse/readonly/ipfs_test.go | 294 ---------- fuse/readonly/mount_unix.go | 20 - fuse/readonly/readonly_unix.go | 296 ---------- 42 files changed, 3386 insertions(+), 2621 deletions(-) rename core/commands/{mount_unix.go => mount.go} (57%) create mode 100644 core/commands/mount/cache.go create mode 100644 core/commands/mount/create.go create mode 100644 core/commands/mount/delete.go create mode 100644 core/commands/mount/dirio.go create mode 100644 core/commands/mount/event.go create mode 100644 core/commands/mount/filesystem.go create mode 100644 core/commands/mount/index.go create mode 100644 core/commands/mount/indexNodes.go create mode 100644 core/commands/mount/interface.go create mode 100644 core/commands/mount/interface/interface.go create mode 100644 core/commands/mount/io.go create mode 100644 core/commands/mount/modify.go create mode 100644 core/commands/mount/probe.go create mode 100644 core/commands/mount/rootNodes.go create mode 100644 core/commands/mount/system_darwin.go create mode 100644 core/commands/mount/system_linux.go create mode 100644 core/commands/mount/system_windows.go create mode 100644 core/commands/mount/utils.go create mode 100644 core/commands/mount/write.go delete mode 100644 core/commands/mount_windows.go delete mode 100644 fuse/ipns/common.go delete mode 100644 fuse/ipns/ipns_test.go delete mode 100644 fuse/ipns/ipns_unix.go delete mode 100644 fuse/ipns/link_unix.go delete mode 100644 fuse/ipns/mount_unix.go delete mode 100644 fuse/mount/fuse.go delete mode 100644 fuse/mount/mount.go delete mode 100644 fuse/node/mount_darwin.go delete mode 100644 fuse/node/mount_nofuse.go delete mode 100644 fuse/node/mount_test.go delete mode 100644 fuse/node/mount_unix.go delete mode 100644 fuse/node/mount_windows.go delete mode 100644 fuse/readonly/doc.go delete mode 100644 fuse/readonly/ipfs_test.go delete mode 100644 fuse/readonly/mount_unix.go delete mode 100644 fuse/readonly/readonly_unix.go diff --git a/cmd/ipfs/Rules.mk b/cmd/ipfs/Rules.mk index d693cd86b0d..ad676981f5a 100644 --- a/cmd/ipfs/Rules.mk +++ b/cmd/ipfs/Rules.mk @@ -7,6 +7,13 @@ CLEAN += $(IPFS_BIN_$(d)) PATH := $(realpath $(d)):$(PATH) +#TODO: review; probably a more elegant way to do this; alternative is fork cgofuse, removing cgo invocations +ifeq ($(WINDOWS),1) + ifeq ($(origin CPATH), undefined) + export CGO_ENABLED = 0 + endif +endif + # disabled for now # depend on *.pb.go files in the repo as Order Only (as they shouldn't be rebuilt if exist) # DPES_OO_$(d) := diagnostics/pb/diagnostics.pb.go exchange/bitswap/message/pb/message.pb.go diff --git a/cmd/ipfs/daemon.go b/cmd/ipfs/daemon.go index e708e773ee9..6b4662b620b 100644 --- a/cmd/ipfs/daemon.go +++ b/cmd/ipfs/daemon.go @@ -8,6 +8,7 @@ import ( "net/http" _ "net/http/pprof" "os" + "path/filepath" "runtime" "sort" "sync" @@ -17,10 +18,10 @@ import ( oldcmds "github.com/ipfs/go-ipfs/commands" "github.com/ipfs/go-ipfs/core" commands "github.com/ipfs/go-ipfs/core/commands" - coreapi "github.com/ipfs/go-ipfs/core/coreapi" + mount "github.com/ipfs/go-ipfs/core/commands/mount" + "github.com/ipfs/go-ipfs/core/coreapi" corehttp "github.com/ipfs/go-ipfs/core/corehttp" corerepo "github.com/ipfs/go-ipfs/core/corerepo" - nodeMount "github.com/ipfs/go-ipfs/fuse/node" fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo" migrate "github.com/ipfs/go-ipfs/repo/fsrepo/migrations" @@ -28,8 +29,8 @@ import ( "gx/ipfs/QmTQuFQWHAWy4wMH6ZyPfGiawA5u9T8rs79FENoV8yXaoS/client_golang/prometheus" ma "gx/ipfs/QmTZBfrPJmjWsCvHEtX5FE6KimVJhsJg5sBbqEFYf4UZtL/go-multiaddr" cmds "gx/ipfs/QmX6AchyJgso1WNamTJMdxfzGiWuYu94K6tF9MJ66rRhAu/go-ipfs-cmds" - "gx/ipfs/Qmc85NSvmSG4Frn9Vb2cBc1rMyULH6D3TNVEfCzSKoUpip/go-multiaddr-net" - "gx/ipfs/Qmde5VP1qUkyQXKCfmEUA7bP64V2HAptbJ7phuPp7jXWwg/go-ipfs-cmdkit" + manet "gx/ipfs/Qmc85NSvmSG4Frn9Vb2cBc1rMyULH6D3TNVEfCzSKoUpip/go-multiaddr-net" + cmdkit "gx/ipfs/Qmde5VP1qUkyQXKCfmEUA7bP64V2HAptbJ7phuPp7jXWwg/go-ipfs-cmdkit" mprome "gx/ipfs/QmfHYMtNSntM6qFhHzLDCyqTX7NNpsfwFgvicJv7L5saAP/go-metrics-prometheus" ) @@ -39,7 +40,6 @@ const ( initOptionKwd = "init" initProfileOptionKwd = "init-profile" ipfsMountKwd = "mount-ipfs" - ipnsMountKwd = "mount-ipns" migrateKwd = "migrate" mountKwd = "mount" offlineKwd = "offline" // global option @@ -158,7 +158,6 @@ Headers. cmdkit.BoolOption(mountKwd, "Mounts IPFS to the filesystem"), cmdkit.BoolOption(writableKwd, "Enable writing objects (with POST, PUT and DELETE)"), cmdkit.StringOption(ipfsMountKwd, "Path to the mountpoint for IPFS (if using --mount). Defaults to config setting."), - cmdkit.StringOption(ipnsMountKwd, "Path to the mountpoint for IPNS (if using --mount). Defaults to config setting."), cmdkit.BoolOption(unrestrictedApiAccessKwd, "Allow API access to unlisted hashes"), cmdkit.BoolOption(unencryptTransportKwd, "Disable transport encryption (for debugging protocols)"), cmdkit.BoolOption(enableGCKwd, "Enable automatic periodic repo garbage collection"), @@ -377,9 +376,6 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment // construct fuse mountpoints - if the user provided the --mount flag mount, _ := req.Options[mountKwd].(bool) - if mount && offline { - return cmdkit.Errorf(cmdkit.ErrClient, "mount is not currently supported in offline mode") - } if mount { if err := mountFuse(req, cctx); err != nil { return err @@ -632,27 +628,40 @@ func mountFuse(req *cmds.Request, cctx *oldcmds.Context) error { return fmt.Errorf("mountFuse: GetConfig() failed: %s", err) } - fsdir, found := req.Options[ipfsMountKwd].(string) + //FIXME: on MacOS supplying --ipfs-mount="~/ipfs-mount" creates a relative directory "./~/ipfs-mount" + mountPoint, found := req.Options[ipfsMountKwd].(string) if !found { - fsdir = cfg.Mounts.IPFS + mountPoint = cfg.Mounts.IPFS } - nsdir, found := req.Options[ipnsMountKwd].(string) - if !found { - nsdir = cfg.Mounts.IPNS + daemon, err := cctx.ConstructNode() + if err != nil { + return fmt.Errorf("mountFuse: ConstructNode() failed: %s", err) } - node, err := cctx.ConstructNode() + //TODO: remove this when done debugging + if daemon.Mount != nil { + return fmt.Errorf("Programmer error detected; daemon is initializing but mountpoint is already set: {%p}%q", daemon.Mount, daemon.Mount.Where()) + } + + api, err := coreapi.NewCoreAPI(daemon) if err != nil { - return fmt.Errorf("mountFuse: ConstructNode() failed: %s", err) + return fmt.Errorf("mountFuse: NewCoreAPI() failed: %s", err) } - err = nodeMount.Mount(node, fsdir, nsdir) + fsi, err := mount.InvokeMount(mountPoint, daemon.FilesRoot, api, cctx.Context()) if err != nil { - return err + return fmt.Errorf("mountFuse: InvokeMount() failed: %s", err) } - fmt.Printf("IPFS mounted at: %s\n", fsdir) - fmt.Printf("IPNS mounted at: %s\n", nsdir) + + daemon.Mount = fsi + + absMount, err := filepath.Abs(mountPoint) + if err != nil { + absMount = mountPoint + } + + fmt.Printf("IPFS mounted at: %s\n", absMount) return nil } diff --git a/core/commands/mount_unix.go b/core/commands/mount.go similarity index 57% rename from core/commands/mount_unix.go rename to core/commands/mount.go index 530734f1660..7fbf2ed0efb 100644 --- a/core/commands/mount_unix.go +++ b/core/commands/mount.go @@ -1,27 +1,22 @@ -// +build !windows,!nofuse +// +build !nofuse package commands import ( + "errors" "fmt" - "io" - cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv" - nodeMount "github.com/ipfs/go-ipfs/fuse/node" + "github.com/ipfs/go-ipfs/core/commands/cmdenv" + mount "github.com/ipfs/go-ipfs/core/commands/mount" + "github.com/ipfs/go-ipfs/core/coreapi" - config "gx/ipfs/QmUAuYuiafnJRZxDDX7MuruMNsicYNuyub5vUeAcupUBNs/go-ipfs-config" cmds "gx/ipfs/QmX6AchyJgso1WNamTJMdxfzGiWuYu94K6tF9MJ66rRhAu/go-ipfs-cmds" cmdkit "gx/ipfs/Qmde5VP1qUkyQXKCfmEUA7bP64V2HAptbJ7phuPp7jXWwg/go-ipfs-cmdkit" ) -const ( - mountIPFSPathOptionName = "ipfs-path" - mountIPNSPathOptionName = "ipns-path" -) - var MountCmd = &cmds.Command{ Helptext: cmdkit.HelpText{ - Tagline: "Mounts IPFS to the filesystem (read-only).", + Tagline: "Mounts IPFS to the filesystem.", ShortDescription: ` Mount IPFS at a read-only mountpoint on the OS (default: /ipfs and /ipns). All IPFS objects will be accessible under that directory. Note that the @@ -77,53 +72,60 @@ baz `, }, Options: []cmdkit.Option{ - cmdkit.StringOption(mountIPFSPathOptionName, "f", "The path where IPFS should be mounted."), - cmdkit.StringOption(mountIPNSPathOptionName, "n", "The path where IPNS should be mounted."), + cmdkit.StringOption("ipfs-path", "f", "The path where IPFS should be mounted."), + //TODO: this should probably be an argument not an option; or possibly its own command `ipfs unmount [-f --timeout]` + cmdkit.BoolOption("unmount", "u", "Destroy existing mount."), }, - Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { - cfg, err := cmdenv.GetConfig(env) - if err != nil { - return err - } - nd, err := cmdenv.GetNode(env) + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) (err error) { + defer res.Close() + + daemon, err := cmdenv.GetNode(env) if err != nil { return err } - // error if we aren't running node in online mode - if nd.LocalMode() { - return ErrNotOnline + destroy, _ := req.Options["unmount"].(bool) + if destroy { + if daemon.Mount == nil { + return errors.New("IPFS is not mounted") + } + whence := daemon.Mount.Where() + ret := daemon.Mount.Close() + if ret == nil { + cmds.EmitOnce(res, fmt.Sprintf("Successfully unmounted %#q", whence)) + daemon.Mount = nil + } + + return ret } - fsdir, found := req.Options[mountIPFSPathOptionName].(string) - if !found { - fsdir = cfg.Mounts.IPFS // use default value + if daemon.Mount != nil { + //TODO: introduce `mount -f` to automatically do this? + //problem: '-f' overlaps with ipfs-path short param + return fmt.Errorf("IPFS already mounted at: %#q use `ipfs mount -u`", daemon.Mount.Where()) } - // get default mount points - nsdir, found := req.Options[mountIPNSPathOptionName].(string) - if !found { - nsdir = cfg.Mounts.IPNS // NB: be sure to not redeclare! + api, err := coreapi.NewCoreAPI(daemon) + if err != nil { + return err } - err = nodeMount.Mount(nd, fsdir, nsdir) + conf, err := cmdenv.GetConfig(env) if err != nil { return err } - var output config.Mounts - output.IPFS = fsdir - output.IPNS = nsdir - return cmds.EmitOnce(res, &output) - }, - Type: config.Mounts{}, - Encoders: cmds.EncoderMap{ - cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, mounts *config.Mounts) error { - fmt.Fprintf(w, "IPFS mounted at: %s\n", mounts.IPFS) - fmt.Fprintf(w, "IPNS mounted at: %s\n", mounts.IPNS) - - return nil - }), + daemonCtx := daemon.Context() + mountPoint := conf.Mounts.IPFS + filesRoot := daemon.FilesRoot + + fsi, err := mount.InvokeMount(mountPoint, filesRoot, api, daemonCtx) + if err != nil { + return err + } + daemon.Mount = fsi + cmds.EmitOnce(res, fmt.Sprintf("mounted at: %#q", daemon.Mount.Where())) + return nil }, } diff --git a/core/commands/mount/cache.go b/core/commands/mount/cache.go new file mode 100644 index 00000000000..20002c97a52 --- /dev/null +++ b/core/commands/mount/cache.go @@ -0,0 +1,97 @@ +package fusemount + +import ( + cid "gx/ipfs/QmTbxNB1NwDesLmKTscr4udL2tVP7MaxvXnD1D9yX7g3PN/go-cid" + mh "gx/ipfs/QmerPMzPk1mJVowm8KgmoknWa4yCYvvugMPsgWmDNUvDLW/go-multihash" + + lru "gx/ipfs/QmQjMHF8ptRgx4E57UFMiT4YM6kqaJeYxZ1MCDX23aw4rK/golang-lru" +) + +/* OLD +type cidCache interface { + Add(*cid.Cid, fusePath) + Request(*cid.Cid) fusePath + Release(*cid.Cid) +} +*/ + +type cidCache struct { + actual *lru.ARCCache + cb cid.Builder +} + +func (cc *cidCache) Add(nCid cid.Cid, fp fusePath) { + if cidCacheEnabled { + cc.actual.Add(nCid, fp) + } +} + +func (cc *cidCache) Request(nCid cid.Cid) fusePath { + if !cidCacheEnabled { + return nil + } + if v, ok := cc.actual.Get(nCid); ok { + if _, ok = v.(fusePath); !ok { + log.Errorf("Cache entry for %q is not valid: {%T}%#v", nCid, v, v) + return nil + } + return v.(fusePath) + } + return nil +} + +func (cc *cidCache) Release(nCid cid.Cid) { + if !cidCacheEnabled { + return + } + cc.actual.Remove(nCid) +} + +//TODO: size from conf +func (cc *cidCache) Init() error { + if !cidCacheEnabled { + return nil + } + + arc, err := lru.NewARC(100) //NOTE: arbitrary debug size + if err != nil { + return err + } + cc.actual = arc + //TODO: pkg/cid should expose a recommendation hint, i.e. cid.RecommendedDefault + cc.cb = cid.NewPrefixV1(cid.Raw, mh.SHA2_256) + return nil +} + +func (cc *cidCache) ReleasePath(path string) { + if !cidCacheEnabled { + return + } + + pathCid, err := cc.cb.Sum([]byte(path)) + if err != nil { + log.Errorf("Cache - hash error [report this]: %s", err) + return + } + cc.actual.Remove(pathCid) +} + +func (cc *cidCache) RequestPath(path string) fusePath { + if !cidCacheEnabled { + return nil + } + + pathCid, err := cc.cb.Sum([]byte(path)) + if err != nil { + log.Errorf("Cache - hash error [report this]: %s", err) + return nil + } + if node, ok := cc.actual.Get(pathCid); ok { + return node.(fusePath) + } + return nil +} + +func (cc *cidCache) Hash(path string) (cid.Cid, error) { + return cc.cb.Sum([]byte(path)) +} diff --git a/core/commands/mount/create.go b/core/commands/mount/create.go new file mode 100644 index 00000000000..f18fd6ff24a --- /dev/null +++ b/core/commands/mount/create.go @@ -0,0 +1,311 @@ +package fusemount + +import ( + "fmt" + dag "gx/ipfs/QmPJNbVw8o3ohC43ppSXyNXwYKsWShG4zygnirHptfbHri/go-merkledag" + coreoptions "gx/ipfs/QmXLwxifxwfc2bAwq6rdjbYqAsGzWsDE9RM5TWMGtykyj6/interface-go-ipfs-core/options" + mfs "gx/ipfs/Qmb74fRYPgpjYzoBV7PAVNmP3DQaRrh8dHdKE4PwnF3cRx/go-mfs" + unixfs "gx/ipfs/QmcYUTQ7tBZeH1CLsZM2S3xhMEZdvUgXvbjhpMsLDpk3oJ/go-unixfs" + "os" + gopath "path" + + "github.com/billziss-gh/cgofuse/fuse" +) + +//TODO: lock parent? +func (fs *FUSEIPFS) Link(origin, target string) int { + fs.Lock() + defer fs.Unlock() + log.Errorf("Link - Request %q -> %q", origin, target) + + switch parsePathType(target) { + case tIPNS: + log.Errorf("Link - IPNS support not implemented yet %q", target) + return -fuse.EROFS + case tMFS: + fErr, gErr := mfsSymlink(fs.filesRoot, target, target[frs:]) + if gErr != nil { + log.Errorf("Link - mfs error: %s", gErr) + } + return fErr + default: + log.Errorf("Link - Unexpected request %q", target) + return -fuse.ENOENT + } +} + +//TODO: Upon successful completion, link() shall mark for update the st_ctime field of the file. Also, the st_ctime and st_mtime fields of the directory that contains the new entry shall be marked for update. +//TODO: lock parent +func (fs *FUSEIPFS) Symlink(target, linkActual string) int { + fs.Lock() + defer fs.Unlock() + log.Debugf("Symlink - Request %q -> %q", target, linkActual) + + switch parsePathType(linkActual) { + case tIPNS: + log.Errorf("Symlink - IPNS support not implemented yet %q", linkActual) + return -fuse.EROFS + case tMFS: + fErr, gErr := mfsSymlink(fs.filesRoot, target, linkActual[frs:]) + if gErr != nil { + log.Errorf("Symlink - error: %s", gErr) + } + return fErr + default: + log.Errorf("Symlink - Unexpected request %q", linkActual) + return -fuse.ENOENT + } +} + +func mfsSymlink(filesRoot *mfs.Root, target, linkActual string) (int, error) { + linkDir, linkName := gopath.Split(linkActual) + log.Errorf("mfsSymlink 1 - %q -> %q", target, linkActual) + + mfsNode, err := mfs.Lookup(filesRoot, linkDir) + if err != nil { + return -fuse.ENOENT, err + } + log.Errorf("mfsSymlink 2 - %q -> %q", target, linkActual) + mfsDir, ok := mfsNode.(*mfs.Directory) + if !ok { + return -fuse.ENOTDIR, fmt.Errorf("%s was not a directory", linkDir) + } + log.Errorf("mfsSymlink 3 - %q -> %q", target, linkActual) + + dagData, err := unixfs.SymlinkData(target) + if err != nil { + log.Errorf("mfsSymlink I/O sunk %q:%s", linkActual, err) + return -fuse.EIO, err + } + log.Errorf("mfsSymlink 4 - %q -> %q", target, linkActual) + + dagNode := dag.NodeWithData(dagData) + dagNode.SetCidBuilder(mfsDir.GetCidBuilder()) + log.Errorf("mfsSymlink 5 - %q -> %q", target, linkActual) + + if err := mfsDir.AddChild(linkName, dagNode); err != nil { + log.Errorf("mfsSymlink I/O sunk %q:%s", linkActual, err) + return -fuse.EIO, err + } + log.Errorf("mfsSymlink 6 - %q -> %q", target, linkActual) + return 0, nil +} + +//TODO: lock parent +func (fs *FUSEIPFS) Create(path string, flags int, mode uint32) (int, uint64) { + log.Debugf("Create - Request[m:%o]{f:%o} %q", mode, flags, path) + return fs.Open(path, flags) +} + +func (fs *FUSEIPFS) Mknod(path string, mode uint32, dev uint64) int { + fs.Lock() + defer fs.Unlock() + log.Debugf("Mknod - Request [%X]{%X}%q", mode, dev, path) + + // TODO: abstract this: node.PLock(){self.parent.lock();self.lock()} // goes up to soft-root max; i.e ipns-key, mfs-root + + parentPath := gopath.Dir(path) + parent, err := fs.LookupPath(parentPath) + if err != nil { + log.Errorf("Mknod - could not fetch/lock parent for %q", path) + } + parent.Lock() + defer parent.Unlock() + // + fErr, gErr := fs.mknod(path) + if gErr != nil { + log.Errorf("Mknod - %s", gErr) + } + + fs.cc.ReleasePath(parentPath) + return fErr +} + +//TODO: inline this +func (fs FUSEIPFS) mknod(path string) (int, error) { + parsedNode, err := fs.LookupPath(path) + if err == nil { + return -fuse.EEXIST, os.ErrExist + } + if err != os.ErrNotExist { + return -fuse.EIO, err + } + + switch parsedNode.(type) { + case *ipnsKey: + _, keyName := gopath.Split(path) + coreKey, err := fs.core.Key().Generate(fs.ctx, keyName) + if err != nil { + return -fuse.EIO, fmt.Errorf("could not generate IPNS key %q: %s", keyName, err) + } + newRootNode, err := emptyNode(fs.ctx, fs.core.Dag(), unixfs.TFile, nil) + if err != nil { + return -fuse.EIO, fmt.Errorf("could not generate unixdir %q: %s", keyName, err) + } + + err = fs.ipnsDelayedPublish(coreKey, newRootNode) + if err != nil { + return -fuse.EIO, fmt.Errorf("could not publish to key %q: %s", keyName, err) + } + return fuseSuccess, nil + + case *ipnsNode: + return fs.ipnsMknod(path) + + case *mfsNode: + return mfsMknod(fs.filesRoot, path[frs:]) + } + + return -fuse.EROFS, fmt.Errorf("unexpected request {%T}%q", parsedNode, path) +} + +func mfsMknod(filesRoot *mfs.Root, path string) (int, error) { + if _, err := mfs.Lookup(filesRoot, path); err == nil { + return -fuse.EEXIST, fmt.Errorf("%q already exists", path) + } + + dirName, fName := gopath.Split(path) + mfsNode, err := mfs.Lookup(filesRoot, dirName) + if err != nil { + return -fuse.ENOENT, err + } + mfsDir, ok := mfsNode.(*mfs.Directory) + if !ok { + return -fuse.ENOTDIR, fmt.Errorf("%s is not a directory", dirName) + } + + dagNode := dag.NodeWithData(unixfs.FilePBData(nil, 0)) + dagNode.SetCidBuilder(mfsDir.GetCidBuilder()) + + err = mfsDir.AddChild(fName, dagNode) + if err != nil { + log.Errorf("mfsMknod I/O sunk %q:%s", path, err) + return -fuse.EIO, err + } + + return fuseSuccess, nil +} + +func (fs *FUSEIPFS) Mkdir(path string, mode uint32) int { + fs.Lock() + defer fs.Unlock() + log.Debugf("Mkdir - Request {%X}%q", mode, path) + + parentPath := gopath.Dir(path) + parent, err := fs.LookupPath(parentPath) + if err != nil { + log.Errorf("Mkdir - could not fetch/lock parent for %q", path) + } + parent.Lock() + defer parent.Unlock() + defer fs.cc.ReleasePath(parentPath) //TODO: don't do this on failure + + switch parsePathType(path) { + case tMFS: + //TODO: review mkdir opts + Mkdir POSIX specs (are intermediate paths allowed by default?) + if err := mfs.Mkdir(fs.filesRoot, path[frs:], mfs.MkdirOpts{Flush: mfsSync}); err != nil { + if err == mfs.ErrDirExists || err == os.ErrExist { + return -fuse.EEXIST + } + log.Errorf("Mkdir - unexpected error - %s", err) + return -fuse.EACCES + } + return fuseSuccess + case tIPNSKey: //TODO: refresh fs.nameRoots + _, keyName := gopath.Split(path) + coreKey, err := fs.core.Key().Generate(fs.ctx, keyName) + if err != nil { + log.Errorf("Mkdir - could not generate IPNS key %q: %s", keyName, err) + return -fuse.EACCES + } + newRootNode, err := emptyNode(fs.ctx, fs.core.Dag(), unixfs.TDirectory, nil) + if err != nil { + log.Errorf("Mkdir - could not generate unixdir %q: %s", keyName, err) + return -fuse.EACCES + } + + err = fs.ipnsDelayedPublish(coreKey, newRootNode) + if err != nil { + log.Errorf("Mkdir - could not publish to key %q: %s", keyName, err) + return -fuse.EACCES + } + + pbNode, ok := newRootNode.(*dag.ProtoNode) + if !ok { //this should never happen + log.Errorf("IPNS key %q has incompatible type %T", keyName, newRootNode) + fs.nameRoots[keyName] = nil + return -fuse.EACCES + } + + oAPI, err := fs.core.WithOptions(coreoptions.Api.Offline(true)) + if err != nil { + log.Errorf("offline API could not be created: %s", err) + fs.nameRoots[keyName] = nil + return -fuse.EACCES + } + keyRoot, err := mfs.NewRoot(fs.ctx, fs.core.Dag(), pbNode, ipnsPublisher(keyName, oAPI.Name())) + if err != nil { + log.Errorf("IPNS key %q could not be mapped to MFS root: %s", keyName, err) + fs.nameRoots[keyName] = nil + return -fuse.EACCES + } + fs.nameRoots[keyName] = keyRoot + + return fuseSuccess + + case tIPNS: + fErr, gErr := fs.ipnsMkdir(path) + if gErr != nil { + log.Errorf("Mkdir - error: %s", gErr) + } + return fErr + + case tMountRoot, tIPFSRoot, tFilesRoot: + log.Errorf("Mkdir - requested a root entry - %q", path) + return -fuse.EEXIST + } + + log.Errorf("Mkdir - unexpected request %q", path) + return -fuse.ENOENT +} + +func (fs *FUSEIPFS) ipnsMkdir(path string) (int, error) { + keyRoot, subPath, err := fs.ipnsMFSSplit(path) + if err != nil { + return -fuse.EACCES, err + } + + //NOTE: must flush/publish otherwise our resolver is never going to pick up the change + if err := mfs.Mkdir(keyRoot, subPath, mfs.MkdirOpts{Flush: false}); err != nil { + if err == mfs.ErrDirExists || err == os.ErrExist { + return -fuse.EEXIST, err + } + return -fuse.EACCES, err + } + + if err := mfs.FlushPath(keyRoot, subPath); err != nil { + return -fuse.EACCES, err + } + + return fuseSuccess, nil +} + +func (fs *FUSEIPFS) ipnsMknod(path string) (int, error) { + keyRoot, subPath, err := fs.ipnsMFSSplit(path) + if err != nil { + return -fuse.EIO, err + } + blankNode, err := emptyNode(fs.ctx, fs.core.Dag(), unixfs.TFile, nil) + if err != nil { + return -fuse.EIO, err + } + if err := mfs.PutNode(keyRoot, subPath, blankNode); err != nil { + return -fuse.EIO, err + } + + if err := mfs.FlushPath(keyRoot, subPath); err != nil { + return -fuse.EACCES, err + } + + return fuseSuccess, nil +} diff --git a/core/commands/mount/delete.go b/core/commands/mount/delete.go new file mode 100644 index 00000000000..dcf79f5c0fd --- /dev/null +++ b/core/commands/mount/delete.go @@ -0,0 +1,137 @@ +package fusemount + +import ( + "fmt" + mfs "gx/ipfs/Qmb74fRYPgpjYzoBV7PAVNmP3DQaRrh8dHdKE4PwnF3cRx/go-mfs" + gopath "path" + + "github.com/billziss-gh/cgofuse/fuse" +) + +/* TODO +func (*FileSystemBase) Removexattr(path string, name string) int +*/ + +//TODO: review; make sure we invalidate node and handles +//TODO: lock parent? +func (fs *FUSEIPFS) Unlink(path string) int { + fs.Lock() + defer fs.Unlock() + log.Debugf("Unlink - Request %q", path) + /* + lNode, err := fs.parseLocalPath(path) + if err != nil { + log.Errorf("Unlink - path err %s", err) + return -fuse.EINVAL + } + */ + + fsNode, err := fs.LookupPath(path) + if err != nil { + log.Errorf("Unlink - lookup error: %s", err) + return -fuse.ENOENT + } + fsNode.Lock() + defer fsNode.Unlock() + + switch fsNode.(type) { + default: + log.Errorf("Unlink - request in read only section: %q", path) + return -fuse.EROFS + case *ipfsRoot, *ipfsNode: + //TODO: block rm? + log.Errorf("Unlink - request in read only section: %q", path) + return -fuse.EROFS + case *mfsNode: + if err := mfsRemove(fs.filesRoot, path[frs:]); err != nil { + log.Errorf("Unlink - mfs error: %s", err) + return -fuse.EIO + } + + //invalidate cache object + fs.cc.ReleasePath(path) + case *ipnsKey: + _, keyName := gopath.Split(path) + _, err := fs.core.Key().Remove(fs.ctx, keyName) + if err != nil { + log.Errorf("could not remove IPNS key %q: %s", keyName, err) + return -fuse.EIO + } + + case *ipnsNode: + keyRoot, subPath, err := fs.ipnsMFSSplit(path) + if err != nil { + log.Errorf("Unlink - IPNS key error: %s", err) + } + if err := mfsRemove(keyRoot, subPath); err != nil { + log.Errorf("Unlink - mfs error: %s", err) + return -fuse.EIO + } + } + + return fuseSuccess +} + +//TODO: lock parent +func (fs *FUSEIPFS) Rmdir(path string) int { + fs.Lock() + defer fs.Unlock() + log.Debugf("Rmdir - Request %q", path) + + lNode, err := parseLocalPath(path) + if err != nil { + log.Errorf("Rmdir - path err %s", err) + return -fuse.ENOENT + } + + parentPath := gopath.Dir(path) + parent, err := fs.LookupPath(parentPath) + if err != nil { + log.Errorf("Mkdir - could not fetch/lock parent for %q", path) + } + parent.Lock() + defer parent.Unlock() + defer fs.cc.ReleasePath(parentPath) //TODO: don't do this on failure + + defer fs.cc.ReleasePath(path) //TODO: don't do on failure + + switch lNode.(type) { + case *ipnsKey: + _, keyName := gopath.Split(path) + _, err := fs.core.Key().Remove(fs.ctx, keyName) + if err != nil { + log.Errorf("could not remove IPNS key %q: %s", keyName, err) + return -fuse.EIO + } + + case *ipnsNode: + case *ipfsNode: + return -fuse.EROFS + case *mfsNode: + if err := mfsRemove(fs.filesRoot, path[frs:]); err != nil { + log.Errorf("Unlink - DBG EIO %q %s", path, err) + return -fuse.EIO + } + } + return fuseSuccess +} + +func mfsRemove(mRoot *mfs.Root, path string) error { + dir, name := gopath.Split(path) + parent, err := mfs.Lookup(mRoot, dir) + if err != nil { + return fmt.Errorf("parent lookup: %s", err) + } + pDir, ok := parent.(*mfs.Directory) + if !ok { + return fmt.Errorf("no such file or directory: %s", path) + } + + if err = pDir.Unlink(name); err != nil { + return err + } + + //TODO: is it the callers responsibility to flush? + //return pDir.Flush() + return nil +} diff --git a/core/commands/mount/dirio.go b/core/commands/mount/dirio.go new file mode 100644 index 00000000000..0b6c3dce742 --- /dev/null +++ b/core/commands/mount/dirio.go @@ -0,0 +1,340 @@ +package fusemount + +import ( + "context" + "errors" + "fmt" + "io" + "sync" + "time" + + "github.com/billziss-gh/cgofuse/fuse" + + coreiface "gx/ipfs/QmXLwxifxwfc2bAwq6rdjbYqAsGzWsDE9RM5TWMGtykyj6/interface-go-ipfs-core" + coreoptions "gx/ipfs/QmXLwxifxwfc2bAwq6rdjbYqAsGzWsDE9RM5TWMGtykyj6/interface-go-ipfs-core/options" + mfs "gx/ipfs/Qmb74fRYPgpjYzoBV7PAVNmP3DQaRrh8dHdKE4PwnF3cRx/go-mfs" + unixfs "gx/ipfs/QmcYUTQ7tBZeH1CLsZM2S3xhMEZdvUgXvbjhpMsLDpk3oJ/go-unixfs" +) + +//TODO: rename? directoryStreamHandle +type directoryStream struct { + record fusePath + entries []directoryEntry + err error + init *sync.Cond + + //initialize sync.Once + //sem sync.Cond + //stream <-chan DirectoryMessage +} + +//TODO: better name; AsyncDirEntry? +// API format bridge +type DirectoryMessage struct { + directoryEntry + error +} + +//NOTE: purpose is to be cached and updated when needed instead of re-initialized each call for path; thread safe as a consequent +type directoryEntry struct { + //sync.RWMutex + label string + stat *fuse.Stat_t +} + +func (fs *FUSEIPFS) Readdir(path string, + fill func(name string, stat *fuse.Stat_t, ofst int64) bool, + ofst int64, + fh uint64) int { + fs.RLock() + log.Debugf("Readdir - Request +%d[%X]%q", ofst, fh, path) + if ofst < 0 { + fs.RUnlock() + return -fuse.EINVAL + } + + dh, err := fs.LookupDirHandle(fh) + if err != nil { + fs.RUnlock() + log.Errorf("Readdir - [%X]%q: %s", fh, path, err) + if err == errInvalidHandle { + return -fuse.EBADF + } + return -fuse.EIO + } + dh.record.RLock() + fs.RUnlock() + defer dh.record.RUnlock() + + if dh.io.Entries() == 0 { + //TODO: reconsider/discuss this behaviour; dots are not actually required in POSIX or FUSE + // not having them on empty directories causes things like `dir` and `ls` to fail since they look for the dot, but maybe they should since IPFS does not store dot entries in its directories + fill(".", dh.record.Stat(), -1) + return fuseSuccess + } + + for { + select { + case <-fs.ctx.Done(): + return -fuse.EBADF + case msg := <-dh.io.Read(ofst): + err = msg.error + label := msg.directoryEntry.label + stat := msg.directoryEntry.stat + + if err != nil { + if err == io.EOF { + fill(label, stat, -1) + return fuseSuccess + } + log.Errorf("Readdir - %q list err: %s", path, err) + return -fuse.ENOENT + } + + ofst++ + if !fill(label, stat, ofst) { + return fuseSuccess + } + } + } +} + +func (ds *directoryStream) Read(ofst int64) <-chan DirectoryMessage { + msgChan := make(chan DirectoryMessage, 1) + if int64(cap(ds.entries)) <= ofst { + msgChan <- DirectoryMessage{error: fmt.Errorf("invalid offset %d <= %d", cap(ds.entries), ofst)} + return msgChan + } + + if ds.init != nil { + ds.init.L.Lock() + for int64(len(ds.entries)) < ofst || ds.err == nil { + ds.init.Wait() + } + + switch ds.err { + case io.EOF: + ds.init = nil + ds.err = nil + default: + msgChan <- DirectoryMessage{error: ds.err} + return msgChan + } + } + + // NOTE: it it assumed that if len(entries) == ofst and EOF is set, that the entry slot is populated + // if the spooler lies, we'll likely panic + // index change 0 -> 1 + if ofst+1 == int64(len(ds.entries)) { + msgChan <- DirectoryMessage{directoryEntry: ds.entries[ofst], error: io.EOF} + } else { + msgChan <- DirectoryMessage{directoryEntry: ds.entries[ofst]} + } + + return msgChan +} + +//TODO: [readdirplus] stats +//TODO: name: pulse -> timeoutExtender? signalCallback? entryEvent? +//TODO: document/handle this is only intended to be used with non-empty directories; length must be checked by caller +func (ds *directoryStream) spool(ctx context.Context, inStream <-chan DirectoryMessage, pulse func()) { + defer pulse() + entCap := cap(ds.entries) + for i := 0; i != entCap; i++ { + select { + case <-ctx.Done(): + return + case msg := <-inStream: + if msg.error != nil { + ds.err = msg.error + return + } + + if msg.directoryEntry.label == "" { + ds.err = fmt.Errorf("directory contains empty entry label") + return + } + + if fReaddirPlus && msg.directoryEntry.stat == nil { + ds.err = fmt.Errorf("Readdir - stat for %q is not initialized", msg.directoryEntry.label) + return + } + + ds.entries = append(ds.entries, msg.directoryEntry) + pulse() + } + } + ds.err = io.EOF +} + +func (ds *directoryStream) Entries() int { + return cap(ds.entries) +} + +func coreMux(cc <-chan coreiface.LsLink) <-chan DirectoryMessage { + msgChan := make(chan DirectoryMessage) + go func() { + for m := range cc { + ent := directoryEntry{label: m.Link.Name} + msgChan <- DirectoryMessage{directoryEntry: ent, error: m.Err} + } + close(msgChan) + }() + return msgChan +} + +func mfsMux(uc <-chan unixfs.LinkResult) <-chan DirectoryMessage { + msgChan := make(chan DirectoryMessage) + go func() { + for m := range uc { + ent := directoryEntry{label: m.Link.Name} + msgChan <- DirectoryMessage{directoryEntry: ent, error: m.Err} + } + close(msgChan) + }() + return msgChan +} + +//TODO: +func activeDir(fsNode fusePath) FsDirectory { + //TODO: if fsNode.Handles(); ret handles[0].dirio + return nil +} + +func (fs *FUSEIPFS) yieldDirIO(fsNode fusePath) (FsDirectory, error) { + dirStream := &directoryStream{record: fsNode} + switch fsNode.(type) { + default: + return nil, errors.New("unexpected type") + + case *mountRoot: + dirStream.entries = fs.roots + + case *ipfsRoot: + pins, err := fs.core.Pin().Ls(fs.ctx, coreoptions.Pin.Type.Recursive()) + if err != nil { + log.Errorf("ipfsRoot - Ls err: %v", err) + return nil, err + } + + ents := make([]directoryEntry, 0, len(pins)) + for _, pin := range pins { + //TODO: [readdirplus] stats + ents = append(ents, directoryEntry{label: pin.Path().Cid().String()}) + } + dirStream.entries = ents + + case *ipnsRoot: + dirStream.entries = fs.ipnsRootSubnodes() + } + + return dirStream, nil +} + +func (fs *FUSEIPFS) yieldAsyncDirIO(ctx context.Context, timeoutGrace time.Duration, fsNode fusePath) (FsDirectory, error) { + if cached := activeDir(fsNode); cached != nil { + return cached, nil + } + + var inputChan <-chan DirectoryMessage + var entryCount int + + switch fsNode.(type) { + default: + return nil, errors.New("unexpected type") + case *ipfsNode, *ipnsNode: + globalNode := fsNode + if isReference(fsNode) { + var err error + globalNode, err = fs.resolveToGlobal(fsNode) + if err != nil { + return nil, err + } + } + + //TODO: [readdirplus] stats + coreChan, err := fs.core.Unixfs().Ls(ctx, globalNode, coreoptions.Unixfs.ResolveChildren(false)) + if err != nil { + return nil, err + } + + oStat, err := fs.core.Object().Stat(ctx, globalNode) + if err != nil { + return nil, err + } + + entryCount = oStat.NumLinks + if entryCount != 0 { + inputChan = coreMux(coreChan) + } + + case *mfsNode, *mfsRoot, *ipnsKey: + var ( //XXX: there's probably a better way to handle this; go prevents a nice fallthrough + mRoot *mfs.Root + mPath string + ) + if _, ok := fsNode.(*ipnsKey); ok { + var err error + if mRoot, mPath, err = fs.ipnsMFSSplit(fsNode.String()); err != nil { + return nil, err + } + } else { + mRoot = fs.filesRoot + mPath = fsNode.String()[frs:] + } + + mfsChan, count, err := fs.mfsSubNodes(mRoot, mPath) + if err != nil { + return nil, err + } + entryCount = count + if entryCount != 0 { + inputChan = mfsMux(mfsChan) + } + } + + //TODO: move this to doc.go or something + /* Opendir() -> .spool() -> Readdir() real-time, cross thread synchronization + - .init: use to .init.Wait() in reader, until entry slot N or .err is populated + set .init to nil in waiting thread after processing error|EOF + - .spool(): responsible for populating directory entries list and directory's error value + set .err to io.EOF when finished without errors + - timeout.AfterFunc(): responsible for reporting timeout to directory, + cleaning up resources, and sending wake-up event. + Basically an object with this capability: + context.WithDeadline(deadline, func()).Reset(duration) + - pulser(): responsible for extending timeout and sending wake-up event + */ + + backgroundDir := &directoryStream{record: fsNode, entries: make([]directoryEntry, 0, entryCount)} + if entryCount == 0 { + backgroundDir.err = io.EOF + return backgroundDir, nil + } + + backgroundDir.init = sync.NewCond(&sync.Mutex{}) + callContext, cancel := context.WithCancel(ctx) + cancelClosure := func() { + if backgroundDir.err == nil { // don't overwrite error if it existed before the timeout + backgroundDir.err = errors.New("timed out") + } + cancel() + } + + timeout := time.AfterFunc(timeoutGrace, cancelClosure) + pulser := func() { + defer backgroundDir.init.Broadcast() + if backgroundDir.err != nil { + return + } + + if !timeout.Stop() { + <-timeout.C + } + timeout.Reset(timeoutGrace) + } + + backgroundDir.spool(callContext, inputChan, pulser) + + return backgroundDir, nil +} diff --git a/core/commands/mount/event.go b/core/commands/mount/event.go new file mode 100644 index 00000000000..1988d1f6e37 --- /dev/null +++ b/core/commands/mount/event.go @@ -0,0 +1,53 @@ +package fusemount + +import ( + "sync" + "time" +) + +//TODO: config: IPNS record-lifetime, mfs lifetime +func (fs *FUSEIPFS) dbgBackgroundRoutine() { + var nodeType typeToken + + // invalidate cache entries + // TODO refresh nodes in background instead? + + var wg sync.WaitGroup + for { + select { + case <-fs.ctx.Done(): + log.Warning("fs context is done") + return + case <-time.After(10 * time.Second): + nodeType = tIPNSKey + case <-time.After(11 * time.Second): + nodeType = tIPNS + case <-time.After(2 * time.Minute): + nodeType = tMFS + } + + //this is likely suboptimal, quick hacks for debugging + fs.Lock() + wg.Add(2) + go func() { + defer wg.Done() + for _, handle := range fs.fileHandles { + path := handle.record.String() + if parsePathType(path) == nodeType { + fs.cc.ReleasePath(path) + } + } + }() + go func() { + defer wg.Done() + for _, handle := range fs.dirHandles { + path := handle.record.String() + if parsePathType(path) == nodeType { + fs.cc.ReleasePath(path) + } + } + }() + wg.Wait() + fs.Unlock() + } +} diff --git a/core/commands/mount/filesystem.go b/core/commands/mount/filesystem.go new file mode 100644 index 00000000000..177e0b4befd --- /dev/null +++ b/core/commands/mount/filesystem.go @@ -0,0 +1,519 @@ +package fusemount + +import ( + "context" + "errors" + "fmt" + "io" + "runtime" + "sync" + + "github.com/billziss-gh/cgofuse/fuse" + mi "github.com/ipfs/go-ipfs/core/commands/mount/interface" + + dag "gx/ipfs/QmPJNbVw8o3ohC43ppSXyNXwYKsWShG4zygnirHptfbHri/go-merkledag" + coreiface "gx/ipfs/QmXLwxifxwfc2bAwq6rdjbYqAsGzWsDE9RM5TWMGtykyj6/interface-go-ipfs-core" + coreoptions "gx/ipfs/QmXLwxifxwfc2bAwq6rdjbYqAsGzWsDE9RM5TWMGtykyj6/interface-go-ipfs-core/options" + mfs "gx/ipfs/Qmb74fRYPgpjYzoBV7PAVNmP3DQaRrh8dHdKE4PwnF3cRx/go-mfs" + logging "gx/ipfs/QmbkT7eMTyXfpeyB3ZMxxcxg7XH8t6uXp49jqzz4HB7BGF/go-log" +) + +// NOTE: readdirplus isn't supported on all platforms, being aware of this reduces duplicate metadata construction +// alter InvokeMount mount to have opts struct opts{core:coreapi, readdir:bool...}) +var ( + log = logging.Logger("mount") + + //dbg stuff + fReaddirPlus = false //TODO: this has to be passed to us, this is only set here for debugging + mfsSync = false + cidCacheEnabled = true + + errNoLink = errors.New("not a symlink") + errInvalidHandle = errors.New("invalid handle") + errNoKey = errors.New("key not found") + errInvalidPath = errors.New("invalid path") + errInvalidArg = errors.New("invalid argument") + errReadOnly = errors.New("read only section") +) + +const fuseSuccess = 0 + +//TODO: platform specific routine mountArgs() +func InvokeMount(mountPoint string, filesRoot *mfs.Root, api coreiface.CoreAPI, ctx context.Context) (fsi mi.Interface, err error) { + //TODO: if mountPoint == default; platform switch; win32 = \\.\ipfs\$mountpoint + + hcf := make(chan error) + //TODO: mux the parent context with out own context, for now they're the same + fs := &FUSEIPFS{core: api, filesRoot: filesRoot, signal: hcf, parentCtx: ctx, ctx: ctx, mountPoint: mountPoint} + fs.fuseHost = fuse.NewFileSystemHost(fs) + + //TODO: fsh.SetCapReaddirPlus(true) + fs.fuseHost.SetCapCaseInsensitive(false) + + //FIXME: cgofuse has its own signal/interrupt handler; need to ctrl+c twice + go func() { + defer func() { + //TODO: insert platform specific separation here + if r := recover(); r != nil { + if typedR, ok := r.(string); ok { + if typedR == "cgofuse: cannot find winfsp" { + err = errors.New("WinFSP(http://www.secfs.net/winfsp/) is required for mount on this platform, but it was not found") + } + err = errors.New(typedR) + hcf <- err + return + } + err = fmt.Errorf("Mount panicked! %v", r) + hcf <- err + } + }() + + //triggers init() which returns on signal channel + if runtime.GOOS == "windows" { + //FIXME: Volume prefix result in WinFSP err c000000d + // ^ breaks UNC paths + //fsh.Mount(mountPoint, []string{"-o uid=-1,gid=-1,fstypename=IPFS", "--VolumePrefix=\\ipfs"}) + fs.fuseHost.Mount(mountPoint, []string{"-o uid=-1,gid=-1,fstypename=IPFS"}) + } else { + fs.fuseHost.Mount(mountPoint, nil) + } + }() + + if err = <-hcf; err != nil { + log.Error(err) + return + } + + fs.active = true + fsi = fs + return +} + +type FUSEIPFS struct { + // index's (lookup) lock + // RLock should be retained for lookups + // Lock should be retained when altering index (including cache) + sync.RWMutex + + // provided by InvokeMount() + core coreiface.CoreAPI + filesRoot *mfs.Root + //for compliance with go-ipfs daemon + parentCtx context.Context + signal chan error + mountPoint string + fuseHost *fuse.FileSystemHost // pointer to "self"; used to die via self.fuseHost.Unmount() + + // set in init + active bool + ctx context.Context + roots []directoryEntry + cc cidCache + mountTime fuse.Timespec + + // NOTE: heap equivalent; prevent Go gc claiming our objects between FS operations + fileHandles map[uint64]*fileHandle + dirHandles map[uint64]*dirHandle + nameRoots map[string]*mfs.Root //TODO: see note on nameYieldFileIO +} + +/* +func (ri *rootIndex) requestFh() uint64 { + ri.Lock() + for { + ri.indexCursor++ + if ri.indexCursor <= specialIndexEnd { + ri.indexCursor = passiveIndexLen + } + + if _, ok := ri.activeHandles[ri.indexCursor]; !ok { // empty index + ri.activeHandles[ri.indexCursor] = nil + ri.Unlock() + return ri.indexCursor + } + } +} +*/ + +func (fs *FUSEIPFS) Init() { + fs.Lock() + defer fs.Unlock() + log.Debug("init") + + // return error on channel set by our invoker + var chanErr error + defer func() { + fs.signal <- chanErr + }() + + if chanErr = fs.cc.Init(); chanErr != nil { + log.Errorf("[FATAL] cache could not be initialized: %s", chanErr) + return + } + + fs.roots = []directoryEntry{ + directoryEntry{label: "ipfs"}, + directoryEntry{label: "ipns"}, + directoryEntry{label: filesNamespace}, + } + + oAPI, chanErr := fs.core.WithOptions(coreoptions.Api.Offline(true)) + if chanErr != nil { + log.Errorf("[FATAL] offline API could not be initialized: %s", chanErr) + return + } + + nameKeys, chanErr := fs.core.Key().List(fs.ctx) + if chanErr != nil { + log.Errorf("[FATAL] IPNS keys could not be listed: %s", chanErr) + return + } + + fs.nameRoots = make(map[string]*mfs.Root) + for _, key := range nameKeys { + keyNode, scopedErr := oAPI.ResolveNode(fs.ctx, key.Path()) + if scopedErr != nil { + log.Warning("IPNS key %q could not be resolved: %s", key.Name(), scopedErr) + fs.nameRoots[key.Name()] = nil + continue + } + + pbNode, ok := keyNode.(*dag.ProtoNode) + if !ok { + log.Warningf("IPNS key %q has incompatible type %T", key.Name(), keyNode) + fs.nameRoots[key.Name()] = nil + continue + } + + keyRoot, scopedErr := mfs.NewRoot(fs.ctx, fs.core.Dag(), pbNode, ipnsPublisher(key.Name(), oAPI.Name())) + if scopedErr != nil { + log.Warningf("IPNS key %q could not be mapped to MFS root: %s", key.Name(), scopedErr) + fs.nameRoots[key.Name()] = nil + continue + } + fs.nameRoots[key.Name()] = keyRoot + } + + fs.fileHandles = make(map[uint64]*fileHandle) + fs.dirHandles = make(map[uint64]*dirHandle) + + fs.mountTime = fuse.Now() + + //TODO: implement for real + go fs.dbgBackgroundRoutine() + log.Debug("init finished") +} + +type fileHandle struct { + record fusePath + io FsFile +} + +type dirHandle struct { + record fusePath + io FsDirectory +} + +func (fs *FUSEIPFS) Open(path string, flags int) (int, uint64) { + fs.Lock() + defer fs.Unlock() + log.Debugf("Open - Request {%X}%q", flags, path) + + if fs.AvailableHandles(aFiles) == 0 { + log.Error("Open - all handle slots are filled") + return -fuse.ENFILE, invalidIndex + } + + if flags&fuse.O_CREAT != 0 { + if flags&fuse.O_EXCL != 0 { + _, err := fs.LookupPath(path) + if err == nil { + log.Errorf("Open/Create - exclusive flag provided but %q already exists", path) + return -fuse.EEXIST, invalidIndex + } + } + fErr, gErr := fs.mknod(path) + if gErr != nil { + log.Errorf("Open/Create - %q: %s", path, gErr) + return fErr, invalidIndex + } + } + + fsNode, err := fs.LookupPath(path) + if err != nil { + log.Errorf("Open - path err: %s", err) + return -fuse.ENOENT, invalidIndex + } + fsNode.Lock() + defer fsNode.Unlock() + + fh, err := fs.newFileHandle(fsNode) //TODO: flags + if err != nil { + log.Errorf("Open - sunk %q:%s", fsNode.String(), err) + return -fuse.EIO, invalidIndex + } + + log.Debugf("Open - Assigned [%X]%q", fh, fsNode) + return fuseSuccess, fh +} + +func (fs *FUSEIPFS) Opendir(path string) (int, uint64) { + fs.Lock() + defer fs.Unlock() + log.Debugf("Opendir - Request %q", path) + + if fs.AvailableHandles(aDirectories) == 0 { + log.Error("Opendir - all handle slots are filled") + return -fuse.ENFILE, invalidIndex + } + + fsNode, err := fs.LookupPath(path) + if err != nil { + log.Errorf("Opendir - path err %q: %s", path, err) + return -fuse.ENOENT, invalidIndex + } + fsNode.Lock() + defer fsNode.Unlock() + + if !fReaddirPlus { + fh, err := fs.newDirHandle(fsNode) + if err != nil { + log.Errorf("Opendir - %s", err) + return -fuse.ENOENT, invalidIndex + } + log.Debugf("Opendir - Assigned [%X]%q", fh, fsNode) + return fuseSuccess, fh + } + + return -fuse.EACCES, invalidIndex +} + +//TODO: [educational/compiler] how costly is defer vs {ret = x; unlock; return ret} +func (fs *FUSEIPFS) Release(path string, fh uint64) int { + log.Debugf("Release - [%X]%q", fh, path) + fs.Lock() + defer fs.Unlock() + return fs.releaseFileHandle(fh) +} + +func (fs *FUSEIPFS) Releasedir(path string, fh uint64) int { + log.Debugf("Releasedir - [%X]%q", fh, path) + fs.Lock() + defer fs.Unlock() + return fs.releaseDirHandle(fh) +} + +//TODO: implement +func (fs *FUSEIPFS) Chmod(path string, mode uint32) int { + log.Errorf("Chmod [%X]%q", mode, path) + return 0 +} + +func (fs *FUSEIPFS) Chown(path string, uid, gid uint32) int { + + log.Errorf("Chmod [%d:%d]%q", uid, gid, path) + return 0 +} + +//HCF +func (fs *FUSEIPFS) Destroy() { + log.Debugf("Destroy requested") + + //NOTE: ideally the invoker would release us at some point + // we could require a pointer in InvokeMount() and nil it here though + // fs.daemonPtr = nil; cleanup...; return + fs.active = false + fs.mountPoint = "" + + if !mfsSync { //TODO: do this anyway? + if err := mfs.FlushPath(fs.filesRoot, "/"); err != nil { + log.Errorf("MFS failed to sync: %s", err) + } + } + + //TODO: close our context +} + +func (fs *FUSEIPFS) Flush(path string, fh uint64) int { + fs.RLock() + log.Debugf("Flush - Request [%X]%q", fh, path) + + h, err := fs.LookupFileHandle(fh) + if err != nil { + fs.RUnlock() + log.Errorf("Flush - bad request [%X]%q: %s", fh, path, err) + if err == errInvalidHandle { + return -fuse.EBADF //TODO: we might want to change this to EIO since Flush is called on Close which implies the handle should have been valid + } + return -fuse.EIO + } + h.record.Lock() + fs.RUnlock() + defer h.record.Unlock() + + if !h.record.Mutable() { + return fuseSuccess + } + + return h.io.Sync() +} + +func (fs *FUSEIPFS) Fsync(path string, datasync bool, fh uint64) int { + fs.RLock() + log.Debugf("Fsync - Request [%X]%q", fh, path) + + h, err := fs.LookupFileHandle(fh) + if err != nil { + fs.RUnlock() + log.Errorf("Fsync - bad request [%X]%q: %s", fh, path, err) + if err == errInvalidHandle { + return -fuse.EBADF + } + return -fuse.EIO + } + + h.record.Lock() + fs.RUnlock() + defer h.record.Unlock() + + return h.io.Sync() +} + +func (fs *FUSEIPFS) Fsyncdir(path string, datasync bool, fh uint64) int { + fs.RLock() + log.Errorf("Fsyncdir - Request [%X]%q", fh, path) + + fsDirNode, err := fs.LookupDirHandle(fh) + if err != nil { + fs.RUnlock() + log.Errorf("Fsyncdir - [%X]%q: %s", fh, path, err) + if err == errInvalidHandle { + return -fuse.EBADF + } + return -fuse.EIO + } + fsDirNode.record.Lock() + fs.RUnlock() + defer fsDirNode.record.Unlock() + + /* FIXME: not implemented + fsDirNode, err := dirFromHandle(fh) + if err != nil { + fs.Unlock() + log.Errorf("Fsyncdir - [%X]%q: %s", fh, path, err) + if err == errInvalidHandle { + return -fuse.EBADF + } + return -fuse.EIO + } + + return fsDirNode.Sync() + */ + return fuseSuccess +} + +func (fs *FUSEIPFS) Getxattr(path, name string) (int, []byte) { + log.Errorf("Getxattr") + return -fuse.ENOSYS, nil +} + +func (fs *FUSEIPFS) Listxattr(path string, fill func(name string) bool) int { + log.Errorf("Listxattr") + return -fuse.ENOSYS +} + +func (fs *FUSEIPFS) Removexattr(path, name string) int { + log.Errorf("Removexattr") + return -fuse.ENOSYS +} + +func (fs *FUSEIPFS) Setxattr(path, name string, value []byte, flags int) int { + log.Errorf("Setxattr") + return -fuse.ENOSYS +} + +func (fs *FUSEIPFS) Readlink(path string) (int, string) { + fs.RLock() + log.Debugf("Readlink - Request %q", path) + fsNode, err := fs.LookupPath(path) + if err != nil { + log.Debugf("Readlink - lookup: %s", err) + fs.RUnlock() + return -fuse.ENOENT, "" + } + + fsNode.RLock() + fs.RUnlock() + defer fsNode.RUnlock() + + if isDevice(fsNode) { + return -fuse.EINVAL, "" + } + + if isReference(fsNode) { + var err error + fsNode, err = fs.resolveToGlobal(fsNode) + if err != nil { + log.Errorf("Readlink - node resolution error: %s", err) + return -fuse.EIO, "" + } + } + + target, err := fs.fuseReadlink(fsNode) + if err != nil { + if err == errNoLink { + log.Errorf("Readlink - %q is not a symlink", path) + return -fuse.EINVAL, "" + } + log.Errorf("Readlink - unexpected link resolution error: %s", err) + return -fuse.EIO, "" + } + + return len(target), target +} + +type handleErrorPair struct { + fhi uint64 + err error +} + +//TODO: test this +func (fs *FUSEIPFS) refreshFileSiblings(fh uint64, h *fileHandle) (failed []handleErrorPair) { + handles := *h.record.Handles() + if len(handles) == 1 && handles[0] == fh { + return + } + + for _, fhi := range handles { + if fhi == fh { + continue + } + curFh, err := fs.LookupFileHandle(fhi) + if err != nil { + failed = append(failed, handleErrorPair{fhi, err}) + continue + } + + oCur, err := curFh.io.Seek(0, io.SeekCurrent) + if err != nil { + failed = append(failed, handleErrorPair{fhi, err}) + continue + } + err = curFh.io.Close() + if err != nil { + failed = append(failed, handleErrorPair{fhi, err}) + continue + } + curFh.io = nil + + posixIo, err := fs.yieldFileIO(h.record) + if err != nil { + failed = append(failed, handleErrorPair{fhi, err}) + continue + } + + posixIo.Seek(oCur, io.SeekStart) + curFh.io = posixIo + } + return +} diff --git a/core/commands/mount/index.go b/core/commands/mount/index.go new file mode 100644 index 00000000000..9735f2dc360 --- /dev/null +++ b/core/commands/mount/index.go @@ -0,0 +1,343 @@ +package fusemount + +import ( + "errors" + "fmt" + "os" + gopath "path" + "strings" + + ds "gx/ipfs/QmUadX5EcvrBmxAV9sE7wUWtWSqxns5K84qKJBixmcT1w9/go-datastore" + coreiface "gx/ipfs/QmXLwxifxwfc2bAwq6rdjbYqAsGzWsDE9RM5TWMGtykyj6/interface-go-ipfs-core" + coreoptions "gx/ipfs/QmXLwxifxwfc2bAwq6rdjbYqAsGzWsDE9RM5TWMGtykyj6/interface-go-ipfs-core/options" + ipld "gx/ipfs/QmZ6nzCLwGLVfRzYLpD7pW6UNuBDKEcA2imJtVpbEx2rxy/go-ipld-format" + mfs "gx/ipfs/Qmb74fRYPgpjYzoBV7PAVNmP3DQaRrh8dHdKE4PwnF3cRx/go-mfs" +) + +const ( + invalidIndex = ^uint64(0) + + filesNamespace = "files" + filesRootPath = "/" + filesNamespace + filesRootPrefix = filesRootPath + "/" + frs = len(filesRootPath) +) + +//TODO: remove alias +type typeToken = uint64 + +//TODO: cleanup +const ( + tMountRoot typeToken = iota + tIPFSRoot + tIPNSRoot + tFilesRoot + tRoots + tIPFS + tIPLD + tImmutable + tIPNS + tIPNSKey + tMFS + tMutable + tUnknown +) + +func resolveMFS(filesRoot *mfs.Root, path string) (ipld.Node, error) { + mfsNd, err := mfs.Lookup(filesRoot, path) + if err != nil { + return nil, err + } + ipldNode, err := mfsNd.GetNode() + if err != nil { + return nil, err + } + return ipldNode, nil +} + +func (fs *FUSEIPFS) resolveIpns(path string) (string, error) { + pathKey, remainder, err := ipnsSplit(path) + if err != nil { + return "", err + } + + oAPI, err := fs.core.WithOptions(coreoptions.Api.Offline(true)) + if err != nil { + log.Errorf("API error: %v", err) + return "", err + } + + var nameAPI coreiface.NameAPI + globalPath := path + coreKey, err := resolveKeyName(fs.ctx, oAPI.Key(), pathKey) + switch err { + case nil: // locally owned keys are resolved offline + globalPath = gopath.Join(coreKey.Path().String(), remainder) + nameAPI = oAPI.Name() + + case errNoKey: // paths without named keys are valid, but looked up via network instead of locally + nameAPI = fs.core.Name() + + case ds.ErrNotFound: // a key was generated, but not published to / initialized + return "", fmt.Errorf("IPNS key %q has no value", pathKey) + default: + return "", err + } + + //NOTE: the returned path is not guaranteed to exist + resolvedPath, err := nameAPI.Resolve(fs.ctx, globalPath) + if err != nil { + return "", err + } + //log.Errorf("dbg: %q -> %q -> %q", path, globalPath, resolvedPath) + return resolvedPath.String(), nil +} + +//TODO: see how IPLD selectors handle this kind of parsing +func parsePathType(path string) typeToken { + switch { + case path == "/": + return tMountRoot + case path == "/ipfs": + return tIPFSRoot + case path == "/ipns": + return tIPNSRoot + case path == filesRootPath: + return tFilesRoot + case strings.HasPrefix(path, "/ipld/"): + return tIPLD + case strings.HasPrefix(path, "/ipfs/"): + return tIPFS + case strings.HasPrefix(path, filesRootPrefix): + return tMFS + case strings.HasPrefix(path, "/ipns/"): + if len(strings.Split(path, "/")) == 3 { + return tIPNSKey + } + return tIPNS + /* NIY + case strings.HasPrefix(path, "/api/"): + return tAPI + */ + } + + return tUnknown +} + +//operator, operator! +func parseLocalPath(path string) (fusePath, error) { + switch parsePathType(path) { + case tMountRoot: + return &mountRoot{rootBase: rootBase{ + recordBase: crb("/")}}, nil + case tIPFSRoot: + return &ipfsRoot{rootBase: rootBase{ + recordBase: crb("/ipfs")}}, nil + case tIPNSRoot: + return &ipnsRoot{rootBase: rootBase{ + recordBase: crb("/ipns")}}, nil + case tFilesRoot: + return &mfsRoot{rootBase: rootBase{ + recordBase: crb(filesRootPath)}}, nil + case tIPFS, tIPLD: + return &ipfsNode{recordBase: crb(path)}, nil + case tMFS: + return &mfsNode{mutableBase: mutableBase{ + recordBase: crb(path)}}, nil + case tIPNSKey: + return &ipnsKey{ipnsNode{mutableBase: mutableBase{ + recordBase: crb(path)}}}, nil + case tIPNS: + return &ipnsNode{mutableBase: mutableBase{ + recordBase: crb(path)}}, nil + case tUnknown: + switch strings.Count(path, "/") { + case 0: + return nil, errors.New("invalid request") + case 1: + return nil, errors.New("invalid root request") + case 2: + return nil, errors.New("invalid root namespace") + } + } + + return nil, fmt.Errorf("unexpected request %q", path) +} + +func crb(path string) recordBase { + return recordBase{path: path, handles: &[]uint64{}} +} + +func (fs *FUSEIPFS) parent(node fusePath) (fusePath, error) { + if _, ok := node.(*mountRoot); ok { + return node, nil + } + + path := node.String() + i := len(path) - 1 + for i != 0 && path[i] != '/' { + i-- + } + if i == 0 { + return fs.LookupPath("/") + } + + return fs.LookupPath(path[:i]) +} + +func (fs *FUSEIPFS) resolveToGlobal(node fusePath) (fusePath, error) { + switch node.(type) { + case *mfsNode: + //contacts API node + ipldNode, err := resolveMFS(fs.filesRoot, node.String()[frs:]) + if err != nil { + return nil, err + } + + if cachedNode := fs.cc.Request(ipldNode.Cid()); cachedNode != nil { + return cachedNode, nil + } + + //TODO: will ipld always be valid here? is there a better way to retrieve the path? + globalNode, err := fs.LookupPath(gopath.Join("/ipld/", ipldNode.String())) + if err != nil { + return nil, err + } + + fs.cc.Add(ipldNode.Cid(), globalNode) + return globalNode, nil + + case *ipnsNode, *ipnsKey: + resolvedPath, err := fs.resolveIpns(node.String()) //contacts API node + if err != nil { + return nil, err + } + + //TODO: test if the core handles recursion protection here; IPNS->IPNS->... + return fs.LookupPath(resolvedPath) + } + + return nil, fmt.Errorf("unexpected reference-node type %T", node) +} + +func isReference(fsNode fusePath) bool { + switch fsNode.(type) { + case *mfsNode, *ipnsNode, *ipnsKey: + return true + default: + return false + } +} + +func isDevice(fsNode fusePath) bool { + switch fsNode.(type) { + case *mountRoot, *ipfsRoot, *ipnsRoot, *mfsRoot: + return true + default: + return false + } +} + +//NOTE: caller should retain FS (R)Lock +func (fs *FUSEIPFS) LookupFileHandle(fh uint64) (handle *fileHandle, err error) { + err = errInvalidHandle + if fh == invalidIndex { + return + } + + defer func() { + if r := recover(); r != nil { + log.Errorf("Lookup recovered from panic, likely invalid handle: %v", r) + handle = nil + err = errInvalidHandle + } + }() + + //TODO: enable when stable + //L0 direct cast 🤠 + //return (*fileHandle)(unsafe.Pointer(uintptr(fh))), nil + + //L1 handle -> lookup -> node + if hs, ok := fs.fileHandles[fh]; ok { + if hs.record != nil { + return hs, nil + } + //TODO: return separate error? handleInvalidated (handle was active but became bad) vs handleInvalid (never existed in the first place) + } + return nil, errInvalidHandle +} + +//NOTE: caller should retain FS (R)Lock +func (fs *FUSEIPFS) LookupDirHandle(fh uint64) (handle *dirHandle, err error) { + err = errInvalidHandle + if fh == invalidIndex { + return + } + + defer func() { + if r := recover(); r != nil { + log.Errorf("Lookup recovered from panic, likely invalid handle: %v", r) + handle = nil + err = errInvalidHandle + } + }() + + //TODO: enable when stable + //L0 direct cast 🤠 + //return (*dirHandle)(unsafe.Pointer(uintptr(fh))), nil + + //L1 handle -> lookup -> node + if hs, ok := fs.dirHandles[fh]; ok { + if hs.record != nil { + return hs, nil + } + } + return nil, errInvalidHandle +} + +//NOTE: caller should retain FS (R)Lock +func (fs *FUSEIPFS) LookupPath(path string) (fusePath, error) { + if path == "" { + return nil, errInvalidArg + } + + //L1 path -> cid -> cache -?> record + pathCid, err := fs.cc.Hash(path) + if err != nil { + log.Errorf("cache: %s", err) + } else if cachedNode := fs.cc.Request(pathCid); cachedNode != nil { + return cachedNode, nil + } + + //L2 path -> full parse+construction + parsedNode, err := parseLocalPath(path) + if err != nil { + return nil, err + } + + if !isDevice(parsedNode) { + if !fs.exists(parsedNode) { + return parsedNode, os.ErrNotExist //NOTE: node is still a valid structure ready for use (i.e. useful for creation/type inspection) + } + } + + fs.cc.Add(pathCid, parsedNode) + return parsedNode, nil +} + +func (fs *FUSEIPFS) exists(parsedNode fusePath) bool { + globalNode := parsedNode + var err error + if isReference(parsedNode) { + + globalNode, err = fs.resolveToGlobal(parsedNode) + if err != nil { + return false + } + } + if _, err = fs.core.ResolvePath(fs.ctx, globalNode); err != nil { //contacts API node and possibly the network + return false + } + + return true +} diff --git a/core/commands/mount/indexNodes.go b/core/commands/mount/indexNodes.go new file mode 100644 index 00000000000..07c08b70cdd --- /dev/null +++ b/core/commands/mount/indexNodes.go @@ -0,0 +1,73 @@ +package fusemount + +import ( + "strings" + "sync" + + "github.com/billziss-gh/cgofuse/fuse" +) + +type recordBase struct { + sync.RWMutex + path string + + metadata fuse.Stat_t + handles *[]uint64 +} + +type mutableBase struct { + recordBase +} + +type ipfsNode struct { + recordBase + //fd coreiface.UnixfsFile + //initOnce sync.Once + //fStat *fuse.Stat_t +} + +type ipnsNode struct { + mutableBase +} + +type ipnsKey struct { + ipnsNode +} + +type mfsNode struct { + mutableBase + //fd mfs.FileDescriptor +} + +func (rb *recordBase) String() string { + return rb.path +} + +func (rb *recordBase) Handles() *[]uint64 { + return rb.handles +} + +func (rb *recordBase) Stat() *fuse.Stat_t { + return &rb.metadata +} + +//TODO: make a note somewhere that generic functions assume valid structs; define what "valid" means +func (rb *recordBase) Namespace() string { + i := strings.IndexRune(rb.path[1:], '/') + if i == -1 { + return "root" + } + return rb.path[1:i] +} + +func (mn *mfsNode) Namespace() string { + return filesNamespace +} + +func (rb *recordBase) Mutable() bool { + return false +} + +func (mb *mutableBase) Mutable() bool { + return true +} diff --git a/core/commands/mount/interface.go b/core/commands/mount/interface.go new file mode 100644 index 00000000000..34e8768b712 --- /dev/null +++ b/core/commands/mount/interface.go @@ -0,0 +1,187 @@ +package fusemount + +import ( + "fmt" + "io" + "sync" + "time" + "unsafe" + + coreiface "gx/ipfs/QmXLwxifxwfc2bAwq6rdjbYqAsGzWsDE9RM5TWMGtykyj6/interface-go-ipfs-core" + + "github.com/billziss-gh/cgofuse/fuse" +) + +type RWLocker interface { + sync.Locker + RLock() + RUnlock() +} + +type FsFile interface { + io.Reader + io.Seeker + io.Closer + Size() (int64, error) + Write(buff []byte, ofst int64) (int, error) + Sync() int + Truncate(size int64) (int, error) +} + +type FsDirectory interface { + //Parent() Directory + //Entries(ofst int64) <-chan directoryEntry + Entries() int + Read(offset int64) <-chan DirectoryMessage +} + +type fusePath interface { + coreiface.Path + RWLocker + + //TODO: reconsider approach + Stat() *fuse.Stat_t + Handles() *[]uint64 +} + +// FIXME: even if it's unlikely, we need to assure addr == invalidIndex is never true +func (fs *FUSEIPFS) newDirHandle(fsNode fusePath) (uint64, error) { + //TODO: check path/node is actually a directory + //return -fuse.ENOTDIR + + var ( + fsDir FsDirectory + err error + ) + if canAsync(fsNode) { + const timeout = 2 * time.Second //reset per entry in stream reader routine; TODO: configurable + fsDir, err = fs.yieldAsyncDirIO(fs.ctx, timeout, fsNode) // Read inherits this context + } else { + fsDir, err = fs.yieldDirIO(fsNode) + } + + if err != nil { + return invalidIndex, fmt.Errorf("could not yield directory IO: %s", err) + } + + hs := &dirHandle{record: fsNode, io: fsDir} + fh := uint64(uintptr(unsafe.Pointer(hs))) + *fsNode.Handles() = append(*fsNode.Handles(), fh) + fs.dirHandles[fh] = hs + return fh, nil +} + +func (fs *FUSEIPFS) newFileHandle(fsNode fusePath) (uint64, error) { + pIo, err := fs.yieldFileIO(fsNode) + if err != nil { + return invalidIndex, err + } + + hs := &fileHandle{record: fsNode, io: pIo} + + fh := uint64(uintptr(unsafe.Pointer(hs))) + *fsNode.Handles() = append(*fsNode.Handles(), fh) + fs.fileHandles[fh] = hs + return fh, nil +} + +func (fs *FUSEIPFS) releaseFileHandle(fh uint64) (ret int) { + if fh == invalidIndex { + log.Errorf("releaseHandle - input handle is invalid") + return -fuse.EBADF + } + + defer func() { + if r := recover(); r != nil { + log.Errorf("releaseHandle recovered from panic, likely invalid handle: %v", r) + ret = -fuse.EBADF + } + }() + + hs := (*fileHandle)(unsafe.Pointer(uintptr(fh))) + + handleGroup := hs.record.Handles() + for i, cFh := range *handleGroup { + if cFh == fh { + *handleGroup = append((*handleGroup)[:i], (*handleGroup)[i+1:]...) + + if hs.io != nil { + if err := hs.io.Close(); err != nil { + log.Error(err) + } + } + + //Go runtime free-able + fs.fileHandles[fh] = nil + delete(fs.fileHandles, fh) + ret = fuseSuccess + return + } + } + log.Errorf("releaseHandle - handle was detected as valid but was not associated with node %q", hs.record.String()) + ret = -fuse.EBADF + return +} + +func (fs *FUSEIPFS) releaseDirHandle(fh uint64) (ret int) { + if fh == invalidIndex { + log.Errorf("releaseDirHandle - input handle is invalid") + ret = -fuse.EBADF + return + } + + defer func() { + if r := recover(); r != nil { + log.Errorf("releaseDirHandle recovered from panic, likely invalid handle: %v", r) + ret = -fuse.EBADF + } + }() + hs := (*dirHandle)(unsafe.Pointer(uintptr(fh))) + + handleGroup := hs.record.Handles() + for i, cFh := range *handleGroup { + if cFh == fh { + *handleGroup = append((*handleGroup)[:i], (*handleGroup)[i+1:]...) + + //Go runtime free-able + fs.dirHandles[fh] = nil + delete(fs.dirHandles, fh) + ret = fuseSuccess + return + } + } + log.Errorf("releaseDirHandle - handle was detected as valid but was not associated with node %q", hs.record.String()) + ret = -fuse.EBADF + return +} + +type AvailType = bool + +const ( + aFiles AvailType = false + aDirectories AvailType = true +) + +//NOTE: caller should retain FS Lock +func (fs *FUSEIPFS) AvailableHandles(directories AvailType) uint64 { + if directories { + return (invalidIndex - 1) - uint64(len(fs.dirHandles)) + } + return (invalidIndex - 1) - uint64(len(fs.fileHandles)) +} + +func (fs *FUSEIPFS) Close() error { + //log.whatever + if !fs.fuseHost.Unmount() { + return fmt.Errorf("Could not unmount %q, reason unknown", fs.mountPoint) + } + return nil +} + +func (fs *FUSEIPFS) IsActive() bool { + return fs.active +} + +func (fs *FUSEIPFS) Where() string { + return fs.mountPoint +} diff --git a/core/commands/mount/interface/interface.go b/core/commands/mount/interface/interface.go new file mode 100644 index 00000000000..b69d9411570 --- /dev/null +++ b/core/commands/mount/interface/interface.go @@ -0,0 +1,13 @@ +package mountinterface + +import ( + "github.com/billziss-gh/cgofuse/fuse" +) + +//TODO: docs; this is only necessary to prevent cyclical imports in core; necessary for daemon to retain knowledge and control of mountpoint +type Interface interface { + fuse.FileSystemInterface + IsActive() bool + Where() string + Close() error +} diff --git a/core/commands/mount/io.go b/core/commands/mount/io.go new file mode 100644 index 00000000000..e4d001ba875 --- /dev/null +++ b/core/commands/mount/io.go @@ -0,0 +1,442 @@ +package fusemount + +import ( + "context" + "errors" + "fmt" + "io" + gopath "path" + + coreiface "gx/ipfs/QmXLwxifxwfc2bAwq6rdjbYqAsGzWsDE9RM5TWMGtykyj6/interface-go-ipfs-core" + coreoptions "gx/ipfs/QmXLwxifxwfc2bAwq6rdjbYqAsGzWsDE9RM5TWMGtykyj6/interface-go-ipfs-core/options" + + "github.com/billziss-gh/cgofuse/fuse" + + files "gx/ipfs/QmQmhotPUzVrMEWNK3x1R5jQ5ZHWyL7tVUrmRPjrBrvyCb/go-ipfs-files" + chunk "gx/ipfs/QmYmZ81dU5nnmBFy5MmktXLZpt8QCWhRJd6M1uxVF6vke8/go-ipfs-chunker" + mfs "gx/ipfs/Qmb74fRYPgpjYzoBV7PAVNmP3DQaRrh8dHdKE4PwnF3cRx/go-mfs" + "gx/ipfs/QmcYUTQ7tBZeH1CLsZM2S3xhMEZdvUgXvbjhpMsLDpk3oJ/go-unixfs/mod" +) + +//TODO: set atime on success; use defer to check return +func (fs *FUSEIPFS) Read(path string, buff []byte, ofst int64, fh uint64) int { + fs.RLock() + log.Debugf("Read - Request [%X]{+%d}%q", fh, ofst, path) + + if ofst < 0 { + log.Errorf("Read - Invalid offset {%d}[%X]%q", ofst, fh, path) + fs.RUnlock() + return -fuse.EINVAL + } + + //TODO: [everywhere] change handle lookups to be like this; just to reduce + /* + fh, err := LookupFileHandle(fh) + if err { ... } + fh.record.Lock() + ... + requiresFusePath(fh.record) + requiresIo(fh.io) + ... + fh.Record.Unlock() + */ + + h, err := fs.LookupFileHandle(fh) + if err != nil { + fs.RUnlock() + log.Errorf("Read - [%X]%q: %s", fh, path, err) + if err == errInvalidHandle { + return -fuse.EBADF + } + return -fuse.EIO + } + + h.record.Lock() // write lock; handle cursor is modified + fs.RUnlock() + defer h.record.Unlock() + + //TODO: inspect need to flush here + //if fh != handle.lastCaller { flush } + + if fileBound, err := h.io.Size(); err == nil { + if ofst >= fileBound { + return 0 // this is unique from fuseSuccess + } + } + + if ofst != 0 { + _, err = h.io.Seek(ofst, io.SeekStart) + if err != nil { + log.Errorf("Read - seek error: %s", err) + return -fuse.EIO + } + } + + readBytes, err := h.io.Read(buff) + if err != nil && err != io.EOF { + log.Errorf("Read - error: %s", err) + } + return readBytes +} + +func (fs *FUSEIPFS) yieldFileIO(fsNode fusePath) (FsFile, error) { + //TODO: cast to concrete type and change yield parameters to accept them + switch fsNode.(type) { + case *mfsNode: + return mfsYieldFileIO(fs.filesRoot, fsNode.String()[frs:]) + case *ipfsNode: + return fs.coreYieldFileIO(fsNode) + case *ipnsKey: + return fs.keyYieldFileIO(fsNode) + case *ipnsNode: + return fs.nameYieldFileIO(fsNode) + default: + return nil, fmt.Errorf("unexpected IO request {%T}%q", fsNode, fsNode.String()) + } +} + +func mfsYieldFileIO(filesRoot *mfs.Root, path string) (FsFile, error) { + mfsNode, err := mfs.Lookup(filesRoot, path) + if err != nil { + return nil, err + } + + mfsFile, ok := mfsNode.(*mfs.File) + if !ok { + return nil, fmt.Errorf("File IO requested for non-file, type: %v %q", mfsNode.Type(), path) + } + + //TODO: change arity of yield to accept i/o request flag + return &mfsFileIo{ff: mfsFile, ioFlags: mfs.Flags{Read: true, Write: true, Sync: mfsSync}}, nil +} + +//TODO: we'll have to pass and store write flags on this; for now rely on 🤠 to maintain permissions +//TODO: some kind of local write buffer +type mfsFileIo struct { + ff *mfs.File + //fd mfs.FileDescriptor + + //XXX: this is not ideal, we're duplicating state here to circumvent mfs's 1 (writable) file descriptor limit + //this is likely suboptimal + ioFlags mfs.Flags + cursor int64 +} + +//allows for multiple handles to a single mfs node +func (mio *mfsFileIo) mfsOpenShim() (mfs.FileDescriptor, error) { + fd, err := mio.ff.Open(mio.ioFlags) + if err != nil { + return nil, err + } + if mio.cursor != 0 { + mio.cursor, err = fd.Seek(mio.cursor, io.SeekStart) + if err != nil { + return nil, err + } + } + return fd, nil +} + +func (mio *mfsFileIo) Size() (int64, error) { + fd, err := mio.ff.Open(mio.ioFlags) + if err != nil { + log.Errorf("mio Size I/O sunk %X:%s", fd, err) + return int64(-fuse.EIO), err + } + + defer fd.Close() + return fd.Size() +} + +func (mio *mfsFileIo) Close() error { + return nil + //return mio.fd.Close() +} + +func (mio *mfsFileIo) Seek(offset int64, whence int) (int64, error) { + if whence > io.SeekEnd { + return int64(-fuse.EINVAL), errors.New("invalid whence value") + } + + if offset == 0 { + return mio.cursor, nil + } + + switch whence { + case io.SeekStart: + mio.cursor = offset + case io.SeekCurrent: + mio.cursor += offset + case io.SeekEnd: + if offset > 0 { + return int64(-fuse.EINVAL), errors.New("invalid offset value") + } + s, err := mio.Size() //TODO: avoid re-opening fd in Size() if we can + if err != nil { + log.Errorf("mio Seek I/O sunk: %s", err) + return int64(-fuse.EIO), err + } + mio.cursor = s + offset + } + + return fuseSuccess, nil +} + +func (mio *mfsFileIo) Read(buff []byte) (int, error) { + fd, err := mio.mfsOpenShim() + if err != nil { + log.Errorf("mio Read I/O sunk %X:%s", fd, err) + return -fuse.EIO, err + } + defer fd.Close() + + readBytes, err := fd.Read(buff) + if readBytes >= 1 { + mio.cursor += int64(readBytes) + } + if err != nil { + if err == io.EOF { + return readBytes, err + } + + log.Errorf("mio Read I/O sunk %X:%s", fd, err) + return -fuse.EIO, err + } + return readBytes, nil +} + +//TODO: look into this; speak with shcomatis +// API syncs on close by default; see mfsOpenShim(); every op should force a sync as a result of that +// ideally we want to only sync on demand +func (mio *mfsFileIo) Sync() int { + /* + if err := mio.fd.Flush(); err != nil { + return -fuse.EIO + } + */ + return fuseSuccess +} + +func (mio *mfsFileIo) Write(buff []byte, ofst int64) (int, error) { + var ( + written int + err error + ) + + fd, err := mio.mfsOpenShim() + if err != nil { + log.Errorf("mio Write I/O sunk %X:%s", fd, err) + return -fuse.EIO, err + } + defer fd.Close() + + if ofst == 0 && mio.cursor == 0 { + written, err = fd.Write(buff) + } else { + written, err = fd.WriteAt(buff, ofst) + } + if err != nil { + log.Errorf("mio Write I/O sunk %X:%s", fd, err) + return -fuse.EIO, err + } + mio.cursor += int64(written) + + return written, nil +} + +func (mio *mfsFileIo) Truncate(size int64) (int, error) { + fd, err := mio.mfsOpenShim() + if err != nil { + log.Errorf("mio Truncate I/O sunk %X:%s", fd, err) + return -fuse.EIO, err + } + defer fd.Close() + + err = fd.Truncate(size) + if err != nil { + return -fuse.EIO, err + } + return fuseSuccess, nil +} + +func (fs *FUSEIPFS) coreYieldFileIO(curNode coreiface.Path) (FsFile, error) { + var err error + apiNode, err := fs.core.Unixfs().Get(fs.ctx, curNode) + if err != nil { + return nil, err + } + + fIo, ok := apiNode.(files.File) + if !ok { + return nil, fmt.Errorf("%q is not a file", curNode.String()) + } + + return &corePIo{fd: fIo}, nil +} + +type corePIo struct { + fd files.File +} + +//FIXME read broken on large files sometimes? +//MFS too +func (cio *corePIo) Read(buff []byte) (int, error) { + readBytes, err := cio.fd.Read(buff) + if err != nil { + if err == io.EOF { + return readBytes, err + } + log.Errorf("cio Read I/O sunk %s", err) + return -fuse.EIO, err + } + return readBytes, nil +} + +func (cio *corePIo) Close() error { + return cio.fd.Close() +} + +func (cio *corePIo) Seek(offset int64, whence int) (int64, error) { + return cio.fd.Seek(offset, whence) +} + +func (cio *corePIo) Size() (int64, error) { + return cio.fd.Size() +} + +func (cio *corePIo) Write(buff []byte, ofst int64) (int, error) { + return -fuse.EROFS, fmt.Errorf("Write requested on read only path") +} + +func (cio *corePIo) Sync() int { + log.Warning("Sync called on read only file") + return -fuse.EINVAL +} + +func (cio *corePIo) Truncate(int64) (int, error) { + return -fuse.EROFS, errReadOnly +} + +//TODO: [fs] free MFS roots when no references are using them instead of loading them all forever +// instantiate on demand +func (fs *FUSEIPFS) nameYieldFileIO(fsNode fusePath) (FsFile, error) { + keyRoot, subPath, err := fs.ipnsMFSSplit(fsNode.String()) + if err != nil { + globalNode, err := fs.resolveToGlobal(fsNode) + if err != nil { + return nil, err + } + return fs.coreYieldFileIO(globalNode) + } + return mfsYieldFileIO(keyRoot, subPath) +} + +type keyFileIo struct { + key coreiface.Key + name coreiface.NameAPI + mod *mod.DagModifier +} + +func (fs *FUSEIPFS) keyYieldFileIO(fsNode fusePath) (FsFile, error) { + _, keyName := gopath.Split(fsNode.String()) + + oAPI, err := fs.core.WithOptions(coreoptions.Api.Offline(true)) + if err != nil { + return nil, err + } + + coreKey, err := resolveKeyName(fs.ctx, oAPI.Key(), keyName) + if err != nil { + return nil, err + } + + ipldNode, err := oAPI.ResolveNode(fs.ctx, coreKey.Path()) + if err != nil { + return nil, err + } + + dmod, err := mod.NewDagModifier(fs.ctx, ipldNode, oAPI.Dag(), chunk.DefaultSplitter) + if err != nil { + return nil, err + } + + return &keyFileIo{key: coreKey, name: oAPI.Name(), mod: dmod}, nil +} + +func (kio *keyFileIo) Write(buff []byte, ofst int64) (int, error) { + var ( + written int + err error + ) + + if ofst == 0 { + written, err = kio.mod.Write(buff) + } else { + written, err = kio.mod.WriteAt(buff, ofst) + } + if err != nil { + return -fuse.EIO, err + } + + //TODO: [investigate] core.ResolveNode deadlocks if we write and publish this node, but don't commit it to the dag service + /* + if err = kio.mod.Sync(); err != nil { + return -fuse.EIO, err + } + */ + + nd, err := kio.mod.GetNode() + if err != nil { + return -fuse.EIO, err + } + + corePath, err := coreiface.ParsePath(nd.String()) + if err != nil { + return -fuse.EIO, err + } + + _, err = kio.name.Publish(context.TODO(), corePath, coreoptions.Name.Key(kio.key.Name()), coreoptions.Name.AllowOffline(true)) + if err != nil { + return -fuse.EIO, err + } + + return written, nil +} + +func (kio *keyFileIo) Read(buff []byte) (int, error) { + readBytes, err := kio.mod.Read(buff) + if err != nil { + if err == io.EOF { + return readBytes, err + } + log.Errorf("kio Read I/O sunk %s", err) + return -fuse.EIO, err + } + return readBytes, nil +} + +func (*keyFileIo) Close() error { + return nil +} + +func (kio *keyFileIo) Seek(offset int64, whence int) (int64, error) { + return kio.mod.Seek(offset, whence) +} + +func (kio *keyFileIo) Size() (int64, error) { + return kio.mod.Size() +} + +func (kio *keyFileIo) Sync() int { + if err := kio.mod.Sync(); err != nil { + return -fuse.EIO + } + return fuseSuccess +} + +func (kio *keyFileIo) Truncate(size int64) (int, error) { + if err := kio.mod.Truncate(size); err != nil { + return -fuse.EIO, err + } + return fuseSuccess, nil +} diff --git a/core/commands/mount/modify.go b/core/commands/mount/modify.go new file mode 100644 index 00000000000..100d0ef0ca1 --- /dev/null +++ b/core/commands/mount/modify.go @@ -0,0 +1,139 @@ +package fusemount + +import ( + "github.com/billziss-gh/cgofuse/fuse" + + mfs "gx/ipfs/Qmb74fRYPgpjYzoBV7PAVNmP3DQaRrh8dHdKE4PwnF3cRx/go-mfs" +) + +/* TODO +func (*FileSystemBase) Setxattr(path string, name string, value []byte, flags int) int +*/ + +//FIXME: obtain lock on oldpath, invalidate handles, etc. +//TODO: see what parts mfs errors on internally; don't reimplement +func (fs *FUSEIPFS) Rename(oldpath, newpath string) int { + fs.Lock() + defer fs.Unlock() + if oldpath == newpath { + return fuseSuccess + } + + //TODO: If either the old or new argument names a symbolic link, rename() shall operate on the symbolic link itself, and shall not resolve the last component of the argument. + + /* TODO + if type(oldpath) != type(newpath) { + if type(newpath) == directory { + return -fuse.EISDIR + } else { + return -fuse.ENOTDIR + } + } + */ + + /* TODO: current + lookup src + check src.mode for type + if exists; fusePut(srcNode, dstPath) + + ipldPut(src *ipld.Node, dst string) + lookup dst + if exists; compare types + check dst.mode for write access; create() + + + */ + + //oldNode, newNode := fs + + //TODO: enforce: Write access permission is required for both the directory containing old and the directory containing new. + if err := mfs.Mv(fs.filesRoot, oldpath[frs:], newpath[frs:]); err != nil { + log.Errorf("Rename - %s", err) + return -fuse.ENOENT //TODO: real error + } + return fuseSuccess +} + +/* inline this +func (fs FUSEIPFS) ipldPut(nd *ipld.Node, path string) { + target, err := fs.LookupPath(path) + if err == nil { + //file exists, check type compatibility + } + if err != nil && err != os.ErrNotExist { + //bad things + } +} +*/ + +//TODO: document; filesystem locks; FS writes = would alter Lookup(), node writes = alters node data (meta or actual) +func (fs *FUSEIPFS) Utimens(path string, tmsp []fuse.Timespec) int { + fs.RLock() + log.Debugf("Utimens - Request %v %q", tmsp, path) + + fsNode, err := fs.LookupPath(path) + if err != nil { + log.Error(err) //TODO: msg + fs.RUnlock() + return -fuse.ENOENT + } + + fsNode.Lock() + fs.RUnlock() + fStat := fsNode.Stat() + fStat.Atim = tmsp[0] + fStat.Mtim = tmsp[1] + fsNode.Unlock() + return fuseSuccess +} + +func (fs *FUSEIPFS) Truncate(path string, size int64, fh uint64) int { + fs.Lock() + defer fs.Unlock() + log.Debugf("Truncate - req [%X]{%d}%q", fh, size, path) + + if size < 0 { + return -fuse.EINVAL + } + + /* TODO [POISX] + if size > max-size { + return -fuse.EFBIG + } + */ + + if h, err := fs.LookupFileHandle(fh); err == nil { + h.record.Lock() + defer h.record.Unlock() + fErr, gErr := h.io.Truncate(size) + if gErr != nil { + log.Errorf("Truncate - [%X]%q:%s", fh, path, gErr) + } else { + h.record.Stat().Size = size + } + return fErr + } + + fsNode, err := fs.LookupPath(path) + if err != nil { + log.Errorf("Truncate - %q:%s", path, err) + return -fuse.ENOENT + } + + fsNode.Lock() + defer fsNode.Unlock() + + io, err := fs.yieldFileIO(fsNode) + if err != nil { + log.Errorf("Truncate - %q:%s", path, err) + return -fuse.EIO + } + + fErr, gErr := io.Truncate(size) + if gErr != nil { + log.Errorf("Truncate - %q:%s", path, gErr) + } else { + fsNode.Stat().Size = size + } + return fErr +} diff --git a/core/commands/mount/probe.go b/core/commands/mount/probe.go new file mode 100644 index 00000000000..a98f538fe05 --- /dev/null +++ b/core/commands/mount/probe.go @@ -0,0 +1,222 @@ +package fusemount + +import ( + "fmt" + + "github.com/billziss-gh/cgofuse/fuse" + + config "gx/ipfs/QmUAuYuiafnJRZxDDX7MuruMNsicYNuyub5vUeAcupUBNs/go-ipfs-config" + mfs "gx/ipfs/Qmb74fRYPgpjYzoBV7PAVNmP3DQaRrh8dHdKE4PwnF3cRx/go-mfs" + unixfs "gx/ipfs/QmcYUTQ7tBZeH1CLsZM2S3xhMEZdvUgXvbjhpMsLDpk3oJ/go-unixfs" + uio "gx/ipfs/QmcYUTQ7tBZeH1CLsZM2S3xhMEZdvUgXvbjhpMsLDpk3oJ/go-unixfs/io" +) + +/* TODO +func (*FileSystemBase) Getxattr(path string, name string) (int, []byte) +func (*FileSystemBase) Listxattr(path string, fill func(name string) bool) int +*/ + +//TODO: implement for real +//NOTE: systems manual: "If pathname is a symbolic link, it is dereferenced." +func (fs *FUSEIPFS) Access(path string, mask uint32) int { + log.Debugf("Access - Request %q{%d}", path, mask) + return fuseSuccess + //return -fuse.EACCES +} + +func (fs *FUSEIPFS) Statfs(path string, stat *fuse.Statfs_t) int { + log.Debugf("Statfs - Request %q", path) + target, err := config.DataStorePath("") //TODO: review + if err != nil { + log.Errorf("Statfs - Config err %q: %v", path, err) + return -fuse.ENOENT //TODO: proper error + } + + err = fs.fuseFreeSize(stat, target) + if err != nil { + log.Errorf("Statfs - Size err %q: %v", target, err) + return -fuse.ENOENT //TODO: proper error + } + return fuseSuccess +} + +//FIXME: we need to initialize children if target is a directory +// ^only on readdirplus though +func (fs *FUSEIPFS) Getattr(path string, fStat *fuse.Stat_t, fh uint64) int { + fs.RLock() + log.Debugf("Getattr - Request [%X]%q", fh, path) + + /* TODO: we need a way to distinguish file and directory handles + if fh != invalidIndex { + curNode, err = fs.LookupHandle(fh) + } else { + curNode, err = fs.LookupPath(path) + } + */ + + fsNode, err := fs.LookupPath(path) + if err != nil { + if !platformException(path) { + log.Warningf("Getattr - Lookup error %q: %s", path, err) + } + fs.RUnlock() + return -fuse.ENOENT + } + fsNode.Lock() + fs.RUnlock() + defer fsNode.Unlock() + + //NOTE [2018.12.26]: [uid] + /* fuse.Getcontext only contains data in callstack under: + - Mknod + - Mkdir + - Getattr + - Open + - OpenDir + - Create + TODO: we need to retain values from chmod and chown and not overwrite them here + */ + + nodeStat := fsNode.Stat() + if nodeStat == nil { + log.Errorf("Getattr - node %q was not initialized properly", fsNode) + return -fuse.EIO + } + + if nodeStat.Mode != 0 { // active node retrieved from lookup + *fStat = *nodeStat + fStat.Uid, fStat.Gid, _ = fuse.Getcontext() + return fuseSuccess + } + + // Local permissions + var permissionBits uint32 = 0555 + switch fsNode.(type) { + case *mfsNode, *mfsRoot, *ipnsRoot, *ipnsKey: + permissionBits |= 0220 + case *ipnsNode: + keyName, _, _ := ipnsSplit(path) + if _, err := resolveKeyName(fs.ctx, fs.core.Key(), keyName); err == nil { // we own this path/key locally + permissionBits |= 0220 + } + } + nodeStat.Mode = permissionBits + + // POSIX type + sizes + switch fsNode.(type) { + case *mountRoot, *ipfsRoot, *ipnsRoot, *mfsRoot: + nodeStat.Mode |= fuse.S_IFDIR + default: + globalNode := fsNode + if isReference(fsNode) { + globalNode, err = fs.resolveToGlobal(fsNode) + if err != nil { + log.Errorf("Getattr - reference node %q could not be resolved: %s ", fsNode, err) + return -fuse.EIO + } + } + + ipldNode, err := fs.core.ResolveNode(fs.ctx, globalNode) + if err != nil { + log.Errorf("Getattr - reference node %q could not be resolved: %s ", fsNode, err) + return -fuse.EIO + } + ufsNode, err := unixfs.ExtractFSNode(ipldNode) + if err != nil { + log.Errorf("Getattr - reference node %q could not be transformed into UnixFS type: %s ", fsNode, err) + return -fuse.EIO + } + + switch ufsNode.Type() { + case unixfs.TFile: + nodeStat.Mode |= fuse.S_IFREG + case unixfs.TDirectory: + nodeStat.Mode |= fuse.S_IFDIR + case unixfs.TSymlink: + nodeStat.Mode |= fuse.S_IFLNK + default: + log.Errorf("Getattr - unexpected node type {%T}%q", ufsNode, globalNode) + return -fuse.EIO + } + + if bs := ufsNode.BlockSizes(); len(bs) != 0 { + nodeStat.Blksize = int64(bs[0]) + } + nodeStat.Size = int64(ufsNode.FileSize()) + } + + // Time + now := fuse.Now() + switch fsNode.(type) { + case *mountRoot, *ipfsRoot, *ipnsRoot, *mfsRoot: + nodeStat.Birthtim = fs.mountTime + default: + nodeStat.Birthtim = now + } + + nodeStat.Atim, nodeStat.Mtim, nodeStat.Ctim = now, now, now //!!! + + *fStat = *nodeStat + fStat.Uid, fStat.Gid, _ = fuse.Getcontext() + return fuseSuccess +} + +func canAsync(fsNd fusePath) bool { + switch fsNd.(type) { + case *ipfsNode, *ipnsNode, *ipnsKey, *mfsNode, *mfsRoot: + return true + } + return false +} + +func (fs *FUSEIPFS) ipnsRootSubnodes() []directoryEntry { + keys, err := fs.core.Key().List(fs.ctx) + if err != nil { + log.Errorf("ipnsRoot - Key err: %v", err) + return nil + } + + ents := make([]directoryEntry, 0, len(keys)) + if !fReaddirPlus { + for _, key := range keys { + ents = append(ents, directoryEntry{label: key.Name()}) + } + return ents + } + //TODO [readdirplus] + return nil +} + +//TODO: accept context arg +func (fs *FUSEIPFS) mfsSubNodes(filesRoot *mfs.Root, path string) (<-chan unixfs.LinkResult, int, error) { + //log.Errorf("mfsSubNodes dbg dir %q", path) + mfsNd, err := mfs.Lookup(filesRoot, path) + if err != nil { + return nil, 0, err + } + + //mfsDir, ok := mfsNd.(*mfs.Directory) + _, ok := mfsNd.(*mfs.Directory) + if !ok { + return nil, 0, fmt.Errorf("mfs %q not a directory", path) + } + + ipldNd, err := mfsNd.GetNode() + if err != nil { + return nil, 0, err + } + + iStat, err := ipldNd.Stat() + if err != nil { + return nil, 0, err + } + entries := iStat.NumLinks + // [2019.02.06] MFS's ForEachEntry is not async, so this conflicts with our Readdir timeout timer + //go mfsDir.ForEachEntry(fs.ctx, muxMessage) + + unixDir, err := uio.NewDirectoryFromNode(fs.core.Dag(), ipldNd) + if err != nil { + return nil, 0, err + } + return unixDir.EnumLinksAsync(fs.ctx), entries, nil +} diff --git a/core/commands/mount/rootNodes.go b/core/commands/mount/rootNodes.go new file mode 100644 index 00000000000..8c9a983d746 --- /dev/null +++ b/core/commands/mount/rootNodes.go @@ -0,0 +1,33 @@ +package fusemount + +type rootBase struct { + recordBase + //mountTime *fuse.Timespec +} + +//type rootList [tRoots]fusePath +type mountRoot struct { + rootBase + //rootIndices rootList +} + +//should inherit from directory entries +type ipfsRoot struct { + rootBase + //TODO: review below + //sync.Mutex + //subnodes []fuseStatPair + //lastUpdated time.Time +} + +type ipnsRoot struct { + rootBase + //sync.Mutex + //keys []coreiface.Key + //subnodes []fuseStatPair + //lastUpdated time.Time +} + +type mfsRoot struct { + rootBase +} diff --git a/core/commands/mount/system_darwin.go b/core/commands/mount/system_darwin.go new file mode 100644 index 00000000000..00aaaae7e48 --- /dev/null +++ b/core/commands/mount/system_darwin.go @@ -0,0 +1,28 @@ +package fusemount + +import ( + "syscall" + + "github.com/billziss-gh/cgofuse/fuse" +) + +//TODO: untested on darwin, struct is different +func (fs *FUSEIPFS) fuseFreeSize(fStatfs *fuse.Statfs_t, path string) error { + sysStat := &syscall.Statfs_t{} + if err := syscall.Statfs(path, sysStat); err != nil { + return err + } + + fStatfs.Fsid = uint64(sysStat.Fsid.Val[0])<<32 | uint64(sysStat.Fsid.Val[1]) + + fStatfs.Bsize = uint64(sysStat.Bsize) + fStatfs.Blocks = sysStat.Blocks + fStatfs.Bfree = sysStat.Bfree + fStatfs.Bavail = sysStat.Bavail + fStatfs.Files = sysStat.Files + fStatfs.Ffree = sysStat.Ffree + fStatfs.Frsize = uint64(sysStat.Bsize) //TODO: review this; should be standard on this platform but needs to be checked again + fStatfs.Flag = uint64(sysStat.Flags) + fStatfs.Namemax = uint64(syscall.NAME_MAX) + return nil +} diff --git a/core/commands/mount/system_linux.go b/core/commands/mount/system_linux.go new file mode 100644 index 00000000000..c148d9891d0 --- /dev/null +++ b/core/commands/mount/system_linux.go @@ -0,0 +1,27 @@ +package fusemount + +import ( + "syscall" + + "github.com/billziss-gh/cgofuse/fuse" +) + +func (fs *FUSEIPFS) fuseFreeSize(fStatfs *fuse.Statfs_t, path string) error { + sysStat := &syscall.Statfs_t{} + if err := syscall.Statfs(path, sysStat); err != nil { + return err + } + + fStatfs.Fsid = uint64(sysStat.Fsid.X__val[0])<<32 | uint64(sysStat.Fsid.X__val[1]) + + fStatfs.Bsize = uint64(sysStat.Bsize) + fStatfs.Blocks = sysStat.Blocks + fStatfs.Bfree = sysStat.Bfree + fStatfs.Bavail = sysStat.Bavail + fStatfs.Files = sysStat.Files + fStatfs.Ffree = sysStat.Ffree + fStatfs.Frsize = uint64(sysStat.Frsize) + fStatfs.Flag = uint64(sysStat.Flags) + fStatfs.Namemax = uint64(sysStat.Namelen) + return nil +} diff --git a/core/commands/mount/system_windows.go b/core/commands/mount/system_windows.go new file mode 100644 index 00000000000..2d337180f84 --- /dev/null +++ b/core/commands/mount/system_windows.go @@ -0,0 +1,86 @@ +package fusemount + +import ( + "unsafe" + + "gx/ipfs/QmVGjyM9i2msKvLXwh9VosCTgP4mL91kC7hDmqnwTTx6Hu/sys/windows" + + "github.com/billziss-gh/cgofuse/fuse" +) + +//try to extract these into other pkg(s) +const LOAD_LIBRARY_SEARCH_SYSTEM32 = 0x00000800 + +func loadSystemDLL(name string) (*windows.DLL, error) { + modHandle, err := windows.LoadLibraryEx(name, 0, LOAD_LIBRARY_SEARCH_SYSTEM32) + if err != nil { + return nil, err + } + return &windows.DLL{Name: name, Handle: modHandle}, nil +} + +func (fs *FUSEIPFS) fuseFreeSize(fStatfs *fuse.Statfs_t, path string) error { + mod, err := loadSystemDLL("kernel32.dll") + if err != nil { + return err + } + defer mod.Release() + proc, err := mod.FindProc("GetDiskFreeSpaceExW") + if err != nil { + return err + } + + var ( + FreeBytesAvailableToCaller, + TotalNumberOfBytes, + TotalNumberOfFreeBytes uint64 + + SectorsPerCluster, + BytesPerSector uint16 + //NumberOfFreeClusters, + //TotalNumberOfClusters uint16 + ) + pathPtr, err := windows.UTF16PtrFromString(path) + if err != nil { + return err //check syscall.EINVAL in caller; NUL byte in string + } + + r1, _, wErr := proc.Call(uintptr(unsafe.Pointer(pathPtr)), + uintptr(unsafe.Pointer(&FreeBytesAvailableToCaller)), + uintptr(unsafe.Pointer(&TotalNumberOfBytes)), + uintptr(unsafe.Pointer(&TotalNumberOfFreeBytes)), + ) + if r1 == 0 { + return wErr + } + + proc, _ = mod.FindProc("GetDiskFreeSpaceW") + r1, _, wErr = proc.Call(uintptr(unsafe.Pointer(pathPtr)), + uintptr(unsafe.Pointer(&SectorsPerCluster)), + uintptr(unsafe.Pointer(&BytesPerSector)), + //uintptr(unsafe.Pointer(&NumberOfFreeClusters)), + 0, + //uintptr(unsafe.Pointer(&TotalNumberOfClusters)), + 0, + ) + if r1 == 0 { + return wErr + } + + fStatfs.Bsize = uint64(SectorsPerCluster * BytesPerSector) + fStatfs.Frsize = uint64(BytesPerSector) + fStatfs.Blocks = TotalNumberOfBytes / uint64(BytesPerSector) + fStatfs.Bfree = TotalNumberOfFreeBytes / (uint64(BytesPerSector)) + fStatfs.Bavail = FreeBytesAvailableToCaller / (uint64(BytesPerSector)) + fStatfs.Files = ^uint64(0) + fStatfs.Ffree = fs.AvailableHandles(aFiles) + fStatfs.Favail = fStatfs.Ffree + + /* TODO + fStatfs.Fsid + fStatfs.Flag + fStatfs.Namemax + */ + + return nil +} diff --git a/core/commands/mount/utils.go b/core/commands/mount/utils.go new file mode 100644 index 00000000000..86951a848cb --- /dev/null +++ b/core/commands/mount/utils.go @@ -0,0 +1,181 @@ +package fusemount + +import ( + "context" + "errors" + "fmt" + gopath "path" + "strings" + + dag "gx/ipfs/QmPJNbVw8o3ohC43ppSXyNXwYKsWShG4zygnirHptfbHri/go-merkledag" + cid "gx/ipfs/QmTbxNB1NwDesLmKTscr4udL2tVP7MaxvXnD1D9yX7g3PN/go-cid" + coreiface "gx/ipfs/QmXLwxifxwfc2bAwq6rdjbYqAsGzWsDE9RM5TWMGtykyj6/interface-go-ipfs-core" + coreoptions "gx/ipfs/QmXLwxifxwfc2bAwq6rdjbYqAsGzWsDE9RM5TWMGtykyj6/interface-go-ipfs-core/options" + ipld "gx/ipfs/QmZ6nzCLwGLVfRzYLpD7pW6UNuBDKEcA2imJtVpbEx2rxy/go-ipld-format" + mfs "gx/ipfs/Qmb74fRYPgpjYzoBV7PAVNmP3DQaRrh8dHdKE4PwnF3cRx/go-mfs" + unixfs "gx/ipfs/QmcYUTQ7tBZeH1CLsZM2S3xhMEZdvUgXvbjhpMsLDpk3oJ/go-unixfs" + uio "gx/ipfs/QmcYUTQ7tBZeH1CLsZM2S3xhMEZdvUgXvbjhpMsLDpk3oJ/go-unixfs/io" + upb "gx/ipfs/QmcYUTQ7tBZeH1CLsZM2S3xhMEZdvUgXvbjhpMsLDpk3oJ/go-unixfs/pb" +) + +//const mutableFlags = fuse.O_WRONLY | fuse.O_RDWR | fuse.O_APPEND | fuse.O_CREAT | fuse.O_TRUNC + +func platformException(path string) bool { + //TODO: add detection for common platform path patterns to avoid flooding error log + /* + macos: + .DS_Store + NT: + Autorun.inf + desktop.ini + Thumbs.db + *.exe.Config + *.exe.lnk + ... + */ + //TODO: move this to a build constraint file filter_windows.go + switch strings.ToLower(gopath.Base(path)) { + case "autorun.inf", "desktop.ini", "folder.jpg", "folder.gif", "thumbs.db": + return true + } + return strings.HasSuffix(path, ".exe.Config") +} + +func (fs *FUSEIPFS) fuseReadlink(fsNode fusePath) (string, error) { + + ipldNode, err := fs.core.ResolveNode(fs.ctx, fsNode) + if err != nil { + return "", err + } + + unixNode, err := unixfs.ExtractFSNode(ipldNode) + if err != nil { + return "", err + } + + if unixNode.Type() != unixfs.TSymlink { + return "", errNoLink + } + + return string(unixNode.Data()), nil +} + +func unixAddChild(ctx context.Context, dagSrv coreiface.APIDagService, rootNode ipld.Node, path string, node ipld.Node) (ipld.Node, error) { + rootDir, err := uio.NewDirectoryFromNode(dagSrv, rootNode) + if err != nil { + return nil, err + } + + err = rootDir.AddChild(ctx, path, node) + if err != nil { + return nil, err + } + + newRoot, err := rootDir.GetNode() + if err != nil { + return nil, err + } + + if err := dagSrv.Add(ctx, newRoot); err != nil { + return nil, err + } + return newRoot, nil +} + +func resolveKeyName(ctx context.Context, api coreiface.KeyAPI, keyString string) (coreiface.Key, error) { + if keyString == "self" { + return api.Self(ctx) + } + + keys, err := api.List(ctx) + if err != nil { + return nil, err + } + for _, key := range keys { + if keyString == key.Name() { + return key, nil + } + } + + return nil, errNoKey +} + +//TODO: remove this and inline publishing? +func (fs *FUSEIPFS) ipnsDelayedPublish(key coreiface.Key, node ipld.Node) error { + oAPI, err := fs.core.WithOptions(coreoptions.Api.Offline(true)) + if err != nil { + return err + } + + coreTarget, err := coreiface.ParsePath(node.String()) + if err != nil { + return err + } + + _, err = oAPI.Name().Publish(fs.ctx, coreTarget, coreoptions.Name.Key(key.Name()), coreoptions.Name.AllowOffline(true)) + if err != nil { + return err + } + + //TODO: go {grace timer based on key; publish to network } + return nil +} + +//TODO: reconsider parameters +func ipnsPublisher(keyName string, nameAPI coreiface.NameAPI) func(context.Context, cid.Cid) error { + return func(ctx context.Context, rootCid cid.Cid) error { + //log.Errorf("publish request; key:%q cid:%q", keyName, rootCid) + _, err := nameAPI.Publish(ctx, coreiface.IpfsPath(rootCid), coreoptions.Name.Key(keyName), coreoptions.Name.AllowOffline(true)) + //log.Errorf("published %q to %q", ent.Value(), ent.Name()) + return err + } +} + +//TODO: do this on initialization of IPNS keys; embed in struct +func (fs FUSEIPFS) ipnsMFSSplit(path string) (*mfs.Root, string, error) { + keyName, subPath, _ := ipnsSplit(path) + keyRoot := fs.nameRoots[keyName] + if keyRoot == nil { + return nil, "", fmt.Errorf("mfs root for key %s not initialized", keyName) + } + return keyRoot, subPath, nil +} + +//XXX +func emptyNode(ctx context.Context, dagAPI coreiface.APIDagService, nodeType upb.Data_DataType, filePrefix *cid.Prefix) (ipld.Node, error) { + if nodeType == unixfs.TFile { + eFile := dag.NodeWithData(unixfs.FilePBData(nil, 0)) + if filePrefix != nil { + eFile.SetCidBuilder(filePrefix.WithCodec(filePrefix.GetCodec())) + } + if err := dagAPI.Add(ctx, eFile); err != nil { + return nil, err + } + return eFile, nil + } else if nodeType == unixfs.TDirectory { + eDir, err := uio.NewDirectory(dagAPI).GetNode() + if err != nil { + return nil, err + } + return eDir, nil + } else { + return nil, errors.New("unexpected node type") + } + +} + +//TODO: docs; return: key, path, error +//TODO: check if there's overlap with go-path +func ipnsSplit(path string) (string, string, error) { + splitPath := strings.Split(path, "/") + if len(splitPath) < 3 { + return "", "", errInvalidPath + } + + key := splitPath[2] + index := strings.Index(path, key) + len(key) + if index != len(path) { + return key, path[index+1:], nil //strip leading '/' + } + return key, "", nil +} diff --git a/core/commands/mount/write.go b/core/commands/mount/write.go new file mode 100644 index 00000000000..e0c30e70461 --- /dev/null +++ b/core/commands/mount/write.go @@ -0,0 +1,66 @@ +package fusemount + +import ( + "io" + + "github.com/billziss-gh/cgofuse/fuse" +) + +//TODO: [cleanup] move to another source file; no need for single function +//TODO: set c|mtim; see note on Read() +func (fs *FUSEIPFS) Write(path string, buff []byte, ofst int64, fh uint64) int { + fs.RLock() + log.Debugf("Write - Request {%d}[%X]%q", ofst, fh, path) + + if ofst < 0 { + log.Errorf("Write - Invalid offset {%d}[%X]%q", ofst, fh, path) + fs.RUnlock() + return -fuse.EINVAL + } + + h, err := fs.LookupFileHandle(fh) + if err != nil { + fs.RUnlock() + log.Errorf("Write - Lookup failed [%X]%q: %s", fh, path, err) + if err == errInvalidHandle { + return -fuse.EBADF + } + return -fuse.EIO + } + h.record.Lock() + fs.RUnlock() + defer h.record.Unlock() + fStat := h.record.Stat() + + oCur, err := h.io.Seek(0, io.SeekCurrent) + if err != nil { + log.Errorf("Write - cursor error: %s", err) + return -fuse.EIO + } + + written, err := h.io.Write(buff, ofst) + if err != nil { + log.Errorf("Write - error %q: %s", path, err) + /* Callers responsibility? + if err = pIo.Close(); err != nil { + log.Errorf("Write - Close error %s", err) + } + */ + return -fuse.EIO + } + + if oCur+int64(written) > fStat.Size { // write extended file + fStat.Size += int64(written) + } + + // when the same path has multiple open handles we need to swap the IO for them + errPair := fs.refreshFileSiblings(fh, h) + if len(errPair) != 0 { + for _, e := range errPair { + log.Errorf("Write - handle update failed for %X: %s", e.fhi, e.err) + } + return -fuse.EIO + } + + return written +} diff --git a/core/commands/mount_nofuse.go b/core/commands/mount_nofuse.go index 5fd13202032..f6f487f66db 100644 --- a/core/commands/mount_nofuse.go +++ b/core/commands/mount_nofuse.go @@ -1,4 +1,4 @@ -// +build !windows,nofuse +// +build nofuse package commands diff --git a/core/commands/mount_windows.go b/core/commands/mount_windows.go deleted file mode 100644 index f4a56a5a141..00000000000 --- a/core/commands/mount_windows.go +++ /dev/null @@ -1,19 +0,0 @@ -package commands - -import ( - "errors" - - cmds "gx/ipfs/QmX6AchyJgso1WNamTJMdxfzGiWuYu94K6tF9MJ66rRhAu/go-ipfs-cmds" - cmdkit "gx/ipfs/Qmde5VP1qUkyQXKCfmEUA7bP64V2HAptbJ7phuPp7jXWwg/go-ipfs-cmdkit" -) - -var MountCmd = &cmds.Command{ - Helptext: cmdkit.HelpText{ - Tagline: "Not yet implemented on Windows.", - ShortDescription: "Not yet implemented on Windows. :(", - }, - - Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { - return errors.New("Mount isn't compatible with Windows yet") - }, -} diff --git a/core/core.go b/core/core.go index c5fbfe4c1c2..7b9ca423bcf 100644 --- a/core/core.go +++ b/core/core.go @@ -21,9 +21,9 @@ import ( "time" version "github.com/ipfs/go-ipfs" + mount "github.com/ipfs/go-ipfs/core/commands/mount/interface" rp "github.com/ipfs/go-ipfs/exchange/reprovide" filestore "github.com/ipfs/go-ipfs/filestore" - mount "github.com/ipfs/go-ipfs/fuse/mount" namesys "github.com/ipfs/go-ipfs/namesys" ipnsrp "github.com/ipfs/go-ipfs/namesys/republisher" p2p "github.com/ipfs/go-ipfs/p2p" @@ -108,10 +108,10 @@ type IpfsNode struct { Repo repo.Repo // Local node - Pinning pin.Pinner // the pinning manager - Mounts Mounts // current mount state, if any. - PrivateKey ic.PrivKey // the local node's private Key - PNetFingerprint []byte // fingerprint of private network + Pinning pin.Pinner // the pinning manager + Mount mount.Interface // current mount state, if any. + PrivateKey ic.PrivKey // the local node's private Key + PNetFingerprint []byte // fingerprint of private network // Services Peerstore pstore.Peerstore // storage for other Peer instances @@ -149,14 +149,6 @@ type IpfsNode struct { localModeSet bool } -// Mounts defines what the node's mount state is. This should -// perhaps be moved to the daemon or mount. It's here because -// it needs to be accessible across daemon requests. -type Mounts struct { - Ipfs mount.Mount - Ipns mount.Mount -} - func (n *IpfsNode) startOnlineServices(ctx context.Context, routingOption RoutingOption, hostOption HostOption, do DiscoveryOption, pubsub, ipnsps, mplex bool) error { if n.PeerHost != nil { // already online. return errors.New("node already online") @@ -683,11 +675,8 @@ func (n *IpfsNode) teardown() error { closers = append(closers, n.Exchange) } - if n.Mounts.Ipfs != nil && !n.Mounts.Ipfs.IsActive() { - closers = append(closers, mount.Closer(n.Mounts.Ipfs)) - } - if n.Mounts.Ipns != nil && !n.Mounts.Ipns.IsActive() { - closers = append(closers, mount.Closer(n.Mounts.Ipns)) + if n.Mount != nil { + closers = append(closers, n.Mount) } if n.DHT != nil { diff --git a/core/coreapi/coreapi.go b/core/coreapi/coreapi.go index 5b379a11f53..a233c3f7aff 100644 --- a/core/coreapi/coreapi.go +++ b/core/coreapi/coreapi.go @@ -188,9 +188,6 @@ func (api *CoreAPI) WithOptions(opts ...options.ApiOption) (coreiface.CoreAPI, e } subApi.checkPublishAllowed = func() error { - if n.Mounts.Ipns != nil && n.Mounts.Ipns.IsActive() { - return errors.New("cannot manually publish while IPNS is mounted") - } return nil } diff --git a/fuse/ipns/common.go b/fuse/ipns/common.go deleted file mode 100644 index 32fc4987034..00000000000 --- a/fuse/ipns/common.go +++ /dev/null @@ -1,34 +0,0 @@ -package ipns - -import ( - "context" - - "github.com/ipfs/go-ipfs/core" - nsys "github.com/ipfs/go-ipfs/namesys" - path "gx/ipfs/QmQAgv6Gaoe2tQpcabqwKXKChp2MZ7i3UXv9DqTTaxCaTR/go-path" - ci "gx/ipfs/QmTW4SdgBWq9GjsBsHeUx8WuGxzhgzAf88UMH2w62PC8yK/go-libp2p-crypto" - ft "gx/ipfs/QmcYUTQ7tBZeH1CLsZM2S3xhMEZdvUgXvbjhpMsLDpk3oJ/go-unixfs" -) - -// InitializeKeyspace sets the ipns record for the given key to -// point to an empty directory. -func InitializeKeyspace(n *core.IpfsNode, key ci.PrivKey) error { - ctx, cancel := context.WithCancel(n.Context()) - defer cancel() - - emptyDir := ft.EmptyDirNode() - - err := n.Pinning.Pin(ctx, emptyDir, false) - if err != nil { - return err - } - - err = n.Pinning.Flush() - if err != nil { - return err - } - - pub := nsys.NewIpnsPublisher(n.Routing, n.Repo.Datastore()) - - return pub.Publish(ctx, key, path.FromCid(emptyDir.Cid())) -} diff --git a/fuse/ipns/ipns_test.go b/fuse/ipns/ipns_test.go deleted file mode 100644 index 1b6dcf800c0..00000000000 --- a/fuse/ipns/ipns_test.go +++ /dev/null @@ -1,486 +0,0 @@ -// +build !nofuse - -package ipns - -import ( - "bytes" - "context" - "fmt" - "io/ioutil" - mrand "math/rand" - "os" - "sync" - "testing" - - core "github.com/ipfs/go-ipfs/core" - - u "gx/ipfs/QmNohiVssaPw3KVLZik59DBVGTSm2dGvYT9eoXt5DQ36Yz/go-ipfs-util" - fstest "gx/ipfs/QmSJBsmLP1XMjv8hxYg2rUMdPDB7YUpyBo9idjrJ6Cmq6F/fuse/fs/fstestutil" - ci "gx/ipfs/QmWapVoHjtKhn4MhvKNoPTkJKADFGACfXPFnt7combwp5W/go-testutil/ci" - racedet "gx/ipfs/Qmf7HqcW7LtCi1W8y2bdx2eJpze74jkbKqpByxgXikdbLF/go-detect-race" -) - -func maybeSkipFuseTests(t *testing.T) { - if ci.NoFuse() { - t.Skip("Skipping FUSE tests") - } -} - -func randBytes(size int) []byte { - b := make([]byte, size) - u.NewTimeSeededRand().Read(b) - return b -} - -func mkdir(t *testing.T, path string) { - err := os.Mkdir(path, os.ModeDir) - if err != nil { - t.Fatal(err) - } -} - -func writeFile(t *testing.T, size int, path string) []byte { - return writeFileData(t, randBytes(size), path) -} - -func writeFileData(t *testing.T, data []byte, path string) []byte { - fi, err := os.Create(path) - if err != nil { - t.Fatal(err) - } - - n, err := fi.Write(data) - if err != nil { - t.Fatal(err) - } - - if n != len(data) { - t.Fatal("Didnt write proper amount!") - } - - err = fi.Close() - if err != nil { - t.Fatal(err) - } - - return data -} - -func verifyFile(t *testing.T, path string, wantData []byte) { - isData, err := ioutil.ReadFile(path) - if err != nil { - t.Fatal(err) - } - if len(isData) != len(wantData) { - t.Fatal("Data not equal - length check failed") - } - if !bytes.Equal(isData, wantData) { - t.Fatal("Data not equal") - } -} - -func checkExists(t *testing.T, path string) { - _, err := os.Stat(path) - if err != nil { - t.Fatal(err) - } -} - -func closeMount(mnt *mountWrap) { - if err := recover(); err != nil { - log.Error("Recovered panic") - log.Error(err) - } - mnt.Close() -} - -type mountWrap struct { - *fstest.Mount - Fs *FileSystem -} - -func (m *mountWrap) Close() error { - m.Fs.Destroy() - m.Mount.Close() - return nil -} - -func setupIpnsTest(t *testing.T, node *core.IpfsNode) (*core.IpfsNode, *mountWrap) { - maybeSkipFuseTests(t) - - var err error - if node == nil { - node, err = core.NewNode(context.Background(), nil) - if err != nil { - t.Fatal(err) - } - - err = InitializeKeyspace(node, node.PrivateKey) - if err != nil { - t.Fatal(err) - } - } - - fs, err := NewFileSystem(node, node.PrivateKey, "", "") - if err != nil { - t.Fatal(err) - } - mnt, err := fstest.MountedT(t, fs, nil) - if err != nil { - t.Fatal(err) - } - - return node, &mountWrap{ - Mount: mnt, - Fs: fs, - } -} - -func TestIpnsLocalLink(t *testing.T) { - nd, mnt := setupIpnsTest(t, nil) - defer mnt.Close() - name := mnt.Dir + "/local" - - checkExists(t, name) - - linksto, err := os.Readlink(name) - if err != nil { - t.Fatal(err) - } - - if linksto != nd.Identity.Pretty() { - t.Fatal("Link invalid") - } -} - -// Test writing a file and reading it back -func TestIpnsBasicIO(t *testing.T) { - if testing.Short() { - t.SkipNow() - } - nd, mnt := setupIpnsTest(t, nil) - defer closeMount(mnt) - - fname := mnt.Dir + "/local/testfile" - data := writeFile(t, 10, fname) - - rbuf, err := ioutil.ReadFile(fname) - if err != nil { - t.Fatal(err) - } - - if !bytes.Equal(rbuf, data) { - t.Fatal("Incorrect Read!") - } - - fname2 := mnt.Dir + "/" + nd.Identity.Pretty() + "/testfile" - rbuf, err = ioutil.ReadFile(fname2) - if err != nil { - t.Fatal(err) - } - - if !bytes.Equal(rbuf, data) { - t.Fatal("Incorrect Read!") - } -} - -// Test to make sure file changes persist over mounts of ipns -func TestFilePersistence(t *testing.T) { - if testing.Short() { - t.SkipNow() - } - node, mnt := setupIpnsTest(t, nil) - - fname := "/local/atestfile" - data := writeFile(t, 127, mnt.Dir+fname) - - mnt.Close() - - t.Log("Closed, opening new fs") - _, mnt = setupIpnsTest(t, node) - defer mnt.Close() - - rbuf, err := ioutil.ReadFile(mnt.Dir + fname) - if err != nil { - t.Fatal(err) - } - - if !bytes.Equal(rbuf, data) { - t.Fatalf("File data changed between mounts! sizes differ: %d != %d", len(data), len(rbuf)) - } -} - -func TestMultipleDirs(t *testing.T) { - node, mnt := setupIpnsTest(t, nil) - - t.Log("make a top level dir") - dir1 := "/local/test1" - mkdir(t, mnt.Dir+dir1) - - checkExists(t, mnt.Dir+dir1) - - t.Log("write a file in it") - data1 := writeFile(t, 4000, mnt.Dir+dir1+"/file1") - - verifyFile(t, mnt.Dir+dir1+"/file1", data1) - - t.Log("sub directory") - mkdir(t, mnt.Dir+dir1+"/dir2") - - checkExists(t, mnt.Dir+dir1+"/dir2") - - t.Log("file in that subdirectory") - data2 := writeFile(t, 5000, mnt.Dir+dir1+"/dir2/file2") - - verifyFile(t, mnt.Dir+dir1+"/dir2/file2", data2) - - mnt.Close() - t.Log("closing mount, then restarting") - - _, mnt = setupIpnsTest(t, node) - - checkExists(t, mnt.Dir+dir1) - - verifyFile(t, mnt.Dir+dir1+"/file1", data1) - - verifyFile(t, mnt.Dir+dir1+"/dir2/file2", data2) - mnt.Close() -} - -// Test to make sure the filesystem reports file sizes correctly -func TestFileSizeReporting(t *testing.T) { - if testing.Short() { - t.SkipNow() - } - _, mnt := setupIpnsTest(t, nil) - defer mnt.Close() - - fname := mnt.Dir + "/local/sizecheck" - data := writeFile(t, 5555, fname) - - finfo, err := os.Stat(fname) - if err != nil { - t.Fatal(err) - } - - if finfo.Size() != int64(len(data)) { - t.Fatal("Read incorrect size from stat!") - } -} - -// Test to make sure you cant create multiple entries with the same name -func TestDoubleEntryFailure(t *testing.T) { - if testing.Short() { - t.SkipNow() - } - _, mnt := setupIpnsTest(t, nil) - defer mnt.Close() - - dname := mnt.Dir + "/local/thisisadir" - err := os.Mkdir(dname, 0777) - if err != nil { - t.Fatal(err) - } - - err = os.Mkdir(dname, 0777) - if err == nil { - t.Fatal("Should have gotten error one creating new directory.") - } -} - -func TestAppendFile(t *testing.T) { - if testing.Short() { - t.SkipNow() - } - _, mnt := setupIpnsTest(t, nil) - defer mnt.Close() - - fname := mnt.Dir + "/local/file" - data := writeFile(t, 1300, fname) - - fi, err := os.OpenFile(fname, os.O_RDWR|os.O_APPEND, 0666) - if err != nil { - t.Fatal(err) - } - - nudata := randBytes(500) - - n, err := fi.Write(nudata) - if err != nil { - t.Fatal(err) - } - err = fi.Close() - if err != nil { - t.Fatal(err) - } - - if n != len(nudata) { - t.Fatal("Failed to write enough bytes.") - } - - data = append(data, nudata...) - - rbuf, err := ioutil.ReadFile(fname) - if err != nil { - t.Fatal(err) - } - if !bytes.Equal(rbuf, data) { - t.Fatal("Data inconsistent!") - } -} - -func TestConcurrentWrites(t *testing.T) { - if testing.Short() { - t.SkipNow() - } - _, mnt := setupIpnsTest(t, nil) - defer mnt.Close() - - nactors := 4 - filesPerActor := 400 - fileSize := 2000 - - data := make([][][]byte, nactors) - - if racedet.WithRace() { - nactors = 2 - filesPerActor = 50 - } - - wg := sync.WaitGroup{} - for i := 0; i < nactors; i++ { - data[i] = make([][]byte, filesPerActor) - wg.Add(1) - go func(n int) { - defer wg.Done() - for j := 0; j < filesPerActor; j++ { - out := writeFile(t, fileSize, mnt.Dir+fmt.Sprintf("/local/%dFILE%d", n, j)) - data[n][j] = out - } - }(i) - } - wg.Wait() - - for i := 0; i < nactors; i++ { - for j := 0; j < filesPerActor; j++ { - verifyFile(t, mnt.Dir+fmt.Sprintf("/local/%dFILE%d", i, j), data[i][j]) - } - } -} - -func TestFSThrash(t *testing.T) { - files := make(map[string][]byte) - - if testing.Short() { - t.SkipNow() - } - _, mnt := setupIpnsTest(t, nil) - defer mnt.Close() - - base := mnt.Dir + "/local" - dirs := []string{base} - dirlock := sync.RWMutex{} - filelock := sync.Mutex{} - - ndirWorkers := 2 - nfileWorkers := 2 - - ndirs := 100 - nfiles := 200 - - wg := sync.WaitGroup{} - - // Spawn off workers to make directories - for i := 0; i < ndirWorkers; i++ { - wg.Add(1) - go func(worker int) { - defer wg.Done() - for j := 0; j < ndirs; j++ { - dirlock.RLock() - n := mrand.Intn(len(dirs)) - dir := dirs[n] - dirlock.RUnlock() - - newDir := fmt.Sprintf("%s/dir%d-%d", dir, worker, j) - err := os.Mkdir(newDir, os.ModeDir) - if err != nil { - t.Fatal(err) - } - dirlock.Lock() - dirs = append(dirs, newDir) - dirlock.Unlock() - } - }(i) - } - - // Spawn off workers to make files - for i := 0; i < nfileWorkers; i++ { - wg.Add(1) - go func(worker int) { - defer wg.Done() - for j := 0; j < nfiles; j++ { - dirlock.RLock() - n := mrand.Intn(len(dirs)) - dir := dirs[n] - dirlock.RUnlock() - - newFileName := fmt.Sprintf("%s/file%d-%d", dir, worker, j) - - data := writeFile(t, 2000+mrand.Intn(5000), newFileName) - filelock.Lock() - files[newFileName] = data - filelock.Unlock() - } - }(i) - } - - wg.Wait() - for name, data := range files { - out, err := ioutil.ReadFile(name) - if err != nil { - t.Fatal(err) - } - - if !bytes.Equal(data, out) { - t.Fatal("Data didnt match") - } - } -} - -// Test writing a medium sized file one byte at a time -func TestMultiWrite(t *testing.T) { - - if testing.Short() { - t.SkipNow() - } - - _, mnt := setupIpnsTest(t, nil) - defer mnt.Close() - - fpath := mnt.Dir + "/local/file" - fi, err := os.Create(fpath) - if err != nil { - t.Fatal(err) - } - - data := randBytes(1001) - for i := 0; i < len(data); i++ { - n, err := fi.Write(data[i : i+1]) - if err != nil { - t.Fatal(err) - } - if n != 1 { - t.Fatal("Somehow wrote the wrong number of bytes! (n != 1)") - } - } - fi.Close() - - rbuf, err := ioutil.ReadFile(fpath) - if err != nil { - t.Fatal(err) - } - - if !bytes.Equal(rbuf, data) { - t.Fatal("File on disk did not match bytes written") - } -} diff --git a/fuse/ipns/ipns_unix.go b/fuse/ipns/ipns_unix.go deleted file mode 100644 index 8a69e83dfcb..00000000000 --- a/fuse/ipns/ipns_unix.go +++ /dev/null @@ -1,594 +0,0 @@ -// +build !nofuse - -// package fuse/ipns implements a fuse filesystem that interfaces -// with ipns, the naming system for ipfs. -package ipns - -import ( - "context" - "errors" - "fmt" - "io" - "os" - - core "github.com/ipfs/go-ipfs/core" - namesys "github.com/ipfs/go-ipfs/namesys" - dag "gx/ipfs/QmPJNbVw8o3ohC43ppSXyNXwYKsWShG4zygnirHptfbHri/go-merkledag" - path "gx/ipfs/QmQAgv6Gaoe2tQpcabqwKXKChp2MZ7i3UXv9DqTTaxCaTR/go-path" - ft "gx/ipfs/QmcYUTQ7tBZeH1CLsZM2S3xhMEZdvUgXvbjhpMsLDpk3oJ/go-unixfs" - - fuse "gx/ipfs/QmSJBsmLP1XMjv8hxYg2rUMdPDB7YUpyBo9idjrJ6Cmq6F/fuse" - fs "gx/ipfs/QmSJBsmLP1XMjv8hxYg2rUMdPDB7YUpyBo9idjrJ6Cmq6F/fuse/fs" - ci "gx/ipfs/QmTW4SdgBWq9GjsBsHeUx8WuGxzhgzAf88UMH2w62PC8yK/go-libp2p-crypto" - cid "gx/ipfs/QmTbxNB1NwDesLmKTscr4udL2tVP7MaxvXnD1D9yX7g3PN/go-cid" - peer "gx/ipfs/QmYVXrKrKHDC9FobgmcmshCDyWwdrfwfanNQN4oxJ9Fk3h/go-libp2p-peer" - mfs "gx/ipfs/Qmb74fRYPgpjYzoBV7PAVNmP3DQaRrh8dHdKE4PwnF3cRx/go-mfs" - logging "gx/ipfs/QmbkT7eMTyXfpeyB3ZMxxcxg7XH8t6uXp49jqzz4HB7BGF/go-log" -) - -func init() { - if os.Getenv("IPFS_FUSE_DEBUG") != "" { - fuse.Debug = func(msg interface{}) { - fmt.Println(msg) - } - } -} - -var log = logging.Logger("fuse/ipns") - -// FileSystem is the readwrite IPNS Fuse Filesystem. -type FileSystem struct { - Ipfs *core.IpfsNode - RootNode *Root -} - -// NewFileSystem constructs new fs using given core.IpfsNode instance. -func NewFileSystem(ipfs *core.IpfsNode, sk ci.PrivKey, ipfspath, ipnspath string) (*FileSystem, error) { - - kmap := map[string]ci.PrivKey{ - "local": sk, - } - root, err := CreateRoot(ipfs, kmap, ipfspath, ipnspath) - if err != nil { - return nil, err - } - - return &FileSystem{Ipfs: ipfs, RootNode: root}, nil -} - -// Root constructs the Root of the filesystem, a Root object. -func (f *FileSystem) Root() (fs.Node, error) { - log.Debug("filesystem, get root") - return f.RootNode, nil -} - -func (f *FileSystem) Destroy() { - err := f.RootNode.Close() - if err != nil { - log.Errorf("Error Shutting Down Filesystem: %s\n", err) - } -} - -// Root is the root object of the filesystem tree. -type Root struct { - Ipfs *core.IpfsNode - Keys map[string]ci.PrivKey - - // Used for symlinking into ipfs - IpfsRoot string - IpnsRoot string - LocalDirs map[string]fs.Node - Roots map[string]*keyRoot - - LocalLinks map[string]*Link -} - -func ipnsPubFunc(ipfs *core.IpfsNode, k ci.PrivKey) mfs.PubFunc { - return func(ctx context.Context, c cid.Cid) error { - return ipfs.Namesys.Publish(ctx, k, path.FromCid(c)) - } -} - -func loadRoot(ctx context.Context, rt *keyRoot, ipfs *core.IpfsNode, name string) (fs.Node, error) { - p, err := path.ParsePath("/ipns/" + name) - if err != nil { - log.Errorf("mkpath %s: %s", name, err) - return nil, err - } - - node, err := core.Resolve(ctx, ipfs.Namesys, ipfs.Resolver, p) - switch err { - case nil: - case namesys.ErrResolveFailed: - node = ft.EmptyDirNode() - default: - log.Errorf("looking up %s: %s", p, err) - return nil, err - } - - pbnode, ok := node.(*dag.ProtoNode) - if !ok { - return nil, dag.ErrNotProtobuf - } - - root, err := mfs.NewRoot(ctx, ipfs.DAG, pbnode, ipnsPubFunc(ipfs, rt.k)) - if err != nil { - return nil, err - } - - rt.root = root - - return &Directory{dir: root.GetDirectory()}, nil -} - -type keyRoot struct { - k ci.PrivKey - alias string - root *mfs.Root -} - -func CreateRoot(ipfs *core.IpfsNode, keys map[string]ci.PrivKey, ipfspath, ipnspath string) (*Root, error) { - ldirs := make(map[string]fs.Node) - roots := make(map[string]*keyRoot) - links := make(map[string]*Link) - for alias, k := range keys { - pid, err := peer.IDFromPrivateKey(k) - if err != nil { - return nil, err - } - name := pid.Pretty() - - kr := &keyRoot{k: k, alias: alias} - fsn, err := loadRoot(ipfs.Context(), kr, ipfs, name) - if err != nil { - return nil, err - } - - roots[name] = kr - ldirs[name] = fsn - - // set up alias symlink - links[alias] = &Link{ - Target: name, - } - } - - return &Root{ - Ipfs: ipfs, - IpfsRoot: ipfspath, - IpnsRoot: ipnspath, - Keys: keys, - LocalDirs: ldirs, - LocalLinks: links, - Roots: roots, - }, nil -} - -// Attr returns file attributes. -func (*Root) Attr(ctx context.Context, a *fuse.Attr) error { - log.Debug("Root Attr") - a.Mode = os.ModeDir | 0111 // -rw+x - return nil -} - -// Lookup performs a lookup under this node. -func (s *Root) Lookup(ctx context.Context, name string) (fs.Node, error) { - switch name { - case "mach_kernel", ".hidden", "._.": - // Just quiet some log noise on OS X. - return nil, fuse.ENOENT - } - - if lnk, ok := s.LocalLinks[name]; ok { - return lnk, nil - } - - nd, ok := s.LocalDirs[name] - if ok { - switch nd := nd.(type) { - case *Directory: - return nd, nil - case *FileNode: - return nd, nil - default: - return nil, fuse.EIO - } - } - - // other links go through ipns resolution and are symlinked into the ipfs mountpoint - ipnsName := "/ipns/" + name - resolved, err := s.Ipfs.Namesys.Resolve(s.Ipfs.Context(), ipnsName) - if err != nil { - log.Warningf("ipns: namesys resolve error: %s", err) - return nil, fuse.ENOENT - } - - segments := resolved.Segments() - if segments[0] == "ipfs" { - p := path.Join(resolved.Segments()[1:]) - return &Link{s.IpfsRoot + "/" + p}, nil - } - - log.Error("Invalid path.Path: ", resolved) - return nil, errors.New("invalid path from ipns record") -} - -func (r *Root) Close() error { - for _, mr := range r.Roots { - err := mr.root.Close() - if err != nil { - return err - } - } - return nil -} - -// Forget is called when the filesystem is unmounted. probably. -// see comments here: http://godoc.org/bazil.org/fuse/fs#FSDestroyer -func (r *Root) Forget() { - err := r.Close() - if err != nil { - log.Error(err) - } -} - -// ReadDirAll reads a particular directory. Will show locally available keys -// as well as a symlink to the peerID key -func (r *Root) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { - log.Debug("Root ReadDirAll") - - var listing []fuse.Dirent - for alias, k := range r.Keys { - pid, err := peer.IDFromPrivateKey(k) - if err != nil { - continue - } - ent := fuse.Dirent{ - Name: pid.Pretty(), - Type: fuse.DT_Dir, - } - link := fuse.Dirent{ - Name: alias, - Type: fuse.DT_Link, - } - listing = append(listing, ent, link) - } - return listing, nil -} - -// Directory is wrapper over an mfs directory to satisfy the fuse fs interface -type Directory struct { - dir *mfs.Directory -} - -type FileNode struct { - fi *mfs.File -} - -// File is wrapper over an mfs file to satisfy the fuse fs interface -type File struct { - fi mfs.FileDescriptor -} - -// Attr returns the attributes of a given node. -func (d *Directory) Attr(ctx context.Context, a *fuse.Attr) error { - log.Debug("Directory Attr") - a.Mode = os.ModeDir | 0555 - a.Uid = uint32(os.Getuid()) - a.Gid = uint32(os.Getgid()) - return nil -} - -// Attr returns the attributes of a given node. -func (fi *FileNode) Attr(ctx context.Context, a *fuse.Attr) error { - log.Debug("File Attr") - size, err := fi.fi.Size() - if err != nil { - // In this case, the dag node in question may not be unixfs - return fmt.Errorf("fuse/ipns: failed to get file.Size(): %s", err) - } - a.Mode = os.FileMode(0666) - a.Size = uint64(size) - a.Uid = uint32(os.Getuid()) - a.Gid = uint32(os.Getgid()) - return nil -} - -// Lookup performs a lookup under this node. -func (s *Directory) Lookup(ctx context.Context, name string) (fs.Node, error) { - child, err := s.dir.Child(name) - if err != nil { - // todo: make this error more versatile. - return nil, fuse.ENOENT - } - - switch child := child.(type) { - case *mfs.Directory: - return &Directory{dir: child}, nil - case *mfs.File: - return &FileNode{fi: child}, nil - default: - // NB: if this happens, we do not want to continue, unpredictable behaviour - // may occur. - panic("invalid type found under directory. programmer error.") - } -} - -// ReadDirAll reads the link structure as directory entries -func (dir *Directory) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { - var entries []fuse.Dirent - listing, err := dir.dir.List(ctx) - if err != nil { - return nil, err - } - for _, entry := range listing { - dirent := fuse.Dirent{Name: entry.Name} - - switch mfs.NodeType(entry.Type) { - case mfs.TDir: - dirent.Type = fuse.DT_Dir - case mfs.TFile: - dirent.Type = fuse.DT_File - } - - entries = append(entries, dirent) - } - - if len(entries) > 0 { - return entries, nil - } - return nil, fuse.ENOENT -} - -func (fi *File) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error { - _, err := fi.fi.Seek(req.Offset, io.SeekStart) - if err != nil { - return err - } - - fisize, err := fi.fi.Size() - if err != nil { - return err - } - - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - - readsize := min(req.Size, int(fisize-req.Offset)) - n, err := fi.fi.CtxReadFull(ctx, resp.Data[:readsize]) - resp.Data = resp.Data[:n] - return err -} - -func (fi *File) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.WriteResponse) error { - // TODO: at some point, ensure that WriteAt here respects the context - wrote, err := fi.fi.WriteAt(req.Data, req.Offset) - if err != nil { - return err - } - resp.Size = wrote - return nil -} - -func (fi *File) Flush(ctx context.Context, req *fuse.FlushRequest) error { - errs := make(chan error, 1) - go func() { - errs <- fi.fi.Flush() - }() - select { - case err := <-errs: - return err - case <-ctx.Done(): - return ctx.Err() - } -} - -func (fi *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) error { - if req.Valid.Size() { - cursize, err := fi.fi.Size() - if err != nil { - return err - } - if cursize != int64(req.Size) { - err := fi.fi.Truncate(int64(req.Size)) - if err != nil { - return err - } - } - } - return nil -} - -// Fsync flushes the content in the file to disk. -func (fi *FileNode) Fsync(ctx context.Context, req *fuse.FsyncRequest) error { - // This needs to perform a *full* flush because, in MFS, a write isn't - // persisted until the root is updated. - errs := make(chan error, 1) - go func() { - errs <- fi.fi.Flush() - }() - select { - case err := <-errs: - return err - case <-ctx.Done(): - return ctx.Err() - } -} - -func (fi *File) Forget() { - // TODO(steb): this seems like a place where we should be *uncaching*, not flushing. - err := fi.fi.Flush() - if err != nil { - log.Debug("forget file error: ", err) - } -} - -func (dir *Directory) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) { - child, err := dir.dir.Mkdir(req.Name) - if err != nil { - return nil, err - } - - return &Directory{dir: child}, nil -} - -func (fi *FileNode) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) { - fd, err := fi.fi.Open(mfs.Flags{ - Read: req.Flags.IsReadOnly() || req.Flags.IsReadWrite(), - Write: req.Flags.IsWriteOnly() || req.Flags.IsReadWrite(), - Sync: true, - }) - if err != nil { - return nil, err - } - - if req.Flags&fuse.OpenTruncate != 0 { - if req.Flags.IsReadOnly() { - log.Error("tried to open a readonly file with truncate") - return nil, fuse.ENOTSUP - } - log.Info("Need to truncate file!") - err := fd.Truncate(0) - if err != nil { - return nil, err - } - } else if req.Flags&fuse.OpenAppend != 0 { - log.Info("Need to append to file!") - if req.Flags.IsReadOnly() { - log.Error("tried to open a readonly file with append") - return nil, fuse.ENOTSUP - } - - _, err := fd.Seek(0, io.SeekEnd) - if err != nil { - log.Error("seek reset failed: ", err) - return nil, err - } - } - - return &File{fi: fd}, nil -} - -func (fi *File) Release(ctx context.Context, req *fuse.ReleaseRequest) error { - return fi.fi.Close() -} - -func (dir *Directory) Create(ctx context.Context, req *fuse.CreateRequest, resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) { - // New 'empty' file - nd := dag.NodeWithData(ft.FilePBData(nil, 0)) - err := dir.dir.AddChild(req.Name, nd) - if err != nil { - return nil, nil, err - } - - child, err := dir.dir.Child(req.Name) - if err != nil { - return nil, nil, err - } - - fi, ok := child.(*mfs.File) - if !ok { - return nil, nil, errors.New("child creation failed") - } - - nodechild := &FileNode{fi: fi} - - fd, err := fi.Open(mfs.Flags{ - Read: req.Flags.IsReadOnly() || req.Flags.IsReadWrite(), - Write: req.Flags.IsWriteOnly() || req.Flags.IsReadWrite(), - Sync: true, - }) - if err != nil { - return nil, nil, err - } - - return nodechild, &File{fi: fd}, nil -} - -func (dir *Directory) Remove(ctx context.Context, req *fuse.RemoveRequest) error { - err := dir.dir.Unlink(req.Name) - if err != nil { - return fuse.ENOENT - } - return nil -} - -// Rename implements NodeRenamer -func (dir *Directory) Rename(ctx context.Context, req *fuse.RenameRequest, newDir fs.Node) error { - cur, err := dir.dir.Child(req.OldName) - if err != nil { - return err - } - - err = dir.dir.Unlink(req.OldName) - if err != nil { - return err - } - - switch newDir := newDir.(type) { - case *Directory: - nd, err := cur.GetNode() - if err != nil { - return err - } - - err = newDir.dir.AddChild(req.NewName, nd) - if err != nil { - return err - } - case *FileNode: - log.Error("Cannot move node into a file!") - return fuse.EPERM - default: - log.Error("Unknown node type for rename target dir!") - return errors.New("unknown fs node type") - } - return nil -} - -func min(a, b int) int { - if a < b { - return a - } - return b -} - -// to check that out Node implements all the interfaces we want -type ipnsRoot interface { - fs.Node - fs.HandleReadDirAller - fs.NodeStringLookuper -} - -var _ ipnsRoot = (*Root)(nil) - -type ipnsDirectory interface { - fs.HandleReadDirAller - fs.Node - fs.NodeCreater - fs.NodeMkdirer - fs.NodeRemover - fs.NodeRenamer - fs.NodeStringLookuper -} - -var _ ipnsDirectory = (*Directory)(nil) - -type ipnsFile interface { - fs.HandleFlusher - fs.HandleReader - fs.HandleWriter - fs.HandleReleaser -} - -type ipnsFileNode interface { - fs.Node - fs.NodeFsyncer - fs.NodeOpener -} - -var _ ipnsFileNode = (*FileNode)(nil) -var _ ipnsFile = (*File)(nil) diff --git a/fuse/ipns/link_unix.go b/fuse/ipns/link_unix.go deleted file mode 100644 index e89112f3255..00000000000 --- a/fuse/ipns/link_unix.go +++ /dev/null @@ -1,28 +0,0 @@ -// +build !nofuse - -package ipns - -import ( - "context" - "os" - - "gx/ipfs/QmSJBsmLP1XMjv8hxYg2rUMdPDB7YUpyBo9idjrJ6Cmq6F/fuse" - "gx/ipfs/QmSJBsmLP1XMjv8hxYg2rUMdPDB7YUpyBo9idjrJ6Cmq6F/fuse/fs" -) - -type Link struct { - Target string -} - -func (l *Link) Attr(ctx context.Context, a *fuse.Attr) error { - log.Debug("Link attr.") - a.Mode = os.ModeSymlink | 0555 - return nil -} - -func (l *Link) Readlink(ctx context.Context, req *fuse.ReadlinkRequest) (string, error) { - log.Debugf("ReadLink: %s", l.Target) - return l.Target, nil -} - -var _ fs.NodeReadlinker = (*Link)(nil) diff --git a/fuse/ipns/mount_unix.go b/fuse/ipns/mount_unix.go deleted file mode 100644 index e6b551b2dbe..00000000000 --- a/fuse/ipns/mount_unix.go +++ /dev/null @@ -1,26 +0,0 @@ -// +build linux darwin freebsd netbsd openbsd -// +build !nofuse - -package ipns - -import ( - core "github.com/ipfs/go-ipfs/core" - mount "github.com/ipfs/go-ipfs/fuse/mount" -) - -// Mount mounts ipns at a given location, and returns a mount.Mount instance. -func Mount(ipfs *core.IpfsNode, ipnsmp, ipfsmp string) (mount.Mount, error) { - cfg, err := ipfs.Repo.Config() - if err != nil { - return nil, err - } - - allow_other := cfg.Mounts.FuseAllowOther - - fsys, err := NewFileSystem(ipfs, ipfs.PrivateKey, ipfsmp, ipnsmp) - if err != nil { - return nil, err - } - - return mount.NewMount(ipfs.Process(), fsys, ipnsmp, allow_other) -} diff --git a/fuse/mount/fuse.go b/fuse/mount/fuse.go deleted file mode 100644 index 9167159b73c..00000000000 --- a/fuse/mount/fuse.go +++ /dev/null @@ -1,162 +0,0 @@ -// +build !nofuse -// +build !windows - -package mount - -import ( - "errors" - "fmt" - "sync" - "time" - - "gx/ipfs/QmSF8fPo3jgVBAy8fpdjjYqgG87dkJgUprRBHRd2tmfgpP/goprocess" - "gx/ipfs/QmSJBsmLP1XMjv8hxYg2rUMdPDB7YUpyBo9idjrJ6Cmq6F/fuse" - "gx/ipfs/QmSJBsmLP1XMjv8hxYg2rUMdPDB7YUpyBo9idjrJ6Cmq6F/fuse/fs" -) - -var ErrNotMounted = errors.New("not mounted") - -// mount implements go-ipfs/fuse/mount -type mount struct { - mpoint string - filesys fs.FS - fuseConn *fuse.Conn - - active bool - activeLock *sync.RWMutex - - proc goprocess.Process -} - -// Mount mounts a fuse fs.FS at a given location, and returns a Mount instance. -// parent is a ContextGroup to bind the mount's ContextGroup to. -func NewMount(p goprocess.Process, fsys fs.FS, mountpoint string, allow_other bool) (Mount, error) { - var conn *fuse.Conn - var err error - - if allow_other { - conn, err = fuse.Mount(mountpoint, fuse.AllowOther()) - } else { - conn, err = fuse.Mount(mountpoint) - } - - if err != nil { - return nil, err - } - - m := &mount{ - mpoint: mountpoint, - fuseConn: conn, - filesys: fsys, - active: false, - activeLock: &sync.RWMutex{}, - proc: goprocess.WithParent(p), // link it to parent. - } - m.proc.SetTeardown(m.unmount) - - // launch the mounting process. - if err := m.mount(); err != nil { - m.Unmount() // just in case. - return nil, err - } - - return m, nil -} - -func (m *mount) mount() error { - log.Infof("Mounting %s", m.MountPoint()) - - errs := make(chan error, 1) - go func() { - // fs.Serve blocks until the filesystem is unmounted. - err := fs.Serve(m.fuseConn, m.filesys) - log.Debugf("%s is unmounted", m.MountPoint()) - if err != nil { - log.Debugf("fs.Serve returned (%s)", err) - errs <- err - } - m.setActive(false) - }() - - // wait for the mount process to be done, or timed out. - select { - case <-time.After(MountTimeout): - return fmt.Errorf("mounting %s timed out", m.MountPoint()) - case err := <-errs: - return err - case <-m.fuseConn.Ready: - } - - // check if the mount process has an error to report - if err := m.fuseConn.MountError; err != nil { - return err - } - - m.setActive(true) - - log.Infof("Mounted %s", m.MountPoint()) - return nil -} - -// umount is called exactly once to unmount this service. -// note that closing the connection will not always unmount -// properly. If that happens, we bring out the big guns -// (mount.ForceUnmountManyTimes, exec unmount). -func (m *mount) unmount() error { - log.Infof("Unmounting %s", m.MountPoint()) - - // try unmounting with fuse lib - err := fuse.Unmount(m.MountPoint()) - if err == nil { - m.setActive(false) - return nil - } - log.Warningf("fuse unmount err: %s", err) - - // try closing the fuseConn - err = m.fuseConn.Close() - if err == nil { - m.setActive(false) - return nil - } - log.Warningf("fuse conn error: %s", err) - - // try mount.ForceUnmountManyTimes - if err := ForceUnmountManyTimes(m, 10); err != nil { - return err - } - - log.Infof("Seemingly unmounted %s", m.MountPoint()) - m.setActive(false) - return nil -} - -func (m *mount) Process() goprocess.Process { - return m.proc -} - -func (m *mount) MountPoint() string { - return m.mpoint -} - -func (m *mount) Unmount() error { - if !m.IsActive() { - return ErrNotMounted - } - - // call Process Close(), which calls unmount() exactly once. - return m.proc.Close() -} - -func (m *mount) IsActive() bool { - m.activeLock.RLock() - defer m.activeLock.RUnlock() - - return m.active -} - -func (m *mount) setActive(a bool) { - m.activeLock.Lock() - m.active = a - m.activeLock.Unlock() -} diff --git a/fuse/mount/mount.go b/fuse/mount/mount.go deleted file mode 100644 index bb30412763b..00000000000 --- a/fuse/mount/mount.go +++ /dev/null @@ -1,107 +0,0 @@ -// package mount provides a simple abstraction around a mount point -package mount - -import ( - "fmt" - "io" - "os/exec" - "runtime" - "time" - - goprocess "gx/ipfs/QmSF8fPo3jgVBAy8fpdjjYqgG87dkJgUprRBHRd2tmfgpP/goprocess" - logging "gx/ipfs/QmbkT7eMTyXfpeyB3ZMxxcxg7XH8t6uXp49jqzz4HB7BGF/go-log" -) - -var log = logging.Logger("mount") - -var MountTimeout = time.Second * 5 - -// Mount represents a filesystem mount -type Mount interface { - // MountPoint is the path at which this mount is mounted - MountPoint() string - - // Unmounts the mount - Unmount() error - - // Checks if the mount is still active. - IsActive() bool - - // Process returns the mount's Process to be able to link it - // to other processes. Unmount upon closing. - Process() goprocess.Process -} - -// ForceUnmount attempts to forcibly unmount a given mount. -// It does so by calling diskutil or fusermount directly. -func ForceUnmount(m Mount) error { - point := m.MountPoint() - log.Warningf("Force-Unmounting %s...", point) - - cmd, err := UnmountCmd(point) - if err != nil { - return err - } - - errc := make(chan error, 1) - go func() { - defer close(errc) - - // try vanilla unmount first. - if err := exec.Command("umount", point).Run(); err == nil { - return - } - - // retry to unmount with the fallback cmd - errc <- cmd.Run() - }() - - select { - case <-time.After(7 * time.Second): - return fmt.Errorf("umount timeout") - case err := <-errc: - return err - } -} - -// UnmountCmd creates an exec.Cmd that is GOOS-specific -// for unmount a FUSE mount -func UnmountCmd(point string) (*exec.Cmd, error) { - switch runtime.GOOS { - case "darwin": - return exec.Command("diskutil", "umount", "force", point), nil - case "linux": - return exec.Command("fusermount", "-u", point), nil - default: - return nil, fmt.Errorf("unmount: unimplemented") - } -} - -// ForceUnmountManyTimes attempts to forcibly unmount a given mount, -// many times. It does so by calling diskutil or fusermount directly. -// Attempts a given number of times. -func ForceUnmountManyTimes(m Mount, attempts int) error { - var err error - for i := 0; i < attempts; i++ { - err = ForceUnmount(m) - if err == nil { - return err - } - - <-time.After(time.Millisecond * 500) - } - return fmt.Errorf("unmount %s failed after 10 seconds of trying", m.MountPoint()) -} - -type closer struct { - M Mount -} - -func (c *closer) Close() error { - log.Warning(" (c *closer) Close(),", c.M.MountPoint()) - return c.M.Unmount() -} - -func Closer(m Mount) io.Closer { - return &closer{m} -} diff --git a/fuse/node/mount_darwin.go b/fuse/node/mount_darwin.go deleted file mode 100644 index 512eb1f1299..00000000000 --- a/fuse/node/mount_darwin.go +++ /dev/null @@ -1,243 +0,0 @@ -// +build !nofuse - -package node - -import ( - "bytes" - "fmt" - "os/exec" - "runtime" - "strings" - - core "github.com/ipfs/go-ipfs/core" - - unix "gx/ipfs/QmVGjyM9i2msKvLXwh9VosCTgP4mL91kC7hDmqnwTTx6Hu/sys/unix" - "gx/ipfs/QmYRGECuvQnRX73fcvPnGbYijBcGN2HbKZQ7jh26qmLiHG/semver" -) - -func init() { - // this is a hack, but until we need to do it another way, this works. - platformFuseChecks = darwinFuseCheckVersion -} - -// dontCheckOSXFUSEConfigKey is a key used to let the user tell us to -// skip fuse checks. -var dontCheckOSXFUSEConfigKey = "DontCheckOSXFUSE" - -// fuseVersionPkg is the go pkg url for fuse-version -var fuseVersionPkg = "github.com/jbenet/go-fuse-version/fuse-version" - -// errStrFuseRequired is returned when we're sure the user does not have fuse. -var errStrFuseRequired = `OSXFUSE not found. - -OSXFUSE is required to mount, please install it. -NOTE: Version 2.7.2 or higher required; prior versions are known to kernel panic! -It is recommended you install it from the OSXFUSE website: - - http://osxfuse.github.io/ - -For more help, see: - - https://github.com/ipfs/go-ipfs/issues/177 -` - -// errStrNoFuseHeaders is included in the output of `go get ` if there -// are no fuse headers. this means they dont have OSXFUSE installed. -var errStrNoFuseHeaders = "no such file or directory: '/usr/local/lib/libosxfuse.dylib'" - -var errStrUpgradeFuse = `OSXFUSE version %s not supported. - -OSXFUSE versions <2.7.2 are known to cause kernel panics! -Please upgrade to the latest OSXFUSE version. -It is recommended you install it from the OSXFUSE website: - - http://osxfuse.github.io/ - -For more help, see: - - https://github.com/ipfs/go-ipfs/issues/177 -` - -var errStrNeedFuseVersion = `unable to check fuse version. - -Dear User, - -Before mounting, we must check your version of OSXFUSE. We are protecting -you from a nasty kernel panic we found in OSXFUSE versions <2.7.2.[1]. To -make matters worse, it's harder than it should be to check whether you have -the right version installed...[2]. We've automated the process with the -help of a little tool. We tried to install it, but something went wrong[3]. -Please install it yourself by running: - - go get %s - -You can also stop ipfs from running these checks and use whatever OSXFUSE -version you have by running: - - ipfs --json config %s true - -[1]: https://github.com/ipfs/go-ipfs/issues/177 -[2]: https://github.com/ipfs/go-ipfs/pull/533 -[3]: %s -` - -var errStrFailedToRunFuseVersion = `unable to check fuse version. - -Dear User, - -Before mounting, we must check your version of OSXFUSE. We are protecting -you from a nasty kernel panic we found in OSXFUSE versions <2.7.2.[1]. To -make matters worse, it's harder than it should be to check whether you have -the right version installed...[2]. We've automated the process with the -help of a little tool. We tried to run it, but something went wrong[3]. -Please, try to run it yourself with: - - go get %s - fuse-version - -You should see something like this: - - > fuse-version - fuse-version -only agent - OSXFUSE.AgentVersion: 2.7.3 - -Just make sure the number is 2.7.2 or higher. You can then stop ipfs from -trying to run these checks with: - - ipfs config %s true - -[1]: https://github.com/ipfs/go-ipfs/issues/177 -[2]: https://github.com/ipfs/go-ipfs/pull/533 -[3]: %s -` - -var errStrFixConfig = `config key invalid: %s %v -You may be able to get this error to go away by setting it again: - - ipfs config %s true - -Either way, please tell us at: http://github.com/ipfs/go-ipfs/issues -` - -func darwinFuseCheckVersion(node *core.IpfsNode) error { - // on OSX, check FUSE version. - if runtime.GOOS != "darwin" { - return nil - } - - ov, errGFV := tryGFV() - if errGFV != nil { - // if we failed AND the user has told us to ignore the check we - // continue. this is in case fuse-version breaks or the user cannot - // install it, but is sure their fuse version will work. - if skip, err := userAskedToSkipFuseCheck(node); err != nil { - return err - } else if skip { - return nil // user told us not to check version... ok.... - } else { - return errGFV - } - } - - log.Debug("mount: osxfuse version:", ov) - - min := semver.MustParse("2.7.2") - curr, err := semver.Make(ov) - if err != nil { - return err - } - - if curr.LT(min) { - return fmt.Errorf(errStrUpgradeFuse, ov) - } - return nil -} - -func tryGFV() (string, error) { - // first try sysctl. it may work! - ov, err := trySysctl() - if err == nil { - return ov, nil - } - log.Debug(err) - - return tryGFVFromFuseVersion() -} - -func trySysctl() (string, error) { - v, err := unix.Sysctl("osxfuse.version.number") - if err != nil { - log.Debug("mount: sysctl osxfuse.version.number:", "failed") - return "", err - } - log.Debug("mount: sysctl osxfuse.version.number:", v) - return v, nil -} - -func tryGFVFromFuseVersion() (string, error) { - if err := ensureFuseVersionIsInstalled(); err != nil { - return "", err - } - - cmd := exec.Command("fuse-version", "-q", "-only", "agent", "-s", "OSXFUSE") - out := new(bytes.Buffer) - cmd.Stdout = out - if err := cmd.Run(); err != nil { - return "", fmt.Errorf(errStrFailedToRunFuseVersion, fuseVersionPkg, dontCheckOSXFUSEConfigKey, err) - } - - return out.String(), nil -} - -func ensureFuseVersionIsInstalled() error { - // see if fuse-version is there - if _, err := exec.LookPath("fuse-version"); err == nil { - return nil // got it! - } - - // try installing it... - log.Debug("fuse-version: no fuse-version. attempting to install.") - cmd := exec.Command("go", "get", "github.com/jbenet/go-fuse-version/fuse-version") - cmdout := new(bytes.Buffer) - cmd.Stdout = cmdout - cmd.Stderr = cmdout - if err := cmd.Run(); err != nil { - // Ok, install fuse-version failed. is it they dont have fuse? - cmdoutstr := cmdout.String() - if strings.Contains(cmdoutstr, errStrNoFuseHeaders) { - // yes! it is! they dont have fuse! - return fmt.Errorf(errStrFuseRequired) - } - - log.Debug("fuse-version: failed to install.") - s := err.Error() + "\n" + cmdoutstr - return fmt.Errorf(errStrNeedFuseVersion, fuseVersionPkg, dontCheckOSXFUSEConfigKey, s) - } - - // ok, try again... - if _, err := exec.LookPath("fuse-version"); err != nil { - log.Debug("fuse-version: failed to install?") - return fmt.Errorf(errStrNeedFuseVersion, fuseVersionPkg, dontCheckOSXFUSEConfigKey, err) - } - - log.Debug("fuse-version: install success") - return nil -} - -func userAskedToSkipFuseCheck(node *core.IpfsNode) (skip bool, err error) { - val, err := node.Repo.GetConfigKey(dontCheckOSXFUSEConfigKey) - if err != nil { - return false, nil // failed to get config value. dont skip check. - } - - switch val := val.(type) { - case string: - return val == "true", nil - case bool: - return val, nil - default: - // got config value, but it's invalid... dont skip check, ask the user to fix it... - return false, fmt.Errorf(errStrFixConfig, dontCheckOSXFUSEConfigKey, val, - dontCheckOSXFUSEConfigKey) - } -} diff --git a/fuse/node/mount_nofuse.go b/fuse/node/mount_nofuse.go deleted file mode 100644 index 7f824ef3e12..00000000000 --- a/fuse/node/mount_nofuse.go +++ /dev/null @@ -1,13 +0,0 @@ -// +build !windows,nofuse - -package node - -import ( - "errors" - - core "github.com/ipfs/go-ipfs/core" -) - -func Mount(node *core.IpfsNode, fsdir, nsdir string) error { - return errors.New("not compiled in") -} diff --git a/fuse/node/mount_test.go b/fuse/node/mount_test.go deleted file mode 100644 index 366a9ce8749..00000000000 --- a/fuse/node/mount_test.go +++ /dev/null @@ -1,86 +0,0 @@ -// +build !nofuse - -package node - -import ( - "io/ioutil" - "os" - "testing" - "time" - - "context" - - core "github.com/ipfs/go-ipfs/core" - ipns "github.com/ipfs/go-ipfs/fuse/ipns" - mount "github.com/ipfs/go-ipfs/fuse/mount" - - ci "gx/ipfs/QmWapVoHjtKhn4MhvKNoPTkJKADFGACfXPFnt7combwp5W/go-testutil/ci" -) - -func maybeSkipFuseTests(t *testing.T) { - if ci.NoFuse() { - t.Skip("Skipping FUSE tests") - } -} - -func mkdir(t *testing.T, path string) { - err := os.Mkdir(path, os.ModeDir|os.ModePerm) - if err != nil { - t.Fatal(err) - } -} - -// Test externally unmounting, then trying to unmount in code -func TestExternalUnmount(t *testing.T) { - if testing.Short() { - t.SkipNow() - } - - // TODO: needed? - maybeSkipFuseTests(t) - - node, err := core.NewNode(context.Background(), nil) - if err != nil { - t.Fatal(err) - } - - err = ipns.InitializeKeyspace(node, node.PrivateKey) - if err != nil { - t.Fatal(err) - } - - // get the test dir paths (/tmp/fusetestXXXX) - dir, err := ioutil.TempDir("", "fusetest") - if err != nil { - t.Fatal(err) - } - - ipfsDir := dir + "/ipfs" - ipnsDir := dir + "/ipns" - mkdir(t, ipfsDir) - mkdir(t, ipnsDir) - - err = Mount(node, ipfsDir, ipnsDir) - if err != nil { - t.Fatal(err) - } - - // Run shell command to externally unmount the directory - cmd, err := mount.UnmountCmd(ipfsDir) - if err != nil { - t.Fatal(err) - } - - if err := cmd.Run(); err != nil { - t.Fatal(err) - } - - // TODO(noffle): it takes a moment for the goroutine that's running fs.Serve to be notified and do its cleanup. - time.Sleep(time.Millisecond * 100) - - // Attempt to unmount IPFS; it should unmount successfully. - err = node.Mounts.Ipfs.Unmount() - if err != mount.ErrNotMounted { - t.Fatal("Unmount should have failed") - } -} diff --git a/fuse/node/mount_unix.go b/fuse/node/mount_unix.go deleted file mode 100644 index 4e2f732da31..00000000000 --- a/fuse/node/mount_unix.go +++ /dev/null @@ -1,114 +0,0 @@ -// +build !windows,!nofuse - -package node - -import ( - "errors" - "fmt" - "strings" - "sync" - - core "github.com/ipfs/go-ipfs/core" - ipns "github.com/ipfs/go-ipfs/fuse/ipns" - mount "github.com/ipfs/go-ipfs/fuse/mount" - rofs "github.com/ipfs/go-ipfs/fuse/readonly" - - logging "gx/ipfs/QmbkT7eMTyXfpeyB3ZMxxcxg7XH8t6uXp49jqzz4HB7BGF/go-log" -) - -var log = logging.Logger("node") - -// fuseNoDirectory used to check the returning fuse error -const fuseNoDirectory = "fusermount: failed to access mountpoint" - -// fuseExitStatus1 used to check the returning fuse error -const fuseExitStatus1 = "fusermount: exit status 1" - -// platformFuseChecks can get overridden by arch-specific files -// to run fuse checks (like checking the OSXFUSE version) -var platformFuseChecks = func(*core.IpfsNode) error { - return nil -} - -func Mount(node *core.IpfsNode, fsdir, nsdir string) error { - // check if we already have live mounts. - // if the user said "Mount", then there must be something wrong. - // so, close them and try again. - if node.Mounts.Ipfs != nil && node.Mounts.Ipfs.IsActive() { - node.Mounts.Ipfs.Unmount() - } - if node.Mounts.Ipns != nil && node.Mounts.Ipns.IsActive() { - node.Mounts.Ipns.Unmount() - } - - if err := platformFuseChecks(node); err != nil { - return err - } - - return doMount(node, fsdir, nsdir) -} - -func doMount(node *core.IpfsNode, fsdir, nsdir string) error { - fmtFuseErr := func(err error, mountpoint string) error { - s := err.Error() - if strings.Contains(s, fuseNoDirectory) { - s = strings.Replace(s, `fusermount: "fusermount:`, "", -1) - s = strings.Replace(s, `\n", exit status 1`, "", -1) - return errors.New(s) - } - if s == fuseExitStatus1 { - s = fmt.Sprintf("fuse failed to access mountpoint %s", mountpoint) - return errors.New(s) - } - return err - } - - // this sync stuff is so that both can be mounted simultaneously. - var fsmount, nsmount mount.Mount - var err1, err2 error - - var wg sync.WaitGroup - - wg.Add(1) - go func() { - defer wg.Done() - fsmount, err1 = rofs.Mount(node, fsdir) - }() - - if node.OnlineMode() { - wg.Add(1) - go func() { - defer wg.Done() - nsmount, err2 = ipns.Mount(node, nsdir, fsdir) - }() - } - - wg.Wait() - - if err1 != nil { - log.Errorf("error mounting: %s", err1) - } - - if err2 != nil { - log.Errorf("error mounting: %s", err2) - } - - if err1 != nil || err2 != nil { - if fsmount != nil { - fsmount.Unmount() - } - if nsmount != nil { - nsmount.Unmount() - } - - if err1 != nil { - return fmtFuseErr(err1, fsdir) - } - return fmtFuseErr(err2, nsdir) - } - - // setup node state, so that it can be cancelled - node.Mounts.Ipfs = fsmount - node.Mounts.Ipns = nsmount - return nil -} diff --git a/fuse/node/mount_windows.go b/fuse/node/mount_windows.go deleted file mode 100644 index ce89deddb23..00000000000 --- a/fuse/node/mount_windows.go +++ /dev/null @@ -1,11 +0,0 @@ -package node - -import ( - "github.com/ipfs/go-ipfs/core" -) - -func Mount(node *core.IpfsNode, fsdir, nsdir string) error { - // TODO - // currently a no-op, but we don't want to return an error - return nil -} diff --git a/fuse/readonly/doc.go b/fuse/readonly/doc.go deleted file mode 100644 index 1a7e779fe94..00000000000 --- a/fuse/readonly/doc.go +++ /dev/null @@ -1,3 +0,0 @@ -// package fuse/readonly implements a fuse filesystem to access files -// stored inside of ipfs. -package readonly diff --git a/fuse/readonly/ipfs_test.go b/fuse/readonly/ipfs_test.go deleted file mode 100644 index b84d370997e..00000000000 --- a/fuse/readonly/ipfs_test.go +++ /dev/null @@ -1,294 +0,0 @@ -// +build !nofuse - -package readonly - -import ( - "bytes" - "context" - "errors" - "fmt" - "io/ioutil" - "math/rand" - "os" - "path" - "strings" - "sync" - "testing" - - core "github.com/ipfs/go-ipfs/core" - coreapi "github.com/ipfs/go-ipfs/core/coreapi" - coremock "github.com/ipfs/go-ipfs/core/mock" - - u "gx/ipfs/QmNohiVssaPw3KVLZik59DBVGTSm2dGvYT9eoXt5DQ36Yz/go-ipfs-util" - dag "gx/ipfs/QmPJNbVw8o3ohC43ppSXyNXwYKsWShG4zygnirHptfbHri/go-merkledag" - files "gx/ipfs/QmQmhotPUzVrMEWNK3x1R5jQ5ZHWyL7tVUrmRPjrBrvyCb/go-ipfs-files" - fstest "gx/ipfs/QmSJBsmLP1XMjv8hxYg2rUMdPDB7YUpyBo9idjrJ6Cmq6F/fuse/fs/fstestutil" - ci "gx/ipfs/QmWapVoHjtKhn4MhvKNoPTkJKADFGACfXPFnt7combwp5W/go-testutil/ci" - iface "gx/ipfs/QmXLwxifxwfc2bAwq6rdjbYqAsGzWsDE9RM5TWMGtykyj6/interface-go-ipfs-core" - chunker "gx/ipfs/QmYmZ81dU5nnmBFy5MmktXLZpt8QCWhRJd6M1uxVF6vke8/go-ipfs-chunker" - ipld "gx/ipfs/QmZ6nzCLwGLVfRzYLpD7pW6UNuBDKEcA2imJtVpbEx2rxy/go-ipld-format" - importer "gx/ipfs/QmcYUTQ7tBZeH1CLsZM2S3xhMEZdvUgXvbjhpMsLDpk3oJ/go-unixfs/importer" - uio "gx/ipfs/QmcYUTQ7tBZeH1CLsZM2S3xhMEZdvUgXvbjhpMsLDpk3oJ/go-unixfs/io" -) - -func maybeSkipFuseTests(t *testing.T) { - if ci.NoFuse() { - t.Skip("Skipping FUSE tests") - } -} - -func randObj(t *testing.T, nd *core.IpfsNode, size int64) (ipld.Node, []byte) { - buf := make([]byte, size) - u.NewTimeSeededRand().Read(buf) - read := bytes.NewReader(buf) - obj, err := importer.BuildTrickleDagFromReader(nd.DAG, chunker.DefaultSplitter(read)) - if err != nil { - t.Fatal(err) - } - - return obj, buf -} - -func setupIpfsTest(t *testing.T, node *core.IpfsNode) (*core.IpfsNode, *fstest.Mount) { - maybeSkipFuseTests(t) - - var err error - if node == nil { - node, err = coremock.NewMockNode() - if err != nil { - t.Fatal(err) - } - } - - fs := NewFileSystem(node) - mnt, err := fstest.MountedT(t, fs, nil) - if err != nil { - t.Fatal(err) - } - - return node, mnt -} - -// Test writing an object and reading it back through fuse -func TestIpfsBasicRead(t *testing.T) { - if testing.Short() { - t.SkipNow() - } - nd, mnt := setupIpfsTest(t, nil) - defer mnt.Close() - - fi, data := randObj(t, nd, 10000) - k := fi.Cid() - fname := path.Join(mnt.Dir, k.String()) - rbuf, err := ioutil.ReadFile(fname) - if err != nil { - t.Fatal(err) - } - - if !bytes.Equal(rbuf, data) { - t.Fatal("Incorrect Read!") - } -} - -func getPaths(t *testing.T, ipfs *core.IpfsNode, name string, n *dag.ProtoNode) []string { - if len(n.Links()) == 0 { - return []string{name} - } - var out []string - for _, lnk := range n.Links() { - child, err := lnk.GetNode(ipfs.Context(), ipfs.DAG) - if err != nil { - t.Fatal(err) - } - - childpb, ok := child.(*dag.ProtoNode) - if !ok { - t.Fatal(dag.ErrNotProtobuf) - } - - sub := getPaths(t, ipfs, path.Join(name, lnk.Name), childpb) - out = append(out, sub...) - } - return out -} - -// Perform a large number of concurrent reads to stress the system -func TestIpfsStressRead(t *testing.T) { - if testing.Short() { - t.SkipNow() - } - nd, mnt := setupIpfsTest(t, nil) - defer mnt.Close() - - api, err := coreapi.NewCoreAPI(nd) - if err != nil { - t.Fatal(err) - } - - var nodes []ipld.Node - var paths []string - - nobj := 50 - ndiriter := 50 - - // Make a bunch of objects - for i := 0; i < nobj; i++ { - fi, _ := randObj(t, nd, rand.Int63n(50000)) - nodes = append(nodes, fi) - paths = append(paths, fi.Cid().String()) - } - - // Now make a bunch of dirs - for i := 0; i < ndiriter; i++ { - db := uio.NewDirectory(nd.DAG) - for j := 0; j < 1+rand.Intn(10); j++ { - name := fmt.Sprintf("child%d", j) - - err := db.AddChild(nd.Context(), name, nodes[rand.Intn(len(nodes))]) - if err != nil { - t.Fatal(err) - } - } - newdir, err := db.GetNode() - if err != nil { - t.Fatal(err) - } - - err = nd.DAG.Add(nd.Context(), newdir) - if err != nil { - t.Fatal(err) - } - - nodes = append(nodes, newdir) - npaths := getPaths(t, nd, newdir.Cid().String(), newdir.(*dag.ProtoNode)) - paths = append(paths, npaths...) - } - - // Now read a bunch, concurrently - wg := sync.WaitGroup{} - errs := make(chan error) - - for s := 0; s < 4; s++ { - wg.Add(1) - go func() { - defer wg.Done() - - for i := 0; i < 2000; i++ { - item, _ := iface.ParsePath(paths[rand.Intn(len(paths))]) - - relpath := strings.Replace(item.String(), item.Namespace(), "", 1) - fname := path.Join(mnt.Dir, relpath) - - rbuf, err := ioutil.ReadFile(fname) - if err != nil { - errs <- err - } - - //nd.Context() is never closed which leads to - //hitting 8128 goroutine limit in go test -race mode - ctx, cancelFunc := context.WithCancel(context.Background()) - - read, err := api.Unixfs().Get(ctx, item) - if err != nil { - errs <- err - } - - data, err := ioutil.ReadAll(read.(files.File)) - if err != nil { - errs <- err - } - - cancelFunc() - - if !bytes.Equal(rbuf, data) { - errs <- errors.New("incorrect read") - } - } - }() - } - - go func() { - wg.Wait() - close(errs) - }() - - for err := range errs { - if err != nil { - t.Fatal(err) - } - } -} - -// Test writing a file and reading it back -func TestIpfsBasicDirRead(t *testing.T) { - if testing.Short() { - t.SkipNow() - } - nd, mnt := setupIpfsTest(t, nil) - defer mnt.Close() - - // Make a 'file' - fi, data := randObj(t, nd, 10000) - - // Make a directory and put that file in it - db := uio.NewDirectory(nd.DAG) - err := db.AddChild(nd.Context(), "actual", fi) - if err != nil { - t.Fatal(err) - } - - d1nd, err := db.GetNode() - if err != nil { - t.Fatal(err) - } - - err = nd.DAG.Add(nd.Context(), d1nd) - if err != nil { - t.Fatal(err) - } - - dirname := path.Join(mnt.Dir, d1nd.Cid().String()) - fname := path.Join(dirname, "actual") - rbuf, err := ioutil.ReadFile(fname) - if err != nil { - t.Fatal(err) - } - - dirents, err := ioutil.ReadDir(dirname) - if err != nil { - t.Fatal(err) - } - if len(dirents) != 1 { - t.Fatal("Bad directory entry count") - } - if dirents[0].Name() != "actual" { - t.Fatal("Bad directory entry") - } - - if !bytes.Equal(rbuf, data) { - t.Fatal("Incorrect Read!") - } -} - -// Test to make sure the filesystem reports file sizes correctly -func TestFileSizeReporting(t *testing.T) { - if testing.Short() { - t.SkipNow() - } - nd, mnt := setupIpfsTest(t, nil) - defer mnt.Close() - - fi, data := randObj(t, nd, 10000) - k := fi.Cid() - - fname := path.Join(mnt.Dir, k.String()) - - finfo, err := os.Stat(fname) - if err != nil { - t.Fatal(err) - } - - if finfo.Size() != int64(len(data)) { - t.Fatal("Read incorrect size from stat!") - } -} diff --git a/fuse/readonly/mount_unix.go b/fuse/readonly/mount_unix.go deleted file mode 100644 index ab794545629..00000000000 --- a/fuse/readonly/mount_unix.go +++ /dev/null @@ -1,20 +0,0 @@ -// +build linux darwin freebsd netbsd openbsd -// +build !nofuse - -package readonly - -import ( - core "github.com/ipfs/go-ipfs/core" - mount "github.com/ipfs/go-ipfs/fuse/mount" -) - -// Mount mounts IPFS at a given location, and returns a mount.Mount instance. -func Mount(ipfs *core.IpfsNode, mountpoint string) (mount.Mount, error) { - cfg, err := ipfs.Repo.Config() - if err != nil { - return nil, err - } - allow_other := cfg.Mounts.FuseAllowOther - fsys := NewFileSystem(ipfs) - return mount.NewMount(ipfs.Process(), fsys, mountpoint, allow_other) -} diff --git a/fuse/readonly/readonly_unix.go b/fuse/readonly/readonly_unix.go deleted file mode 100644 index 7203e862a74..00000000000 --- a/fuse/readonly/readonly_unix.go +++ /dev/null @@ -1,296 +0,0 @@ -// +build linux darwin freebsd netbsd openbsd -// +build !nofuse - -package readonly - -import ( - "context" - "fmt" - "io" - "os" - "syscall" - - core "github.com/ipfs/go-ipfs/core" - mdag "gx/ipfs/QmPJNbVw8o3ohC43ppSXyNXwYKsWShG4zygnirHptfbHri/go-merkledag" - path "gx/ipfs/QmQAgv6Gaoe2tQpcabqwKXKChp2MZ7i3UXv9DqTTaxCaTR/go-path" - ft "gx/ipfs/QmcYUTQ7tBZeH1CLsZM2S3xhMEZdvUgXvbjhpMsLDpk3oJ/go-unixfs" - uio "gx/ipfs/QmcYUTQ7tBZeH1CLsZM2S3xhMEZdvUgXvbjhpMsLDpk3oJ/go-unixfs/io" - - fuse "gx/ipfs/QmSJBsmLP1XMjv8hxYg2rUMdPDB7YUpyBo9idjrJ6Cmq6F/fuse" - fs "gx/ipfs/QmSJBsmLP1XMjv8hxYg2rUMdPDB7YUpyBo9idjrJ6Cmq6F/fuse/fs" - lgbl "gx/ipfs/QmUbSLukzZYZvEYxynj9Dtd1WrGLxxg9R4U68vCMPWHmRU/go-libp2p-loggables" - ipld "gx/ipfs/QmZ6nzCLwGLVfRzYLpD7pW6UNuBDKEcA2imJtVpbEx2rxy/go-ipld-format" - logging "gx/ipfs/QmbkT7eMTyXfpeyB3ZMxxcxg7XH8t6uXp49jqzz4HB7BGF/go-log" -) - -var log = logging.Logger("fuse/ipfs") - -// FileSystem is the readonly IPFS Fuse Filesystem. -type FileSystem struct { - Ipfs *core.IpfsNode -} - -// NewFileSystem constructs new fs using given core.IpfsNode instance. -func NewFileSystem(ipfs *core.IpfsNode) *FileSystem { - return &FileSystem{Ipfs: ipfs} -} - -// Root constructs the Root of the filesystem, a Root object. -func (f FileSystem) Root() (fs.Node, error) { - return &Root{Ipfs: f.Ipfs}, nil -} - -// Root is the root object of the filesystem tree. -type Root struct { - Ipfs *core.IpfsNode -} - -// Attr returns file attributes. -func (*Root) Attr(ctx context.Context, a *fuse.Attr) error { - a.Mode = os.ModeDir | 0111 // -rw+x - return nil -} - -// Lookup performs a lookup under this node. -func (s *Root) Lookup(ctx context.Context, name string) (fs.Node, error) { - log.Debugf("Root Lookup: '%s'", name) - switch name { - case "mach_kernel", ".hidden", "._.": - // Just quiet some log noise on OS X. - return nil, fuse.ENOENT - } - - p, err := path.ParsePath(name) - if err != nil { - log.Debugf("fuse failed to parse path: %q: %s", name, err) - return nil, fuse.ENOENT - } - - nd, err := s.Ipfs.Resolver.ResolvePath(ctx, p) - if err != nil { - // todo: make this error more versatile. - return nil, fuse.ENOENT - } - - switch nd := nd.(type) { - case *mdag.ProtoNode, *mdag.RawNode: - return &Node{Ipfs: s.Ipfs, Nd: nd}, nil - default: - log.Error("fuse node was not a protobuf node") - return nil, fuse.ENOTSUP - } - -} - -// ReadDirAll reads a particular directory. Disallowed for root. -func (*Root) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { - log.Debug("read Root") - return nil, fuse.EPERM -} - -// Node is the core object representing a filesystem tree node. -type Node struct { - Ipfs *core.IpfsNode - Nd ipld.Node - cached *ft.FSNode -} - -func (s *Node) loadData() error { - if pbnd, ok := s.Nd.(*mdag.ProtoNode); ok { - fsn, err := ft.FSNodeFromBytes(pbnd.Data()) - if err != nil { - return err - } - s.cached = fsn - } - return nil -} - -// Attr returns the attributes of a given node. -func (s *Node) Attr(ctx context.Context, a *fuse.Attr) error { - log.Debug("Node attr") - if rawnd, ok := s.Nd.(*mdag.RawNode); ok { - a.Mode = 0444 - a.Size = uint64(len(rawnd.RawData())) - a.Blocks = 1 - return nil - } - - if s.cached == nil { - if err := s.loadData(); err != nil { - return fmt.Errorf("readonly: loadData() failed: %s", err) - } - } - switch s.cached.Type() { - case ft.TDirectory, ft.THAMTShard: - a.Mode = os.ModeDir | 0555 - case ft.TFile: - size := s.cached.FileSize() - a.Mode = 0444 - a.Size = uint64(size) - a.Blocks = uint64(len(s.Nd.Links())) - case ft.TRaw: - a.Mode = 0444 - a.Size = uint64(len(s.cached.Data())) - a.Blocks = uint64(len(s.Nd.Links())) - case ft.TSymlink: - a.Mode = 0777 | os.ModeSymlink - a.Size = uint64(len(s.cached.Data())) - default: - return fmt.Errorf("invalid data type - %s", s.cached.Type()) - } - return nil -} - -// Lookup performs a lookup under this node. -func (s *Node) Lookup(ctx context.Context, name string) (fs.Node, error) { - log.Debugf("Lookup '%s'", name) - link, _, err := uio.ResolveUnixfsOnce(ctx, s.Ipfs.DAG, s.Nd, []string{name}) - switch err { - case os.ErrNotExist, mdag.ErrLinkNotFound: - // todo: make this error more versatile. - return nil, fuse.ENOENT - default: - log.Errorf("fuse lookup %q: %s", name, err) - return nil, fuse.EIO - case nil: - // noop - } - - nd, err := s.Ipfs.DAG.Get(ctx, link.Cid) - switch err { - case ipld.ErrNotFound: - default: - log.Errorf("fuse lookup %q: %s", name, err) - return nil, err - case nil: - // noop - } - - return &Node{Ipfs: s.Ipfs, Nd: nd}, nil -} - -// ReadDirAll reads the link structure as directory entries -func (s *Node) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { - log.Debug("Node ReadDir") - dir, err := uio.NewDirectoryFromNode(s.Ipfs.DAG, s.Nd) - if err != nil { - return nil, err - } - - var entries []fuse.Dirent - err = dir.ForEachLink(ctx, func(lnk *ipld.Link) error { - n := lnk.Name - if len(n) == 0 { - n = lnk.Cid.String() - } - nd, err := s.Ipfs.DAG.Get(ctx, lnk.Cid) - if err != nil { - log.Warning("error fetching directory child node: ", err) - } - - t := fuse.DT_Unknown - switch nd := nd.(type) { - case *mdag.RawNode: - t = fuse.DT_File - case *mdag.ProtoNode: - if fsn, err := ft.FSNodeFromBytes(nd.Data()); err != nil { - log.Warning("failed to unmarshal protonode data field:", err) - } else { - switch fsn.Type() { - case ft.TDirectory, ft.THAMTShard: - t = fuse.DT_Dir - case ft.TFile, ft.TRaw: - t = fuse.DT_File - case ft.TSymlink: - t = fuse.DT_Link - case ft.TMetadata: - log.Error("metadata object in fuse should contain its wrapped type") - default: - log.Error("unrecognized protonode data type: ", fsn.Type()) - } - } - } - entries = append(entries, fuse.Dirent{Name: n, Type: t}) - return nil - }) - if err != nil { - return nil, err - } - - if len(entries) > 0 { - return entries, nil - } - return nil, fuse.ENOENT -} - -func (s *Node) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error { - // TODO: is nil the right response for 'bug off, we aint got none' ? - resp.Xattr = nil - return nil -} - -func (s *Node) Readlink(ctx context.Context, req *fuse.ReadlinkRequest) (string, error) { - if s.cached == nil || s.cached.Type() != ft.TSymlink { - return "", fuse.Errno(syscall.EINVAL) - } - return string(s.cached.Data()), nil -} - -func (s *Node) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error { - c := s.Nd.Cid() - - // setup our logging event - lm := make(lgbl.DeferredMap) - lm["fs"] = "ipfs" - lm["key"] = func() interface{} { return c.String() } - lm["req_offset"] = req.Offset - lm["req_size"] = req.Size - defer log.EventBegin(ctx, "fuseRead", lm).Done() - - r, err := uio.NewDagReader(ctx, s.Nd, s.Ipfs.DAG) - if err != nil { - return err - } - o, err := r.Seek(req.Offset, io.SeekStart) - lm["res_offset"] = o - if err != nil { - return err - } - - buf := resp.Data[:min(req.Size, int(int64(r.Size())-req.Offset))] - n, err := io.ReadFull(r, buf) - if err != nil && err != io.EOF { - return err - } - resp.Data = resp.Data[:n] - lm["res_size"] = n - return nil // may be non-nil / not succeeded -} - -// to check that out Node implements all the interfaces we want -type roRoot interface { - fs.Node - fs.HandleReadDirAller - fs.NodeStringLookuper -} - -var _ roRoot = (*Root)(nil) - -type roNode interface { - fs.HandleReadDirAller - fs.HandleReader - fs.Node - fs.NodeStringLookuper - fs.NodeReadlinker - fs.NodeGetxattrer -} - -var _ roNode = (*Node)(nil) - -func min(a, b int) int { - if a < b { - return a - } - return b -} From 660314bcb55d1c33d52920073e7a661415e9beeb Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Wed, 6 Mar 2019 14:31:08 -0500 Subject: [PATCH 2/2] WIP License: MIT Signed-off-by: Dominic Della Valle --- core/commands/mount/create.go | 233 ++-------- core/commands/mount/delete.go | 125 ++---- core/commands/mount/dirio.go | 386 ++++++++--------- core/commands/mount/event.go | 52 --- core/commands/mount/filesystem.go | 529 +++++++++++++++-------- core/commands/mount/index.go | 382 ++++++---------- core/commands/mount/indexNodes.go | 601 +++++++++++++++++++++++++- core/commands/mount/interface.go | 209 ++++----- core/commands/mount/io.go | 132 +++--- core/commands/mount/modify.go | 64 +-- core/commands/mount/probe.go | 200 ++++----- core/commands/mount/rootNodes.go | 154 ++++++- core/commands/mount/system_linux.go | 5 + core/commands/mount/system_windows.go | 6 +- core/commands/mount/utils.go | 234 ++++++++-- 15 files changed, 1920 insertions(+), 1392 deletions(-) diff --git a/core/commands/mount/create.go b/core/commands/mount/create.go index f18fd6ff24a..8d8957559f4 100644 --- a/core/commands/mount/create.go +++ b/core/commands/mount/create.go @@ -3,7 +3,6 @@ package fusemount import ( "fmt" dag "gx/ipfs/QmPJNbVw8o3ohC43ppSXyNXwYKsWShG4zygnirHptfbHri/go-merkledag" - coreoptions "gx/ipfs/QmXLwxifxwfc2bAwq6rdjbYqAsGzWsDE9RM5TWMGtykyj6/interface-go-ipfs-core/options" mfs "gx/ipfs/Qmb74fRYPgpjYzoBV7PAVNmP3DQaRrh8dHdKE4PwnF3cRx/go-mfs" unixfs "gx/ipfs/QmcYUTQ7tBZeH1CLsZM2S3xhMEZdvUgXvbjhpMsLDpk3oJ/go-unixfs" "os" @@ -19,10 +18,7 @@ func (fs *FUSEIPFS) Link(origin, target string) int { log.Errorf("Link - Request %q -> %q", origin, target) switch parsePathType(target) { - case tIPNS: - log.Errorf("Link - IPNS support not implemented yet %q", target) - return -fuse.EROFS - case tMFS: + case tFAPI: fErr, gErr := mfsSymlink(fs.filesRoot, target, target[frs:]) if gErr != nil { log.Errorf("Link - mfs error: %s", gErr) @@ -42,10 +38,7 @@ func (fs *FUSEIPFS) Symlink(target, linkActual string) int { log.Debugf("Symlink - Request %q -> %q", target, linkActual) switch parsePathType(linkActual) { - case tIPNS: - log.Errorf("Symlink - IPNS support not implemented yet %q", linkActual) - return -fuse.EROFS - case tMFS: + case tFAPI: fErr, gErr := mfsSymlink(fs.filesRoot, target, linkActual[frs:]) if gErr != nil { log.Errorf("Symlink - error: %s", gErr) @@ -101,211 +94,67 @@ func (fs *FUSEIPFS) Mknod(path string, mode uint32, dev uint64) int { fs.Lock() defer fs.Unlock() log.Debugf("Mknod - Request [%X]{%X}%q", mode, dev, path) - - // TODO: abstract this: node.PLock(){self.parent.lock();self.lock()} // goes up to soft-root max; i.e ipns-key, mfs-root - - parentPath := gopath.Dir(path) - parent, err := fs.LookupPath(parentPath) - if err != nil { - log.Errorf("Mknod - could not fetch/lock parent for %q", path) - } - parent.Lock() - defer parent.Unlock() - // - fErr, gErr := fs.mknod(path) + fErr, gErr := fs.Mk(path, mode, unixfs.TFile) if gErr != nil { - log.Errorf("Mknod - %s", gErr) + log.Errorf("Mknod - %q: %s", path, gErr) } - - fs.cc.ReleasePath(parentPath) return fErr } -//TODO: inline this -func (fs FUSEIPFS) mknod(path string) (int, error) { - parsedNode, err := fs.LookupPath(path) - if err == nil { - return -fuse.EEXIST, os.ErrExist - } - if err != os.ErrNotExist { - return -fuse.EIO, err - } - - switch parsedNode.(type) { - case *ipnsKey: - _, keyName := gopath.Split(path) - coreKey, err := fs.core.Key().Generate(fs.ctx, keyName) - if err != nil { - return -fuse.EIO, fmt.Errorf("could not generate IPNS key %q: %s", keyName, err) - } - newRootNode, err := emptyNode(fs.ctx, fs.core.Dag(), unixfs.TFile, nil) - if err != nil { - return -fuse.EIO, fmt.Errorf("could not generate unixdir %q: %s", keyName, err) - } - - err = fs.ipnsDelayedPublish(coreKey, newRootNode) - if err != nil { - return -fuse.EIO, fmt.Errorf("could not publish to key %q: %s", keyName, err) - } - return fuseSuccess, nil - - case *ipnsNode: - return fs.ipnsMknod(path) - - case *mfsNode: - return mfsMknod(fs.filesRoot, path[frs:]) - } - - return -fuse.EROFS, fmt.Errorf("unexpected request {%T}%q", parsedNode, path) -} - -func mfsMknod(filesRoot *mfs.Root, path string) (int, error) { - if _, err := mfs.Lookup(filesRoot, path); err == nil { - return -fuse.EEXIST, fmt.Errorf("%q already exists", path) - } - - dirName, fName := gopath.Split(path) - mfsNode, err := mfs.Lookup(filesRoot, dirName) - if err != nil { - return -fuse.ENOENT, err - } - mfsDir, ok := mfsNode.(*mfs.Directory) - if !ok { - return -fuse.ENOTDIR, fmt.Errorf("%s is not a directory", dirName) - } - - dagNode := dag.NodeWithData(unixfs.FilePBData(nil, 0)) - dagNode.SetCidBuilder(mfsDir.GetCidBuilder()) - - err = mfsDir.AddChild(fName, dagNode) - if err != nil { - log.Errorf("mfsMknod I/O sunk %q:%s", path, err) - return -fuse.EIO, err - } - - return fuseSuccess, nil -} - +//TODO: wrap this; overlaps with mknod func (fs *FUSEIPFS) Mkdir(path string, mode uint32) int { fs.Lock() defer fs.Unlock() log.Debugf("Mkdir - Request {%X}%q", mode, path) - - parentPath := gopath.Dir(path) - parent, err := fs.LookupPath(parentPath) - if err != nil { - log.Errorf("Mkdir - could not fetch/lock parent for %q", path) + fErr, gErr := fs.Mk(path, mode, unixfs.TDirectory) + if gErr != nil { + log.Errorf("Mkdir - %q: %s", path, gErr) } - parent.Lock() - defer parent.Unlock() - defer fs.cc.ReleasePath(parentPath) //TODO: don't do this on failure - - switch parsePathType(path) { - case tMFS: - //TODO: review mkdir opts + Mkdir POSIX specs (are intermediate paths allowed by default?) - if err := mfs.Mkdir(fs.filesRoot, path[frs:], mfs.MkdirOpts{Flush: mfsSync}); err != nil { - if err == mfs.ErrDirExists || err == os.ErrExist { - return -fuse.EEXIST - } - log.Errorf("Mkdir - unexpected error - %s", err) - return -fuse.EACCES - } - return fuseSuccess - case tIPNSKey: //TODO: refresh fs.nameRoots - _, keyName := gopath.Split(path) - coreKey, err := fs.core.Key().Generate(fs.ctx, keyName) - if err != nil { - log.Errorf("Mkdir - could not generate IPNS key %q: %s", keyName, err) - return -fuse.EACCES - } - newRootNode, err := emptyNode(fs.ctx, fs.core.Dag(), unixfs.TDirectory, nil) - if err != nil { - log.Errorf("Mkdir - could not generate unixdir %q: %s", keyName, err) - return -fuse.EACCES - } - - err = fs.ipnsDelayedPublish(coreKey, newRootNode) - if err != nil { - log.Errorf("Mkdir - could not publish to key %q: %s", keyName, err) - return -fuse.EACCES - } + return fErr +} - pbNode, ok := newRootNode.(*dag.ProtoNode) - if !ok { //this should never happen - log.Errorf("IPNS key %q has incompatible type %T", keyName, newRootNode) - fs.nameRoots[keyName] = nil - return -fuse.EACCES +func (fs *FUSEIPFS) Mk(path string, mode uint32, nodeType FsType) (ret int, err error) { + defer func() { + if nodeType == unixfs.TDirectory && ret == -fuse.EIO { + ret = -fuse.EACCES // same interface, different return values } + }() - oAPI, err := fs.core.WithOptions(coreoptions.Api.Offline(true)) - if err != nil { - log.Errorf("offline API could not be created: %s", err) - fs.nameRoots[keyName] = nil - return -fuse.EACCES - } - keyRoot, err := mfs.NewRoot(fs.ctx, fs.core.Dag(), pbNode, ipnsPublisher(keyName, oAPI.Name())) - if err != nil { - log.Errorf("IPNS key %q could not be mapped to MFS root: %s", keyName, err) - fs.nameRoots[keyName] = nil - return -fuse.EACCES - } - fs.nameRoots[keyName] = keyRoot + //NOTE: mode is not expected to be portable, except for FIFO pipes + // which are not implemented, but could be through named p2p sockets in theory + // handling of allowing the OS to create directories with mknod could also be handled here - return fuseSuccess - - case tIPNS: - fErr, gErr := fs.ipnsMkdir(path) - if gErr != nil { - log.Errorf("Mkdir - error: %s", gErr) - } - return fErr + //TODO: attain parent lock first - case tMountRoot, tIPFSRoot, tFilesRoot: - log.Errorf("Mkdir - requested a root entry - %q", path) - return -fuse.EEXIST + fsNode, err := fs.shallowLookupPath(path) + switch err { + case nil: + ret, err = -fuse.EEXIST, os.ErrExist + return + case os.ErrNotExist: + break + default: + ret, err = -fuse.EIO, err + return } - log.Errorf("Mkdir - unexpected request %q", path) - return -fuse.ENOENT -} + fsNode.Lock() + defer fsNode.Unlock() -func (fs *FUSEIPFS) ipnsMkdir(path string) (int, error) { - keyRoot, subPath, err := fs.ipnsMFSSplit(path) - if err != nil { - return -fuse.EACCES, err - } + callContext, cancel := deriveCallContext(fs.ctx) + defer cancel() - //NOTE: must flush/publish otherwise our resolver is never going to pick up the change - if err := mfs.Mkdir(keyRoot, subPath, mfs.MkdirOpts{Flush: false}); err != nil { - if err == mfs.ErrDirExists || err == os.ErrExist { - return -fuse.EEXIST, err - } - return -fuse.EACCES, err + if ret, err = fsNode.Create(callContext, unixfs.TFile); err != nil { + return } - if err := mfs.FlushPath(keyRoot, subPath); err != nil { - return -fuse.EACCES, err + var nodeStat *fuse.Stat_t + if nodeStat, err = fsNode.InitMetadata(callContext); err != nil { + ret = -fuse.EIO + return } - return fuseSuccess, nil -} - -func (fs *FUSEIPFS) ipnsMknod(path string) (int, error) { - keyRoot, subPath, err := fs.ipnsMFSSplit(path) - if err != nil { - return -fuse.EIO, err - } - blankNode, err := emptyNode(fs.ctx, fs.core.Dag(), unixfs.TFile, nil) - if err != nil { - return -fuse.EIO, err - } - if err := mfs.PutNode(keyRoot, subPath, blankNode); err != nil { - return -fuse.EIO, err - } - - if err := mfs.FlushPath(keyRoot, subPath); err != nil { - return -fuse.EACCES, err - } + // TODO update parent meta - return fuseSuccess, nil + return } diff --git a/core/commands/mount/delete.go b/core/commands/mount/delete.go index dcf79f5c0fd..9eb80c3a327 100644 --- a/core/commands/mount/delete.go +++ b/core/commands/mount/delete.go @@ -1,8 +1,10 @@ package fusemount import ( + "errors" "fmt" mfs "gx/ipfs/Qmb74fRYPgpjYzoBV7PAVNmP3DQaRrh8dHdKE4PwnF3cRx/go-mfs" + unixfs "gx/ipfs/QmcYUTQ7tBZeH1CLsZM2S3xhMEZdvUgXvbjhpMsLDpk3oJ/go-unixfs" gopath "path" "github.com/billziss-gh/cgofuse/fuse" @@ -18,120 +20,85 @@ func (fs *FUSEIPFS) Unlink(path string) int { fs.Lock() defer fs.Unlock() log.Debugf("Unlink - Request %q", path) - /* - lNode, err := fs.parseLocalPath(path) - if err != nil { - log.Errorf("Unlink - path err %s", err) - return -fuse.EINVAL - } - */ - fsNode, err := fs.LookupPath(path) - if err != nil { - log.Errorf("Unlink - lookup error: %s", err) - return -fuse.ENOENT - } - fsNode.Lock() - defer fsNode.Unlock() - - switch fsNode.(type) { - default: - log.Errorf("Unlink - request in read only section: %q", path) - return -fuse.EROFS - case *ipfsRoot, *ipfsNode: - //TODO: block rm? - log.Errorf("Unlink - request in read only section: %q", path) - return -fuse.EROFS - case *mfsNode: - if err := mfsRemove(fs.filesRoot, path[frs:]); err != nil { - log.Errorf("Unlink - mfs error: %s", err) - return -fuse.EIO - } - - //invalidate cache object - fs.cc.ReleasePath(path) - case *ipnsKey: - _, keyName := gopath.Split(path) - _, err := fs.core.Key().Remove(fs.ctx, keyName) - if err != nil { - log.Errorf("could not remove IPNS key %q: %s", keyName, err) - return -fuse.EIO - } - - case *ipnsNode: - keyRoot, subPath, err := fs.ipnsMFSSplit(path) - if err != nil { - log.Errorf("Unlink - IPNS key error: %s", err) - } - if err := mfsRemove(keyRoot, subPath); err != nil { - log.Errorf("Unlink - mfs error: %s", err) - return -fuse.EIO - } + fErr, gErr := fs.Remove(path, unixfs.TFile) + if gErr != nil { + log.Errorf("Unlink - %q: %s", path, gErr) } - return fuseSuccess + return fErr } -//TODO: lock parent func (fs *FUSEIPFS) Rmdir(path string) int { fs.Lock() defer fs.Unlock() log.Debugf("Rmdir - Request %q", path) - lNode, err := parseLocalPath(path) - if err != nil { - log.Errorf("Rmdir - path err %s", err) - return -fuse.ENOENT + fErr, gErr := fs.Remove(path, unixfs.TDirectory) + if gErr != nil { + log.Errorf("Rmdir - %q: %s", path, gErr) } + return fErr +} + +func (fs *FUSEIPFS) Remove(path string, nodeType FsType) (int, error) { + //TODO: wrap parent locking and cache release somehow; unlink, mk, et al. need this too parentPath := gopath.Dir(path) parent, err := fs.LookupPath(parentPath) if err != nil { - log.Errorf("Mkdir - could not fetch/lock parent for %q", path) + return -fuse.ENOENT, errors.New("could not fetch/lock parent path") } parent.Lock() defer parent.Unlock() - defer fs.cc.ReleasePath(parentPath) //TODO: don't do this on failure - defer fs.cc.ReleasePath(path) //TODO: don't do on failure + fsNode, err := fs.shallowLookupPath(path) + if err != nil { + return -fuse.ENOENT, err + } + + nodeStat, err := fsNode.Stat() - switch lNode.(type) { - case *ipnsKey: - _, keyName := gopath.Split(path) - _, err := fs.core.Key().Remove(fs.ctx, keyName) + //Rmdir + if nodeType == unixfs.TDirectory { if err != nil { - log.Errorf("could not remove IPNS key %q: %s", keyName, err) - return -fuse.EIO + return -fuse.EACCES, err + } + if !typeCheck(nodeStat.Mode, nodeType) { + return -fuse.ENOTDIR, errIOType } - case *ipnsNode: - case *ipfsNode: - return -fuse.EROFS - case *mfsNode: - if err := mfsRemove(fs.filesRoot, path[frs:]); err != nil { - log.Errorf("Unlink - DBG EIO %q %s", path, err) - return -fuse.EIO + //TODO: check if empty at FS level? + } else { + if err != nil { + return -fuse.EIO, err } } - return fuseSuccess + + callContext, cancel := deriveCallContext(fs.ctx) + defer cancel() + + fErr, gErr := fsNode.Remove(callContext) + if gErr == nil { + fs.cc.ReleasePath(parentPath) + fs.cc.ReleasePath(path) + } + return fErr, gErr } -func mfsRemove(mRoot *mfs.Root, path string) error { +func mfsRemove(mRoot *mfs.Root, path string) (int, error) { dir, name := gopath.Split(path) parent, err := mfs.Lookup(mRoot, dir) if err != nil { - return fmt.Errorf("parent lookup: %s", err) + return -fuse.ENOENT, fmt.Errorf("parent lookup: %s", err) } pDir, ok := parent.(*mfs.Directory) if !ok { - return fmt.Errorf("no such file or directory: %s", path) + return -fuse.ENOTDIR, errIOType } - if err = pDir.Unlink(name); err != nil { - return err + return -fuse.EIO, err } - //TODO: is it the callers responsibility to flush? - //return pDir.Flush() - return nil + return fuseSuccess, nil } diff --git a/core/commands/mount/dirio.go b/core/commands/mount/dirio.go index 0b6c3dce742..8da138d79cb 100644 --- a/core/commands/mount/dirio.go +++ b/core/commands/mount/dirio.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + gopath "path" "sync" "time" @@ -18,8 +19,9 @@ import ( //TODO: rename? directoryStreamHandle type directoryStream struct { - record fusePath - entries []directoryEntry + record fusePath + //entries []directoryEntry + entries []fusePath err error init *sync.Cond @@ -28,18 +30,27 @@ type directoryStream struct { //stream <-chan DirectoryMessage } -//TODO: better name; AsyncDirEntry? -// API format bridge -type DirectoryMessage struct { - directoryEntry +// API format bridges +type directoryStringEntry struct { + string error } +type directoryFuseEntry struct { + fusePath + error +} + +func (ds *directoryStream) Record() fusePath { + return ds.record +} + +func (ds *directoryStream) Lock() { + //TODO: figure out who calls this; I think we need it for re-init of shared array + ds.record.Lock() +} -//NOTE: purpose is to be cached and updated when needed instead of re-initialized each call for path; thread safe as a consequent -type directoryEntry struct { - //sync.RWMutex - label string - stat *fuse.Stat_t +func (ds *directoryStream) Unlock() { + ds.record.Unlock() } func (fs *FUSEIPFS) Readdir(path string, @@ -53,7 +64,7 @@ func (fs *FUSEIPFS) Readdir(path string, return -fuse.EINVAL } - dh, err := fs.LookupDirHandle(fh) + ioIf, err := fs.getIo(fh, unixfs.TDirectory) if err != nil { fs.RUnlock() log.Errorf("Readdir - [%X]%q: %s", fh, path, err) @@ -62,87 +73,124 @@ func (fs *FUSEIPFS) Readdir(path string, } return -fuse.EIO } - dh.record.RLock() + dio := ioIf.(FsDirectory) + dio.Lock() + defer dio.Unlock() fs.RUnlock() - defer dh.record.RUnlock() - if dh.io.Entries() == 0 { - //TODO: reconsider/discuss this behaviour; dots are not actually required in POSIX or FUSE - // not having them on empty directories causes things like `dir` and `ls` to fail since they look for the dot, but maybe they should since IPFS does not store dot entries in its directories - fill(".", dh.record.Stat(), -1) + //NOTE: dots are not required in the standard; we only fill them for compatibility with programs that don't expect truly empty directories + if dio.Entries() == 0 { + fStat, err := dio.Record().Stat() + if err != nil { + log.Errorf("Readdir - %q can't fetch metadata: %s", path, err) + return -fuse.EBADF // node must be invalid in some way + } + fill(".", fStat, -1) return fuseSuccess } - for { - select { - case <-fs.ctx.Done(): - return -fuse.EBADF - case msg := <-dh.io.Read(ofst): - err = msg.error - label := msg.directoryEntry.label - stat := msg.directoryEntry.stat - - if err != nil { - if err == io.EOF { - fill(label, stat, -1) - return fuseSuccess - } - log.Errorf("Readdir - %q list err: %s", path, err) - return -fuse.ENOENT - } + ctx, cancel := context.WithCancel(fs.ctx) + defer cancel() + for msg := range dio.Read(ctx, ofst) { + if msg.error != nil && msg.error != io.EOF { + log.Errorf("Readdir - %q entry err: %s", path, msg.error) + return -fuse.ENOENT + } - ofst++ - if !fill(label, stat, ofst) { - return fuseSuccess - } + label := gopath.Base(msg.fusePath.String()) + fStat, err := msg.fusePath.Stat() + if err != nil { + log.Errorf("Readdir - %q can't fetch metadata for %q: %s", path, label, err) + return -fuse.ENOENT + } + + ofst++ + if !fill(label, fStat, ofst) { + return fuseSuccess // fill signaled an early exit } } + return fuseSuccess } -func (ds *directoryStream) Read(ofst int64) <-chan DirectoryMessage { - msgChan := make(chan DirectoryMessage, 1) - if int64(cap(ds.entries)) <= ofst { - msgChan <- DirectoryMessage{error: fmt.Errorf("invalid offset %d <= %d", cap(ds.entries), ofst)} +func (ds *directoryStream) Read(ctx context.Context, ofst int64) <-chan directoryFuseEntry { + msgChan := make(chan directoryFuseEntry, 1) + entCap := cap(ds.entries) + if int(ofst) > entCap { + msgChan <- directoryFuseEntry{error: fmt.Errorf("invalid offset:%d entries:%d", ofst, entCap)} return msgChan } - if ds.init != nil { - ds.init.L.Lock() - for int64(len(ds.entries)) < ofst || ds.err == nil { - ds.init.Wait() - } + //FIXME: we need to be able to cancel this + go func() { + out: + for i := int(ofst); i != entCap; i++ { + if ds.err != io.EOF { // directory isn't fully populated + ds.init.L.Lock() + for len(ds.entries) < i || ds.err == nil { + ds.init.Wait() // wait until offset is populated or error + } - switch ds.err { - case io.EOF: - ds.init = nil - ds.err = nil - default: - msgChan <- DirectoryMessage{error: ds.err} - return msgChan - } - } + switch ds.err { + case io.EOF: + // all entries are ready; init is nil + break + case nil: + // entry at offset is ready, but spool is not finished yet + ds.init.L.Unlock() + break + default: + // spool reported an error + ds.init.L.Unlock() + msgChan <- directoryFuseEntry{error: ds.err} + return + } + } - // NOTE: it it assumed that if len(entries) == ofst and EOF is set, that the entry slot is populated - // if the spooler lies, we'll likely panic - // index change 0 -> 1 - if ofst+1 == int64(len(ds.entries)) { - msgChan <- DirectoryMessage{directoryEntry: ds.entries[ofst], error: io.EOF} - } else { - msgChan <- DirectoryMessage{directoryEntry: ds.entries[ofst]} - } + msg := directoryFuseEntry{fusePath: ds.entries[i-1]} + if i == entCap { + msg.error = io.EOF + } + select { + case <-ctx.Done(): + msg.error = ctx.Err() + msgChan <- msg + break out + case msgChan <- msg: + continue + } + } + close(msgChan) + }() return msgChan } -//TODO: [readdirplus] stats //TODO: name: pulse -> timeoutExtender? signalCallback? entryEvent? -//TODO: document/handle this is only intended to be used with non-empty directories; length must be checked by caller -func (ds *directoryStream) spool(ctx context.Context, inStream <-chan DirectoryMessage, pulse func()) { - defer pulse() +func (ds *directoryStream) spool(tctx timerContext, inStream <-chan directoryStringEntry, pulse func()) { + defer tctx.Cancel() + defer func() { + if ds.err != io.EOF { + pulse() + } + }() + + lf := tctx.Value(lookupKey{}) + if lf == nil { + ds.err = fmt.Errorf("lookup function not provided (via context) to directory spooler") + return + } + lookup, ok := lf.(lookupFn) + if !ok { + ds.err = fmt.Errorf("provided lookup function does not match signature") + return + } + + var wg sync.WaitGroup entCap := cap(ds.entries) for i := 0; i != entCap; i++ { select { - case <-ctx.Done(): + case <-tctx.Done(): + ds.err = tctx.Err() return case msg := <-inStream: if msg.error != nil { @@ -150,45 +198,55 @@ func (ds *directoryStream) spool(ctx context.Context, inStream <-chan DirectoryM return } - if msg.directoryEntry.label == "" { - ds.err = fmt.Errorf("directory contains empty entry label") - return - } + label := msg.string - if fReaddirPlus && msg.directoryEntry.stat == nil { - ds.err = fmt.Errorf("Readdir - stat for %q is not initialized", msg.directoryEntry.label) + if label == "" { + ds.err = fmt.Errorf("directory contains empty entry label") return } - ds.entries = append(ds.entries, msg.directoryEntry) - pulse() + wg.Add(1) // fan out + go func(i int) { + defer wg.Done() + fsNode, err := lookup(label) + if err != nil { + ds.err = fmt.Errorf("directory entry %q lookup err: %s", label, err) + return + } + ds.init.L.Lock() + ds.entries = append(ds.entries, fsNode) + if i == entCap-1 { + ds.err = io.EOF + } + ds.init.L.Unlock() + pulse() + }(i) } } - ds.err = io.EOF + wg.Wait() + ds.init = nil } func (ds *directoryStream) Entries() int { return cap(ds.entries) } -func coreMux(cc <-chan coreiface.LsLink) <-chan DirectoryMessage { - msgChan := make(chan DirectoryMessage) +func coreMux(cc <-chan coreiface.LsLink) <-chan directoryStringEntry { + msgChan := make(chan directoryStringEntry) go func() { for m := range cc { - ent := directoryEntry{label: m.Link.Name} - msgChan <- DirectoryMessage{directoryEntry: ent, error: m.Err} + msgChan <- directoryStringEntry{string: m.Link.Name, error: m.Err} } close(msgChan) }() return msgChan } -func mfsMux(uc <-chan unixfs.LinkResult) <-chan DirectoryMessage { - msgChan := make(chan DirectoryMessage) +func mfsMux(uc <-chan unixfs.LinkResult) <-chan directoryStringEntry { + msgChan := make(chan directoryStringEntry) go func() { for m := range uc { - ent := directoryEntry{label: m.Link.Name} - msgChan <- DirectoryMessage{directoryEntry: ent, error: m.Err} + msgChan <- directoryStringEntry{string: m.Link.Name, error: m.Err} } close(msgChan) }() @@ -201,140 +259,60 @@ func activeDir(fsNode fusePath) FsDirectory { return nil } -func (fs *FUSEIPFS) yieldDirIO(fsNode fusePath) (FsDirectory, error) { - dirStream := &directoryStream{record: fsNode} - switch fsNode.(type) { - default: - return nil, errors.New("unexpected type") - - case *mountRoot: - dirStream.entries = fs.roots +func mfsYieldDirIO(ctx context.Context, mRoot *mfs.Root, mPath string, entryTimeout time.Duration) (FsDirectory, error) { + //NOTE: this special timeout context is extended in backgroundDir + asyncContext := deriveTimerContext(ctx, entryTimeout) - case *ipfsRoot: - pins, err := fs.core.Pin().Ls(fs.ctx, coreoptions.Pin.Type.Recursive()) - if err != nil { - log.Errorf("ipfsRoot - Ls err: %v", err) - return nil, err - } - - ents := make([]directoryEntry, 0, len(pins)) - for _, pin := range pins { - //TODO: [readdirplus] stats - ents = append(ents, directoryEntry{label: pin.Path().Cid().String()}) - } - dirStream.entries = ents - - case *ipnsRoot: - dirStream.entries = fs.ipnsRootSubnodes() + mfsChan, entryCount, err := mfsSubNodes(asyncContext, mRoot, mPath) + if err != nil { + return nil, err } - return dirStream, nil + return backgroundDir(asyncContext, entryCount, mfsChan) } -func (fs *FUSEIPFS) yieldAsyncDirIO(ctx context.Context, timeoutGrace time.Duration, fsNode fusePath) (FsDirectory, error) { - if cached := activeDir(fsNode); cached != nil { - return cached, nil +func coreYieldDirIO(ctx context.Context, corePath coreiface.Path, core coreiface.CoreAPI, entryTimeout time.Duration) (FsDirectory, error) { + callContext, cancel := deriveCallContext(ctx) + oStat, err := core.Object().Stat(callContext, corePath) + if err != nil { + cancel() + return nil, err } + cancel() - var inputChan <-chan DirectoryMessage - var entryCount int - - switch fsNode.(type) { - default: - return nil, errors.New("unexpected type") - case *ipfsNode, *ipnsNode: - globalNode := fsNode - if isReference(fsNode) { - var err error - globalNode, err = fs.resolveToGlobal(fsNode) - if err != nil { - return nil, err - } - } - - //TODO: [readdirplus] stats - coreChan, err := fs.core.Unixfs().Ls(ctx, globalNode, coreoptions.Unixfs.ResolveChildren(false)) - if err != nil { - return nil, err - } - - oStat, err := fs.core.Object().Stat(ctx, globalNode) - if err != nil { - return nil, err - } - - entryCount = oStat.NumLinks - if entryCount != 0 { - inputChan = coreMux(coreChan) - } - - case *mfsNode, *mfsRoot, *ipnsKey: - var ( //XXX: there's probably a better way to handle this; go prevents a nice fallthrough - mRoot *mfs.Root - mPath string - ) - if _, ok := fsNode.(*ipnsKey); ok { - var err error - if mRoot, mPath, err = fs.ipnsMFSSplit(fsNode.String()); err != nil { - return nil, err - } - } else { - mRoot = fs.filesRoot - mPath = fsNode.String()[frs:] - } - - mfsChan, count, err := fs.mfsSubNodes(mRoot, mPath) - if err != nil { - return nil, err - } - entryCount = count - if entryCount != 0 { - inputChan = mfsMux(mfsChan) - } + //NOTE: this special timeout context is extended in backgroundDir + asyncContext := deriveTimerContext(ctx, entryTimeout) + coreChan, err := core.Unixfs().Ls(asyncContext, corePath, coreoptions.Unixfs.ResolveChildren(false)) + if err != nil { + return nil, err } - //TODO: move this to doc.go or something - /* Opendir() -> .spool() -> Readdir() real-time, cross thread synchronization - - .init: use to .init.Wait() in reader, until entry slot N or .err is populated - set .init to nil in waiting thread after processing error|EOF - - .spool(): responsible for populating directory entries list and directory's error value - set .err to io.EOF when finished without errors - - timeout.AfterFunc(): responsible for reporting timeout to directory, - cleaning up resources, and sending wake-up event. - Basically an object with this capability: - context.WithDeadline(deadline, func()).Reset(duration) - - pulser(): responsible for extending timeout and sending wake-up event - */ - - backgroundDir := &directoryStream{record: fsNode, entries: make([]directoryEntry, 0, entryCount)} + return backgroundDir(asyncContext, oStat.NumLinks, coreMux(coreChan)) +} + +//TODO: refine docs +/* backgroundDir documentation, real-time based failure, intended to be initialized once in the background and shared, reset upon change +- .init: use to .init.Wait() in reader, until entry slot N or .err is populated + if .err == io.EOF then .init == nil and should not be used (even if Wait() was previously called) +- .spool(): responsible for populating .entries and .err + set .err to io.EOF when all entries are processed +- entryCallback(): called after entry is processed and immediately before .init is freed +*/ +func backgroundDir(tctx timerContext, entryCount int, inputChan <-chan directoryStringEntry) (FsDirectory, error) { + backgroundDir := &directoryStream{entries: make([]fusePath, 0, entryCount)} if entryCount == 0 { - backgroundDir.err = io.EOF + backgroundDir.err = errors.New("empty directory stream") + tctx.Cancel() return backgroundDir, nil } backgroundDir.init = sync.NewCond(&sync.Mutex{}) - callContext, cancel := context.WithCancel(ctx) - cancelClosure := func() { - if backgroundDir.err == nil { // don't overwrite error if it existed before the timeout - backgroundDir.err = errors.New("timed out") - } - cancel() - } - - timeout := time.AfterFunc(timeoutGrace, cancelClosure) pulser := func() { - defer backgroundDir.init.Broadcast() - if backgroundDir.err != nil { - return - } - - if !timeout.Stop() { - <-timeout.C - } - timeout.Reset(timeoutGrace) + tctx.Reset() // extend context timeout + backgroundDir.init.Broadcast() // wake up .Read() if it's waiting } - backgroundDir.spool(callContext, inputChan, pulser) + backgroundDir.spool(tctx, inputChan, pulser) return backgroundDir, nil } diff --git a/core/commands/mount/event.go b/core/commands/mount/event.go index 1988d1f6e37..94f0161ef15 100644 --- a/core/commands/mount/event.go +++ b/core/commands/mount/event.go @@ -1,53 +1 @@ package fusemount - -import ( - "sync" - "time" -) - -//TODO: config: IPNS record-lifetime, mfs lifetime -func (fs *FUSEIPFS) dbgBackgroundRoutine() { - var nodeType typeToken - - // invalidate cache entries - // TODO refresh nodes in background instead? - - var wg sync.WaitGroup - for { - select { - case <-fs.ctx.Done(): - log.Warning("fs context is done") - return - case <-time.After(10 * time.Second): - nodeType = tIPNSKey - case <-time.After(11 * time.Second): - nodeType = tIPNS - case <-time.After(2 * time.Minute): - nodeType = tMFS - } - - //this is likely suboptimal, quick hacks for debugging - fs.Lock() - wg.Add(2) - go func() { - defer wg.Done() - for _, handle := range fs.fileHandles { - path := handle.record.String() - if parsePathType(path) == nodeType { - fs.cc.ReleasePath(path) - } - } - }() - go func() { - defer wg.Done() - for _, handle := range fs.dirHandles { - path := handle.record.String() - if parsePathType(path) == nodeType { - fs.cc.ReleasePath(path) - } - } - }() - wg.Wait() - fs.Unlock() - } -} diff --git a/core/commands/mount/filesystem.go b/core/commands/mount/filesystem.go index 177e0b4befd..ac31abc2b17 100644 --- a/core/commands/mount/filesystem.go +++ b/core/commands/mount/filesystem.go @@ -4,18 +4,22 @@ import ( "context" "errors" "fmt" - "io" + "os" + gopath "path" "runtime" "sync" + "time" + "unsafe" "github.com/billziss-gh/cgofuse/fuse" mi "github.com/ipfs/go-ipfs/core/commands/mount/interface" - dag "gx/ipfs/QmPJNbVw8o3ohC43ppSXyNXwYKsWShG4zygnirHptfbHri/go-merkledag" coreiface "gx/ipfs/QmXLwxifxwfc2bAwq6rdjbYqAsGzWsDE9RM5TWMGtykyj6/interface-go-ipfs-core" coreoptions "gx/ipfs/QmXLwxifxwfc2bAwq6rdjbYqAsGzWsDE9RM5TWMGtykyj6/interface-go-ipfs-core/options" mfs "gx/ipfs/Qmb74fRYPgpjYzoBV7PAVNmP3DQaRrh8dHdKE4PwnF3cRx/go-mfs" logging "gx/ipfs/QmbkT7eMTyXfpeyB3ZMxxcxg7XH8t6uXp49jqzz4HB7BGF/go-log" + unixfs "gx/ipfs/QmcYUTQ7tBZeH1CLsZM2S3xhMEZdvUgXvbjhpMsLDpk3oJ/go-unixfs" + upb "gx/ipfs/QmcYUTQ7tBZeH1CLsZM2S3xhMEZdvUgXvbjhpMsLDpk3oJ/go-unixfs/pb" ) // NOTE: readdirplus isn't supported on all platforms, being aware of this reduces duplicate metadata construction @@ -28,15 +32,47 @@ var ( mfsSync = false cidCacheEnabled = true - errNoLink = errors.New("not a symlink") - errInvalidHandle = errors.New("invalid handle") - errNoKey = errors.New("key not found") - errInvalidPath = errors.New("invalid path") - errInvalidArg = errors.New("invalid argument") - errReadOnly = errors.New("read only section") + //TODO: replace errNoLink (likely with errIOType) + errNoLink = errors.New("not a symlink") + errInvalidHandle = errors.New("invalid handle") + errNoKey = errors.New("key not found") + errInvalidPath = errors.New("invalid path") + errInvalidArg = errors.New("invalid argument") + errReadOnly = errors.New("read only section") + errIOType = errors.New("requested IO for node does not match node type") + errUnexpected = errors.New("unexpected node type") + errNotInitialized = errors.New("node metadata is not initialized") + errRoot = errors.New("root initialization exception") + errRecurse = errors.New("hit recursion limit") ) -const fuseSuccess = 0 +type typeToken = uint +type FsType = upb.Data_DataType + +const ( + fuseSuccess = 0 +) + +const ( + invalidIndex = ^uint64(0) + + filesNamespace = "files" + filesRootPath = "/" + filesNamespace + filesRootPrefix = filesRootPath + "/" + frs = len(filesRootPath) //TODO: remove this; makes purpose less obvious where used +) + +//TODO: configurable +const ( + callTimeout = 10 * time.Second + entryTimeout = callTimeout + ipnsTTL = 30 * time.Second +) + +//context key tokens +type lookupKey struct{} +type dagKey struct{} +type lookupFn func(string) (fusePath, error) //TODO: platform specific routine mountArgs() func InvokeMount(mountPoint string, filesRoot *mfs.Root, api coreiface.CoreAPI, ctx context.Context) (fsi mi.Interface, err error) { @@ -107,14 +143,35 @@ type FUSEIPFS struct { // set in init active bool ctx context.Context - roots []directoryEntry + nameRoots nameRootIndex + handles fsHandles + //TODO: ipnsKeys map[string:keyname]{fusePath,IoType?} cc cidCache mountTime fuse.Timespec // NOTE: heap equivalent; prevent Go gc claiming our objects between FS operations - fileHandles map[uint64]*fileHandle - dirHandles map[uint64]*dirHandle - nameRoots map[string]*mfs.Root //TODO: see note on nameYieldFileIO + //nameRoots map[string]*mfs.Root //TODO: see note on nameYieldFileIO + + /* implementation note + Handles are addresses of IO objects/interfaces + to assure they're not garbage collected, we store the IO's record in the filesystem scope during Open() + the record itself stores the IO object on it + in a map who's index is the same int (address of the IO object) + */ + /* + Open(){ + inode, iointerface, err := record.YieldIo(type) { + io, err := yieldIo(record) + inode := &io + record.handles[ + } + fs.handles[inode] = record + } + Read(){ + io, err := record.GetIo(fh) + io.Read() + } + */ } /* @@ -151,12 +208,15 @@ func (fs *FUSEIPFS) Init() { return } - fs.roots = []directoryEntry{ - directoryEntry{label: "ipfs"}, - directoryEntry{label: "ipns"}, - directoryEntry{label: filesNamespace}, + var subroots = [...]string{"/ipfs", "/ipns", filesRootPrefix} + //TODO + keycount + ipns keys + var ipnsKeys []coreiface.Key + if ipnsKeys, chanErr = fs.core.Key().List(fs.ctx); chanErr != nil { + log.Errorf("[FATAL] ipns keys could not be initialized: %s", chanErr) + return } + //TODO: shadow var check; must assign to chanErr oAPI, chanErr := fs.core.WithOptions(coreoptions.Api.Offline(true)) if chanErr != nil { log.Errorf("[FATAL] offline API could not be initialized: %s", chanErr) @@ -169,49 +229,11 @@ func (fs *FUSEIPFS) Init() { return } - fs.nameRoots = make(map[string]*mfs.Root) - for _, key := range nameKeys { - keyNode, scopedErr := oAPI.ResolveNode(fs.ctx, key.Path()) - if scopedErr != nil { - log.Warning("IPNS key %q could not be resolved: %s", key.Name(), scopedErr) - fs.nameRoots[key.Name()] = nil - continue - } - - pbNode, ok := keyNode.(*dag.ProtoNode) - if !ok { - log.Warningf("IPNS key %q has incompatible type %T", key.Name(), keyNode) - fs.nameRoots[key.Name()] = nil - continue - } - - keyRoot, scopedErr := mfs.NewRoot(fs.ctx, fs.core.Dag(), pbNode, ipnsPublisher(key.Name(), oAPI.Name())) - if scopedErr != nil { - log.Warningf("IPNS key %q could not be mapped to MFS root: %s", key.Name(), scopedErr) - fs.nameRoots[key.Name()] = nil - continue - } - fs.nameRoots[key.Name()] = keyRoot - } - - fs.fileHandles = make(map[uint64]*fileHandle) - fs.dirHandles = make(map[uint64]*dirHandle) - + fs.nameRoots = &mfsSharedIndex{roots: make(map[string]*mfsReference, len(nameKeys)+1)} // +Files API + fs.handles = make(fsHandles) fs.mountTime = fuse.Now() - //TODO: implement for real - go fs.dbgBackgroundRoutine() - log.Debug("init finished") -} - -type fileHandle struct { - record fusePath - io FsFile -} - -type dirHandle struct { - record fusePath - io FsDirectory + log.Debug("init finished: %s", fs.mountTime) } func (fs *FUSEIPFS) Open(path string, flags int) (int, uint64) { @@ -219,26 +241,72 @@ func (fs *FUSEIPFS) Open(path string, flags int) (int, uint64) { defer fs.Unlock() log.Debugf("Open - Request {%X}%q", flags, path) - if fs.AvailableHandles(aFiles) == 0 { + if fs.AvailableHandles() == 0 { log.Error("Open - all handle slots are filled") return -fuse.ENFILE, invalidIndex } +lookup: + fsNode, indexErr := fs.shallowLookupPath(path) + var nodeStat *fuse.Stat_t + if indexErr == nil { + var err error + nodeStat, err = fsNode.Stat() + if err != nil { + log.Errorf("Open - %q: %s", path, err) + return -fuse.EACCES, invalidIndex + } + } + + // POSIX specifications + if flags&O_NOFOLLOW != 0 { + if indexErr == nil { + if nodeStat.Mode&fuse.S_IFMT == fuse.S_IFLNK { + log.Errorf("Open - nofollow requested but %q is a link", path) + return -fuse.ELOOP, invalidIndex + } + } + } + if flags&fuse.O_CREAT != 0 { - if flags&fuse.O_EXCL != 0 { - _, err := fs.LookupPath(path) - if err == nil { - log.Errorf("Open/Create - exclusive flag provided but %q already exists", path) + switch indexErr { + case os.ErrNotExist: + nodeType := unixfs.TFile + if flags&O_DIRECTORY != 0 { + nodeType = unixfs.TDirectory + } + + callContext, cancel := deriveCallContext(fs.ctx) + defer cancel() + fErr, gErr := fsNode.Create(callContext, nodeType) + if gErr != nil { + log.Errorf("Create - %q: %s", path, gErr) + return fErr, invalidIndex + } + // node was created API side, clear create bits, jump back, and open it FS side + // respecting link restrictions + flags &^= (fuse.O_EXCL | fuse.O_CREAT) + goto lookup + + case nil: + if flags&fuse.O_EXCL != 0 { + log.Errorf("Create - exclusive flag provided but %q already exists", path) return -fuse.EEXIST, invalidIndex } - } - fErr, gErr := fs.mknod(path) - if gErr != nil { - log.Errorf("Open/Create - %q: %s", path, gErr) - return fErr, invalidIndex + + if nodeStat.Mode&fuse.S_IFMT == fuse.S_IFDIR { + if flags&O_DIRECTORY == 0 { + log.Error("Create - regular file requested but %q resolved to an existing directory", path) + return -fuse.EISDIR, invalidIndex + } + } + default: + log.Errorf("Create - unexpected %q: %s", path, indexErr) + return -fuse.EACCES, invalidIndex } } + // Open proper -- resolves reference nodes fsNode, err := fs.LookupPath(path) if err != nil { log.Errorf("Open - path err: %s", err) @@ -247,14 +315,49 @@ func (fs *FUSEIPFS) Open(path string, flags int) (int, uint64) { fsNode.Lock() defer fsNode.Unlock() - fh, err := fs.newFileHandle(fsNode) //TODO: flags + nodeStat, err = fsNode.Stat() if err != nil { - log.Errorf("Open - sunk %q:%s", fsNode.String(), err) + log.Errorf("Open - node %q not initialized", path) + return -fuse.EACCES, invalidIndex + } + + if nodeStat.Mode&fuse.S_IFMT != fuse.S_IFLNK { + return -fuse.ELOOP, invalidIndex //NOTE: this should never happen, lookup should resolve all + } + + // if request is dir but node is dir + if (flags&O_DIRECTORY != 0) && (nodeStat.Mode&fuse.S_IFMT != fuse.S_IFDIR) { + log.Error("Open - Directory requested but %q does not resolve to a directory", path) + return -fuse.ENOTDIR, invalidIndex + } + + // if request was file but node is dir + if (flags&O_DIRECTORY == 0) && (nodeStat.Mode&fuse.S_IFMT == fuse.S_IFDIR) { + log.Error("Open - regular file requested but %q resolved to a directory", path) + return -fuse.EISDIR, invalidIndex + } + + callContext, cancel := deriveCallContext(fs.ctx) + defer cancel() + + // io is an interface that points to a struct (generic/void*) + io, err := fsNode.YieldIo(callContext, unixfs.TFile) + if err != nil { + log.Errorf("Open - IO err %q %s", path, err) return -fuse.EIO, invalidIndex } - log.Debugf("Open - Assigned [%X]%q", fh, fsNode) - return fuseSuccess, fh + // the address of io itself must remain the same across calls + // as we are sharing it with the OS + // we use the interface structure itself so that + // on our side we can change data sources + // without invalidating handles on the OS/client side + ifPtr := &io // void *ifPtr = (FsFile*) io; + handle := uint64(uintptr(unsafe.Pointer(ifPtr))) // uint64_t handle = &ifPtr; + fsNode.Handles()[handle] = ifPtr //GC prevention of our double pointer; free upon Release() + + log.Debugf("Open - Assigned [%X]%q", handle, fsNode) + return fuseSuccess, handle } func (fs *FUSEIPFS) Opendir(path string) (int, uint64) { @@ -262,7 +365,7 @@ func (fs *FUSEIPFS) Opendir(path string) (int, uint64) { defer fs.Unlock() log.Debugf("Opendir - Request %q", path) - if fs.AvailableHandles(aDirectories) == 0 { + if fs.AvailableHandles() == 0 { log.Error("Opendir - all handle slots are filled") return -fuse.ENFILE, invalidIndex } @@ -275,32 +378,61 @@ func (fs *FUSEIPFS) Opendir(path string) (int, uint64) { fsNode.Lock() defer fsNode.Unlock() - if !fReaddirPlus { - fh, err := fs.newDirHandle(fsNode) - if err != nil { - log.Errorf("Opendir - %s", err) - return -fuse.ENOENT, invalidIndex - } - log.Debugf("Opendir - Assigned [%X]%q", fh, fsNode) - return fuseSuccess, fh + lookupFn := func(child string) (fusePath, error) { + //return fs.LookupPath(gopath.Join(path, child)) + return fs.shallowLookupPath(gopath.Join(path, child)) } - return -fuse.EACCES, invalidIndex + directoryContext := context.WithValue(fs.ctx, lookupKey{}, lookupFn) + io, err := fsNode.YieldIo(directoryContext, unixfs.TDirectory) + if err != nil { + log.Errorf("Opendir - IO err %q %s", path, err) + return -fuse.EACCES, invalidIndex + } + + ifPtr := &io // see comments on Open() + handle := uint64(uintptr(unsafe.Pointer(ifPtr))) + fsNode.Handles()[handle] = ifPtr + return fuseSuccess, handle } -//TODO: [educational/compiler] how costly is defer vs {ret = x; unlock; return ret} func (fs *FUSEIPFS) Release(path string, fh uint64) int { log.Debugf("Release - [%X]%q", fh, path) fs.Lock() defer fs.Unlock() - return fs.releaseFileHandle(fh) + + fsNode, ok := fs.handles[fh] + if !ok { + log.Errorf("Release - handle %X is invalid", fh) + return -fuse.EBADF + } + fErr, gErr := fsNode.DestroyIo(fh, unixfs.TFile) + if gErr != nil { + log.Errorf("Release - %s", gErr) + } + + // invalidate/free handle regardless of grace + delete(fs.handles, fh) + return fErr } func (fs *FUSEIPFS) Releasedir(path string, fh uint64) int { log.Debugf("Releasedir - [%X]%q", fh, path) fs.Lock() defer fs.Unlock() - return fs.releaseDirHandle(fh) + fsNode, ok := fs.handles[fh] + if !ok { + log.Errorf("Releasedir - handle %X is invalid", fh) + return -fuse.EBADF + } + fErr, gErr := fsNode.DestroyIo(fh, unixfs.TDirectory) + if gErr != nil { + log.Errorf("Releasedir - %s", gErr) + } + + // invalidate/free handle regardless of grace + delete(fs.handles, fh) + return fErr } //TODO: implement @@ -310,7 +442,6 @@ func (fs *FUSEIPFS) Chmod(path string, mode uint32) int { } func (fs *FUSEIPFS) Chown(path string, uid, gid uint32) int { - log.Errorf("Chmod [%d:%d]%q", uid, gid, path) return 0 } @@ -332,84 +463,79 @@ func (fs *FUSEIPFS) Destroy() { } //TODO: close our context + //TODO: close all handles } func (fs *FUSEIPFS) Flush(path string, fh uint64) int { - fs.RLock() + fs.Lock() + defer fs.Unlock() log.Debugf("Flush - Request [%X]%q", fh, path) - h, err := fs.LookupFileHandle(fh) - if err != nil { - fs.RUnlock() - log.Errorf("Flush - bad request [%X]%q: %s", fh, path, err) - if err == errInvalidHandle { - return -fuse.EBADF //TODO: we might want to change this to EIO since Flush is called on Close which implies the handle should have been valid - } - return -fuse.EIO - } - h.record.Lock() - fs.RUnlock() - defer h.record.Unlock() - - if !h.record.Mutable() { - return fuseSuccess + fErr, gErr := syncWrap(fh, 0, true) //XXX: optional parameter 0 is ignored from flush + if gErr != nil { + log.Errorf("Flush - [%X]%q: %s", fh, path, gErr) } - - return h.io.Sync() + return fErr } func (fs *FUSEIPFS) Fsync(path string, datasync bool, fh uint64) int { - fs.RLock() + fs.Lock() + defer fs.Unlock() log.Debugf("Fsync - Request [%X]%q", fh, path) - h, err := fs.LookupFileHandle(fh) - if err != nil { - fs.RUnlock() - log.Errorf("Fsync - bad request [%X]%q: %s", fh, path, err) - if err == errInvalidHandle { - return -fuse.EBADF - } - return -fuse.EIO + fErr, gErr := syncWrap(fh, unixfs.TFile, false) + if gErr != nil { + log.Errorf("Fsync - [%X]%q: %s", fh, path, gErr) } - - h.record.Lock() - fs.RUnlock() - defer h.record.Unlock() - - return h.io.Sync() + return fErr } func (fs *FUSEIPFS) Fsyncdir(path string, datasync bool, fh uint64) int { - fs.RLock() + fs.Lock() + defer fs.Unlock() log.Errorf("Fsyncdir - Request [%X]%q", fh, path) - fsDirNode, err := fs.LookupDirHandle(fh) + fErr, gErr := syncWrap(fh, unixfs.TDirectory, false) + if gErr != nil { + log.Errorf("Fsyncdir - [%X]%q: %s", fh, path, gErr) + } + return fErr +} + +func syncWrap(fh uint64, nodeType FsType, isFlush bool) (int, error) { + fsNode, io, err := invertedLookup(fh) if err != nil { - fs.RUnlock() - log.Errorf("Fsyncdir - [%X]%q: %s", fh, path, err) if err == errInvalidHandle { - return -fuse.EBADF + return -fuse.EBADF, err } - return -fuse.EIO + return -fuse.EIO, err + } + fsNode.Lock() + defer fsNode.Unlock() + if !fsNode.Mutable() { + return fuseSuccess, nil } - fsDirNode.record.Lock() - fs.RUnlock() - defer fsDirNode.record.Unlock() - /* FIXME: not implemented - fsDirNode, err := dirFromHandle(fh) + fStat, err := fsNode.Stat() if err != nil { - fs.Unlock() - log.Errorf("Fsyncdir - [%X]%q: %s", fh, path, err) - if err == errInvalidHandle { - return -fuse.EBADF + return -fuse.EIO, err + } + + if !isFlush { // flush is an untyped request + if !typeCheck(fStat.Mode, nodeType) { + return -fuse.EINVAL, errIOType } - return -fuse.EIO } - return fsDirNode.Sync() - */ - return fuseSuccess + switch nodeType { + case unixfs.TFile: + return io.(FsFile).Sync() + case unixfs.TDirectory: + //NOOP + return fuseSuccess, nil + default: + return -fuse.EIO, errUnexpected + } } func (fs *FUSEIPFS) Getxattr(path, name string) (int, []byte) { @@ -446,74 +572,105 @@ func (fs *FUSEIPFS) Readlink(path string) (int, string) { fs.RUnlock() defer fsNode.RUnlock() - if isDevice(fsNode) { + //TODO: + //lIo, err := fsNode.YieldIo(nil, unixfs.TSymlink) + // + + fStat, err := fsNode.Stat() + if err != nil { + log.Errorf("Readlink - %q: %s", path, err) + return -fuse.EIO, "" + } + + if !typeCheck(fStat.Mode, unixfs.TSymlink) { + log.Errorf("Readlink - %q is not a symlink", path) return -fuse.EINVAL, "" } - if isReference(fsNode) { - var err error - fsNode, err = fs.resolveToGlobal(fsNode) - if err != nil { - log.Errorf("Readlink - node resolution error: %s", err) - return -fuse.EIO, "" - } + //TODO: need global path for core + + callContext, cancel := deriveCallContext(fs.ctx) + defer cancel() + ipldNode, err := fs.core.ResolveNode(callContext, fsNode) + if err != nil { + log.Errorf("Readlink - node resolution error: %s", err) + return 0, "" } - target, err := fs.fuseReadlink(fsNode) + ufsNode, err := unixfs.ExtractFSNode(ipldNode) if err != nil { - if err == errNoLink { - log.Errorf("Readlink - %q is not a symlink", path) - return -fuse.EINVAL, "" - } - log.Errorf("Readlink - unexpected link resolution error: %s", err) return -fuse.EIO, "" } - return len(target), target -} + target := string(ufsNode.Data()) + tLen := len(target) + if tLen != int(fStat.Size) { + log.Errorf("Readlink - target size mismatched node:%d != target:%d", fStat.Size, tLen) + return -fuse.EIO, "" + } -type handleErrorPair struct { - fhi uint64 - err error + return tLen, target } -//TODO: test this -func (fs *FUSEIPFS) refreshFileSiblings(fh uint64, h *fileHandle) (failed []handleErrorPair) { - handles := *h.record.Handles() - if len(handles) == 1 && handles[0] == fh { +//NOTE: caller should retain FS (R)Lock +// caller need not do asserting check; it's rolled into an error +func (fs *FUSEIPFS) getIo(fh uint64, ioType FsType) (io interface{}, err error) { + err = errInvalidHandle + if fh == invalidIndex { return } - for _, fhi := range handles { - if fhi == fh { - continue + defer func() { + if r := recover(); r != nil { + log.Errorf("getIo recovered from panic, likely invalid handle: %v", r) + io = nil + err = errInvalidHandle } - curFh, err := fs.LookupFileHandle(fhi) - if err != nil { - failed = append(failed, handleErrorPair{fhi, err}) - continue + }() + + // L0 dereference address; equivalent to io = *fh with trap + // uint -> cast to untyped pointer -> pointer is dereferenced -> type is checked + switch ioType { + case unixfs.TFile: + if io, ok := (*(*interface{})(unsafe.Pointer(uintptr(fh)))).(FsFile); ok { + return io, nil } + return nil, errIOType + case unixfs.TDirectory: + if io, ok := (*(*interface{})(unsafe.Pointer(uintptr(fh)))).(FsDirectory); ok { - oCur, err := curFh.io.Seek(0, io.SeekCurrent) - if err != nil { - failed = append(failed, handleErrorPair{fhi, err}) - continue + return io, nil } - err = curFh.io.Close() - if err != nil { - failed = append(failed, handleErrorPair{fhi, err}) - continue + return nil, errIOType + case unixfs.TSymlink: + if io, ok := (*(*interface{})(unsafe.Pointer(uintptr(fh)))).(FsLink); ok { + return io, nil } - curFh.io = nil + return nil, errIOType + default: + return nil, errUnexpected + } - posixIo, err := fs.yieldFileIO(h.record) - if err != nil { - failed = append(failed, handleErrorPair{fhi, err}) - continue + // L1 double map lookup + // record lookup -> io lookup -> io + /* + if record, ok := fs.handles[fh]; ok { + if io, ok := record.Handles()[fh]; ok { + switch ioType { + case unixfs.TFile: + if _, ok := io.(FsFile); ok { + return io, nil + } + return nil, errIOType + case unixfs.TDirectory: + if _, ok := io.(FsDirectory); ok { + return io, nil + } + return nil, errIOType + default: + return nil, errUnexpected + } + } } - - posixIo.Seek(oCur, io.SeekStart) - curFh.io = posixIo - } - return + */ } diff --git a/core/commands/mount/index.go b/core/commands/mount/index.go index 9735f2dc360..a28c5161f67 100644 --- a/core/commands/mount/index.go +++ b/core/commands/mount/index.go @@ -4,152 +4,78 @@ import ( "errors" "fmt" "os" - gopath "path" "strings" + "unsafe" - ds "gx/ipfs/QmUadX5EcvrBmxAV9sE7wUWtWSqxns5K84qKJBixmcT1w9/go-datastore" - coreiface "gx/ipfs/QmXLwxifxwfc2bAwq6rdjbYqAsGzWsDE9RM5TWMGtykyj6/interface-go-ipfs-core" - coreoptions "gx/ipfs/QmXLwxifxwfc2bAwq6rdjbYqAsGzWsDE9RM5TWMGtykyj6/interface-go-ipfs-core/options" - ipld "gx/ipfs/QmZ6nzCLwGLVfRzYLpD7pW6UNuBDKEcA2imJtVpbEx2rxy/go-ipld-format" - mfs "gx/ipfs/Qmb74fRYPgpjYzoBV7PAVNmP3DQaRrh8dHdKE4PwnF3cRx/go-mfs" -) - -const ( - invalidIndex = ^uint64(0) + unixfs "gx/ipfs/QmcYUTQ7tBZeH1CLsZM2S3xhMEZdvUgXvbjhpMsLDpk3oJ/go-unixfs" - filesNamespace = "files" - filesRootPath = "/" + filesNamespace - filesRootPrefix = filesRootPath + "/" - frs = len(filesRootPath) + "github.com/billziss-gh/cgofuse/fuse" ) -//TODO: remove alias -type typeToken = uint64 - -//TODO: cleanup const ( - tMountRoot typeToken = iota - tIPFSRoot - tIPNSRoot - tFilesRoot - tRoots + tRoot typeToken = iota + tIPNSKey + tFAPI tIPFS tIPLD - tImmutable tIPNS - tIPNSKey - tMFS - tMutable tUnknown ) -func resolveMFS(filesRoot *mfs.Root, path string) (ipld.Node, error) { - mfsNd, err := mfs.Lookup(filesRoot, path) - if err != nil { - return nil, err - } - ipldNode, err := mfsNd.GetNode() - if err != nil { - return nil, err - } - return ipldNode, nil -} - -func (fs *FUSEIPFS) resolveIpns(path string) (string, error) { - pathKey, remainder, err := ipnsSplit(path) - if err != nil { - return "", err - } - - oAPI, err := fs.core.WithOptions(coreoptions.Api.Offline(true)) - if err != nil { - log.Errorf("API error: %v", err) - return "", err - } - - var nameAPI coreiface.NameAPI - globalPath := path - coreKey, err := resolveKeyName(fs.ctx, oAPI.Key(), pathKey) - switch err { - case nil: // locally owned keys are resolved offline - globalPath = gopath.Join(coreKey.Path().String(), remainder) - nameAPI = oAPI.Name() - - case errNoKey: // paths without named keys are valid, but looked up via network instead of locally - nameAPI = fs.core.Name() - - case ds.ErrNotFound: // a key was generated, but not published to / initialized - return "", fmt.Errorf("IPNS key %q has no value", pathKey) - default: - return "", err - } - - //NOTE: the returned path is not guaranteed to exist - resolvedPath, err := nameAPI.Resolve(fs.ctx, globalPath) - if err != nil { - return "", err - } - //log.Errorf("dbg: %q -> %q -> %q", path, globalPath, resolvedPath) - return resolvedPath.String(), nil -} - -//TODO: see how IPLD selectors handle this kind of parsing func parsePathType(path string) typeToken { + + slashCount := len(strings.Split(path, "/")) switch { - case path == "/": - return tMountRoot - case path == "/ipfs": - return tIPFSRoot - case path == "/ipns": - return tIPNSRoot - case path == filesRootPath: - return tFilesRoot + case slashCount == 1: // `/`, `/ipfs`, ... + return tRoot case strings.HasPrefix(path, "/ipld/"): return tIPLD case strings.HasPrefix(path, "/ipfs/"): return tIPFS case strings.HasPrefix(path, filesRootPrefix): - return tMFS + return tFAPI case strings.HasPrefix(path, "/ipns/"): - if len(strings.Split(path, "/")) == 3 { - return tIPNSKey - } return tIPNS - /* NIY - case strings.HasPrefix(path, "/api/"): - return tAPI - */ } return tUnknown } -//operator, operator! -func parseLocalPath(path string) (fusePath, error) { - switch parsePathType(path) { - case tMountRoot: - return &mountRoot{rootBase: rootBase{ - recordBase: crb("/")}}, nil - case tIPFSRoot: - return &ipfsRoot{rootBase: rootBase{ - recordBase: crb("/ipfs")}}, nil - case tIPNSRoot: - return &ipnsRoot{rootBase: rootBase{ - recordBase: crb("/ipns")}}, nil - case tFilesRoot: - return &mfsRoot{rootBase: rootBase{ - recordBase: crb(filesRootPath)}}, nil +func parseFusePath(fs *FUSEIPFS, subsystemType typeToken, path string) (fusePath, error) { + switch subsystemType { case tIPFS, tIPLD: return &ipfsNode{recordBase: crb(path)}, nil - case tMFS: - return &mfsNode{mutableBase: mutableBase{ - recordBase: crb(path)}}, nil - case tIPNSKey: - return &ipnsKey{ipnsNode{mutableBase: mutableBase{ - recordBase: crb(path)}}}, nil + case tFAPI: + return &filesAPINode{mfsNode{root: fs.filesRoot, recordBase: crb(path[len(filesRootPath):])}}, nil case tIPNS: - return &ipnsNode{mutableBase: mutableBase{ - recordBase: crb(path)}}, nil + nn := &ipnsNode{ipfsNode: ipfsNode{recordBase: crb(path)}} + + //NOTE: path is assumed valid because of tIPNS from previous parser + keyComponent := strings.Split(path, "/")[2] + index := strings.Index(path, keyComponent) + len(keyComponent) + subPath := path[index:] + + callContext, cancel := deriveCallContext(fs.ctx) + defer cancel() + var err error + switch nn.key, err = resolveKeyName(callContext, fs.core.Key(), keyComponent); err { + case nil: // key is owned and found, use mfs methods internally for write access + //nn.ipfsNode.recordBase.path = subPath // mfs expects paths relative to its own root + //nn.subsystem = nn.mfsNode + case errNoKey: + // non-owned keys are valid, but read only + //nn.subsystem = nn.ipfsNode + break + default: + return nil, err + } + + if nn.key != nil && len(subPath) == 0 { // promote `/ipns/ownedKey` to subroot "/" + //nn.path = path[:1] + return &ipnsSubroot{ipnsNode: *nn}, nil + } + return nn, nil + case tUnknown: switch strings.Count(path, "/") { case 0: @@ -165,179 +91,119 @@ func parseLocalPath(path string) (fusePath, error) { } func crb(path string) recordBase { - return recordBase{path: path, handles: &[]uint64{}} + return recordBase{path: path, ioHandles: make(nodeHandles)} } -func (fs *FUSEIPFS) parent(node fusePath) (fusePath, error) { - if _, ok := node.(*mountRoot); ok { - return node, nil +//FIXME: implicit locks +//NOTE: caller should retain FS (R)Lock +func (fs *FUSEIPFS) shallowLookupPath(path string) (fusePath, error) { + if path == "" { + return nil, errInvalidArg } - path := node.String() - i := len(path) - 1 - for i != 0 && path[i] != '/' { - i-- - } - if i == 0 { - return fs.LookupPath("/") + //L1 path -> cid -> cache -?> record + pathCid, err := fs.cc.Hash(path) + if err != nil { + log.Errorf("cache: %s", err) //TODO: remove; debug + } else if cachedNode := fs.cc.Request(pathCid); cachedNode != nil { + //TODO: check record TTL conditions here + //FIXME if expired, reset IO, drop through to parse logic + return cachedNode, nil } - return fs.LookupPath(path[:i]) -} - -func (fs *FUSEIPFS) resolveToGlobal(node fusePath) (fusePath, error) { - switch node.(type) { - case *mfsNode: - //contacts API node - ipldNode, err := resolveMFS(fs.filesRoot, node.String()[frs:]) - if err != nil { - return nil, err - } - - if cachedNode := fs.cc.Request(ipldNode.Cid()); cachedNode != nil { - return cachedNode, nil - } + //L2 check open handles for active paths + //TODO/FIXME: important; shared mutex is required - //TODO: will ipld always be valid here? is there a better way to retrieve the path? - globalNode, err := fs.LookupPath(gopath.Join("/ipld/", ipldNode.String())) - if err != nil { + //L3 parse string path, construct typed-node + var parsedNode fusePath + switch apiType := parsePathType(path); apiType { + case tRoot: + if parsedNode, err = parseRootPath(fs, path); err != nil { return nil, err } - - fs.cc.Add(ipldNode.Cid(), globalNode) - return globalNode, nil - - case *ipnsNode, *ipnsKey: - resolvedPath, err := fs.resolveIpns(node.String()) //contacts API node - if err != nil { + default: + if parsedNode, err = parseFusePath(fs, apiType, path); err != nil { return nil, err } - //TODO: test if the core handles recursion protection here; IPNS->IPNS->... - return fs.LookupPath(resolvedPath) + case tUnknown: + return nil, errUnexpected } - return nil, fmt.Errorf("unexpected reference-node type %T", node) -} - -func isReference(fsNode fusePath) bool { - switch fsNode.(type) { - case *mfsNode, *ipnsNode, *ipnsKey: - return true - default: - return false + // populate node's required data + callContext, cancel := deriveCallContext(fs.ctx) + defer cancel() + if nodeStat, err := parsedNode.InitMetadata(callContext); err != nil { + if err == os.ErrNotExist { // NOTE: non-existent nodes are still valid for creation operations + return parsedNode, err + } + return nil, err } -} -func isDevice(fsNode fusePath) bool { - switch fsNode.(type) { - case *mountRoot, *ipfsRoot, *ipnsRoot, *mfsRoot: - return true - default: - return false - } + fs.cc.Add(pathCid, parsedNode) + return parsedNode, err } -//NOTE: caller should retain FS (R)Lock -func (fs *FUSEIPFS) LookupFileHandle(fh uint64) (handle *fileHandle, err error) { - err = errInvalidHandle - if fh == invalidIndex { - return - } - - defer func() { - if r := recover(); r != nil { - log.Errorf("Lookup recovered from panic, likely invalid handle: %v", r) - handle = nil - err = errInvalidHandle +const linkRecurseLimit = 255 //FIXME: arbitrary debug value +func (fs *FUSEIPFS) LookupPath(path string) (fsNode fusePath, err error) { + targetPath := path + for depth := 0; depth != linkRecurseLimit; depth++ { + fsNode, err = fs.shallowLookupPath(targetPath) + if err != nil { + return + } + var nodeStat *fuse.Stat_t + nodeStat, err = fsNode.Stat() + if err != nil { + return } - }() - //TODO: enable when stable - //L0 direct cast 🤠 - //return (*fileHandle)(unsafe.Pointer(uintptr(fh))), nil + if nodeStat.Mode&fuse.S_IFMT != fuse.S_IFLNK { + return + } - //L1 handle -> lookup -> node - if hs, ok := fs.fileHandles[fh]; ok { - if hs.record != nil { - return hs, nil + // if node is link, resolve to its target + var targetNode fusePath + callContext, cancel := deriveCallContext(fs.ctx) + defer cancel() + ioIf, ioErr := fsNode.YieldIo(callContext, unixfs.TSymlink) + if ioErr != nil { + err = ioErr + return } - //TODO: return separate error? handleInvalidated (handle was active but became bad) vs handleInvalid (never existed in the first place) - } - return nil, errInvalidHandle -} -//NOTE: caller should retain FS (R)Lock -func (fs *FUSEIPFS) LookupDirHandle(fh uint64) (handle *dirHandle, err error) { - err = errInvalidHandle - if fh == invalidIndex { - return + targetPath := ioIf.(FsLink).Target() } + err = errRecurse + return +} +func invertedLookup(fh uint64) (fp fusePath, io interface{}, err error) { defer func() { if r := recover(); r != nil { - log.Errorf("Lookup recovered from panic, likely invalid handle: %v", r) - handle = nil + log.Errorf("invertedLookup recovered from panic, likely invalid handle: %v", r) + fp = nil + io = nil err = errInvalidHandle } }() - //TODO: enable when stable - //L0 direct cast 🤠 - //return (*dirHandle)(unsafe.Pointer(uintptr(fh))), nil - - //L1 handle -> lookup -> node - if hs, ok := fs.dirHandles[fh]; ok { - if hs.record != nil { - return hs, nil - } - } - return nil, errInvalidHandle -} - -//NOTE: caller should retain FS (R)Lock -func (fs *FUSEIPFS) LookupPath(path string) (fusePath, error) { - if path == "" { - return nil, errInvalidArg - } - - //L1 path -> cid -> cache -?> record - pathCid, err := fs.cc.Hash(path) - if err != nil { - log.Errorf("cache: %s", err) - } else if cachedNode := fs.cc.Request(pathCid); cachedNode != nil { - return cachedNode, nil - } - - //L2 path -> full parse+construction - parsedNode, err := parseLocalPath(path) - if err != nil { - return nil, err + if io, ok := (*(*interface{})(unsafe.Pointer(uintptr(fh)))).(FsRecord); ok { + fp = io.Record() + return fp, io, nil } - - if !isDevice(parsedNode) { - if !fs.exists(parsedNode) { - return parsedNode, os.ErrNotExist //NOTE: node is still a valid structure ready for use (i.e. useful for creation/type inspection) - } - } - - fs.cc.Add(pathCid, parsedNode) - return parsedNode, nil + err = errUnexpected + return } -func (fs *FUSEIPFS) exists(parsedNode fusePath) bool { - globalNode := parsedNode - var err error - if isReference(parsedNode) { - - globalNode, err = fs.resolveToGlobal(parsedNode) - if err != nil { - return false - } - } - if _, err = fs.core.ResolvePath(fs.ctx, globalNode); err != nil { //contacts API node and possibly the network - return false - } - - return true +/* +func updateStale(ctx context.Context, fsNode fusePath) error { + //check if node.metadata == stale + /* if is + store type bits + re-init with Stat() + if non-exist or type bits changed; return err + for each node handle + fh = fsNode.YieldIo(unixType) } +*/ diff --git a/core/commands/mount/indexNodes.go b/core/commands/mount/indexNodes.go index 07c08b70cdd..29f09ad67d1 100644 --- a/core/commands/mount/indexNodes.go +++ b/core/commands/mount/indexNodes.go @@ -1,6 +1,16 @@ package fusemount import ( + "context" + "fmt" + dag "gx/ipfs/QmPJNbVw8o3ohC43ppSXyNXwYKsWShG4zygnirHptfbHri/go-merkledag" + coreiface "gx/ipfs/QmXLwxifxwfc2bAwq6rdjbYqAsGzWsDE9RM5TWMGtykyj6/interface-go-ipfs-core" + ipld "gx/ipfs/QmZ6nzCLwGLVfRzYLpD7pW6UNuBDKEcA2imJtVpbEx2rxy/go-ipld-format" + mfs "gx/ipfs/Qmb74fRYPgpjYzoBV7PAVNmP3DQaRrh8dHdKE4PwnF3cRx/go-mfs" + unixfs "gx/ipfs/QmcYUTQ7tBZeH1CLsZM2S3xhMEZdvUgXvbjhpMsLDpk3oJ/go-unixfs" + "os" + gopath "path" + "runtime" "strings" "sync" @@ -12,43 +22,246 @@ type recordBase struct { path string metadata fuse.Stat_t - handles *[]uint64 + //handles *[]uint64 + ioHandles nodeHandles + //openFunc nodeOpenFunc } -type mutableBase struct { +type nodeOpenFunc func(ctx context.Context) (io interface{}, handle uint64, err error) + +type ipfsNode struct { recordBase + core coreiface.CoreAPI } -type ipfsNode struct { +type mfsNode struct { recordBase - //fd coreiface.UnixfsFile - //initOnce sync.Once - //fStat *fuse.Stat_t + root *mfs.Root + //fd mfs.FileDescriptor } type ipnsNode struct { - mutableBase + ipfsNode + mfsNode + + /* FIXME + would be nice to have a union type, comprised of (ipfsNode, mfsNode) + where we can modify the vtable + so we don't have to generate a wrapper shim + i.e. make this compiler legal: + `node.Create()` == node.(ipfsNode, mfsNode).Create() + otherwise, go generate + + if node.haskey { n.vtable = mfs }; else { n.vtable = core } + */ + key coreiface.Key // may be nil for read only IO access + fsRootIndex nameRootIndex // for shared mfs root object storage at FS level } -type ipnsKey struct { +type ipnsSubroot struct { ipnsNode } -type mfsNode struct { - mutableBase - //fd mfs.FileDescriptor +type keyNode struct { + filesAPINode + key coreiface.Key // TODO: api returns static object, we need to maintain it + // fetch it in real time via api when needed, store string instead +} + +type filesAPINode struct { + mfsNode +} + +func (nn *ipnsNode) String() string { + if nn.key != nil { + return gopath.Join(nn.key.Path(), nn.path) + } + return nn.path +} + +func (_ *ipnsNode) NameSpace() string { + return "ipns" +} + +func (_ *filesAPINode) Namespace() string { + return filesNamespace +} + +func (fn *filesAPINode) String() string { + return gopath.Join("/", fn.Namespace(), fn.path) } func (rb *recordBase) String() string { return rb.path } -func (rb *recordBase) Handles() *[]uint64 { - return rb.handles +func (rb *recordBase) Handles() nodeHandles { + return rb.ioHandles } -func (rb *recordBase) Stat() *fuse.Stat_t { - return &rb.metadata +func (rb *recordBase) Stat() (*fuse.Stat_t, error) { + if rb.metadata.Mode == 0 { + return nil, errNotInitialized + } + return &rb.metadata, nil +} + +func (rb *recordBase) Remove(_ context.Context) (int, error) { + return -fuse.EROFS, errReadOnly +} + +func (fn *filesAPINode) Remove(_ context.Context) (int, error) { + return mfsRemove(fn.root, fn.path[len(filesRootPath):]) +} + +func (mn *mfsNode) Remove(_ context.Context) (int, error) { + return mfsRemove(mn.root, mn.path) +} + +func (rb *recordBase) Create(_ context.Context, _ FsType) (int, error) { + return -fuse.EROFS, errReadOnly +} + +func (nr *ipnsSubroot) Create(ctx context.Context, nodeType FsType) (int, error) { + keyComponent := strings.Split(nr.path, "/")[2] + _, err := resolveKeyName(ctx, nr.core.Key(), keyComponent) + switch err { + case nil: + return -fuse.EEXIST, os.ErrExist + case errNoKey: + break + default: + return -fuse.EIO, err + } + + //NOTE: we rely on API to not clobber + nr.key, err = nr.core.Key().Generate(ctx, keyComponent) + if err != nil { + log.Errorf("DBG: {%T}%#v", err, err) + return -fuse.EIO, fmt.Errorf("could not generate key %q: %s", keyComponent, err) + } + + ipldNode, err := emptyNode(ctx, nr.core.Dag(), nodeType) + if err != nil { + return -fuse.EIO, fmt.Errorf("could not generate ipld node for %q: %s", keyComponent, err) + } + + if err = ipnsDelayedPublish(ctx, nr.key, ipldNode); err != nil { + return -fuse.EIO, fmt.Errorf("could not publish to key %q: %s", keyComponent, err) + } + return fuseSuccess, nil +} + +func (nn *ipnsNode) Create(ctx context.Context, nodeType FsType) (int, error) { + nn.fsRootIndex.Lock() + defer nn.fsRootIndex.Unlock() + + if nn.key == nil { + return -fuse.EROFS, errReadOnly + } + + //TODO: fetch mroot + //mroot, err := + if err != nil { + } + // nn.root = mroot + return nn.mfsRoot.Create(ctx, nodeType) + + if nn.root, err = nn.fsRootIndex.Request(nn.key.Name()); err != nil { + return -fuse.EIO, err + } + nn.fsRootIndex.Release(nn.key.Name()) + //return mfsMknod(mroot, subPath) + return nn.mfsNode.Create(ctx) + + _, keyName := gopath.Split(nn.path) + newRootNode, err := emptyNode(fs.ctx, fs.core.Dag(), unixfs.TFile, nil) + if err != nil { + return -fuse.EIO, fmt.Errorf("could not generate unixdir %q: %s", keyName, err) + } + + err = fs.ipnsDelayedPublish(coreKey, newRootNode) + if err != nil { + return -fuse.EIO, fmt.Errorf("could not publish to key %q: %s", keyName, err) + } + return fuseSuccess, nil + + //*ipnsNode: + return fs.ipnsMknod(path) + +} + +func (mn *mfsNode) Create(ctx context.Context, nodeType FsType) (int, error) { + //mfs lookup + if _, err := mfs.Lookup(mn.root, mn.path); err == nil { + return -fuse.EEXIST, os.ErrExist + } + + parentPath, childName := gopath.Split(mn.path) + mfsParent, err := mfs.Lookup(mn.root, parentPath) + if err != nil { + return -fuse.ENOENT, err + } + parentDir, ok := mfsNode.(*mfs.Directory) + if !ok { + return -fuse.ENOTDIR, fmt.Errorf("%s is not a directory", parentPath) + } + + var ipldNode *ipld.Node + + switch nodeType { + case unixfs.TFile: + dagFile := dag.NodeWithData(unixfs.FilePBData(nil, 0)) + dagFile.SetCidBuilder(parentDir.GetCidBuilder()) + ipldNode = dagFile + case unixfs.TDirectory: + //TODO: review mkdir opts + Mkdir POSIX specs (are intermediate paths allowed by default?) + if err = mfs.Mkdir(mn.root, mn.path, mfs.MkdirOpts{Flush: mfsSync}); err != nil { + if err == mfs.ErrDirExists || err == os.ErrExist { + return -fuse.EEXIST, os.ErrExist + } + return -fuse.EACCES, err + } + return fuseSuccess, nil + default: + return -fuse.EINVAL, errUnexpected + } + + if err = mfs.PutNode(mn.root, mn.path, ipldNode); err != nil { + return -fuse.EIO, err + } + + err = mfsDir.AddChild(fName, dagNode) + if err != nil { + return -fuse.EIO, err + } + + return fuseSuccess, nil + +} + +func (nn *ipnsNode) Remove(ctx context.Context) (int, error) { + keyName, subPath := ipnsSplit(nn.path) + if subPath == "" { + if nn.key == nil { + return -fuse.EPERM, errNoKey + } + _, err := nn.core.Key().Remove(ctx, keyName) + if err != nil { + return -fuse.EIO, fmt.Errorf("could not remove key %q: %s", keyName, err) + } + nn.key = nil + return fuseSuccess, nil + } + + nn.fsRootIndex.Lock() + defer nn.fsRootIndex.Unlock() + mroot, err := nn.fsRootIndex.Request(keyName) + if err != nil { + return -fuse.EIO, err + } + nn.fsRootIndex.Release(keyName) + return mfsRemove(mroot, subPath) } //TODO: make a note somewhere that generic functions assume valid structs; define what "valid" means @@ -60,14 +273,362 @@ func (rb *recordBase) Namespace() string { return rb.path[1:i] } -func (mn *mfsNode) Namespace() string { - return filesNamespace +func (*recordBase) Mutable() bool { + return false +} + +func (*ipnsNode) Mutable() bool { + return true +} + +// pedantic way of saying Unix permissions 0777 and 0555 +const IRWXA = fuse.S_IRWXU | fuse.S_IRWXG | fuse.S_IRWXO +const IRXA = IRWXA &^ (fuse.S_IWUSR | fuse.S_IWGRP | fuse.S_IWOTH) + +//TODO: document this upstream (cgofuse) +/* fuse.Getcontext only contains (useful) data in callstack under: +- Mknod +- Mkdir +- Getattr +- Open +- OpenDir +- Create +*/ + +func (rb *recordBase) InitMetadata(_ context.Context) (*fuse.Stat_t, error) { + now := fuse.Now() + rb.metadata.Birthtim, rb.metadata.Atim, rb.metadata.Mtim, rb.metadata.Ctim = now, now, now, now //!!!! + rb.metadata.Mode |= IRXA + return &rb.metadata, nil +} + +func (in *ipfsNode) InitMetadata(ctx context.Context) (*fuse.Stat_t, error) { + fStat, err := in.recordBase.InitMetadata(ctx) + if err != nil { + return fStat, err + } + + corePath, err := coreiface.ParsePath(in.path) + if err != nil { + return fStat, err + } + + ipldNode, err := in.core.ResolveNode(ctx, corePath) + if err != nil { + return fStat, err + } + err = ipldStat(fStat, ipldNode) + return fStat, err +} + +func (mn *mfsNode) InitMetadata(_ context.Context) (*fuse.Stat_t, error) { + fStat, err := mn.recordBase.InitMetadata(nil) + if err != nil { + return fStat, err + } + + err = mfsStat(fStat, mn.root, mn.path) + return fStat, err +} + +func (nr *ipnsSubroot) InitMetadata(ctx context.Context) (*fuse.Stat_t, error) { + fStat, err := in.recordBase.InitMetadata(ctx) + if err != nil { + return fStat, err + } + + ipldNode, err := resolveIpns(ctx, nr.String(), nr.core) + if err != nil { + return fStat, err + } + + if err = ipldStat(fStat, ipldNode); err != nil { + return fStat, err + } + + if nn.key != nil && fStat.Mode&fuse.S_IFMT == fuse.S_IFDIR { + //init MFS here? + nr.path = nr.path[:1] // mfs relative + } + + /* + // wrap ipld node as mfs root construct + if nn.root, err = mfsFromKey(ctx, coreKey.Name(), nn.core); err != nil { + return -fuse.EACCES, err + } + nn.path = nn.path[:1] // promote path to root "/" + nn.key = coreKey // store key on fusePath + + nn.fsRootIndex.Register(coreKey.Name(), mroot) + runtime.SetFinalizer(nn, ipnsNodeReleaseRoot) + + return fuseSuccess, nil + + */ +} + +func (nn *ipnsNode) InitMetadata(ctx context.Context) (*fuse.Stat_t, error) { + /* TODO: move the TTL out of stat, into bacgkround thread; refresh things at the FS level + last := time.Unix(nn.metadata.mtim.Sec, nn.metadata.mtim.Nsec) + if time.Since(last) < ipnsTTL && in.metadata.Mode != 0 { + return &in.metadata, nil + } + */ + + if nn.key == nil { + return nn.ipfsNode.InitMetadata(ctx) + } + + // if we own the key; check fs-level mfs index instead of IPFS + var err error + if nn.root, err = mfsFromNode(ctx, nn); err != nil { + return nn.metadata, err + } + + nn.fsRootIndex.Lock() + defer nn.fsRootIndex.Unlock() + nn.root, err = nn.fsRootIndex.Request(nn.key.Name()) + switch err { + case errNotInitialized: + if nn.root, err = mfsFromKey(ctx, nn.key, nn.core); err != nil { + return nil, err + } + + nn.fsRootIndex.Register(nn.key.Name(), nn.root) + runtime.SetFinalizer(nn, ipnsNodeReleaseRoot) + + case nil: + break + default: + return nil, err + } + + err = mfsStat(&nn.metadata, mroot, subPath) + return &nn.metadata, err +} + +func (l *link) InitMetadata(_ context.Context) (*fuse.Stat_t, error) { + l.metadata.Mode = fuse.S_IFLNK | IRXA + l.metadata.Size = len(l.target) +} + +func (rb *recordBase) typeCheck(nodeType FsType) (err error) { + if !typeCheck(rb.metadata.Mode, nodeType) { + return errIOType + } + return nil +} + +func (in *ipfsNode) YieldIo(ctx context.Context, nodeType FsType) (io interface{}, err error) { + if err := in.recordBase.typeCheck(nodeType); err != nil { + return nil, err + } + + switch nodeType { + case unixfs.TFile: + return coreYieldFileIO(ctx, in, in.core.Unixfs()) + case unixfs.TDirectory: + return coreYieldDirIO(ctx, in, in.core, entryTimeout) + + /* TODO + case unixfs.TSymlink: + ipldNode, err := in.core.ResolveNode(ctx, in) + if err != nil { + return nil, err + } + target, err := ipldReadLink(ipldNode) + if err != nil { + return nil, err + } + return &link{target: target} + */ + default: + return nil, errUnexpected + } } -func (rb *recordBase) Mutable() bool { +func (nn *ipnsNode) YieldIo(ctx context.Context, nodeType FsType) (interface{}, error) { + if err := nn.recordBase.typeCheck(nodeType); err != nil { + return nil, err + } + + if nn.key == nil { // use core for non-keyed paths + return nn.ipfsNode.YieldIo(ctx, nodeType) + } + + // check that our key is still valid + if err = checkAPIKeystore(ctx, nn.core.Key(), nn.key); err != nil { + nn.key = nil + return nil, err + } + + if nn.path == "/" { + switch nn.metadata.Mode & fuse.S_IFMT { + case fuse.S_IFREG: + return keyYieldFileIO(ctx, nn.key, nn.core) + case fuse.S_IFLNK: + var ( + ipldNode ipld.Node + target string + lnk *link + ) + if ipldNode, err = nn.core.ResolveNode(ctx, nn.key.Path()); err != nil { + goto linkEnd + } + + if target, err = ipldReadLink(ipldNode); err != nil { + goto linkEnd + } + lnk = &link{target: target} + _, err = lnk.InitMetadata(ctx) + + linkEnd: + return lnk, err + + case fuse.S_IFDIR: + // fallback to MFS IO handler to list out root contents + break + default: + return nil, errUnexpected + } + } + + //handle other nodes via MFS + nn.fsRootIndex.Lock() + defer nn.fsRootIndex.Unlock() + nn.root, err = nn.fsRootIndex.Request(keyName) + switch err { + case nil: + break + default: + return nil, err + case errNotInitialized: + nn.root, err = ipnsToMFSRoot(ctx, key.Path(), nn.core) + if err != nil { + return nil, err + } + + nn.fsRootIndex.Register(keyName, mroot) + if nn.root, err = nn.fsRootIndex.Request(keyName); err != nil { + return nil, err + } + } + + nnIO, err := nn.mfsNode.YieldIo(ctx) + if err != nil { + runtime.SetFinalizer(nnIO, ipnsKeyRootFree) + } + return nnIO, err +} + +func (mn *mfsNode) YieldIo(ctx context.Context, nodeType FsType) (interface{}, error) { + if err := in.recordBase.typeCheck(nodeType); err != nil { + return nil, err + } + + switch nodeType { + case unixfs.TFile: + return mfsYieldFileIO(mn.root, mn.path) + case unixfs.TDirectory: + ctx = context.WithValue(ctx, dagKey{}, mn.core.Dag()) //TODO: [2019.03.26] see note inside mfsSubNodes + return mfsYieldDirIO(ctx, mn.root, mn.path, timeoutGrace, mn.core.Dag()) + case unixfs.TSymlink: + fallthrough + default: + return nil, errUnexpected + } +} + +func (rb *recordBase) DestroyIo(fh uint64, nodeType FsType) (ret int, err error) { + io, ok := rb.ioHandles[fh] + if !ok { + return -fuse.EBADF, fmt.Errorf("handle %X for %q is valid for record but not IO", fh, rb.path) + } + + //FIXME: we need to set the finalizer for IO objects at initialization to call close in the rare event we error on these + switch nodeType { + case unixfs.TFile: + if fio, ok := io.(*FsFile); !ok { + ret = -fuse.EIO + err = fmt.Errorf("handle %X for %q exists but type is mismatched{%T}", fh, rb.path, io) + } else { + ret = fio.Close() + } + case unixfs.TDirectory: + if dio, ok := io.(*FsDirectory); !ok { + ret = -fuse.EACCES + err = fmt.Errorf("handle %X for %q exists but type is mismatched{%T}", fh, rb.path, io) + } else { + ret = dio.Close() + } + default: + ret = -fuse.EBADF + err = fmt.Errorf("handle %X for %q exists but type requested was unexpected{%#v}", fh, rb.path, nodeType) + } + + // invalidate/free handle regardless of grace + delete(rb.ioHandles, fh) + return +} + +/* +switch nodeType { + case unixfs.TFile: + case unixfs.TDirectory: + case unixfs.TSymlink: + } +*/ + +func typeCheck(pMode uint32, iType FsType) bool { + pMode &= fuse.S_IFMT + switch iType { + case unixfs.TFile: + if pMode == fuse.S_IFREG { + return true + } + case unixfs.TDirectory, unixfs.THAMTShard: + if pMode == fuse.S_IFDIR { + return true + } + case unixfs.TSymlink: + if pMode == fuse.S_IFLNK { + return true + } + } return false } -func (mb *mutableBase) Mutable() bool { - return true +// link's target replaces its original path at the FS level +// link's metadata remains separate from target's +type link struct { + fusePath + metadata fuse.Stat_t + target string +} + +func (ln *link) String() string { + return ln.target +} + +func (ln *link) Target() string { + return ln.target +} + +//TODO: log these to test +func ipnsNodeReleaseRoot(nn *ipnsNode) { + nn.fsRootIndex.Lock() + defer nn.fsRootIndex.Unlock() + nn.fsRootIndex.Release(nn.key.Name()) +} + +//XXX +func ipnsIoReleaseRoot(io FsDirectory) { + fp := io.Record() + nn, ok := fp.(*ipnsNode) + if !ok { + return + } + nn.fsRootIndex.Lock() + defer nn.fsRootIndex.Unlock() + nn.fsRootIndex.Release(nn.key.Name()) } diff --git a/core/commands/mount/interface.go b/core/commands/mount/interface.go index 34e8768b712..58d3cb2a13f 100644 --- a/core/commands/mount/interface.go +++ b/core/commands/mount/interface.go @@ -1,173 +1,73 @@ package fusemount import ( + "context" "fmt" "io" "sync" - "time" - "unsafe" coreiface "gx/ipfs/QmXLwxifxwfc2bAwq6rdjbYqAsGzWsDE9RM5TWMGtykyj6/interface-go-ipfs-core" + mfs "gx/ipfs/Qmb74fRYPgpjYzoBV7PAVNmP3DQaRrh8dHdKE4PwnF3cRx/go-mfs" "github.com/billziss-gh/cgofuse/fuse" ) +type fsHandles map[uint64]fusePath +type nodeHandles map[uint64]interface{} // *FsFile | *FsDirectory + +type fusePath interface { + coreiface.Path + RWLocker + + InitMetadata(context.Context) (*fuse.Stat_t, error) + Stat() (*fuse.Stat_t, error) + YieldIo(ctx context.Context, nodeType FsType) (io interface{}, err error) + Handles() nodeHandles + DestroyIo(handle uint64, nodeType FsType) (fuseReturn int, goErr error) + Remove(context.Context) (int, error) + Create(context.Context, FsType) (int, error) + //GetIo(handle uint64) (io interface{}, err error) // FsFile | FsDirectory + //Exists() bool +} + type RWLocker interface { sync.Locker RLock() RUnlock() } +//NOTE: (int, error) pairs are translations of the appropriate return values across APIs, int for FUSE, error for Go type FsFile interface { io.Reader io.Seeker io.Closer + sync.Locker Size() (int64, error) Write(buff []byte, ofst int64) (int, error) - Sync() int + Sync() (int, error) Truncate(size int64) (int, error) + Record() fusePath } type FsDirectory interface { //Parent() Directory - //Entries(ofst int64) <-chan directoryEntry + sync.Locker Entries() int - Read(offset int64) <-chan DirectoryMessage -} - -type fusePath interface { - coreiface.Path - RWLocker - - //TODO: reconsider approach - Stat() *fuse.Stat_t - Handles() *[]uint64 -} - -// FIXME: even if it's unlikely, we need to assure addr == invalidIndex is never true -func (fs *FUSEIPFS) newDirHandle(fsNode fusePath) (uint64, error) { - //TODO: check path/node is actually a directory - //return -fuse.ENOTDIR - - var ( - fsDir FsDirectory - err error - ) - if canAsync(fsNode) { - const timeout = 2 * time.Second //reset per entry in stream reader routine; TODO: configurable - fsDir, err = fs.yieldAsyncDirIO(fs.ctx, timeout, fsNode) // Read inherits this context - } else { - fsDir, err = fs.yieldDirIO(fsNode) - } - - if err != nil { - return invalidIndex, fmt.Errorf("could not yield directory IO: %s", err) - } - - hs := &dirHandle{record: fsNode, io: fsDir} - fh := uint64(uintptr(unsafe.Pointer(hs))) - *fsNode.Handles() = append(*fsNode.Handles(), fh) - fs.dirHandles[fh] = hs - return fh, nil + Read(ctx context.Context, offset int64) <-chan directoryFuseEntry + Record() fusePath } -func (fs *FUSEIPFS) newFileHandle(fsNode fusePath) (uint64, error) { - pIo, err := fs.yieldFileIO(fsNode) - if err != nil { - return invalidIndex, err - } - - hs := &fileHandle{record: fsNode, io: pIo} - - fh := uint64(uintptr(unsafe.Pointer(hs))) - *fsNode.Handles() = append(*fsNode.Handles(), fh) - fs.fileHandles[fh] = hs - return fh, nil +type FsLink interface { + Target() string // Link } -func (fs *FUSEIPFS) releaseFileHandle(fh uint64) (ret int) { - if fh == invalidIndex { - log.Errorf("releaseHandle - input handle is invalid") - return -fuse.EBADF - } - - defer func() { - if r := recover(); r != nil { - log.Errorf("releaseHandle recovered from panic, likely invalid handle: %v", r) - ret = -fuse.EBADF - } - }() - - hs := (*fileHandle)(unsafe.Pointer(uintptr(fh))) - - handleGroup := hs.record.Handles() - for i, cFh := range *handleGroup { - if cFh == fh { - *handleGroup = append((*handleGroup)[:i], (*handleGroup)[i+1:]...) - - if hs.io != nil { - if err := hs.io.Close(); err != nil { - log.Error(err) - } - } - - //Go runtime free-able - fs.fileHandles[fh] = nil - delete(fs.fileHandles, fh) - ret = fuseSuccess - return - } - } - log.Errorf("releaseHandle - handle was detected as valid but was not associated with node %q", hs.record.String()) - ret = -fuse.EBADF - return +type FsRecord interface { + Record() fusePath } -func (fs *FUSEIPFS) releaseDirHandle(fh uint64) (ret int) { - if fh == invalidIndex { - log.Errorf("releaseDirHandle - input handle is invalid") - ret = -fuse.EBADF - return - } - - defer func() { - if r := recover(); r != nil { - log.Errorf("releaseDirHandle recovered from panic, likely invalid handle: %v", r) - ret = -fuse.EBADF - } - }() - hs := (*dirHandle)(unsafe.Pointer(uintptr(fh))) - - handleGroup := hs.record.Handles() - for i, cFh := range *handleGroup { - if cFh == fh { - *handleGroup = append((*handleGroup)[:i], (*handleGroup)[i+1:]...) - - //Go runtime free-able - fs.dirHandles[fh] = nil - delete(fs.dirHandles, fh) - ret = fuseSuccess - return - } - } - log.Errorf("releaseDirHandle - handle was detected as valid but was not associated with node %q", hs.record.String()) - ret = -fuse.EBADF - return -} - -type AvailType = bool - -const ( - aFiles AvailType = false - aDirectories AvailType = true -) - //NOTE: caller should retain FS Lock -func (fs *FUSEIPFS) AvailableHandles(directories AvailType) uint64 { - if directories { - return (invalidIndex - 1) - uint64(len(fs.dirHandles)) - } - return (invalidIndex - 1) - uint64(len(fs.fileHandles)) +func (fs *FUSEIPFS) AvailableHandles() uint64 { + return (invalidIndex - 1) - uint64(len(fs.handles)) } func (fs *FUSEIPFS) Close() error { @@ -185,3 +85,46 @@ func (fs *FUSEIPFS) IsActive() bool { func (fs *FUSEIPFS) Where() string { return fs.mountPoint } + +type nameRootIndex interface { + sync.Locker + Register(string, *mfs.Root) + Request(string) (*mfs.Root, error) + Release(string) +} + +type mfsSharedIndex struct { + sync.Mutex + roots map[string]*mfsReference +} + +type mfsReference struct { + root *mfs.Root + int // refcount +} + +func (mi *mfsSharedIndex) Register(subrootPath string, mroot *mfs.Root) { + mi.roots[subrootPath] = &mfsReference{*mfs.Roots: mroot, int: 1} +} + +func (mi *mfsSharedIndex) Request(subrootPath string) (*mfs.Root, error) { + index, ok := mi.roots[subrootPath] + if !ok || index == nil { + return nil, errNotInitialized + } + index.int++ + return index.root, nil +} + +func (mi *mfsSharedIndex) Release(subrootPath string) { + index, ok := mi.roots[subrootPath] + if !ok || index == nil { + log.Errorf("shared index %q not found", subrootPath) + return // panic? + } + if index.int--; index.int == 0 { + delete(mi.roots, subrootPath) + } + + //= mfsReference{*mfs.Roots: mroot} +} diff --git a/core/commands/mount/io.go b/core/commands/mount/io.go index e4d001ba875..3b20f282c22 100644 --- a/core/commands/mount/io.go +++ b/core/commands/mount/io.go @@ -5,16 +5,17 @@ import ( "errors" "fmt" "io" - gopath "path" coreiface "gx/ipfs/QmXLwxifxwfc2bAwq6rdjbYqAsGzWsDE9RM5TWMGtykyj6/interface-go-ipfs-core" coreoptions "gx/ipfs/QmXLwxifxwfc2bAwq6rdjbYqAsGzWsDE9RM5TWMGtykyj6/interface-go-ipfs-core/options" + ipld "gx/ipfs/QmZ6nzCLwGLVfRzYLpD7pW6UNuBDKEcA2imJtVpbEx2rxy/go-ipld-format" "github.com/billziss-gh/cgofuse/fuse" files "gx/ipfs/QmQmhotPUzVrMEWNK3x1R5jQ5ZHWyL7tVUrmRPjrBrvyCb/go-ipfs-files" chunk "gx/ipfs/QmYmZ81dU5nnmBFy5MmktXLZpt8QCWhRJd6M1uxVF6vke8/go-ipfs-chunker" mfs "gx/ipfs/Qmb74fRYPgpjYzoBV7PAVNmP3DQaRrh8dHdKE4PwnF3cRx/go-mfs" + unixfs "gx/ipfs/QmcYUTQ7tBZeH1CLsZM2S3xhMEZdvUgXvbjhpMsLDpk3oJ/go-unixfs" "gx/ipfs/QmcYUTQ7tBZeH1CLsZM2S3xhMEZdvUgXvbjhpMsLDpk3oJ/go-unixfs/mod" ) @@ -41,7 +42,7 @@ func (fs *FUSEIPFS) Read(path string, buff []byte, ofst int64, fh uint64) int { fh.Record.Unlock() */ - h, err := fs.LookupFileHandle(fh) + ioIf, err := fs.getIo(fh, unixfs.TFile) if err != nil { fs.RUnlock() log.Errorf("Read - [%X]%q: %s", fh, path, err) @@ -50,52 +51,37 @@ func (fs *FUSEIPFS) Read(path string, buff []byte, ofst int64, fh uint64) int { } return -fuse.EIO } - - h.record.Lock() // write lock; handle cursor is modified + fio := ioIf.(FsFile) + fio.Lock() + defer fio.Unlock() fs.RUnlock() - defer h.record.Unlock() //TODO: inspect need to flush here //if fh != handle.lastCaller { flush } - if fileBound, err := h.io.Size(); err == nil { + if fileBound, err := fio.Size(); err == nil { if ofst >= fileBound { return 0 // this is unique from fuseSuccess } } if ofst != 0 { - _, err = h.io.Seek(ofst, io.SeekStart) + _, err = fio.Seek(ofst, io.SeekStart) if err != nil { log.Errorf("Read - seek error: %s", err) return -fuse.EIO } } - readBytes, err := h.io.Read(buff) + readBytes, err := fio.Read(buff) if err != nil && err != io.EOF { log.Errorf("Read - error: %s", err) } return readBytes } -func (fs *FUSEIPFS) yieldFileIO(fsNode fusePath) (FsFile, error) { - //TODO: cast to concrete type and change yield parameters to accept them - switch fsNode.(type) { - case *mfsNode: - return mfsYieldFileIO(fs.filesRoot, fsNode.String()[frs:]) - case *ipfsNode: - return fs.coreYieldFileIO(fsNode) - case *ipnsKey: - return fs.keyYieldFileIO(fsNode) - case *ipnsNode: - return fs.nameYieldFileIO(fsNode) - default: - return nil, fmt.Errorf("unexpected IO request {%T}%q", fsNode, fsNode.String()) - } -} - -func mfsYieldFileIO(filesRoot *mfs.Root, path string) (FsFile, error) { +//TODO: accept i/o flags +func mfsYieldFileIO(filesRoot *mfs.Root, path string) (FsFile, uint64, error) { mfsNode, err := mfs.Lookup(filesRoot, path) if err != nil { return nil, err @@ -105,21 +91,24 @@ func mfsYieldFileIO(filesRoot *mfs.Root, path string) (FsFile, error) { if !ok { return nil, fmt.Errorf("File IO requested for non-file, type: %v %q", mfsNode.Type(), path) } - - //TODO: change arity of yield to accept i/o request flag - return &mfsFileIo{ff: mfsFile, ioFlags: mfs.Flags{Read: true, Write: true, Sync: mfsSync}}, nil + return mfsFileIo{ff: mfsFile, ioFlags: mfs.Flags{Read: true, Write: true, Sync: mfsSync}}, nil + //handle := uint64(uintptr(unsafe.Pointer(&io))) + //return io, handle, nil } //TODO: we'll have to pass and store write flags on this; for now rely on 🤠 to maintain permissions //TODO: some kind of local write buffer type mfsFileIo struct { - ff *mfs.File - //fd mfs.FileDescriptor - //XXX: this is not ideal, we're duplicating state here to circumvent mfs's 1 (writable) file descriptor limit //this is likely suboptimal - ioFlags mfs.Flags cursor int64 + ff *mfs.File + record fusePath + ioFlags mfs.Flags +} + +func (mio *mfsFileIo) Record() fusePath { + return mio.record } //allows for multiple handles to a single mfs node @@ -208,13 +197,11 @@ func (mio *mfsFileIo) Read(buff []byte) (int, error) { //TODO: look into this; speak with shcomatis // API syncs on close by default; see mfsOpenShim(); every op should force a sync as a result of that // ideally we want to only sync on demand -func (mio *mfsFileIo) Sync() int { - /* - if err := mio.fd.Flush(); err != nil { - return -fuse.EIO - } - */ - return fuseSuccess +func (mio *mfsFileIo) Sync() (int, error) { + if err := mio.fd.Flush(); err != nil { + return -fuse.EIO, err + } + return fuseSuccess, nil } func (mio *mfsFileIo) Write(buff []byte, ofst int64) (int, error) { @@ -259,9 +246,9 @@ func (mio *mfsFileIo) Truncate(size int64) (int, error) { return fuseSuccess, nil } -func (fs *FUSEIPFS) coreYieldFileIO(curNode coreiface.Path) (FsFile, error) { +func coreYieldFileIO(ctx context.Context, corePath coreiface.Path, uAPI coreiface.UnixfsAPI) (FsFile, error) { var err error - apiNode, err := fs.core.Unixfs().Get(fs.ctx, curNode) + apiNode, err := uAPI.Get(ctx, corePath) if err != nil { return nil, err } @@ -271,15 +258,38 @@ func (fs *FUSEIPFS) coreYieldFileIO(curNode coreiface.Path) (FsFile, error) { return nil, fmt.Errorf("%q is not a file", curNode.String()) } - return &corePIo{fd: fIo}, nil + return corePIo{fd: fIo}, nil +} + +func ipldReadLink(ipldNode *ipld.Node) (string, error) { + ufsNode, err := unixfs.ExtractFSNode(ipldNode) + if err != nil { + return "", err + } + if ufsNode.Type() != unixfs.TSymlink { + return "", errIOType + } + + return string(ufsNode.Data()), nil } type corePIo struct { - fd files.File + fd files.File + record fusePath +} + +func (cio *corePIo) Record() fusePath { + return cio.record +} + +func (cio *corePIo) Lock() { + cio.record.Lock() +} + +func (cio *corePIo) Unlock() { + cio.record.Unlock() } -//FIXME read broken on large files sometimes? -//MFS too func (cio *corePIo) Read(buff []byte) (int, error) { readBytes, err := cio.fd.Read(buff) if err != nil { @@ -305,12 +315,11 @@ func (cio *corePIo) Size() (int64, error) { } func (cio *corePIo) Write(buff []byte, ofst int64) (int, error) { - return -fuse.EROFS, fmt.Errorf("Write requested on read only path") + return -fuse.EROFS, errReadOnly } -func (cio *corePIo) Sync() int { - log.Warning("Sync called on read only file") - return -fuse.EINVAL +func (cio *corePIo) Sync() (int, error) { + return -fuse.EINVAL, errReadOnly } func (cio *corePIo) Truncate(int64) (int, error) { @@ -319,7 +328,7 @@ func (cio *corePIo) Truncate(int64) (int, error) { //TODO: [fs] free MFS roots when no references are using them instead of loading them all forever // instantiate on demand -func (fs *FUSEIPFS) nameYieldFileIO(fsNode fusePath) (FsFile, error) { +func nameYieldFileIO(path string) (FsFile, uint64, error) { keyRoot, subPath, err := fs.ipnsMFSSplit(fsNode.String()) if err != nil { globalNode, err := fs.resolveToGlobal(fsNode) @@ -337,30 +346,23 @@ type keyFileIo struct { mod *mod.DagModifier } -func (fs *FUSEIPFS) keyYieldFileIO(fsNode fusePath) (FsFile, error) { - _, keyName := gopath.Split(fsNode.String()) - - oAPI, err := fs.core.WithOptions(coreoptions.Api.Offline(true)) +func keyYieldFileIO(ctx context.Context, coreKey coreiface.Key, core coreiface.CoreAPI) (FsFile, error) { + coreKey, err := resolveKeyName(ctx, core.Key(), keyName) if err != nil { return nil, err } - coreKey, err := resolveKeyName(fs.ctx, oAPI.Key(), keyName) + ipldNode, err := core.ResolveNode(ctx, coreKey.Path()) if err != nil { return nil, err } - ipldNode, err := oAPI.ResolveNode(fs.ctx, coreKey.Path()) + dmod, err := mod.NewDagModifier(ctx, ipldNode, core.Dag(), chunk.DefaultSplitter) if err != nil { return nil, err } - dmod, err := mod.NewDagModifier(fs.ctx, ipldNode, oAPI.Dag(), chunk.DefaultSplitter) - if err != nil { - return nil, err - } - - return &keyFileIo{key: coreKey, name: oAPI.Name(), mod: dmod}, nil + return &keyFileIo{key: coreKey, name: core.Name(), mod: dmod}, nil } func (kio *keyFileIo) Write(buff []byte, ofst int64) (int, error) { @@ -427,11 +429,11 @@ func (kio *keyFileIo) Size() (int64, error) { return kio.mod.Size() } -func (kio *keyFileIo) Sync() int { +func (kio *keyFileIo) Sync() (int, error) { if err := kio.mod.Sync(); err != nil { - return -fuse.EIO + return -fuse.EIO, err } - return fuseSuccess + return fuseSuccess, nil } func (kio *keyFileIo) Truncate(size int64) (int, error) { diff --git a/core/commands/mount/modify.go b/core/commands/mount/modify.go index 100d0ef0ca1..ecc79fbc004 100644 --- a/core/commands/mount/modify.go +++ b/core/commands/mount/modify.go @@ -4,6 +4,7 @@ import ( "github.com/billziss-gh/cgofuse/fuse" mfs "gx/ipfs/Qmb74fRYPgpjYzoBV7PAVNmP3DQaRrh8dHdKE4PwnF3cRx/go-mfs" + unixfs "gx/ipfs/QmcYUTQ7tBZeH1CLsZM2S3xhMEZdvUgXvbjhpMsLDpk3oJ/go-unixfs" ) /* TODO @@ -102,38 +103,51 @@ func (fs *FUSEIPFS) Truncate(path string, size int64, fh uint64) int { } */ - if h, err := fs.LookupFileHandle(fh); err == nil { - h.record.Lock() - defer h.record.Unlock() - fErr, gErr := h.io.Truncate(size) - if gErr != nil { - log.Errorf("Truncate - [%X]%q:%s", fh, path, gErr) - } else { - h.record.Stat().Size = size - } - return fErr - } + callContext, cancel := deriveCallContext(fs.ctx) + defer cancel() - fsNode, err := fs.LookupPath(path) - if err != nil { - log.Errorf("Truncate - %q:%s", path, err) - return -fuse.ENOENT - } + var fsNode fusePath + var ioIf interface{} + ioIf, err = fs.getIo(fh, unixfs.TFile) + switch err { + case nil: + fsNode = ioIf.(FsFile).Record() + case errInvalidHandle: // truncate() is allowed on paths that are not open; create temporary io + if fsNode, err = fs.LookupPath(path); err != nil { + log.Errorf("Truncate - %q:%s", path, err) + return -fuse.ENOENT - fsNode.Lock() - defer fsNode.Unlock() - - io, err := fs.yieldFileIO(fsNode) - if err != nil { - log.Errorf("Truncate - %q:%s", path, err) + } + ioIf, err = fsNode.YieldIo(callContext, unixfs.TFile) + switch err { + case nil: + break + case errIOType: + log.Errorf("Truncate - [%X]%q:%s", fh, path, errIOType) + return -fuse.EISDIR + default: + log.Errorf("Truncate - [%X]%q:%s", fh, path, err) + return -fuse.EIO + } + default: + log.Errorf("Truncate - [%X]%q:%s", fh, path, err) return -fuse.EIO } + fsNode.Lock() + defer fsNode.Unlock() - fErr, gErr := io.Truncate(size) + fErr, gErr := ioIf.(FsFile).Truncate(size) if gErr != nil { - log.Errorf("Truncate - %q:%s", path, gErr) + log.Errorf("Truncate - [%X]%q:%s", fh, path, gErr) } else { - fsNode.Stat().Size = size + nodeStat, err := fsNode.Stat(callContext) + if err != nil { + log.Errorf("Truncate - [%X]%q:%s", fh, path, err) + return -fuse.EIO + } + now := fuse.Now() + nodeStat.Size = size + nodeStat.Mtim, nodeStat.Ctim, nodeStat.Atim = now, now, now // calm down } return fErr } diff --git a/core/commands/mount/probe.go b/core/commands/mount/probe.go index a98f538fe05..510d35d456d 100644 --- a/core/commands/mount/probe.go +++ b/core/commands/mount/probe.go @@ -1,11 +1,15 @@ package fusemount import ( + "context" "fmt" + "os" "github.com/billziss-gh/cgofuse/fuse" config "gx/ipfs/QmUAuYuiafnJRZxDDX7MuruMNsicYNuyub5vUeAcupUBNs/go-ipfs-config" + coreiface "gx/ipfs/QmXLwxifxwfc2bAwq6rdjbYqAsGzWsDE9RM5TWMGtykyj6/interface-go-ipfs-core" + ipld "gx/ipfs/QmZ6nzCLwGLVfRzYLpD7pW6UNuBDKEcA2imJtVpbEx2rxy/go-ipld-format" mfs "gx/ipfs/Qmb74fRYPgpjYzoBV7PAVNmP3DQaRrh8dHdKE4PwnF3cRx/go-mfs" unixfs "gx/ipfs/QmcYUTQ7tBZeH1CLsZM2S3xhMEZdvUgXvbjhpMsLDpk3oJ/go-unixfs" uio "gx/ipfs/QmcYUTQ7tBZeH1CLsZM2S3xhMEZdvUgXvbjhpMsLDpk3oJ/go-unixfs/io" @@ -56,106 +60,29 @@ func (fs *FUSEIPFS) Getattr(path string, fStat *fuse.Stat_t, fh uint64) int { fsNode, err := fs.LookupPath(path) if err != nil { - if !platformException(path) { - log.Warningf("Getattr - Lookup error %q: %s", path, err) + if err == os.ErrNotExist { + if !platformException(path) { + log.Warningf("Getattr - %q not found", path) + } + fs.RUnlock() + return -fuse.ENOENT } + log.Errorf("Getattr - Lookup error %q: %s", path, err) fs.RUnlock() - return -fuse.ENOENT + return -fuse.EIO } fsNode.Lock() fs.RUnlock() defer fsNode.Unlock() - //NOTE [2018.12.26]: [uid] - /* fuse.Getcontext only contains data in callstack under: - - Mknod - - Mkdir - - Getattr - - Open - - OpenDir - - Create - TODO: we need to retain values from chmod and chown and not overwrite them here - */ - - nodeStat := fsNode.Stat() - if nodeStat == nil { - log.Errorf("Getattr - node %q was not initialized properly", fsNode) - return -fuse.EIO - } - - if nodeStat.Mode != 0 { // active node retrieved from lookup - *fStat = *nodeStat - fStat.Uid, fStat.Gid, _ = fuse.Getcontext() - return fuseSuccess - } - - // Local permissions - var permissionBits uint32 = 0555 - switch fsNode.(type) { - case *mfsNode, *mfsRoot, *ipnsRoot, *ipnsKey: - permissionBits |= 0220 - case *ipnsNode: - keyName, _, _ := ipnsSplit(path) - if _, err := resolveKeyName(fs.ctx, fs.core.Key(), keyName); err == nil { // we own this path/key locally - permissionBits |= 0220 - } - } - nodeStat.Mode = permissionBits - - // POSIX type + sizes - switch fsNode.(type) { - case *mountRoot, *ipfsRoot, *ipnsRoot, *mfsRoot: - nodeStat.Mode |= fuse.S_IFDIR - default: - globalNode := fsNode - if isReference(fsNode) { - globalNode, err = fs.resolveToGlobal(fsNode) - if err != nil { - log.Errorf("Getattr - reference node %q could not be resolved: %s ", fsNode, err) - return -fuse.EIO - } - } - - ipldNode, err := fs.core.ResolveNode(fs.ctx, globalNode) - if err != nil { - log.Errorf("Getattr - reference node %q could not be resolved: %s ", fsNode, err) - return -fuse.EIO - } - ufsNode, err := unixfs.ExtractFSNode(ipldNode) - if err != nil { - log.Errorf("Getattr - reference node %q could not be transformed into UnixFS type: %s ", fsNode, err) - return -fuse.EIO - } - - switch ufsNode.Type() { - case unixfs.TFile: - nodeStat.Mode |= fuse.S_IFREG - case unixfs.TDirectory: - nodeStat.Mode |= fuse.S_IFDIR - case unixfs.TSymlink: - nodeStat.Mode |= fuse.S_IFLNK - default: - log.Errorf("Getattr - unexpected node type {%T}%q", ufsNode, globalNode) - return -fuse.EIO - } - - if bs := ufsNode.BlockSizes(); len(bs) != 0 { - nodeStat.Blksize = int64(bs[0]) + nodeStat, err := fsNode.Stat() + if err != nil { + if err == os.ErrNotExist { + return -fuse.ENOENT } - nodeStat.Size = int64(ufsNode.FileSize()) - } - - // Time - now := fuse.Now() - switch fsNode.(type) { - case *mountRoot, *ipfsRoot, *ipnsRoot, *mfsRoot: - nodeStat.Birthtim = fs.mountTime - default: - nodeStat.Birthtim = now + log.Errorf("Getattr - %q stat err", fsNode, err) + return -fuse.EIO } - - nodeStat.Atim, nodeStat.Mtim, nodeStat.Ctim = now, now, now //!!! - *fStat = *nodeStat fStat.Uid, fStat.Gid, _ = fuse.Getcontext() return fuseSuccess @@ -169,34 +96,28 @@ func canAsync(fsNd fusePath) bool { return false } -func (fs *FUSEIPFS) ipnsRootSubnodes() []directoryEntry { - keys, err := fs.core.Key().List(fs.ctx) - if err != nil { - log.Errorf("ipnsRoot - Key err: %v", err) - return nil - } +func mfsSubNodes(ctx context.Context, mRoot *mfs.Root, path string) (<-chan directoryStringEntry, int, error) { - ents := make([]directoryEntry, 0, len(keys)) - if !fReaddirPlus { - for _, key := range keys { - ents = append(ents, directoryEntry{label: key.Name()}) - } - return ents + //NOTE: [2019.03.26] MFS's ForEachEntry is not async, as such we sidestep it and use unixfs directly + // if ForEachEntry becomes async we do not need the dag service directly + di := ctx.Value(dagKey{}) + if di == nil { + return nil, 0, fmt.Errorf("context does not contain dag in value") } - //TODO [readdirplus] - return nil -} + dag, ok := di.(*coreiface.APIDagService) + if !ok { + return nil, 0, fmt.Errorf("context value is not a valid dag service") + } + // -//TODO: accept context arg -func (fs *FUSEIPFS) mfsSubNodes(filesRoot *mfs.Root, path string) (<-chan unixfs.LinkResult, int, error) { - //log.Errorf("mfsSubNodes dbg dir %q", path) - mfsNd, err := mfs.Lookup(filesRoot, path) + mfsNd, err := mfs.Lookup(mRoot, path) if err != nil { return nil, 0, err } //mfsDir, ok := mfsNd.(*mfs.Directory) - _, ok := mfsNd.(*mfs.Directory) + //mfsDir.ForEachEntry(tctx, process(entry)) + _, ok = mfsNd.(*mfs.Directory) if !ok { return nil, 0, fmt.Errorf("mfs %q not a directory", path) } @@ -211,12 +132,61 @@ func (fs *FUSEIPFS) mfsSubNodes(filesRoot *mfs.Root, path string) (<-chan unixfs return nil, 0, err } entries := iStat.NumLinks - // [2019.02.06] MFS's ForEachEntry is not async, so this conflicts with our Readdir timeout timer - //go mfsDir.ForEachEntry(fs.ctx, muxMessage) - unixDir, err := uio.NewDirectoryFromNode(fs.core.Dag(), ipldNd) + unixDir, err := uio.NewDirectoryFromNode(dag, ipldNd) if err != nil { return nil, 0, err } - return unixDir.EnumLinksAsync(fs.ctx), entries, nil + + return coreMux(unixDir.EnumLinksAsync(ctx)), entries, nil +} + +func ipldStat(fStat *fuse.Stat_t, node ipld.Node) error { + ufsNode, err := unixfs.ExtractFSNode(ipldNode) + if err != nil { + return err + } + + switch t := ufsNode.Type(); t { + case unixfs.TFile: + fStat.Mode |= fuse.S_IFREG + fStat.Size = int64(ufsNode.FileSize()) + case unixfs.TDirectory, unixfs.THAMTShard: + fStat.Mode |= fuse.S_IFDIR + nodeStat, err := node.Stat() + if err != nil { + return err + } + fStat.Size = nodeStat.NumLinks // NOTE: we're using this as the child count; off_t is not defined for directories in standard + case unixfs.TSymlink: + fStat.Mode |= fuse.S_IFLNK + fStat.Size = len(string(ufsNode.Data())) + default: + return fmt.Errorf("unexpected node type %d", t) + } + + if bs := ufsNode.BlockSizes(); len(bs) != 0 { + fStat.Blksize = int64(bs[0]) //NOTE: this value is to be used as a hint only; subsequent child block size may differ + } + return nil +} + +func mfsStat(fStat *fuse.Stat_t, mroot *mfs.Root, path string) error { + mfsNode, err := mfs.Lookup(mroot, path) + if err != nil { + return err + } + + ipldNode, err := mfsNode.GetNode() + if err != nil { + return err + } + + if err = ipldStat(fStat, ipldNode); err != nil { + return err + } + + fStat.Mode |= fuse.S_IWUSR + + return nil } diff --git a/core/commands/mount/rootNodes.go b/core/commands/mount/rootNodes.go index 8c9a983d746..b199339b730 100644 --- a/core/commands/mount/rootNodes.go +++ b/core/commands/mount/rootNodes.go @@ -1,33 +1,147 @@ package fusemount -type rootBase struct { +import ( + "context" + coreiface "gx/ipfs/QmXLwxifxwfc2bAwq6rdjbYqAsGzWsDE9RM5TWMGtykyj6/interface-go-ipfs-core" + coreoptions "gx/ipfs/QmXLwxifxwfc2bAwq6rdjbYqAsGzWsDE9RM5TWMGtykyj6/interface-go-ipfs-core/options" + + "github.com/billziss-gh/cgofuse/fuse" + "github.com/ipfs/go-ipfs/core/coreapi" +) + +//TODO: document registering with this +type softDirRoot struct { recordBase - //mountTime *fuse.Timespec + //fs *FUSEIPFS } -//type rootList [tRoots]fusePath type mountRoot struct { - rootBase - //rootIndices rootList + softDirRoot + subroots []string +} + +type pinRoot struct { + softDirRoot + pinAPI coreapi.PinAPI +} + +type keyRoot struct { + softDirRoot + keyAPI coreiface.KeyAPI +} + +func (sd *softDirRoot) InitMetadata(_ context.Context) (*fuse.Stat_t, error) { + sd.recordBase.metadata.Mode = fuse.S_IFDIR | IRXA + return &sd.recordBase.metadata, nil +} + +func (kr *keyRoot) InitMetadata(ctx context.Context) (*fuse.Stat_t, error) { + nodeStat, err := kr.softDirRoot.InitMetadata(ctx) + if err != nil { + return nodeStat, err + } + nodeStat.Mode |= fuse.S_IWUSR + return nodeStat, nil +} + +func (pr *pinRoot) YieldIo(ctx context.Context, nodeType FsType) (io interface{}, err error) { + if err := in.recordBase.typeCheck(nodeType); err != nil { + return nil, err + } + + pins, err := pr.pinAPI.Ls(ctx, coreoptions.Pin.Type.Recursive()) + if err != nil { + return nil, err + } + + pinChan := make(chan directoryStringEntry) + asyncContext := deriveTimerContext(ctx, entryTimeout) + go func() { + defer close(pinChan) + for _, pin := range pins { + select { + case <-asyncContext.Done(): + return + case pinChan <- directoryStringEntry{string: pin.String()}: + continue + } + } + + }() + return backgroundDir(asyncContext, len(pins), pinChan) } -//should inherit from directory entries -type ipfsRoot struct { - rootBase - //TODO: review below - //sync.Mutex - //subnodes []fuseStatPair - //lastUpdated time.Time +func (kr *keyRoot) YieldIo(ctx context.Context, nodeType FsType) (io interface{}, err error) { + if err := in.recordBase.typeCheck(nodeType); err != nil { + return nil, err + } + + keys, err := kr.keyAPI.List(ctx) + if err != nil { + return nil, err + } + + keyChan := make(chan directoryStringEntry) + asyncContext := deriveTimerContext(ctx, entryTimeout) + go func() { + defer close(keyChan) + for _, key := range keys { + select { + case <-asyncContext.Done(): + return + case keyChan <- directoryStringEntry{string: key.Name()}: + continue + } + } + + }() + return backgroundDir(asyncContext, len(keys), keyChan) } -type ipnsRoot struct { - rootBase - //sync.Mutex - //keys []coreiface.Key - //subnodes []fuseStatPair - //lastUpdated time.Time +func (mr *mountRoot) YieldIo(ctx context.Context, nodeType FsType) (io interface{}, err error) { + if err := in.recordBase.typeCheck(nodeType); err != nil { + return nil, err + } + + rootChan := make(chan directoryStringEntry) + asyncContext := deriveTimerContext(ctx, entryTimeout) + go func() { + defer close(rootChan) + for _, subroot := range mr.subroots { + select { + case <-asyncContext.Done(): + return + case rootChan <- directoryStringEntry{string: subroot}: + continue + } + } + }() + return backgroundDir(asyncContext, len(mr.subroots), rootChan) +} + +/* +mountroot: entries: func() lookup([]static-stringl-list) +*/ + +func parseRootPath(fs *FUSEIPFS, path string) (fusePath, error) { + //pass in root via context on init + //use root on object self to reinit self + switch path { + case filesRootPrefix: + return mfsNode{root: fs.filesRoot}, nil + case "/ipns": + return &keyRoot{keyAPI: fs.core.Key(), softDirRoot: csd(path, fs.mountTime)}, nil + case "/ipfs": + return &pinRoot{pinAPI: fs.core.Pin(), softDirRoot: csd(path, fs.mountTime)}, nil + case "/": + return &mountRoot{subroots: []string{"/ipfs", "/ipns", filesRootPrefix}, + csd(path, fs.mountTime)}, nil + } } -type mfsRoot struct { - rootBase +func csd(path string, now fuse.Timespec) softDirRoot { + sd := softDirRoot{recordBase: crb(path)} + meta := &sd.recordBase.metadata + meta.Birthtim, meta.Atim, meta.Mtim, meta.Ctim = now, now, now, now + return sd } diff --git a/core/commands/mount/system_linux.go b/core/commands/mount/system_linux.go index c148d9891d0..95df834efc1 100644 --- a/core/commands/mount/system_linux.go +++ b/core/commands/mount/system_linux.go @@ -6,6 +6,11 @@ import ( "github.com/billziss-gh/cgofuse/fuse" ) +const ( + O_DIRECTORY = syscall.O_DIRECTORY + O_NOFOLLOW = syscall.O_NOFOLLOW +) + func (fs *FUSEIPFS) fuseFreeSize(fStatfs *fuse.Statfs_t, path string) error { sysStat := &syscall.Statfs_t{} if err := syscall.Statfs(path, sysStat); err != nil { diff --git a/core/commands/mount/system_windows.go b/core/commands/mount/system_windows.go index 2d337180f84..15bed5e0007 100644 --- a/core/commands/mount/system_windows.go +++ b/core/commands/mount/system_windows.go @@ -9,7 +9,11 @@ import ( ) //try to extract these into other pkg(s) -const LOAD_LIBRARY_SEARCH_SYSTEM32 = 0x00000800 +const ( + LOAD_LIBRARY_SEARCH_SYSTEM32 = 0x00000800 + O_DIRECTORY = 0x00000001 // Nt FILE_DIRECTORY_FILE + O_NOFOLLOW = 0 // TODO: check WINAPI for equivalent +) func loadSystemDLL(name string) (*windows.DLL, error) { modHandle, err := windows.LoadLibraryEx(name, 0, LOAD_LIBRARY_SEARCH_SYSTEM32) diff --git a/core/commands/mount/utils.go b/core/commands/mount/utils.go index 86951a848cb..d15a445c6c1 100644 --- a/core/commands/mount/utils.go +++ b/core/commands/mount/utils.go @@ -2,13 +2,14 @@ package fusemount import ( "context" - "errors" "fmt" gopath "path" "strings" + "time" dag "gx/ipfs/QmPJNbVw8o3ohC43ppSXyNXwYKsWShG4zygnirHptfbHri/go-merkledag" cid "gx/ipfs/QmTbxNB1NwDesLmKTscr4udL2tVP7MaxvXnD1D9yX7g3PN/go-cid" + ds "gx/ipfs/QmUadX5EcvrBmxAV9sE7wUWtWSqxns5K84qKJBixmcT1w9/go-datastore" coreiface "gx/ipfs/QmXLwxifxwfc2bAwq6rdjbYqAsGzWsDE9RM5TWMGtykyj6/interface-go-ipfs-core" coreoptions "gx/ipfs/QmXLwxifxwfc2bAwq6rdjbYqAsGzWsDE9RM5TWMGtykyj6/interface-go-ipfs-core/options" ipld "gx/ipfs/QmZ6nzCLwGLVfRzYLpD7pW6UNuBDKEcA2imJtVpbEx2rxy/go-ipld-format" @@ -41,45 +42,61 @@ func platformException(path string) bool { return strings.HasSuffix(path, ".exe.Config") } -func (fs *FUSEIPFS) fuseReadlink(fsNode fusePath) (string, error) { - - ipldNode, err := fs.core.ResolveNode(fs.ctx, fsNode) +func unixAddChild(ctx context.Context, dagSrv coreiface.APIDagService, rootNode ipld.Node, path string, node ipld.Node) (ipld.Node, error) { + rootDir, err := uio.NewDirectoryFromNode(dagSrv, rootNode) if err != nil { - return "", err + return nil, err } - unixNode, err := unixfs.ExtractFSNode(ipldNode) + err = rootDir.AddChild(ctx, path, node) if err != nil { - return "", err + return nil, err } - if unixNode.Type() != unixfs.TSymlink { - return "", errNoLink + newRoot, err := rootDir.GetNode() + if err != nil { + return nil, err } - return string(unixNode.Data()), nil + if err := dagSrv.Add(ctx, newRoot); err != nil { + return nil, err + } + return newRoot, nil } -func unixAddChild(ctx context.Context, dagSrv coreiface.APIDagService, rootNode ipld.Node, path string, node ipld.Node) (ipld.Node, error) { - rootDir, err := uio.NewDirectoryFromNode(dagSrv, rootNode) +func resolveIpns(ctx context.Context, path string, core coreiface.CoreAPI) (ipld.Node, error) { + pathKey, subPath := ipnsSplit(path) + + oAPI, err := core.WithOptions(coreoptions.Api.Offline(true)) if err != nil { return nil, err } - err = rootDir.AddChild(ctx, path, node) - if err != nil { + var nameAPI coreiface.NameAPI + globalPath := path + coreKey, err := resolveKeyName(ctx, oAPI.Key(), pathKey) + switch err { + case nil: // locally owned keys are resolved offline + globalPath = gopath.Join(coreKey.Path().String(), subPath) + nameAPI = oAPI.Name() + + case errNoKey: // paths without owned keys are valid, but looked up via network instead of locally + nameAPI = core.Name() + + case ds.ErrNotFound: // API conflict; A key exists, but holds no value (generated but not published to) + return nil, fmt.Errorf("IPNS key %q has no value: %s", pathKey, err) + default: return nil, err } - newRoot, err := rootDir.GetNode() + resolvedPath, err := nameAPI.Resolve(ctx, globalPath, coreoptions.Name.Cache(true)) if err != nil { return nil, err } - if err := dagSrv.Add(ctx, newRoot); err != nil { - return nil, err - } - return newRoot, nil + // target resolution is done with online core regardless of ownership + // (local node may not have target path, but not target/key data) + return core.ResolveNode(ctx, resolvedPath) } func resolveKeyName(ctx context.Context, api coreiface.KeyAPI, keyString string) (coreiface.Key, error) { @@ -92,7 +109,7 @@ func resolveKeyName(ctx context.Context, api coreiface.KeyAPI, keyString string) return nil, err } for _, key := range keys { - if keyString == key.Name() { + if keyString == key.Name() || keyString == key.Id() { return key, nil } } @@ -101,7 +118,7 @@ func resolveKeyName(ctx context.Context, api coreiface.KeyAPI, keyString string) } //TODO: remove this and inline publishing? -func (fs *FUSEIPFS) ipnsDelayedPublish(key coreiface.Key, node ipld.Node) error { +func ipnsDelayedPublish(ctx context.Context, key coreiface.Key, node ipld.Node) error { oAPI, err := fs.core.WithOptions(coreoptions.Api.Offline(true)) if err != nil { return err @@ -112,7 +129,7 @@ func (fs *FUSEIPFS) ipnsDelayedPublish(key coreiface.Key, node ipld.Node) error return err } - _, err = oAPI.Name().Publish(fs.ctx, coreTarget, coreoptions.Name.Key(key.Name()), coreoptions.Name.AllowOffline(true)) + _, err = oAPI.Name().Publish(ctx, coreTarget, coreoptions.Name.Key(key.Name()), coreoptions.Name.AllowOffline(true)) if err != nil { return err } @@ -121,7 +138,6 @@ func (fs *FUSEIPFS) ipnsDelayedPublish(key coreiface.Key, node ipld.Node) error return nil } -//TODO: reconsider parameters func ipnsPublisher(keyName string, nameAPI coreiface.NameAPI) func(context.Context, cid.Cid) error { return func(ctx context.Context, rootCid cid.Cid) error { //log.Errorf("publish request; key:%q cid:%q", keyName, rootCid) @@ -142,40 +158,174 @@ func (fs FUSEIPFS) ipnsMFSSplit(path string) (*mfs.Root, string, error) { } //XXX -func emptyNode(ctx context.Context, dagAPI coreiface.APIDagService, nodeType upb.Data_DataType, filePrefix *cid.Prefix) (ipld.Node, error) { - if nodeType == unixfs.TFile { +func emptyNode(ctx context.Context, dagAPI coreiface.APIDagService, nodeType upb.Data_DataType) (ipld.Node, error) { + switch nodeType { + case unixfs.TFile: eFile := dag.NodeWithData(unixfs.FilePBData(nil, 0)) - if filePrefix != nil { - eFile.SetCidBuilder(filePrefix.WithCodec(filePrefix.GetCodec())) - } if err := dagAPI.Add(ctx, eFile); err != nil { return nil, err } return eFile, nil - } else if nodeType == unixfs.TDirectory { + case unixfs.TDirectory: eDir, err := uio.NewDirectory(dagAPI).GetNode() if err != nil { return nil, err } return eDir, nil - } else { - return nil, errors.New("unexpected node type") + default: + return nil, errUnexpected } - } -//TODO: docs; return: key, path, error -//TODO: check if there's overlap with go-path -func ipnsSplit(path string) (string, string, error) { +//TODO: docs; return: key, path +func ipnsSplit(path string) (string, string) { splitPath := strings.Split(path, "/") - if len(splitPath) < 3 { - return "", "", errInvalidPath - } - key := splitPath[2] index := strings.Index(path, key) + len(key) - if index != len(path) { - return key, path[index+1:], nil //strip leading '/' + return key, path[index:] +} + +func deriveCallContext(ctx context.Context) (context.Context, context.CancelFunc) { + return context.WithTimeout(ctx, callTimeout) +} + +type timerContextActual struct { + context.Context + cancel context.CancelFunc + timer time.Timer + grace time.Duration +} + +func (tctx *timerContextActual) Reset() { + if !tctx.timer.Stop() { + <-tctx.timer.C + } + tctx.timer.Reset(tctx.grace) +} + +type timerContext interface { + context.Context + Reset() + Cancel() +} + +func deriveTimerContext(ctx context.Context, grace time.Duration) timerContext { + asyncContext, cancel := context.WithCancel(ctx) + timer := time.AfterFunc(grace, cancel()) + tctx := timerContextActual{context.Context: asyncContext, cancel: cancel, grace: grace, timer: timer} + + return tctx +} + +func checkAPIKeystore(ctx context.Context, keyAPI coreiface.KeyAPI, coreKey coreiface.Key) error { + coreKey, err := resolveKeyName(ctx, keyAPI, coreKey.Name()) + switch err { + default: + return err + case errNoKey: + return errNoKey + case nil: // path contains key we own + if api, arg := coreKey.ID(), coreKey.ID(); api != arg { + return fmt.Errorf("key ID conflict, daemon:%q File System:%q", api, arg) + } + return nil + } +} + +func mfsFromKey(ctx context.Context, coreKey coreiface.Key, core coreiface.CoreAPI) (*mfs.Root, error) { + ipldNode, err := core.ResolveNode(ctx, coreKey.Path()) //TODO: offline this + if err != nil { + return nil, err + } + + pbNode, ok := ipldNode.(*dag.ProtoNode) + if !ok { + return nil, fmt.Errorf("key %q has incompatible type %T", coreKey.Name(), ipldNode) + } + + return mfs.NewRoot(ctx, core.Dag(), pbNode, ipnsPublisher(coreKey.Name(), core.Name())) +} + +func initOrGetMFSKeyRoot(ctx context.Context, keyName string, nr nameRootIndex) (*mfs.Root, error) { + nr.Lock() + defer nr.Unlock() + mroot, err := nr.Request(keyName) + switch err { + case errNotInitialized: + //init mfs + nn.core.ResolveNode(ctx, key.Path()) //TODO: offline this? + pbNode, ok := keyNode.(*dag.ProtoNode) + if !ok { + return nil, fmt.Errorf("key %q has incompatible type %T", keyName, keyNode) + } + + //continue modifying this struct + keyRoot, err := mfs.NewRoot(ctx, nn.core.Dag(), pbNode, ipnsPublisher(key.Name(), nn.core.Name())) + //nn.core.Name().Resolve(ctx, key.Path.Stringname string, opts ...options.NameResolveOption) (Path, error) + + //resolve key to node, node to key io + + //register() + default: + return nil, err + } + +} + +/* +func lockUp(path string, lookup lookupFn) (unlock func()) { + components := strings.Split(path, "/") + + nodeLocks := make([]func(), len(components)) + var wg sync.WaitGroup + wg.Add(len(components)) + + for i := len(components); i >= 0; i-- { + go func(i int) { + defer wg.Done() + node, err := lookup(components[i]) + if err != nil { + return + } + node.Lock() + nodeLocks = append(nodeLocks, node.Unlock) + }(i) + } + for curPath := gopath.Dir(path); curPath != "/"; { + } + + unlock = func() { + for _, unlock := range nodeLocks { + unlock() + } + } + return +} +*/ +/* +func decap(string, subsystemType typeToken) string { + switch subsystemType + /ipns/key/whatever -> /whatever + /ipfs/Qm... -> /Qm... +} +*/ + +//TODO: check how MFS uses this context +func ipnsToMFSRoot(ctx context.Context, path string, core coreiface.CoreAPI) (*mfs.Root, error) { + + keyName, _ := ipnsSplit(path) + + ipldNode, err := resolveIpns(ctx, path, core) + if err != nil { + return nil, err + } + pbNode, ok := ipldNode.(*dag.ProtoNode) + if !ok { + return nil, fmt.Errorf("key %q points to incompatible type %T", keyName, ipldNode) + } + mroot, err := mfs.NewRoot(ctx, core.Dag(), pbNode, ipnsPublisher(keyName, core.Name())) + if err != nil { + return nil, err } - return key, "", nil + return mroot, nil }