forked from grailbio/base
/
fsnodefuse.go
81 lines (75 loc) · 4.26 KB
/
fsnodefuse.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
// fsnodefuse implements github.com/hanwen/go-fuse/v2/fs for fsnode.T.
// It's a work-in-progress. No correctness or stability is guaranteed. Or even suggested.
//
// fsnode.Parent naturally becomes a directory. fsnode.Leaf becomes a file. Support for FUSE
// operations on that file depends on what Leaf.Open returns. If that fsctx.File is also
// spliceio.ReaderAt:
// FUSE file supports concurrent, random reads and uses splices to reduce
// userspace <-> kernelspace memory copying.
// ioctx.ReaderAt:
// FUSE file supports concurrent, random reads.
// Otherwise:
// FUSE file supports in-order, contiguous reads only. That is, each read must
// start where the previous one ended. At fsctx.File EOF, file size is recorded
// and then overrides what fsctx.File.Stat() reports for future getattr calls,
// so users can see they're done reading.
// TODO: Decide if there's a better place for this feature.
package fsnodefuse
import (
"fmt"
"runtime"
"github.com/Schaudge/grailbase/file/fsnode"
"github.com/Schaudge/grailbase/file/internal/kernel"
"github.com/hanwen/go-fuse/v2/fs"
"github.com/hanwen/go-fuse/v2/fuse"
)
// NewRoot creates a FUSE inode whose contents are the given fsnode.T.
// Note that this inode must be mounted with options from ConfigureRequiredMountOptions.
func NewRoot(node fsnode.T) fs.InodeEmbedder {
switch n := node.(type) {
case fsnode.Parent:
return &dirInode{n: n}
case fsnode.Leaf:
// TODO(josh): Test this path.
return ®Inode{n: n}
}
panic(fmt.Sprintf("unrecognized fsnode type: %T, %[1]v", node))
}
// ConfigureRequiredMountOptions sets values in opts to be compatible with fsnodefuse's
// implementation. Users of NewRoot must use these options, and they should call this last,
// to make sure the required options take effect.
func ConfigureRequiredMountOptions(opts *fuse.MountOptions) {
opts.MaxReadAhead = kernel.MaxReadAhead
}
// ConfigureDefaultMountOptions provides defaults that callers may want to start with, for performance.
func ConfigureDefaultMountOptions(opts *fuse.MountOptions) {
// Increase MaxBackground from its default value (12) to improve S3 read performance.
//
// Empirically, while reading a 30 GiB files in chunks in parallel, the number of concurrent
// reads processed by our FUSE server [1] was ~12 with the default, corresponding to poor
// network utilization (only 500 Mb/s on m5d.4x in EC2); it rises to ~120 after, and network
// read bandwidth rises to >7 Gb/s, close to the speed of reading directly from S3 with
// this machine (~9 Gb/s).
//
// libfuse documentation [2] suggests that this limits the number of kernel readahead
// requests, so raising the limit may allow kernel readahead for every chunk, which could
// plausibly explain the performance benefit. (There's also mention of large direct I/O
// requests from userspace; josh@ did not think his Go test program was using direct I/O for
// this benchmark, but maybe he just didn't know).
//
// This particular value is a somewhat-informed guess. We'd like it to be high enough to
// admit all the parallelism that applications may profitably want. EC2 instances generally
// have <1 Gb/s network bandwidth per CPU (m5n.24x is around that, and non-'n' types have
// several times less), and S3 connections are limited to ~700 Mb/s [3], so just a couple of
// read chunks per CPU are sufficient to be I/O-bound for large objects. Many smaller object
// reads tend to not reach maximum bandwidth, so applications may increase parallelism,
// so we set our limit several times higher.
// TODO: Run more benchmarks (like github.com/Schaudge/grailbase/file/filebench) and tune.
//
// [1] As measured by simple logging: https://gitlab.com/grailbio/grail/-/merge_requests/8292/diffs?commit_id=7681acfcac836b92eaca60eb567245b32b81ec50
// [2] https://web.archive.org/web/20220815053939/https://libfuse.github.io/doxygen/structfuse__conn__info.html#a5f9e695735727343448ae1e1a86dfa03
// [3] 85-90 MB/s: https://web.archive.org/web/20220325121400/https://docs.aws.amazon.com/AmazonS3/latest/userguide/optimizing-performance-design-patterns.html#optimizing-performance-parallelization
opts.MaxBackground = 16 * runtime.NumCPU()
// We don't use extended attributes so we can skip these requests to improve performance.
opts.DisableXAttrs = true
}