From 0d024ed29a925d436e2d5f2410fd0a72d95f2b5e Mon Sep 17 00:00:00 2001 From: Shayon Mukherjee Date: Wed, 15 Apr 2026 14:56:34 -0400 Subject: [PATCH] gofer: add pluggable provider interface for custom filesystem backends Building a custom gofer (e.g. for network-backed storage, encrypted filesystems, or tiered caches) currently requires forking the runsc binary and copying/maintaining unexported setup and seccomp code. This adds a Provider interface that lets custom filesystem backends register with the stock gofer and serve LisaFS connections for specific mounts without forking. The interface follows the socket.Provider pattern: NewServer returns (nil, nil) to decline a mount, and the first registered provider that returns a non-nil server handles it. NewServer receives the sandbox's OCI runtime spec so providers can read per-mount configuration from spec.Annotations (endpoints, volume keys, auth modes) without a side-channel. Stock fsgofer remains the default when no provider claims a mount. SeccompRules lets providers declare additional syscalls, merged via filter.Rules() before installation. Zero behavior change when no providers are registered: the stock fsgofer path runs unchanged, identical to today. This follows the same pattern as the network plugin (config.NetworkPlugin): inactive when not configured, no impact on the default path. New package runsc/gofer/provider defines the Provider interface and registration. The gofer command (runsc/cmd/gofer.go) iterates registered providers for each mount before falling through to fsgofer. The seccomp filter (runsc/fsgofer/filter) gains InstallWithExtra for merging provider rules with the stock allowlist. Also adds documentation in g3doc/user_guide/filesystem.md and pkg/lisafs/README.md describing how to use the provider interface. Depends on #12923 (GoferMountConf in specutils). --- g3doc/user_guide/filesystem.md | 27 ++++ pkg/lisafs/README.md | 7 + runsc/cmd/BUILD | 3 + runsc/cmd/gofer.go | 53 ++++++- runsc/fsgofer/filter/filter.go | 11 +- runsc/gofer/BUILD | 4 + runsc/gofer/provider/BUILD | 31 ++++ runsc/gofer/provider/provider.go | 60 ++++++++ runsc/gofer/provider/provider_test.go | 213 ++++++++++++++++++++++++++ 9 files changed, 402 insertions(+), 7 deletions(-) create mode 100644 runsc/gofer/BUILD create mode 100644 runsc/gofer/provider/BUILD create mode 100644 runsc/gofer/provider/provider.go create mode 100644 runsc/gofer/provider/provider_test.go diff --git a/g3doc/user_guide/filesystem.md b/g3doc/user_guide/filesystem.md index 6a592c08eb..fd6af80b3f 100644 --- a/g3doc/user_guide/filesystem.md +++ b/g3doc/user_guide/filesystem.md @@ -273,3 +273,30 @@ runsc --root=/path/to/rootdir debug --mount erofs:{source}:{destination} ``` [Production guide]: production.md + +## Custom Gofer Providers + +The stock gofer serves host filesystem mounts via LISAFS. For workloads that +need a different filesystem backend (a network-backed store, an encrypted +filesystem, a tiered cache), the `runsc/gofer/provider` package lets you plug in +a custom `lisafs.ServerImpl` without forking the runsc binary. + +A provider registers itself at startup and claims mounts by path. For example, a +binary might register one provider that handles mounts under `/storage` via an +HTTP blob service, and another that handles `/encrypted` via a local encryption +layer. Unclaimed mounts fall through to the stock fsgofer as usual. + +To build a custom gofer: + +1. Implement the `provider.Provider` interface: `Name`, `NewServer`, and + `SeccompRules`. +2. Register your provider in `init()` or early `main()`. +3. Build a binary that imports `runsc/cli` and your provider package. + +The provider's `NewServer` returns a `*lisafs.Server` for mounts it handles, or +`(nil, nil)` to decline. Per-mount configuration (endpoints, volume keys) is +passed via OCI annotations on the spec. The `SeccompRules` method declares any +additional syscalls the provider needs beyond the stock gofer allowlist (e.g. +for networking). See the `runsc/gofer/provider` package documentation and +`pkg/lisafs/testsuite` for testing custom `ServerImpl` implementations. + diff --git a/pkg/lisafs/README.md b/pkg/lisafs/README.md index 0f76092f76..c5ad17da35 100644 --- a/pkg/lisafs/README.md +++ b/pkg/lisafs/README.md @@ -42,6 +42,13 @@ accessed/mutated via RPCs by LISAFS clients. The server is a trusted process. For security reasons, the server must assume that the client can be potentially compromised and act maliciously. +The server-side interface is `ServerImpl` (defined in `server.go`). The stock +implementation is `runsc/fsgofer`, which serves host filesystem mounts. The +`runsc/gofer/provider` package allows registering alternative `ServerImpl` +implementations for specific mounts, so that a custom gofer binary can serve +some mounts from a different backend (e.g. a network filesystem) while the +stock fsgofer handles the rest. + #### Concurrency The server must execute file system operations under appropriate concurrency diff --git a/runsc/cmd/BUILD b/runsc/cmd/BUILD index abf94a9722..8984e20e15 100644 --- a/runsc/cmd/BUILD +++ b/runsc/cmd/BUILD @@ -91,10 +91,12 @@ go_library( "//pkg/cpuid", "//pkg/fd", "//pkg/hostarch", + "//pkg/lisafs", "//pkg/log", "//pkg/metric", "//pkg/prometheus", "//pkg/ring0", + "//pkg/seccomp", "//pkg/sentry/control", "//pkg/sentry/devices/nvproxy/nvconf", "//pkg/sentry/devices/tpuproxy", @@ -119,6 +121,7 @@ go_library( "//runsc/flag", "//runsc/fsgofer", "//runsc/fsgofer/filter", + "//runsc/gofer/provider", "//runsc/metricserver/containermetrics", "//runsc/mitigate", "//runsc/profile", diff --git a/runsc/cmd/gofer.go b/runsc/cmd/gofer.go index bb465c0d9d..4d01546a3b 100644 --- a/runsc/cmd/gofer.go +++ b/runsc/cmd/gofer.go @@ -25,7 +25,9 @@ import ( "github.com/google/subcommands" specs "github.com/opencontainers/runtime-spec/specs-go" "golang.org/x/sys/unix" + "gvisor.dev/gvisor/pkg/lisafs" "gvisor.dev/gvisor/pkg/log" + "gvisor.dev/gvisor/pkg/seccomp" "gvisor.dev/gvisor/pkg/unet" "gvisor.dev/gvisor/pkg/urpc" "gvisor.dev/gvisor/runsc/cmd/sandboxsetup" @@ -35,6 +37,7 @@ import ( "gvisor.dev/gvisor/runsc/flag" "gvisor.dev/gvisor/runsc/fsgofer" "gvisor.dev/gvisor/runsc/fsgofer/filter" + "gvisor.dev/gvisor/runsc/gofer/provider" "gvisor.dev/gvisor/runsc/profile" "gvisor.dev/gvisor/runsc/specutils" ) @@ -293,7 +296,11 @@ func (g *Gofer) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomm DirectFS: conf.DirectFS, CgoEnabled: config.CgoEnabled, } - if err := filter.Install(opts); err != nil { + extraRules := seccomp.NewSyscallRules() + for _, p := range provider.Registered() { + extraRules.Merge(p.SeccompRules()) + } + if err := filter.Install(opts, extraRules); err != nil { util.Fatalf("installing seccomp filters: %v", err) } @@ -305,6 +312,8 @@ func (g *Gofer) serve(spec *specs.Spec, conf *config.Config, root string, ruid i sock *unet.Socket mountPath string readonly bool + mountConf specutils.GoferMountConf + mount *specs.Mount } cfgs := make([]connectionConfig, 0, len(spec.Mounts)+1) server := fsgofer.NewLisafsServer(fsgofer.Config{ @@ -327,14 +336,16 @@ func (g *Gofer) serve(spec *specs.Spec, conf *config.Config, root string, ruid i sock: sandboxsetup.NewSocket(ioFDs[0]), mountPath: "/", // fsgofer process is always chroot()ed. So serve root. readonly: spec.Root.Readonly || rootfsConf.ShouldUseOverlayfs(), + mountConf: rootfsConf, }) log.Infof("Serving %q mapped to %q on FD %d (ro: %t)", "/", root, ioFDs[0], cfgs[0].readonly) ioFDs = ioFDs[1:] } mountIdx := 1 // first one is the root - for _, m := range spec.Mounts { - if !specutils.HasMountConfig(m) { + for i := range spec.Mounts { + m := &spec.Mounts[i] + if !specutils.HasMountConfig(*m) { continue } mountConf := g.mountConfs[mountIdx] @@ -356,6 +367,8 @@ func (g *Gofer) serve(spec *specs.Spec, conf *config.Config, root string, ruid i sock: sandboxsetup.NewSocket(ioFD), mountPath: m.Destination, readonly: readonly, + mountConf: mountConf, + mount: m, }) log.Infof("Serving %q mapped on FD %d (ro: %t)", m.Destination, ioFD, readonly) } @@ -372,15 +385,45 @@ func (g *Gofer) serve(spec *specs.Spec, conf *config.Config, root string, ruid i log.Infof("Serving /dev mapped on FD %d (ro: false)", g.devIoFD) } + var providerServers []*lisafs.Server + seen := map[*lisafs.Server]bool{} for _, cfg := range cfgs { - conn, err := server.CreateConnection(cfg.sock, cfg.mountPath, cfg.readonly) + var srv *lisafs.Server + // /dev is always served by the stock fsgofer. + if cfg.mountPath != "/dev" { + for _, p := range provider.Registered() { + var err error + srv, err = p.NewServer(spec, cfg.mount, cfg.mountPath, cfg.mountConf, cfg.readonly) + if err != nil { + util.Fatalf("provider %s for %q: %v", p.Name(), cfg.mountPath, err) + } + if srv != nil { + if !seen[srv] { + seen[srv] = true + providerServers = append(providerServers, srv) + } + log.Infof("Serving %q via provider %s on FD %d", cfg.mountPath, p.Name(), cfg.sock.FD()) + break + } + } + } + if srv == nil { + srv = &server.Server + } + conn, err := srv.CreateConnection(cfg.sock, cfg.mountPath, cfg.readonly) if err != nil { util.Fatalf("starting connection on FD %d for gofer mount failed: %v", cfg.sock.FD(), err) } - server.StartConnection(conn) + srv.StartConnection(conn) } server.Wait() + for _, ps := range providerServers { + ps.Wait() + } server.Destroy() + for _, ps := range providerServers { + ps.Destroy() + } log.Infof("All lisafs servers exited.") if g.stopProfiling != nil { g.stopProfiling() diff --git a/runsc/fsgofer/filter/filter.go b/runsc/fsgofer/filter/filter.go index 12b3aab5b8..63e1a93f81 100644 --- a/runsc/fsgofer/filter/filter.go +++ b/runsc/fsgofer/filter/filter.go @@ -72,10 +72,17 @@ func Rules(opt Options) seccomp.SyscallRules { return s } -// Install installs seccomp filters. -func Install(opt Options) error { +// Install installs seccomp filters. Any extras are merged into the stock +// allowlist before installatio +func Install(opt Options, extras ...seccomp.SyscallRules) error { s := Rules(opt) + for _, extra := range extras { + s.Merge(extra) + } + return install(s) +} +func install(s seccomp.SyscallRules) error { program := &seccomp.Program{ RuleSets: []seccomp.RuleSet{ { diff --git a/runsc/gofer/BUILD b/runsc/gofer/BUILD new file mode 100644 index 0000000000..0374e2dd19 --- /dev/null +++ b/runsc/gofer/BUILD @@ -0,0 +1,4 @@ +package( + default_applicable_licenses = ["//:license"], + licenses = ["notice"], +) diff --git a/runsc/gofer/provider/BUILD b/runsc/gofer/provider/BUILD new file mode 100644 index 0000000000..3d6322d808 --- /dev/null +++ b/runsc/gofer/provider/BUILD @@ -0,0 +1,31 @@ +load("//tools:defs.bzl", "go_library", "go_test") + +package( + default_applicable_licenses = ["//:license"], + licenses = ["notice"], +) + +go_library( + name = "provider", + srcs = ["provider.go"], + visibility = ["//runsc:__subpackages__"], + deps = [ + "//pkg/lisafs", + "//pkg/seccomp", + "//runsc/specutils", + "@com_github_opencontainers_runtime_spec//specs-go:go_default_library", + ], +) + +go_test( + name = "provider_test", + size = "small", + srcs = ["provider_test.go"], + library = ":provider", + deps = [ + "//pkg/lisafs", + "//pkg/seccomp", + "//runsc/specutils", + "@com_github_opencontainers_runtime_spec//specs-go:go_default_library", + ], +) diff --git a/runsc/gofer/provider/provider.go b/runsc/gofer/provider/provider.go new file mode 100644 index 0000000000..3cdeac3419 --- /dev/null +++ b/runsc/gofer/provider/provider.go @@ -0,0 +1,60 @@ +// Copyright 2026 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package provider defines an interface for pluggable gofer filesystem +// backends. The stock fsgofer handles any mount no Provider claims. +package provider + +import ( + specs "github.com/opencontainers/runtime-spec/specs-go" + "gvisor.dev/gvisor/pkg/lisafs" + "gvisor.dev/gvisor/pkg/seccomp" + "gvisor.dev/gvisor/runsc/specutils" +) + +// Provider is implemented by alternative LisaFS backends. It follows the +// same pattern as socket.Provider: the first registered Provider whose +// NewServer returns a non-nil server handles the mount. +type Provider interface { + // Name identifies the provider in log messages. + Name() string + + // NewServer returns a LisaFS server for the given mount, or + // (nil, nil) if this provider does not handle it. A non-nil error + // means the provider claims the mount but failed to initialize. + // + // mount is nil for the root filesystem (root is not present in + // spec.Mounts). Per-sandbox config may be read from spec.Annotations. + // + // A Provider may return the same *lisafs.Server across calls to + // multiplex mounts onto one server; the caller deduplicates. + NewServer(spec *specs.Spec, mount *specs.Mount, mountPath string, conf specutils.GoferMountConf, readonly bool) (*lisafs.Server, error) + + // SeccompRules returns additional rules to merge into the stock + // gofer's seccomp allowlist. + SeccompRules() seccomp.SyscallRules +} + +var registered []Provider + +// Register adds p to the provider list. Must be called during init or +// early in main, before Registered is iterated. +func Register(p Provider) { + registered = append(registered, p) +} + +// Registered returns all registered providers in registration order. +func Registered() []Provider { + return registered +} diff --git a/runsc/gofer/provider/provider_test.go b/runsc/gofer/provider/provider_test.go new file mode 100644 index 0000000000..390904d3d8 --- /dev/null +++ b/runsc/gofer/provider/provider_test.go @@ -0,0 +1,213 @@ +// Copyright 2026 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package provider + +import ( + "errors" + "strings" + "testing" + + specs "github.com/opencontainers/runtime-spec/specs-go" + "gvisor.dev/gvisor/pkg/lisafs" + "gvisor.dev/gvisor/pkg/seccomp" + "gvisor.dev/gvisor/runsc/specutils" +) + +type fakeProvider struct { + name string + prefix string + server *lisafs.Server + calls int + lastSpec *specs.Spec + lastMount *specs.Mount + lastPath string + lastConf specutils.GoferMountConf + lastReadonly bool + seccomp seccomp.SyscallRules +} + +func (f *fakeProvider) Name() string { return f.name } + +func (f *fakeProvider) NewServer(spec *specs.Spec, mount *specs.Mount, mountPath string, conf specutils.GoferMountConf, readonly bool) (*lisafs.Server, error) { + f.calls++ + f.lastSpec = spec + f.lastMount = mount + f.lastPath = mountPath + f.lastConf = conf + f.lastReadonly = readonly + if !strings.HasPrefix(mountPath, f.prefix) { + return nil, nil + } + if f.server == nil { + f.server = &lisafs.Server{} + } + return f.server, nil +} + +func (f *fakeProvider) SeccompRules() seccomp.SyscallRules { + return f.seccomp +} + +func TestRegisterAndRegistered(t *testing.T) { + registered = nil + + p1 := &fakeProvider{name: "p1", prefix: "/storage"} + p2 := &fakeProvider{name: "p2", prefix: "/data"} + Register(p1) + Register(p2) + + all := Registered() + if len(all) != 2 { + t.Fatalf("got %d providers, want 2", len(all)) + } + if all[0] != p1 || all[1] != p2 { + t.Fatalf("providers not in registration order") + } +} + +func firstNonNil(spec *specs.Spec, mount *specs.Mount, path string, conf specutils.GoferMountConf, readonly bool) (Provider, *lisafs.Server, error) { + for _, p := range Registered() { + srv, err := p.NewServer(spec, mount, path, conf, readonly) + if err != nil { + return p, nil, err + } + if srv != nil { + return p, srv, nil + } + } + return nil, nil, nil +} + +func TestFirstNonNilServerWins(t *testing.T) { + registered = nil + + p1 := &fakeProvider{name: "first", prefix: "/storage"} + p2 := &fakeProvider{name: "second", prefix: "/storage"} + Register(p1) + Register(p2) + + winner, srv, err := firstNonNil(&specs.Spec{}, nil, "/storage/vol", specutils.GoferMountConf{}, false) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if srv == nil { + t.Fatalf("expected a non-nil server") + } + if winner.Name() != "first" { + t.Fatalf("winner = %q, want %q", winner.Name(), "first") + } + if p1.calls != 1 { + t.Fatalf("p1 calls = %d, want 1", p1.calls) + } + if p2.calls != 0 { + t.Fatalf("p2 calls = %d, want 0 (iteration should have stopped)", p2.calls) + } +} + +func TestDeclineFallsThrough(t *testing.T) { + registered = nil + + p1 := &fakeProvider{name: "storage", prefix: "/storage"} + p2 := &fakeProvider{name: "data", prefix: "/data"} + Register(p1) + Register(p2) + + winner, srv, err := firstNonNil(&specs.Spec{}, nil, "/tmp", specutils.GoferMountConf{}, false) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if winner != nil || srv != nil { + t.Fatalf("expected no provider to claim /tmp, got %v / %v", winner, srv) + } + if p1.calls != 1 || p2.calls != 1 { + t.Fatalf("calls = (%d, %d), want (1, 1)", p1.calls, p2.calls) + } +} + +type errorProvider struct { + name string + err error +} + +func (e *errorProvider) Name() string { return e.name } +func (e *errorProvider) NewServer(*specs.Spec, *specs.Mount, string, specutils.GoferMountConf, bool) (*lisafs.Server, error) { + return nil, e.err +} +func (e *errorProvider) SeccompRules() seccomp.SyscallRules { return seccomp.NewSyscallRules() } + +func TestErrorStopsIteration(t *testing.T) { + registered = nil + + wantErr := errors.New("boom") + p1 := &errorProvider{name: "broken", err: wantErr} + p2 := &fakeProvider{name: "fallback", prefix: "/storage"} + Register(p1) + Register(p2) + + _, srv, err := firstNonNil(&specs.Spec{}, nil, "/storage/vol", specutils.GoferMountConf{}, false) + if !errors.Is(err, wantErr) { + t.Fatalf("err = %v, want %v", err, wantErr) + } + if srv != nil { + t.Fatalf("expected nil server on error, got %v", srv) + } + if p2.calls != 0 { + t.Fatalf("p2 calls = %d, want 0", p2.calls) + } +} + +func TestArgumentsPassThrough(t *testing.T) { + registered = nil + + p := &fakeProvider{name: "captor", prefix: "/storage"} + Register(p) + + spec := &specs.Spec{Annotations: map[string]string{"dev.example.endpoint": "http://file-service:8080"}} + mount := &specs.Mount{Destination: "/storage/vol", Source: "vol-abc", Type: "bind", Options: []string{"rw"}} + conf := specutils.GoferMountConf{Upper: specutils.NoOverlay, Lower: specutils.Lisafs} + + if _, _, err := firstNonNil(spec, mount, "/storage/vol", conf, true); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if p.lastSpec != spec { + t.Fatalf("spec not forwarded: got %p, want %p", p.lastSpec, spec) + } + if p.lastMount != mount { + t.Fatalf("mount not forwarded: got %p, want %p", p.lastMount, mount) + } + if p.lastPath != "/storage/vol" { + t.Fatalf("mountPath = %q, want %q", p.lastPath, "/storage/vol") + } + if !p.lastReadonly { + t.Fatalf("readonly = false, want true") + } + if p.lastConf != conf { + t.Fatalf("conf = %v, want %v", p.lastConf, conf) + } +} + +func TestNilMountForwarded(t *testing.T) { + registered = nil + + p := &fakeProvider{name: "catch-all", prefix: "/"} + Register(p) + + if _, _, err := firstNonNil(&specs.Spec{}, nil, "/", specutils.GoferMountConf{}, false); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if p.lastMount != nil { + t.Fatalf("lastMount = %v, want nil", p.lastMount) + } +}