Skip to content

Commit

Permalink
virtiofs-nydus: add lazyload support for kata with clh
Browse files Browse the repository at this point in the history 
As kata with qemu has supported lazyload, so this pr aims to
bring lazyload ability to kata with clh.

Fixes #3654

Signed-off-by: luodaowen.backend <luodaowen.backend@bytedance.com>
  • Loading branch information
@luodw
luodw committed Feb 19, 2022
1 parent d0c8eb7 commit 3175aad
Show file tree
Hide file tree
Showing 10 changed files with 108 additions and 63 deletions.
6 changes: 3 additions & 3 deletions docs/how-to/how-to-use-virtio-fs-nydus-with-kata.md
View file
Expand Up @@ -2,7 +2,7 @@

## Introduction

Refer to [kata-`nydus`-design](../design/kata-nydus-design.md)
Refer to [kata-`nydus`-design](../design/kata-nydus-design.md) for introduction and `nydus` has supported Kata Containers with hypervisor `QEMU` and `CLH` currently.

## How to

Expand All @@ -16,15 +16,15 @@ You can use Kata Containers with `nydus` as follows,

4. Use [kata-containers](https://github.com/kata-containers/kata-containers) `latest` branch to compile and build `kata-containers.img`;

5. Update `configuration-qemu.toml` to include:
5. Update `configuration-qemu.toml` or `configuration-clh.toml`to include:

```toml
shared_fs = "virtio-fs-nydus"
virtio_fs_daemon = "<nydusd binary path>"
virtio_fs_extra_args = []
```

6. run `crictl run -r kata-qemu nydus-container.yaml nydus-sandbox.yaml`;
6. run `crictl run -r kata nydus-container.yaml nydus-sandbox.yaml`;

The `nydus-sandbox.yaml` looks like below:

Expand Down
2 changes: 2 additions & 0 deletions src/runtime/Makefile
View file
Expand Up @@ -163,6 +163,7 @@ DEFENTROPYSOURCE := /dev/urandom
DEFVALIDENTROPYSOURCES := [\"/dev/urandom\",\"/dev/random\",\"\"]

DEFDISABLEBLOCK := false
DEFSHAREDFS_CLH_VIRTIOFS := virtio-fs
DEFSHAREDFS_QEMU_VIRTIOFS := virtio-fs
DEFVIRTIOFSDAEMON := $(LIBEXECDIR)/kata-qemu/virtiofsd
DEFVALIDVIRTIOFSDAEMONPATHS := [\"$(DEFVIRTIOFSDAEMON)\"]
Expand Down Expand Up @@ -437,6 +438,7 @@ USER_VARS += DEFDISABLEBLOCK
USER_VARS += DEFBLOCKSTORAGEDRIVER_ACRN
USER_VARS += DEFBLOCKSTORAGEDRIVER_FC
USER_VARS += DEFBLOCKSTORAGEDRIVER_QEMU
USER_VARS += DEFSHAREDFS_CLH_VIRTIOFS
USER_VARS += DEFSHAREDFS_QEMU_VIRTIOFS
USER_VARS += DEFVIRTIOFSDAEMON
USER_VARS += DEFVALIDVIRTIOFSDAEMONPATHS
Expand Down
5 changes: 5 additions & 0 deletions src/runtime/config/configuration-clh.toml.in
View file
Expand Up @@ -70,6 +70,11 @@ default_memory = @DEFMEMSZ@
# This is will determine the times that memory will be hotadded to sandbox/VM.
#memory_slots = @DEFMEMSLOTS@

# Shared file system type:
# - virtio-fs (default)
# - virtio-fs-nydus
shared_fs = "@DEFSHAREDFS_CLH_VIRTIOFS@"

# Path to vhost-user-fs daemon.
virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@"

Expand Down
22 changes: 12 additions & 10 deletions src/runtime/pkg/katautils/config.go
View file
Expand Up @@ -426,7 +426,7 @@ func (h hypervisor) sharedFS() (string, error) {
supportedSharedFS := []string{config.Virtio9P, config.VirtioFS, config.VirtioFSNydus}

if h.SharedFS == "" {
return config.Virtio9P, nil
return config.VirtioFS, nil
}

for _, fs := range supportedSharedFS {
Expand Down Expand Up @@ -644,14 +644,9 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
return vc.HypervisorConfig{}, err
}

if sharedFS == config.VirtioFS && h.VirtioFSDaemon == "" {
if (sharedFS == config.VirtioFS || sharedFS == config.VirtioFSNydus) && h.VirtioFSDaemon == "" {
return vc.HypervisorConfig{},
errors.New("cannot enable virtio-fs without daemon path in configuration file")
}

if sharedFS == config.VirtioFSNydus && h.VirtioFSDaemon == "" {
return vc.HypervisorConfig{},
errors.New("cannot enable virtio nydus without nydusd daemon path in configuration file")
fmt.Errorf("cannot enable %s without daemon path in configuration file", sharedFS)
}

if vSock, err := utils.SupportsVsocks(); !vSock {
Expand Down Expand Up @@ -822,11 +817,18 @@ func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
return vc.HypervisorConfig{}, err
}

sharedFS := config.VirtioFS
sharedFS, err := h.sharedFS()
if err != nil {
return vc.HypervisorConfig{}, err
}

if sharedFS != config.VirtioFS && sharedFS != config.VirtioFSNydus {
return vc.HypervisorConfig{}, errors.New("clh only support virtio-fs or virtio-fs-nydus")
}

if h.VirtioFSDaemon == "" {
return vc.HypervisorConfig{},
errors.New("virtio-fs daemon path is missing in configuration file")
fmt.Errorf("cannot enable %s without daemon path in configuration file", sharedFS)
}

return vc.HypervisorConfig{
Expand Down
4 changes: 4 additions & 0 deletions src/runtime/pkg/katautils/config_test.go
View file
Expand Up @@ -633,6 +633,8 @@ func TestNewQemuHypervisorConfig(t *testing.T) {
PCIeRootPort: pcieRootPort,
RxRateLimiterMaxRate: rxRateLimiterMaxRate,
TxRateLimiterMaxRate: txRateLimiterMaxRate,
SharedFS: "virtio-fs",
VirtioFSDaemon: filepath.Join(dir, "virtiofsd"),
}

files := []string{hypervisorPath, kernelPath, imagePath}
Expand Down Expand Up @@ -1388,6 +1390,8 @@ func TestUpdateRuntimeConfigurationVMConfig(t *testing.T) {
Image: "/",
Firmware: "/",
FirmwareVolume: "/",
SharedFS: "virtio-fs",
VirtioFSDaemon: "/usr/libexec/kata-qemu/virtiofsd",
},
},
}
Expand Down
89 changes: 56 additions & 33 deletions src/runtime/virtcontainers/clh.go
View file
Expand Up @@ -144,27 +144,27 @@ func (c *clhClientApi) VmRemoveDevicePut(ctx context.Context, vmRemoveDevice chc
// Cloud hypervisor state
//
type CloudHypervisorState struct {
apiSocket string
PID int
VirtiofsdPID int
state clhState
apiSocket string
PID int
VirtiofsDaemonPid int
state clhState
}

func (s *CloudHypervisorState) reset() {
s.PID = 0
s.VirtiofsdPID = 0
s.VirtiofsDaemonPid = 0
s.state = clhNotReady
}

type cloudHypervisor struct {
console console.Console
virtiofsd VirtiofsDaemon
APIClient clhClient
ctx context.Context
id string
vmconfig chclient.VmConfig
state CloudHypervisorState
config HypervisorConfig
console console.Console
virtiofsDaemon VirtiofsDaemon
APIClient clhClient
ctx context.Context
id string
vmconfig chclient.VmConfig
state CloudHypervisorState
config HypervisorConfig
}

var clhKernelParams = []Param{
Expand Down Expand Up @@ -198,6 +198,10 @@ func (clh *cloudHypervisor) setConfig(config *HypervisorConfig) error {
return nil
}

func (clh *cloudHypervisor) nydusdAPISocketPath(id string) (string, error) {
return utils.BuildSocketPath(clh.config.VMStorePath, id, nydusdAPISock)
}

// For cloudHypervisor this call only sets the internal structure up.
// The VM will be created and started through StartVM().
func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, network Network, hypervisorConfig *HypervisorConfig) error {
Expand All @@ -223,8 +227,8 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, network Net

if clh.state.PID > 0 {
clh.Logger().WithField("function", "CreateVM").Info("Sandbox already exist, loading from state")
clh.virtiofsd = &virtiofsd{
PID: clh.state.VirtiofsdPID,
clh.virtiofsDaemon = &virtiofsd{
PID: clh.state.VirtiofsDaemonPid,
sourcePath: hypervisorConfig.SharedPath,
debug: clh.config.Debug,
socketPath: virtiofsdSocketPath,
Expand Down Expand Up @@ -349,7 +353,7 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, network Net
ApiInternal: chclient.NewAPIClient(cfg).DefaultApi,
}

clh.virtiofsd = &virtiofsd{
clh.virtiofsDaemon = &virtiofsd{
path: clh.config.VirtioFSDaemon,
sourcePath: filepath.Join(GetSharePath(clh.id)),
socketPath: virtiofsdSocketPath,
Expand All @@ -358,6 +362,25 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, network Net
cache: clh.config.VirtioFSCache,
}

if clh.config.SharedFS == config.VirtioFSNydus {
apiSockPath, err := clh.nydusdAPISocketPath(clh.id)
if err != nil {
clh.Logger().WithError(err).Error("Invalid api socket path for nydusd")
return err
}
nd := &nydusd{
path: clh.config.VirtioFSDaemon,
sockPath: virtiofsdSocketPath,
apiSockPath: apiSockPath,
sourcePath: filepath.Join(GetSharePath(clh.id)),
debug: clh.config.Debug,
extraArgs: clh.config.VirtioFSExtraArgs,
startFn: startInShimNS,
}
nd.setupShareDirFn = nd.setupPassthroughFS
clh.virtiofsDaemon = nd
}

if clh.config.SGXEPCSize > 0 {
epcSection := chclient.NewSgxEpcConfig("kata-epc", clh.config.SGXEPCSize)
epcSection.Prefault = func(b bool) *bool { return &b }(true)
Expand Down Expand Up @@ -389,8 +412,8 @@ func (clh *cloudHypervisor) StartVM(ctx context.Context, timeout int) error {
return err
}

if clh.virtiofsd == nil {
return errors.New("Missing virtiofsd configuration")
if clh.virtiofsDaemon == nil {
return errors.New("Missing virtiofsDaemon configuration")
}

// This needs to be done as late as possible, just before launching
Expand All @@ -402,23 +425,23 @@ func (clh *cloudHypervisor) StartVM(ctx context.Context, timeout int) error {
}
defer label.SetProcessLabel("")

if clh.config.SharedFS == config.VirtioFS {
clh.Logger().WithField("function", "StartVM").Info("Starting virtiofsd")
pid, err := clh.virtiofsd.Start(ctx, func() {
if clh.config.SharedFS == config.VirtioFS || clh.config.SharedFS == config.VirtioFSNydus {
clh.Logger().WithField("function", "StartVM").Info("Starting virtiofsDaemon")
pid, err := clh.virtiofsDaemon.Start(ctx, func() {
clh.StopVM(ctx, false)
})
if err != nil {
return err
}
clh.state.VirtiofsdPID = pid
clh.state.VirtiofsDaemonPid = pid
} else {
return errors.New("cloud-hypervisor only supports virtio based file sharing")
}

pid, err := clh.launchClh()
if err != nil {
if shutdownErr := clh.virtiofsd.Stop(ctx); shutdownErr != nil {
clh.Logger().WithError(shutdownErr).Warn("error shutting down Virtiofsd")
if shutdownErr := clh.virtiofsDaemon.Stop(ctx); shutdownErr != nil {
clh.Logger().WithError(shutdownErr).Warn("error shutting down VirtiofsDaemon")
}
return fmt.Errorf("failed to launch cloud-hypervisor: %q", err)
}
Expand Down Expand Up @@ -759,14 +782,14 @@ func (clh *cloudHypervisor) toGrpc(ctx context.Context) ([]byte, error) {
func (clh *cloudHypervisor) Save() (s hv.HypervisorState) {
s.Pid = clh.state.PID
s.Type = string(ClhHypervisor)
s.VirtiofsDaemonPid = clh.state.VirtiofsdPID
s.VirtiofsDaemonPid = clh.state.VirtiofsDaemonPid
s.APISocket = clh.state.apiSocket
return
}

func (clh *cloudHypervisor) Load(s hv.HypervisorState) {
clh.state.PID = s.Pid
clh.state.VirtiofsdPID = s.VirtiofsDaemonPid
clh.state.VirtiofsDaemonPid = s.VirtiofsDaemonPid
clh.state.apiSocket = s.APISocket
}

Expand All @@ -790,7 +813,7 @@ func (clh *cloudHypervisor) GetPids() []int {
}

func (clh *cloudHypervisor) GetVirtioFsPid() *int {
return &clh.state.VirtiofsdPID
return &clh.state.VirtiofsDaemonPid
}

func (clh *cloudHypervisor) AddDevice(ctx context.Context, devInfo interface{}, devType DeviceType) error {
Expand Down Expand Up @@ -872,13 +895,13 @@ func (clh *cloudHypervisor) terminate(ctx context.Context, waitOnly bool) (err e
return err
}

if clh.virtiofsd == nil {
return errors.New("virtiofsd config is nil, failed to stop it")
if clh.virtiofsDaemon == nil {
return errors.New("virtiofsDaemon config is nil, failed to stop it")
}

clh.Logger().Debug("stop virtiofsd")
if err = clh.virtiofsd.Stop(ctx); err != nil {
clh.Logger().WithError(err).Error("failed to stop virtiofsd")
clh.Logger().Debug("stop virtiofsDaemon")
if err = clh.virtiofsDaemon.Stop(ctx); err != nil {
clh.Logger().WithError(err).Error("failed to stop virtiofsDaemon")
}

return
Expand Down Expand Up @@ -1181,7 +1204,7 @@ func (clh *cloudHypervisor) addNet(e Endpoint) error {

// Add shared Volume using virtiofs
func (clh *cloudHypervisor) addVolume(volume types.Volume) error {
if clh.config.SharedFS != config.VirtioFS {
if clh.config.SharedFS != config.VirtioFS && clh.config.SharedFS != config.VirtioFSNydus {
return fmt.Errorf("shared fs method not supported %s", clh.config.SharedFS)
}

Expand Down
8 changes: 4 additions & 4 deletions src/runtime/virtcontainers/clh_test.go
View file
Expand Up @@ -296,7 +296,7 @@ func TestClhCreateVM(t *testing.T) {
assert.Exactly(clhConfig, clh.config)
}

func TestClooudHypervisorStartSandbox(t *testing.T) {
func TestCloudHypervisorStartSandbox(t *testing.T) {
assert := assert.New(t)
clhConfig, err := newClhConfig()
assert.NoError(err)
Expand All @@ -308,9 +308,9 @@ func TestClooudHypervisorStartSandbox(t *testing.T) {
clhConfig.RunStorePath = store.RunStoragePath()

clh := &cloudHypervisor{
config: clhConfig,
APIClient: &clhClientMock{},
virtiofsd: &virtiofsdMock{},
config: clhConfig,
APIClient: &clhClientMock{},
virtiofsDaemon: &virtiofsdMock{},
}

err = clh.StartVM(context.Background(), 10)
Expand Down
23 changes: 17 additions & 6 deletions src/runtime/virtcontainers/kata_agent.go
View file
Expand Up @@ -1272,13 +1272,24 @@ func (k *kataAgent) rollbackFailingContainerCreation(ctx context.Context, c *Con
}
}

func (k *kataAgent) buildContainerRootfsWithNydus(sandbox *Sandbox, c *Container, rootPathParent string) (*grpc.Storage, error) {
if sandbox.GetHypervisorType() != string(QemuHypervisor) {
// qemu is supported first, other hypervisors will next
// https://github.com/kata-containers/kata-containers/issues/2724
func getVirtiofsDaemonForNydus(sandbox *Sandbox) (VirtiofsDaemon, error) {
var virtiofsDaemon VirtiofsDaemon
switch sandbox.GetHypervisorType() {
case string(QemuHypervisor):
virtiofsDaemon = sandbox.hypervisor.(*qemu).virtiofsDaemon
case string(ClhHypervisor):
virtiofsDaemon = sandbox.hypervisor.(*cloudHypervisor).virtiofsDaemon
default:
return nil, errNydusdNotSupport
}
q, _ := sandbox.hypervisor.(*qemu)
return virtiofsDaemon, nil
}

func (k *kataAgent) buildContainerRootfsWithNydus(sandbox *Sandbox, c *Container, rootPathParent string) (*grpc.Storage, error) {
virtiofsDaemon, err := getVirtiofsDaemonForNydus(sandbox)
if err != nil {
return nil, err
}
extraOption, err := parseExtraOption(c.rootFs.Options)
if err != nil {
return nil, err
Expand All @@ -1290,7 +1301,7 @@ func (k *kataAgent) buildContainerRootfsWithNydus(sandbox *Sandbox, c *Container
}
k.Logger().Infof("nydus option: %v", extraOption)
// mount lowerdir to guest /run/kata-containers/shared/images/<cid>/lowerdir
if err := q.virtiofsDaemon.Mount(*mountOpt); err != nil {
if err := virtiofsDaemon.Mount(*mountOpt); err != nil {
return nil, err
}
rootfs := &grpc.Storage{}
Expand Down
10 changes: 4 additions & 6 deletions src/runtime/virtcontainers/mount.go
View file
Expand Up @@ -390,13 +390,11 @@ func bindUnmountContainerSnapshotDir(ctx context.Context, sharedDir, cID string)

func nydusContainerCleanup(ctx context.Context, sharedDir string, c *Container) error {
sandbox := c.sandbox
if sandbox.GetHypervisorType() != string(QemuHypervisor) {
// qemu is supported first, other hypervisors will next
// https://github.com/kata-containers/kata-containers/issues/2724
return errNydusdNotSupport
virtiofsDaemon, err := getVirtiofsDaemonForNydus(sandbox)
if err != nil {
return err
}
q, _ := sandbox.hypervisor.(*qemu)
if err := q.virtiofsDaemon.Umount(rafsMountPath(c.id)); err != nil {
if err := virtiofsDaemon.Umount(rafsMountPath(c.id)); err != nil {
return errors.Wrap(err, "umount rafs failed")
}
if err := bindUnmountContainerSnapshotDir(ctx, sharedDir, c.id); err != nil {
Expand Down

0 comments on commit 3175aad

Please sign in to comment.