Skip to content

Commit

Permalink
s390x: add virtio-blk-ccw
Browse files Browse the repository at this point in the history 
Add virtio-blk-ccw support

Fixes: kata-containers#1153

Signed-off-by: Alice Frosi <afrosi@de.ibm.com>
Reviewed-by: Jan Schintag <jan.schintag@de.ibm.com>
  • Loading branch information
@jschintag
jschintag committed Aug 29, 2019
1 parent 47bfc06 commit 2d7de41
Show file tree
Hide file tree
Showing 6 changed files with 83 additions and 60 deletions.
2 changes: 1 addition & 1 deletion pkg/katautils/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -340,7 +340,7 @@ func (h hypervisor) defaultBridges() uint32 {
}

func (h hypervisor) blockDeviceDriver() (string, error) {
supportedBlockDrivers := []string{config.VirtioSCSI, config.VirtioBlock, config.VirtioMmio, config.Nvdimm}
supportedBlockDrivers := []string{config.VirtioSCSI, config.VirtioBlock, config.VirtioMmio, config.Nvdimm, config.VirtioBlockCCW}

if h.BlockDeviceDriver == "" {
return defaultBlockDeviceDriver, nil
Expand Down
6 changes: 6 additions & 0 deletions virtcontainers/device/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,9 @@ const (
// VirtioBlock means use virtio-blk for hotplugging drives
VirtioBlock = "virtio-blk"

// VirtioBlockCCW means use virtio-blk for hotplugging drives
VirtioBlockCCW = "virtio-blk-ccw"

// VirtioSCSI means use virtio-scsi for hotplugging drives
VirtioSCSI = "virtio-scsi"

Expand Down Expand Up @@ -138,6 +141,9 @@ type BlockDrive struct {

// VirtPath at which the device appears inside the VM, outside of the container mount namespace
VirtPath string

// DevNo identifies the css bus id for virtio-blk-ccw
DevNo string
}

// VFIODeviceType indicates VFIO device type
Expand Down
4 changes: 4 additions & 0 deletions virtcontainers/device/drivers/block.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,8 @@ func (device *BlockDevice) Attach(devReceiver api.DeviceReceiver) (err error) {
switch customOptions["block-driver"] {
case "virtio-blk":
globalIdx = index
case "virtio-blk-ccw":
globalIdx = index
case "virtio-mmio":
//With firecracker the rootfs for the VM itself
//sits at /dev/vda and consumes the first index.
Expand Down Expand Up @@ -164,6 +166,7 @@ func (device *BlockDevice) Save() persistapi.DeviceState {
SCSIAddr: drive.SCSIAddr,
NvdimmID: drive.NvdimmID,
VirtPath: drive.VirtPath,
DevNo: drive.DevNo,
}
}
return ds
Expand All @@ -188,6 +191,7 @@ func (device *BlockDevice) Load(ds persistapi.DeviceState) {
SCSIAddr: bd.SCSIAddr,
NvdimmID: bd.NvdimmID,
VirtPath: bd.VirtPath,
DevNo: bd.DevNo,
}
}

Expand Down
4 changes: 4 additions & 0 deletions virtcontainers/device/manager/manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ const (
VirtioMmio string = "virtio-mmio"
// VirtioBlock indicates block driver is virtio-blk based
VirtioBlock string = "virtio-blk"
// VirtioBlockCCW indicates block driver is virtio-blk-ccw based
VirtioBlockCCW string = "virtio-blk-ccw"
// VirtioSCSI indicates block driver is virtio-scsi based
VirtioSCSI string = "virtio-scsi"
// Nvdimm indicates block driver is nvdimm based
Expand Down Expand Up @@ -66,6 +68,8 @@ func NewDeviceManager(blockDriver string, devices []api.Device) api.DeviceManage
dm.blockDriver = VirtioBlock
} else if blockDriver == Nvdimm {
dm.blockDriver = Nvdimm
} else if blockDriver == VirtioBlockCCW {
dm.blockDriver = VirtioBlockCCW
} else {
dm.blockDriver = VirtioSCSI
}
Expand Down
44 changes: 32 additions & 12 deletions virtcontainers/kata_agent.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ var (
kata9pDevType = "9p"
kataMmioBlkDevType = "mmioblk"
kataBlkDevType = "blk"
kataBlkCCWDevType = "blk-ccw"
kataSCSIDevType = "scsi"
kataNvdimmDevType = "nvdimm"
kataVirtioFSDevType = "virtio-fs"
Expand Down Expand Up @@ -1106,6 +1107,9 @@ func (k *kataAgent) appendDevices(deviceList []*grpc.Device, c *Container) []*gr
kataDevice.Type = kataMmioBlkDevType
kataDevice.Id = d.VirtPath
kataDevice.VmPath = d.VirtPath
case config.VirtioBlockCCW:
kataDevice.Type = kataBlkCCWDevType
kataDevice.Id = d.DevNo
case config.VirtioBlock:
kataDevice.Type = kataBlkDevType
kataDevice.Id = d.PCIAddr
Expand Down Expand Up @@ -1160,22 +1164,29 @@ func (k *kataAgent) buildContainerRootfs(sandbox *Sandbox, c *Container, rootPat
k.Logger().Error("malformed block drive")
return nil, fmt.Errorf("malformed block drive")
}

if sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioMmio {
switch {
case sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioMmio:
rootfs.Driver = kataMmioBlkDevType
rootfs.Source = blockDrive.VirtPath
} else if sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlock {
case sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlockCCW:
rootfs.Driver = kataBlkCCWDevType
rootfs.Source = blockDrive.DevNo
case sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlock:
rootfs.Driver = kataBlkDevType
if blockDrive.PCIAddr == "" {
rootfs.Source = blockDrive.VirtPath
} else {
rootfs.Source = blockDrive.PCIAddr
}

} else {
case sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioSCSI:

rootfs.Driver = kataSCSIDevType
rootfs.Source = blockDrive.SCSIAddr
default:
return nil, fmt.Errorf("Unknown block device driver: %s", sandbox.config.HypervisorConfig.BlockDeviceDriver)
}

rootfs.MountPoint = rootPathParent
rootfs.Fstype = c.state.Fstype

Expand Down Expand Up @@ -1281,7 +1292,10 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process,
// Note this call modifies the list of container devices to make sure
// all hotplugged devices are unplugged, so this needs be done
// after devices passed with --device are handled.
volumeStorages := k.handleBlockVolumes(c)
volumeStorages, err := k.handleBlockVolumes(c)
if err != nil {
return nil, err
}
if err := k.replaceOCIMountsForStorages(ociSpec, volumeStorages); err != nil {
return nil, err
}
Expand Down Expand Up @@ -1400,7 +1414,7 @@ func (k *kataAgent) handleLocalStorage(mounts []specs.Mount, sandboxID string, r

// handleBlockVolumes handles volumes that are block devices files
// by passing the block devices as Storage to the agent.
func (k *kataAgent) handleBlockVolumes(c *Container) []*grpc.Storage {
func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) {

var volumeStorages []*grpc.Storage

Expand All @@ -1418,7 +1432,7 @@ func (k *kataAgent) handleBlockVolumes(c *Container) []*grpc.Storage {
if !c.sandbox.supportNewStore() {
if err := c.storeDevices(); err != nil {
k.Logger().WithField("device", id).WithError(err).Error("store device failed")
return nil
return nil, err
}
}

Expand All @@ -1427,22 +1441,28 @@ func (k *kataAgent) handleBlockVolumes(c *Container) []*grpc.Storage {
device := c.sandbox.devManager.GetDeviceByID(id)
if device == nil {
k.Logger().WithField("device", id).Error("failed to find device by id")
return nil
return nil, fmt.Errorf("Failed to find device by id (id=%s)", id)
}
blockDrive, ok := device.GetDeviceInfo().(*config.BlockDrive)
if !ok || blockDrive == nil {
k.Logger().Error("malformed block drive")
continue
}
if c.sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlock {
switch {
case c.sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlockCCW:
vol.Driver = kataBlkCCWDevType
vol.Source = blockDrive.DevNo
case c.sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlock:
vol.Driver = kataBlkDevType
vol.Source = blockDrive.PCIAddr
} else if c.sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioMmio {
case c.sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioMmio:
vol.Driver = kataMmioBlkDevType
vol.Source = blockDrive.VirtPath
} else {
case c.sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioSCSI:
vol.Driver = kataSCSIDevType
vol.Source = blockDrive.SCSIAddr
default:
return nil, fmt.Errorf("Unknown block device driver: %s", c.sandbox.config.HypervisorConfig.BlockDeviceDriver)
}

vol.MountPoint = m.Destination
Expand All @@ -1452,7 +1472,7 @@ func (k *kataAgent) handleBlockVolumes(c *Container) []*grpc.Storage {
volumeStorages = append(volumeStorages, vol)
}

return volumeStorages
return volumeStorages, nil
}

// handlePidNamespace checks if Pid namespace for a container needs to be shared with its sandbox
Expand Down
83 changes: 36 additions & 47 deletions virtcontainers/qemu.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -986,38 +986,6 @@ func (q *qemu) qmpShutdown() {
}
}

func (q *qemu) addDeviceToBridge(ID string) (string, types.Bridge, error) {
var err error
var addr uint32

if len(q.state.Bridges) == 0 {
return "", types.Bridge{}, errors.New("failed to get available address from bridges")
}

// looking for an empty address in the bridges
for _, b := range q.state.Bridges {
addr, err = b.AddDevice(ID)
if err == nil {
return fmt.Sprintf("%02x", addr), b, nil
}
}

return "", types.Bridge{}, fmt.Errorf("no more bridge slots available")
}

func (q *qemu) removeDeviceFromBridge(ID string) error {
var err error
for _, b := range q.state.Bridges {
err = b.RemoveDevice(ID)
if err == nil {
// device was removed correctly
return nil
}
}

return err
}

func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, devID string) (err error) {
if q.config.BlockDeviceDriver == config.Nvdimm {
var blocksize int64
Expand Down Expand Up @@ -1052,26 +1020,43 @@ func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, dev
}
}()

if q.config.BlockDeviceDriver == config.VirtioBlock {
switch {
case q.config.BlockDeviceDriver == config.VirtioBlockCCW:
driver := "virtio-blk-ccw"

addr, bridge, err := q.arch.addDeviceToBridge(drive.ID, types.CCW)
if err != nil {
return err
}
var devNoHotplug string
devNoHotplug, err = bridge.AddressFormatCCW(addr)
if err != nil {
return err
}
drive.DevNo, err = bridge.AddressFormatCCWForVirtServer(addr)
if err != nil {
return err
}
if err = q.qmpMonitorCh.qmp.ExecuteDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, devNoHotplug, "", true, false); err != nil {
return err
}
case q.config.BlockDeviceDriver == config.VirtioBlock:
driver := "virtio-blk-pci"
addr, bridge, err := q.addDeviceToBridge(drive.ID)
addr, bridge, err := q.arch.addDeviceToBridge(drive.ID, types.PCI)
if err != nil {
return err
}

defer func() {
if err != nil {
q.removeDeviceFromBridge(drive.ID)
q.arch.removeDeviceFromBridge(drive.ID)
}
}()

// PCI address is in the format bridge-addr/device-addr eg. "03/02"
drive.PCIAddr = fmt.Sprintf("%02x", bridge.Addr) + "/" + addr

if err = q.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, addr, bridge.ID, romFile, 0, true, defaultDisableModern); err != nil {
return err
}
} else {
case q.config.BlockDeviceDriver == config.VirtioSCSI:
driver := "scsi-hd"

// Bus exposed by the SCSI Controller
Expand All @@ -1086,6 +1071,8 @@ func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, dev
if err = q.qmpMonitorCh.qmp.ExecuteSCSIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, bus, romFile, scsiID, lun, true, defaultDisableModern); err != nil {
return err
}
default:
return fmt.Errorf("Block device %s not recognized", q.config.BlockDeviceDriver)
}

return nil
Expand All @@ -1103,7 +1090,7 @@ func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error
err = q.hotplugAddBlockDevice(drive, op, devID)
} else {
if q.config.BlockDeviceDriver == config.VirtioBlock {
if err := q.removeDeviceFromBridge(drive.ID); err != nil {
if err := q.arch.removeDeviceFromBridge(drive.ID); err != nil {
return err
}
}
Expand Down Expand Up @@ -1143,14 +1130,14 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err erro
}
}

addr, bridge, err := q.addDeviceToBridge(devID)
addr, bridge, err := q.arch.addDeviceToBridge(devID, types.PCI)
if err != nil {
return err
}

defer func() {
if err != nil {
q.removeDeviceFromBridge(devID)
q.arch.removeDeviceFromBridge(devID)
}
}()

Expand All @@ -1164,7 +1151,7 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err erro
}
} else {
if !q.state.HotplugVFIOOnRootBus {
if err := q.removeDeviceFromBridge(devID); err != nil {
if err := q.arch.removeDeviceFromBridge(devID); err != nil {
return err
}
}
Expand Down Expand Up @@ -1230,14 +1217,14 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) {
}
}()

addr, bridge, err := q.addDeviceToBridge(tap.ID)
addr, bridge, err := q.arch.addDeviceToBridge(tap.ID, types.PCI)
if err != nil {
return err
}

defer func() {
if err != nil {
q.removeDeviceFromBridge(tap.ID)
q.arch.removeDeviceFromBridge(tap.ID)
}
}()

Expand All @@ -1250,12 +1237,14 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) {
return err
}
if machine.Type == QemuCCWVirtio {
return q.qmpMonitorCh.qmp.ExecuteNetCCWDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), bridge.ID, int(q.config.NumVCPUs))
devNoHotplug := fmt.Sprintf("fe.%x.%x", bridge.Addr, addr)
return q.qmpMonitorCh.qmp.ExecuteNetCCWDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), devNoHotplug, int(q.config.NumVCPUs))
}
return q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), addr, bridge.ID, romFile, int(q.config.NumVCPUs), defaultDisableModern)

}

if err := q.removeDeviceFromBridge(tap.ID); err != nil {
if err := q.arch.removeDeviceFromBridge(tap.ID); err != nil {
return err
}

Expand Down

0 comments on commit 2d7de41

Please sign in to comment.