qemu: guest VM vsock device support

factory/direct/direct.go

@@ -10,11 +10,12 @@ type directFactory struct {
 	config hypervisor.BootConfig
 }
 
-func New(cpu, mem int, kernel, initrd string) base.Factory {
+func New(cpu, mem int, kernel, initrd string, vsock bool) base.Factory {
 	b := hypervisor.BootConfig{
 		CPU:          cpu,
 		Memory:       mem,
 		HotAddCpuMem: true,
+		EnableVsock:  vsock,
 		Kernel:       kernel,
 		Initrd:       initrd,
 	}

factory/factory.go

@@ -6,6 +6,7 @@ package factory
 
 import (
 	"encoding/json"
+
 	"github.com/golang/glog"
 	"github.com/hyperhq/runv/factory/base"
 	"github.com/hyperhq/runv/factory/cache"
@@ -24,6 +25,7 @@ type Factory interface {
 type FactoryConfig struct {
 	Cache    int  `json:"cache"`
 	Template bool `json:"template"`
+	Vsock    bool `json:"vsock"`
 	Cpu      int  `json:"cpu"`
 	Memory   int  `json:"memory"`
 }
@@ -33,16 +35,16 @@ func NewFromConfigs(kernel, initrd string, configs []FactoryConfig) Factory {
 	for i, c := range configs {
 		var b base.Factory
 		if c.Template {
-			b = template.New(hypervisor.BaseDir+"/template", c.Cpu, c.Memory, kernel, initrd)
+			b = template.New(hypervisor.BaseDir+"/template", c.Cpu, c.Memory, kernel, initrd, c.Vsock)
 		} else {
-			b = direct.New(c.Cpu, c.Memory, kernel, initrd)
+			b = direct.New(c.Cpu, c.Memory, kernel, initrd, c.Vsock)
 		}
 		bases[i] = cache.New(c.Cache, b)
 	}
 
 	if len(bases) == 0 {
 		// skip GetVm from the base factory
-		return single.New(direct.New(1000000, 1000000, kernel, initrd))
+		return single.New(direct.New(1000000, 1000000, kernel, initrd, false))
 	} else if len(bases) == 1 {
 		return single.New(bases[0])
 	} else {
@@ -51,7 +53,7 @@ func NewFromConfigs(kernel, initrd string, configs []FactoryConfig) Factory {
 }
 
 // vmFactoryPolicy = [FactoryConfig,]*FactoryConfig
-// FactoryConfig   = {["cache":NUMBER,]["template":true|false,]"cpu":NUMBER,"memory":NUMBER}
+// FactoryConfig   = {["cache":NUMBER,]["template":true|false,]["vsock":true|false,]"cpu":NUMBER,"memory":NUMBER}
 func NewFromPolicy(kernel, initrd string, policy string) Factory {
 	var configs []FactoryConfig
 	jsonString := "[" + policy + "]"

factory/template/template.go

@@ -16,7 +16,7 @@ type templateFactory struct {
 	s *template.TemplateVmConfig
 }
 
-func New(templateRoot string, cpu, mem int, kernel, initrd string) base.Factory {
+func New(templateRoot string, cpu, mem int, kernel, initrd string, vsock bool) base.Factory {
 	var vmName string
 
 	for {
@@ -25,11 +25,11 @@ func New(templateRoot string, cpu, mem int, kernel, initrd string) base.Factory
 			break
 		}
 	}
-	s, err := template.CreateTemplateVM(templateRoot+"/"+vmName, vmName, cpu, mem, kernel, initrd)
+	s, err := template.CreateTemplateVM(templateRoot+"/"+vmName, vmName, cpu, mem, kernel, initrd, vsock)
 	if err != nil {
 		glog.Infof("failed to create template factory: %v", err)
 		glog.Infof("use direct factory instead")
-		return direct.New(cpu, mem, kernel, initrd)
+		return direct.New(cpu, mem, kernel, initrd, vsock)
 	}
 	return &templateFactory{s: s}
 }

hypervisor/constants.go

@@ -10,6 +10,9 @@ const (
 	DefaultInitrd   = "/var/lib/hyper/hyper-initrd.img"
 	DetachKeys      = "ctrl-p,ctrl-q"
 
+	HyperVsockCtlPort = 2718
+	HyperVsockMsgPort = 2719
+
 	// cpu/mem hotplug constants
 	DefaultMaxCpus = 8     // CONFIG_NR_CPUS hyperstart.git/build/kernel_config
 	DefaultMaxMem  = 32768 // size in MiB

hypervisor/context.go

@@ -44,6 +44,7 @@ type VmContext struct {
 	TtySockName     string
 	ConsoleSockName string
 	ShareDir        string
+	GuestCid        uint32 // guest vsock cid
 
 	pciAddr int //next available pci addr for pci hotplug
 	scsiId  int //next available scsi id for scsi hotplug
@@ -151,6 +152,7 @@ func (ctx *VmContext) reset() {
 
 	ctx.pciAddr = PciAddrFrom
 	ctx.scsiId = 0
+	ctx.GuestCid = 0
 	//do not reset attach id here, let it increase
 
 	ctx.userSpec = nil
@@ -169,7 +171,7 @@ func (ctx *VmContext) nextScsiId() int {
 	return id
 }
 
-func (ctx *VmContext) nextPciAddr() int {
+func (ctx *VmContext) NextPciAddr() int {
 	ctx.lock.Lock()
 	addr := ctx.pciAddr
 	ctx.pciAddr++

hypervisor/devicemap.go

@@ -293,7 +293,7 @@ func (ctx *VmContext) setVolumeInfo(info *VolumeInfo) {
 func (ctx *VmContext) allocateNetworks() {
 	for i := range ctx.progress.adding.networks {
 		name := fmt.Sprintf("eth%d", i)
-		addr := ctx.nextPciAddr()
+		addr := ctx.NextPciAddr()
 		if len(ctx.userSpec.Interfaces) > 0 {
 			go ctx.ConfigureInterface(i, addr, name, ctx.userSpec.Interfaces[i], ctx.Hub)
 		} else {

hypervisor/driver.go

@@ -2,10 +2,11 @@ package hypervisor
 
 import (
 	"errors"
+	"os"
+
 	"github.com/hyperhq/runv/hypervisor/network"
 	"github.com/hyperhq/runv/hypervisor/pod"
 	"github.com/hyperhq/runv/hypervisor/types"
-	"os"
 )
 
 type BootConfig struct {
@@ -14,6 +15,7 @@ type BootConfig struct {
 	HotAddCpuMem     bool
 	BootToBeTemplate bool
 	BootFromTemplate bool
+	EnableVsock      bool
 	MemoryPath       string
 	DevicesStatePath string
 	Kernel           string

hypervisor/events.go

@@ -33,7 +33,7 @@ type InitFailedEvent struct {
 }
 
 type InitConnectedEvent struct {
-	conn *net.UnixConn
+	conn net.Conn
 }
 
 type GetPodStatsCommand struct {

hypervisor/hypervisor.go

@@ -43,8 +43,8 @@ func VmLoop(vmId string, hub chan VmEvent, client chan *types.VmResponse, boot *
 	}
 
 	//launch routines
-	context.startSocks()
 	context.DCtx.Launch(context)
+	context.startSocks()
 
 	context.loop()
 }

hypervisor/init_comm.go

@@ -82,7 +82,7 @@ func NewVmMessage(m *hyperstartapi.DecodedMessage) []byte {
 	return msg
 }
 
-func ReadVmMessage(conn *net.UnixConn) (*hyperstartapi.DecodedMessage, error) {
+func ReadVmMessage(conn net.Conn) (*hyperstartapi.DecodedMessage, error) {
 	needRead := 8
 	length := 0
 	read := 0
@@ -121,7 +121,16 @@ func ReadVmMessage(conn *net.UnixConn) (*hyperstartapi.DecodedMessage, error) {
 }
 
 func waitInitReady(ctx *VmContext) {
-	conn, err := utils.UnixSocketConnect(ctx.HyperSockName)
+	var (
+		conn net.Conn
+		err  error
+	)
+
+	if ctx.GuestCid > 0 {
+		conn, err = utils.VmSocketConnect(ctx.GuestCid, HyperVsockCtlPort)
+	} else {
+		conn, err = utils.UnixSocketConnect(ctx.HyperSockName)
+	}
 	if err != nil {
 		glog.Error("Cannot connect to hyper socket ", err.Error())
 		ctx.Hub <- &InitFailedEvent{
@@ -133,15 +142,15 @@ func waitInitReady(ctx *VmContext) {
 	if ctx.Boot.BootFromTemplate {
 		glog.Info("boot from template")
 		ctx.PauseState = PauseStatePaused
-		ctx.Hub <- &InitConnectedEvent{conn: conn.(*net.UnixConn)}
-		go waitCmdToInit(ctx, conn.(*net.UnixConn))
+		ctx.Hub <- &InitConnectedEvent{conn: conn}
+		go waitCmdToInit(ctx, conn)
 		// TODO call getVMHyperstartAPIVersion(ctx) after unpaused
 		return
 	}
 
 	glog.Info("Wating for init messages...")
 
-	msg, err := ReadVmMessage(conn.(*net.UnixConn))
+	msg, err := ReadVmMessage(conn)
 	if err != nil {
 		glog.Error("read init message failed... ", err.Error())
 		ctx.Hub <- &InitFailedEvent{
@@ -150,8 +159,8 @@ func waitInitReady(ctx *VmContext) {
 		conn.Close()
 	} else if msg.Code == hyperstartapi.INIT_READY {
 		glog.Info("Get init ready message")
-		ctx.Hub <- &InitConnectedEvent{conn: conn.(*net.UnixConn)}
-		go waitCmdToInit(ctx, conn.(*net.UnixConn))
+		ctx.Hub <- &InitConnectedEvent{conn: conn}
+		go waitCmdToInit(ctx, conn)
 		if !ctx.Boot.BootToBeTemplate {
 			getVMHyperstartAPIVersion(ctx)
 		}
@@ -165,7 +174,16 @@ func waitInitReady(ctx *VmContext) {
 }
 
 func connectToInit(ctx *VmContext) {
-	conn, err := utils.UnixSocketConnect(ctx.HyperSockName)
+	var (
+		conn net.Conn
+		err  error
+	)
+
+	if ctx.GuestCid > 0 {
+		conn, err = utils.VmSocketConnect(ctx.GuestCid, HyperVsockCtlPort)
+	} else {
+		conn, err = utils.UnixSocketConnect(ctx.HyperSockName)
+	}
 	if err != nil {
 		glog.Error("Cannot re-connect to hyper socket ", err.Error())
 		ctx.Hub <- &InitFailedEvent{
@@ -174,7 +192,7 @@ func connectToInit(ctx *VmContext) {
 		return
 	}
 
-	go waitCmdToInit(ctx, conn.(*net.UnixConn))
+	go waitCmdToInit(ctx, conn)
 	getVMHyperstartAPIVersion(ctx)
 }
 
@@ -200,7 +218,7 @@ func getVMHyperstartAPIVersion(ctx *VmContext) error {
 	return nil
 }
 
-func waitCmdToInit(ctx *VmContext, init *net.UnixConn) {
+func waitCmdToInit(ctx *VmContext, init net.Conn) {
 	looping := true
 	cmds := []*hyperstartCmd{}
 
@@ -349,7 +367,7 @@ func waitCmdToInit(ctx *VmContext, init *net.UnixConn) {
 	}
 }
 
-func waitInitAck(ctx *VmContext, init *net.UnixConn) {
+func waitInitAck(ctx *VmContext, init net.Conn) {
 	for {
 		res, err := ReadVmMessage(init)
 		if err != nil {

hypervisor/lazy.go

@@ -100,7 +100,7 @@ func (ctx *VmContext) lazyPrepareDevice(cmd *RunPodCommand) bool {
 func (ctx *VmContext) lazyAllocateNetworks() error {
 	for i := range ctx.progress.adding.networks {
 		name := fmt.Sprintf("eth%d", i)
-		addr := ctx.nextPciAddr()
+		addr := ctx.NextPciAddr()
 		nic, err := ctx.allocateInterface(i, addr, name)
 		if err != nil {
 			return err