diff --git a/cmd/vm-control/createVm.go b/cmd/vm-control/createVm.go index d9793ad7..ca359c8e 100644 --- a/cmd/vm-control/createVm.go +++ b/cmd/vm-control/createVm.go @@ -96,13 +96,14 @@ func createVmOnHypervisor(hypervisor string, logger log.DebugLogger) error { request := hyper_proto.CreateVmRequest{ DhcpTimeout: *dhcpTimeout, VmInfo: hyper_proto.VmInfo{ - Hostname: *vmHostname, - MemoryInMiB: uint64(memory >> 20), - MilliCPUs: *milliCPUs, - OwnerGroups: ownerGroups, - OwnerUsers: ownerUsers, - Tags: vmTags, - SubnetId: *subnetId, + Hostname: *vmHostname, + MemoryInMiB: uint64(memory >> 20), + MilliCPUs: *milliCPUs, + OwnerGroups: ownerGroups, + OwnerUsers: ownerUsers, + Tags: vmTags, + SecondarySubnetIDs: secondarySubnetIDs, + SubnetId: *subnetId, }, MinimumFreeBytes: uint64(minFreeBytes), RoundupPower: *roundupPower, diff --git a/cmd/vm-control/main.go b/cmd/vm-control/main.go index fa9f25c5..706b12b4 100644 --- a/cmd/vm-control/main.go +++ b/cmd/vm-control/main.go @@ -50,6 +50,7 @@ var ( probePortNum = flag.Uint("probePortNum", 0, "Port number on VM to probe") probeTimeout = flag.Duration("probeTimeout", time.Minute*5, "Time to wait before timing out on probing VM port") + secondarySubnetIDs flagutil.StringList secondaryVolumeSizes flagutil.StringList subnetId = flag.String("subnetId", "", "Subnet ID to launch VM in") @@ -77,6 +78,7 @@ func init() { "minimum number of free bytes in root volume") flag.Var(&ownerGroups, "ownerGroups", "Groups who own the VM") flag.Var(&ownerUsers, "ownerUsers", "Extra users who own the VM") + flag.Var(&secondarySubnetIDs, "secondarySubnetIDs", "Secondary Subnet IDs") flag.Var(&secondaryVolumeSizes, "secondaryVolumeSizes", "Sizes for secondary volumes") flag.Var(&vmTags, "vmTags", "Tags to apply to VM") diff --git a/hypervisor/manager/vm.go b/hypervisor/manager/vm.go index f2739204..a22c7ebe 100644 --- a/hypervisor/manager/vm.go +++ b/hypervisor/manager/vm.go @@ -169,19 +169,40 @@ func (m *Manager) acknowledgeVm(ipAddr net.IP, func (m *Manager) allocateVm(req proto.CreateVmRequest, authInfo *srpc.AuthInformation) (*vmInfoType, error) { + subnetIDs := map[string]struct{}{req.SubnetId: struct{}{}} + for _, subnetId := range req.SecondarySubnetIDs { + if subnetId == "" { + return nil, + errors.New("cannot give unspecified secondary subnet ID") + } + if _, ok := subnetIDs[subnetId]; ok { + return nil, + fmt.Errorf("subnet: %s specified multiple times", subnetId) + } + subnetIDs[subnetId] = struct{}{} + } address, subnetId, err := m.getFreeAddress(req.SubnetId, authInfo) if err != nil { return nil, err } - freeAddress := true + addressesToFree := []proto.Address{address} defer func() { - if freeAddress { + for _, address := range addressesToFree { err := m.releaseAddressInPool(address) if err != nil { m.Logger.Println(err) } } }() + var secondaryAddresses []proto.Address + for _, subnetId := range req.SecondarySubnetIDs { + secondaryAddress, _, err := m.getFreeAddress(subnetId, authInfo) + if err != nil { + return nil, err + } + secondaryAddresses = append(secondaryAddresses, secondaryAddress) + addressesToFree = append(addressesToFree, secondaryAddress) + } m.mutex.Lock() defer m.mutex.Unlock() if err := m.checkSufficientCPUWithLock(req.MilliCPUs); err != nil { @@ -198,17 +219,19 @@ func (m *Manager) allocateVm(req proto.CreateVmRequest, } vm := &vmInfoType{ VmInfo: proto.VmInfo{ - Address: address, - Hostname: req.Hostname, - ImageName: req.ImageName, - ImageURL: req.ImageURL, - MemoryInMiB: req.MemoryInMiB, - MilliCPUs: req.MilliCPUs, - OwnerGroups: req.OwnerGroups, - SpreadVolumes: req.SpreadVolumes, - State: proto.StateStarting, - Tags: req.Tags, - SubnetId: subnetId, + Address: address, + Hostname: req.Hostname, + ImageName: req.ImageName, + ImageURL: req.ImageURL, + MemoryInMiB: req.MemoryInMiB, + MilliCPUs: req.MilliCPUs, + OwnerGroups: req.OwnerGroups, + SpreadVolumes: req.SpreadVolumes, + SecondaryAddresses: secondaryAddresses, + SecondarySubnetIDs: req.SecondarySubnetIDs, + State: proto.StateStarting, + SubnetId: subnetId, + Tags: req.Tags, }, manager: m, dirname: path.Join(m.StateDir, "VMs", ipAddress), @@ -217,7 +240,7 @@ func (m *Manager) allocateVm(req proto.CreateVmRequest, metadataChannels: make(map[chan<- string]struct{}), } m.vms[ipAddress] = vm - freeAddress = false + addressesToFree = nil return vm, nil } @@ -1060,6 +1083,9 @@ func (m *Manager) migrateVmChecks(vmInfo proto.VmInfo) error { default: return fmt.Errorf("VM state: %s is not stopped/running", vmInfo.State) } + if len(vmInfo.SecondaryAddresses) > 0 { + return errors.New("cannot migrate VM with multiple interfaces") + } m.mutex.RLock() defer m.mutex.RUnlock() if err := m.checkSufficientCPUWithLock(vmInfo.MilliCPUs); err != nil { @@ -1681,6 +1707,11 @@ func (vm *vmInfoType) cleanup() { if err := m.releaseAddressInPoolWithLock(vm.Address); err != nil { m.Logger.Println(err) } + for _, address := range vm.SecondaryAddresses { + if err := m.releaseAddressInPoolWithLock(address); err != nil { + m.Logger.Println(err) + } + } } os.RemoveAll(vm.dirname) for _, volume := range vm.VolumeLocations { @@ -1714,12 +1745,21 @@ func (vm *vmInfoType) delete() { close(ch) } vm.manager.DhcpServer.RemoveLease(vm.Address.IpAddress) + for _, address := range vm.SecondaryAddresses { + vm.manager.DhcpServer.RemoveLease(address.IpAddress) + } vm.manager.mutex.Lock() delete(vm.manager.vms, vm.ipAddress) vm.manager.sendVmInfo(vm.ipAddress, nil) var err error if !vm.Uncommitted { err = vm.manager.releaseAddressInPoolWithLock(vm.Address) + for _, address := range vm.SecondaryAddresses { + err := vm.manager.releaseAddressInPoolWithLock(address) + if err != nil { + vm.manager.Logger.Println(err) + } + } } vm.manager.mutex.Unlock() if err != nil { @@ -1899,6 +1939,9 @@ func (vm *vmInfoType) startManaging(dhcpTimeout time.Duration, return false, nil } vm.manager.DhcpServer.AddLease(vm.Address, vm.Hostname) + for _, address := range vm.SecondaryAddresses { + vm.manager.DhcpServer.AddLease(address, vm.Hostname) + } monitorSock, err := net.Dial("unix", vm.monitorSockname) if err != nil { vm.logger.Debugf(0, "error connecting to: %s: %s\n", @@ -1949,6 +1992,47 @@ func (vm *vmInfoType) startManaging(dhcpTimeout time.Duration, return false, nil } +func (vm *vmInfoType) getBridgesAndOptions(haveManagerLock bool) ( + []string, []string, error) { + if !haveManagerLock { + vm.manager.mutex.RLock() + defer vm.manager.mutex.RUnlock() + } + addresses := make([]proto.Address, 1, len(vm.SecondarySubnetIDs)+1) + addresses[0] = vm.Address + subnetIDs := make([]string, 1, len(vm.SecondarySubnetIDs)+1) + subnetIDs[0] = vm.SubnetId + for index, subnetId := range vm.SecondarySubnetIDs { + addresses = append(addresses, vm.SecondaryAddresses[index]) + subnetIDs = append(subnetIDs, subnetId) + } + var bridges, options []string + for index, address := range addresses { + subnet, ok := vm.manager.subnets[subnetIDs[index]] + if !ok { + return nil, nil, + fmt.Errorf("subnet: %s not found", subnetIDs[index]) + } + var vlanOption string + if bridge, ok := vm.manager.VlanIdToBridge[subnet.VlanId]; !ok { + if bridge, ok = vm.manager.VlanIdToBridge[0]; !ok { + return nil, nil, fmt.Errorf("no usable bridge") + } else { + bridges = append(bridges, bridge) + vlanOption = fmt.Sprintf(",vlan=%d", subnet.VlanId) + } + } else { + bridges = append(bridges, bridge) + } + options = append(options, + "-netdev", fmt.Sprintf("tap,id=net%d,fd=%d%s", + index, index+3, vlanOption), + "-device", fmt.Sprintf("virtio-net-pci,netdev=net%d,mac=%s", + index, address.MacAddress)) + } + return bridges, options, nil +} + func (vm *vmInfoType) startVm(haveManagerLock bool) error { if err := checkAvailableMemory(vm.MemoryInMiB); err != nil { return err @@ -1961,43 +2045,31 @@ func (vm *vmInfoType) startVm(haveManagerLock bool) error { nCpus++ } bootlogFilename := path.Join(vm.dirname, "bootlog") - if !haveManagerLock { - vm.manager.mutex.RLock() - } - subnet, ok := vm.manager.subnets[vm.SubnetId] - if !haveManagerLock { - vm.manager.mutex.RUnlock() - } - if !ok { - return fmt.Errorf("subnet: %s not found", vm.SubnetId) + bridges, netOptions, err := vm.getBridgesAndOptions(haveManagerLock) + if err != nil { + return err } - var bridge string - var vlanOption string - if bridge, ok = vm.manager.VlanIdToBridge[subnet.VlanId]; !ok { - if bridge, ok = vm.manager.VlanIdToBridge[0]; !ok { - return fmt.Errorf("no usable bridge") - } else { - vlanOption = fmt.Sprintf(",vlan=%d", subnet.VlanId) + var tapFiles []*os.File + for _, bridge := range bridges { + tapFile, err := createTapDevice(bridge) + if err != nil { + return fmt.Errorf("error creating tap device: %s", err) } + defer tapFile.Close() + tapFiles = append(tapFiles, tapFile) } - tapFile, err := createTapDevice(bridge) - if err != nil { - return fmt.Errorf("error creating tap device: %s", err) - } - defer tapFile.Close() cmd := exec.Command("qemu-system-x86_64", "-machine", "pc,accel=kvm", "-cpu", "host", // Allow the VM to take full advantage of host CPU. "-nodefaults", "-name", vm.ipAddress, "-m", fmt.Sprintf("%dM", vm.MemoryInMiB), "-smp", fmt.Sprintf("cpus=%d", nCpus), - "-net", "nic,model=virtio,macaddr="+vm.Address.MacAddress, - "-net", "tap,fd=3"+vlanOption, "-serial", "file:"+bootlogFilename, "-chroot", "/tmp", "-runas", vm.manager.Username, "-qmp", "unix:"+vm.monitorSockname+",server,nowait", "-daemonize") + cmd.Args = append(cmd.Args, netOptions...) if vm.manager.ShowVgaConsole { cmd.Args = append(cmd.Args, "-vga", "std") } else { @@ -2013,7 +2085,7 @@ func (vm *vmInfoType) startVm(haveManagerLock bool) error { ",if=virtio") } os.Remove(bootlogFilename) - cmd.ExtraFiles = []*os.File{tapFile} // fd=3 for QEMU. + cmd.ExtraFiles = tapFiles // Start at fd=3 for QEMU. if output, err := cmd.CombinedOutput(); err != nil { return fmt.Errorf("error starting QEMU: %s: %s", err, output) } else if len(output) > 0 { diff --git a/proto/hypervisor/messages.go b/proto/hypervisor/messages.go index 08dc71e6..db279016 100644 --- a/proto/hypervisor/messages.go +++ b/proto/hypervisor/messages.go @@ -411,20 +411,22 @@ type UpdateSubnetsResponse struct { } type VmInfo struct { - Address Address - Hostname string `json:",omitempty"` - ImageName string `json:",omitempty"` - ImageURL string `json:",omitempty"` - MemoryInMiB uint64 - MilliCPUs uint - OwnerGroups []string `json:",omitempty"` - OwnerUsers []string `json:",omitempty"` - SpreadVolumes bool `json:",omitempty"` - State State - Tags tags.Tags `json:",omitempty"` - SubnetId string `json:",omitempty"` - Uncommitted bool `json:",omitempty"` - Volumes []Volume `json:",omitempty"` + Address Address + Hostname string `json:",omitempty"` + ImageName string `json:",omitempty"` + ImageURL string `json:",omitempty"` + MemoryInMiB uint64 + MilliCPUs uint + OwnerGroups []string `json:",omitempty"` + OwnerUsers []string `json:",omitempty"` + SpreadVolumes bool `json:",omitempty"` + State State + Tags tags.Tags `json:",omitempty"` + SecondaryAddresses []Address `json:",omitempty"` + SecondarySubnetIDs []string `json:",omitempty"` + SubnetId string `json:",omitempty"` + Uncommitted bool `json:",omitempty"` + Volumes []Volume `json:",omitempty"` } type Volume struct { diff --git a/proto/hypervisor/methods.go b/proto/hypervisor/methods.go index e59edcc2..da1c0acf 100644 --- a/proto/hypervisor/methods.go +++ b/proto/hypervisor/methods.go @@ -190,6 +190,17 @@ func (left *VmInfo) Equal(right *VmInfo) bool { if !left.Tags.Equal(right.Tags) { return false } + if len(left.SecondaryAddresses) != len(right.SecondaryAddresses) { + return false + } + for index, leftAddress := range left.SecondaryAddresses { + if !leftAddress.Equal(&right.SecondaryAddresses[index]) { + return false + } + } + if !stringSlicesEqual(left.SecondarySubnetIDs, right.SecondarySubnetIDs) { + return false + } if left.SubnetId != right.SubnetId { return false }