From 40a2a3df2854346562aece4a4e9f7ac969c7dbf5 Mon Sep 17 00:00:00 2001 From: Richard Gooch Date: Mon, 10 Dec 2018 08:28:02 -0800 Subject: [PATCH 01/13] Hypervisor: support migrating VMs with multiple network interfaces. --- hypervisor/manager/vm.go | 42 ++++++++++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/hypervisor/manager/vm.go b/hypervisor/manager/vm.go index a22c7ebe..2ddbc4fb 100644 --- a/hypervisor/manager/vm.go +++ b/hypervisor/manager/vm.go @@ -1056,6 +1056,11 @@ func (m *Manager) migrateVm(conn *srpc.Conn, decoder srpc.Decoder, if err := m.registerAddress(vm.Address); err != nil { return err } + for _, address := range vm.SecondaryAddresses { + if err := m.registerAddress(address); err != nil { + return err + } + } vm.doNotWriteOrSend = false vm.Uncommitted = false vm.writeAndSendInfo() @@ -1083,11 +1088,18 @@ func (m *Manager) migrateVmChecks(vmInfo proto.VmInfo) error { default: return fmt.Errorf("VM state: %s is not stopped/running", vmInfo.State) } - if len(vmInfo.SecondaryAddresses) > 0 { - return errors.New("cannot migrate VM with multiple interfaces") - } m.mutex.RLock() defer m.mutex.RUnlock() + for index, address := range vmInfo.SecondaryAddresses { + subnetId := m.getMatchingSubnet(address.IpAddress) + if subnetId == "" { + return fmt.Errorf("no matching subnet for: %s\n", address.IpAddress) + } + if subnetId != vmInfo.SecondarySubnetIDs[index] { + return fmt.Errorf("subnet ID changing from: %s to: %s", + vmInfo.SecondarySubnetIDs[index], subnetId) + } + } if err := m.checkSufficientCPUWithLock(vmInfo.MilliCPUs); err != nil { return err } @@ -1247,8 +1259,8 @@ func (m *Manager) prepareVmForMigration(ipAddr net.IP, if vm.State != proto.StateStopped { return errors.New("VM is not stopped") } - // Block reallocation of address until VM is destroyed, then release - // claim on address. + // Block reallocation of addresses until VM is destroyed, then release + // claims on addresses. vm.Uncommitted = true vm.setState(proto.StateMigrating) if err := m.unregisterAddress(vm.Address); err != nil { @@ -1256,15 +1268,33 @@ func (m *Manager) prepareVmForMigration(ipAddr net.IP, vm.setState(proto.StateStopped) return err } + for _, address := range vm.SecondaryAddresses { + if err := m.unregisterAddress(address); err != nil { + vm.logger.Printf("error unregistering address: %s\n", + address.IpAddress) + vm.Uncommitted = false + vm.setState(proto.StateStopped) + return err + } + } } else { if vm.State != proto.StateMigrating { return errors.New("VM is not migrating") } - // Reclaim address and then allow reallocation if VM is later destroyed. + // Reclaim addresses and then allow reallocation if VM is later + // destroyed. if err := m.registerAddress(vm.Address); err != nil { vm.setState(proto.StateStopped) return err } + for _, address := range vm.SecondaryAddresses { + if err := m.registerAddress(address); err != nil { + vm.logger.Printf("error registering address: %s\n", + address.IpAddress) + vm.setState(proto.StateStopped) + return err + } + } vm.Uncommitted = false vm.setState(proto.StateStopped) } From 4ee393f31900bba6a3687d92ce4228d611a1fe35 Mon Sep 17 00:00:00 2001 From: Richard Gooch Date: Tue, 11 Dec 2018 08:34:00 -0800 Subject: [PATCH 02/13] Hypervisor: support selection of IP address(es) if available. --- hypervisor/manager/addressPool.go | 21 ++++++++++++++++++--- hypervisor/manager/vm.go | 12 +++++++++--- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/hypervisor/manager/addressPool.go b/hypervisor/manager/addressPool.go index d1942ac3..a350972f 100644 --- a/hypervisor/manager/addressPool.go +++ b/hypervisor/manager/addressPool.go @@ -12,6 +12,13 @@ import ( proto "github.com/Symantec/Dominator/proto/hypervisor" ) +func ipIsUnspecified(ipAddr net.IP) bool { + if len(ipAddr) < 1 { + return true + } + return ipAddr.IsUnspecified() +} + func (m *Manager) addAddressesToPool(addresses []proto.Address) error { for index := range addresses { addresses[index].Shrink() @@ -90,7 +97,7 @@ func (m *Manager) loadAddressPool() error { return nil } -func (m *Manager) getFreeAddress(subnetId string, +func (m *Manager) getFreeAddress(ipAddr net.IP, subnetId string, authInfo *srpc.AuthInformation) (proto.Address, string, error) { m.mutex.Lock() defer m.mutex.Unlock() @@ -104,14 +111,22 @@ func (m *Manager) getFreeAddress(subnetId string, subnetAddr := subnet.IpGateway.Mask(subnetMask) foundPos := -1 for index, address := range m.addressPool.Free { + if !ipIsUnspecified(ipAddr) && !ipAddr.Equal(address.IpAddress) { + continue + } if address.IpAddress.Mask(subnetMask).Equal(subnetAddr) { foundPos = index break } } if foundPos < 0 { - return proto.Address{}, "", - fmt.Errorf("no free address in subnet: %s", subnetId) + if ipIsUnspecified(ipAddr) { + return proto.Address{}, "", + fmt.Errorf("no free address in subnet: %s", subnetId) + } else { + return proto.Address{}, "", + fmt.Errorf("address: %s not found in free pool", ipAddr) + } } addressPool := addressPoolType{ Free: make([]proto.Address, 0, len(m.addressPool.Free)-1), diff --git a/hypervisor/manager/vm.go b/hypervisor/manager/vm.go index 2ddbc4fb..c2d92b37 100644 --- a/hypervisor/manager/vm.go +++ b/hypervisor/manager/vm.go @@ -181,7 +181,8 @@ func (m *Manager) allocateVm(req proto.CreateVmRequest, } subnetIDs[subnetId] = struct{}{} } - address, subnetId, err := m.getFreeAddress(req.SubnetId, authInfo) + address, subnetId, err := m.getFreeAddress(req.Address.IpAddress, + req.SubnetId, authInfo) if err != nil { return nil, err } @@ -195,8 +196,13 @@ func (m *Manager) allocateVm(req proto.CreateVmRequest, } }() var secondaryAddresses []proto.Address - for _, subnetId := range req.SecondarySubnetIDs { - secondaryAddress, _, err := m.getFreeAddress(subnetId, authInfo) + for index, subnetId := range req.SecondarySubnetIDs { + var reqIpAddr net.IP + if index < len(req.SecondaryAddresses) { + reqIpAddr = req.SecondaryAddresses[index].IpAddress + } + secondaryAddress, _, err := m.getFreeAddress(reqIpAddr, subnetId, + authInfo) if err != nil { return nil, err } From 192bf45f617ec5f4c77a3f3aa2ea0569cd082d1b Mon Sep 17 00:00:00 2001 From: Richard Gooch Date: Wed, 12 Dec 2018 08:08:37 -0800 Subject: [PATCH 03/13] Add -requestIPs option to vm-control. --- cmd/vm-control/createVm.go | 22 ++++++++++++++++++++++ cmd/vm-control/main.go | 2 ++ 2 files changed, 24 insertions(+) diff --git a/cmd/vm-control/createVm.go b/cmd/vm-control/createVm.go index ca359c8e..02d82a76 100644 --- a/cmd/vm-control/createVm.go +++ b/cmd/vm-control/createVm.go @@ -108,6 +108,28 @@ func createVmOnHypervisor(hypervisor string, logger log.DebugLogger) error { MinimumFreeBytes: uint64(minFreeBytes), RoundupPower: *roundupPower, } + if len(requestIPs) > 0 && requestIPs[0] != "" { + ipAddr := net.ParseIP(requestIPs[0]) + if ipAddr == nil { + return fmt.Errorf("invalid IP address: %s", requestIPs[0]) + } + request.Address.IpAddress = ipAddr + } + if len(requestIPs) > 1 && len(secondarySubnetIDs) > 0 { + request.SecondaryAddresses = make([]hyper_proto.Address, + len(secondarySubnetIDs)) + for index, addr := range requestIPs[1:] { + if addr == "" { + continue + } + ipAddr := net.ParseIP(addr) + if ipAddr == nil { + return fmt.Errorf("invalid IP address: %s", requestIPs[0]) + } + request.SecondaryAddresses[index] = hyper_proto.Address{ + IpAddress: ipAddr} + } + } if sizes, err := parseSizes(secondaryVolumeSizes); err != nil { return err } else { diff --git a/cmd/vm-control/main.go b/cmd/vm-control/main.go index 706b12b4..3356ee40 100644 --- a/cmd/vm-control/main.go +++ b/cmd/vm-control/main.go @@ -54,6 +54,7 @@ var ( secondaryVolumeSizes flagutil.StringList subnetId = flag.String("subnetId", "", "Subnet ID to launch VM in") + requestIPs flagutil.StringList roundupPower = flag.Uint64("roundupPower", 24, "power of 2 to round up root volume size") snapshotRootOnly = flag.Bool("snapshotRootOnly", false, @@ -78,6 +79,7 @@ func init() { "minimum number of free bytes in root volume") flag.Var(&ownerGroups, "ownerGroups", "Groups who own the VM") flag.Var(&ownerUsers, "ownerUsers", "Extra users who own the VM") + flag.Var(&requestIPs, "requestIPs", "Request specific IPs, if available") flag.Var(&secondarySubnetIDs, "secondarySubnetIDs", "Secondary Subnet IDs") flag.Var(&secondaryVolumeSizes, "secondaryVolumeSizes", "Sizes for secondary volumes") From b549f1db76224e611eaa3478fc26a61d83750aee Mon Sep 17 00:00:00 2001 From: Richard Gooch Date: Thu, 13 Dec 2018 08:35:21 -0800 Subject: [PATCH 04/13] Add proto/hypervisor.ChangeAddressPoolRequest.AddressesToRemove field. --- proto/hypervisor/messages.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/proto/hypervisor/messages.go b/proto/hypervisor/messages.go index db279016..a4273b5d 100644 --- a/proto/hypervisor/messages.go +++ b/proto/hypervisor/messages.go @@ -42,7 +42,8 @@ type BecomePrimaryVmOwnerResponse struct { } type ChangeAddressPoolRequest struct { - AddressesToAdd []Address + AddressesToAdd []Address // Will be added to free pool. + AddressesToRemove []Address // Will be removed from free pool. MaximumFreeAddresses map[string]uint // Key: subnet ID. } From ce81592cc8d74a199729ef87dae21ba65b763814 Mon Sep 17 00:00:00 2001 From: Richard Gooch Date: Fri, 14 Dec 2018 08:52:13 -0800 Subject: [PATCH 05/13] Add hypervisor/manager.Manager.RemoveAddressesFromPool() method. --- hypervisor/manager/addressPool.go | 66 +++++++++++++++++++++++++++++++ hypervisor/manager/api.go | 4 ++ 2 files changed, 70 insertions(+) diff --git a/hypervisor/manager/addressPool.go b/hypervisor/manager/addressPool.go index a350972f..68ecec7e 100644 --- a/hypervisor/manager/addressPool.go +++ b/hypervisor/manager/addressPool.go @@ -19,6 +19,37 @@ func ipIsUnspecified(ipAddr net.IP) bool { return ipAddr.IsUnspecified() } +func removeAddresses(addresses []proto.Address, + ipsToRemove, macsToRemove map[string]struct{}, message string) ( + []proto.Address, error) { + newAddresses := make([]proto.Address, 0) + for _, address := range addresses { + keep := true + if len(address.IpAddress) > 0 { + ipAddr := address.IpAddress.String() + if _, ok := ipsToRemove[ipAddr]; ok { + delete(ipsToRemove, ipAddr) + delete(macsToRemove, address.MacAddress) + keep = false + } + } + if _, ok := macsToRemove[address.MacAddress]; ok { + delete(macsToRemove, address.MacAddress) + keep = false + } + if keep { + newAddresses = append(newAddresses, address) + } + } + if len(ipsToRemove) > 0 { + return nil, fmt.Errorf("IPs: %v %s", ipsToRemove, message) + } + if len(macsToRemove) > 0 { + return nil, fmt.Errorf("MACs: %v %s", macsToRemove, message) + } + return newAddresses, nil +} + func (m *Manager) addAddressesToPool(addresses []proto.Address) error { for index := range addresses { addresses[index].Shrink() @@ -178,6 +209,41 @@ func (m *Manager) releaseAddressInPoolWithLock(address proto.Address) error { return m.writeAddressPoolWithLock(m.addressPool, false) } +func (m *Manager) removeAddressesFromPool(addresses []proto.Address) error { + ipsToRemoveFree := make(map[string]struct{}, len(addresses)) + ipsToRemoveRegistered := make(map[string]struct{}, len(addresses)) + macsToRemoveFree := make(map[string]struct{}, len(addresses)) + macsToRemoveRegistered := make(map[string]struct{}, len(addresses)) + for _, address := range addresses { + if len(address.IpAddress) > 0 { + ipsToRemoveFree[address.IpAddress.String()] = struct{}{} + ipsToRemoveRegistered[address.IpAddress.String()] = struct{}{} + } + if address.MacAddress != "" { + macsToRemoveFree[address.MacAddress] = struct{}{} + macsToRemoveRegistered[address.MacAddress] = struct{}{} + } + } + if len(ipsToRemoveFree) < 1 && len(macsToRemoveFree) < 1 { + return nil + } + m.mutex.Lock() + defer m.mutex.Unlock() + freeAddresses, err := removeAddresses(m.addressPool.Free, ipsToRemoveFree, + macsToRemoveFree, "not in free pool") + if err != nil { + return err + } + registeredAddresses, err := removeAddresses(m.addressPool.Registered, + ipsToRemoveRegistered, macsToRemoveRegistered, "not registered") + if err != nil { + return err + } + m.addressPool.Free = freeAddresses + m.addressPool.Registered = registeredAddresses + return m.writeAddressPoolWithLock(m.addressPool, false) +} + func (m *Manager) removeExcessAddressesFromPool(maxFree map[string]uint) error { freeCount := make(map[string]uint) macAddressesToRemove := make(map[string]struct{}) diff --git a/hypervisor/manager/api.go b/hypervisor/manager/api.go index 545ae3ae..0cc03964 100644 --- a/hypervisor/manager/api.go +++ b/hypervisor/manager/api.go @@ -250,6 +250,10 @@ func (m *Manager) PrepareVmForMigration(ipAddr net.IP, return m.prepareVmForMigration(ipAddr, authInfo, accessToken, enable) } +func (m *Manager) RemoveAddressesFromPool(addresses []proto.Address) error { + return m.removeAddressesFromPool(addresses) +} + func (m *Manager) RemoveExcessAddressesFromPool(maxFree map[string]uint) error { return m.removeExcessAddressesFromPool(maxFree) } From 8d3f71fd0bbe54c1f2dee9f05d81751f9439dc3c Mon Sep 17 00:00:00 2001 From: Richard Gooch Date: Sat, 15 Dec 2018 07:34:59 -0800 Subject: [PATCH 06/13] Support removing addresses in Hypervisor.ChangeAddressPool SRPC method. --- hypervisor/rpcd/changeAddressPool.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/hypervisor/rpcd/changeAddressPool.go b/hypervisor/rpcd/changeAddressPool.go index 21c2d0d1..8a13f77b 100644 --- a/hypervisor/rpcd/changeAddressPool.go +++ b/hypervisor/rpcd/changeAddressPool.go @@ -22,6 +22,12 @@ func (t *srpcType) changeAddressPool(conn *srpc.Conn, return err } } + if len(request.AddressesToRemove) > 0 { + err := t.manager.RemoveAddressesFromPool(request.AddressesToRemove) + if err != nil { + return err + } + } if len(request.MaximumFreeAddresses) > 0 { err := t.manager.RemoveExcessAddressesFromPool( request.MaximumFreeAddresses) From 2d747a3a3818d1ba323288a5448d0ca03ed17743 Mon Sep 17 00:00:00 2001 From: Richard Gooch Date: Sun, 16 Dec 2018 07:20:23 -0800 Subject: [PATCH 07/13] Add remove-{ip,mac}-address subcommand to hyper-control tool. --- cmd/hyper-control/main.go | 4 +++ cmd/hyper-control/removeAddress.go | 54 ++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 cmd/hyper-control/removeAddress.go diff --git a/cmd/hyper-control/main.go b/cmd/hyper-control/main.go index 55cd58cb..107967fe 100644 --- a/cmd/hyper-control/main.go +++ b/cmd/hyper-control/main.go @@ -74,6 +74,8 @@ func printUsage() { fmt.Fprintln(os.Stderr, " netboot-machine MACaddr IPaddr [hostname]") fmt.Fprintln(os.Stderr, " reinstall") fmt.Fprintln(os.Stderr, " remove-excess-addresses MaxFreeAddr") + fmt.Fprintln(os.Stderr, " remove-ip-address IPaddr") + fmt.Fprintln(os.Stderr, " remove-mac-address MACaddr") fmt.Fprintln(os.Stderr, " rollout-image name") fmt.Fprintln(os.Stderr, " write-netboot-files hostname dirname") } @@ -99,6 +101,8 @@ var subcommands = []subcommand{ {"netboot-machine", 2, 3, netbootMachineSubcommand}, {"reinstall", 0, 0, reinstallSubcommand}, {"remove-excess-addresses", 1, 1, removeExcessAddressesSubcommand}, + {"remove-ip-address", 1, 1, removeIpAddressSubcommand}, + {"remove-mac-address", 1, 1, removeMacAddressSubcommand}, {"rollout-image", 1, 1, rolloutImageSubcommand}, {"write-netboot-files", 2, 2, writeNetbootFilesSubcommand}, } diff --git a/cmd/hyper-control/removeAddress.go b/cmd/hyper-control/removeAddress.go new file mode 100644 index 00000000..83c22b1c --- /dev/null +++ b/cmd/hyper-control/removeAddress.go @@ -0,0 +1,54 @@ +package main + +import ( + "fmt" + "net" + "os" + + "github.com/Symantec/Dominator/lib/errors" + "github.com/Symantec/Dominator/lib/log" + "github.com/Symantec/Dominator/lib/srpc" + proto "github.com/Symantec/Dominator/proto/hypervisor" +) + +func removeIpAddressSubcommand(args []string, logger log.DebugLogger) { + ipAddr := net.ParseIP(args[0]) + if len(ipAddr) < 4 { + fmt.Fprintf(os.Stderr, "Invalid IP address: %s\n", args[0]) + os.Exit(1) + } + err := removeAddress(proto.Address{IpAddress: ipAddr}, logger) + if err != nil { + fmt.Fprintf(os.Stderr, "Error removing IP address: %s\n", err) + os.Exit(1) + } + os.Exit(0) +} + +func removeMacAddressSubcommand(args []string, logger log.DebugLogger) { + address := proto.Address{MacAddress: args[0]} + err := removeAddress(address, logger) + if err != nil { + fmt.Fprintf(os.Stderr, "Error removing MAC address: %s\n", err) + os.Exit(1) + } + os.Exit(0) +} + +func removeAddress(address proto.Address, logger log.DebugLogger) error { + address.Shrink() + request := proto.ChangeAddressPoolRequest{ + AddressesToRemove: []proto.Address{address}} + var reply proto.ChangeAddressPoolResponse + clientName := fmt.Sprintf("%s:%d", *hypervisorHostname, *hypervisorPortNum) + client, err := srpc.DialHTTP("tcp", clientName, 0) + if err != nil { + return err + } + defer client.Close() + err = client.RequestReply("Hypervisor.ChangeAddressPool", request, &reply) + if err != nil { + return err + } + return errors.New(reply.Error) +} From 99678863f320894f0737a897008d70aff25892bc Mon Sep 17 00:00:00 2001 From: Richard Gooch Date: Mon, 17 Dec 2018 08:09:30 -0800 Subject: [PATCH 08/13] Add proto/fleetmanager.MoveIpAddresses messages. --- proto/fleetmanager/messages.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/proto/fleetmanager/messages.go b/proto/fleetmanager/messages.go index 39954e07..792cbb2c 100644 --- a/proto/fleetmanager/messages.go +++ b/proto/fleetmanager/messages.go @@ -94,6 +94,15 @@ type Machine struct { Tags tags.Tags `json:",omitempty"` } +type MoveIpAddressesRequest struct { + HypervisorHostname string + IpAddresses []net.IP +} + +type MoveIpAddressesResponse struct { + Error string +} + type NetworkEntry struct { Hostname string `json:",omitempty"` HostIpAddress net.IP `json:",omitempty"` From 9fb2fb3ac22e801d131c083324d566760be8ea8f Mon Sep 17 00:00:00 2001 From: Richard Gooch Date: Tue, 18 Dec 2018 08:04:28 -0800 Subject: [PATCH 09/13] Fix bug in hypervisor: send notification when address is removed. --- hypervisor/manager/addressPool.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hypervisor/manager/addressPool.go b/hypervisor/manager/addressPool.go index 68ecec7e..79f01d6a 100644 --- a/hypervisor/manager/addressPool.go +++ b/hypervisor/manager/addressPool.go @@ -241,7 +241,7 @@ func (m *Manager) removeAddressesFromPool(addresses []proto.Address) error { } m.addressPool.Free = freeAddresses m.addressPool.Registered = registeredAddresses - return m.writeAddressPoolWithLock(m.addressPool, false) + return m.writeAddressPoolWithLock(m.addressPool, true) } func (m *Manager) removeExcessAddressesFromPool(maxFree map[string]uint) error { From 7b94c50de0e40a37c2d26efeee5749d085bc89f8 Mon Sep 17 00:00:00 2001 From: Richard Gooch Date: Tue, 18 Dec 2018 23:28:58 -0800 Subject: [PATCH 10/13] Add fleetmanager/hypervisors/fsstorer.Storer.GetHypervisorForIp() method. --- fleetmanager/hypervisors/fsstorer/api.go | 4 ++++ fleetmanager/hypervisors/fsstorer/check.go | 4 ++-- fleetmanager/hypervisors/fsstorer/get.go | 19 +++++++++++++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 fleetmanager/hypervisors/fsstorer/get.go diff --git a/fleetmanager/hypervisors/fsstorer/api.go b/fleetmanager/hypervisors/fsstorer/api.go index b4a8f30e..53f184e4 100644 --- a/fleetmanager/hypervisors/fsstorer/api.go +++ b/fleetmanager/hypervisors/fsstorer/api.go @@ -49,6 +49,10 @@ func (s *Storer) DeleteVm(hypervisor net.IP, ipAddr string) error { return s.deleteVm(hypervisor, ipAddr) } +func (s *Storer) GetHypervisorForIp(addr net.IP) (net.IP, error) { + return s.getHypervisorForIp(addr) +} + func (s *Storer) ListVMs(hypervisor net.IP) ([]string, error) { return s.listVMs(hypervisor) } diff --git a/fleetmanager/hypervisors/fsstorer/check.go b/fleetmanager/hypervisors/fsstorer/check.go index 8574fb89..9e82f8ae 100644 --- a/fleetmanager/hypervisors/fsstorer/check.go +++ b/fleetmanager/hypervisors/fsstorer/check.go @@ -6,11 +6,11 @@ import ( ) func (s *Storer) checkIpIsRegistered(addr net.IP) (bool, error) { - s.mutex.RLock() - defer s.mutex.RUnlock() if ip, err := netIpToIp(addr); err != nil { return false, err } else { + s.mutex.RLock() + defer s.mutex.RUnlock() _, ok := s.ipToHypervisor[ip] return ok, nil } diff --git a/fleetmanager/hypervisors/fsstorer/get.go b/fleetmanager/hypervisors/fsstorer/get.go new file mode 100644 index 00000000..085f5cb1 --- /dev/null +++ b/fleetmanager/hypervisors/fsstorer/get.go @@ -0,0 +1,19 @@ +package fsstorer + +import ( + "net" +) + +func (s *Storer) getHypervisorForIp(addr net.IP) (net.IP, error) { + if ip, err := netIpToIp(addr); err != nil { + return nil, err + } else { + s.mutex.RLock() + hypervisor, ok := s.ipToHypervisor[ip] + s.mutex.RUnlock() + if !ok { + return nil, nil + } + return hypervisor[:], nil + } +} From f70a718a93c19a5ad05073d60d2f4ad499271390 Mon Sep 17 00:00:00 2001 From: Richard Gooch Date: Wed, 19 Dec 2018 08:31:14 -0800 Subject: [PATCH 11/13] Add fleetmanager/hypervisors.Manager.MoveIpAddresses() method. --- fleetmanager/hypervisors/api.go | 5 + fleetmanager/hypervisors/moveIpAddresses.go | 184 ++++++++++++++++++++ fleetmanager/hypervisors/update.go | 2 + 3 files changed, 191 insertions(+) create mode 100644 fleetmanager/hypervisors/moveIpAddresses.go diff --git a/fleetmanager/hypervisors/api.go b/fleetmanager/hypervisors/api.go index 7f1922db..41f30575 100644 --- a/fleetmanager/hypervisors/api.go +++ b/fleetmanager/hypervisors/api.go @@ -43,6 +43,7 @@ type hypervisorType struct { type ipStorer interface { AddIPsForHypervisor(hypervisor net.IP, addrs []net.IP) error CheckIpIsRegistered(addr net.IP) (bool, error) + GetHypervisorForIp(addr net.IP) (net.IP, error) SetIPsForHypervisor(hypervisor net.IP, addrs []net.IP) error UnregisterHypervisor(hypervisor net.IP) error } @@ -146,6 +147,10 @@ func (m *Manager) MakeUpdateChannel(location string) <-chan fm_proto.Update { return m.makeUpdateChannel(location) } +func (m *Manager) MoveIpAddresses(hostname string, ipAddresses []net.IP) error { + return m.moveIpAddresses(hostname, ipAddresses) +} + func (m *Manager) WriteHtml(writer io.Writer) { m.writeHtml(writer) } diff --git a/fleetmanager/hypervisors/moveIpAddresses.go b/fleetmanager/hypervisors/moveIpAddresses.go new file mode 100644 index 00000000..1a372671 --- /dev/null +++ b/fleetmanager/hypervisors/moveIpAddresses.go @@ -0,0 +1,184 @@ +package hypervisors + +import ( + "fmt" + "net" + "time" + + "github.com/Symantec/Dominator/lib/constants" + "github.com/Symantec/Dominator/lib/errors" + "github.com/Symantec/Dominator/lib/net/util" + "github.com/Symantec/Dominator/lib/srpc" + hyper_proto "github.com/Symantec/Dominator/proto/hypervisor" +) + +func (m *Manager) addIp(hypervisorIpAddress, ip net.IP) error { + client, err := srpc.DialHTTP("tcp", + fmt.Sprintf("%s:%d", + hypervisorIpAddress, constants.HypervisorPortNumber), + time.Second*15) + if err != nil { + return err + } + defer client.Close() + request := hyper_proto.ChangeAddressPoolRequest{ + AddressesToAdd: []hyper_proto.Address{{ + IpAddress: ip, + MacAddress: fmt.Sprintf("52:54:%02x:%02x:%02x:%02x", + ip[0], ip[1], ip[2], ip[3]), + }}, + } + var reply hyper_proto.ChangeAddressPoolResponse + err = client.RequestReply("Hypervisor.ChangeAddressPool", request, &reply) + if err != nil { + return err + } + return errors.New(reply.Error) +} + +func (m *Manager) getHealthyHypervisorAddr(hostname string) (net.IP, error) { + hypervisor, err := m.getLockedHypervisor(hostname, false) + if err != nil { + return nil, err + } + defer hypervisor.mutex.RUnlock() + if hypervisor.healthStatus == "marginal" || + hypervisor.healthStatus == "at risk" { + return nil, errors.New("cannot move IPs to unhealthy hypervisor") + } + if len(hypervisor.machine.HostIpAddress) < 1 { + return nil, fmt.Errorf("IP address for: %s not known", hostname) + } + return hypervisor.machine.HostIpAddress, nil +} + +func (m *Manager) markIPsForMigration(ipAddresses []net.IP) error { + m.mutex.Lock() + defer m.mutex.Unlock() + if num := len(m.migratingIPs); num > 0 { + return fmt.Errorf("%d other migrations in progress: %v", + num, m.migratingIPs) + } + for _, ip := range ipAddresses { + if len(ip) > 0 { + m.migratingIPs[ip.String()] = struct{}{} + } + } + return nil +} + +func (m *Manager) moveIpAddresses(hostname string, ipAddresses []net.IP) error { + if !*manageHypervisors { + return errors.New("this is a read-only Fleet Manager") + } + if len(ipAddresses) < 1 { + return nil + } + sourceHypervisorIPs := make([]net.IP, len(ipAddresses)) + for index, ip := range ipAddresses { + ip = util.ShrinkIP(ip) + ipAddresses[index] = ip + sourceHypervisorIp, err := m.storer.GetHypervisorForIp(ip) + if err != nil { + return err + } + sourceHypervisorIPs[index] = sourceHypervisorIp + } + hypervisorIpAddress, err := m.getHealthyHypervisorAddr(hostname) + if err != nil { + return err + } + if err := m.markIPsForMigration(ipAddresses); err != nil { + return err + } + defer m.unmarkIPsForMigration(ipAddresses) + // Move IPs. + for index, ip := range ipAddresses { + err := m.moveIpAddress(hypervisorIpAddress, sourceHypervisorIPs[index], + ip) + if err != nil { + return err + } + } + // Wait for IPs to have moved. + // TODO(rgooch): Change this to watch for the registration events. + stopTime := time.Now().Add(time.Second * 10) + for ; time.Until(stopTime) >= 0; time.Sleep(time.Millisecond * 10) { + allInPlace := true + for _, ip := range ipAddresses { + newHyperIp, err := m.storer.GetHypervisorForIp(ip) + if err != nil { + return err + } + if newHyperIp == nil || !newHyperIp.Equal(hypervisorIpAddress) { + allInPlace = false + break // Not yet registered with the destination Hypervisor. + } + } + if allInPlace { + return nil + } + } + return errors.New("timed out waiting for addresses to move") +} + +func (m *Manager) moveIpAddress(destinationHypervisorIpAddress, + sourceHypervisorIpAddress, ipToMove net.IP) error { + if sourceHypervisorIpAddress != nil { + if sourceHypervisorIpAddress.Equal(destinationHypervisorIpAddress) { + return nil // IP address is already registered to dest Hypervisor. + } + err := m.removeIpAndWait(sourceHypervisorIpAddress, ipToMove) + if err != nil { + return err + } + } + return m.addIp(destinationHypervisorIpAddress, ipToMove) +} + +func (m *Manager) removeIpAndWait(hypervisorIpAddress, ipToMove net.IP) error { + client, err := srpc.DialHTTP("tcp", + fmt.Sprintf("%s:%d", + hypervisorIpAddress, constants.HypervisorPortNumber), + time.Second*15) + if err != nil { + return err + } + defer client.Close() + request := hyper_proto.ChangeAddressPoolRequest{ + AddressesToRemove: []hyper_proto.Address{{IpAddress: ipToMove}}, + } + var reply hyper_proto.ChangeAddressPoolResponse + err = client.RequestReply("Hypervisor.ChangeAddressPool", request, &reply) + if err != nil { + return err + } + if err := errors.New(reply.Error); err != nil { + return fmt.Errorf("error unregistering %s from %s: %s", + ipToMove, hypervisorIpAddress, err) + } + // TODO(rgooch): Change this to watch for the deregistration event. + stopTime := time.Now().Add(time.Second * 10) + for ; time.Until(stopTime) >= 0; time.Sleep(time.Millisecond * 10) { + newHyperIp, err := m.storer.GetHypervisorForIp(ipToMove) + if err != nil { + return err + } + if newHyperIp == nil { + return nil // No longer registered with a Hypervisor. + } + } + return fmt.Errorf( + "timed out waiting for %s to become unregistered from %s", + ipToMove, hypervisorIpAddress) +} + +func (m *Manager) unmarkIPsForMigration(ipAddresses []net.IP) { + m.mutex.Lock() + defer m.mutex.Unlock() + for _, ip := range ipAddresses { + if len(ip) > 0 { + delete(m.migratingIPs, ip.String()) + } + } +} diff --git a/fleetmanager/hypervisors/update.go b/fleetmanager/hypervisors/update.go index c3976a6b..a035e13c 100644 --- a/fleetmanager/hypervisors/update.go +++ b/fleetmanager/hypervisors/update.go @@ -478,6 +478,8 @@ func (m *Manager) getSubnetsForMachine(h *hypervisorType) ( func (m *Manager) processAddressPoolUpdates(h *hypervisorType, update hyper_proto.Update) { if update.HaveAddressPool { + h.logger.Debugf(1, "registered address pool size: %d\n", + len(update.AddressPool)) addresses := make([]net.IP, 0, len(update.AddressPool)) for _, address := range update.AddressPool { addresses = append(addresses, address.IpAddress) From 68c78c77f801600f74263b8b7489362bdbfd3641 Mon Sep 17 00:00:00 2001 From: Richard Gooch Date: Thu, 20 Dec 2018 07:42:46 -0800 Subject: [PATCH 12/13] Add FleetManager.MoveIpAddresses SRPC method. --- fleetmanager/rpcd/moveIpAddresses.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 fleetmanager/rpcd/moveIpAddresses.go diff --git a/fleetmanager/rpcd/moveIpAddresses.go b/fleetmanager/rpcd/moveIpAddresses.go new file mode 100644 index 00000000..11f86d2e --- /dev/null +++ b/fleetmanager/rpcd/moveIpAddresses.go @@ -0,0 +1,19 @@ +package rpcd + +import ( + "github.com/Symantec/Dominator/lib/errors" + "github.com/Symantec/Dominator/lib/srpc" + fm_proto "github.com/Symantec/Dominator/proto/fleetmanager" +) + +func (t *srpcType) MoveIpAddresses(conn *srpc.Conn, + request fm_proto.MoveIpAddressesRequest, + reply *fm_proto.MoveIpAddressesResponse) error { + err := t.hypervisorsManager.MoveIpAddresses(request.HypervisorHostname, + request.IpAddresses) + if err != nil { + *reply = fm_proto.MoveIpAddressesResponse{ + Error: errors.ErrorToString(err)} + } + return nil +} From c8662d9c200b168976e39f2c8e803bd17675092d Mon Sep 17 00:00:00 2001 From: Richard Gooch Date: Fri, 21 Dec 2018 07:29:28 -0800 Subject: [PATCH 13/13] Add move-ip-address subcommand to hyper-control tool. --- cmd/hyper-control/README.md | 4 +++ cmd/hyper-control/main.go | 2 ++ cmd/hyper-control/moveIpAddress.go | 44 ++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+) create mode 100644 cmd/hyper-control/moveIpAddress.go diff --git a/cmd/hyper-control/README.md b/cmd/hyper-control/README.md index 8f9793d6..d86f344a 100644 --- a/cmd/hyper-control/README.md +++ b/cmd/hyper-control/README.md @@ -43,6 +43,7 @@ Some of the sub-commands available are: on a machine - **make-installer-iso**: make a bootable installation ISO (CD-ROM) image for a machine +- **move-ip-address**: move a (free) IP address to a specific *Hypervisor* - **netboot-host**: temporarily enable PXE-based network booting and installing for a machine - **netboot-machine**: temporarily enable PXE-based network booting for a @@ -50,6 +51,9 @@ Some of the sub-commands available are: - **reinstall**: reinstall the local machine. This erases all data - **remove-excess-addresses**: remove free addresses for a specific *Hypervisor* above the specified limit +- **remove-ip-address**: remove a (free) IP address from a specific *Hypervisor* +- **remove-mac-address**: remove a (free) MAC address from a specific + *Hypervisor* - **rollout-image**: safely roll out specified image to all *Hypervisors* in a location - **write-netboot-files**: write the configuration files for installing a diff --git a/cmd/hyper-control/main.go b/cmd/hyper-control/main.go index 107967fe..76f7f49f 100644 --- a/cmd/hyper-control/main.go +++ b/cmd/hyper-control/main.go @@ -70,6 +70,7 @@ func printUsage() { fmt.Fprintln(os.Stderr, " get-updates") fmt.Fprintln(os.Stderr, " installer-shell hostname") fmt.Fprintln(os.Stderr, " make-installer-iso hostname dirname") + fmt.Fprintln(os.Stderr, " move-ip-address IPaddr") fmt.Fprintln(os.Stderr, " netboot-host hostname") fmt.Fprintln(os.Stderr, " netboot-machine MACaddr IPaddr [hostname]") fmt.Fprintln(os.Stderr, " reinstall") @@ -97,6 +98,7 @@ var subcommands = []subcommand{ {"get-updates", 0, 0, getUpdatesSubcommand}, {"installer-shell", 1, 1, installerShellSubcommand}, {"make-installer-iso", 2, 2, makeInstallerIsoSubcommand}, + {"move-ip-address", 1, 1, moveIpAddressSubcommand}, {"netboot-host", 1, 1, netbootHostSubcommand}, {"netboot-machine", 2, 3, netbootMachineSubcommand}, {"reinstall", 0, 0, reinstallSubcommand}, diff --git a/cmd/hyper-control/moveIpAddress.go b/cmd/hyper-control/moveIpAddress.go new file mode 100644 index 00000000..0c75ba0e --- /dev/null +++ b/cmd/hyper-control/moveIpAddress.go @@ -0,0 +1,44 @@ +package main + +import ( + "fmt" + "net" + "os" + + "github.com/Symantec/Dominator/lib/errors" + "github.com/Symantec/Dominator/lib/log" + "github.com/Symantec/Dominator/lib/srpc" + proto "github.com/Symantec/Dominator/proto/fleetmanager" +) + +func moveIpAddressSubcommand(args []string, logger log.DebugLogger) { + if err := moveIpAddress(args[0], logger); err != nil { + fmt.Fprintf(os.Stderr, "Error moving IP address: %s\n", err) + os.Exit(1) + } + os.Exit(0) +} + +func moveIpAddress(addr string, logger log.DebugLogger) error { + ipAddr := net.ParseIP(addr) + if len(ipAddr) < 4 { + return fmt.Errorf("invalid IP address: %s", addr) + } + request := proto.MoveIpAddressesRequest{ + HypervisorHostname: *hypervisorHostname, + IpAddresses: []net.IP{ipAddr}, + } + var reply proto.MoveIpAddressesResponse + clientName := fmt.Sprintf("%s:%d", + *fleetManagerHostname, *fleetManagerPortNum) + client, err := srpc.DialHTTP("tcp", clientName, 0) + if err != nil { + return err + } + defer client.Close() + err = client.RequestReply("FleetManager.MoveIpAddresses", request, &reply) + if err != nil { + return err + } + return errors.New(reply.Error) +}