From 24e75eba98e74dc080ae25592d0f579bd90d49ee Mon Sep 17 00:00:00 2001 From: Zhang Wei Date: Tue, 11 Jul 2017 15:42:53 +0800 Subject: [PATCH 1/3] Implement more rich network functions * Support add ip addr to a specific network interface * Support delete ip address from a network interface * Support modify MTU Signed-off-by: Zhang Wei --- api/descriptions.pb.go | 163 ++++++++++++----------- api/descriptions.proto | 12 +- hyperstart/api/json/constants.go | 1 + hyperstart/api/json/spec.go | 1 + hyperstart/libhyperstart/hyperstart.go | 3 +- hyperstart/libhyperstart/json.go | 12 +- hypervisor/context.go | 21 +++ hypervisor/driver.go | 1 + hypervisor/events.go | 7 +- hypervisor/network.go | 75 ++++++++--- hypervisor/network/network.go | 17 +-- hypervisor/network/network_linux.go | 68 ++++------ hypervisor/persistence.go | 6 +- hypervisor/vbox/network.go | 21 +-- hypervisor/vm.go | 46 ++++++- hypervisor/vm_states.go | 73 +++++++++- supervisor/hyperpod.go | 176 ++++++++++++++++--------- supervisor/proxy/nslistener.go | 61 +++++++-- 18 files changed, 514 insertions(+), 250 deletions(-) diff --git a/api/descriptions.pb.go b/api/descriptions.pb.go index 98f5c843..f6bf4cec 100644 --- a/api/descriptions.pb.go +++ b/api/descriptions.pb.go @@ -307,14 +307,15 @@ func (m *VolumeDescription) GetReadOnly() bool { } type InterfaceDescription struct { - Id string `protobuf:"bytes,1,opt,name=id" json:"id,omitempty"` - Lo bool `protobuf:"varint,2,opt,name=lo" json:"lo,omitempty"` - Bridge string `protobuf:"bytes,3,opt,name=bridge" json:"bridge,omitempty"` - Ip string `protobuf:"bytes,4,opt,name=ip" json:"ip,omitempty"` - Mac string `protobuf:"bytes,5,opt,name=mac" json:"mac,omitempty"` - Gw string `protobuf:"bytes,6,opt,name=gw" json:"gw,omitempty"` - TapName string `protobuf:"bytes,7,opt,name=tapName" json:"tapName,omitempty"` - Options string `protobuf:"bytes,8,opt,name=options" json:"options,omitempty"` + Id string `protobuf:"bytes,1,opt,name=id" json:"id,omitempty"` + Lo bool `protobuf:"varint,2,opt,name=lo" json:"lo,omitempty"` + Bridge string `protobuf:"bytes,3,opt,name=bridge" json:"bridge,omitempty"` + Ip []string `protobuf:"bytes,4,rep,name=ip" json:"ip,omitempty"` + Mac string `protobuf:"bytes,5,opt,name=mac" json:"mac,omitempty"` + Mtu uint64 `protobuf:"varint,6,opt,name=mtu" json:"mtu,omitempty"` + Gw string `protobuf:"bytes,7,opt,name=gw" json:"gw,omitempty"` + TapName string `protobuf:"bytes,8,opt,name=tapName" json:"tapName,omitempty"` + Options string `protobuf:"bytes,9,opt,name=options" json:"options,omitempty"` } func (m *InterfaceDescription) Reset() { *m = InterfaceDescription{} } @@ -343,11 +344,11 @@ func (m *InterfaceDescription) GetBridge() string { return "" } -func (m *InterfaceDescription) GetIp() string { +func (m *InterfaceDescription) GetIp() []string { if m != nil { return m.Ip } - return "" + return nil } func (m *InterfaceDescription) GetMac() string { @@ -357,6 +358,13 @@ func (m *InterfaceDescription) GetMac() string { return "" } +func (m *InterfaceDescription) GetMtu() uint64 { + if m != nil { + return m.Mtu + } + return 0 +} + func (m *InterfaceDescription) GetGw() string { if m != nil { return m.Gw @@ -692,71 +700,72 @@ func init() { func init() { proto.RegisterFile("descriptions.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 1050 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x94, 0x56, 0x6f, 0x6b, 0x1b, 0x47, - 0x13, 0xe7, 0xce, 0x92, 0x25, 0x8d, 0xe2, 0xd8, 0x59, 0xfc, 0x84, 0x25, 0x3c, 0x14, 0x73, 0x4d, - 0x8a, 0x49, 0xc1, 0x2f, 0x1c, 0x68, 0x9a, 0x42, 0xa0, 0x21, 0x09, 0x41, 0xd0, 0xda, 0x62, 0xdd, - 0xb4, 0xf4, 0xe5, 0xea, 0x6e, 0x25, 0x6f, 0x7d, 0xda, 0x3d, 0x76, 0x57, 0x76, 0xd4, 0x0f, 0xd1, - 0xf7, 0xfd, 0x16, 0x7d, 0xd1, 0x8f, 0xd3, 0x2f, 0xd0, 0x6f, 0x51, 0x66, 0x6e, 0xef, 0x7c, 0x96, - 0x9d, 0x96, 0xbc, 0x9b, 0xf9, 0xdd, 0xcc, 0xec, 0xfc, 0x9f, 0x03, 0x56, 0x28, 0x9f, 0x3b, 0x5d, - 0x05, 0x6d, 0x8d, 0x3f, 0xaa, 0x9c, 0x0d, 0x96, 0x6d, 0xc9, 0x4a, 0x67, 0x7f, 0x24, 0xb0, 0x73, - 0x26, 0x4d, 0x31, 0xb3, 0x1f, 0x5e, 0x5b, 0x33, 0xd7, 0x0b, 0xf6, 0x08, 0x86, 0xe7, 0xd6, 0x07, - 0x23, 0x97, 0x8a, 0x27, 0x07, 0xc9, 0xe1, 0x48, 0xb4, 0x3c, 0xdb, 0x83, 0xad, 0xc2, 0x78, 0x9e, - 0x1e, 0x6c, 0x1d, 0x8e, 0x04, 0x92, 0xec, 0x19, 0x8c, 0x8c, 0xd2, 0x8b, 0xf3, 0x99, 0x75, 0x9e, - 0x6f, 0x1d, 0x24, 0x87, 0xe3, 0xe3, 0xff, 0x1d, 0xc9, 0x4a, 0x1f, 0x9d, 0x44, 0xf4, 0x44, 0x85, - 0x2b, 0xeb, 0x2e, 0xbc, 0xb8, 0x96, 0x63, 0x9f, 0x01, 0x14, 0xc6, 0x9f, 0xd6, 0xde, 0xf0, 0x1e, - 0x59, 0xeb, 0x20, 0xec, 0xff, 0x30, 0x2a, 0x8c, 0x3f, 0x53, 0xd2, 0xe5, 0xe7, 0xbc, 0x4f, 0x9f, - 0xaf, 0x81, 0xec, 0xf7, 0x01, 0xec, 0xbf, 0xb6, 0x26, 0x48, 0x6d, 0x94, 0x7b, 0x73, 0x1d, 0x17, - 0xbb, 0x0f, 0xa9, 0x2e, 0xa2, 0xcf, 0xa9, 0x2e, 0x18, 0x83, 0x1e, 0x45, 0x91, 0x12, 0x42, 0x34, - 0xdb, 0x87, 0xbe, 0x5e, 0xca, 0x85, 0x22, 0x5f, 0x47, 0xa2, 0x66, 0xd8, 0x4b, 0xd8, 0x2e, 0xe5, - 0x4c, 0x95, 0xb5, 0x33, 0xe3, 0xe3, 0x27, 0x14, 0xc2, 0x5d, 0x8f, 0x1c, 0x7d, 0x47, 0x72, 0x6f, - 0x4d, 0x70, 0x6b, 0x11, 0x95, 0x30, 0x2d, 0x21, 0xac, 0x79, 0xff, 0x20, 0x39, 0x1c, 0x0a, 0x24, - 0x31, 0x42, 0x1f, 0x6c, 0x75, 0xa6, 0x17, 0x46, 0x96, 0x7c, 0x9b, 0xde, 0xea, 0x20, 0xec, 0x2b, - 0x00, 0x67, 0x6d, 0xf8, 0xd1, 0x96, 0xab, 0xa5, 0xe2, 0x03, 0xca, 0xdb, 0x43, 0x7a, 0xb4, 0x86, - 0x3a, 0x2f, 0x8a, 0x8e, 0x24, 0xe3, 0x30, 0x58, 0xda, 0x95, 0x09, 0x93, 0x82, 0x0f, 0xc9, 0x68, - 0xc3, 0x62, 0xd9, 0x50, 0x6e, 0x2a, 0xc3, 0x39, 0x1f, 0xd5, 0x65, 0x6b, 0x78, 0xf6, 0x18, 0xb6, - 0xde, 0xbf, 0x9b, 0x70, 0xa0, 0x67, 0x18, 0x3d, 0xf3, 0xde, 0x2b, 0xf7, 0xce, 0xd9, 0x55, 0x35, - 0x31, 0x73, 0x2b, 0xf0, 0x33, 0x7b, 0x0e, 0x3d, 0x65, 0x2e, 0x3d, 0x1f, 0x53, 0x0a, 0x3e, 0xff, - 0x78, 0x0a, 0xde, 0x9a, 0xcb, 0x98, 0x00, 0x52, 0x40, 0xa7, 0xb0, 0xc4, 0x85, 0x76, 0xfc, 0x5e, - 0xed, 0x54, 0x64, 0xb1, 0x02, 0x15, 0x3a, 0xb4, 0x53, 0x57, 0x00, 0x69, 0xc4, 0xa4, 0x5b, 0x78, - 0x7e, 0x9f, 0xea, 0x4a, 0x34, 0x7b, 0x02, 0x03, 0x57, 0xea, 0xa5, 0x0e, 0x9e, 0xef, 0xd2, 0xeb, - 0x63, 0x7a, 0x5d, 0x10, 0x26, 0x9a, 0x6f, 0x58, 0x26, 0xbf, 0xf6, 0x79, 0x28, 0xf9, 0xde, 0x7f, - 0x95, 0xe9, 0x8c, 0xe4, 0x62, 0x99, 0x6a, 0x25, 0xf6, 0x2d, 0x0c, 0x2e, 0x29, 0x8d, 0x9e, 0x3f, - 0x20, 0xfd, 0x2f, 0x3e, 0xae, 0x5f, 0xe7, 0x3b, 0x86, 0xd9, 0xa8, 0x61, 0x59, 0xb5, 0xd1, 0x41, - 0xcb, 0x52, 0xff, 0xaa, 0x38, 0xa7, 0x7a, 0x77, 0x90, 0x47, 0x2f, 0x60, 0xdc, 0xe9, 0x0f, 0xec, - 0x8b, 0x0b, 0xb5, 0x8e, 0x1d, 0x89, 0x24, 0xb6, 0xdf, 0xa5, 0x2c, 0x57, 0x4d, 0x4f, 0xd6, 0xcc, - 0x37, 0xe9, 0xd7, 0xc9, 0xa3, 0xe7, 0x30, 0x6a, 0xf3, 0xfa, 0x49, 0x8a, 0x2f, 0x60, 0xdc, 0x09, - 0xf6, 0x93, 0x54, 0xa7, 0x70, 0xaf, 0x1b, 0xe7, 0x1d, 0xba, 0x4f, 0xbb, 0xba, 0xe3, 0xe3, 0xfd, - 0x4e, 0x8b, 0x0a, 0x35, 0x57, 0x4e, 0x99, 0x5c, 0x75, 0x2c, 0x66, 0x7f, 0x25, 0xf0, 0xe0, 0x56, - 0x07, 0xb7, 0x83, 0x98, 0x74, 0x06, 0xf1, 0x21, 0x6c, 0x7b, 0xbb, 0x72, 0x79, 0xe3, 0x56, 0xe4, - 0x10, 0x9f, 0x5b, 0xb7, 0x94, 0x21, 0x4e, 0x68, 0xe4, 0x08, 0xf7, 0x61, 0x5d, 0x29, 0xde, 0x8b, - 0x38, 0x71, 0xec, 0x4b, 0x18, 0xd8, 0xb8, 0x48, 0x86, 0xe4, 0xe3, 0x83, 0x8e, 0x8f, 0xf5, 0x42, - 0x11, 0x8d, 0x04, 0xcb, 0xe0, 0x5e, 0x61, 0xf3, 0x0b, 0xe5, 0xe2, 0xe0, 0x8d, 0xa8, 0x82, 0x37, - 0x30, 0x1a, 0x24, 0x25, 0x8b, 0x53, 0x53, 0xae, 0x69, 0x62, 0x86, 0xa2, 0xe5, 0xb3, 0x3f, 0x13, - 0xd8, 0x9f, 0x98, 0xa0, 0xdc, 0x5c, 0xe6, 0xea, 0xdf, 0x56, 0xcf, 0x7d, 0x48, 0x4b, 0x4b, 0x91, - 0x0d, 0x45, 0x5a, 0x5a, 0xf4, 0x7e, 0xe6, 0x74, 0xd1, 0xee, 0x9d, 0xc8, 0x91, 0x5e, 0x15, 0x23, - 0x4a, 0x75, 0x85, 0x15, 0x58, 0xca, 0x9c, 0x36, 0xc9, 0x48, 0x20, 0x89, 0x12, 0x8b, 0xab, 0xb8, - 0x41, 0xd2, 0xc5, 0x15, 0x0e, 0x5b, 0x90, 0xd5, 0x89, 0x8c, 0x6b, 0x63, 0x24, 0x1a, 0x16, 0xbf, - 0x74, 0x33, 0x31, 0x6a, 0xc3, 0xce, 0x2c, 0xec, 0x4e, 0xad, 0x0b, 0x5d, 0x87, 0xe3, 0x96, 0x47, - 0x98, 0xdc, 0xee, 0x8b, 0x96, 0x67, 0x8f, 0x61, 0x27, 0x6f, 0x66, 0x82, 0x04, 0x52, 0x12, 0xb8, - 0x09, 0xa2, 0x05, 0xba, 0x23, 0xb9, 0x2d, 0x63, 0x50, 0x2d, 0x9f, 0xfd, 0x02, 0x7b, 0x9b, 0xfb, - 0x9f, 0x3d, 0x85, 0x3d, 0x8d, 0xa9, 0x33, 0xb2, 0x6c, 0x30, 0x9e, 0xd0, 0x0e, 0xb8, 0x85, 0xa3, - 0xac, 0xfa, 0xb0, 0x21, 0x5b, 0x1f, 0x9d, 0x5b, 0x78, 0xf6, 0x33, 0xec, 0x6e, 0x34, 0xe4, 0x9d, - 0xfd, 0x76, 0x0c, 0x63, 0x5a, 0x95, 0x53, 0xab, 0x4d, 0xa8, 0xad, 0x8d, 0x8f, 0xf7, 0x3a, 0xbd, - 0xf2, 0x3d, 0x7e, 0x15, 0x5d, 0xa1, 0xec, 0x25, 0x8c, 0x3b, 0xdf, 0xda, 0x6d, 0x96, 0x74, 0xb6, - 0x59, 0xb7, 0x5b, 0xd2, 0x8d, 0x6e, 0xf9, 0x2d, 0x69, 0xe6, 0xeb, 0xb4, 0x9d, 0x83, 0x95, 0x57, - 0xae, 0x31, 0x80, 0x34, 0x1a, 0x58, 0x5a, 0xa3, 0x03, 0xde, 0xcf, 0x3a, 0xc4, 0x96, 0xc7, 0x8a, - 0x5e, 0xa8, 0xb5, 0xd3, 0x66, 0x11, 0x33, 0xdc, 0xb0, 0xec, 0x00, 0xc6, 0xb3, 0x75, 0x50, 0x7e, - 0xaa, 0xdc, 0x99, 0xca, 0xa9, 0x81, 0xfa, 0xa2, 0x0b, 0xe1, 0x5b, 0xda, 0x56, 0x9e, 0x5a, 0xa9, - 0x2f, 0x88, 0xce, 0x14, 0xec, 0xdc, 0xd8, 0xfb, 0x77, 0x3a, 0xb4, 0x0f, 0xfd, 0x05, 0x0a, 0x34, - 0xeb, 0x82, 0x18, 0xac, 0x88, 0x2c, 0x0a, 0x8d, 0x61, 0xc8, 0x92, 0x0c, 0xe0, 0xb9, 0xa7, 0x8a, - 0x6c, 0xe2, 0xd9, 0x1b, 0xd8, 0xae, 0x37, 0x37, 0xda, 0xa7, 0x91, 0x8d, 0xf6, 0x69, 0x60, 0x19, - 0xf4, 0xce, 0xa5, 0x2b, 0xc8, 0x7c, 0x4f, 0x10, 0x8d, 0x98, 0xb7, 0xf3, 0x7a, 0xe4, 0x7b, 0x82, - 0xe8, 0xec, 0xef, 0x04, 0x06, 0x53, 0x67, 0x73, 0xe5, 0xe9, 0x87, 0xa0, 0xdd, 0xd2, 0xd1, 0xd8, - 0x35, 0x80, 0x23, 0x32, 0x29, 0xa2, 0xbb, 0xe9, 0x84, 0xac, 0x61, 0x98, 0x31, 0x67, 0x44, 0x63, - 0x54, 0xe4, 0x5d, 0x9c, 0xb5, 0x9a, 0x61, 0x87, 0xb0, 0xfb, 0xea, 0xa6, 0xf7, 0xf1, 0x77, 0x63, - 0x13, 0xc6, 0x32, 0xfd, 0xa0, 0xdc, 0x52, 0x37, 0xe7, 0x7c, 0x28, 0x5a, 0x1e, 0xdf, 0x7b, 0x85, - 0x17, 0x6d, 0x50, 0x5f, 0x34, 0xa4, 0x11, 0xc3, 0x75, 0xce, 0x87, 0x35, 0xf6, 0x36, 0xde, 0xc9, - 0x9f, 0xe2, 0x9d, 0xac, 0x2f, 0x74, 0xc3, 0xce, 0xb6, 0x69, 0x72, 0x9e, 0xfd, 0x13, 0x00, 0x00, - 0xff, 0xff, 0xb9, 0xd4, 0x30, 0xfc, 0xa7, 0x09, 0x00, 0x00, + // 1060 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x94, 0x56, 0x5d, 0x6e, 0xdb, 0xc6, + 0x13, 0x07, 0x69, 0xc9, 0x92, 0x86, 0xf1, 0xd7, 0xc2, 0xff, 0x60, 0x61, 0xfc, 0x51, 0x18, 0x6c, + 0x52, 0x18, 0x29, 0xe0, 0x07, 0x07, 0x68, 0x9a, 0x02, 0x01, 0x1a, 0x24, 0x46, 0x20, 0xa0, 0xb5, + 0x85, 0x75, 0xd3, 0xa2, 0x8f, 0x14, 0xb9, 0x92, 0xb7, 0xa6, 0x76, 0x89, 0xdd, 0x95, 0x1d, 0xf5, + 0x10, 0x7d, 0xef, 0x2d, 0x7a, 0x8a, 0x9e, 0xa2, 0x17, 0xe8, 0x2d, 0x8a, 0x19, 0x2e, 0x69, 0x5a, + 0x76, 0x5a, 0xe4, 0x6d, 0xe6, 0xc7, 0x99, 0xd9, 0xf9, 0x1e, 0x02, 0x2b, 0xa4, 0xcb, 0xad, 0xaa, + 0xbc, 0x32, 0xda, 0x1d, 0x57, 0xd6, 0x78, 0xc3, 0x36, 0xb2, 0x4a, 0xa5, 0x7f, 0x44, 0xb0, 0x75, + 0x91, 0xe9, 0x62, 0x6a, 0x3e, 0xbc, 0x31, 0x7a, 0xa6, 0xe6, 0xec, 0x00, 0x86, 0x97, 0xc6, 0x79, + 0x9d, 0x2d, 0x24, 0x8f, 0x0e, 0xa3, 0xa3, 0x91, 0x68, 0x79, 0xb6, 0x0b, 0x1b, 0x85, 0x76, 0x3c, + 0x3e, 0xdc, 0x38, 0x1a, 0x09, 0x24, 0xd9, 0x73, 0x18, 0x69, 0xa9, 0xe6, 0x97, 0x53, 0x63, 0x1d, + 0xdf, 0x38, 0x8c, 0x8e, 0x92, 0x93, 0xff, 0x1d, 0x67, 0x95, 0x3a, 0x3e, 0x0b, 0xe8, 0x99, 0xf4, + 0x37, 0xc6, 0x5e, 0x39, 0x71, 0x2b, 0xc7, 0x3e, 0x03, 0x28, 0xb4, 0x3b, 0xaf, 0xbd, 0xe1, 0x3d, + 0xb2, 0xd6, 0x41, 0xd8, 0xff, 0x61, 0x54, 0x68, 0x77, 0x21, 0x33, 0x9b, 0x5f, 0xf2, 0x3e, 0x7d, + 0xbe, 0x05, 0xd2, 0xdf, 0x07, 0xb0, 0xff, 0xc6, 0x68, 0x9f, 0x29, 0x2d, 0xed, 0xdb, 0xdb, 0xb8, + 0xd8, 0x36, 0xc4, 0xaa, 0x08, 0x3e, 0xc7, 0xaa, 0x60, 0x0c, 0x7a, 0x14, 0x45, 0x4c, 0x08, 0xd1, + 0x6c, 0x1f, 0xfa, 0x6a, 0x91, 0xcd, 0x25, 0xf9, 0x3a, 0x12, 0x35, 0xc3, 0x5e, 0xc1, 0x66, 0x99, + 0x4d, 0x65, 0x59, 0x3b, 0x93, 0x9c, 0x3c, 0xa5, 0x10, 0x1e, 0x7a, 0xe4, 0xf8, 0x3b, 0x92, 0x3b, + 0xd5, 0xde, 0xae, 0x44, 0x50, 0xc2, 0xb4, 0x78, 0xbf, 0xe2, 0xfd, 0xc3, 0xe8, 0x68, 0x28, 0x90, + 0xc4, 0x08, 0x9d, 0x37, 0xd5, 0x85, 0x9a, 0xeb, 0xac, 0xe4, 0x9b, 0xf4, 0x56, 0x07, 0x61, 0x5f, + 0x01, 0x58, 0x63, 0xfc, 0x8f, 0xa6, 0x5c, 0x2e, 0x24, 0x1f, 0x50, 0xde, 0x1e, 0xd3, 0xa3, 0x35, + 0xd4, 0x79, 0x51, 0x74, 0x24, 0x19, 0x87, 0xc1, 0xc2, 0x2c, 0xb5, 0x1f, 0x17, 0x7c, 0x48, 0x46, + 0x1b, 0x16, 0xcb, 0x86, 0x72, 0x93, 0xcc, 0x5f, 0xf2, 0x51, 0x5d, 0xb6, 0x86, 0x67, 0x4f, 0x60, + 0xe3, 0xfd, 0xbb, 0x31, 0x07, 0x7a, 0x86, 0xd1, 0x33, 0xef, 0x9d, 0xb4, 0xef, 0xac, 0x59, 0x56, + 0x63, 0x3d, 0x33, 0x02, 0x3f, 0xb3, 0x17, 0xd0, 0x93, 0xfa, 0xda, 0xf1, 0x84, 0x52, 0xf0, 0xf9, + 0xc7, 0x53, 0x70, 0xaa, 0xaf, 0x43, 0x02, 0x48, 0x01, 0x9d, 0xc2, 0x12, 0x17, 0xca, 0xf2, 0x47, + 0xb5, 0x53, 0x81, 0xc5, 0x0a, 0x54, 0xe8, 0xd0, 0x56, 0x5d, 0x01, 0xa4, 0x11, 0xcb, 0xec, 0xdc, + 0xf1, 0x6d, 0xaa, 0x2b, 0xd1, 0xec, 0x29, 0x0c, 0x6c, 0xa9, 0x16, 0xca, 0x3b, 0xbe, 0x43, 0xaf, + 0x27, 0xf4, 0xba, 0x20, 0x4c, 0x34, 0xdf, 0xb0, 0x4c, 0x6e, 0xe5, 0x72, 0x5f, 0xf2, 0xdd, 0xff, + 0x2a, 0xd3, 0x05, 0xc9, 0x85, 0x32, 0xd5, 0x4a, 0xec, 0x5b, 0x18, 0x5c, 0x53, 0x1a, 0x1d, 0xdf, + 0x23, 0xfd, 0x2f, 0x3e, 0xae, 0x5f, 0xe7, 0x3b, 0x84, 0xd9, 0xa8, 0x61, 0x59, 0x95, 0x56, 0x5e, + 0x65, 0xa5, 0xfa, 0x55, 0x72, 0x4e, 0xf5, 0xee, 0x20, 0x07, 0x2f, 0x21, 0xe9, 0xf4, 0x07, 0xf6, + 0xc5, 0x95, 0x5c, 0x85, 0x8e, 0x44, 0x12, 0xdb, 0xef, 0x3a, 0x2b, 0x97, 0x4d, 0x4f, 0xd6, 0xcc, + 0x37, 0xf1, 0xd7, 0xd1, 0xc1, 0x0b, 0x18, 0xb5, 0x79, 0xfd, 0x24, 0xc5, 0x97, 0x90, 0x74, 0x82, + 0xfd, 0x24, 0xd5, 0x09, 0x3c, 0xea, 0xc6, 0xf9, 0x80, 0xee, 0xb3, 0xae, 0x6e, 0x72, 0xb2, 0xdf, + 0x69, 0x51, 0x21, 0x67, 0xd2, 0x4a, 0x9d, 0xcb, 0x8e, 0xc5, 0xf4, 0xaf, 0x08, 0xf6, 0xee, 0x75, + 0x70, 0x3b, 0x88, 0x51, 0x67, 0x10, 0x1f, 0xc3, 0xa6, 0x33, 0x4b, 0x9b, 0x37, 0x6e, 0x05, 0x0e, + 0xf1, 0x99, 0xb1, 0x8b, 0xcc, 0x87, 0x09, 0x0d, 0x1c, 0xe1, 0xce, 0xaf, 0x2a, 0xc9, 0x7b, 0x01, + 0x27, 0x8e, 0x7d, 0x09, 0x03, 0x13, 0x16, 0xc9, 0x90, 0x7c, 0xdc, 0xeb, 0xf8, 0x58, 0x2f, 0x14, + 0xd1, 0x48, 0xb0, 0x14, 0x1e, 0x15, 0x26, 0xbf, 0x92, 0x36, 0x0c, 0xde, 0x88, 0x2a, 0x78, 0x07, + 0xa3, 0x41, 0x92, 0x59, 0x71, 0xae, 0xcb, 0x15, 0x4d, 0xcc, 0x50, 0xb4, 0x7c, 0xfa, 0x67, 0x04, + 0xfb, 0x63, 0xed, 0xa5, 0x9d, 0x65, 0xb9, 0xfc, 0xb7, 0xd5, 0xb3, 0x0d, 0x71, 0x69, 0x28, 0xb2, + 0xa1, 0x88, 0x4b, 0x83, 0xde, 0x4f, 0xad, 0x2a, 0xda, 0xbd, 0x13, 0x38, 0xd2, 0xab, 0xc2, 0x06, + 0x8c, 0x55, 0x85, 0x15, 0x58, 0x64, 0x39, 0x6d, 0x92, 0x91, 0x40, 0x92, 0x10, 0xbf, 0xa4, 0x15, + 0xd2, 0x13, 0x48, 0xa2, 0xce, 0xfc, 0x86, 0x76, 0xc6, 0x48, 0xc4, 0xf3, 0x1b, 0x1c, 0x3f, 0x9f, + 0x55, 0x67, 0x98, 0xe0, 0xb0, 0x13, 0x02, 0x8b, 0x5f, 0x9a, 0xdc, 0xd4, 0x2b, 0xa1, 0x61, 0x53, + 0x03, 0x3b, 0x13, 0x63, 0x7d, 0x37, 0x84, 0xb0, 0xf7, 0x11, 0xa6, 0x40, 0xfa, 0xa2, 0xe5, 0xd9, + 0x13, 0xd8, 0xca, 0x9b, 0x29, 0x21, 0x81, 0x98, 0x04, 0xee, 0x82, 0x68, 0x81, 0x2e, 0x4b, 0x6e, + 0xca, 0x10, 0x66, 0xcb, 0xa7, 0xbf, 0xc0, 0xee, 0xfa, 0x45, 0x60, 0xcf, 0x60, 0x57, 0x61, 0x32, + 0x75, 0x56, 0x36, 0x18, 0x8f, 0x28, 0x15, 0xf7, 0x70, 0x94, 0x95, 0x1f, 0xd6, 0x64, 0xeb, 0x33, + 0x74, 0x0f, 0x4f, 0x7f, 0x86, 0x9d, 0xb5, 0x16, 0x7d, 0xb0, 0x03, 0x4f, 0x20, 0xa1, 0xe5, 0x39, + 0x31, 0x4a, 0xfb, 0xda, 0x5a, 0x72, 0xb2, 0xdb, 0xe9, 0x9e, 0xef, 0xf1, 0xab, 0xe8, 0x0a, 0xa5, + 0xaf, 0x20, 0xe9, 0x7c, 0x6b, 0xf7, 0x5b, 0xd4, 0xd9, 0x6f, 0xdd, 0xfe, 0x89, 0xd7, 0xfa, 0xe7, + 0xb7, 0xa8, 0x99, 0xb8, 0xf3, 0x76, 0x32, 0x96, 0x4e, 0xda, 0xc6, 0x00, 0xd2, 0x68, 0x60, 0x61, + 0xb4, 0xf2, 0x78, 0x51, 0xeb, 0x10, 0x5b, 0x1e, 0x2b, 0x7a, 0x25, 0x57, 0x56, 0xe9, 0x79, 0xc8, + 0x70, 0xc3, 0xb2, 0x43, 0x48, 0xa6, 0x2b, 0x2f, 0xdd, 0x44, 0xda, 0x0b, 0x99, 0xd3, 0x90, 0xf4, + 0x45, 0x17, 0xc2, 0xb7, 0x94, 0xa9, 0x1c, 0x35, 0x57, 0x5f, 0x10, 0x9d, 0x4a, 0xd8, 0xba, 0x73, + 0x09, 0x1e, 0x74, 0x68, 0x1f, 0xfa, 0x73, 0x14, 0x68, 0x16, 0x08, 0x31, 0x58, 0x91, 0xac, 0x28, + 0x14, 0x86, 0x91, 0x95, 0x64, 0x00, 0x7f, 0x00, 0xa8, 0x22, 0xeb, 0x78, 0xfa, 0x16, 0x36, 0xeb, + 0x5d, 0x8e, 0xf6, 0x69, 0x88, 0x83, 0x7d, 0x1a, 0x61, 0x06, 0xbd, 0xcb, 0xcc, 0x16, 0x64, 0xbe, + 0x27, 0x88, 0x46, 0xcc, 0x99, 0x59, 0xbd, 0x04, 0x7a, 0x82, 0xe8, 0xf4, 0xef, 0x08, 0x06, 0x13, + 0x6b, 0x72, 0xe9, 0xe8, 0x17, 0xa1, 0xdd, 0xdb, 0xc1, 0xd8, 0x2d, 0x80, 0x23, 0x32, 0x2e, 0x82, + 0xbb, 0xf1, 0x98, 0xac, 0x61, 0x98, 0x21, 0x67, 0x44, 0x63, 0x54, 0xe4, 0x5d, 0xd8, 0x27, 0x35, + 0xc3, 0x8e, 0x60, 0xe7, 0xf5, 0x5d, 0xef, 0xc3, 0x0f, 0xc8, 0x3a, 0x8c, 0x65, 0xfa, 0x41, 0xda, + 0x85, 0x6a, 0x0e, 0xfc, 0x50, 0xb4, 0x3c, 0xbe, 0xf7, 0x1a, 0x6f, 0xdc, 0xa0, 0xbe, 0x71, 0x48, + 0x23, 0x86, 0x0b, 0x9e, 0x0f, 0x6b, 0xec, 0x34, 0x5c, 0xce, 0x9f, 0xc2, 0xe5, 0x0c, 0x03, 0x1a, + 0xd8, 0xe9, 0x26, 0x4d, 0xce, 0xf3, 0x7f, 0x02, 0x00, 0x00, 0xff, 0xff, 0x18, 0x7b, 0xa1, 0x75, + 0xb9, 0x09, 0x00, 0x00, } diff --git a/api/descriptions.proto b/api/descriptions.proto index 2ecde84e..0fa5522b 100644 --- a/api/descriptions.proto +++ b/api/descriptions.proto @@ -2,6 +2,9 @@ syntax = "proto3"; package api; +// To generate description.pb.go, run: +// $ protoc --go_out=. descriptions.proto + message SandboxConfig { string hostname = 1; repeated string dns = 2; @@ -54,11 +57,12 @@ message InterfaceDescription { string id = 1; bool lo = 2; string bridge = 3; - string ip = 4; + repeated string ip = 4; string mac = 5; - string gw = 6; - string tapName = 7; - string options = 8; + uint64 mtu = 6; + string gw = 7; + string tapName = 8; + string options = 9; } message PortDescription { diff --git a/hyperstart/api/json/constants.go b/hyperstart/api/json/constants.go index f39a938b..875554e4 100644 --- a/hyperstart/api/json/constants.go +++ b/hyperstart/api/json/constants.go @@ -29,6 +29,7 @@ const ( INIT_REMOVECONTAINER INIT_PROCESSASYNCEVENT INIT_SIGNALPROCESS + INIT_DELETEINTERFACE // 25 ) // "hyperstart" is the special container ID for adding processes. diff --git a/hyperstart/api/json/spec.go b/hyperstart/api/json/spec.go index ba9ba498..dc5f3ecb 100644 --- a/hyperstart/api/json/spec.go +++ b/hyperstart/api/json/spec.go @@ -81,6 +81,7 @@ type NetworkInf struct { Device string `json:"device"` IpAddress string `json:"ipAddress"` NetMask string `json:"netMask"` + Mtu uint64 `json:"mtu"` } type Route struct { diff --git a/hyperstart/libhyperstart/hyperstart.go b/hyperstart/libhyperstart/hyperstart.go index 7b834afa..bafcd3df 100644 --- a/hyperstart/libhyperstart/hyperstart.go +++ b/hyperstart/libhyperstart/hyperstart.go @@ -27,6 +27,7 @@ type Hyperstart interface { WriteFile(container, path string, data []byte) error ReadFile(container, path string) ([]byte, error) AddRoute(r []hyperstartapi.Route) error - UpdateInterface(dev, ip, mask string) error + AddInterface(inf *hyperstartapi.NetworkInf) error + DeleteInterface(inf *hyperstartapi.NetworkInf) error OnlineCpuMem() error } diff --git a/hyperstart/libhyperstart/json.go b/hyperstart/libhyperstart/json.go index 604904b3..3de04a01 100644 --- a/hyperstart/libhyperstart/json.go +++ b/hyperstart/libhyperstart/json.go @@ -646,12 +646,12 @@ func (h *jsonBasedHyperstart) AddRoute(r []hyperstartapi.Route) error { return h.hyperstartCommand(hyperstartapi.INIT_SETUPROUTE, hyperstartapi.Routes{Routes: r}) } -func (h *jsonBasedHyperstart) UpdateInterface(dev, ip, mask string) error { - return h.hyperstartCommand(hyperstartapi.INIT_SETUPINTERFACE, hyperstartapi.NetworkInf{ - Device: dev, - IpAddress: ip, - NetMask: mask, - }) +func (h *jsonBasedHyperstart) AddInterface(inf *hyperstartapi.NetworkInf) error { + return h.hyperstartCommand(hyperstartapi.INIT_SETUPINTERFACE, inf) +} + +func (h *jsonBasedHyperstart) DeleteInterface(inf *hyperstartapi.NetworkInf) error { + return h.hyperstartCommand(hyperstartapi.INIT_DELETEINTERFACE, inf) } func (h *jsonBasedHyperstart) TtyWinResize4242(container, process string, row, col uint16) error { diff --git a/hypervisor/context.go b/hypervisor/context.go index 69bd6d6d..f94021ec 100644 --- a/hypervisor/context.go +++ b/hypervisor/context.go @@ -270,6 +270,27 @@ func (ctx *VmContext) RemoveInterface(id string, result chan api.Result) { ctx.networks.removeInterface(id, result) } +func (ctx *VmContext) AddIPAddr(id, ip string) error { + ctx.lock.Lock() + defer ctx.lock.Unlock() + + return ctx.networks.AddIPAddr(id, ip) +} + +func (ctx *VmContext) DeleteIPAddr(id, ip string) error { + ctx.lock.Lock() + defer ctx.lock.Unlock() + + return ctx.networks.DeleteIPAddr(id, ip) +} + +func (ctx *VmContext) UpdateMtu(id string, mtu uint64) error { + ctx.lock.Lock() + defer ctx.lock.Unlock() + + return ctx.networks.UpdateMtu(id, mtu) +} + func (ctx *VmContext) validateContainer(c *api.ContainerDescription) error { for vn, vr := range c.Volumes { if _, ok := ctx.volumes[vn]; !ok { diff --git a/hypervisor/driver.go b/hypervisor/driver.go index 16508a62..0dcc7c76 100644 --- a/hypervisor/driver.go +++ b/hypervisor/driver.go @@ -44,6 +44,7 @@ type GuestNicInfo struct { Ipaddr string Index int Busaddr int + Mtu uint64 } type HypervisorDriver interface { diff --git a/hypervisor/events.go b/hypervisor/events.go index e2986047..9c4f1904 100644 --- a/hypervisor/events.go +++ b/hypervisor/events.go @@ -2,6 +2,8 @@ package hypervisor import ( "os" + + "github.com/hyperhq/runv/api" ) type VmEvent interface { @@ -56,14 +58,15 @@ type InterfaceCreated struct { Id string //user specified in (ref api.InterfaceDescription: a user identifier of interface, user may use this to specify a nic, normally you can use IPAddr as an Id, however, in some driver (probably vbox?), user may not specify the IPAddr.) Index int PCIAddr int + Mtu uint64 Fd *os.File Bridge string HostDevice string DeviceName string MacAddr string - IpAddr string - NetMask string + IpAddr []string RouteTable []*RouteRule + Desc *api.InterfaceDescription } type RouteRule struct { diff --git a/hypervisor/network.go b/hypervisor/network.go index 7388f740..97dcab16 100644 --- a/hypervisor/network.go +++ b/hypervisor/network.go @@ -2,7 +2,6 @@ package hypervisor import ( "fmt" - "net" "sync" "github.com/hyperhq/runv/api" @@ -84,7 +83,7 @@ func (nc *NetworkContext) freeSlot(slot int) { func (nc *NetworkContext) addInterface(inf *api.InterfaceDescription, result chan api.Result) { if inf.Lo { - if inf.Ip == "" { + if inf.Ip == nil || len(inf.Ip) == 0 { estr := fmt.Sprintf("creating an interface without an IP address: %#v", inf) nc.sandbox.Log(ERROR, estr) result <- NewSpecError(inf.Id, estr) @@ -92,11 +91,12 @@ func (nc *NetworkContext) addInterface(inf *api.InterfaceDescription, result cha } i := &InterfaceCreated{ Id: inf.Id, + Mtu: inf.Mtu, DeviceName: DEFAULT_LO_DEVICE_NAME, IpAddr: inf.Ip, - NetMask: "255.255.255.255", + Desc: inf, } - nc.lo[inf.Ip] = i + nc.lo[inf.Ip[0]] = i nc.idMap[inf.Id] = i result <- &api.ResultBase{ @@ -158,7 +158,7 @@ func (nc *NetworkContext) removeInterface(id string, result chan api.Result) { return } else if inf.HostDevice == "" { // a virtual interface delete(nc.idMap, id) - delete(nc.lo, inf.IpAddr) + delete(nc.lo, inf.IpAddr[0]) result <- api.NewResultBase(id, true, "") return } else { @@ -242,6 +242,7 @@ func (nc *NetworkContext) configureInterface(index, pciAddr int, name string, in result <- &DeviceFailed{Session: created} return } + created.Desc = inf h := &HostNicInfo{ Id: created.Id, @@ -253,9 +254,10 @@ func (nc *NetworkContext) configureInterface(index, pciAddr int, name string, in } g := &GuestNicInfo{ Device: created.DeviceName, - Ipaddr: created.IpAddr, + Ipaddr: created.IpAddr[0], Index: created.Index, Busaddr: created.PCIAddr, + Mtu: created.Mtu, } nc.eth[index] = created @@ -282,13 +284,61 @@ func (nc *NetworkContext) getInterface(id string) *InterfaceCreated { return nil } +func (nc *NetworkContext) AddIPAddr(id, ip string) error { + nc.slotLock.RLock() + defer nc.slotLock.RUnlock() + inf, ok := nc.idMap[id] + if !ok { + return ErrNoSuchInf + } + inf.IpAddr = append(inf.IpAddr, ip) + inf.Desc.Ip = append(inf.Desc.Ip, ip) + return nil +} + +func (nc *NetworkContext) DeleteIPAddr(id, ip string) error { + nc.slotLock.RLock() + defer nc.slotLock.RUnlock() + inf, ok := nc.idMap[id] + if !ok { + return ErrNoSuchInf + } + + for i, v := range inf.IpAddr { + if v == ip { + inf.IpAddr = append(inf.IpAddr[:i], inf.IpAddr[i+1:]...) + break + } + } + + for i, v := range inf.Desc.Ip { + if v == ip { + inf.IpAddr = append(inf.Desc.Ip[:i], inf.Desc.Ip[i+1:]...) + break + } + } + return nil +} + +func (nc *NetworkContext) UpdateMtu(id string, mtu uint64) error { + nc.slotLock.RLock() + defer nc.slotLock.RUnlock() + inf, ok := nc.idMap[id] + if !ok { + return ErrNoSuchInf + } + inf.Mtu = mtu + inf.Desc.Mtu = mtu + return nil +} + func (nc *NetworkContext) getIpAddrs() []string { nc.slotLock.RLock() defer nc.slotLock.RUnlock() res := []string{} for _, inf := range nc.eth { - res = append(res, inf.IpAddr) + res = append(res, inf.IpAddr...) } return res @@ -325,13 +375,6 @@ func (nc *NetworkContext) close() { } func interfaceGot(id string, index int, pciAddr int, name string, inf *network.Settings) (*InterfaceCreated, error) { - ip, nw, err := net.ParseCIDR(fmt.Sprintf("%s/%d", inf.IPAddress, inf.IPPrefixLen)) - if err != nil { - return &InterfaceCreated{Index: index, PCIAddr: pciAddr, DeviceName: name}, err - } - var tmp []byte = nw.Mask - var mask net.IP = tmp - rt := []*RouteRule{} /* Route rule is generated automaticly on first interface, * or generated on the gateway configured interface. */ @@ -351,8 +394,8 @@ func interfaceGot(id string, index int, pciAddr int, name string, inf *network.S DeviceName: name, Fd: inf.File, MacAddr: inf.Mac, - IpAddr: ip.String(), - NetMask: mask.String(), + IpAddr: inf.IPAddress, + Mtu: inf.Mtu, RouteTable: rt, }, nil } diff --git a/hypervisor/network/network.go b/hypervisor/network/network.go index 3a805ae5..dee7f40c 100644 --- a/hypervisor/network/network.go +++ b/hypervisor/network/network.go @@ -9,19 +9,20 @@ import ( ) type Settings struct { - Mac string - IPAddress string - IPPrefixLen int - Gateway string - Bridge string - Device string - File *os.File - Automatic bool + Mac string + IPAddress []string + Mtu uint64 + Gateway string + Bridge string + Device string + File *os.File + Automatic bool } const ( DefaultBridgeIface = "hyper0" DefaultBridgeIP = "192.168.123.0/24" + DefaultMtu = 1450 ) var ( diff --git a/hypervisor/network/network_linux.go b/hypervisor/network/network_linux.go index 47fa0022..7d2bde34 100644 --- a/hypervisor/network/network_linux.go +++ b/hypervisor/network/network_linux.go @@ -1049,8 +1049,6 @@ func AllocateAddr(requestedIP string) (*Settings, error) { return nil, err } - maskSize, _ := BridgeIPv4Net.Mask.Size() - mac, err := GenRandomMac() if err != nil { glog.Errorf("Generate Random Mac address failed") @@ -1058,36 +1056,20 @@ func AllocateAddr(requestedIP string) (*Settings, error) { } return &Settings{ - Mac: mac, - IPAddress: ip.String(), - Gateway: BridgeIPv4Net.IP.String(), - Bridge: BridgeIface, - IPPrefixLen: maskSize, - Device: "", - File: nil, - Automatic: true, + Mac: mac, + IPAddress: []string{ip.String()}, + Mtu: DefaultMtu, + Gateway: BridgeIPv4Net.IP.String(), + Bridge: BridgeIface, + Device: "", + File: nil, + Automatic: true, }, nil } func Configure(addrOnly bool, inf *api.InterfaceDescription) (*Settings, error) { - - ip, mask, err := ipParser(inf.Ip) - if err != nil { - glog.Errorf("Parse config IP failed %s", err) - return nil, err - } - - maskSize, _ := mask.Size() - - /* TODO: Move port maps out of the plugging procedure - err = SetupPortMaps(ip.String(), maps) - if err != nil { - glog.Errorf("Setup Port Map failed %s", err) - return nil, err - } - */ - mac := inf.Mac + var err error if mac == "" { mac, err = GenRandomMac() if err != nil { @@ -1098,14 +1080,14 @@ func Configure(addrOnly bool, inf *api.InterfaceDescription) (*Settings, error) if addrOnly { return &Settings{ - Mac: mac, - IPAddress: ip.String(), - Gateway: inf.Gw, - Bridge: inf.Bridge, - IPPrefixLen: maskSize, - Device: inf.TapName, - File: nil, - Automatic: false, + Mac: mac, + IPAddress: inf.Ip, + Mtu: inf.Mtu, + Gateway: inf.Gw, + Bridge: inf.Bridge, + Device: inf.TapName, + File: nil, + Automatic: false, }, nil } @@ -1115,14 +1097,14 @@ func Configure(addrOnly bool, inf *api.InterfaceDescription) (*Settings, error) } return &Settings{ - Mac: mac, - IPAddress: ip.String(), - Gateway: inf.Gw, - Bridge: inf.Bridge, - IPPrefixLen: maskSize, - Device: device, - File: tapFile, - Automatic: false, + Mac: mac, + IPAddress: inf.Ip, + Mtu: inf.Mtu, + Gateway: inf.Gw, + Bridge: inf.Bridge, + Device: device, + File: tapFile, + Automatic: false, }, nil } diff --git a/hypervisor/persistence.go b/hypervisor/persistence.go index b8f0c28a..0638bd35 100644 --- a/hypervisor/persistence.go +++ b/hypervisor/persistence.go @@ -109,7 +109,7 @@ func (ctx *VmContext) dump() (*PersistInfo, error) { Index: nic.Index, PciAddr: nic.PCIAddr, DeviceName: nic.DeviceName, - IpAddr: nic.IpAddr, + IpAddr: nic.IpAddr[0], } nid++ } @@ -120,7 +120,7 @@ func (ctx *VmContext) dump() (*PersistInfo, error) { Index: nic.Index, PciAddr: nic.PCIAddr, DeviceName: nic.DeviceName, - IpAddr: nic.IpAddr, + IpAddr: nic.IpAddr[0], } nid++ } @@ -207,7 +207,7 @@ func (nc *NetworkContext) load(pinfo *PersistInfo) { Index: pi.Index, PCIAddr: pi.PciAddr, DeviceName: pi.DeviceName, - IpAddr: pi.IpAddr, + IpAddr: []string{pi.IpAddr}, } // if empty, may be old data, generate one for compatibility. if ifc.Id == "" { diff --git a/hypervisor/vbox/network.go b/hypervisor/vbox/network.go index 4358675a..fe478a93 100644 --- a/hypervisor/vbox/network.go +++ b/hypervisor/vbox/network.go @@ -61,22 +61,13 @@ func (vd *VBoxDriver) InitNetwork(bIface, bIP string, disableIptables bool) erro } func (vc *VBoxContext) ConfigureNetwork(config *api.InterfaceDescription) (*network.Settings, error) { - ip, ipnet, err := net.ParseCIDR(config.Ip) - if err != nil { - glog.Errorf("Parse interface IP failed %s", err) - return nil, err - } - - maskSize, _ := ipnet.Mask.Size() - return &network.Settings{ - Mac: config.Mac, - IPAddress: ip.String(), - Gateway: config.Gw, - Bridge: "", - IPPrefixLen: maskSize, - Device: "", - File: nil, + Mac: config.Mac, + IPAddress: config.Ip, + Gateway: config.Gw, + Bridge: "", + Device: "", + File: nil, }, nil } diff --git a/hypervisor/vm.go b/hypervisor/vm.go index a87de686..83c2a271 100644 --- a/hypervisor/vm.go +++ b/hypervisor/vm.go @@ -19,6 +19,10 @@ import ( "github.com/hyperhq/runv/lib/utils" ) +var ( + ErrNoSuchInf error = errors.New("No such interface") +) + type Vm struct { Id string @@ -296,6 +300,10 @@ func (vm *Vm) AssociateContainer(container string) (alive bool, err error) { func (vm *Vm) AddRoute() error { routes := vm.ctx.networks.getRoutes() + return vm.AddHyperstartRoutes(routes) +} + +func (vm *Vm) AddHyperstartRoutes(routes []hyperstartapi.Route) error { return vm.ctx.hyperstart.AddRoute(routes) } @@ -315,10 +323,22 @@ func (vm *Vm) AddNic(info *api.InterfaceDescription) error { if vm.ctx.LogLevel(TRACE) { vm.Log(TRACE, "finial vmSpec.Interface is %#v", vm.ctx.networks.getInterface(info.Id)) } - return vm.ctx.updateInterface(info.Id) + return vm.ctx.hyperstartAddInterface(info.Id) +} + +func (vm *Vm) GetNic(id string) (*InterfaceCreated, error) { + inf := vm.ctx.networks.getInterface(id) + if inf == nil { + return nil, ErrNoSuchInf + } + return inf, nil } func (vm *Vm) DeleteNic(id string) error { + if err := vm.ctx.hyperstartDeleteInterface(id); err != nil { + return fmt.Errorf("hyperstart delete interface error: %v", err) + } + client := make(chan api.Result, 1) vm.ctx.RemoveInterface(id, client) @@ -333,6 +353,30 @@ func (vm *Vm) DeleteNic(id string) error { return nil } +func (vm *Vm) AddIPAddr(id, ip string) error { + if err := vm.ctx.AddIPAddr(id, ip); err != nil { + return err + } + + return vm.ctx.hyperstartAddIPAddr(id, ip) +} + +func (vm *Vm) DeleteIPAddr(id, ip string) error { + if err := vm.ctx.DeleteIPAddr(id, ip); err != nil { + return err + } + + return vm.ctx.hyperstartDeleteIPAddr(id, ip) +} + +func (vm *Vm) UpdateMtu(id string, mtu uint64) error { + if err := vm.ctx.UpdateMtu(id, mtu); err != nil { + return err + } + + return vm.ctx.hyperstartUpdateMtu(id, mtu) +} + // TODO: deprecated api, it will be removed after the hyper.git updated func (vm *Vm) AddCpu(totalCpu int) error { return vm.SetCpus(totalCpu) diff --git a/hypervisor/vm_states.go b/hypervisor/vm_states.go index 02e3386b..a3d4943f 100644 --- a/hypervisor/vm_states.go +++ b/hypervisor/vm_states.go @@ -4,11 +4,13 @@ import ( "errors" "fmt" "io" + "net" "strings" "sync" "time" "github.com/hyperhq/hypercontainer-utils/hlog" + hyperstartapi "github.com/hyperhq/runv/hyperstart/api/json" "github.com/hyperhq/runv/hypervisor/types" ) @@ -106,11 +108,78 @@ func (ctx *VmContext) restoreContainer(id string) (alive bool, err error) { return true, nil } -func (ctx *VmContext) updateInterface(id string) error { +func (ctx *VmContext) hyperstartAddInterface(id string) error { if inf := ctx.networks.getInterface(id); inf == nil { return fmt.Errorf("can't find interface whose ID is %s", id) } else { - return ctx.hyperstart.UpdateInterface(inf.DeviceName, inf.IpAddr, inf.NetMask) + for _, addr := range inf.IpAddr { + ip, net, err := net.ParseCIDR(addr) + if err != nil { + return fmt.Errorf("can't parse ip %q: %v", inf.IpAddr, err) + } + if err = ctx.hyperstart.AddInterface(&hyperstartapi.NetworkInf{ + Device: inf.DeviceName, + IpAddress: ip.String(), + NetMask: fmt.Sprintf("%d.%d.%d.%d", net.Mask[0], net.Mask[1], net.Mask[2], net.Mask[3]), + Mtu: inf.Mtu, + }); err != nil { + return err + } + } + return nil + } +} + +func (ctx *VmContext) hyperstartDeleteInterface(id string) error { + if inf := ctx.networks.getInterface(id); inf == nil { + return fmt.Errorf("can't find interface whose ID is %s", id) + } else { + return ctx.hyperstart.DeleteInterface(&hyperstartapi.NetworkInf{ + Device: inf.DeviceName, + }) + } +} + +func (ctx *VmContext) hyperstartAddIPAddr(id, ipaddr string) error { + if inf := ctx.networks.getInterface(id); inf == nil { + return fmt.Errorf("can't find interface whose ID is %s", id) + } else { + ip, net, err := net.ParseCIDR(ipaddr) + if err != nil { + return fmt.Errorf("can't parse ip %q: %v", ipaddr, err) + } + return ctx.hyperstart.AddInterface(&hyperstartapi.NetworkInf{ + Device: inf.DeviceName, + IpAddress: ip.String(), + NetMask: fmt.Sprintf("%d.%d.%d.%d", net.Mask[0], net.Mask[1], net.Mask[2], net.Mask[3]), + }) + } +} + +func (ctx *VmContext) hyperstartDeleteIPAddr(id, ipaddr string) error { + if inf := ctx.networks.getInterface(id); inf == nil { + return fmt.Errorf("can't find interface whose ID is %s", id) + } else { + ip, net, err := net.ParseCIDR(ipaddr) + if err != nil { + return fmt.Errorf("can't parse ip %q: %v", ipaddr, err) + } + return ctx.hyperstart.AddInterface(&hyperstartapi.NetworkInf{ + Device: inf.DeviceName, + IpAddress: fmt.Sprintf("-%s", ip.String()), // start with '-' means delete instead of add + NetMask: fmt.Sprintf("%d.%d.%d.%d", net.Mask[0], net.Mask[1], net.Mask[2], net.Mask[3]), + }) + } +} + +func (ctx *VmContext) hyperstartUpdateMtu(id string, mtu uint64) error { + if inf := ctx.networks.getInterface(id); inf == nil { + return fmt.Errorf("can't find interface whose ID is %s", id) + } else { + return ctx.hyperstart.AddInterface(&hyperstartapi.NetworkInf{ + Device: inf.DeviceName, + Mtu: mtu, + }) } } diff --git a/supervisor/hyperpod.go b/supervisor/hyperpod.go index b8e33fb6..e05690b4 100644 --- a/supervisor/hyperpod.go +++ b/supervisor/hyperpod.go @@ -15,6 +15,7 @@ import ( "github.com/golang/glog" "github.com/hyperhq/runv/api" "github.com/hyperhq/runv/factory" + //hyperstartapi "github.com/hyperhq/runv/hyperstart/api/json" "github.com/hyperhq/runv/hypervisor" "github.com/kardianos/osext" "github.com/opencontainers/runtime-spec/specs-go" @@ -35,9 +36,8 @@ type NetlinkUpdate struct { Addr netlink.AddrUpdate // RouteUpdate is used to pass information back from RouteSubscribe() Route netlink.RouteUpdate - // Veth is used to pass information back from LinkSubscribe(). // We only support veth link at present. - Veth *netlink.Veth + Link netlink.LinkUpdate // UpdateType indicates which part of the netlink information has been changed. UpdateType NetlinkUpdateType @@ -53,12 +53,16 @@ type HyperPod struct { sv *Supervisor nslistener *nsListener + + // networkInited indicates whether the network namespace has already been set or not. + networkInited bool } type InterfaceInfo struct { Index int PeerIndex int Ip string + Mac string } type nsListener struct { @@ -133,7 +137,7 @@ func GetBridgeFromIndex(idx int) (string, string, error) { func (hp *HyperPod) initPodNetwork(c *Container) error { // Only start first container will setup netns - if len(hp.Containers) > 1 { + if hp.networkInited == true { return nil } @@ -184,10 +188,11 @@ func (hp *HyperPod) initPodNetwork(c *Container) error { nicId := strconv.Itoa(info.Index) conf := &api.InterfaceDescription{ - Id: nicId, //ip as an id + Id: nicId, // link index as an id Lo: false, Bridge: bridge, - Ip: info.Ip, + Ip: []string{info.Ip}, + Mac: info.Mac, Options: options, } @@ -212,11 +217,112 @@ func (hp *HyperPod) initPodNetwork(c *Container) error { return err } + hp.networkInited = true + go hp.nsListenerStrap() return nil } +func (hp *HyperPod) handleLinkUpdate(update *NetlinkUpdate) { + link := update.Link + + stringid := fmt.Sprintf("%d", link.Attrs().Index) + infCreated, err := hp.vm.GetNic(stringid) + if err != nil { + glog.Error(err) + return + } + + // only update mtu when link is up + if link.IfInfomsg.Flags&syscall.IFF_UP == 1 { + if infCreated.Mtu != uint64(link.Attrs().MTU) { + glog.V(3).Infof("[netns] MTU changed from %d to %d\n", infCreated.Mtu, link.Attrs().MTU) + if err := hp.vm.UpdateMtu(stringid, uint64(link.Attrs().MTU)); err != nil { + glog.Error("failed to set mtu to %d: %v", uint64(link.Attrs().MTU), err) + } + } + } +} + +func (hp *HyperPod) handleAddrUpdate(update *NetlinkUpdate) { + link := update.Link + stringid := fmt.Sprintf("%d", link.Attrs().Index) + infCreated, getNicErr := hp.vm.GetNic(stringid) + + // true=added false=deleted + if update.Addr.NewAddr == false { + // if we want to delete ip (or remove the whole nic), make sure it exist + if getNicErr != nil { + glog.Errorf("failed to get nic %q: %v", stringid, getNicErr) + return + } + + if len(infCreated.IpAddr) <= 1 { + // if this is last ip, remove the whole nic + if err := hp.vm.DeleteNic(stringid); err != nil { + glog.Errorf("[netns] delete nic failed: %v", err) + return + } + glog.V(3).Infof("interface %s deleted", stringid) + } else { + // remove single ip address + if err := hp.vm.DeleteIPAddr(stringid, update.Addr.LinkAddress.String()); err != nil { + glog.Errorf("[netns] delete ip address failed: %v", err) + return + } + glog.V(3).Infof("IP(%q) of nic %s deleted", update.Addr.LinkAddress.String(), stringid) + } + return + } + + // This is just a sanity check. + // + // The link should be the one which the address on it has been changed. + if link.Attrs().Index != update.Addr.LinkIndex { + glog.Errorf("Get the wrong link with ID %d, expect %d", link.Attrs().Index, update.Addr.LinkIndex) + return + } + + if getNicErr != nil { + if getNicErr != hypervisor.ErrNoSuchInf { + glog.Error(getNicErr) + return + } + + // add a new interface + bridge, options, err := GetBridgeFromIndex(link.Attrs().ParentIndex) + if err != nil { + glog.Error(err) + return + } + + inf := &api.InterfaceDescription{ + Id: strconv.Itoa(link.Attrs().Index), + Lo: false, + Bridge: bridge, + Ip: []string{update.Addr.LinkAddress.String()}, + Mac: link.Attrs().HardwareAddr.String(), + Mtu: uint64(link.Attrs().MTU), + Options: options, + } + + err = hp.vm.AddNic(inf) + if err != nil { + glog.Error(err) + return + } + } else { + // interface exists, we only want to add a new IP address + if err := hp.vm.AddIPAddr(stringid, update.Addr.LinkAddress.String()); err != nil { + glog.Error("failed to add ip address %q to network: %v", err) + return + } + } +} + +func (hp *HyperPod) handleRouteUpdate(update *NetlinkUpdate) {} + func (hp *HyperPod) nsListenerStrap() { listener := hp.nslistener @@ -236,63 +342,14 @@ func (hp *HyperPod) nsListenerStrap() { glog.V(3).Infof("network namespace information of %s has been changed", update.UpdateType) switch update.UpdateType { + case UpdateTypeRoute: case UpdateTypeLink: - link := update.Veth - if link.Attrs().ParentIndex == 0 { - glog.V(3).Infof("The deleted link: %s", link) - err = hp.vm.DeleteNic(strconv.Itoa(link.Attrs().Index)) - if err != nil { - glog.Error(err) - continue - } - - } else { - glog.V(3).Infof("The changed link: %s", link) - } + glog.V(3).Infof("[netns] link has been changed: %#v", update.Link) + hp.handleLinkUpdate(&update) case UpdateTypeAddr: - glog.V(3).Infof("The changed address: %s", update.Addr) - - link := update.Veth - - // If there is a delete operation upon an link, it will also trigger - // the address change event which the link will be NIL since it has - // already been deleted before the address change event be triggered. - if link == nil { - glog.V(3).Info("Link for this address has already been deleted.") - continue - } - - // This is just a sanity check. - // - // The link should be the one which the address on it has been changed. - if link.Attrs().Index != update.Addr.LinkIndex { - glog.Errorf("Get the wrong link with ID %d, expect %d", link.Attrs().Index, update.Addr.LinkIndex) - continue - } - - bridge, options, err := GetBridgeFromIndex(link.Attrs().ParentIndex) - if err != nil { - glog.Error(err) - continue - } - - inf := &api.InterfaceDescription{ - Id: strconv.Itoa(link.Attrs().Index), - Lo: false, - Bridge: bridge, - Ip: update.Addr.LinkAddress.String(), - Options: options, - } - - err = hp.vm.AddNic(inf) - if err != nil { - glog.Error(err) - continue - } - - case UpdateTypeRoute: - + glog.V(3).Infof("[netns] address has been changed: %#v", update.Addr) + hp.handleAddrUpdate(&update) } } } @@ -344,6 +401,7 @@ func (hp *HyperPod) startNsListener() (err error) { enc := gob.NewEncoder(parentPipe) dec := gob.NewDecoder(parentPipe) + gob.Register(&netlink.Veth{}) hp.nslistener = &nsListener{ enc: enc, diff --git a/supervisor/proxy/nslistener.go b/supervisor/proxy/nslistener.go index b79cac13..cb9419ab 100644 --- a/supervisor/proxy/nslistener.go +++ b/supervisor/proxy/nslistener.go @@ -18,6 +18,12 @@ func init() { } func setupNsListener() { + // FOR DEBUG PURPOSE + // If you need to debug nslistener, uncomment these lines + // ==============DEBUG BEGIN===================== + // flag.CommandLine.Parse([]string{"-v", "3", "--log_dir", "/tmp/", "--alsologtostderr"}) + // =============DEBUG END======================= + runtime.LockOSThread() defer runtime.UnlockOSThread() @@ -30,6 +36,7 @@ func setupNsListener() { childPipe := os.NewFile(uintptr(3), "child") enc := gob.NewEncoder(childPipe) dec := gob.NewDecoder(childPipe) + gob.Register(&netlink.Veth{}) /* notify containerd to execute prestart hooks */ if err := enc.Encode("init"); err != nil { @@ -95,6 +102,7 @@ func collectionInterfaceInfo() []supervisor.InterfaceInfo { continue } + mac := link.Attrs().HardwareAddr addrs, err := netlink.AddrList(link, netlink.FAMILY_V4) if err != nil { glog.Error(err) @@ -104,6 +112,7 @@ func collectionInterfaceInfo() []supervisor.InterfaceInfo { for _, addr := range addrs { info := supervisor.InterfaceInfo{ Ip: addr.IPNet.String(), + Mac: mac.String(), Index: link.Attrs().Index, PeerIndex: link.Attrs().ParentIndex, } @@ -120,7 +129,6 @@ func collectionInterfaceInfo() []supervisor.InterfaceInfo { // This function should be put into the main process or somewhere that can be // use to init the network namespace trap. func setupNetworkNsTrap(netNs2Containerd func(supervisor.NetlinkUpdate)) { - // Subscribe for links change event chLink := make(chan netlink.LinkUpdate) doneLink := make(chan struct{}) @@ -159,39 +167,41 @@ func setupNetworkNsTrap(netNs2Containerd func(supervisor.NetlinkUpdate)) { // Link specific func handleLink(update netlink.LinkUpdate, callback func(supervisor.NetlinkUpdate)) { + glog.V(3).Infof("[nslistener] link changed: %#v", update) if update.IfInfomsg.Flags&syscall.IFF_UP == 1 { - fmt.Printf("[Link device up]\tupdateLink is:%+v, flag is:0x%x\n", update.Link.Attrs(), update.IfInfomsg.Flags) + glog.V(3).Infof("[Link device up]\tupdateLink is:%+v, flag is:0x%x\n", update.Link.Attrs(), update.IfInfomsg.Flags) } else { if update.Link.Attrs().ParentIndex == 0 { - fmt.Printf("[Link device !up][Deleted]\tupdateLink is:%+v, flag is:0x%x\n", update.Link.Attrs(), update.IfInfomsg.Flags) + glog.V(3).Infof("[Link device !up][Deleted]\tupdateLink is:%+v, flag is:0x%x\n", update.Link.Attrs(), update.IfInfomsg.Flags) } else { - fmt.Printf("[Link device !up]\tupdateLink is:%+v, flag is:0x%x\n", update.Link.Attrs(), update.IfInfomsg.Flags) + glog.V(3).Infof("[Link device !up]\tupdateLink is:%+v, flag is:0x%x\n", update.Link.Attrs(), update.IfInfomsg.Flags) } } netlinkUpdate := supervisor.NetlinkUpdate{} netlinkUpdate.UpdateType = supervisor.UpdateTypeLink + netlinkUpdate.Link = update // We would like to only handle the veth pair link at present. - if veth, ok := (update.Link).(*netlink.Veth); ok { - netlinkUpdate.Veth = veth + if _, ok := (update.Link).(*netlink.Veth); ok { callback(netlinkUpdate) } } // Address specific func handleAddr(update netlink.AddrUpdate, callback func(supervisor.NetlinkUpdate)) { + glog.V(3).Infof("[nslistener] address changed: %#v", update) if update.NewAddr { - fmt.Printf("[Add a address]") + glog.V(3).Infof("[Add a address]") } else { - fmt.Printf("[Delete a address]") + glog.V(3).Infof("[Delete a address]") } if update.LinkAddress.IP.To4() != nil { - fmt.Printf("[IPv4]\t%+v\n", update) + glog.V(3).Infof("[IPv4]\t%+v\n", update) } else { // We would not like to handle IPv6 at present. - fmt.Printf("[IPv6]\t%+v\n", update) + glog.V(3).Infof("[IPv6]\t%+v\n", update) return } @@ -204,7 +214,7 @@ func handleAddr(update netlink.AddrUpdate, callback func(supervisor.NetlinkUpdat } for _, link := range links { if link.Attrs().Index == update.LinkIndex && link.Type() == "veth" { - netlinkUpdate.Veth = link.(*netlink.Veth) + netlinkUpdate.Link.Link = link break } } @@ -213,19 +223,44 @@ func handleAddr(update netlink.AddrUpdate, callback func(supervisor.NetlinkUpdat // Route specific func handleRoute(update netlink.RouteUpdate, callback func(supervisor.NetlinkUpdate)) { + glog.V(3).Infof("[nslistener] route changed: %#v", update) // Route type is not a bit mask for a couple of values, but a single // unsigned int, that's why we use switch here not the "&" operator. switch update.Type { case syscall.RTM_NEWROUTE: - fmt.Printf("[Create a route]\t%+v\n", update) + glog.V(3).Infof("[Create a route]\t%+v\n", update) case syscall.RTM_DELROUTE: - fmt.Printf("[Remove a route]\t%+v\n", update) + glog.V(3).Infof("[Remove a route]\t%+v\n", update) case syscall.RTM_GETROUTE: fmt.Printf("[Receive info of a route]\t%+v\n", update) } + // we would like to only handle the gateway change + if update.Route.Dst != nil { + fmt.Println("[Route] is not a gateway, skip") + return + } + netlinkUpdate := supervisor.NetlinkUpdate{} netlinkUpdate.Route = update netlinkUpdate.UpdateType = supervisor.UpdateTypeRoute callback(netlinkUpdate) + + // delete all the routes from outside + // The reason that we need to delete all the routes is + // that if we don't do this, we will get two active network interfaces, + // and ping the IP will get duplicate ICMP replies. + routes, err := netlink.RouteList(nil, netlink.FAMILY_V4) + if err != nil { + fmt.Printf("[Get RouteList] Error:%v\n", err) + return + } + + for _, r := range routes { + if err := netlink.RouteDel(&r); err != nil { + fmt.Printf("[Remove route] Error:%v\n", err) + } else { + fmt.Printf("[Remove route] route:%+v\n", r) + } + } } From ffb5656239171181fd26ccf8e9e79612b9bb57fc Mon Sep 17 00:00:00 2001 From: Zhang Wei Date: Wed, 19 Jul 2017 11:22:48 +0800 Subject: [PATCH 2/3] Support add any route into container Enhance network functions: Support add any routes into container Signed-off-by: Zhang Wei --- hypervisor/vm.go | 8 ++++---- hypervisor/vm_states.go | 7 +++++++ supervisor/hyperpod.go | 33 +++++++++++++++++++++++++++++---- supervisor/proxy/nslistener.go | 31 +++++++------------------------ 4 files changed, 47 insertions(+), 32 deletions(-) diff --git a/hypervisor/vm.go b/hypervisor/vm.go index 83c2a271..5a312245 100644 --- a/hypervisor/vm.go +++ b/hypervisor/vm.go @@ -298,13 +298,13 @@ func (vm *Vm) AssociateContainer(container string) (alive bool, err error) { return vm.ctx.restoreContainer(container) } -func (vm *Vm) AddRoute() error { +func (vm *Vm) AddDefaultRoute() error { routes := vm.ctx.networks.getRoutes() - return vm.AddHyperstartRoutes(routes) + return vm.AddRoute(routes) } -func (vm *Vm) AddHyperstartRoutes(routes []hyperstartapi.Route) error { - return vm.ctx.hyperstart.AddRoute(routes) +func (vm *Vm) AddRoute(route []hyperstartapi.Route) error { + return vm.ctx.hyperstartAddRoute(route) } func (vm *Vm) AddNic(info *api.InterfaceDescription) error { diff --git a/hypervisor/vm_states.go b/hypervisor/vm_states.go index a3d4943f..96b1c0c9 100644 --- a/hypervisor/vm_states.go +++ b/hypervisor/vm_states.go @@ -130,6 +130,13 @@ func (ctx *VmContext) hyperstartAddInterface(id string) error { } } +func (ctx *VmContext) hyperstartAddRoute(r []hyperstartapi.Route) error { + if err := ctx.hyperstart.AddRoute(r); err != nil { + return err + } + return nil +} + func (ctx *VmContext) hyperstartDeleteInterface(id string) error { if inf := ctx.networks.getInterface(id); inf == nil { return fmt.Errorf("can't find interface whose ID is %s", id) diff --git a/supervisor/hyperpod.go b/supervisor/hyperpod.go index e05690b4..7b51f076 100644 --- a/supervisor/hyperpod.go +++ b/supervisor/hyperpod.go @@ -15,7 +15,7 @@ import ( "github.com/golang/glog" "github.com/hyperhq/runv/api" "github.com/hyperhq/runv/factory" - //hyperstartapi "github.com/hyperhq/runv/hyperstart/api/json" + hyperstartapi "github.com/hyperhq/runv/hyperstart/api/json" "github.com/hyperhq/runv/hypervisor" "github.com/kardianos/osext" "github.com/opencontainers/runtime-spec/specs-go" @@ -211,7 +211,7 @@ func (hp *HyperPod) initPodNetwork(c *Container) error { } } - err = hp.vm.AddRoute() + err = hp.vm.AddDefaultRoute() if err != nil { glog.Error(err) return err @@ -321,7 +321,30 @@ func (hp *HyperPod) handleAddrUpdate(update *NetlinkUpdate) { } } -func (hp *HyperPod) handleRouteUpdate(update *NetlinkUpdate) {} +func (hp *HyperPod) handleRouteUpdate(update *NetlinkUpdate) { + route := update.Route + if route.Type == syscall.RTM_DELROUTE || route.Type == syscall.RTM_GETROUTE { + glog.V(3).Infof("currently we only support adding new route, delete or query isn't supported") + return + } + + stringid := fmt.Sprintf("%d", route.LinkIndex) + infCreated, err := hp.vm.GetNic(stringid) + if err != nil { + glog.Errorf("failed to get information of nic with id %d", route.LinkIndex) + return + } + + r := hyperstartapi.Route{ + Dest: route.Dst.String(), + Gateway: route.Gw.String(), + Device: infCreated.DeviceName, + } + if err = hp.vm.AddRoute([]hyperstartapi.Route{r}); err != nil { + glog.Errorf("failed to add route: %v", err) + return + } +} func (hp *HyperPod) nsListenerStrap() { listener := hp.nslistener @@ -343,10 +366,11 @@ func (hp *HyperPod) nsListenerStrap() { glog.V(3).Infof("network namespace information of %s has been changed", update.UpdateType) switch update.UpdateType { case UpdateTypeRoute: + glog.V(3).Infof("[netns] route has been changed: %#v", update.Route) + hp.handleRouteUpdate(&update) case UpdateTypeLink: glog.V(3).Infof("[netns] link has been changed: %#v", update.Link) hp.handleLinkUpdate(&update) - case UpdateTypeAddr: glog.V(3).Infof("[netns] address has been changed: %#v", update.Addr) hp.handleAddrUpdate(&update) @@ -392,6 +416,7 @@ func (hp *HyperPod) startNsListener() (err error) { cmd := exec.Command(path) cmd.Args[0] = "containerd-nslistener" cmd.ExtraFiles = append(cmd.ExtraFiles, childPipe) + cmd.SysProcAttr = &syscall.SysProcAttr{Cloneflags: syscall.CLONE_NEWNET} if err = cmd.Start(); err != nil { glog.Errorf("start containerd-nslistener failed: %v", err) return err diff --git a/supervisor/proxy/nslistener.go b/supervisor/proxy/nslistener.go index cb9419ab..57a3292e 100644 --- a/supervisor/proxy/nslistener.go +++ b/supervisor/proxy/nslistener.go @@ -2,9 +2,7 @@ package proxy import ( "encoding/gob" - "fmt" "os" - "runtime" "syscall" "github.com/docker/docker/pkg/reexec" @@ -24,15 +22,6 @@ func setupNsListener() { // flag.CommandLine.Parse([]string{"-v", "3", "--log_dir", "/tmp/", "--alsologtostderr"}) // =============DEBUG END======================= - runtime.LockOSThread() - defer runtime.UnlockOSThread() - - /* create own netns */ - if err := syscall.Unshare(syscall.CLONE_NEWNET); err != nil { - glog.Error(err) - return - } - childPipe := os.NewFile(uintptr(3), "child") enc := gob.NewEncoder(childPipe) dec := gob.NewDecoder(childPipe) @@ -130,7 +119,7 @@ func collectionInterfaceInfo() []supervisor.InterfaceInfo { // use to init the network namespace trap. func setupNetworkNsTrap(netNs2Containerd func(supervisor.NetlinkUpdate)) { // Subscribe for links change event - chLink := make(chan netlink.LinkUpdate) + chLink := make(chan netlink.LinkUpdate, 64) doneLink := make(chan struct{}) defer close(doneLink) if err := netlink.LinkSubscribe(chLink, doneLink); err != nil { @@ -138,7 +127,7 @@ func setupNetworkNsTrap(netNs2Containerd func(supervisor.NetlinkUpdate)) { } // Subscribe for addresses change event - chAddr := make(chan netlink.AddrUpdate) + chAddr := make(chan netlink.AddrUpdate, 64) doneAddr := make(chan struct{}) defer close(doneAddr) if err := netlink.AddrSubscribe(chAddr, doneAddr); err != nil { @@ -146,7 +135,7 @@ func setupNetworkNsTrap(netNs2Containerd func(supervisor.NetlinkUpdate)) { } // Subscribe for route change event - chRoute := make(chan netlink.RouteUpdate) + chRoute := make(chan netlink.RouteUpdate, 64) doneRoute := make(chan struct{}) defer close(doneRoute) if err := netlink.RouteSubscribe(chRoute, doneRoute); err != nil { @@ -232,13 +221,7 @@ func handleRoute(update netlink.RouteUpdate, callback func(supervisor.NetlinkUpd case syscall.RTM_DELROUTE: glog.V(3).Infof("[Remove a route]\t%+v\n", update) case syscall.RTM_GETROUTE: - fmt.Printf("[Receive info of a route]\t%+v\n", update) - } - - // we would like to only handle the gateway change - if update.Route.Dst != nil { - fmt.Println("[Route] is not a gateway, skip") - return + glog.V(3).Infof("[Receive info of a route]\t%+v\n", update) } netlinkUpdate := supervisor.NetlinkUpdate{} @@ -252,15 +235,15 @@ func handleRoute(update netlink.RouteUpdate, callback func(supervisor.NetlinkUpd // and ping the IP will get duplicate ICMP replies. routes, err := netlink.RouteList(nil, netlink.FAMILY_V4) if err != nil { - fmt.Printf("[Get RouteList] Error:%v\n", err) + glog.Errorf("[Get RouteList] Error:%v\n", err) return } for _, r := range routes { if err := netlink.RouteDel(&r); err != nil { - fmt.Printf("[Remove route] Error:%v\n", err) + glog.Errorf("[Remove route] Error:%v\n", err) } else { - fmt.Printf("[Remove route] route:%+v\n", r) + glog.V(3).Infof("[Remove route] route:%+v\n", r) } } } From c78f19f7a108516d59960e3543a023b77bd6c753 Mon Sep 17 00:00:00 2001 From: Zhang Wei Date: Thu, 20 Jul 2017 11:35:49 +0800 Subject: [PATCH 3/3] Bugfix: fix panic when delete nic Fix bug that container panics when doing the network interface deletion work. Also modify nic delete logic a little bit, now we can delete single IP and delete link separately. Signed-off-by: Zhang Wei --- supervisor/hyperpod.go | 46 +++++++++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/supervisor/hyperpod.go b/supervisor/hyperpod.go index 7b51f076..359002d6 100644 --- a/supervisor/hyperpod.go +++ b/supervisor/hyperpod.go @@ -227,6 +227,11 @@ func (hp *HyperPod) initPodNetwork(c *Container) error { func (hp *HyperPod) handleLinkUpdate(update *NetlinkUpdate) { link := update.Link + if link.Link == nil { + glog.Errorf("Link update must have an non-nil Link param!") + return + } + stringid := fmt.Sprintf("%d", link.Attrs().Index) infCreated, err := hp.vm.GetNic(stringid) if err != nil { @@ -234,7 +239,16 @@ func (hp *HyperPod) handleLinkUpdate(update *NetlinkUpdate) { return } - // only update mtu when link is up + // ParentIndex==0 means link is to be deleted + if link.Attrs().ParentIndex == 0 { + if err := hp.vm.DeleteNic(stringid); err != nil { + glog.Errorf("[netns] delete nic failed: %v", err) + return + } + glog.V(3).Infof("interface %s deleted", stringid) + return + } + if link.IfInfomsg.Flags&syscall.IFF_UP == 1 { if infCreated.Mtu != uint64(link.Attrs().MTU) { glog.V(3).Infof("[netns] MTU changed from %d to %d\n", infCreated.Mtu, link.Attrs().MTU) @@ -247,8 +261,17 @@ func (hp *HyperPod) handleLinkUpdate(update *NetlinkUpdate) { func (hp *HyperPod) handleAddrUpdate(update *NetlinkUpdate) { link := update.Link + + // If there is a delete operation upon an link, it will also trigger + // the address change event which the link will be NIL since it has + // already been deleted before the address change event be triggered. + if link.Link == nil { + glog.V(3).Infof("Link for this address has already been deleted.") + return + } + stringid := fmt.Sprintf("%d", link.Attrs().Index) - infCreated, getNicErr := hp.vm.GetNic(stringid) + _, getNicErr := hp.vm.GetNic(stringid) // true=added false=deleted if update.Addr.NewAddr == false { @@ -258,21 +281,12 @@ func (hp *HyperPod) handleAddrUpdate(update *NetlinkUpdate) { return } - if len(infCreated.IpAddr) <= 1 { - // if this is last ip, remove the whole nic - if err := hp.vm.DeleteNic(stringid); err != nil { - glog.Errorf("[netns] delete nic failed: %v", err) - return - } - glog.V(3).Infof("interface %s deleted", stringid) - } else { - // remove single ip address - if err := hp.vm.DeleteIPAddr(stringid, update.Addr.LinkAddress.String()); err != nil { - glog.Errorf("[netns] delete ip address failed: %v", err) - return - } - glog.V(3).Infof("IP(%q) of nic %s deleted", update.Addr.LinkAddress.String(), stringid) + // remove single ip address + if err := hp.vm.DeleteIPAddr(stringid, update.Addr.LinkAddress.String()); err != nil { + glog.Errorf("[netns] delete ip address failed: %v", err) + return } + glog.V(3).Infof("IP(%q) of nic %s deleted", update.Addr.LinkAddress.String(), stringid) return }