From 7bf3da6bf3e87342161a6d3f7c9636db1a30a98c Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Mon, 17 Dec 2018 15:50:52 -0800 Subject: [PATCH 1/9] added changes in cni to support calico policy --- cni/netconfig.go | 1 + cni/network/network.go | 9 +- netlink/ip.go | 11 ++ netlink/link.go | 79 +++++++++++ netlink/netlink_test.go | 25 ++++ netlink/protocol.go | 120 +++++++++++++--- network/calico_endpointclient_linux.go | 187 +++++++++++++++++++++++++ network/endpoint.go | 9 +- network/endpoint_linux.go | 20 ++- network/network.go | 1 + network/network_linux.go | 3 +- 11 files changed, 433 insertions(+), 32 deletions(-) create mode 100644 network/calico_endpointclient_linux.go diff --git a/cni/netconfig.go b/cni/netconfig.go index db335372e0..151b302e70 100644 --- a/cni/netconfig.go +++ b/cni/netconfig.go @@ -37,6 +37,7 @@ type NetworkConfig struct { MultiTenancy bool `json:"multiTenancy,omitempty"` EnableSnatOnHost bool `json:"enableSnatOnHost,omitempty"` EnableExactMatchForPodName bool `json:"enableExactMatchForPodName,omitempty"` + EnableCalicoMode bool `json:"enableCalicoMode,omitempty"` Ipam struct { Type string `json:"type"` Environment string `json:"environment,omitempty"` diff --git a/cni/network/network.go b/cni/network/network.go index 7a4fb8acc3..711053d047 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -360,19 +360,17 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { } else { if !nwCfg.MultiTenancy { // Network already exists. + // Call into IPAM plugin to allocate an address for the endpoint. subnetPrefix := nwInfo.Subnets[0].Prefix.String() log.Printf("[cni-net] Found network %v with subnet %v.", networkId, subnetPrefix) - - // Call into IPAM plugin to allocate an address for the endpoint. nwCfg.Ipam.Subnet = subnetPrefix + result, err = plugin.DelegateAdd(nwCfg.Ipam.Type, nwCfg) if err != nil { err = plugin.Errorf("Failed to allocate address: %v", err) return err } - ipconfig := result.IPs[0] - iface := &cniTypesCurr.Interface{Name: args.IfName} result.Interfaces = append(result.Interfaces, iface) @@ -423,7 +421,8 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { // A runtime must not call ADD twice (without a corresponding DEL) for the same // (network name, container id, name of the interface inside the container) - vethName := fmt.Sprintf("%s%s%s", networkId, k8sContainerID, k8sIfName) + //vethName := fmt.Sprintf("%s%s%s", networkId, k8sContainerID, k8sIfName) + vethName := fmt.Sprintf("%s.%s", k8sNamespace, k8sPodName) setEndpointOptions(cnsNetworkConfig, epInfo, vethName) // Create the endpoint. diff --git a/netlink/ip.go b/netlink/ip.go index 2908dab871..1db1e58c4b 100644 --- a/netlink/ip.go +++ b/netlink/ip.go @@ -11,6 +11,17 @@ import ( "golang.org/x/sys/unix" ) +const ( + RT_SCOPE_UNIVERSE = 0 + RT_SCOPE_SITE = 200 + RT_SCOPE_LINK = 253 + RT_SCOPE_HOST = 254 + RT_SCOPE_NOWHERE = 255 +) +const ( + RTPROT_KERNEL = 2 +) + // GetIpAddressFamily returns the address family of an IP address. func GetIpAddressFamily(ip net.IP) int { if len(ip) <= net.IPv4len { diff --git a/netlink/link.go b/netlink/link.go index 37571fe343..210d9647d3 100644 --- a/netlink/link.go +++ b/netlink/link.go @@ -13,6 +13,41 @@ import ( "golang.org/x/sys/unix" ) +const ( + NDA_UNSPEC = iota + NDA_DST + NDA_LLADDR + NDA_CACHEINFO + NDA_PROBES + NDA_VLAN + NDA_PORT + NDA_VNI + NDA_IFINDEX + NDA_MAX = NDA_IFINDEX +) + +// Neighbor Cache Entry States. +const ( + NUD_NONE = 0x00 + NUD_INCOMPLETE = 0x01 + NUD_REACHABLE = 0x02 + NUD_STALE = 0x04 + NUD_DELAY = 0x08 + NUD_PROBE = 0x10 + NUD_FAILED = 0x20 + NUD_NOARP = 0x40 + NUD_PERMANENT = 0x80 +) + +// Neighbor Flags +const ( + NTF_USE = 0x01 + NTF_SELF = 0x02 + NTF_MASTER = 0x04 + NTF_PROXY = 0x08 + NTF_ROUTER = 0x80 +) + // Link types. const ( LINK_TYPE_BRIDGE = "bridge" @@ -31,6 +66,11 @@ const ( IPVLAN_MODE_MAX ) +const ( + ADD = iota + REMOVE +) + // Link represents a network interface. type Link interface { Info() *LinkInfo @@ -384,3 +424,42 @@ func SetLinkHairpin(bridgeName string, on bool) error { return s.sendAndWaitForAck(req) } + +// SetLinkMaster sets the master (upper) device of a network interface. +func AddorRemoveStaticArp(mode int, name string, ipaddr net.IP, mac net.HardwareAddr) error { + s, err := getSocket() + if err != nil { + return err + } + + var req *message + state := 0 + if mode == ADD { + req = newRequest(unix.RTM_NEWNEIGH, unix.NLM_F_CREATE|unix.NLM_F_REPLACE|unix.NLM_F_ACK) + state = NUD_PERMANENT + } else { + req = newRequest(unix.RTM_DELNEIGH, unix.NLM_F_ACK) + state = NUD_INCOMPLETE + } + + iface, err := net.InterfaceByName(name) + if err != nil { + return err + } + + msg := neighMsg{ + Family: uint8(unix.AF_INET), + Index: uint32(iface.Index), + State: uint16(state), + } + req.addPayload(&msg) + + ipData := ipaddr.To4() + dstData := newRtAttr(NDA_DST, ipData) + req.addPayload(dstData) + + hwData := newRtAttr(NDA_LLADDR, []byte(mac)) + req.addPayload(hwData) + + return s.sendAndWaitForAck(req) +} diff --git a/netlink/netlink_test.go b/netlink/netlink_test.go index 325eadc985..ec82351c07 100644 --- a/netlink/netlink_test.go +++ b/netlink/netlink_test.go @@ -234,3 +234,28 @@ func TestSetLinkHairpin(t *testing.T) { t.Errorf("DeleteLink failed: %+v", err) } } + +func TestAddRemoveStaticArp(t *testing.T) { + _, err := addDummyInterface(ifName) + if err != nil { + t.Errorf("addDummyInterface failed: %v", err) + } + + ip := net.ParseIP("192.168.0.2") + mac, _ := net.ParseMAC("aa:b3:4d:5e:e2:4a") + + err = AddorRemoveStaticArp(ADD, ifName, ip, mac) + if err != nil { + t.Errorf("ret val %v", err) + } + + err = AddorRemoveStaticArp(REMOVE, ifName, ip, mac) + if err != nil { + t.Errorf("ret val %v", err) + } + + err = DeleteLink(ifName) + if err != nil { + t.Errorf("DeleteLink failed: %+v", err) + } +} diff --git a/netlink/protocol.go b/netlink/protocol.go index 4acbd686d1..18aa638539 100644 --- a/netlink/protocol.go +++ b/netlink/protocol.go @@ -30,6 +30,40 @@ type serializable interface { length() int } +// +// Netlink message +// + +// Generic netlink message +type message struct { + unix.NlMsghdr + data []byte + payload []serializable +} + +// Generic netlink message attribute +type attribute struct { + unix.NlAttr + value []byte + children []serializable +} + +// Neighbor entry message strutcure +type neighMsg struct { + Family uint8 + Index uint32 + State uint16 + Flags uint8 + Type uint8 +} + +// rta attribute structure +type rtAttr struct { + unix.RtAttr + Data []byte + children []serializable +} + // Byte encoder var encoder binary.ByteOrder @@ -43,17 +77,6 @@ func initEncoder() { } } -// -// Netlink message -// - -// Generic netlink message -type message struct { - unix.NlMsghdr - data []byte - payload []serializable -} - // Creates a new netlink message. func newMessage(msgType int, flags int) *message { return &message{ @@ -127,14 +150,6 @@ func (msg *message) getAttributes(body serializable) []*attribute { // // Netlink message attribute // - -// Generic netlink message attribute -type attribute struct { - unix.NlAttr - value []byte - children []serializable -} - // Creates a new attribute. func newAttribute(attrType int, value []byte) *attribute { return &attribute{ @@ -339,3 +354,70 @@ func (rt *rtMsg) serialize() []byte { func (rt *rtMsg) length() int { return unix.SizeofRtMsg } + +// serialize neighbor message +func (msg *neighMsg) serialize() []byte { + return (*(*[unsafe.Sizeof(*msg)]byte)(unsafe.Pointer(msg)))[:] +} + +func (msg *neighMsg) length() int { + return int(unsafe.Sizeof(*msg)) +} + +// creates new rta attr message +func newRtAttr(attrType int, data []byte) *rtAttr { + return &rtAttr{ + RtAttr: unix.RtAttr{ + Type: uint16(attrType), + }, + children: []serializable{}, + Data: data, + } +} + +// align rta attributes +func rtaAlignOf(attrlen int) int { + return (attrlen + unix.RTA_ALIGNTO - 1) & ^(unix.RTA_ALIGNTO - 1) +} + +// serialize rta message +func (rta *rtAttr) serialize() []byte { + length := rta.length() + buf := make([]byte, rtaAlignOf(length)) + + next := 4 + if rta.Data != nil { + copy(buf[next:], rta.Data) + next += rtaAlignOf(len(rta.Data)) + } + if len(rta.children) > 0 { + for _, child := range rta.children { + childBuf := child.serialize() + copy(buf[next:], childBuf) + next += rtaAlignOf(len(childBuf)) + } + } + + if l := uint16(length); l != 0 { + encoder.PutUint16(buf[0:2], l) + } + encoder.PutUint16(buf[2:4], rta.Type) + return buf +} + +func (rta *rtAttr) length() int { + if len(rta.children) == 0 { + return (unix.SizeofRtAttr + len(rta.Data)) + } + + l := 0 + for _, child := range rta.children { + l += rtaAlignOf(child.length()) + } + l += unix.SizeofRtAttr + return rtaAlignOf(l + len(rta.Data)) +} + +func (rta *rtAttr) addChild(attr serializable) { + rta.children = append(rta.children, attr) +} diff --git a/network/calico_endpointclient_linux.go b/network/calico_endpointclient_linux.go new file mode 100644 index 0000000000..665b0ac774 --- /dev/null +++ b/network/calico_endpointclient_linux.go @@ -0,0 +1,187 @@ +package network + +import ( + "fmt" + "net" + + "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/netlink" + "github.com/Azure/azure-container-networking/network/epcommon" + "github.com/Azure/azure-container-networking/platform" +) + +const ( + FAKE_GW_IP = "169.254.1.1/32" + DEFAULT_GW = "0.0.0.0/0" +) + +type CalicoEndpointClient struct { + bridgeName string + hostPrimaryIfName string + hostVethName string + containerVethName string + hostPrimaryMac net.HardwareAddr + containerMac net.HardwareAddr + hostVethMac net.HardwareAddr + mode string +} + +func NewCalicoEndpointClient( + extIf *externalInterface, + hostVethName string, + containerVethName string, + mode string, +) *CalicoEndpointClient { + + client := &CalicoEndpointClient{ + bridgeName: extIf.BridgeName, + hostPrimaryIfName: extIf.Name, + hostVethName: hostVethName, + containerVethName: containerVethName, + hostPrimaryMac: extIf.MacAddress, + mode: mode, + } + + return client +} + +func (client *CalicoEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { + if err := epcommon.CreateEndpoint(client.hostVethName, client.containerVethName); err != nil { + return err + } + + containerIf, err := net.InterfaceByName(client.containerVethName) + if err != nil { + return err + } + + client.containerMac = containerIf.HardwareAddr + + hostVethIf, err := net.InterfaceByName(client.hostVethName) + if err != nil { + return err + } + + client.hostVethMac = hostVethIf.HardwareAddr + + return nil +} + +func (client *CalicoEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error { + var routeInfoList []RouteInfo + + // ip route add dev + // This route is needed for incoming packets to pod to route via hostveth + for _, ipAddr := range epInfo.IPAddresses { + var routeInfo RouteInfo + ipNet := net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(32, 32)} + log.Printf("[net] Adding route for the ip %v", ipNet.String()) + routeInfo.Dst = ipNet + routeInfoList = append(routeInfoList, routeInfo) + if err := addRoutes(client.hostVethName, routeInfoList); err != nil { + return err + } + } + return nil +} + +func (client *CalicoEndpointClient) DeleteEndpointRules(ep *endpoint) { + var routeInfoList []RouteInfo + + // ip route del dev + // Deleting the route set up for routing the incoming packets to pod + for _, ipAddr := range ep.IPAddresses { + var routeInfo RouteInfo + ipNet := net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(32, 32)} + log.Printf("[net] Deleting route for the ip %v", ipNet.String()) + routeInfo.Dst = ipNet + routeInfoList = append(routeInfoList, routeInfo) + deleteRoutes(client.hostVethName, routeInfoList) + } +} + +func (client *CalicoEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo, nsID uintptr) error { + // Move the container interface to container's network namespace. + log.Printf("[net] Setting link %v netns %v.", client.containerVethName, epInfo.NetNsPath) + if err := netlink.SetLinkNetNs(client.containerVethName, nsID); err != nil { + return err + } + + return nil +} + +func (client *CalicoEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) error { + if err := epcommon.SetupContainerInterface(client.containerVethName, epInfo.IfName); err != nil { + return err + } + + client.containerVethName = epInfo.IfName + + return nil +} + +func (client *CalicoEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { + if err := epcommon.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { + return err + } + + var routeInfo RouteInfo + var routeInfoList []RouteInfo + + // ip route add 169.254.1.1/32 dev eth0 scope link + gwIP, gwIPNet, _ := net.ParseCIDR(FAKE_GW_IP) + routeInfo.Dst = *gwIPNet + routeInfo.Scope = netlink.RT_SCOPE_LINK + routeInfoList = append(routeInfoList, routeInfo) + + // ip route add default gw 169.254.1.1 dev eth0 + routeInfo = RouteInfo{} + _, defIPNet, _ := net.ParseCIDR(DEFAULT_GW) + routeInfo.Dst = *defIPNet + routeInfo.Gw = net.ParseIP(gwIP.String()) + routeInfoList = append(routeInfoList, routeInfo) + + // add the above routes + if err := addRoutes(client.containerVethName, routeInfoList); err != nil { + return err + } + + routeInfoList = routeInfoList[:0] + + // Removing the route added by setipaddress while assigning IP to interface + for _, ipAddr := range epInfo.IPAddresses { + routeInfo = RouteInfo{} + ip, ipNet, _ := net.ParseCIDR(ipAddr.String()) + log.Printf("Removing route %v", ipNet.String()) + routeInfo.Dst = *ipNet + routeInfo.Scope = netlink.RT_SCOPE_LINK + routeInfo.Src = ip + routeInfo.Protocol = netlink.RTPROT_KERNEL + routeInfoList = append(routeInfoList, routeInfo) + } + + // delete the above route + if err := deleteRoutes(client.containerVethName, routeInfoList); err != nil { + log.Printf("Deleting route failed with err %v", err) + } + + // set arp entry for fake gateway in pod + arpCmd := fmt.Sprintf("arp -s 169.254.1.1 %v", client.hostVethMac.String()) + _, err := platform.ExecuteCommand(arpCmd) + if err != nil { + log.Printf("Setting static arp for ip 169.254.1.1 mac %v failed with error %v", client) + } + + return nil +} + +func (client *CalicoEndpointClient) DeleteEndpoints(ep *endpoint) error { + log.Printf("[net] Deleting veth pair %v %v.", ep.HostIfName, ep.IfName) + err := netlink.DeleteLink(ep.HostIfName) + if err != nil { + log.Printf("[net] Failed to delete veth pair %v: %v.", ep.HostIfName, err) + return err + } + + return nil +} diff --git a/network/endpoint.go b/network/endpoint.go index 1a10d9277b..62d129e651 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -54,9 +54,12 @@ type EndpointInfo struct { // RouteInfo contains information about an IP route. type RouteInfo struct { - Dst net.IPNet - Gw net.IP - DevName string + Dst net.IPNet + Src net.IP + Gw net.IP + Protocol int + DevName string + Scope int } // NewEndpoint creates a new endpoint in the network. diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index cf85a5ad1a..94f102c0ee 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -71,7 +71,7 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { log.Printf("Generate veth name based on the key provided") key := epInfo.Data[OptVethName].(string) vethname := generateVethName(key) - hostIfName = fmt.Sprintf("%s%s", hostVEthInterfacePrefix, vethname) + hostIfName = fmt.Sprintf("%s%s", "cali", vethname) contIfName = fmt.Sprintf("%s%s2", hostVEthInterfacePrefix, vethname) } else { // Create a veth pair. @@ -87,8 +87,12 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { hostIfName, contIfName, vlanid) - } else { + } else if nw.Mode != opModeCalico { + log.Printf("Bridge client") epClient = NewLinuxBridgeEndpointClient(nw.extIf, hostIfName, contIfName, nw.Mode) + } else { + log.Printf("calico client") + epClient = NewCalicoEndpointClient(nw.extIf, hostIfName, contIfName, nw.Mode) } // Cleanup on failure. @@ -201,8 +205,10 @@ func (nw *network) deleteEndpointImpl(ep *endpoint) error { if ep.VlanID != 0 { epInfo := ep.getInfo() epClient = NewOVSEndpointClient(nw.extIf, epInfo, ep.HostIfName, "", ep.VlanID) - } else { + } else if nw.Mode != opModeCalico { epClient = NewLinuxBridgeEndpointClient(nw.extIf, ep.HostIfName, "", nw.Mode) + } else { + epClient = NewCalicoEndpointClient(nw.extIf, ep.HostIfName, "", nw.Mode) } epClient.DeleteEndpointRules(ep) @@ -232,8 +238,11 @@ func addRoutes(interfaceName string, routes []RouteInfo) error { nlRoute := &netlink.Route{ Family: netlink.GetIpAddressFamily(route.Gw), Dst: &route.Dst, + Src: route.Src, Gw: route.Gw, LinkIndex: ifIndex, + Scope: route.Scope, + Protocol: route.Protocol, } if err := netlink.AddIpRoute(nlRoute); err != nil { @@ -253,7 +262,7 @@ func deleteRoutes(interfaceName string, routes []RouteInfo) error { interfaceIf, _ := net.InterfaceByName(interfaceName) for _, route := range routes { - log.Printf("[ovs] Adding IP route %+v to link %v.", route, interfaceName) + log.Printf("[ovs] Deleting IP route %+v to link %v.", route, interfaceName) if route.DevName != "" { devIf, _ := net.InterfaceByName(route.DevName) @@ -265,8 +274,11 @@ func deleteRoutes(interfaceName string, routes []RouteInfo) error { nlRoute := &netlink.Route{ Family: netlink.GetIpAddressFamily(route.Gw), Dst: &route.Dst, + Src: route.Src, Gw: route.Gw, LinkIndex: ifIndex, + Scope: route.Scope, + Protocol: route.Protocol, } if err := netlink.DeleteIpRoute(nlRoute); err != nil { diff --git a/network/network.go b/network/network.go index 0e68c490b4..01ce2bf796 100644 --- a/network/network.go +++ b/network/network.go @@ -15,6 +15,7 @@ const ( // Operational modes. opModeBridge = "bridge" opModeTunnel = "tunnel" + opModeCalico = "calico" opModeDefault = opModeTunnel ) diff --git a/network/network_linux.go b/network/network_linux.go index 4d49a634d2..2a90489ef1 100644 --- a/network/network_linux.go +++ b/network/network_linux.go @@ -54,7 +54,8 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt if opt != nil && opt[VlanIDKey] != nil { vlanid, _ = strconv.Atoi(opt[VlanIDKey].(string)) } - + case opModeCalico: + break default: return nil, errNetworkModeInvalid } From bfac3512ba9076c4de2c3f9f60d110a2e821ed31 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Mon, 17 Dec 2018 18:32:37 -0800 Subject: [PATCH 2/9] cni for calico policy controller --- cni/network/network.go | 11 +++++--- netlink/link.go | 35 -------------------------- network/calico_endpointclient_linux.go | 12 ++++----- network/endpoint_linux.go | 12 ++++++--- 4 files changed, 22 insertions(+), 48 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 2121f81c19..0613aa41bc 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -27,7 +27,8 @@ const ( dockerNetworkOption = "com.docker.network.generic" // Supported IP version. Currently support only IPv4 - ipVersion = "4" + ipVersion = "4" + opModeCalico = "calico" ) // NetPlugin represents the CNI network plugin. @@ -167,6 +168,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { result *cniTypesCurr.Result azIpamResult *cniTypesCurr.Result err error + vethName string nwCfg *cni.NetworkConfig epInfo *network.EndpointInfo iface *cniTypesCurr.Interface @@ -454,8 +456,11 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { // A runtime must not call ADD twice (without a corresponding DEL) for the same // (network name, container id, name of the interface inside the container) - //vethName := fmt.Sprintf("%s%s%s", networkId, k8sContainerID, k8sIfName) - vethName := fmt.Sprintf("%s.%s", k8sNamespace, k8sPodName) + if nwCfg.Mode == opModeCalico { + vethName = fmt.Sprintf("%s.%s", k8sNamespace, k8sPodName) + } else { + vethName = fmt.Sprintf("%s%s%s", networkId, k8sContainerID, k8sIfName) + } setEndpointOptions(cnsNetworkConfig, epInfo, vethName) // Create the endpoint. diff --git a/netlink/link.go b/netlink/link.go index 9e015261a6..dfbdbaaa48 100644 --- a/netlink/link.go +++ b/netlink/link.go @@ -13,41 +13,6 @@ import ( "golang.org/x/sys/unix" ) -const ( - NDA_UNSPEC = iota - NDA_DST - NDA_LLADDR - NDA_CACHEINFO - NDA_PROBES - NDA_VLAN - NDA_PORT - NDA_VNI - NDA_IFINDEX - NDA_MAX = NDA_IFINDEX -) - -// Neighbor Cache Entry States. -const ( - NUD_NONE = 0x00 - NUD_INCOMPLETE = 0x01 - NUD_REACHABLE = 0x02 - NUD_STALE = 0x04 - NUD_DELAY = 0x08 - NUD_PROBE = 0x10 - NUD_FAILED = 0x20 - NUD_NOARP = 0x40 - NUD_PERMANENT = 0x80 -) - -// Neighbor Flags -const ( - NTF_USE = 0x01 - NTF_SELF = 0x02 - NTF_MASTER = 0x04 - NTF_PROXY = 0x08 - NTF_ROUTER = 0x80 -) - // Link types. const ( LINK_TYPE_BRIDGE = "bridge" diff --git a/network/calico_endpointclient_linux.go b/network/calico_endpointclient_linux.go index 665b0ac774..f8137c637a 100644 --- a/network/calico_endpointclient_linux.go +++ b/network/calico_endpointclient_linux.go @@ -1,13 +1,11 @@ package network import ( - "fmt" "net" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netlink" "github.com/Azure/azure-container-networking/network/epcommon" - "github.com/Azure/azure-container-networking/platform" ) const ( @@ -152,7 +150,7 @@ func (client *CalicoEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo for _, ipAddr := range epInfo.IPAddresses { routeInfo = RouteInfo{} ip, ipNet, _ := net.ParseCIDR(ipAddr.String()) - log.Printf("Removing route %v", ipNet.String()) + log.Printf("[net] Removing route %v", ipNet.String()) routeInfo.Dst = *ipNet routeInfo.Scope = netlink.RT_SCOPE_LINK routeInfo.Src = ip @@ -162,14 +160,14 @@ func (client *CalicoEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo // delete the above route if err := deleteRoutes(client.containerVethName, routeInfoList); err != nil { - log.Printf("Deleting route failed with err %v", err) + log.Printf("[net] Deleting route failed with err %v", err) } // set arp entry for fake gateway in pod - arpCmd := fmt.Sprintf("arp -s 169.254.1.1 %v", client.hostVethMac.String()) - _, err := platform.ExecuteCommand(arpCmd) + err := netlink.AddOrRemoveStaticArp(netlink.ADD, client.containerVethName, gwIP, client.hostVethMac) if err != nil { - log.Printf("Setting static arp for ip 169.254.1.1 mac %v failed with error %v", client) + log.Printf("[net] Setting static arp for ip %v mac %v failed with error %v", gwIP.String(), client.hostVethMac, err) + return err } return nil diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index bfa1449ebc..c7ca517257 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -21,6 +21,8 @@ const ( // Prefix for host virtual network interface names. hostVEthInterfacePrefix = commonInterfacePrefix + "v" + calicoPrefix = "cali" + // Prefix for container network interface names. containerInterfacePrefix = "eth" ) @@ -71,7 +73,11 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { log.Printf("Generate veth name based on the key provided") key := epInfo.Data[OptVethName].(string) vethname := generateVethName(key) - hostIfName = fmt.Sprintf("%s%s", "cali", vethname) + if nw.Mode == opModeCalico { + hostIfName = fmt.Sprintf("%s%s", calicoPrefix, vethname) + } else { + hostIfName = fmt.Sprintf("%s%s", hostVEthInterfacePrefix, vethname) + } contIfName = fmt.Sprintf("%s%s2", hostVEthInterfacePrefix, vethname) } else { // Create a veth pair. @@ -255,7 +261,7 @@ func addRoutes(interfaceName string, routes []RouteInfo) error { if !strings.Contains(strings.ToLower(err.Error()), "file exists") { return err } else { - log.Printf("route already exists") + log.Printf("[net] route already exists") } } } @@ -268,7 +274,7 @@ func deleteRoutes(interfaceName string, routes []RouteInfo) error { interfaceIf, _ := net.InterfaceByName(interfaceName) for _, route := range routes { - log.Printf("[ovs] Deleting IP route %+v from link %v.", route, interfaceName) + log.Printf("[net] Deleting IP route %+v from link %v.", route, interfaceName) if route.DevName != "" { devIf, _ := net.InterfaceByName(route.DevName) From 733a8b44e399ac687680717c87be1e6bed1745e6 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Mon, 17 Dec 2018 18:36:54 -0800 Subject: [PATCH 3/9] removed unused parameter --- cni/netconfig.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cni/netconfig.go b/cni/netconfig.go index 3bb4f3202d..5fe49ecd74 100644 --- a/cni/netconfig.go +++ b/cni/netconfig.go @@ -49,7 +49,6 @@ type NetworkConfig struct { EnableSnatOnHost bool `json:"enableSnatOnHost,omitempty"` EnableExactMatchForPodName bool `json:"enableExactMatchForPodName,omitempty"` CNSUrl string `json:"cnsurl,omitempty"` - EnableCalicoMode bool `json:"enableCalicoMode,omitempty"` Ipam struct { Type string `json:"type"` Environment string `json:"environment,omitempty"` From ac8aed257dd85efd7e6306b639ecfab621bdabd7 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Mon, 17 Dec 2018 18:37:50 -0800 Subject: [PATCH 4/9] minor fix --- cni/network/network.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cni/network/network.go b/cni/network/network.go index 0613aa41bc..a155f5cdc3 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -388,16 +388,17 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { } else { if !nwCfg.MultiTenancy { // Network already exists. - // Call into IPAM plugin to allocate an address for the endpoint. subnetPrefix := nwInfo.Subnets[0].Prefix.String() log.Printf("[cni-net] Found network %v with subnet %v.", networkId, subnetPrefix) nwCfg.Ipam.Subnet = subnetPrefix + // Call into IPAM plugin to allocate an address for the endpoint. result, err = plugin.DelegateAdd(nwCfg.Ipam.Type, nwCfg) if err != nil { err = plugin.Errorf("Failed to allocate address: %v", err) return err } + ipconfig := result.IPs[0] iface := &cniTypesCurr.Interface{Name: args.IfName} result.Interfaces = append(result.Interfaces, iface) From d183b53432f23e40c411b91aaa0cc51f3858e825 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Fri, 21 Dec 2018 12:56:24 -0800 Subject: [PATCH 5/9] addressed review comments --- cni/network/network.go | 6 +- network/endpoint_linux.go | 28 +++++--- network/network.go | 8 +-- network/network_linux.go | 2 +- ...go => transparent_endpointclient_linux.go} | 69 +++++-------------- 5 files changed, 47 insertions(+), 66 deletions(-) rename network/{calico_endpointclient_linux.go => transparent_endpointclient_linux.go} (60%) diff --git a/cni/network/network.go b/cni/network/network.go index a155f5cdc3..56b5c9076b 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -27,8 +27,8 @@ const ( dockerNetworkOption = "com.docker.network.generic" // Supported IP version. Currently support only IPv4 - ipVersion = "4" - opModeCalico = "calico" + ipVersion = "4" + opModeTransparent = "transparent" ) // NetPlugin represents the CNI network plugin. @@ -457,7 +457,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { // A runtime must not call ADD twice (without a corresponding DEL) for the same // (network name, container id, name of the interface inside the container) - if nwCfg.Mode == opModeCalico { + if nwCfg.Mode == opModeTransparent { vethName = fmt.Sprintf("%s.%s", k8sNamespace, k8sPodName) } else { vethName = fmt.Sprintf("%s%s%s", networkId, k8sContainerID, k8sIfName) diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index c7ca517257..3fdd3ea738 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -21,7 +21,7 @@ const ( // Prefix for host virtual network interface names. hostVEthInterfacePrefix = commonInterfacePrefix + "v" - calicoPrefix = "cali" + transPrefix = "cali" // Prefix for container network interface names. containerInterfacePrefix = "eth" @@ -73,8 +73,8 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { log.Printf("Generate veth name based on the key provided") key := epInfo.Data[OptVethName].(string) vethname := generateVethName(key) - if nw.Mode == opModeCalico { - hostIfName = fmt.Sprintf("%s%s", calicoPrefix, vethname) + if nw.Mode == opModeTransparent { + hostIfName = fmt.Sprintf("%s%s", transPrefix, vethname) } else { hostIfName = fmt.Sprintf("%s%s", hostVEthInterfacePrefix, vethname) } @@ -87,18 +87,19 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { } if vlanid != 0 { + log.Printf("OVS client") epClient = NewOVSEndpointClient( nw.extIf, epInfo, hostIfName, contIfName, vlanid) - } else if nw.Mode != opModeCalico { + } else if nw.Mode != opModeTransparent { log.Printf("Bridge client") epClient = NewLinuxBridgeEndpointClient(nw.extIf, hostIfName, contIfName, nw.Mode) } else { - log.Printf("calico client") - epClient = NewCalicoEndpointClient(nw.extIf, hostIfName, contIfName, nw.Mode) + log.Printf("Transparent client") + epClient = NewTransparentEndpointClient(nw.extIf, hostIfName, contIfName, nw.Mode) } // Cleanup on failure. @@ -217,10 +218,10 @@ func (nw *network) deleteEndpointImpl(ep *endpoint) error { if ep.VlanID != 0 { epInfo := ep.getInfo() epClient = NewOVSEndpointClient(nw.extIf, epInfo, ep.HostIfName, "", ep.VlanID) - } else if nw.Mode != opModeCalico { + } else if nw.Mode != opModeTransparent { epClient = NewLinuxBridgeEndpointClient(nw.extIf, ep.HostIfName, "", nw.Mode) } else { - epClient = NewCalicoEndpointClient(nw.extIf, ep.HostIfName, "", nw.Mode) + epClient = NewTransparentEndpointClient(nw.extIf, ep.HostIfName, "", nw.Mode) } epClient.DeleteEndpointRules(ep) @@ -434,3 +435,14 @@ func updateRoutes(existingEp *EndpointInfo, targetEp *EndpointInfo) error { return nil } + +func getDefaultGateway(routes []RouteInfo) net.IP { + _, defDstIP, _ := net.ParseCIDR("0.0.0.0/0") + for _, route := range routes { + if route.Dst.String() == defDstIP.String() { + return route.Gw + } + } + + return nil +} diff --git a/network/network.go b/network/network.go index 29b86414b4..e0a11fe567 100644 --- a/network/network.go +++ b/network/network.go @@ -14,10 +14,10 @@ import ( const ( // Operational modes. - opModeBridge = "bridge" - opModeTunnel = "tunnel" - opModeCalico = "calico" - opModeDefault = opModeTunnel + opModeBridge = "bridge" + opModeTunnel = "tunnel" + opModeTransparent = "transparent" + opModeDefault = opModeTunnel ) // ExternalInterface is a host network interface that bridges containers to external networks. diff --git a/network/network_linux.go b/network/network_linux.go index 60a4fc37f2..6da8aaa704 100644 --- a/network/network_linux.go +++ b/network/network_linux.go @@ -52,7 +52,7 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt if opt != nil && opt[VlanIDKey] != nil { vlanid, _ = strconv.Atoi(opt[VlanIDKey].(string)) } - case opModeCalico: + case opModeTransparent: break default: return nil, errNetworkModeInvalid diff --git a/network/calico_endpointclient_linux.go b/network/transparent_endpointclient_linux.go similarity index 60% rename from network/calico_endpointclient_linux.go rename to network/transparent_endpointclient_linux.go index f8137c637a..ff223865d2 100644 --- a/network/calico_endpointclient_linux.go +++ b/network/transparent_endpointclient_linux.go @@ -13,7 +13,7 @@ const ( DEFAULT_GW = "0.0.0.0/0" ) -type CalicoEndpointClient struct { +type TransparentEndpointClient struct { bridgeName string hostPrimaryIfName string hostVethName string @@ -24,14 +24,14 @@ type CalicoEndpointClient struct { mode string } -func NewCalicoEndpointClient( +func NewTransparentEndpointClient( extIf *externalInterface, hostVethName string, containerVethName string, mode string, -) *CalicoEndpointClient { +) *TransparentEndpointClient { - client := &CalicoEndpointClient{ + client := &TransparentEndpointClient{ bridgeName: extIf.BridgeName, hostPrimaryIfName: extIf.Name, hostVethName: hostVethName, @@ -43,7 +43,7 @@ func NewCalicoEndpointClient( return client } -func (client *CalicoEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { +func (client *TransparentEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { if err := epcommon.CreateEndpoint(client.hostVethName, client.containerVethName); err != nil { return err } @@ -65,7 +65,7 @@ func (client *CalicoEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { return nil } -func (client *CalicoEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error { +func (client *TransparentEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error { var routeInfoList []RouteInfo // ip route add dev @@ -83,7 +83,7 @@ func (client *CalicoEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error return nil } -func (client *CalicoEndpointClient) DeleteEndpointRules(ep *endpoint) { +func (client *TransparentEndpointClient) DeleteEndpointRules(ep *endpoint) { var routeInfoList []RouteInfo // ip route del dev @@ -98,7 +98,7 @@ func (client *CalicoEndpointClient) DeleteEndpointRules(ep *endpoint) { } } -func (client *CalicoEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo, nsID uintptr) error { +func (client *TransparentEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo, nsID uintptr) error { // Move the container interface to container's network namespace. log.Printf("[net] Setting link %v netns %v.", client.containerVethName, epInfo.NetNsPath) if err := netlink.SetLinkNetNs(client.containerVethName, nsID); err != nil { @@ -108,7 +108,7 @@ func (client *CalicoEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointI return nil } -func (client *CalicoEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) error { +func (client *TransparentEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) error { if err := epcommon.SetupContainerInterface(client.containerVethName, epInfo.IfName); err != nil { return err } @@ -118,62 +118,31 @@ func (client *CalicoEndpointClient) SetupContainerInterfaces(epInfo *EndpointInf return nil } -func (client *CalicoEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { +func (client *TransparentEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { if err := epcommon.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { return err } - var routeInfo RouteInfo - var routeInfoList []RouteInfo - - // ip route add 169.254.1.1/32 dev eth0 scope link - gwIP, gwIPNet, _ := net.ParseCIDR(FAKE_GW_IP) - routeInfo.Dst = *gwIPNet - routeInfo.Scope = netlink.RT_SCOPE_LINK - routeInfoList = append(routeInfoList, routeInfo) - - // ip route add default gw 169.254.1.1 dev eth0 - routeInfo = RouteInfo{} - _, defIPNet, _ := net.ParseCIDR(DEFAULT_GW) - routeInfo.Dst = *defIPNet - routeInfo.Gw = net.ParseIP(gwIP.String()) - routeInfoList = append(routeInfoList, routeInfo) - - // add the above routes - if err := addRoutes(client.containerVethName, routeInfoList); err != nil { + if err := addRoutes(client.containerVethName, epInfo.Routes); err != nil { return err } - routeInfoList = routeInfoList[:0] - - // Removing the route added by setipaddress while assigning IP to interface - for _, ipAddr := range epInfo.IPAddresses { - routeInfo = RouteInfo{} - ip, ipNet, _ := net.ParseCIDR(ipAddr.String()) - log.Printf("[net] Removing route %v", ipNet.String()) - routeInfo.Dst = *ipNet - routeInfo.Scope = netlink.RT_SCOPE_LINK - routeInfo.Src = ip - routeInfo.Protocol = netlink.RTPROT_KERNEL - routeInfoList = append(routeInfoList, routeInfo) - } - - // delete the above route - if err := deleteRoutes(client.containerVethName, routeInfoList); err != nil { - log.Printf("[net] Deleting route failed with err %v", err) + gw := getDefaultGateway(epInfo.Routes) + if gw == nil { + log.Printf("Default gateway not found in routes") + return nil } - // set arp entry for fake gateway in pod - err := netlink.AddOrRemoveStaticArp(netlink.ADD, client.containerVethName, gwIP, client.hostVethMac) + log.Printf("Add static arp entry in pod ip %v mac %v", gw.String(), client.hostVethMac) + err := netlink.AddOrRemoveStaticArp(netlink.ADD, client.containerVethName, gw, client.hostVethMac) if err != nil { - log.Printf("[net] Setting static arp for ip %v mac %v failed with error %v", gwIP.String(), client.hostVethMac, err) - return err + log.Printf("[net] Setting static arp for ip %v mac %v failed with error %v", gw.String(), client.hostVethMac, err) } return nil } -func (client *CalicoEndpointClient) DeleteEndpoints(ep *endpoint) error { +func (client *TransparentEndpointClient) DeleteEndpoints(ep *endpoint) error { log.Printf("[net] Deleting veth pair %v %v.", ep.HostIfName, ep.IfName) err := netlink.DeleteLink(ep.HostIfName) if err != nil { From f96330501499737e1ea9f4f95b4c3b7114fd04ef Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Wed, 26 Dec 2018 11:14:34 -0800 Subject: [PATCH 6/9] addressed review comments --- cni/network/network.go | 2 ++ network/endpoint_linux.go | 7 +------ 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 56b5c9076b..664a6cb2f6 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -458,6 +458,8 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { // A runtime must not call ADD twice (without a corresponding DEL) for the same // (network name, container id, name of the interface inside the container) if nwCfg.Mode == opModeTransparent { + // This mechanism of using only namespace and name is not unique for different incarnations of POD/container. + // It will result in unpredictable behavior if API server decides to reorder DELETE and ADD call for new incarnation of same POD. vethName = fmt.Sprintf("%s.%s", k8sNamespace, k8sPodName) } else { vethName = fmt.Sprintf("%s%s%s", networkId, k8sContainerID, k8sIfName) diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 3fdd3ea738..e95768843f 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -21,6 +21,7 @@ const ( // Prefix for host virtual network interface names. hostVEthInterfacePrefix = commonInterfacePrefix + "v" + // Prefix for host veth interfaces in transparent mode. transPrefix = "cali" // Prefix for container network interface names. @@ -251,11 +252,8 @@ func addRoutes(interfaceName string, routes []RouteInfo) error { nlRoute := &netlink.Route{ Family: netlink.GetIpAddressFamily(route.Gw), Dst: &route.Dst, - Src: route.Src, Gw: route.Gw, LinkIndex: ifIndex, - Scope: route.Scope, - Protocol: route.Protocol, } if err := netlink.AddIpRoute(nlRoute); err != nil { @@ -287,11 +285,8 @@ func deleteRoutes(interfaceName string, routes []RouteInfo) error { nlRoute := &netlink.Route{ Family: netlink.GetIpAddressFamily(route.Gw), Dst: &route.Dst, - Src: route.Src, Gw: route.Gw, LinkIndex: ifIndex, - Scope: route.Scope, - Protocol: route.Protocol, } if err := netlink.DeleteIpRoute(nlRoute); err != nil { From 4f7226629bcb8f76b1bfde7a1d800debb7f58214 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Fri, 4 Jan 2019 16:02:34 -0800 Subject: [PATCH 7/9] modified vethname generation and the hostbveth prefix --- cni/network/network.go | 11 ++--------- network/endpoint_linux.go | 9 +-------- network/transparent_endpointclient_linux.go | 15 +++++++++++++++ 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 664a6cb2f6..0166acf228 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -27,8 +27,7 @@ const ( dockerNetworkOption = "com.docker.network.generic" // Supported IP version. Currently support only IPv4 - ipVersion = "4" - opModeTransparent = "transparent" + ipVersion = "4" ) // NetPlugin represents the CNI network plugin. @@ -457,13 +456,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { // A runtime must not call ADD twice (without a corresponding DEL) for the same // (network name, container id, name of the interface inside the container) - if nwCfg.Mode == opModeTransparent { - // This mechanism of using only namespace and name is not unique for different incarnations of POD/container. - // It will result in unpredictable behavior if API server decides to reorder DELETE and ADD call for new incarnation of same POD. - vethName = fmt.Sprintf("%s.%s", k8sNamespace, k8sPodName) - } else { - vethName = fmt.Sprintf("%s%s%s", networkId, k8sContainerID, k8sIfName) - } + vethName = fmt.Sprintf("%s%s%s", networkId, k8sContainerID, k8sIfName) setEndpointOptions(cnsNetworkConfig, epInfo, vethName) // Create the endpoint. diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index e95768843f..a8e64e4dac 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -21,9 +21,6 @@ const ( // Prefix for host virtual network interface names. hostVEthInterfacePrefix = commonInterfacePrefix + "v" - // Prefix for host veth interfaces in transparent mode. - transPrefix = "cali" - // Prefix for container network interface names. containerInterfacePrefix = "eth" ) @@ -74,11 +71,7 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { log.Printf("Generate veth name based on the key provided") key := epInfo.Data[OptVethName].(string) vethname := generateVethName(key) - if nw.Mode == opModeTransparent { - hostIfName = fmt.Sprintf("%s%s", transPrefix, vethname) - } else { - hostIfName = fmt.Sprintf("%s%s", hostVEthInterfacePrefix, vethname) - } + hostIfName = fmt.Sprintf("%s%s", hostVEthInterfacePrefix, vethname) contIfName = fmt.Sprintf("%s%s2", hostVEthInterfacePrefix, vethname) } else { // Create a veth pair. diff --git a/network/transparent_endpointclient_linux.go b/network/transparent_endpointclient_linux.go index ff223865d2..5c18a78599 100644 --- a/network/transparent_endpointclient_linux.go +++ b/network/transparent_endpointclient_linux.go @@ -1,11 +1,13 @@ package network import ( + "fmt" "net" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netlink" "github.com/Azure/azure-container-networking/network/epcommon" + "github.com/Azure/azure-container-networking/platform" ) const ( @@ -43,6 +45,12 @@ func NewTransparentEndpointClient( return client } +func setArpProxy(ifName string) error { + cmd := fmt.Sprintf("echo 1 > /proc/sys/net/ipv4/conf/%v/proxy_arp", ifName) + _, err := platform.ExecuteCommand(cmd) + return err +} + func (client *TransparentEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { if err := epcommon.CreateEndpoint(client.hostVethName, client.containerVethName); err != nil { return err @@ -80,6 +88,13 @@ func (client *TransparentEndpointClient) AddEndpointRules(epInfo *EndpointInfo) return err } } + + log.Printf("calling setArpProxy for %v", client.hostVethName) + if err := setArpProxy(client.hostVethName); err != nil { + log.Printf("setArpProxy failed with: %v", err) + return err + } + return nil } From 9c49caa76c175cca862bf6c4b51d881cfaf8c54a Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Fri, 4 Jan 2019 16:11:09 -0800 Subject: [PATCH 8/9] removed setting arp for default gw --- network/transparent_endpointclient_linux.go | 22 ++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/network/transparent_endpointclient_linux.go b/network/transparent_endpointclient_linux.go index 5c18a78599..8f130eac01 100644 --- a/network/transparent_endpointclient_linux.go +++ b/network/transparent_endpointclient_linux.go @@ -142,17 +142,17 @@ func (client *TransparentEndpointClient) ConfigureContainerInterfacesAndRoutes(e return err } - gw := getDefaultGateway(epInfo.Routes) - if gw == nil { - log.Printf("Default gateway not found in routes") - return nil - } - - log.Printf("Add static arp entry in pod ip %v mac %v", gw.String(), client.hostVethMac) - err := netlink.AddOrRemoveStaticArp(netlink.ADD, client.containerVethName, gw, client.hostVethMac) - if err != nil { - log.Printf("[net] Setting static arp for ip %v mac %v failed with error %v", gw.String(), client.hostVethMac, err) - } + // gw := getDefaultGateway(epInfo.Routes) + // if gw == nil { + // log.Printf("Default gateway not found in routes") + // return nil + // } + + // log.Printf("Add static arp entry in pod ip %v mac %v", gw.String(), client.hostVethMac) + // err := netlink.AddOrRemoveStaticArp(netlink.ADD, client.containerVethName, gw, client.hostVethMac) + // if err != nil { + // log.Printf("[net] Setting static arp for ip %v mac %v failed with error %v", gw.String(), client.hostVethMac, err) + // } return nil } From 522e1941f9f4ae60dc9352e8b4fa501ecb08727b Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Fri, 4 Jan 2019 16:12:06 -0800 Subject: [PATCH 9/9] minor fix --- network/transparent_endpointclient_linux.go | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/network/transparent_endpointclient_linux.go b/network/transparent_endpointclient_linux.go index 8f130eac01..e41c51d5fe 100644 --- a/network/transparent_endpointclient_linux.go +++ b/network/transparent_endpointclient_linux.go @@ -138,23 +138,7 @@ func (client *TransparentEndpointClient) ConfigureContainerInterfacesAndRoutes(e return err } - if err := addRoutes(client.containerVethName, epInfo.Routes); err != nil { - return err - } - - // gw := getDefaultGateway(epInfo.Routes) - // if gw == nil { - // log.Printf("Default gateway not found in routes") - // return nil - // } - - // log.Printf("Add static arp entry in pod ip %v mac %v", gw.String(), client.hostVethMac) - // err := netlink.AddOrRemoveStaticArp(netlink.ADD, client.containerVethName, gw, client.hostVethMac) - // if err != nil { - // log.Printf("[net] Setting static arp for ip %v mac %v failed with error %v", gw.String(), client.hostVethMac, err) - // } - - return nil + return addRoutes(client.containerVethName, epInfo.Routes) } func (client *TransparentEndpointClient) DeleteEndpoints(ep *endpoint) error {