Merge pull request #517 from abstractmj/1.17.x-qcloud-eip-fix

1.17.x qcloud eip fix

bcs-services/bcs-network/qcloud-eip/conf/conf.go

@@ -35,14 +35,16 @@ type NetArgs struct {
 // NetConf net config
 type NetConf struct {
 	types.NetConf
-	Master     string   `json:"master"`
-	ENIPrefix  string   `json:"eniPrefix"`
-	ClusterID  string   `json:"clusterId"`
-	Region     string   `json:"region"`
-	Secret     string   `json:"secret"`
-	UUID       string   `json:"uuid"`
-	SubnetID   string   `json:"subnetId,omitempty"`
-	MTU        int      `json:"mtu,omitempty"`
+	TencentCloudCVMDomain string `json:"tencentcloudCVMDomain"`
+	TencentCloudVPCDomain string `json:"tencentcloudVPCDomain"`
+	Master                string `json:"master"`
+	ENIPrefix             string `json:"eniPrefix"`
+	ClusterID             string `json:"clusterId"`
+	Region                string `json:"region"`
+	Secret                string `json:"secret"`
+	UUID                  string `json:"uuid"`
+	SubnetID              string `json:"subnetId,omitempty"`
+	MTU                   int    `json:"mtu,omitempty"`
 	NetService *NetArgs `json:"netservice,omitempty"`
 	Args       *bcsconf.CNIArgs
 }

bcs-services/bcs-network/qcloud-eip/eip/eip.go

@@ -13,9 +13,6 @@
 package eip
 
 import (
-	"bk-bcs/bcs-common/common/blog"
-	netsvc "bk-bcs/bcs-services/bcs-netservice/pkg/netservice/types"
-	"bk-bcs/bcs-services/bcs-network/qcloud-eip/conf"
 	"fmt"
 	"net"
 	"os"
@@ -29,11 +26,17 @@ import (
 	"github.com/containernetworking/plugins/pkg/ipam"
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/vishvananda/netlink"
+
+	"bk-bcs/bcs-common/common/blog"
+	netsvc "bk-bcs/bcs-services/bcs-netservice/pkg/netservice/types"
+	"bk-bcs/bcs-services/bcs-network/qcloud-eip/conf"
 )
 
 const (
 	// start eni route table id
 	startENIRouteTableID = 100
+
+	mainRouteTableID = 254
 )
 
 // EIP object for use tencent network interface
@@ -99,30 +102,16 @@ func (eip *EIP) Init(file string, eniNum int, ipNum int) {
 		os.Exit(1)
 	}
 
-	// if there is already some network interface name starting with netConf.ENIPrefix,
-	// it means someone has already initialized the network of this cvm,
-	// so stop the init action
-	for i := 0; i < eniNum; i++ {
-		eniName := fmt.Sprintf("%s%d", netConf.ENIPrefix, i)
-		blog.Infof("check eni %s......", eniName)
-		eniIPAddr, _, _ := getIPAddrByName(eniName)
-		if len(eniIPAddr) != 0 {
-			blog.Errorf("%s is existed, no need to init", eniName)
-			os.Exit(1)
-		}
-	}
-
 	// record route tables ids
 	var routeTableIDs []int
+	// map for link name to route table id
+	routeTableIDMap := make(map[string]int)
 	// record all available ips
 	secondaryIPMap := make(map[string][]string)
 	// according to the demanded network interface number and ip address number,
 	// apply certain ip addresses for each newly applied network interface
 	for i := 0; i < eniNum; i++ {
-		// for each newly applied network interface, create a route table in later steps,
-		// here we just calculate the route table id and record it in an array
-		tableID := startENIRouteTableID + i
-		routeTableIDs = append(routeTableIDs, tableID)
+
 		// for each eni, there is a IP num limitation
 		// apply (limitation - 1) ip, because the default primary ip
 		ipLimit := getMaxPrivateIPNumPerENI(int(*instance.CPU), int(*instance.Memory))
@@ -149,20 +138,40 @@ func (eip *EIP) Init(file string, eniNum int, ipNum int) {
 		// record other applied ip addresses
 		blog.Infof("set up eni %s ......", eniLocalInterfaceName)
 		var secondaryIPs []string
+		var primaryIP string
 		for _, privateIPObj := range newENI.PrivateIpAddressSet {
 			if *privateIPObj.Primary {
-				err = setupNetworkInterface(*privateIPObj.PrivateIpAddress, *subnet.CidrBlock, eniLocalInterfaceName, *newENI.MacAddress)
-				if err != nil {
-					blog.Errorf("set up networkinterface %s with ip %s mac %s failed, err %s", eniLocalInterfaceName, *privateIPObj.PrivateIpAddress, *newENI.MacAddress, err.Error())
-					os.Exit(1)
-				}
-				blog.Infof("set up eni with primary ip %s done", *privateIPObj.PrivateIpAddress)
+				primaryIP = *privateIPObj.PrivateIpAddress
 			} else {
 				blog.Infof("get secondary ip %s", *privateIPObj.PrivateIpAddress)
 				secondaryIPs = append(secondaryIPs, *privateIPObj.PrivateIpAddress)
 			}
 		}
 		secondaryIPMap[strings.ToLower(*newENI.MacAddress)] = secondaryIPs
+
+		// if there is already some network interface name starting with netConf.ENIPrefix,
+		// it means someone has already initialized the network of this cvm,
+		// so stop the init action
+		linkName := fmt.Sprintf("%s%d", netConf.ENIPrefix, i)
+		blog.Infof("check eni %s......", linkName)
+		eniIPAddr, _, _ := getIPAddrByName(linkName)
+		if len(eniIPAddr) != 0 {
+			blog.Errorf("%s is existed, no need to set up", linkName)
+			continue
+		}
+		// set up eni
+		err = setupNetworkInterface(primaryIP, *subnet.CidrBlock, eniLocalInterfaceName, *newENI.MacAddress)
+		if err != nil {
+			blog.Errorf("set up networkinterface %s with ip %s mac %s failed, err %s", eniLocalInterfaceName, primaryIP, *newENI.MacAddress, err.Error())
+			os.Exit(1)
+		}
+		blog.Infof("set up eni with primary ip %s done", primaryIP)
+
+		// for each newly applied network interface, create a route table in later steps,
+		// here we just calculate the route table id and record it in an array
+		tableID := startENIRouteTableID + i
+		routeTableIDs = append(routeTableIDs, tableID)
+		routeTableIDMap[linkName] = tableID
 	}
 
 	// init network environment
@@ -204,11 +213,10 @@ func (eip *EIP) Init(file string, eniNum int, ipNum int) {
 	}
 	gw := ip.NextIP(cidrIP)
 	// add default route into each route tables
-	for index, id := range routeTableIDs {
-		linkName := fmt.Sprintf("%s%d", netConf.ENIPrefix, index)
-		link, err := netlink.LinkByName(linkName)
+	for eniName, id := range routeTableIDMap {
+		link, err := netlink.LinkByName(eniName)
 		if err != nil {
-			blog.Errorf("get link by name %s failed, err %s", linkName, err.Error())
+			blog.Errorf("get link by name %s failed, err %s", eniName, err.Error())
 			os.Exit(1)
 		}
 		defaultRouteRule := &netlink.Route{
@@ -270,10 +278,28 @@ func (eip *EIP) Init(file string, eniNum int, ipNum int) {
 	}
 	blog.Infof("register ip pool to netservice done")
 
+	// get ip pool in netservice
+	existedPool, err := netSvcClient.GetPool(pool)
+	if err != nil {
+		blog.Errorf("get netservice pool failed, err %s", err.Error())
+		os.Exit(1)
+	}
+	existedIPs := make(map[string]string)
+	for _, ip := range existedPool.Reserved {
+		existedIPs[ip] = ip
+	}
+	for _, ip := range existedPool.Active {
+		existedIPs[ip] = ip
+	}
+
 	// add newly ip addresses into ip pool in netservice
 	blog.Infof("update ip instances to netservice")
 	for mac, ips := range secondaryIPMap {
 		for _, ip := range ips {
+			if _, ok := existedIPs[ip]; ok {
+				blog.Infof("ip %s already in pool, skip", ip)
+				continue
+			}
 			ipIns := new(netsvc.IPInst)
 			ipIns.IPAddr = ip
 			ipIns.MacAddr = mac
@@ -555,6 +581,7 @@ func (eip *EIP) Release(file string) {
 	err = eip.doDeregister(netConf)
 	if err != nil {
 		blog.Errorf("do deregister failed, err %s", err.Error())
+		os.Exit(1)
 	}
 
 	// create cvm client
@@ -688,6 +715,7 @@ func configureHostNS(hostIfName string, ipNet *net.IPNet, routeTableID int) erro
 	ruleToTable := netlink.NewRule()
 	ruleToTable.Dst = ipNet
 	ruleToTable.Table = routeTableID
+	ruleToTable.Priority = 2048
 	err = netlink.RuleDel(ruleToTable)
 	if err != nil {
 		blog.Warnf("clean old rule to table %s failed, err %s", ruleToTable.String(), err.Error())
@@ -701,6 +729,7 @@ func configureHostNS(hostIfName string, ipNet *net.IPNet, routeTableID int) erro
 	ruleFromTaskgroup := netlink.NewRule()
 	ruleFromTaskgroup.Src = ipNet
 	ruleFromTaskgroup.Table = routeTableID
+	ruleFromTaskgroup.Priority = 2048
 	err = netlink.RuleDel(ruleFromTaskgroup)
 	if err != nil {
 		blog.Warnf("clean old rule from taskgroup %s failed, err %s", ruleToTable.String(), err.Error())

bcs-services/bcs-network/qcloud-eip/eip/instance.go

@@ -34,6 +34,12 @@ func newInstanceClient(conf *conf.NetConf) *instanceClient {
 		conf.UUID,
 	)
 	cpf := profile.NewClientProfile()
+
+	// set tencentcloud domain
+	if len(conf.TencentCloudCVMDomain) != 0 {
+		cpf.HttpProfile.Endpoint = conf.TencentCloudCVMDomain
+	}
+
 	client, err := cvm.NewClient(credential, conf.Region, cpf)
 	if err != nil {
 		blog.Errorf("new instance client failed, err %s", err.Error())

bcs-services/bcs-network/qcloud-eip/eip/netservice.go

@@ -40,7 +40,7 @@ func NewNetSvcClient(conf *conf.NetArgs) (*NetSvcClient, error) {
 	if len(conf.PubKey) == 0 && len(conf.Key) == 0 && len(conf.Ca) == 0 {
 		client, clientErr = netservice.NewClient()
 	} else {
-		client, clientErr = netservice.NewTLSClient(conf.Ca, conf.Key, conf.PubKey, static.ServerCertPwd)
+		client, clientErr = netservice.NewTLSClient(conf.Ca, conf.Key, conf.PubKey, static.ClientCertPwd)
 	}
 	if clientErr != nil {
 		return nil, clientErr
@@ -73,6 +73,15 @@ func (c *NetSvcClient) CreateOrUpdatePool(pool *netsvc.NetPool) error {
 	return c.client.UpdatePool(pool)
 }
 
+// GetPool get net pool
+func (c *NetSvcClient) GetPool(pool *netsvc.NetPool) (*netsvc.NetPool, error) {
+	p, err := c.client.GetPool(pool.Cluster, pool.Net)
+	if err != nil {
+		return nil, err
+	}
+	return p[0], nil
+}
+
 // UpdateIPInstance update ip instance
 func (c *NetSvcClient) UpdateIPInstance(ins *netsvc.IPInst) error {
 	err := c.client.UpdateIPInstance(ins)

bcs-services/bcs-network/qcloud-eip/eip/util.go

@@ -13,13 +13,48 @@
 package eip
 
 import (
-	"bk-bcs/bcs-common/common/blog"
 	"fmt"
 	"net"
 	"strconv"
 	"strings"
+	"golang.org/x/sys/unix"
+
+	"github.com/vishvananda/netlink"
+
+	"bk-bcs/bcs-common/common/blog"
 )
 
+func getBridgeRoutes(tableID int, bridgeName string) ([]netlink.Route, error) {
+	link, err := netlink.LinkByName(bridgeName)
+	if err != nil {
+		blog.Errorf("there is no bridge with name %s", bridgeName)
+		return nil, fmt.Errorf("there is no bridge with name %s", bridgeName)
+	}
+	addr, _, _ := getIPAddrByName(bridgeName)
+	if len(addr) == 0 {
+		blog.Errorf("get bridge %s addr failed", bridgeName)
+		return nil, fmt.Errorf("get bridge %s addr failed", bridgeName)
+	}
+	ipObj := net.ParseIP(addr)
+	if ipObj == nil {
+		blog.Errorf("parse ip %s return nil", addr)
+		return nil, fmt.Errorf("parse ip %s return nil", addr)
+	}
+	bridgeRouteRule := netlink.Route{
+		LinkIndex: link.Attrs().Index,
+		Scope:     netlink.SCOPE_LINK,
+		Src:       ipObj,
+		Table:     tableID,
+	}
+	routes, err := netlink.RouteListFiltered(unix.AF_INET, &bridgeRouteRule,
+		netlink.RT_FILTER_TABLE|netlink.RT_FILTER_SCOPE|netlink.RT_FILTER_OIF|netlink.RT_FILTER_SRC)
+	if err != nil {
+		blog.Errorf("failed to list route list with route %+v , err %s", bridgeRouteRule, err.Error())
+		return nil, err
+	}
+	return routes, nil
+}
+
 //GetIPAddrByName return eni ip address, mask, mac address
 func getIPAddrByName(name string) (string, int, string) {
 	//get ip address

bcs-services/bcs-network/qcloud-eip/eip/vpc.go

@@ -55,6 +55,12 @@ func newVPCClient(conf *conf.NetConf, vpcID string) *vpcClient {
 		conf.UUID,
 	)
 	cpf := profile.NewClientProfile()
+
+	// set tencentcloud domain
+	if len(conf.TencentCloudVPCDomain) != 0 {
+		cpf.HttpProfile.Endpoint = conf.TencentCloudVPCDomain
+	}
+
 	client, err := vpc.NewClient(credential, conf.Region, cpf)
 	if err != nil {
 		blog.Errorf("new vpc client failed, err %s", err.Error())