Skip to content

Commit

Permalink
Add adaption for dualstack, part of controller process.
Browse files Browse the repository at this point in the history 
Co-authored-by: feixiang43 <352447595@qq.com>
  • Loading branch information
@hongzhen-ma
hongzhen-ma and feixiang43 committed Dec 7, 2020
1 parent 7b5dcf5 commit d64e693
Show file tree
Hide file tree
Showing 15 changed files with 1,048 additions and 374 deletions.
4 changes: 4 additions & 0 deletions pkg/apis/kubeovn/v1/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,10 @@ type SubnetStatus struct {

AvailableIPs float64 `json:"availableIPs"`
UsingIPs float64 `json:"usingIPs"`
V4AvailableIPs float64 `json:"v4availableIPs"`
V4UsingIPs float64 `json:"v4usingIPs"`
V6AvailableIPs float64 `json:"v6availableIPs"`
V6UsingIPs float64 `json:"v6usingIPs"`
ActivateGateway string `json:"activateGateway"`
}

Expand Down
4 changes: 2 additions & 2 deletions pkg/controller/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ func ParseFlags() (*Configuration, error) {
}

if config.DefaultGateway == "" {
gw, err := util.FirstSubnetIP(config.DefaultCIDR)
gw, err := util.GetGwByCidr(config.DefaultCIDR)
if err != nil {
return nil, err
}
Expand All @@ -152,7 +152,7 @@ func ParseFlags() (*Configuration, error) {
}

if config.NodeSwitchGateway == "" {
gw, err := util.FirstSubnetIP(config.NodeSwitchCIDR)
gw, err := util.GetGwByCidr(config.NodeSwitchCIDR)
if err != nil {
return nil, err
}
Expand Down
6 changes: 3 additions & 3 deletions pkg/controller/init.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ func (c *Controller) initNodeSwitch() error {
Provider: util.OvnProvider,
CIDRBlock: c.config.NodeSwitchCIDR,
Gateway: c.config.NodeSwitchGateway,
ExcludeIps: []string{c.config.NodeSwitchGateway},
ExcludeIps: strings.Split(c.config.NodeSwitchGateway, ","),
Protocol: util.CheckProtocol(c.config.NodeSwitchCIDR),
DisableInterConnection: true,
},
Expand Down Expand Up @@ -244,7 +244,7 @@ func (c *Controller) InitIPAM() error {
if isPodAlive(pod) &&
pod.Annotations[util.AllocatedAnnotation] == "true" &&
pod.Annotations[util.LogicalSwitchAnnotation] != "" {
_, _, err := c.ipam.GetStaticAddress(
_, _, _, err := c.ipam.GetStaticAddress(
fmt.Sprintf("%s/%s", pod.Namespace, pod.Name),
pod.Annotations[util.IpAddressAnnotation],
pod.Annotations[util.MacAddressAnnotation],
Expand All @@ -263,7 +263,7 @@ func (c *Controller) InitIPAM() error {
for _, node := range nodes {
if node.Annotations[util.AllocatedAnnotation] == "true" {
portName := fmt.Sprintf("node-%s", node.Name)
_, _, err := c.ipam.GetStaticAddress(portName, node.Annotations[util.IpAddressAnnotation],
_, _, _, err := c.ipam.GetStaticAddress(portName, node.Annotations[util.IpAddressAnnotation],
node.Annotations[util.MacAddressAnnotation],
node.Annotations[util.LogicalSwitchAnnotation])
if err != nil {
Expand Down
142 changes: 78 additions & 64 deletions pkg/controller/node.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -198,12 +198,12 @@ func (c *Controller) handleAddNode(key string) error {
return err
}

var ip, mac string
var v4IP, v6IP, mac string
portName := fmt.Sprintf("node-%s", key)
if node.Annotations[util.AllocatedAnnotation] == "true" {
return nil
} else {
ip, mac, err = c.ipam.GetRandomAddress(portName, c.config.NodeSwitch)
v4IP, v6IP, mac, err = c.ipam.GetRandomAddress(portName, c.config.NodeSwitch)
if err != nil {
return err
}
Expand All @@ -214,16 +214,20 @@ func (c *Controller) handleAddNode(key string) error {
return err
}

if err := c.ovnClient.CreatePort(c.config.NodeSwitch, portName, ip, subnet.Spec.CIDRBlock, mac, tag, false); err != nil {
ipStr := util.GetStringIP(v4IP, v6IP)
if err := c.ovnClient.CreatePort(c.config.NodeSwitch, portName, ipStr, subnet.Spec.CIDRBlock, mac, tag, false); err != nil {
return err
}

// There is only one nodeAddr temp
nodeAddr := getNodeInternalIP(node)
if util.CheckProtocol(nodeAddr) == util.CheckProtocol(ip) {
err = c.ovnClient.AddStaticRoute("", nodeAddr, strings.Split(ip, "/")[0], c.config.ClusterRouter)
if err != nil {
klog.Errorf("failed to add static router from node to ovn0 %v", err)
return err
for _, ip := range strings.Split(ipStr, ",") {
if util.CheckProtocol(nodeAddr) == util.CheckProtocol(ip) {
err = c.ovnClient.AddStaticRoute("", nodeAddr, ip, c.config.ClusterRouter)
if err != nil {
klog.Errorf("failed to add static router from node to ovn0 %v", err)
return err
}
}
}

Expand All @@ -237,7 +241,8 @@ func (c *Controller) handleAddNode(key string) error {
if len(node.Annotations) == 0 {
op = "add"
}
node.Annotations[util.IpAddressAnnotation] = ip

node.Annotations[util.IpAddressAnnotation] = ipStr
node.Annotations[util.MacAddressAnnotation] = mac
node.Annotations[util.CidrAnnotation] = subnet.Spec.CIDRBlock
node.Annotations[util.GatewayAnnotation] = subnet.Spec.Gateway
Expand All @@ -252,59 +257,9 @@ func (c *Controller) handleAddNode(key string) error {
return err
}

ipCr, err := c.config.KubeOvnClient.KubeovnV1().IPs().Get(fmt.Sprintf("node-%s", key), metav1.GetOptions{})
if err != nil {
if k8serrors.IsNotFound(err) {
_, err := c.config.KubeOvnClient.KubeovnV1().IPs().Create(&kubeovnv1.IP{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("node-%s", key),
Labels: map[string]string{
util.SubnetNameLabel: c.config.NodeSwitch,
c.config.NodeSwitch: "",
},
},
Spec: kubeovnv1.IPSpec{
PodName: key,
Subnet: c.config.NodeSwitch,
NodeName: key,
IPAddress: ip,
MacAddress: mac,
AttachIPs: []string{},
AttachMacs: []string{},
AttachSubnets: []string{},
},
})
if err != nil {
errMsg := fmt.Errorf("failed to create ip crd for %s, %v", ip, err)
klog.Error(errMsg)
return errMsg
}
} else {
errMsg := fmt.Errorf("failed to get ip crd for %s, %v", ip, err)
klog.Error(errMsg)
return errMsg
}
} else {
if ipCr.Labels != nil {
ipCr.Labels[util.SubnetNameLabel] = c.config.NodeSwitch
} else {
ipCr.Labels = map[string]string{
util.SubnetNameLabel: c.config.NodeSwitch,
}
}
ipCr.Spec.PodName = key
ipCr.Spec.Namespace = ""
ipCr.Spec.Subnet = c.config.NodeSwitch
ipCr.Spec.NodeName = key
ipCr.Spec.IPAddress = ip
ipCr.Spec.MacAddress = mac
ipCr.Spec.ContainerID = ""
_, err := c.config.KubeOvnClient.KubeovnV1().IPs().Update(ipCr)
if err != nil {
errMsg := fmt.Errorf("failed to create ip crd for %s, %v", ip, err)
klog.Error(errMsg)
return errMsg
}
if err := c.createOrUpdateCrdIPs(key, ipStr, mac); err != nil {
klog.Errorf("failed to create or update IPs node-%s %v", key, err)
return err
}

return nil
Expand All @@ -326,8 +281,8 @@ func (c *Controller) handleDeleteNode(key string) error {
}

addresses := c.ipam.GetPodAddress(portName)
if len(addresses) > 0 {
if err := c.ovnClient.DeleteStaticRouteByNextHop(addresses[0].Ip); err != nil {
for _, addr := range addresses {
if err := c.ovnClient.DeleteStaticRouteByNextHop(addr.Ip); err != nil {
return err
}
}
Expand Down Expand Up @@ -390,3 +345,62 @@ func getNodeInternalIP(node *v1.Node) string {
}
return nodeAddr
}

func (c *Controller) createOrUpdateCrdIPs(key, ip, mac string) error {
ipCr, err := c.config.KubeOvnClient.KubeovnV1().IPs().Get(fmt.Sprintf("node-%s", key), metav1.GetOptions{})
if err != nil {
if k8serrors.IsNotFound(err) {
_, err := c.config.KubeOvnClient.KubeovnV1().IPs().Create(&kubeovnv1.IP{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("node-%s", key),
Labels: map[string]string{
util.SubnetNameLabel: c.config.NodeSwitch,
c.config.NodeSwitch: "",
},
},
Spec: kubeovnv1.IPSpec{
PodName: key,
Subnet: c.config.NodeSwitch,
NodeName: key,
IPAddress: ip,
MacAddress: mac,
AttachIPs: []string{},
AttachMacs: []string{},
AttachSubnets: []string{},
},
})
if err != nil {
errMsg := fmt.Errorf("failed to create ip crd for %s, %v", ip, err)
klog.Error(errMsg)
return errMsg
}
} else {
errMsg := fmt.Errorf("failed to get ip crd for %s, %v", ip, err)
klog.Error(errMsg)
return errMsg
}
} else {
if ipCr.Labels != nil {
ipCr.Labels[util.SubnetNameLabel] = c.config.NodeSwitch
} else {
ipCr.Labels = map[string]string{
util.SubnetNameLabel: c.config.NodeSwitch,
}
}
ipCr.Spec.PodName = key
ipCr.Spec.Namespace = ""
ipCr.Spec.Subnet = c.config.NodeSwitch
ipCr.Spec.NodeName = key
ipCr.Spec.IPAddress = ip
ipCr.Spec.MacAddress = mac
ipCr.Spec.ContainerID = ""
_, err := c.config.KubeOvnClient.KubeovnV1().IPs().Update(ipCr)
if err != nil {
errMsg := fmt.Errorf("failed to create ip crd for %s, %v", ip, err)
klog.Error(errMsg)
return errMsg
}
}

return nil
}

0 comments on commit d64e693

Please sign in to comment.