Skip to content

Commit

Permalink
Add support for IPv6 only mode
Browse files Browse the repository at this point in the history 
Automatically switch to IPv6 only mode if first node-ip is IPv6 address

Signed-off-by: Olli Janatuinen <olli.janatuinen@gmail.com>
  • Loading branch information
@olljanat
olljanat committed Nov 28, 2021
1 parent 0c1f816 commit f1651b4
Show file tree
Hide file tree
Showing 6 changed files with 149 additions and 15 deletions.
14 changes: 11 additions & 3 deletions pkg/agent/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -418,7 +418,6 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
nodeConfig.AgentConfig.ClusterDomain = controlConfig.ClusterDomain
nodeConfig.AgentConfig.ResolvConf = locateOrGenerateResolvConf(envInfo)
nodeConfig.AgentConfig.ClientCA = clientCAFile
nodeConfig.AgentConfig.ListenAddress = "0.0.0.0"
nodeConfig.AgentConfig.KubeConfigKubelet = kubeconfigKubelet
nodeConfig.AgentConfig.KubeConfigKubeProxy = kubeconfigKubeproxy
nodeConfig.AgentConfig.KubeConfigK3sController = kubeconfigK3sController
Expand Down Expand Up @@ -460,8 +459,13 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N

nodeConfig.AgentConfig.NodeIPs = nodeIPs
nodeIP, err := util.GetFirst4(nodeIPs)
nodeConfig.AgentConfig.ListenAddress = "0.0.0.0"
if err != nil {
return nil, errors.Wrap(err, "cannot configure IPv4 node-ip")
nodeIP, err = util.GetFirst6(nodeIPs)
if err != nil {
return nil, errors.Wrap(err, "cannot configure IPv4/IPv6 node-ip")
}
nodeConfig.AgentConfig.ListenAddress = "::"
}
nodeConfig.AgentConfig.NodeIP = nodeIP.String()

Expand All @@ -476,10 +480,14 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
}

// if configured, set NodeExternalIP to the first IPv4 address, for legacy clients
// unless only IPv6 address given
if len(nodeConfig.AgentConfig.NodeExternalIPs) > 0 {
nodeExternalIP, err := util.GetFirst4(nodeConfig.AgentConfig.NodeExternalIPs)
if err != nil {
return nil, errors.Wrap(err, "cannot configure IPv4 node-external-ip")
nodeExternalIP, err = util.GetFirst6(nodeConfig.AgentConfig.NodeExternalIPs)
if err != nil {
return nil, errors.Wrap(err, "cannot configure IPv4/IPv6 node-external-ip")
}
}
nodeConfig.AgentConfig.NodeExternalIP = nodeExternalIP.String()
}
Expand Down
8 changes: 7 additions & 1 deletion pkg/agent/run.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,14 @@ func run(ctx context.Context, cfg cmds.Agent, proxy proxy.Proxy) error {
if err != nil {
return errors.Wrap(err, "failed to validate node-ip")
}
serviceIPv6 := false
for _, cidr := range nodeConfig.AgentConfig.ServiceCIDRs {
if utilsnet.IsIPv6CIDR(cidr) {
serviceIPv6 = true
}
}

enableIPv6 := dualCluster || dualService || dualNode
enableIPv6 := dualCluster || dualService || dualNode || serviceIPv6
conntrackConfig, err := getConntrackConfig(nodeConfig)
if err != nil {
return errors.Wrap(err, "failed to validate kube-proxy conntrack configuration")
Expand Down
74 changes: 67 additions & 7 deletions pkg/cli/server/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -224,16 +224,25 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
if serverConfig.ControlConfig.PrivateIP == "" && len(cmds.AgentConfig.NodeIP) != 0 {
// ignoring the error here is fine since etcd will fall back to the interface's IPv4 address
serverConfig.ControlConfig.PrivateIP, _ = util.GetFirst4String(cmds.AgentConfig.NodeIP)
if serverConfig.ControlConfig.PrivateIP == "" {
serverConfig.ControlConfig.PrivateIP, _ = util.GetFirst6String(cmds.AgentConfig.NodeIP)
}
}

// if not set, try setting advertise-ip from agent node-external-ip
if serverConfig.ControlConfig.AdvertiseIP == "" && len(cmds.AgentConfig.NodeExternalIP) != 0 {
serverConfig.ControlConfig.AdvertiseIP, _ = util.GetFirst4String(cmds.AgentConfig.NodeExternalIP)
if serverConfig.ControlConfig.AdvertiseIP == "" {
serverConfig.ControlConfig.AdvertiseIP, _ = util.GetFirst6String(cmds.AgentConfig.NodeExternalIP)
}
}

// if not set, try setting advertise-up from agent node-ip
if serverConfig.ControlConfig.AdvertiseIP == "" && len(cmds.AgentConfig.NodeIP) != 0 {
serverConfig.ControlConfig.AdvertiseIP, _ = util.GetFirst4String(cmds.AgentConfig.NodeIP)
if serverConfig.ControlConfig.AdvertiseIP == "" {
serverConfig.ControlConfig.AdvertiseIP, _ = util.GetFirst6String(cmds.AgentConfig.NodeIP)
}
}

// if we ended up with any advertise-ips, ensure they're added to the SAN list;
Expand All @@ -251,14 +260,22 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
return err
}
serverConfig.ControlConfig.ServerNodeName = nodeName
serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, "127.0.0.1", "localhost", nodeName)
serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, "127.0.0.1", "::1", "localhost", nodeName)
for _, ip := range nodeIPs {
serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, ip.String())
}

// configure ClusterIPRanges
if len(cmds.ServerConfig.ClusterCIDR) == 0 {
cmds.ServerConfig.ClusterCIDR.Set("10.42.0.0/16")
clusterCIDR := "10.42.0.0/16"

// default to IPv6 ClusterIPRange on IPv6 nodes
_, ipv4Err := util.GetFirst4(nodeIPs)
_, ipv6Err := util.GetFirst6(nodeIPs)
if ipv6Err == nil && ipv4Err != nil {
clusterCIDR = "fd:42::/56"
}
cmds.ServerConfig.ClusterCIDR.Set(clusterCIDR)
}
for _, cidr := range cmds.ServerConfig.ClusterCIDR {
for _, v := range strings.Split(cidr, ",") {
Expand All @@ -271,15 +288,27 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
}

// set ClusterIPRange to the first IPv4 block, for legacy clients
// unless only IPv6 range given
clusterIPRange, err := util.GetFirst4Net(serverConfig.ControlConfig.ClusterIPRanges)
if err != nil {
return errors.Wrap(err, "cannot configure IPv4 cluster-cidr")
clusterIPRange, err = util.GetFirst6Net(serverConfig.ControlConfig.ClusterIPRanges)
if err != nil {
return errors.Wrap(err, "cannot configure IPv4/IPv6 cluster-cidr")
}
}
serverConfig.ControlConfig.ClusterIPRange = clusterIPRange

// configure ServiceIPRanges
if len(cmds.ServerConfig.ServiceCIDR) == 0 {
cmds.ServerConfig.ServiceCIDR.Set("10.43.0.0/16")
serviceCIDR := "10.43.0.0/16"

// default to IPv6 ServiceIPRange on IPv6 nodes
_, ipv4Err := util.GetFirst4(nodeIPs)
_, ipv6Err := util.GetFirst6(nodeIPs)
if ipv6Err == nil && ipv4Err != nil {
serviceCIDR = "fd:43::/112"
}
cmds.ServerConfig.ServiceCIDR.Set(serviceCIDR)
}
for _, cidr := range cmds.ServerConfig.ServiceCIDR {
for _, v := range strings.Split(cidr, ",") {
Expand All @@ -292,9 +321,13 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
}

// set ServiceIPRange to the first IPv4 block, for legacy clients
// unless only IPv6 range given
serviceIPRange, err := util.GetFirst4Net(serverConfig.ControlConfig.ServiceIPRanges)
if err != nil {
return errors.Wrap(err, "cannot configure IPv4 service-cidr")
serviceIPRange, err = util.GetFirst6Net(serverConfig.ControlConfig.ServiceIPRanges)
if err != nil {
return errors.Wrap(err, "cannot configure IPv4/IPv6 service-cidr")
}
}
serverConfig.ControlConfig.ServiceIPRange = serviceIPRange

Expand All @@ -312,7 +345,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont

// If cluster-dns CLI arg is not set, we set ClusterDNS address to be the first IPv4 ServiceCIDR network + 10,
// i.e. when you set service-cidr to 192.168.0.0/16 and don't provide cluster-dns, it will be set to 192.168.0.10
// If there are no IPv4 ServiceCIDRs, an error will be raised.
// If there are no IPv4 ServiceCIDRs, an IPv6 ServiceCIDRs will be used.
if len(cmds.ServerConfig.ClusterDNS) == 0 {
clusterDNS, err := utilsnet.GetIndexedIP(serverConfig.ControlConfig.ServiceIPRange, 10)
if err != nil {
Expand All @@ -331,9 +364,13 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
}
}
// Set ClusterDNS to the first IPv4 address, for legacy clients
// unless only IPv6 range given
clusterDNS, err := util.GetFirst4(serverConfig.ControlConfig.ClusterDNSs)
if err != nil {
return errors.Wrap(err, "cannot configure IPv4 cluster-dns address")
clusterDNS, err = util.GetFirst6(serverConfig.ControlConfig.ClusterDNSs)
if err != nil {
return errors.Wrap(err, "cannot configure IPv4/IPv6 cluster-dns address")
}
}
serverConfig.ControlConfig.ClusterDNS = clusterDNS
}
Expand Down Expand Up @@ -445,6 +482,10 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
ip := serverConfig.ControlConfig.BindAddress
if ip == "" {
ip = "127.0.0.1"
_, err := util.GetFirst6([]net.IP{nodeIPs[0]})
if err == nil {
ip = "[::1]"
}
}

url := fmt.Sprintf("https://%s:%d", ip, serverConfig.ControlConfig.SupervisorPort)
Expand Down Expand Up @@ -505,6 +546,25 @@ func validateNetworkConfiguration(serverConfig server.Config) error {
return errors.New("dual-stack cluster-dns is not supported")
}

serviceIPv4 := false
serviceIPv6 := false
for _, cidr := range serverConfig.ControlConfig.ServiceIPRanges {
if utilsnet.IsIPv4CIDR(cidr) {
serviceIPv4 = true
}
if utilsnet.IsIPv6CIDR(cidr) {
serviceIPv6 = true
}
}
if serviceIPv6 && !serviceIPv4 {
if serverConfig.ControlConfig.DisableNPC == false {
return errors.New("network policy enforcement is not compatible with IPv6 only operation; server must be restarted with --disable-network-policy")
}
if serverConfig.ControlConfig.FlannelBackend != config.FlannelBackendNone {
return errors.New("Flannel is not compatible with IPv6 only operation; server must be restarted with --flannel-backend=none")
}
}

return nil
}

Expand Down
14 changes: 12 additions & 2 deletions pkg/daemons/agent/agent_linux.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,14 @@ func checkRuntimeEndpoint(cfg *config.Agent, argsMap map[string]string) {
}

func kubeProxyArgs(cfg *config.Agent) map[string]string {
bindAddress := "127.0.0.1"
_, err := util.GetFirst6String([]string{cfg.NodeIP})
if err == nil {
bindAddress = "::1"
}
argsMap := map[string]string{
"proxy-mode": "iptables",
"healthz-bind-address": "127.0.0.1",
"healthz-bind-address": bindAddress,
"kubeconfig": cfg.KubeConfigKubeProxy,
"cluster-cidr": util.JoinIPNets(cfg.ClusterCIDRs),
"conntrack-max-per-core": "0",
Expand All @@ -55,8 +60,13 @@ func kubeProxyArgs(cfg *config.Agent) map[string]string {
}

func kubeletArgs(cfg *config.Agent) map[string]string {
bindAddress := "127.0.0.1"
_, err := util.GetFirst6String([]string{cfg.NodeIP})
if err == nil {
bindAddress = "::1"
}
argsMap := map[string]string{
"healthz-bind-address": "127.0.0.1",
"healthz-bind-address": bindAddress,
"read-only-port": "0",
"cluster-domain": cfg.ClusterDomain,
"kubeconfig": cfg.KubeConfigKubelet,
Expand Down
14 changes: 12 additions & 2 deletions pkg/daemons/agent/agent_windows.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,14 @@ func checkRuntimeEndpoint(cfg *config.Agent, argsMap map[string]string) {
}

func kubeProxyArgs(cfg *config.Agent) map[string]string {
bindAddress := "127.0.0.1"
_, err := util.GetFirst6String([]string{cfg.NodeIP})
if err == nil {
bindAddress = "::1"
}
argsMap := map[string]string{
"proxy-mode": "kernelspace",
"healthz-bind-address": "127.0.0.1",
"healthz-bind-address": bindAddress,
"kubeconfig": cfg.KubeConfigKubeProxy,
"cluster-cidr": util.JoinIPNets(cfg.ClusterCIDRs),
}
Expand All @@ -47,8 +52,13 @@ func kubeProxyArgs(cfg *config.Agent) map[string]string {
}

func kubeletArgs(cfg *config.Agent) map[string]string {
bindAddress := "127.0.0.1"
_, err := util.GetFirst6String([]string{cfg.NodeIP})
if err == nil {
bindAddress = "::1"
}
argsMap := map[string]string{
"healthz-bind-address": "127.0.0.1",
"healthz-bind-address": bindAddress,
"read-only-port": "0",
"cluster-domain": cfg.ClusterDomain,
"kubeconfig": cfg.KubeConfigKubelet,
Expand Down
40 changes: 40 additions & 0 deletions pkg/util/net.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,46 @@ func JoinIP4Nets(elems []*net.IPNet) string {
return strings.Join(strs, ",")
}

// GetFirst6 returns the first IPv6 address from the list of IP addresses.
// If no IPv6 addresses are found, an error is raised.
func GetFirst6(elems []net.IP) (net.IP, error) {
for _, elem := range elems {
if elem == nil || elem.To16() == nil {
continue
}
return elem, nil
}
return nil, errors.New("no IPv6 address found")
}

// GetFirst6Net returns the first IPv4 network from the list of IP networks.
// If no IPv6 addresses are found, an error is raised.
func GetFirst6Net(elems []*net.IPNet) (*net.IPNet, error) {
for _, elem := range elems {
if elem == nil || elem.IP.To16() == nil {
continue
}
return elem, nil
}
return nil, errors.New("no IPv6 CIDRs found")
}

// GetFirst6String returns the first IPv6 address from a list of IP address strings.
// If no IPv6 addresses are found, an error is raised.
func GetFirst6String(elems []string) (string, error) {
ips := []net.IP{}
for _, elem := range elems {
for _, v := range strings.Split(elem, ",") {
ips = append(ips, net.ParseIP(v))
}
}
ip, err := GetFirst6(ips)
if err != nil {
return "", err
}
return ip.String(), nil
}

// JoinIP6Nets stringifies and joins a list of IPv6 networks with commas.
func JoinIP6Nets(elems []*net.IPNet) string {
var strs []string
Expand Down

0 comments on commit f1651b4

Please sign in to comment.