Skip to content

Commit

Permalink
Add support for IPv6 only mode
Browse files Browse the repository at this point in the history 
Automatically switch to IPv6 only mode if first node-ip is IPv6 address

Signed-off-by: Olli Janatuinen <olli.janatuinen@gmail.com>
  • Loading branch information
@olljanat
olljanat committed Feb 10, 2022
1 parent 830c330 commit 966f4d6
Show file tree
Hide file tree
Showing 6 changed files with 187 additions and 23 deletions.
11 changes: 6 additions & 5 deletions pkg/agent/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -425,7 +425,6 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
nodeConfig.AgentConfig.ClusterDomain = controlConfig.ClusterDomain
nodeConfig.AgentConfig.ResolvConf = locateOrGenerateResolvConf(envInfo)
nodeConfig.AgentConfig.ClientCA = clientCAFile
nodeConfig.AgentConfig.ListenAddress = "0.0.0.0"
nodeConfig.AgentConfig.KubeConfigKubelet = kubeconfigKubelet
nodeConfig.AgentConfig.KubeConfigKubeProxy = kubeconfigKubeproxy
nodeConfig.AgentConfig.KubeConfigK3sController = kubeconfigK3sController
Expand Down Expand Up @@ -466,18 +465,20 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
nodeConfig.Certificate = servingCert

nodeConfig.AgentConfig.NodeIPs = nodeIPs
nodeIP, err := util.GetFirst4(nodeIPs)
nodeIP, listenAddress, _, err := util.GetFirstIP(nodeIPs)
if err != nil {
return nil, errors.Wrap(err, "cannot configure IPv4 node-ip")
return nil, errors.Wrap(err, "cannot configure IPv4/IPv6 node-ip")
}
nodeConfig.AgentConfig.NodeIP = nodeIP.String()
nodeConfig.AgentConfig.ListenAddress = listenAddress
nodeConfig.AgentConfig.NodeExternalIPs = nodeExternalIPs

// if configured, set NodeExternalIP to the first IPv4 address, for legacy clients
// unless only IPv6 address given
if len(nodeConfig.AgentConfig.NodeExternalIPs) > 0 {
nodeExternalIP, err := util.GetFirst4(nodeConfig.AgentConfig.NodeExternalIPs)
nodeExternalIP, _, _, err := util.GetFirstIP(nodeConfig.AgentConfig.NodeExternalIPs)
if err != nil {
return nil, errors.Wrap(err, "cannot configure IPv4 node-external-ip")
return nil, errors.Wrap(err, "cannot configure IPv4/IPv6 node-external-ip")
}
nodeConfig.AgentConfig.NodeExternalIP = nodeExternalIP.String()
}
Expand Down
4 changes: 3 additions & 1 deletion pkg/agent/run.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,10 @@ func run(ctx context.Context, cfg cmds.Agent, proxy proxy.Proxy) error {
if err != nil {
return errors.Wrap(err, "failed to validate node-ip")
}
serviceIPv6 := utilsnet.IsIPv6CIDR(nodeConfig.AgentConfig.ServiceCIDR)
clusterIPv6 := utilsnet.IsIPv6CIDR(nodeConfig.AgentConfig.ClusterCIDR)

enableIPv6 := dualCluster || dualService || dualNode
enableIPv6 := dualCluster || dualService || dualNode || serviceIPv6 || clusterIPv6
conntrackConfig, err := getConntrackConfig(nodeConfig)
if err != nil {
return errors.Wrap(err, "failed to validate kube-proxy conntrack configuration")
Expand Down
52 changes: 39 additions & 13 deletions pkg/cli/server/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -198,17 +198,17 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont

if serverConfig.ControlConfig.PrivateIP == "" && len(cmds.AgentConfig.NodeIP) != 0 {
// ignoring the error here is fine since etcd will fall back to the interface's IPv4 address
serverConfig.ControlConfig.PrivateIP, _ = util.GetFirst4String(cmds.AgentConfig.NodeIP)
serverConfig.ControlConfig.PrivateIP, _, _ = util.GetFirstString(cmds.AgentConfig.NodeIP)
}

// if not set, try setting advertise-ip from agent node-external-ip
if serverConfig.ControlConfig.AdvertiseIP == "" && len(cmds.AgentConfig.NodeExternalIP) != 0 {
serverConfig.ControlConfig.AdvertiseIP, _ = util.GetFirst4String(cmds.AgentConfig.NodeExternalIP)
serverConfig.ControlConfig.AdvertiseIP, _, _ = util.GetFirstString(cmds.AgentConfig.NodeExternalIP)
}

// if not set, try setting advertise-ip from agent node-ip
if serverConfig.ControlConfig.AdvertiseIP == "" && len(cmds.AgentConfig.NodeIP) != 0 {
serverConfig.ControlConfig.AdvertiseIP, _ = util.GetFirst4String(cmds.AgentConfig.NodeIP)
serverConfig.ControlConfig.AdvertiseIP, _, _ = util.GetFirstString(cmds.AgentConfig.NodeIP)
}

// if we ended up with any advertise-ips, ensure they're added to the SAN list;
Expand All @@ -226,14 +226,19 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
return err
}
serverConfig.ControlConfig.ServerNodeName = nodeName
serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, "127.0.0.1", "localhost", nodeName)
serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, "127.0.0.1", "::1", "localhost", nodeName)
for _, ip := range nodeIPs {
serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, ip.String())
}

// configure ClusterIPRanges
_, _, IPv6only, _ := util.GetFirstIP(nodeIPs)
if len(cmds.ServerConfig.ClusterCIDR) == 0 {
cmds.ServerConfig.ClusterCIDR.Set("10.42.0.0/16")
clusterCIDR := "10.42.0.0/16"
if IPv6only {
clusterCIDR = "fd:42::/56"
}
cmds.ServerConfig.ClusterCIDR.Set(clusterCIDR)
}
for _, cidr := range cmds.ServerConfig.ClusterCIDR {
for _, v := range strings.Split(cidr, ",") {
Expand All @@ -246,15 +251,20 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
}

// set ClusterIPRange to the first IPv4 block, for legacy clients
clusterIPRange, err := util.GetFirst4Net(serverConfig.ControlConfig.ClusterIPRanges)
// unless only IPv6 range given
clusterIPRange, err := util.GetFirstNet(serverConfig.ControlConfig.ClusterIPRanges)
if err != nil {
return errors.Wrap(err, "cannot configure IPv4 cluster-cidr")
return errors.Wrap(err, "cannot configure IPv4/IPv6 cluster-cidr")
}
serverConfig.ControlConfig.ClusterIPRange = clusterIPRange

// configure ServiceIPRanges
if len(cmds.ServerConfig.ServiceCIDR) == 0 {
cmds.ServerConfig.ServiceCIDR.Set("10.43.0.0/16")
serviceCIDR := "10.43.0.0/16"
if IPv6only {
serviceCIDR = "fd:43::/112"
}
cmds.ServerConfig.ServiceCIDR.Set(serviceCIDR)
}
for _, cidr := range cmds.ServerConfig.ServiceCIDR {
for _, v := range strings.Split(cidr, ",") {
Expand All @@ -267,9 +277,10 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
}

// set ServiceIPRange to the first IPv4 block, for legacy clients
serviceIPRange, err := util.GetFirst4Net(serverConfig.ControlConfig.ServiceIPRanges)
// unless only IPv6 range given
serviceIPRange, err := util.GetFirstNet(serverConfig.ControlConfig.ServiceIPRanges)
if err != nil {
return errors.Wrap(err, "cannot configure IPv4 service-cidr")
return errors.Wrap(err, "cannot configure IPv4/IPv6 service-cidr")
}
serverConfig.ControlConfig.ServiceIPRange = serviceIPRange

Expand All @@ -287,7 +298,8 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont

// If cluster-dns CLI arg is not set, we set ClusterDNS address to be the first IPv4 ServiceCIDR network + 10,
// i.e. when you set service-cidr to 192.168.0.0/16 and don't provide cluster-dns, it will be set to 192.168.0.10
// If there are no IPv4 ServiceCIDRs, an error will be raised.
// If there are no IPv4 ServiceCIDRs, an IPv6 ServiceCIDRs will be used.
// If neither of IPv4 or IPv6 are found an error is raised.
if len(cmds.ServerConfig.ClusterDNS) == 0 {
clusterDNS, err := utilsnet.GetIndexedIP(serverConfig.ControlConfig.ServiceIPRange, 10)
if err != nil {
Expand All @@ -306,9 +318,10 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
}
}
// Set ClusterDNS to the first IPv4 address, for legacy clients
clusterDNS, err := util.GetFirst4(serverConfig.ControlConfig.ClusterDNSs)
// unless only IPv6 range given
clusterDNS, _, _, err := util.GetFirstIP(serverConfig.ControlConfig.ClusterDNSs)
if err != nil {
return errors.Wrap(err, "cannot configure IPv4 cluster-dns address")
return errors.Wrap(err, "cannot configure IPv4/IPv6 cluster-dns address")
}
serverConfig.ControlConfig.ClusterDNS = clusterDNS
}
Expand Down Expand Up @@ -457,6 +470,9 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
ip := serverConfig.ControlConfig.BindAddress
if ip == "" {
ip = "127.0.0.1"
if IPv6only {
ip = "[::1]"
}
}

url := fmt.Sprintf("https://%s:%d", ip, serverConfig.ControlConfig.SupervisorPort)
Expand Down Expand Up @@ -517,6 +533,16 @@ func validateNetworkConfiguration(serverConfig server.Config) error {
return errors.New("dual-stack cluster-dns is not supported")
}

IPv6OnlyService, _ := util.IsIPv6OnlyCIDRs(serverConfig.ControlConfig.ServiceIPRanges)
if IPv6OnlyService {
if serverConfig.ControlConfig.DisableNPC == false {
return errors.New("network policy enforcement is not compatible with IPv6 only operation; server must be restarted with --disable-network-policy")
}
if serverConfig.ControlConfig.FlannelBackend != config.FlannelBackendNone {
return errors.New("Flannel is not compatible with IPv6 only operation; server must be restarted with --flannel-backend=none")
}
}

return nil
}

Expand Down
14 changes: 12 additions & 2 deletions pkg/daemons/agent/agent_linux.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,14 @@ func checkRuntimeEndpoint(cfg *config.Agent, argsMap map[string]string) {
}

func kubeProxyArgs(cfg *config.Agent) map[string]string {
bindAddress := "127.0.0.1"
_, IPv6only, _ := util.GetFirstString([]string{cfg.NodeIP})
if IPv6only {
bindAddress = "::1"
}
argsMap := map[string]string{
"proxy-mode": "iptables",
"healthz-bind-address": "127.0.0.1",
"healthz-bind-address": bindAddress,
"kubeconfig": cfg.KubeConfigKubeProxy,
"cluster-cidr": util.JoinIPNets(cfg.ClusterCIDRs),
"conntrack-max-per-core": "0",
Expand All @@ -55,8 +60,13 @@ func kubeProxyArgs(cfg *config.Agent) map[string]string {
}

func kubeletArgs(cfg *config.Agent) map[string]string {
bindAddress := "127.0.0.1"
_, IPv6only, _ := util.GetFirstString([]string{cfg.NodeIP})
if IPv6only {
bindAddress = "::1"
}
argsMap := map[string]string{
"healthz-bind-address": "127.0.0.1",
"healthz-bind-address": bindAddress,
"read-only-port": "0",
"cluster-domain": cfg.ClusterDomain,
"kubeconfig": cfg.KubeConfigKubelet,
Expand Down
14 changes: 12 additions & 2 deletions pkg/daemons/agent/agent_windows.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,14 @@ func checkRuntimeEndpoint(cfg *config.Agent, argsMap map[string]string) {
}

func kubeProxyArgs(cfg *config.Agent) map[string]string {
bindAddress := "127.0.0.1"
_, IPv6only, _ := util.GetFirstString(cfg.NodeIP)
if IPv6only {
bindAddress = "::1"
}
argsMap := map[string]string{
"proxy-mode": "kernelspace",
"healthz-bind-address": "127.0.0.1",
"healthz-bind-address": bindAddress,
"kubeconfig": cfg.KubeConfigKubeProxy,
"cluster-cidr": util.JoinIPNets(cfg.ClusterCIDRs),
}
Expand All @@ -45,8 +50,13 @@ func kubeProxyArgs(cfg *config.Agent) map[string]string {
}

func kubeletArgs(cfg *config.Agent) map[string]string {
bindAddress := "127.0.0.1"
_, IPv6only, _ := util.GetFirstString(cfg.NodeIP)
if IPv6only {
bindAddress = "::1"
}
argsMap := map[string]string{
"healthz-bind-address": "127.0.0.1",
"healthz-bind-address": bindAddress,
"read-only-port": "0",
"cluster-domain": cfg.ClusterDomain,
"kubeconfig": cfg.KubeConfigKubelet,
Expand Down
115 changes: 115 additions & 0 deletions pkg/util/net.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,46 @@ func JoinIP4Nets(elems []*net.IPNet) string {
return strings.Join(strs, ",")
}

// GetFirst6 returns the first IPv6 address from the list of IP addresses.
// If no IPv6 addresses are found, an error is raised.
func GetFirst6(elems []net.IP) (net.IP, error) {
for _, elem := range elems {
if elem == nil || elem.To16() == nil {
continue
}
return elem, nil
}
return nil, errors.New("no IPv6 address found")
}

// GetFirst6Net returns the first IPv4 network from the list of IP networks.
// If no IPv6 addresses are found, an error is raised.
func GetFirst6Net(elems []*net.IPNet) (*net.IPNet, error) {
for _, elem := range elems {
if elem == nil || elem.IP.To16() == nil {
continue
}
return elem, nil
}
return nil, errors.New("no IPv6 CIDRs found")
}

// GetFirst6String returns the first IPv6 address from a list of IP address strings.
// If no IPv6 addresses are found, an error is raised.
func GetFirst6String(elems []string) (string, error) {
ips := []net.IP{}
for _, elem := range elems {
for _, v := range strings.Split(elem, ",") {
ips = append(ips, net.ParseIP(v))
}
}
ip, err := GetFirst6(ips)
if err != nil {
return "", err
}
return ip.String(), nil
}

// JoinIP6Nets stringifies and joins a list of IPv6 networks with commas.
func JoinIP6Nets(elems []*net.IPNet) string {
var strs []string
Expand Down Expand Up @@ -141,3 +181,78 @@ func ParseStringSliceToIPs(s cli.StringSlice) ([]net.IP, error) {

return ips, nil
}

// GetFirstIP returns the first IPv4 address from the list of IP addresses.
// If no IPv4 addresses are found, returns the first IPv6 address
// if neither of IPv4 or IPv6 are found an error is raised.
// Additionally matching listen address and IP version is returned.
func GetFirstIP(nodeIPs []net.IP) (net.IP, string, bool, error) {
nodeIP, err := GetFirst4(nodeIPs)
ListenAddress := "0.0.0.0"
IPv6only := false
if err != nil {
nodeIP, err = GetFirst6(nodeIPs)
if err != nil {
return nil, "", false, err
}
ListenAddress = "::"
IPv6only = true
}
return nodeIP, ListenAddress, IPv6only, nil
}

// GetFirstNet returns the first IPv4 network from the list of IP networks.
// If no IPv4 addresses are found, returns the first IPv6 address
// if neither of IPv4 or IPv6 are found an error is raised.
func GetFirstNet(elems []*net.IPNet) (*net.IPNet, error) {
serviceIPRange, err := GetFirst4Net(elems)
if err != nil {
serviceIPRange, err = GetFirst6Net(elems)
if err != nil {
return nil, err
}
}
return serviceIPRange, nil
}

// GetFirstString returns the first IP4 address from a list of IP address strings.
// If no IPv4 addresses are found, returns the first IPv6 address
// if neither of IPv4 or IPv6 are found an error is raised.
func GetFirstString(elems []string) (string, bool, error) {
ip, err := GetFirst4String(elems)
IPv6only := false
if err != nil {
ip, err = GetFirst6String(elems)
if err != nil {
return "", false, err
}
IPv6only = true
}
return ip, IPv6only, nil
}

// IsIPv6OnlyCIDRs returns if
// - all are valid cidrs
// - at least one cidr from v6 family is found
// - v4 family cidr is not found
func IsIPv6OnlyCIDRs(cidrs []*net.IPNet) (bool, error) {
v4Found := false
v6Found := false
for _, cidr := range cidrs {
if cidr == nil {
return false, fmt.Errorf("cidr %v is invalid", cidr)
}

if v4Found && v6Found {
continue
}

if cidr.IP != nil && cidr.IP.To4() == nil {
v6Found = true
continue
}
v4Found = true
}

return !v4Found && v6Found, nil
}

0 comments on commit 966f4d6

Please sign in to comment.